[ { "path": ".editorconfig", "content": "# https://EditorConfig.org\n\n# Top-most EditorConfig file\nroot = true\n\n# Unix-style newlines with a newline ending every file, utf-8 charset\n[*]\nend_of_line = lf\ninsert_final_newline = true\ntrim_trailing_whitespace = true\ncharset = utf-8\nindent_style = space\nindent_size = 4\n\n[*.md]\nindent_size = 2\n\n[Makefile]\nindent_style = tab\n\n[prompts/*.txt]\ninsert_final_newline = unset\n" }, { "path": ".github/pull_request_template.md", "content": "*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.*\n" }, { "path": ".github/workflows/ci.yml", "content": "name: CI\n\non:\n push:\n branches: [ master ]\n pull_request:\n branches: [ master ]\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}\n cancel-in-progress: true\n\njobs:\n build:\n strategy:\n matrix:\n os: [ubuntu-latest, macos-latest, windows-latest]\n libraries: [shared, static]\n\n runs-on: ${{ matrix.os }}\n\n steps:\n - name: Clone\n uses: actions/checkout@v6\n\n - name: Dependencies for Ubuntu\n if: matrix.os == 'ubuntu-latest'\n run: |\n sudo apt-get update\n sudo apt-get install llvm\n\n - name: Add msbuild to PATH\n if: matrix.os == 'windows-latest'\n uses: microsoft/setup-msbuild@v2\n\n - name: Create Build Environment\n run: mkdir build\n\n - name: Configure CMake\n working-directory: ./build\n run: cmake ..\n ${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }}\n ${{ matrix.libraries == 'static' && '-DBUILD_SHARED_LIBS=OFF' || '-DBUILD_SHARED_LIBS=ON' }}\n -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed\n -DGGML_METAL=OFF\n\n - name: Build\n working-directory: ./build\n run: cmake --build . ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}\n\n - name: Test\n working-directory: ./build\n run: ctest --verbose --timeout 900 ${{ contains(matrix.os, 'windows') && '--build-config Release' || '' }}\n\n - name: Install\n working-directory: ./build\n run: cmake --build . --target install ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}\n\n - name: Test CMake config\n run: |\n mkdir test-cmake\n cmake -S examples/test-cmake -B test-cmake -DCMAKE_PREFIX_PATH=${{ github.workspace }}/installed ${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }}\n cmake --build test-cmake ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}\n\n# TODO: simplify the following workflows using a matrix\n ggml-ci-x64-cpu-low-perf:\n runs-on: ubuntu-22.04\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: ccache\n uses: ggml-org/ccache-action@v1.2.16\n with:\n key: ggml-ci-x64-cpu-low-perf\n evict-old-files: 1d\n\n - name: Dependencies\n id: depends\n run: |\n sudo apt-get update\n sudo apt-get install build-essential libcurl4-openssl-dev\n\n - name: Test\n id: ggml-ci\n run: |\n LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n ggml-ci-arm64-cpu-low-perf:\n runs-on: ubuntu-22.04-arm\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: ccache\n uses: ggml-org/ccache-action@v1.2.16\n with:\n key: ggml-ci-arm64-cpu-low-perf\n evict-old-files: 1d\n\n - name: Dependencies\n id: depends\n run: |\n sudo apt-get update\n sudo apt-get install build-essential libcurl4-openssl-dev\n\n - name: Test\n id: ggml-ci\n run: |\n LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n ggml-ci-x64-cpu-high-perf:\n runs-on: ubuntu-22.04\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: ccache\n uses: ggml-org/ccache-action@v1.2.16\n with:\n key: ggml-ci-x64-cpu-high-perf\n evict-old-files: 1d\n\n - name: Dependencies\n id: depends\n run: |\n sudo apt-get update\n sudo apt-get install build-essential libcurl4-openssl-dev\n\n - name: Test\n id: ggml-ci\n run: |\n LLAMA_ARG_THREADS=$(nproc) bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n ggml-ci-arm64-cpu-high-perf:\n runs-on: ubuntu-22.04-arm\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: ccache\n uses: ggml-org/ccache-action@v1.2.16\n with:\n key: ggml-ci-arm64-cpu-high-perf\n evict-old-files: 1d\n\n - name: Dependencies\n id: depends\n run: |\n sudo apt-get update\n sudo apt-get install build-essential libcurl4-openssl-dev\n\n - name: Test\n id: ggml-ci\n run: |\n LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n ggml-ci-arm64-cpu-high-perf-sve:\n runs-on: ubuntu-22.04-arm\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: ccache\n uses: ggml-org/ccache-action@v1.2.16\n with:\n key: ggml-ci-arm64-cpu-high-perf-sve\n evict-old-files: 1d\n\n - name: Dependencies\n id: depends\n run: |\n sudo apt-get update\n sudo apt-get install build-essential libcurl4-openssl-dev\n\n - name: Test\n id: ggml-ci\n run: |\n LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n ggml-ci-x64-nvidia-cuda:\n runs-on: [self-hosted, Linux, X64, NVIDIA]\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: Test\n id: ggml-ci\n run: |\n nvidia-smi\n GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml\n\n ggml-ci-x64-nvidia-vulkan-cm:\n runs-on: [self-hosted, Linux, X64, NVIDIA]\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: Test\n id: ggml-ci\n run: |\n vulkaninfo --summary\n GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml\n\n ggml-ci-x64-nvidia-vulkan-cm2:\n runs-on: [self-hosted, Linux, X64, NVIDIA, COOPMAT2]\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: Test\n id: ggml-ci\n run: |\n vulkaninfo --summary\n GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml\n\n # TODO: provision AMX-compatible machine\n #ggml-ci-x64-cpu-amx:\n # runs-on: [self-hosted, Linux, X64, CPU, AMX]\n\n # steps:\n # - name: Clone\n # id: checkout\n # uses: actions/checkout@v6\n\n # - name: Test\n # id: ggml-ci\n # run: |\n # bash ./ci/run.sh ~/results/ggml /mnt/ggml\n\n ggml-ci-mac-metal:\n runs-on: [self-hosted, macOS, ARM64]\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: Test\n id: ggml-ci\n run: |\n GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml\n\n ggml-ci-mac-vulkan:\n runs-on: [self-hosted, macOS, ARM64]\n\n steps:\n - name: Clone\n id: checkout\n uses: actions/checkout@v6\n\n - name: Test\n id: ggml-ci\n run: |\n vulkaninfo --summary\n GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml\n" }, { "path": ".github/workflows/release.yml", "content": "name: Release\n\non:\n push:\n tags:\n - 'v*'\n\njobs:\n release:\n runs-on: ubuntu-latest\n permissions:\n contents: write\n\n steps:\n - name: Checkout code\n uses: actions/checkout@v6\n\n - name: Create Release\n id: create_release\n uses: ggml-org/action-create-release@v1\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n with:\n tag_name: ${{ github.ref_name }}\n release_name: ${{ github.ref }}\n draft: false\n prerelease: false\n" }, { "path": ".gitignore", "content": "build/\nbuild-*/\nout/\ntmp/\nmodels/\nmodels-mnt\n\ncompile_commands.json\nCMakeSettings.json\n.vs/\n.vscode/\n.idea/\n.clangd\n\n.venv/\nggml_env/\n.exrc\n.cache\n.DS_Store\n.stablelm\n.gpt-2\n\nsrc/arm_neon.h\ntests/arm_neon.h\n\nzig-out/\nzig-cache/\n\n*.o\n*.d\n*.dot\n\n*.sw?\n\n__pycache__/\n\n# Model files\nggml-model-f16.bin\n*.bat\n" }, { "path": ".gitmodules", "content": "" }, { "path": "AUTHORS", "content": "# date: Tue Feb 4 13:03:51 EET 2025\n# this file is auto-generated by scripts/gen-authors.sh\n\n0cc4m \n65a <10104049+65a@users.noreply.github.com>\nAT \nAbhilash Majumder <30946547+abhilash1910@users.noreply.github.com>\nAdam Tazi <52357206+ad1tazi@users.noreply.github.com>\nAdrien Gallouët \nAdrien Gallouët \nAhmad Tameem <113388789+Tameem-10xE@users.noreply.github.com>\nAidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com>\nAidanBeltonS \nAkarshan Biswas \nAkarshan Biswas \nAlbert Jin \nAlberto Cabrera Pérez \nAlberto Cabrera Pérez \nAlex Azarov \nAlex O'Connell <35843486+acon96@users.noreply.github.com>\nAlex von Gluck IV \nAmbientL <107641468+AmbientL@users.noreply.github.com>\nAmirAli Mirian <37371367+amiralimi@users.noreply.github.com>\nAnanta Bastola \nAndreas (Andi) Kunar \nAndreas Kieslinger <47689530+aendk@users.noreply.github.com>\nAndrei \nAndrew Minh Nguyen <40281306+amqdn@users.noreply.github.com>\nAndrii Ryzhkov \nArjun \nAshraful Islam \nAstariul <43774355+astariul@users.noreply.github.com>\nAsukaMinato \nAvi Lumelsky \nBart Pelle <3662930+Velocity-@users.noreply.github.com>\nBen Ashbaugh \nBernhard M. Wiedemann \nBorislav Stanimirov \nBrad Ito \nBrad Murray <59848399+bradmurray-dt@users.noreply.github.com>\nBrian \nBryan Lozano \nCarolinabanana <140120812+Carolinabanana@users.noreply.github.com>\nCarterLi999 <664681047@qq.com>\nCebtenzzre \nChangyeon Kim \nCharles Xu <63788048+chaxu01@users.noreply.github.com>\nCharles Xu \nChen Xi \nChen Xi \nChenguang Li <87689256+noemotiovon@users.noreply.github.com>\nChris Elrod \nChristian Kastner \nClint Herron \nConrad Kramer \nCordeiro <1471463+ocordeiro@users.noreply.github.com>\nCristiano Calcagno \nDAN™ \nDan Forbes \nDan Johansson <164997844+eddnjjn@users.noreply.github.com>\nDan Johansson \nDaniel Bevenius \nDaniel Ziegenberg \nDaniele <57776841+daniandtheweb@users.noreply.github.com>\nDaulet Zhanguzin \nDave \nDave Airlie \nDave Airlie \nDavid Miller \nDavidKorczynski \nDavidson Francis \nDibakar Gope \nDidzis Gosko \nDiego Devesa \nDiogo \nDjip007 <3705339+Djip007@users.noreply.github.com>\nDjip007 \nDou Xinpeng <15529241576@163.com>\nDou Xinpeng <81913537+Dou-Git@users.noreply.github.com>\nDr. Tom Murphy VII Ph.D <499244+tom7@users.noreply.github.com>\nEbey Abraham \nEldar Yusupov \nEmmanuel Durand \nEngininja2 <139037756+Engininja2@users.noreply.github.com>\nEric Zhang <34133756+EZForever@users.noreply.github.com>\nErik Scholz \nEttore Di Giacinto \nEve <139727413+netrunnereve@users.noreply.github.com>\nF1L1P <78918286+F1L1Pv2@users.noreply.github.com>\nFaisal Zaghloul \nFantasyGmm <16450052+FantasyGmm@users.noreply.github.com>\nFelix \nFinn Voorhees \nFirstTimeEZ <179362031+FirstTimeEZ@users.noreply.github.com>\nFrankie Robertson \nGainLee \nGeorge Hindle \nGeorgi Gerganov \nGilad S <7817232+giladgd@users.noreply.github.com>\nGilad S \nGilad S. <7817232+giladgd@users.noreply.github.com>\nGuillaume Wenzek \nHalalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com>\nHaus1 \nHerman Semenov \nHimariO \nHirochika Matsumoto \nHong Bo PENG \nHugo Rosenkranz-Costa \nHyunsung Lee \nIGUILIZ Salah-Eddine <76955987+salahiguiliz@users.noreply.github.com>\nIan Bull \nIhar Hrachyshka \nIkko Eltociear Ashimine \nIvan \nIvan Filipov <159561759+vanaka11@users.noreply.github.com>\nIvan Stepanov \nIvan Zdane \nJack Mousseau \nJack Vial \nJacobLinCool \nJakob Frick \nJan Ploski \nJared Van Bortel \nJeff Bolz \nJeffrey Quesnelle \nJeroen Mostert \nJiahao Li \nJidongZhang-THU <1119708529@qq.com>\nJiří Podivín <66251151+jpodivin@users.noreply.github.com>\nJo Liss \nJoe Todd \nJohannes Gäßler \nJohn Balis \nJosh Bleecher Snyder \nJudd \nJun Hee Yoo \nJunil Kim \nJustina Cho \nJustine Tunney \nJustine Tunney \nKarol Kontny <82021046+kkontny@users.noreply.github.com>\nKawrakow <48489457+ikawrakow@users.noreply.github.com>\nKevin Gibbons \nKonstantin Zhuravlyov \nKylin <56434533+KyL0N@users.noreply.github.com>\nLoganDark \nLoganDark \nLostRuins <39025047+LostRuins@users.noreply.github.com>\nLukas Möller \nM Refi D.A <24388107+refinism@users.noreply.github.com>\nM. Yusuf Sarıgöz \nMa Mingfei \nMahesh Madhav <67384846+heshpdx@users.noreply.github.com>\nMaiHD \nMark Zhuang \nMarkus Tavenrath \nMasaya, Kato <62578291+msy-kato@users.noreply.github.com>\nMathieu Baudier \nMathijs de Bruin \nMatt Stephenson \nMax Krasnyansky \nMax Krasnyansky \nMayank Kumar Pal \nMeng, Hengyu \nMengqing Cao \nMetal Whale <45712559+metalwhale@users.noreply.github.com>\nMichael Klimenko \nMichael Podvitskiy \nMichael Verrilli \nMolly Sophia \nNatsu \nNeo Zhang <14088817+arthw@users.noreply.github.com>\nNeo Zhang Jianyu \nNeuman Vong \nNevin \nNicholai Tukanov \nNico Bosshard \nNicolò Scipione \nNikita Sarychev <42014488+sARY77@users.noreply.github.com>\nNouamane Tazi \nOlivier Chafik \nOlivier Chafik \nOndřej Čertík \nOuadie EL FAROUKI \nPAB \nPaul Tsochantaris \nPeter \nPhilpax \nPierre Alexandre SCHEMBRI \nPlamen Minev \nPlaydev \nPrashant Vithule <119530321+Vithulep@users.noreply.github.com>\nPrzemysław Pawełczyk \nR0CKSTAR \nR0CKSTAR \nRadoslav Gerganov \nRadosław Gryta \nRavindra Marella \nRay Cromwell \nReinforce-II \nRémy Oudompheng \nReza Rezvan \nRick G <26732651+TheFlipbook@users.noreply.github.com>\nRiverZhou \nRobert Ormandi <52251610+ormandi@users.noreply.github.com>\nRomain Biessy \nRonsor \nRotem Dan \nRyan Hitchman \nSRHMorris <69468379+SRHMorris@users.noreply.github.com>\nSXX \nSalvatore Mesoraca \nSam Spilsbury \nSanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com>\nSanttu Keskinen \nSergio López \nSergio López \nShanshan Shen <467638484@qq.com>\nShijie <821898965@qq.com>\nShupei Fan \nSiddharth Ramakrishnan \nSigbjørn Skjæret \nSkyler Celestinian-Sterling <80314197+Celestinian@users.noreply.github.com>\nSlava Primenko \nSrihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>\nSteward Garcia <57494570+FSSRepo@users.noreply.github.com>\nSupreet Sethi \nTakuya Takeuchi \nTamotsu Takahashi \nTanmay \nTanmay Sachan \nTimothy Cronin <40186632+4imothy@users.noreply.github.com>\nTom Bailey \nTom Jobbins <784313+TheBloke@users.noreply.github.com>\nTony Wasserka <4840017+neobrain@users.noreply.github.com>\nTristan Druyen \nTyé singwa <92231658+tye-singwa@users.noreply.github.com>\nUEXTM.com <84163508+uextm@users.noreply.github.com>\nWillCorticesAI <150854901+WillCorticesAI@users.noreply.github.com>\nWilliam Tambellini \nWilliam Tambellini \nXiaotaoChen \nXinpeng Dou <81913537+Dou-Git@users.noreply.github.com>\nXuan Son Nguyen \nYavor Ivanov \nYavorGIvanov \nYilong Guo \nYilong Guo \nYuri Khrustalev \nZhenwei Jin <109658203+kylo5aby@users.noreply.github.com>\nZhiyuan Li \nZhiyuan Li \na3sh <38979186+A3shTnT@users.noreply.github.com>\nag2s20150909 <19373730+ag2s20150909@users.noreply.github.com>\nagray3 \namd-dwang \namritahs-ibm \napcameron <37645737+apcameron@users.noreply.github.com>\nappvoid <78444142+appvoid@users.noreply.github.com>\nariez-xyz <41232910+ariez-xyz@users.noreply.github.com>\nautomaticcat \nbandoti <141645996+bandoti@users.noreply.github.com>\nbmwl \nbobqianic <129547291+bobqianic@users.noreply.github.com>\nbssrdf \nchengchi \ncompilade <113953597+compilade@users.noreply.github.com>\ncompilade \nddpasa <112642920+ddpasa@users.noreply.github.com>\ndenersc \ndscripka \nfitzsim \nfj-y-saito <85871716+fj-y-saito@users.noreply.github.com>\nfraxy-v <65565042+fraxy-v@users.noreply.github.com>\ngn64 \ngoerch \ngoldwaving <77494627+goldwaving@users.noreply.github.com>\nhaopeng <657407891@qq.com>\nhidenorly \nhipudding \nhydai \nissixx <46835150+issixx@users.noreply.github.com>\njaeminSon \njdomke <28772296+jdomke@users.noreply.github.com>\njiez <373447296@qq.com>\njohnson442 <56517414+johnson442@users.noreply.github.com>\njunchao-loongson <68935141+junchao-loongson@users.noreply.github.com>\nk.h.lai \nkatsu560 <118887472+katsu560@users.noreply.github.com>\nklosax <131523366+klosax@users.noreply.github.com>\nkunnis \nl3utterfly \nle.chang \nleejet <31925346+leejet@users.noreply.github.com>\nleejet \nleo-pony \nlhez \nliuwei-git <14815172+liuwei-git@users.noreply.github.com>\nluoyu-intel \nmagicse \nmahorozte <41834471+mahorozte@users.noreply.github.com>\nmashizora <30516315+mashizora@users.noreply.github.com>\nmatt23654 \nmatteo \nochafik \notaGran \npengxin99 \npikalover6 <49179590+pikalover6@users.noreply.github.com>\npostmasters \nsjinzh \nskirodev <57715494+skirodev@users.noreply.github.com>\nslaren \nsnadampal <87143774+snadampal@users.noreply.github.com>\nsomeone13574 <81528246+someone13574@users.noreply.github.com>\nstduhpf \ntaher <8665427+nullhook@users.noreply.github.com>\ntexmex76 <40733439+texmex76@users.noreply.github.com>\nthe-crypt-keeper <84680712+the-crypt-keeper@users.noreply.github.com>\nthewh1teagle <61390950+thewh1teagle@users.noreply.github.com>\nucag.li \nulatekh \nuvos \nuvos \nwangshuai09 <391746016@qq.com>\nwoachk <24752637+woachk@users.noreply.github.com>\nxctan \nyangyaofei \nyuri@FreeBSD \nzhentaoyu \nzhouwg <6889919+zhouwg@users.noreply.github.com>\nzhouwg \n谢乃闻 \n布客飞龙 <562826179@qq.com>\n旺旺碎冰冰 <38837039+Cyberhan123@users.noreply.github.com>\n" }, { "path": "CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit target directories.\nproject(\"ggml\" C CXX ASM)\n\n### GGML Version\nset(GGML_VERSION_MAJOR 0)\nset(GGML_VERSION_MINOR 9)\nset(GGML_VERSION_PATCH 8)\nset(GGML_VERSION_BASE \"${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}\")\n\nfind_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)\nif(GIT_EXE)\n # Get current git commit hash\n execute_process(COMMAND ${GIT_EXE} rev-parse --short HEAD\n WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n OUTPUT_VARIABLE GGML_BUILD_COMMIT\n OUTPUT_STRIP_TRAILING_WHITESPACE\n ERROR_QUIET\n )\n\n # Check if the working directory is dirty (i.e., has uncommitted changes)\n execute_process(COMMAND ${GIT_EXE} diff-index --quiet HEAD -- .\n WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n RESULT_VARIABLE GGML_GIT_DIRTY\n ERROR_QUIET\n )\nendif()\n\nset(GGML_VERSION \"${GGML_VERSION_BASE}\")\n\nif(NOT GGML_BUILD_COMMIT)\n set(GGML_BUILD_COMMIT \"unknown\")\nendif()\n\n# Build the commit string with optional dirty flag\nif(DEFINED GGML_GIT_DIRTY AND GGML_GIT_DIRTY EQUAL 1)\n set(GGML_BUILD_COMMIT \"${GGML_BUILD_COMMIT}-dirty\")\nendif()\n\ninclude(CheckIncludeFileCXX)\n\nset(CMAKE_EXPORT_COMPILE_COMMANDS ON)\n\nif (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)\n set(CMAKE_BUILD_TYPE Release CACHE STRING \"Build type\" FORCE)\n set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS \"Debug\" \"Release\" \"MinSizeRel\" \"RelWithDebInfo\")\nendif()\n\nif (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)\n set(GGML_STANDALONE ON)\n\n set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n\n # configure project version\n # TODO\nelse()\n set(GGML_STANDALONE OFF)\n\n if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY)\n set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n endif()\nendif()\n\nif (EMSCRIPTEN)\n set(BUILD_SHARED_LIBS_DEFAULT OFF)\n\n option(GGML_WASM_SINGLE_FILE \"ggml: embed WASM inside the generated ggml.js\" ON)\nelse()\n if (MINGW)\n set(BUILD_SHARED_LIBS_DEFAULT OFF)\n else()\n set(BUILD_SHARED_LIBS_DEFAULT ON)\n endif()\nendif()\n\n# remove the lib prefix on win32 mingw\nif (WIN32)\n set(CMAKE_STATIC_LIBRARY_PREFIX \"\")\n set(CMAKE_SHARED_LIBRARY_PREFIX \"\")\n set(CMAKE_SHARED_MODULE_PREFIX \"\")\nendif()\n\noption(BUILD_SHARED_LIBS \"ggml: build shared libraries\" ${BUILD_SHARED_LIBS_DEFAULT})\noption(GGML_BACKEND_DL \"ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)\" OFF)\nset(GGML_BACKEND_DIR \"\" CACHE PATH \"ggml: directory to load dynamic backends from (requires GGML_BACKEND_DL\")\n\n#\n# option list\n#\n\n# TODO: mark all options as advanced when not GGML_STANDALONE\n\nif (APPLE)\n set(GGML_METAL_DEFAULT ON)\n set(GGML_BLAS_DEFAULT ON)\n set(GGML_BLAS_VENDOR_DEFAULT \"Apple\")\nelse()\n set(GGML_METAL_DEFAULT OFF)\n set(GGML_BLAS_DEFAULT OFF)\n set(GGML_BLAS_VENDOR_DEFAULT \"Generic\")\nendif()\n\nif (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH})\n message(STATUS \"Setting GGML_NATIVE_DEFAULT to OFF\")\n set(GGML_NATIVE_DEFAULT OFF)\nelse()\n set(GGML_NATIVE_DEFAULT ON)\nendif()\n\n# defaults\nif (NOT GGML_LLAMAFILE_DEFAULT)\n set(GGML_LLAMAFILE_DEFAULT OFF)\nendif()\n\nif (NOT GGML_CUDA_GRAPHS_DEFAULT)\n set(GGML_CUDA_GRAPHS_DEFAULT OFF)\nendif()\n\n# general\noption(GGML_STATIC \"ggml: static link libraries\" OFF)\noption(GGML_NATIVE \"ggml: optimize the build for the current system\" ${GGML_NATIVE_DEFAULT})\noption(GGML_LTO \"ggml: enable link time optimization\" OFF)\noption(GGML_CCACHE \"ggml: use ccache if available\" ON)\n\n# debug\noption(GGML_ALL_WARNINGS \"ggml: enable all compiler warnings\" ON)\noption(GGML_ALL_WARNINGS_3RD_PARTY \"ggml: enable all compiler warnings in 3rd party libs\" OFF)\noption(GGML_GPROF \"ggml: enable gprof\" OFF)\n\n# build\noption(GGML_FATAL_WARNINGS \"ggml: enable -Werror flag\" OFF)\n\n# sanitizers\noption(GGML_SANITIZE_THREAD \"ggml: enable thread sanitizer\" OFF)\noption(GGML_SANITIZE_ADDRESS \"ggml: enable address sanitizer\" OFF)\noption(GGML_SANITIZE_UNDEFINED \"ggml: enable undefined sanitizer\" OFF)\n\n# instruction set specific\nif (GGML_NATIVE OR NOT GGML_NATIVE_DEFAULT)\n set(INS_ENB OFF)\nelse()\n set(INS_ENB ON)\nendif()\n\nmessage(DEBUG \"GGML_NATIVE : ${GGML_NATIVE}\")\nmessage(DEBUG \"GGML_NATIVE_DEFAULT : ${GGML_NATIVE_DEFAULT}\")\nmessage(DEBUG \"INS_ENB : ${INS_ENB}\")\n\noption(GGML_CPU_HBM \"ggml: use memkind for CPU HBM\" OFF)\noption(GGML_CPU_REPACK \"ggml: use runtime weight conversion of Q4_0 to Q4_X_X\" ON)\noption(GGML_CPU_KLEIDIAI \"ggml: use KleidiAI optimized kernels if applicable\" OFF)\noption(GGML_SSE42 \"ggml: enable SSE 4.2\" ${INS_ENB})\noption(GGML_AVX \"ggml: enable AVX\" ${INS_ENB})\noption(GGML_AVX_VNNI \"ggml: enable AVX-VNNI\" OFF)\noption(GGML_AVX2 \"ggml: enable AVX2\" ${INS_ENB})\noption(GGML_BMI2 \"ggml: enable BMI2\" ${INS_ENB})\noption(GGML_AVX512 \"ggml: enable AVX512F\" OFF)\noption(GGML_AVX512_VBMI \"ggml: enable AVX512-VBMI\" OFF)\noption(GGML_AVX512_VNNI \"ggml: enable AVX512-VNNI\" OFF)\noption(GGML_AVX512_BF16 \"ggml: enable AVX512-BF16\" OFF)\nif (NOT MSVC)\n # in MSVC F16C and FMA is implied with AVX2/AVX512\n option(GGML_FMA \"ggml: enable FMA\" ${INS_ENB})\n option(GGML_F16C \"ggml: enable F16C\" ${INS_ENB})\n # MSVC does not seem to support AMX\n option(GGML_AMX_TILE \"ggml: enable AMX-TILE\" OFF)\n option(GGML_AMX_INT8 \"ggml: enable AMX-INT8\" OFF)\n option(GGML_AMX_BF16 \"ggml: enable AMX-BF16\" OFF)\nendif()\noption(GGML_LASX \"ggml: enable lasx\" ON)\noption(GGML_LSX \"ggml: enable lsx\" ON)\noption(GGML_RVV \"ggml: enable rvv\" ON)\noption(GGML_RV_ZFH \"ggml: enable riscv zfh\" ON)\noption(GGML_RV_ZVFH \"ggml: enable riscv zvfh\" ON)\noption(GGML_RV_ZICBOP \"ggml: enable riscv zicbop\" ON)\noption(GGML_RV_ZIHINTPAUSE \"ggml: enable riscv zihintpause \" ON)\noption(GGML_XTHEADVECTOR \"ggml: enable xtheadvector\" OFF)\noption(GGML_VXE \"ggml: enable vxe\" ${GGML_NATIVE})\n\noption(GGML_CPU_ALL_VARIANTS \"ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)\" OFF)\nset(GGML_CPU_ARM_ARCH \"\" CACHE STRING \"ggml: CPU architecture for ARM\")\nset(GGML_CPU_POWERPC_CPUTYPE \"\" CACHE STRING \"ggml: CPU type for PowerPC\")\n\n# ggml core\nset(GGML_SCHED_MAX_COPIES \"4\" CACHE STRING \"ggml: max input copies for pipeline parallelism\")\noption(GGML_CPU \"ggml: enable CPU backend\" ON)\noption(GGML_SCHED_NO_REALLOC \"ggml: disallow reallocations in ggml-alloc (for debugging)\" OFF)\n\n# 3rd party libs / backends\noption(GGML_ACCELERATE \"ggml: enable Accelerate framework\" ON)\noption(GGML_BLAS \"ggml: use BLAS\" ${GGML_BLAS_DEFAULT})\nset(GGML_BLAS_VENDOR ${GGML_BLAS_VENDOR_DEFAULT} CACHE STRING\n \"ggml: BLAS library vendor\")\noption(GGML_LLAMAFILE \"ggml: use LLAMAFILE\" ${GGML_LLAMAFILE_DEFAULT})\n\noption(GGML_CUDA \"ggml: use CUDA\" OFF)\noption(GGML_MUSA \"ggml: use MUSA\" OFF)\noption(GGML_CUDA_FORCE_MMQ \"ggml: use mmq kernels instead of cuBLAS\" OFF)\noption(GGML_CUDA_FORCE_CUBLAS \"ggml: always use cuBLAS instead of mmq kernels\" OFF)\nset (GGML_CUDA_PEER_MAX_BATCH_SIZE \"128\" CACHE STRING\n \"ggml: max. batch size for using peer access\")\noption(GGML_CUDA_NO_PEER_COPY \"ggml: do not use peer to peer copies\" OFF)\noption(GGML_CUDA_NO_VMM \"ggml: do not try to use CUDA VMM\" OFF)\noption(GGML_CUDA_FA \"ggml: compile ggml FlashAttention CUDA kernels\" ON)\noption(GGML_CUDA_FA_ALL_QUANTS \"ggml: compile all quants for FlashAttention\" OFF)\noption(GGML_CUDA_GRAPHS \"ggml: use CUDA graphs (llama.cpp only)\" ${GGML_CUDA_GRAPHS_DEFAULT})\nset (GGML_CUDA_COMPRESSION_MODE \"size\" CACHE STRING\n \"ggml: cuda link binary compression mode; requires cuda 12.8+\")\nset_property(CACHE GGML_CUDA_COMPRESSION_MODE PROPERTY STRINGS \"none;speed;balance;size\")\n\noption(GGML_HIP \"ggml: use HIP\" OFF)\noption(GGML_HIP_GRAPHS \"ggml: use HIP graph, experimental, slow\" OFF)\noption(GGML_HIP_NO_VMM \"ggml: do not try to use HIP VMM\" ON)\noption(GGML_HIP_ROCWMMA_FATTN \"ggml: enable rocWMMA for FlashAttention\" OFF)\noption(GGML_HIP_MMQ_MFMA \"ggml: enable MFMA MMA for CDNA in MMQ\" ON)\noption(GGML_HIP_EXPORT_METRICS \"ggml: enable kernel perf metrics output\" OFF)\noption(GGML_MUSA_GRAPHS \"ggml: use MUSA graph, experimental, unstable\" OFF)\noption(GGML_MUSA_MUDNN_COPY \"ggml: enable muDNN for accelerated copy\" OFF)\noption(GGML_VULKAN \"ggml: use Vulkan\" OFF)\noption(GGML_VULKAN_CHECK_RESULTS \"ggml: run Vulkan op checks\" OFF)\noption(GGML_VULKAN_DEBUG \"ggml: enable Vulkan debug output\" OFF)\noption(GGML_VULKAN_MEMORY_DEBUG \"ggml: enable Vulkan memory debug output\" OFF)\noption(GGML_VULKAN_SHADER_DEBUG_INFO \"ggml: enable Vulkan shader debug info\" OFF)\noption(GGML_VULKAN_VALIDATE \"ggml: enable Vulkan validation\" OFF)\noption(GGML_VULKAN_RUN_TESTS \"ggml: run Vulkan tests\" OFF)\noption(GGML_WEBGPU \"ggml: use WebGPU\" OFF)\noption(GGML_WEBGPU_DEBUG \"ggml: enable WebGPU debug output\" OFF)\noption(GGML_WEBGPU_CPU_PROFILE \"ggml: enable WebGPU profiling (CPU)\" OFF)\noption(GGML_WEBGPU_GPU_PROFILE \"ggml: enable WebGPU profiling (GPU)\" OFF)\noption(GGML_WEBGPU_JSPI \"ggml: use JSPI for WebGPU\" ON)\noption(GGML_ZDNN \"ggml: use zDNN\" OFF)\noption(GGML_VIRTGPU \"ggml: use the VirtGPU/Virglrenderer API Remoting frontend\" OFF)\noption(GGML_VIRTGPU_BACKEND \"ggml: build the VirtGPU/Virglrenderer API Remoting backend\" OFF)\noption(GGML_METAL \"ggml: use Metal\" ${GGML_METAL_DEFAULT})\noption(GGML_METAL_NDEBUG \"ggml: disable Metal debugging\" OFF)\noption(GGML_METAL_SHADER_DEBUG \"ggml: compile Metal with -fno-fast-math\" OFF)\noption(GGML_METAL_EMBED_LIBRARY \"ggml: embed Metal library\" ${GGML_METAL})\nset (GGML_METAL_MACOSX_VERSION_MIN \"\" CACHE STRING\n \"ggml: metal minimum macOS version\")\nset (GGML_METAL_STD \"\" CACHE STRING \"ggml: metal standard version (-std flag)\")\noption(GGML_OPENMP \"ggml: use OpenMP\" ON)\noption(GGML_RPC \"ggml: use RPC\" OFF)\noption(GGML_SYCL \"ggml: use SYCL\" OFF)\noption(GGML_SYCL_F16 \"ggml: use 16 bit floats for sycl calculations\" OFF)\noption(GGML_SYCL_GRAPH \"ggml: enable graphs in the SYCL backend\" ON)\noption(GGML_SYCL_DNN \"ggml: enable oneDNN in the SYCL backend\" ON)\nset (GGML_SYCL_TARGET \"INTEL\" CACHE STRING\n \"ggml: sycl target device\")\nset (GGML_SYCL_DEVICE_ARCH \"\" CACHE STRING\n \"ggml: sycl device architecture\")\n\noption(GGML_OPENVINO \"ggml: use OPENVINO\" OFF)\n\noption(GGML_OPENCL \"ggml: use OpenCL\" OFF)\noption(GGML_OPENCL_PROFILING \"ggml: use OpenCL profiling (increases overhead)\" OFF)\noption(GGML_OPENCL_EMBED_KERNELS \"ggml: embed kernels\" ON)\noption(GGML_OPENCL_USE_ADRENO_KERNELS \"ggml: use optimized kernels for Adreno\" ON)\nset (GGML_OPENCL_TARGET_VERSION \"300\" CACHE STRING\n \"ggml: OpenCL API version to target\")\n\noption(GGML_HEXAGON \"ggml: enable Hexagon backend\" OFF)\nset(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING \"ggml: quantize group size (32, 64, or 128)\")\n\n# toolchain for vulkan-shaders-gen\nset (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN \"\" CACHE FILEPATH \"ggml: toolchain file for vulkan-shaders-gen\")\n\noption(GGML_ZENDNN \"ggml: use ZenDNN\" OFF)\noption(ZENDNN_ROOT \"ggml: path to ZenDNN installation\" \"\")\n\n# extra artifacts\noption(GGML_BUILD_TESTS \"ggml: build tests\" ${GGML_STANDALONE})\noption(GGML_BUILD_EXAMPLES \"ggml: build examples\" ${GGML_STANDALONE})\n\n#\n# dependencies\n#\n\nset(CMAKE_C_STANDARD 11)\nset(CMAKE_C_STANDARD_REQUIRED true)\n\nset(CMAKE_CXX_STANDARD 17)\nset(CMAKE_CXX_STANDARD_REQUIRED true)\n\nset(THREADS_PREFER_PTHREAD_FLAG ON)\n\nfind_package(Threads REQUIRED)\n\ninclude(GNUInstallDirs)\n\n#\n# build the library\n#\n\nadd_subdirectory(src)\n\n#\n# tests and examples\n#\n\nif (GGML_BUILD_TESTS)\n enable_testing()\n add_subdirectory(tests)\nendif ()\n\nif (GGML_BUILD_EXAMPLES)\n add_subdirectory(examples)\nendif ()\n\n#\n# install\n#\n\ninclude(CMakePackageConfigHelpers)\n\n# all public headers\nset(GGML_PUBLIC_HEADERS\n include/ggml.h\n include/ggml-cpu.h\n include/ggml-alloc.h\n include/ggml-backend.h\n include/ggml-blas.h\n include/ggml-cann.h\n include/ggml-cpp.h\n include/ggml-cuda.h\n include/ggml-opt.h\n include/ggml-metal.h\n include/ggml-rpc.h\n include/ggml-virtgpu.h\n include/ggml-sycl.h\n include/ggml-vulkan.h\n include/ggml-webgpu.h\n include/ggml-zendnn.h\n include/ggml-openvino.h\n include/gguf.h)\n\nset_target_properties(ggml PROPERTIES PUBLIC_HEADER \"${GGML_PUBLIC_HEADERS}\")\n#if (GGML_METAL)\n# set_target_properties(ggml PROPERTIES RESOURCE \"${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal\")\n#endif()\ninstall(TARGETS ggml LIBRARY PUBLIC_HEADER)\ninstall(TARGETS ggml-base LIBRARY)\n\nif (GGML_STANDALONE)\n configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in\n ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc\n @ONLY)\n\n install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc\n DESTINATION share/pkgconfig)\nendif()\n\n#\n# Create CMake package\n#\n\n\n\n# Capture variables prefixed with GGML_.\n\nset(variable_set_statements\n\"\n####### Expanded from @GGML_VARIABLES_EXPANED@ by configure_package_config_file() #######\n####### Any changes to this file will be overwritten by the next CMake run #######\n\n\")\n\nset(GGML_SHARED_LIB ${BUILD_SHARED_LIBS})\n\nget_cmake_property(all_variables VARIABLES)\nforeach(variable_name IN LISTS all_variables)\n if(variable_name MATCHES \"^GGML_\")\n string(REPLACE \";\" \"\\\\;\"\n variable_value \"${${variable_name}}\")\n\n set(variable_set_statements\n \"${variable_set_statements}set(${variable_name} \\\"${variable_value}\\\")\\n\")\n endif()\nendforeach()\n\nset(GGML_VARIABLES_EXPANDED ${variable_set_statements})\n\n# Create the CMake package and set install location.\n\nset(GGML_INSTALL_VERSION ${GGML_VERSION})\nset(GGML_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH \"Location of header files\")\nset(GGML_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH \"Location of library files\")\nset(GGML_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH \"Location of binary files\")\n\nconfigure_package_config_file(\n ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-config.cmake.in\n ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake\n INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml\n PATH_VARS GGML_INCLUDE_INSTALL_DIR\n GGML_LIB_INSTALL_DIR\n GGML_BIN_INSTALL_DIR)\n\nwrite_basic_package_version_file(\n ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake\n VERSION ${GGML_INSTALL_VERSION}\n COMPATIBILITY SameMajorVersion)\n\ntarget_compile_definitions(ggml-base PRIVATE\n GGML_VERSION=\"${GGML_INSTALL_VERSION}\"\n GGML_COMMIT=\"${GGML_BUILD_COMMIT}\"\n)\nmessage(STATUS \"ggml version: ${GGML_INSTALL_VERSION}\")\nmessage(STATUS \"ggml commit: ${GGML_BUILD_COMMIT}\")\n\ninstall(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake\n ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake\n DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml)\n\nif (MSVC)\n set(MSVC_WARNING_FLAGS\n /wd4005 # Macro redefinition\n /wd4244 # Conversion from one type to another type, possible loss of data\n /wd4267 # Conversion from 'size_t' to a smaller type, possible loss of data\n /wd4305 # Conversion from 'type1' to 'type2', possible loss of data\n /wd4566 # Conversion from 'char' to 'wchar_t', possible loss of data\n /wd4996 # Disable POSIX deprecation warnings\n /wd4702 # Unreachable code warnings\n )\n set(MSVC_COMPILE_OPTIONS\n \"$<$:/utf-8>\"\n \"$<$:/utf-8>\"\n )\n function(configure_msvc_target target_name)\n if(TARGET ${target_name})\n target_compile_options(${target_name} PRIVATE ${MSVC_WARNING_FLAGS})\n target_compile_options(${target_name} PRIVATE ${MSVC_COMPILE_OPTIONS})\n endif()\n endfunction()\n\n configure_msvc_target(ggml-base)\n configure_msvc_target(ggml)\n configure_msvc_target(ggml-cpu)\n configure_msvc_target(ggml-cpu-x64)\n configure_msvc_target(ggml-cpu-sse42)\n configure_msvc_target(ggml-cpu-sandybridge)\n # __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512\n # skipping ggml-cpu-ivybridge\n # skipping ggml-cpu-piledriver\n configure_msvc_target(ggml-cpu-haswell)\n configure_msvc_target(ggml-cpu-skylakex)\n configure_msvc_target(ggml-cpu-cannonlake)\n configure_msvc_target(ggml-cpu-cascadelake)\n configure_msvc_target(ggml-cpu-icelake)\n # MSVC 2022 doesn't support BF16 intrinsics without `/arch:AVX10.1` ?!\n # https://learn.microsoft.com/en-us/cpp/intrinsics/x64-amd64-intrinsics-list?view=msvc-170\n # https://learn.microsoft.com/en-us/cpp/build/reference/arch-x64?view=msvc-170\n # skipping ggml-cpu-cooperlake\n # skipping ggml-cpu-zen4\n configure_msvc_target(ggml-cpu-alderlake)\n # MSVC doesn't support AMX\n # skipping ggml-cpu-sapphirerapids\n\n if (GGML_BUILD_EXAMPLES)\n configure_msvc_target(common-ggml)\n configure_msvc_target(common)\n\n configure_msvc_target(mnist-common)\n configure_msvc_target(mnist-eval)\n configure_msvc_target(mnist-train)\n\n configure_msvc_target(gpt-2-ctx)\n configure_msvc_target(gpt-2-alloc)\n configure_msvc_target(gpt-2-backend)\n configure_msvc_target(gpt-2-sched)\n configure_msvc_target(gpt-2-quantize)\n configure_msvc_target(gpt-2-batched)\n\n configure_msvc_target(gpt-j)\n configure_msvc_target(gpt-j-quantize)\n\n configure_msvc_target(magika)\n configure_msvc_target(yolov3-tiny)\n configure_msvc_target(sam)\n\n configure_msvc_target(simple-ctx)\n configure_msvc_target(simple-backend)\n endif()\n\n if (GGML_BUILD_TESTS)\n configure_msvc_target(test-mul-mat)\n configure_msvc_target(test-arange)\n configure_msvc_target(test-backend-ops)\n configure_msvc_target(test-cont)\n configure_msvc_target(test-conv-transpose)\n configure_msvc_target(test-conv-transpose-1d)\n configure_msvc_target(test-conv1d)\n configure_msvc_target(test-conv2d)\n configure_msvc_target(test-conv2d-dw)\n configure_msvc_target(test-customop)\n configure_msvc_target(test-dup)\n configure_msvc_target(test-opt)\n configure_msvc_target(test-pool)\n endif ()\nendif()\n" }, { "path": "CONTRIBUTING.md", "content": "Please use [llama.cpp's contribution guidelines](https://github.com/ggml-org/llama.cpp/blob/master/CONTRIBUTING.md) for this project.\n\n*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.*\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2023-2026 The ggml authors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "# ggml\n\n[Manifesto](https://github.com/ggerganov/llama.cpp/discussions/205)\n\nTensor library for machine learning\n\n***Note that this project is under active development. \\\nSome of the development is currently happening in the [llama.cpp](https://github.com/ggerganov/llama.cpp) and [whisper.cpp](https://github.com/ggerganov/whisper.cpp) repos***\n\n## Features\n\n- Low-level cross-platform implementation\n- Integer quantization support\n- Broad hardware support\n- Automatic differentiation\n- ADAM and L-BFGS optimizers\n- No third-party dependencies\n- Zero memory allocations during runtime\n\n## Build\n\n```bash\ngit clone https://github.com/ggml-org/ggml\ncd ggml\n\n# install python dependencies in a virtual environment\npython3.10 -m venv .venv\nsource .venv/bin/activate\npip install -r requirements.txt\n\n# build the examples\nmkdir build && cd build\ncmake ..\ncmake --build . --config Release -j 8\n```\n\n## GPT inference (example)\n\n```bash\n# run the GPT-2 small 117M model\n../examples/gpt-2/download-ggml-model.sh 117M\n./bin/gpt-2-backend -m models/gpt-2-117M/ggml-model.bin -p \"This is an example\"\n```\n\nFor more information, checkout the corresponding programs in the [examples](examples) folder.\n\n## Resources\n\n- [Introduction to ggml](https://huggingface.co/blog/introduction-to-ggml)\n- [The GGUF file format](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md)\n" }, { "path": "ci/run.sh", "content": "#/bin/bash\n#\n# sample usage:\n#\n# mkdir tmp\n#\n# # CPU-only build\n# bash ./ci/run.sh ./tmp/results ./tmp/mnt\n#\n# # with CUDA support\n# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n#\n# # With SYCL support\n# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n#\n\nif [ -z \"$2\" ]; then\n echo \"usage: $0 \"\n exit 1\nfi\n\nmkdir -p \"$1\"\nmkdir -p \"$2\"\n\nOUT=$(realpath \"$1\")\nMNT=$(realpath \"$2\")\n\nrm -v $OUT/*.log\nrm -v $OUT/*.exit\nrm -v $OUT/*.md\n\nsd=`dirname $0`\ncd $sd/../\nSRC=`pwd`\n\nCMAKE_EXTRA=\"\"\nCTEST_EXTRA=\"\"\n\nif [ ! -z ${GG_BUILD_METAL} ]; then\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_METAL=ON\"\nfi\n\nif [ ! -z ${GG_BUILD_CUDA} ]; then\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_CUDA=ON\"\n\n if command -v nvidia-smi >/dev/null 2>&1; then\n CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')\n if [[ -n \"$CUDA_ARCH\" && \"$CUDA_ARCH\" =~ ^[0-9]+$ ]]; then\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}\"\n else\n echo \"Warning: Using fallback CUDA architectures\"\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89\"\n fi\n else\n echo \"Error: nvidia-smi not found, cannot build with CUDA\"\n exit 1\n fi\nfi\n\nif [ ! -z ${GG_BUILD_ROCM} ]; then\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_HIP=ON\"\n if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then\n echo \"Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)\"\n exit 1\n fi\n\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DAMDGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}\"\nfi\n\nif [ ! -z ${GG_BUILD_SYCL} ]; then\n if [ -z ${ONEAPI_ROOT} ]; then\n echo \"Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:\"\n echo \"source /opt/intel/oneapi/setvars.sh\"\n exit 1\n fi\n # Use only main GPU\n export ONEAPI_DEVICE_SELECTOR=\"level_zero:0\"\n # Enable sysman for correct memory reporting\n export ZES_ENABLE_SYSMAN=1\n # to circumvent precision issues on CPY operations\n export SYCL_PROGRAM_COMPILE_OPTIONS=\"-cl-fp32-correctly-rounded-divide-sqrt\"\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON\"\nfi\n\nif [ ! -z ${GG_BUILD_VULKAN} ]; then\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_VULKAN=1\"\n\n # if on Mac, disable METAL\n if [[ \"$OSTYPE\" == \"darwin\"* ]]; then\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_METAL=OFF -DGGML_BLAS=OFF\"\n fi\n\nfi\n\nif [ ! -z ${GG_BUILD_WEBGPU} ]; then\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_WEBGPU=1\"\nfi\n\nif [ ! -z ${GG_BUILD_MUSA} ]; then\n # Use qy1 by default (MTT S80)\n MUSA_ARCH=${MUSA_ARCH:-21}\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}\"\nfi\n\nif [ ! -z ${GG_BUILD_NO_SVE} ]; then\n # arm 9 and newer enables sve by default, adjust these flags depending on the cpu used\n CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm\"\nfi\n\n## helpers\n\n# download a file if it does not exist or if it is outdated\nfunction gg_wget {\n local out=$1\n local url=$2\n\n local cwd=`pwd`\n\n mkdir -p $out\n cd $out\n\n # should not re-download if file is the same\n wget -nv -N $url\n\n cd $cwd\n}\n\nfunction gg_printf {\n printf -- \"$@\" >> $OUT/README.md\n}\n\nfunction gg_run {\n ci=$1\n\n set -o pipefail\n set -x\n\n gg_run_$ci | tee $OUT/$ci.log\n cur=$?\n echo \"$cur\" > $OUT/$ci.exit\n\n set +x\n set +o pipefail\n\n gg_sum_$ci\n\n ret=$((ret | cur))\n}\n\n## ci\n\n# ctest_debug\n\nfunction gg_run_ctest_debug {\n cd ${SRC}\n\n rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug\n\n set -e\n\n (time cmake -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log\n (time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log\n\n (time ctest ${CTEST_EXTRA} --output-on-failure -E \"test-opt|test-backend-ops\" ) 2>&1 | tee -a $OUT/${ci}-ctest.log\n\n set +e\n}\n\nfunction gg_sum_ctest_debug {\n gg_printf '### %s\\n\\n' \"${ci}\"\n\n gg_printf 'Runs ctest in debug mode\\n'\n gg_printf '- status: %s\\n' \"$(cat $OUT/${ci}.exit)\"\n gg_printf '```\\n'\n gg_printf '%s\\n' \"$(cat $OUT/${ci}-ctest.log)\"\n gg_printf '```\\n'\n gg_printf '\\n'\n}\n\n# ctest_release\n\nfunction gg_run_ctest_release {\n cd ${SRC}\n\n rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release\n\n set -e\n\n (time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log\n (time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log\n\n if [ -z $GG_BUILD_LOW_PERF ]; then\n (time ctest ${CTEST_EXTRA} --output-on-failure ) 2>&1 | tee -a $OUT/${ci}-ctest.log\n else\n (time ctest ${CTEST_EXTRA} --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log\n fi\n\n set +e\n}\n\nfunction gg_sum_ctest_release {\n gg_printf '### %s\\n\\n' \"${ci}\"\n\n gg_printf 'Runs ctest in release mode\\n'\n gg_printf '- status: %s\\n' \"$(cat $OUT/${ci}.exit)\"\n gg_printf '```\\n'\n gg_printf '%s\\n' \"$(cat $OUT/${ci}-ctest.log)\"\n gg_printf '```\\n'\n}\n\n# gpt_2\n\nfunction gg_run_gpt_2 {\n cd ${SRC}\n\n gg_wget models-mnt/gpt-2 https://huggingface.co/ggerganov/ggml/resolve/main/ggml-model-gpt-2-117M.bin\n\n cd build-ci-release\n\n set -e\n\n model=\"../models-mnt/gpt-2/ggml-model-gpt-2-117M.bin\"\n prompts=\"../examples/prompts/gpt-2.txt\"\n\n (time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -tt ${prompts} ) 2>&1 | tee -a $OUT/${ci}-tg.log\n (time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -p \"I believe the meaning of life is\") 2>&1 | tee -a $OUT/${ci}-tg.log\n (time ./bin/gpt-2-sched --model ${model} -s 1234 -n 64 -p \"I believe the meaning of life is\") 2>&1 | tee -a $OUT/${ci}-tg.log\n\n (time ./bin/gpt-2-batched --model ${model} -s 1234 -n 64 -np 8 -p \"I believe the meaning of life is\") 2>&1 | tee -a $OUT/${ci}-tg.log\n\n set +e\n}\n\nfunction gg_sum_gpt_2 {\n gg_printf '### %s\\n\\n' \"${ci}\"\n\n gg_printf 'Runs short GPT-2 text generation\\n'\n gg_printf '- status: %s\\n' \"$(cat $OUT/${ci}.exit)\"\n gg_printf '```\\n'\n gg_printf '%s\\n' \"$(cat $OUT/${ci}-tg.log)\"\n gg_printf '```\\n'\n}\n\n# TODO: update\n## mnist\n#\n#function gg_run_mnist {\n# cd ${SRC}\n#\n# cd build-ci-release\n#\n# set -e\n#\n# mkdir -p models/mnist\n# python3 ../examples/mnist/convert-h5-to-ggml.py ../examples/mnist/models/mnist/mnist_model.state_dict\n#\n# model_f32=\"./models/mnist/ggml-model-f32.bin\"\n# samples=\"../examples/mnist/models/mnist/t10k-images.idx3-ubyte\"\n#\n# # first command runs and exports \"mnist.ggml\", the second command runs the exported model\n#\n# (time ./bin/mnist ${model_f32} ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log\n# (time ./bin/mnist-cpu ./mnist.ggml ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log\n#\n# set +e\n#}\n#\n#function gg_sum_mnist {\n# gg_printf '### %s\\n\\n' \"${ci}\"\n#\n# gg_printf 'MNIST\\n'\n# gg_printf '- status: %s\\n' \"$(cat $OUT/${ci}.exit)\"\n# gg_printf '```\\n'\n# gg_printf '%s\\n' \"$(cat $OUT/${ci}-mnist.log)\"\n# gg_printf '```\\n'\n#}\n\n# sam\n\nfunction gg_run_sam {\n cd ${SRC}\n\n gg_wget models-mnt/sam/ https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth\n gg_wget models-mnt/sam/ https://raw.githubusercontent.com/YavorGIvanov/sam.cpp/ceafb7467bff7ec98e0c4f952e58a9eb8fd0238b/img.jpg\n\n cd build-ci-release\n\n set -e\n\n path_models=\"../models-mnt/sam/\"\n model_f16=\"${path_models}/ggml-model-f16.bin\"\n img_0=\"${path_models}/img.jpg\"\n\n python3 ../examples/sam/convert-pth-to-ggml.py ${path_models}/sam_vit_b_01ec64.pth ${path_models}/ 1\n\n # Test default parameters\n (time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 ) 2>&1 | tee -a $OUT/${ci}-main.log\n grep -q \"point prompt\" $OUT/${ci}-main.log\n grep -q \"bbox (371, 436), (144, 168)\" $OUT/${ci}-main.log ||\n grep -q \"bbox (370, 439), (144, 168)\" $OUT/${ci}-main.log\n\n # Test box prompt and single mask output\n (time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 -b 368,144,441,173 -sm) 2>&1 | tee -a $OUT/${ci}-main.log\n grep -q \"box prompt\" $OUT/${ci}-main.log\n grep -q \"bbox (370, 439), (144, 169)\" $OUT/${ci}-main.log ||\n grep -q \"bbox (370, 439), (144, 168)\" $OUT/${ci}-main.log\n\n set +e\n}\n\nfunction gg_sum_sam {\n gg_printf '### %s\\n\\n' \"${ci}\"\n\n gg_printf 'Run SAM\\n'\n gg_printf '- status: %s\\n' \"$(cat $OUT/${ci}.exit)\"\n gg_printf '```\\n'\n gg_printf '%s\\n' \"$(cat $OUT/${ci}-main.log)\"\n gg_printf '```\\n'\n}\n\n# yolo\n\nfunction gg_run_yolo {\n cd ${SRC}\n\n gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/yolov3-tiny.weights\n gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/dog.jpg\n\n cd build-ci-release\n cp -r ../examples/yolo/data .\n\n set -e\n\n path_models=\"../models-mnt/yolo/\"\n\n python3 ../examples/yolo/convert-yolov3-tiny.py ${path_models}/yolov3-tiny.weights\n\n (time ./bin/yolov3-tiny -m yolov3-tiny.gguf -i ${path_models}/dog.jpg ) 2>&1 | tee -a $OUT/${ci}-main.log\n\n grep -qE \"dog: (55|56|57|58|59)%\" $OUT/${ci}-main.log\n grep -qE \"car: (50|51|52|53|54)%\" $OUT/${ci}-main.log\n grep -qE \"truck: (54|55|56|57|58)%\" $OUT/${ci}-main.log\n grep -qE \"bicycle: (57|58|59|60|61)%\" $OUT/${ci}-main.log\n\n set +e\n}\n\nfunction gg_sum_yolo {\n gg_printf '### %s\\n\\n' \"${ci}\"\n\n gg_printf 'Run YOLO\\n'\n gg_printf '- status: %s\\n' \"$(cat $OUT/${ci}.exit)\"\n gg_printf '```\\n'\n gg_printf '%s\\n' \"$(cat $OUT/${ci}-main.log)\"\n gg_printf '```\\n'\n}\n\n## main\n\nif true ; then\n # Create symlink: ./ggml/models-mnt -> $MNT/models/models-mnt\n rm -rf ${SRC}/models-mnt\n mnt_models=${MNT}/models\n mkdir -p ${mnt_models}\n ln -sfn ${mnt_models} ${SRC}/models-mnt\n\n # Create a fresh python3 venv and enter it\n if ! python3 -m venv \"$MNT/venv\"; then\n echo \"Error: Failed to create Python virtual environment at $MNT/venv.\"\n exit 1\n fi\n source \"$MNT/venv/bin/activate\"\n\n pip install -r ${SRC}/requirements.txt --disable-pip-version-check\nfi\n\n\nret=0\n\ntest $ret -eq 0 && gg_run ctest_debug\ntest $ret -eq 0 && gg_run ctest_release\n\ntest $ret -eq 0 && gg_run gpt_2\n#test $ret -eq 0 && gg_run mnist\ntest $ret -eq 0 && gg_run sam\ntest $ret -eq 0 && gg_run yolo\n\nif [ -z $GG_BUILD_LOW_PERF ]; then\n # run tests meant for low-perf runners\n date\nfi\n\ncat $OUT/README.md\n\nexit $ret\n" }, { "path": "cmake/GitVars.cmake", "content": "find_package(Git)\n\n# the commit's SHA1\nexecute_process(COMMAND\n \"${GIT_EXECUTABLE}\" describe --match=NeVeRmAtCh --always --abbrev=8\n WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n OUTPUT_VARIABLE GIT_SHA1\n ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n\n# the date of the commit\nexecute_process(COMMAND\n \"${GIT_EXECUTABLE}\" log -1 --format=%ad --date=local\n WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n OUTPUT_VARIABLE GIT_DATE\n ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n\n# the subject of the commit\nexecute_process(COMMAND\n \"${GIT_EXECUTABLE}\" log -1 --format=%s\n WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n OUTPUT_VARIABLE GIT_COMMIT_SUBJECT\n ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n" }, { "path": "cmake/common.cmake", "content": "function(ggml_get_flags CCID CCVER)\n set(C_FLAGS \"\")\n set(CXX_FLAGS \"\")\n\n if (CCID MATCHES \"Clang\")\n set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return)\n set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)\n\n if (\n (CCID STREQUAL \"Clang\" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR\n (CCID STREQUAL \"AppleClang\" AND CCVER VERSION_GREATER_EQUAL 7.3.0)\n )\n list(APPEND C_FLAGS -Wdouble-promotion)\n endif()\n elseif (CCID STREQUAL \"GNU\")\n set(C_FLAGS -Wdouble-promotion)\n set(CXX_FLAGS -Wno-array-bounds)\n\n if (CCVER VERSION_GREATER_EQUAL 8.1.0)\n list(APPEND CXX_FLAGS -Wextra-semi)\n endif()\n endif()\n\n set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE)\n set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)\nendfunction()\n\nfunction(ggml_get_system_arch)\n if (CMAKE_OSX_ARCHITECTURES STREQUAL \"arm64\" OR\n CMAKE_GENERATOR_PLATFORM_LWR STREQUAL \"arm64\" OR\n (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND\n CMAKE_SYSTEM_PROCESSOR MATCHES \"^(aarch64|arm.*|ARM64)$\"))\n set(GGML_SYSTEM_ARCH \"ARM\" PARENT_SCOPE)\n elseif (CMAKE_OSX_ARCHITECTURES STREQUAL \"x86_64\" OR\n CMAKE_GENERATOR_PLATFORM_LWR MATCHES \"^(x86_64|i686|amd64|x64|win32)$\" OR\n (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND\n CMAKE_SYSTEM_PROCESSOR MATCHES \"^(x86_64|i686|AMD64|amd64)$\"))\n set(GGML_SYSTEM_ARCH \"x86\" PARENT_SCOPE)\n elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"ppc|power\")\n set(GGML_SYSTEM_ARCH \"PowerPC\" PARENT_SCOPE)\n elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"loongarch64\")\n set(GGML_SYSTEM_ARCH \"loongarch64\" PARENT_SCOPE)\n elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"riscv64\")\n set(GGML_SYSTEM_ARCH \"riscv64\" PARENT_SCOPE)\n elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"s390x\")\n set(GGML_SYSTEM_ARCH \"s390x\" PARENT_SCOPE)\n else()\n set(GGML_SYSTEM_ARCH \"UNKNOWN\" PARENT_SCOPE)\n endif()\nendfunction()\n" }, { "path": "cmake/ggml-config.cmake.in", "content": "@PACKAGE_INIT@\n\n@GGML_VARIABLES_EXPANDED@\n\n# Find all dependencies before creating any target.\ninclude(CMakeFindDependencyMacro)\nfind_dependency(Threads)\nif (NOT GGML_SHARED_LIB)\n set(GGML_CPU_INTERFACE_LINK_LIBRARIES \"\")\n set(GGML_CPU_INTERFACE_LINK_OPTIONS \"\")\n\n if (APPLE AND GGML_ACCELERATE)\n find_library(ACCELERATE_FRAMEWORK Accelerate)\n if(NOT ACCELERATE_FRAMEWORK)\n set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)\n return()\n endif()\n list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${ACCELERATE_FRAMEWORK})\n endif()\n\n if (GGML_OPENMP_ENABLED)\n find_dependency(OpenMP)\n list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_C OpenMP::OpenMP_CXX)\n endif()\n\n if (GGML_CPU_HBM)\n find_library(memkind memkind)\n if(NOT memkind)\n set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)\n return()\n endif()\n list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES memkind)\n endif()\n\n if (GGML_BLAS)\n find_dependency(BLAS)\n list(APPEND GGML_BLAS_INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES})\n list(APPEND GGML_BLAS_INTERFACE_LINK_OPTIONS ${BLAS_LINKER_FLAGS})\n endif()\n\n if (GGML_CUDA)\n set(GGML_CUDA_INTERFACE_LINK_LIBRARIES \"\")\n find_dependency(CUDAToolkit)\n if (GGML_STATIC)\n list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $)\n if (WIN32)\n list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $ $)\n else()\n list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $ $)\n endif()\n endif()\n if (NOT GGML_CUDA_NO_VMM)\n list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $)\n endif()\n endif()\n\n if (GGML_METAL)\n find_library(FOUNDATION_LIBRARY Foundation)\n find_library(METAL_FRAMEWORK Metal)\n find_library(METALKIT_FRAMEWORK MetalKit)\n if(NOT FOUNDATION_LIBRARY OR NOT METAL_FRAMEWORK OR NOT METALKIT_FRAMEWORK)\n set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)\n return()\n endif()\n set(GGML_METAL_INTERFACE_LINK_LIBRARIES\n ${FOUNDATION_LIBRARY} ${METAL_FRAMEWORK} ${METALKIT_FRAMEWORK})\n endif()\n\n if (GGML_OPENCL)\n find_dependency(OpenCL)\n set(GGML_OPENCL_INTERFACE_LINK_LIBRARIES $)\n endif()\n\n if (GGML_VULKAN)\n find_dependency(Vulkan)\n set(GGML_VULKAN_INTERFACE_LINK_LIBRARIES $)\n endif()\n\n if (GGML_HIP)\n find_dependency(hip)\n find_dependency(hipblas)\n find_dependency(rocblas)\n set(GGML_HIP_INTERFACE_LINK_LIBRARIES hip::host roc::rocblas roc::hipblas)\n endif()\n\n if (GGML_SYCL)\n set(GGML_SYCL_INTERFACE_LINK_LIBRARIES \"\")\n find_package(DNNL)\n if (${DNNL_FOUND} AND GGML_SYCL_TARGET STREQUAL \"INTEL\")\n list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES DNNL::dnnl)\n endif()\n if (WIN32)\n find_dependency(IntelSYCL)\n find_dependency(MKL)\n list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL)\n endif()\n endif()\nendif()\n\nset_and_check(GGML_INCLUDE_DIR \"@PACKAGE_GGML_INCLUDE_INSTALL_DIR@\")\nset_and_check(GGML_LIB_DIR \"@PACKAGE_GGML_LIB_INSTALL_DIR@\")\n#set_and_check(GGML_BIN_DIR \"@PACKAGE_GGML_BIN_INSTALL_DIR@\")\n\nif(NOT TARGET ggml::ggml)\n find_package(Threads REQUIRED)\n\n find_library(GGML_LIBRARY ggml\n REQUIRED\n HINTS ${GGML_LIB_DIR}\n NO_CMAKE_FIND_ROOT_PATH)\n\n add_library(ggml::ggml UNKNOWN IMPORTED)\n set_target_properties(ggml::ggml\n PROPERTIES\n IMPORTED_LOCATION \"${GGML_LIBRARY}\")\n\n find_library(GGML_BASE_LIBRARY ggml-base\n REQUIRED\n HINTS ${GGML_LIB_DIR}\n NO_CMAKE_FIND_ROOT_PATH)\n\n add_library(ggml::ggml-base UNKNOWN IMPORTED)\n set_target_properties(ggml::ggml-base\n PROPERTIES\n IMPORTED_LOCATION \"${GGML_BASE_LIBRARY}\")\n\n set(_ggml_all_targets \"\")\n if (NOT GGML_BACKEND_DL)\n foreach(_ggml_backend ${GGML_AVAILABLE_BACKENDS})\n string(REPLACE \"-\" \"_\" _ggml_backend_pfx \"${_ggml_backend}\")\n string(TOUPPER \"${_ggml_backend_pfx}\" _ggml_backend_pfx)\n\n find_library(${_ggml_backend_pfx}_LIBRARY ${_ggml_backend}\n REQUIRED\n HINTS ${GGML_LIB_DIR}\n NO_CMAKE_FIND_ROOT_PATH)\n\n message(STATUS \"Found ${${_ggml_backend_pfx}_LIBRARY}\")\n\n add_library(ggml::${_ggml_backend} UNKNOWN IMPORTED)\n set_target_properties(ggml::${_ggml_backend}\n PROPERTIES\n INTERFACE_INCLUDE_DIRECTORIES \"${GGML_INCLUDE_DIR}\"\n IMPORTED_LINK_INTERFACE_LANGUAGES \"CXX\"\n IMPORTED_LOCATION \"${${_ggml_backend_pfx}_LIBRARY}\"\n INTERFACE_COMPILE_FEATURES c_std_90\n POSITION_INDEPENDENT_CODE ON)\n\n string(REGEX MATCH \"^ggml-cpu\" is_cpu_variant \"${_ggml_backend}\")\n if(is_cpu_variant)\n list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES \"ggml::ggml-base\")\n set_target_properties(ggml::${_ggml_backend}\n PROPERTIES\n INTERFACE_LINK_LIBRARIES \"${GGML_CPU_INTERFACE_LINK_LIBRARIES}\")\n\n if(GGML_CPU_INTERFACE_LINK_OPTIONS)\n set_target_properties(ggml::${_ggml_backend}\n PROPERTIES\n INTERFACE_LINK_OPTIONS \"${GGML_CPU_INTERFACE_LINK_OPTIONS}\")\n endif()\n\n else()\n list(APPEND ${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES \"ggml::ggml-base\")\n set_target_properties(ggml::${_ggml_backend}\n PROPERTIES\n INTERFACE_LINK_LIBRARIES \"${${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES}\")\n\n if(${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS)\n set_target_properties(ggml::${_ggml_backend}\n PROPERTIES\n INTERFACE_LINK_OPTIONS \"${${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS}\")\n endif()\n endif()\n\n list(APPEND _ggml_all_targets ggml::${_ggml_backend})\n endforeach()\n endif()\n\n list(APPEND GGML_INTERFACE_LINK_LIBRARIES ggml::ggml-base \"${_ggml_all_targets}\")\n set_target_properties(ggml::ggml\n PROPERTIES\n INTERFACE_LINK_LIBRARIES \"${GGML_INTERFACE_LINK_LIBRARIES}\")\n\n add_library(ggml::all INTERFACE IMPORTED)\n set_target_properties(ggml::all\n PROPERTIES\n INTERFACE_LINK_LIBRARIES \"${_ggml_all_targets}\")\n\nendif()\n\ncheck_required_components(ggml)\n" }, { "path": "docs/gguf.md", "content": "# GGUF\n\nGGUF is a file format for storing models for inference with GGML and executors based on GGML. GGUF is a binary format that is designed for fast loading and saving of models, and for ease of reading. Models are traditionally developed using PyTorch or another framework, and then converted to GGUF for use in GGML.\n\nIt is a successor file format to GGML, GGMF and GGJT, and is designed to be unambiguous by containing all the information needed to load a model. It is also designed to be extensible, so that new information can be added to models without breaking compatibility.\n\nFor more information about the motivation behind GGUF, see [Historical State of Affairs](#historical-state-of-affairs).\n\n## Specification\n\nGGUF is a format based on the existing GGJT, but makes a few changes to the format to make it more extensible and easier to use. The following features are desired:\n\n- Single-file deployment: they can be easily distributed and loaded, and do not require any external files for additional information.\n- Extensible: new features can be added to GGML-based executors/new information can be added to GGUF models without breaking compatibility with existing models.\n- `mmap` compatibility: models can be loaded using `mmap` for fast loading and saving.\n- Easy to use: models can be easily loaded and saved using a small amount of code, with no need for external libraries, regardless of the language used.\n- Full information: all information needed to load a model is contained in the model file, and no additional information needs to be provided by the user.\n\nThe key difference between GGJT and GGUF is the use of a key-value structure for the hyperparameters (now referred to as metadata), rather than a list of untyped values. This allows for new metadata to be added without breaking compatibility with existing models, and to annotate the model with additional information that may be useful for inference or for identifying the model.\n\n### GGUF Naming Convention\n\nGGUF follow a naming convention of `.gguf` where each component is delimitated by a `-` if present. Ultimately this is intended to make it easier for humans to at a glance get the most important details of a model. It is not intended to be perfectly parsable in the field due to the diversity of existing gguf filenames.\n\nThe components are:\n1. **BaseName**: A descriptive name for the model base type or architecture.\n - This can be derived from gguf metadata `general.basename` substituting spaces for dashes.\n1. **SizeLabel**: Parameter weight class (useful for leader boards) represented as `x`\n - This can be derived from gguf metadata `general.size_label` if available or calculated if missing.\n - Rounded decimal point is supported in count with a single letter scale prefix to assist in floating point exponent shown below\n - `Q`: Quadrillion parameters.\n - `T`: Trillion parameters.\n - `B`: Billion parameters.\n - `M`: Million parameters.\n - `K`: Thousand parameters.\n - Additional `-` can be appended as needed to indicate other attributes of interest\n1. **FineTune**: A descriptive name for the model fine tuning goal (e.g. Chat, Instruct, etc...)\n - This can be derived from gguf metadata `general.finetune` substituting spaces for dashes.\n1. **Version**: (Optional) Denotes the model version number, formatted as `v.`\n - If model is missing a version number then assume `v1.0` (First Public Release)\n - This can be derived from gguf metadata `general.version`\n1. **Encoding**: Indicates the weights encoding scheme that was applied to the model. Content, type mixture and arrangement however are determined by user code and can vary depending on project needs.\n1. **Type**: Indicates the kind of gguf file and the intended purpose for it\n - If missing, then file is by default a typical gguf tensor model file\n - `LoRA` : GGUF file is a LoRA adapter\n - `vocab` : GGUF file with only vocab data and metadata\n1. **Shard**: (Optional) Indicates and denotes that the model has been split into multiple shards, formatted as `-of-`.\n - *ShardNum* : Shard position in this model. Must be 5 digits padded by zeros.\n - Shard number always starts from `00001` onwards (e.g. First shard always starts at `00001-of-XXXXX` rather than `00000-of-XXXXX`).\n - *ShardTotal* : Total number of shards in this model. Must be 5 digits padded by zeros.\n\n\n#### Validating Above Naming Convention\n\nAt a minimum all model files should have at least BaseName, SizeLabel, Version, in order to be easily validated as a file that is keeping with the GGUF Naming Convention. An example of this issue is that it is easy for Encoding to be mistaken as a FineTune if Version is omitted.\n\nTo validate you can use this regular expression `^(?[A-Za-z0-9\\s]*(?:(?:-(?:(?:[A-Za-z\\s][A-Za-z0-9\\s]*)|(?:[0-9\\s]*)))*))-(?:(?(?:\\d+x)?(?:\\d+\\.)?\\d+[A-Za-z](?:-[A-Za-z]+(\\d+\\.)?\\d+[A-Za-z]+)?)(?:-(?[A-Za-z0-9\\s-]+))?)?-(?:(?v\\d+(?:\\.\\d+)*))(?:-(?(?!LoRA|vocab)[\\w_]+))?(?:-(?LoRA|vocab))?(?:-(?\\d{5}-of-\\d{5}))?\\.gguf$` which will check that you got the minimum BaseName, SizeLabel and Version present in the correct order.\n\nFor example:\n\n * `Mixtral-8x7B-v0.1-KQ2.gguf`:\n - Model Name: Mixtral\n - Expert Count: 8\n - Parameter Count: 7B\n - Version Number: v0.1\n - Weight Encoding Scheme: KQ2\n\n * `Hermes-2-Pro-Llama-3-8B-F16.gguf`:\n - Model Name: Hermes 2 Pro Llama 3\n - Expert Count: 0\n - Parameter Count: 8B\n - Version Number: v1.0\n - Weight Encoding Scheme: F16\n - Shard: N/A\n\n * `Grok-100B-v1.0-Q4_0-00003-of-00009.gguf`\n - Model Name: Grok\n - Expert Count: 0\n - Parameter Count: 100B\n - Version Number: v1.0\n - Weight Encoding Scheme: Q4_0\n - Shard: 3 out of 9 total shards\n\n\n
Example Node.js Regex Function\n\n```js\n#!/usr/bin/env node\nconst ggufRegex = /^(?[A-Za-z0-9\\s]*(?:(?:-(?:(?:[A-Za-z\\s][A-Za-z0-9\\s]*)|(?:[0-9\\s]*)))*))-(?:(?(?:\\d+x)?(?:\\d+\\.)?\\d+[A-Za-z](?:-[A-Za-z]+(\\d+\\.)?\\d+[A-Za-z]+)?)(?:-(?[A-Za-z0-9\\s-]+))?)?-(?:(?v\\d+(?:\\.\\d+)*))(?:-(?(?!LoRA|vocab)[\\w_]+))?(?:-(?LoRA|vocab))?(?:-(?\\d{5}-of-\\d{5}))?\\.gguf$/;\n\nfunction parseGGUFFilename(filename) {\n const match = ggufRegex.exec(filename);\n if (!match)\n return null;\n const {BaseName = null, SizeLabel = null, FineTune = null, Version = \"v1.0\", Encoding = null, Type = null, Shard = null} = match.groups;\n return {BaseName: BaseName, SizeLabel: SizeLabel, FineTune: FineTune, Version: Version, Encoding: Encoding, Type: Type, Shard: Shard};\n}\n\nconst testCases = [\n {filename: 'Mixtral-8x7B-v0.1-KQ2.gguf', expected: { BaseName: 'Mixtral', SizeLabel: '8x7B', FineTune: null, Version: 'v0.1', Encoding: 'KQ2', Type: null, Shard: null}},\n {filename: 'Grok-100B-v1.0-Q4_0-00003-of-00009.gguf', expected: { BaseName: 'Grok', SizeLabel: '100B', FineTune: null, Version: 'v1.0', Encoding: 'Q4_0', Type: null, Shard: \"00003-of-00009\"}},\n {filename: 'Hermes-2-Pro-Llama-3-8B-v1.0-F16.gguf', expected: { BaseName: 'Hermes-2-Pro-Llama-3', SizeLabel: '8B', FineTune: null, Version: 'v1.0', Encoding: 'F16', Type: null, Shard: null}},\n {filename: 'Phi-3-mini-3.8B-ContextLength4k-instruct-v1.0.gguf', expected: { BaseName: 'Phi-3-mini', SizeLabel: '3.8B-ContextLength4k', FineTune: 'instruct', Version: 'v1.0', Encoding: null, Type: null, Shard: null}},\n {filename: 'not-a-known-arrangement.gguf', expected: null},\n];\n\ntestCases.forEach(({ filename, expected }) => {\n const result = parseGGUFFilename(filename);\n const passed = JSON.stringify(result) === JSON.stringify(expected);\n console.log(`${filename}: ${passed ? \"PASS\" : \"FAIL\"}`);\n if (!passed) {\n console.log(result);\n console.log(expected);\n }\n});\n```\n\n
\n\n\n### File Structure\n\n![image](https://github.com/ggerganov/ggml/assets/1991296/c3623641-3a1d-408e-bfaf-1b7c4e16aa63)\n*diagram by [@mishig25](https://github.com/mishig25) (GGUF v3)*\n\nGGUF files are structured as follows. They use a global alignment specified in the `general.alignment` metadata field, referred to as `ALIGNMENT` below. Where required, the file is padded with `0x00` bytes to the next multiple of `general.alignment`.\n\nFields, including arrays, are written sequentially without alignment unless otherwise specified.\n\nModels are little-endian by default. They can also come in big-endian for use with big-endian computers; in this case, all values (including metadata values and tensors) will also be big-endian. At the time of writing, there is no way to determine if a model is big-endian; this may be rectified in future versions. If no additional information is provided, assume the model is little-endian.\n\n```c\nenum ggml_type: uint32_t {\n GGML_TYPE_F32 = 0,\n GGML_TYPE_F16 = 1,\n GGML_TYPE_Q4_0 = 2,\n GGML_TYPE_Q4_1 = 3,\n // GGML_TYPE_Q4_2 = 4, support has been removed\n // GGML_TYPE_Q4_3 = 5, support has been removed\n GGML_TYPE_Q5_0 = 6,\n GGML_TYPE_Q5_1 = 7,\n GGML_TYPE_Q8_0 = 8,\n GGML_TYPE_Q8_1 = 9,\n GGML_TYPE_Q2_K = 10,\n GGML_TYPE_Q3_K = 11,\n GGML_TYPE_Q4_K = 12,\n GGML_TYPE_Q5_K = 13,\n GGML_TYPE_Q6_K = 14,\n GGML_TYPE_Q8_K = 15,\n GGML_TYPE_IQ2_XXS = 16,\n GGML_TYPE_IQ2_XS = 17,\n GGML_TYPE_IQ3_XXS = 18,\n GGML_TYPE_IQ1_S = 19,\n GGML_TYPE_IQ4_NL = 20,\n GGML_TYPE_IQ3_S = 21,\n GGML_TYPE_IQ2_S = 22,\n GGML_TYPE_IQ4_XS = 23,\n GGML_TYPE_I8 = 24,\n GGML_TYPE_I16 = 25,\n GGML_TYPE_I32 = 26,\n GGML_TYPE_I64 = 27,\n GGML_TYPE_F64 = 28,\n GGML_TYPE_IQ1_M = 29,\n GGML_TYPE_BF16 = 30,\n // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files\n // GGML_TYPE_Q4_0_4_8 = 32,\n // GGML_TYPE_Q4_0_8_8 = 33,\n GGML_TYPE_TQ1_0 = 34,\n GGML_TYPE_TQ2_0 = 35,\n // GGML_TYPE_IQ4_NL_4_4 = 36,\n // GGML_TYPE_IQ4_NL_4_8 = 37,\n // GGML_TYPE_IQ4_NL_8_8 = 38,\n GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)\n GGML_TYPE_COUNT = 40,\n};\n\nenum gguf_metadata_value_type: uint32_t {\n // The value is a 8-bit unsigned integer.\n GGUF_METADATA_VALUE_TYPE_UINT8 = 0,\n // The value is a 8-bit signed integer.\n GGUF_METADATA_VALUE_TYPE_INT8 = 1,\n // The value is a 16-bit unsigned little-endian integer.\n GGUF_METADATA_VALUE_TYPE_UINT16 = 2,\n // The value is a 16-bit signed little-endian integer.\n GGUF_METADATA_VALUE_TYPE_INT16 = 3,\n // The value is a 32-bit unsigned little-endian integer.\n GGUF_METADATA_VALUE_TYPE_UINT32 = 4,\n // The value is a 32-bit signed little-endian integer.\n GGUF_METADATA_VALUE_TYPE_INT32 = 5,\n // The value is a 32-bit IEEE754 floating point number.\n GGUF_METADATA_VALUE_TYPE_FLOAT32 = 6,\n // The value is a boolean.\n // 1-byte value where 0 is false and 1 is true.\n // Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy.\n GGUF_METADATA_VALUE_TYPE_BOOL = 7,\n // The value is a UTF-8 non-null-terminated string, with length prepended.\n GGUF_METADATA_VALUE_TYPE_STRING = 8,\n // The value is an array of other values, with the length and type prepended.\n ///\n // Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes.\n GGUF_METADATA_VALUE_TYPE_ARRAY = 9,\n // The value is a 64-bit unsigned little-endian integer.\n GGUF_METADATA_VALUE_TYPE_UINT64 = 10,\n // The value is a 64-bit signed little-endian integer.\n GGUF_METADATA_VALUE_TYPE_INT64 = 11,\n // The value is a 64-bit IEEE754 floating point number.\n GGUF_METADATA_VALUE_TYPE_FLOAT64 = 12,\n};\n\n// A string in GGUF.\nstruct gguf_string_t {\n // The length of the string, in bytes.\n uint64_t len;\n // The string as a UTF-8 non-null-terminated string.\n char string[len];\n};\n\nunion gguf_metadata_value_t {\n uint8_t uint8;\n int8_t int8;\n uint16_t uint16;\n int16_t int16;\n uint32_t uint32;\n int32_t int32;\n float float32;\n uint64_t uint64;\n int64_t int64;\n double float64;\n bool bool_;\n gguf_string_t string;\n struct {\n // Any value type is valid, including arrays.\n gguf_metadata_value_type type;\n // Number of elements, not bytes\n uint64_t len;\n // The array of values.\n gguf_metadata_value_t array[len];\n } array;\n};\n\nstruct gguf_metadata_kv_t {\n // The key of the metadata. It is a standard GGUF string, with the following caveats:\n // - It must be a valid ASCII string.\n // - It must be a hierarchical key, where each segment is `lower_snake_case` and separated by a `.`.\n // - It must be at most 2^16-1/65535 bytes long.\n // Any keys that do not follow these rules are invalid.\n gguf_string_t key;\n\n // The type of the value.\n // Must be one of the `gguf_metadata_value_type` values.\n gguf_metadata_value_type value_type;\n // The value.\n gguf_metadata_value_t value;\n};\n\nstruct gguf_header_t {\n // Magic number to announce that this is a GGUF file.\n // Must be `GGUF` at the byte level: `0x47` `0x47` `0x55` `0x46`.\n // Your executor might do little-endian byte order, so it might be\n // check for 0x46554747 and letting the endianness cancel out.\n // Consider being *very* explicit about the byte order here.\n uint32_t magic;\n // The version of the format implemented.\n // Must be `3` for version described in this spec, which introduces big-endian support.\n //\n // This version should only be increased for structural changes to the format.\n // Changes that do not affect the structure of the file should instead update the metadata\n // to signify the change.\n uint32_t version;\n // The number of tensors in the file.\n // This is explicit, instead of being included in the metadata, to ensure it is always present\n // for loading the tensors.\n uint64_t tensor_count;\n // The number of metadata key-value pairs.\n uint64_t metadata_kv_count;\n // The metadata key-value pairs.\n gguf_metadata_kv_t metadata_kv[metadata_kv_count];\n};\n\nuint64_t align_offset(uint64_t offset) {\n return offset + (ALIGNMENT - (offset % ALIGNMENT)) % ALIGNMENT;\n}\n\nstruct gguf_tensor_info_t {\n // The name of the tensor. It is a standard GGUF string, with the caveat that\n // it must be at most 64 bytes long.\n gguf_string_t name;\n // The number of dimensions in the tensor.\n // Currently at most 4, but this may change in the future.\n uint32_t n_dimensions;\n // The dimensions of the tensor.\n uint64_t dimensions[n_dimensions];\n // The type of the tensor.\n ggml_type type;\n // The offset of the tensor's data in this file in bytes.\n //\n // This offset is relative to `tensor_data`, not to the start\n // of the file, to make it easier for writers to write the file.\n // Readers should consider exposing this offset relative to the\n // file to make it easier to read the data.\n //\n // Must be a multiple of `ALIGNMENT`. That is, `align_offset(offset) == offset`.\n uint64_t offset;\n};\n\nstruct gguf_file_t {\n // The header of the file.\n gguf_header_t header;\n\n // Tensor infos, which can be used to locate the tensor data.\n gguf_tensor_info_t tensor_infos[header.tensor_count];\n\n // Padding to the nearest multiple of `ALIGNMENT`.\n //\n // That is, if `sizeof(header) + sizeof(tensor_infos)` is not a multiple of `ALIGNMENT`,\n // this padding is added to make it so.\n //\n // This can be calculated as `align_offset(position) - position`, where `position` is\n // the position of the end of `tensor_infos` (i.e. `sizeof(header) + sizeof(tensor_infos)`).\n uint8_t _padding[];\n\n // Tensor data.\n //\n // This is arbitrary binary data corresponding to the weights of the model. This data should be close\n // or identical to the data in the original model file, but may be different due to quantization or\n // other optimizations for inference. Any such deviations should be recorded in the metadata or as\n // part of the architecture definition.\n //\n // Each tensor's data must be stored within this array, and located through its `tensor_infos` entry.\n // The offset of each tensor's data must be a multiple of `ALIGNMENT`, and the space between tensors\n // should be padded to `ALIGNMENT` bytes.\n uint8_t tensor_data[];\n};\n```\n\n## Standardized key-value pairs\n\nThe following key-value pairs are standardized. This list may grow in the future as more use cases are discovered. Where possible, names are shared with the original model definitions to make it easier to map between the two.\n\nNot all of these are required, but they are all recommended. Keys that are required are bolded. For omitted pairs, the reader should assume that the value is unknown and either default or error as appropriate.\n\nThe community can develop their own key-value pairs to carry additional data. However, these should be namespaced with the relevant community name to avoid collisions. For example, the `rustformers` community might use `rustformers.` as a prefix for all of their keys.\n\nIf a particular community key is widely used, it may be promoted to a standardized key.\n\nBy convention, most counts/lengths/etc are `uint64` unless otherwise specified. This is to allow for larger models to be supported in the future. Some models may use `uint32` for their values; it is recommended that readers support both.\n\n### General\n\n#### Required\n\n- **`general.architecture: string`**: describes what architecture this model implements. All lowercase ASCII, with only `[a-z0-9]+` characters allowed. Known values include:\n - `llama`\n - `mpt`\n - `gptneox`\n - `gptj`\n - `gpt2`\n - `bloom`\n - `falcon`\n - `mamba`\n - `rwkv`\n- **`general.quantization_version: uint32`**: The version of the quantization format. Not required if the model is not quantized (i.e. no tensors are quantized). If any tensors are quantized, this _must_ be present. This is separate to the quantization scheme of the tensors itself; the quantization version may change without changing the scheme's name (e.g. the quantization scheme is Q5_K, and the quantization version is 4).\n- **`general.alignment: uint32`**: the global alignment to use, as described above. This can vary to allow for different alignment schemes, but it must be a multiple of 8. Some writers may not write the alignment. If the alignment is **not** specified, assume it is `32`.\n\n#### General metadata\n\n- `general.name: string`: The name of the model. This should be a human-readable name that can be used to identify the model. It should be unique within the community that the model is defined in.\n- `general.author: string`: The author of the model.\n- `general.version: string`: The version of the model.\n- `general.organization: string`: The organization of the model.\n- `general.basename: string`: The base model name / architecture of the model\n- `general.finetune: string`: What has the base model been optimized toward.\n- `general.description: string`: free-form description of the model including anything that isn't covered by the other fields\n- `general.quantized_by: string`: The name of the individual who quantized the model\n- `general.size_label: string`: Size class of the model, such as number of weights and experts. (Useful for leader boards)\n- `general.license: string`: License of the model, expressed as a [SPDX license expression](https://spdx.github.io/spdx-spec/v2-draft/SPDX-license-expressions/) (e.g. `\"MIT OR Apache-2.0`). Do not include any other information, such as the license text or the URL to the license.\n- `general.license.name: string`: Human friendly license name\n- `general.license.link: string`: URL to the license.\n- `general.url: string`: URL to the model's homepage. This can be a GitHub repo, a paper, etc.\n- `general.doi: string`: Digital Object Identifier (DOI) https://www.doi.org/\n- `general.uuid: string`: [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)\n- `general.repo_url: string`: URL to the model's repository such as a GitHub repo or HuggingFace repo\n- `general.tags: string[]`: List of tags that can be used as search terms for a search engine or social media\n- `general.languages: string[]`: What languages can the model speak. Encoded as [ISO 639](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) two letter codes\n- `general.datasets: string[]`: Links or references to datasets that the model was trained upon\n- `general.file_type: uint32`: An enumerated value describing the type of the majority of the tensors in the file. Optional; can be inferred from the tensor types.\n - `ALL_F32 = 0`\n - `MOSTLY_F16 = 1`\n - `MOSTLY_Q4_0 = 2`\n - `MOSTLY_Q4_1 = 3`\n - `MOSTLY_Q4_1_SOME_F16 = 4`\n - `MOSTLY_Q4_2 = 5` (support removed)\n - `MOSTLY_Q4_3 = 6` (support removed)\n - `MOSTLY_Q8_0 = 7`\n - `MOSTLY_Q5_0 = 8`\n - `MOSTLY_Q5_1 = 9`\n - `MOSTLY_Q2_K = 10`\n - `MOSTLY_Q3_K_S = 11`\n - `MOSTLY_Q3_K_M = 12`\n - `MOSTLY_Q3_K_L = 13`\n - `MOSTLY_Q4_K_S = 14`\n - `MOSTLY_Q4_K_M = 15`\n - `MOSTLY_Q5_K_S = 16`\n - `MOSTLY_Q5_K_M = 17`\n - `MOSTLY_Q6_K = 18`\n\n#### Source metadata\n\nInformation about where this model came from. This is useful for tracking the provenance of the model, and for finding the original source if the model is modified. For a model that was converted from GGML, for example, these keys would point to the model that was converted from.\n\n- `general.source.url: string`: URL to the source of the model's homepage. This can be a GitHub repo, a paper, etc.\n- `general.source.doi: string`: Source Digital Object Identifier (DOI) https://www.doi.org/\n- `general.source.uuid: string`: Source [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)\n- `general.source.repo_url: string`: URL to the source of the model's repository such as a GitHub repo or HuggingFace repo\n\n- `general.base_model.count: uint32`: Number of parent models\n- `general.base_model.{id}.name: string`: The name of the parent model.\n- `general.base_model.{id}.author: string`: The author of the parent model.\n- `general.base_model.{id}.version: string`: The version of the parent model.\n- `general.base_model.{id}.organization: string`: The organization of the parent model.\n- `general.base_model.{id}.url: string`: URL to the source of the parent model's homepage. This can be a GitHub repo, a paper, etc.\n- `general.base_model.{id}.doi: string`: Parent Digital Object Identifier (DOI) https://www.doi.org/\n- `general.base_model.{id}.uuid: string`: Parent [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)\n- `general.base_model.{id}.repo_url: string`: URL to the source of the parent model's repository such as a GitHub repo or HuggingFace repo\n\n### LLM\n\nIn the following, `[llm]` is used to fill in for the name of a specific LLM architecture. For example, `llama` for LLaMA, `mpt` for MPT, etc. If mentioned in an architecture's section, it is required for that architecture, but not all keys are required for all architectures. Consult the relevant section for more information.\n\n- `[llm].context_length: uint64`: Also known as `n_ctx`. length of the context (in tokens) that the model was trained on. For most architectures, this is the hard limit on the length of the input. Architectures, like RWKV, that are not reliant on transformer-style attention may be able to handle larger inputs, but this is not guaranteed.\n- `[llm].embedding_length: uint64`: Also known as `n_embd`. Embedding layer size.\n- `[llm].block_count: uint64`: The number of blocks of attention+feed-forward layers (i.e. the bulk of the LLM). Does not include the input or embedding layers.\n- `[llm].feed_forward_length: uint64`: Also known as `n_ff`. The length of the feed-forward layer.\n- `[llm].use_parallel_residual: bool`: Whether or not the parallel residual logic should be used.\n- `[llm].tensor_data_layout: string`: When a model is converted to GGUF, tensors may be rearranged to improve performance. This key describes the layout of the tensor data. This is not required; if not present, it is assumed to be `reference`.\n - `reference`: tensors are laid out in the same order as the original model\n - further options can be found for each architecture in their respective sections\n- `[llm].expert_count: uint32`: Number of experts in MoE models (optional for non-MoE arches).\n- `[llm].expert_used_count: uint32`: Number of experts used during each token token evaluation (optional for non-MoE arches).\n\n#### Attention\n\n- `[llm].attention.head_count: uint64`: Also known as `n_head`. Number of attention heads.\n- `[llm].attention.head_count_kv: uint64`: The number of heads per group used in Grouped-Query-Attention. If not present or if present and equal to `[llm].attention.head_count`, the model does not use GQA.\n- `[llm].attention.max_alibi_bias: float32`: The maximum bias to use for ALiBI.\n- `[llm].attention.clamp_kqv: float32`: Value (`C`) to clamp the values of the `Q`, `K`, and `V` tensors between (`[-C, C]`).\n- `[llm].attention.layer_norm_epsilon: float32`: Layer normalization epsilon.\n- `[llm].attention.layer_norm_rms_epsilon: float32`: Layer RMS normalization epsilon.\n- `[llm].attention.key_length: uint32`: The optional size of a key head, $d_k$. If not specified, it will be `n_embd / n_head`.\n- `[llm].attention.value_length: uint32`: The optional size of a value head, $d_v$. If not specified, it will be `n_embd / n_head`.\n\n#### RoPE\n\n- `[llm].rope.dimension_count: uint64`: The number of rotary dimensions for RoPE.\n- `[llm].rope.freq_base: float32`: The base frequency for RoPE.\n\n##### Scaling\n\nThe following keys describe RoPE scaling parameters:\n\n- `[llm].rope.scaling.type: string`: Can be `none`, `linear`, or `yarn`.\n- `[llm].rope.scaling.factor: float32`: A scale factor for RoPE to adjust the context length.\n- `[llm].rope.scaling.original_context_length: uint32_t`: The original context length of the base model.\n- `[llm].rope.scaling.finetuned: bool`: True if model has been finetuned with RoPE scaling.\n\nNote that older models may not have these keys, and may instead use the following key:\n\n- `[llm].rope.scale_linear: float32`: A linear scale factor for RoPE to adjust the context length.\n\nIt is recommended that models use the newer keys if possible, as they are more flexible and allow for more complex scaling schemes. Executors will need to support both indefinitely.\n\n#### SSM\n\n- `[llm].ssm.conv_kernel: uint32`: The size of the rolling/shift state.\n- `[llm].ssm.inner_size: uint32`: The embedding size of the states.\n- `[llm].ssm.state_size: uint32`: The size of the recurrent state.\n- `[llm].ssm.time_step_rank: uint32`: The rank of time steps.\n\n#### Models\n\nThe following sections describe the metadata for each model architecture. Each key specified _must_ be present.\n\n##### LLaMA\n\n- `llama.context_length`\n- `llama.embedding_length`\n- `llama.block_count`\n- `llama.feed_forward_length`\n- `llama.rope.dimension_count`\n- `llama.attention.head_count`\n- `llama.attention.layer_norm_rms_epsilon`\n\n###### Optional\n\n- `llama.rope.scale`\n- `llama.attention.head_count_kv`\n- `llama.tensor_data_layout`:\n - `Meta AI original pth`:\n ```python\n def permute(weights: NDArray, n_head: int) -> NDArray:\n return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])\n .swapaxes(1, 2)\n .reshape(weights.shape))\n ```\n- `llama.expert_count`\n- `llama.expert_used_count`\n\n##### MPT\n\n- `mpt.context_length`\n- `mpt.embedding_length`\n- `mpt.block_count`\n- `mpt.attention.head_count`\n- `mpt.attention.alibi_bias_max`\n- `mpt.attention.clip_kqv`\n- `mpt.attention.layer_norm_epsilon`\n\n##### GPT-NeoX\n\n- `gptneox.context_length`\n- `gptneox.embedding_length`\n- `gptneox.block_count`\n- `gptneox.use_parallel_residual`\n- `gptneox.rope.dimension_count`\n- `gptneox.attention.head_count`\n- `gptneox.attention.layer_norm_epsilon`\n\n###### Optional\n\n- `gptneox.rope.scale`\n\n##### GPT-J\n\n- `gptj.context_length`\n- `gptj.embedding_length`\n- `gptj.block_count`\n- `gptj.rope.dimension_count`\n- `gptj.attention.head_count`\n- `gptj.attention.layer_norm_epsilon`\n\n###### Optional\n\n- `gptj.rope.scale`\n\n##### GPT-2\n\n- `gpt2.context_length`\n- `gpt2.embedding_length`\n- `gpt2.block_count`\n- `gpt2.attention.head_count`\n- `gpt2.attention.layer_norm_epsilon`\n\n##### BLOOM\n\n- `bloom.context_length`\n- `bloom.embedding_length`\n- `bloom.block_count`\n- `bloom.feed_forward_length`\n- `bloom.attention.head_count`\n- `bloom.attention.layer_norm_epsilon`\n\n##### Falcon\n\n- `falcon.context_length`\n- `falcon.embedding_length`\n- `falcon.block_count`\n- `falcon.attention.head_count`\n- `falcon.attention.head_count_kv`\n- `falcon.attention.use_norm`\n- `falcon.attention.layer_norm_epsilon`\n\n###### Optional\n\n- `falcon.tensor_data_layout`:\n\n - `jploski` (author of the original GGML implementation of Falcon):\n\n ```python\n # The original query_key_value tensor contains n_head_kv \"kv groups\",\n # each consisting of n_head/n_head_kv query weights followed by one key\n # and one value weight (shared by all query heads in the kv group).\n # This layout makes it a big pain to work with in GGML.\n # So we rearrange them here,, so that we have n_head query weights\n # followed by n_head_kv key weights followed by n_head_kv value weights,\n # in contiguous fashion.\n\n if \"query_key_value\" in src:\n qkv = model[src].view(\n n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)\n\n q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)\n k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)\n v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)\n\n model[src] = torch.cat((q,k,v)).reshape_as(model[src])\n ```\n\n##### Mamba\n\n- `mamba.context_length`\n- `mamba.embedding_length`\n- `mamba.block_count`\n- `mamba.ssm.conv_kernel`\n- `mamba.ssm.inner_size`\n- `mamba.ssm.state_size`\n- `mamba.ssm.time_step_rank`\n- `mamba.attention.layer_norm_rms_epsilon`\n\n##### RWKV\n\nThe vocabulary size is the same as the number of rows in the `head` matrix.\n\n- `rwkv.architecture_version: uint32`: The only allowed value currently is 4. Version 5 is expected to appear some time in the future.\n- `rwkv.context_length: uint64`: Length of the context used during training or fine-tuning. RWKV is able to handle larger context than this limit, but the output quality may suffer.\n- `rwkv.block_count: uint64`\n- `rwkv.embedding_length: uint64`\n- `rwkv.feed_forward_length: uint64`\n\n##### Whisper\n\nKeys that do not have types defined should be assumed to share definitions with `llm.` keys.\n(For example, `whisper.context_length` is equivalent to `llm.context_length`.)\nThis is because they are both transformer models.\n\n- `whisper.encoder.context_length`\n- `whisper.encoder.embedding_length`\n- `whisper.encoder.block_count`\n- `whisper.encoder.mels_count: uint64`\n- `whisper.encoder.attention.head_count`\n\n- `whisper.decoder.context_length`\n- `whisper.decoder.embedding_length`\n- `whisper.decoder.block_count`\n- `whisper.decoder.attention.head_count`\n\n#### Prompting\n\n**TODO**: Include prompt format, and/or metadata about how it should be used (instruction, conversation, autocomplete, etc).\n\n### LoRA\n\n**TODO**: Figure out what metadata is needed for LoRA. Probably desired features:\n\n- match an existing model exactly, so that it can't be misapplied\n- be marked as a LoRA so executors won't try to run it by itself\n\nShould this be an architecture, or should it share the details of the original model with additional fields to mark it as a LoRA?\n\n### Tokenizer\n\nThe following keys are used to describe the tokenizer of the model. It is recommended that model authors support as many of these as possible, as it will allow for better tokenization quality with supported executors.\n\n#### GGML\n\nGGML supports an embedded vocabulary that enables inference of the model, but implementations of tokenization using this vocabulary (i.e. `llama.cpp`'s tokenizer) may have lower accuracy than the original tokenizer used for the model. When a more accurate tokenizer is available and supported, it should be used instead.\n\nIt is not guaranteed to be standardized across models, and may change in the future. It is recommended that model authors use a more standardized tokenizer if possible.\n\n- `tokenizer.ggml.model: string`: The name of the tokenizer model.\n - `llama`: Llama style SentencePiece (tokens and scores extracted from HF `tokenizer.model`)\n - `replit`: Replit style SentencePiece (tokens and scores extracted from HF `spiece.model`)\n - `gpt2`: GPT-2 / GPT-NeoX style BPE (tokens extracted from HF `tokenizer.json`)\n - `rwkv`: RWKV tokenizer\n- `tokenizer.ggml.tokens: array[string]`: A list of tokens indexed by the token ID used by the model.\n- `tokenizer.ggml.scores: array[float32]`: If present, the score/probability of each token. If not present, all tokens are assumed to have equal probability. If present, it must have the same length and index as `tokens`.\n- `tokenizer.ggml.token_type: array[int32]`: The token type (1=normal, 2=unknown, 3=control, 4=user defined, 5=unused, 6=byte). If present, it must have the same length and index as `tokens`.\n- `tokenizer.ggml.merges: array[string]`: If present, the merges of the tokenizer. If not present, the tokens are assumed to be atomic.\n- `tokenizer.ggml.added_tokens: array[string]`: If present, tokens that were added after training.\n\n##### Special tokens\n\n- `tokenizer.ggml.bos_token_id: uint32`: Beginning of sequence marker\n- `tokenizer.ggml.eos_token_id: uint32`: End of sequence marker\n- `tokenizer.ggml.unknown_token_id: uint32`: Unknown token\n- `tokenizer.ggml.separator_token_id: uint32`: Separator token\n- `tokenizer.ggml.padding_token_id: uint32`: Padding token\n\n#### Hugging Face\n\nHugging Face maintains their own `tokenizers` library that supports a wide variety of tokenizers. If your executor uses this library, it may be able to use the model's tokenizer directly.\n\n- `tokenizer.huggingface.json: string`: the entirety of the HF `tokenizer.json` for a given model (e.g. ). Included for compatibility with executors that support HF tokenizers directly.\n\n#### Other\n\nOther tokenizers may be used, but are not necessarily standardized. They may be executor-specific. They will be documented here as they are discovered/further developed.\n\n- `tokenizer.rwkv.world: string`: a RWKV World tokenizer, like [this](https://github.com/BlinkDL/ChatRWKV/blob/main/tokenizer/rwkv_vocab_v20230424.txt). This text file should be included verbatim.\n- `tokenizer.chat_template : string`: a Jinja template that specifies the input format expected by the model. For more details see: \n\n### Computation graph\n\nThis is a future extension and still needs to be discussed, and may necessitate a new GGUF version. At the time of writing, the primary blocker is the stabilization of the computation graph format.\n\nA sample computation graph of GGML nodes could be included in the model itself, allowing an executor to run the model without providing its own implementation of the architecture. This would allow for a more consistent experience across executors, and would allow for more complex architectures to be supported without requiring the executor to implement them.\n\n## Standardized tensor names\n\nTo minimize complexity and maximize compatibility, it is recommended that models using the transformer architecture use the following naming convention for their tensors:\n\n### Base layers\n\n`AA.weight` `AA.bias`\n\nwhere `AA` can be:\n\n- `token_embd`: Token embedding layer\n- `pos_embd`: Position embedding layer\n- `output_norm`: Output normalization layer\n- `output`: Output layer\n\n### Attention and feed-forward layer blocks\n\n`blk.N.BB.weight` `blk.N.BB.bias`\n\nwhere N signifies the block number a layer belongs to, and where `BB` could be:\n\n- `attn_norm`: Attention normalization layer\n- `attn_norm_2`: Attention normalization layer\n- `attn_qkv`: Attention query-key-value layer\n- `attn_q`: Attention query layer\n- `attn_k`: Attention key layer\n- `attn_v`: Attention value layer\n- `attn_output`: Attention output layer\n\n- `ffn_norm`: Feed-forward network normalization layer\n- `ffn_up`: Feed-forward network \"up\" layer\n- `ffn_gate`: Feed-forward network \"gate\" layer\n- `ffn_down`: Feed-forward network \"down\" layer\n- `ffn_gate_inp`: Expert-routing layer for the Feed-forward network in MoE models\n- `ffn_gate_exp`: Feed-forward network \"gate\" layer per expert in MoE models\n- `ffn_down_exp`: Feed-forward network \"down\" layer per expert in MoE models\n- `ffn_up_exp`: Feed-forward network \"up\" layer per expert in MoE models\n\n- `ssm_in`: State space model input projections layer\n- `ssm_conv1d`: State space model rolling/shift layer\n- `ssm_x`: State space model selective parametrization layer\n- `ssm_a`: State space model state compression layer\n- `ssm_d`: State space model skip connection layer\n- `ssm_dt`: State space model time step layer\n- `ssm_out`: State space model output projection layer\n\n## Version History\n\nThis document is actively updated to describe the current state of the metadata, and these changes are not tracked outside of the commits.\n\nHowever, the format _itself_ has changed. The following sections describe the changes to the format itself.\n\n### v3\n\nAdds big-endian support.\n\n### v2\n\nMost countable values (lengths, etc) were changed from `uint32` to `uint64` to allow for larger models to be supported in the future.\n\n### v1\n\nInitial version.\n\n## Historical State of Affairs\n\nThe following information is provided for context, but is not necessary to understand the rest of this document.\n\n### Overview\n\nAt present, there are three GGML file formats floating around for LLMs:\n\n- **GGML** (unversioned): baseline format, with no versioning or alignment.\n- **GGMF** (versioned): the same as GGML, but with versioning. Only one version exists.\n- **GGJT**: Aligns the tensors to allow for use with `mmap`, which requires alignment. v1, v2 and v3 are identical, but the latter versions use a different quantization scheme that is incompatible with previous versions.\n\nGGML is primarily used by the examples in `ggml`, while GGJT is used by `llama.cpp` models. Other executors may use any of the three formats, but this is not 'officially' supported.\n\nThese formats share the same fundamental structure:\n\n- a magic number with an optional version number\n- model-specific hyperparameters, including\n - metadata about the model, such as the number of layers, the number of heads, etc.\n - a `ftype` that describes the type of the majority of the tensors,\n - for GGML files, the quantization version is encoded in the `ftype` divided by 1000\n- an embedded vocabulary, which is a list of strings with length prepended. The GGMF/GGJT formats embed a float32 score next to the strings.\n- finally, a list of tensors with their length-prepended name, type, and (aligned, in the case of GGJT) tensor data\n\nNotably, this structure does not identify what model architecture the model belongs to, nor does it offer any flexibility for changing the structure of the hyperparameters. This means that the only way to add new hyperparameters is to add them to the end of the list, which is a breaking change for existing models.\n\n### Drawbacks\n\nUnfortunately, over the last few months, there are a few issues that have become apparent with the existing models:\n\n- There's no way to identify which model architecture a given model is for, because that information isn't present\n - Similarly, existing programs cannot intelligently fail upon encountering new architectures\n- Adding or removing any new hyperparameters is a breaking change, which is impossible for a reader to detect without using heuristics\n- Each model architecture requires its own conversion script to their architecture's variant of GGML\n- Maintaining backwards compatibility without breaking the structure of the format requires clever tricks, like packing the quantization version into the ftype, which are not guaranteed to be picked up by readers/writers, and are not consistent between the two formats\n\n### Why not other formats?\n\nThere are a few other formats that could be used, but issues include:\n\n- requiring additional dependencies to load or save the model, which is complicated in a C environment\n- limited or no support for 4-bit quantization\n- existing cultural expectations (e.g. whether or not the model is a directory or a file)\n- lack of support for embedded vocabularies\n- lack of control over direction of future development\n\nUltimately, it is likely that GGUF will remain necessary for the foreseeable future, and it is better to have a single format that is well-documented and supported by all executors than to contort an existing format to fit the needs of GGML.\n" }, { "path": "examples/CMakeLists.txt", "content": "if (GGML_ALL_WARNINGS)\n if (NOT MSVC)\n set(cxx_flags\n # TODO(marella): Add other warnings.\n -Wpedantic\n -Wunused-variable\n -Wno-unused-function\n -Wno-multichar\n )\n add_compile_options(\"$<$:${cxx_flags}>\")\n endif()\nendif()\n\nadd_library(common STATIC common.cpp)\ntarget_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})\n\nadd_library(common-ggml STATIC common-ggml.cpp)\ntarget_link_libraries(common-ggml PRIVATE ggml)\ntarget_include_directories(common-ggml PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})\n\nadd_subdirectory(yolo)\n\nif (NOT GGML_BACKEND_DL)\n add_subdirectory(gpt-2)\n add_subdirectory(gpt-j)\n add_subdirectory(mnist)\n add_subdirectory(sam)\n add_subdirectory(simple)\n add_subdirectory(magika)\nendif()\n\nif (GGML_METAL)\n add_subdirectory(perf-metal)\nendif()\n" }, { "path": "examples/common-ggml.cpp", "content": "#include \"common-ggml.h\"\n\n#include \n#include \n\nstatic const std::map GGML_FTYPE_MAP = {\n {\"q4_0\", GGML_FTYPE_MOSTLY_Q4_0},\n {\"q4_1\", GGML_FTYPE_MOSTLY_Q4_1},\n {\"q5_0\", GGML_FTYPE_MOSTLY_Q5_0},\n {\"q5_1\", GGML_FTYPE_MOSTLY_Q5_1},\n {\"q8_0\", GGML_FTYPE_MOSTLY_Q8_0},\n {\"q2_k\", GGML_FTYPE_MOSTLY_Q2_K},\n {\"q3_k\", GGML_FTYPE_MOSTLY_Q3_K},\n {\"q4_k\", GGML_FTYPE_MOSTLY_Q4_K},\n {\"q5_k\", GGML_FTYPE_MOSTLY_Q5_K},\n {\"q6_k\", GGML_FTYPE_MOSTLY_Q6_K},\n};\n\nvoid ggml_print_ftypes(FILE * fp) {\n for (auto it = GGML_FTYPE_MAP.begin(); it != GGML_FTYPE_MAP.end(); it++) {\n fprintf(fp, \" type = \\\"%s\\\" or %d\\n\", it->first.c_str(), it->second);\n }\n}\n\nenum ggml_ftype ggml_parse_ftype(const char * str) {\n enum ggml_ftype ftype;\n if (str[0] == 'q') {\n const auto it = GGML_FTYPE_MAP.find(str);\n if (it == GGML_FTYPE_MAP.end()) {\n fprintf(stderr, \"%s: unknown ftype '%s'\\n\", __func__, str);\n return GGML_FTYPE_UNKNOWN;\n }\n ftype = it->second;\n } else {\n ftype = (enum ggml_ftype) atoi(str);\n }\n\n return ftype;\n}\n\nbool ggml_common_quantize_0(\n std::ifstream & finp,\n std::ofstream & fout,\n const ggml_ftype ftype,\n const std::vector & to_quant,\n const std::vector & to_skip) {\n\n ggml_type qtype = GGML_TYPE_F32;\n\n switch (ftype) {\n case GGML_FTYPE_MOSTLY_Q4_0: qtype = GGML_TYPE_Q4_0; break;\n case GGML_FTYPE_MOSTLY_Q4_1: qtype = GGML_TYPE_Q4_1; break;\n case GGML_FTYPE_MOSTLY_Q5_0: qtype = GGML_TYPE_Q5_0; break;\n case GGML_FTYPE_MOSTLY_Q5_1: qtype = GGML_TYPE_Q5_1; break;\n case GGML_FTYPE_MOSTLY_Q8_0: qtype = GGML_TYPE_Q8_0; break;\n case GGML_FTYPE_MOSTLY_Q2_K: qtype = GGML_TYPE_Q2_K; break;\n case GGML_FTYPE_MOSTLY_Q3_K: qtype = GGML_TYPE_Q3_K; break;\n case GGML_FTYPE_MOSTLY_Q4_K: qtype = GGML_TYPE_Q4_K; break;\n case GGML_FTYPE_MOSTLY_Q5_K: qtype = GGML_TYPE_Q5_K; break;\n case GGML_FTYPE_MOSTLY_Q6_K: qtype = GGML_TYPE_Q6_K; break;\n case GGML_FTYPE_UNKNOWN:\n case GGML_FTYPE_ALL_F32:\n case GGML_FTYPE_MOSTLY_F16:\n case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16:\n case GGML_FTYPE_MOSTLY_IQ2_XXS:\n case GGML_FTYPE_MOSTLY_IQ2_XS:\n case GGML_FTYPE_MOSTLY_IQ2_S:\n case GGML_FTYPE_MOSTLY_IQ3_XXS:\n case GGML_FTYPE_MOSTLY_IQ3_S:\n case GGML_FTYPE_MOSTLY_IQ1_S:\n case GGML_FTYPE_MOSTLY_IQ4_NL:\n case GGML_FTYPE_MOSTLY_IQ4_XS:\n case GGML_FTYPE_MOSTLY_IQ1_M:\n case GGML_FTYPE_MOSTLY_BF16:\n case GGML_FTYPE_MOSTLY_MXFP4:\n case GGML_FTYPE_MOSTLY_NVFP4:\n {\n fprintf(stderr, \"%s: invalid model type %d\\n\", __func__, ftype);\n return false;\n }\n };\n\n if (!ggml_is_quantized(qtype)) {\n fprintf(stderr, \"%s: invalid quantization type %d (%s)\\n\", __func__, qtype, ggml_type_name(qtype));\n return false;\n }\n\n size_t total_size_org = 0;\n size_t total_size_new = 0;\n\n std::vector work;\n\n std::vector data_u8;\n std::vector data_f16;\n std::vector data_f32;\n\n while (true) {\n int32_t n_dims;\n int32_t length;\n int32_t ttype;\n\n finp.read(reinterpret_cast(&n_dims), sizeof(n_dims));\n finp.read(reinterpret_cast(&length), sizeof(length));\n finp.read(reinterpret_cast(&ttype), sizeof(ttype));\n\n if (finp.eof()) {\n break;\n }\n\n int32_t nelements = 1;\n int32_t ne[4] = { 1, 1, 1, 1 };\n for (int i = 0; i < n_dims; ++i) {\n finp.read (reinterpret_cast(&ne[i]), sizeof(ne[i]));\n nelements *= ne[i];\n }\n\n std::string name(length, 0);\n finp.read (&name[0], length);\n\n printf(\"%64s - [%5d, %5d, %5d], type = %6s \", name.data(), ne[0], ne[1], ne[2], ggml_type_name((ggml_type) ttype));\n\n bool quantize = false;\n\n // check if we should quantize this tensor\n for (const auto & s : to_quant) {\n if (std::regex_match(name, std::regex(s))) {\n quantize = true;\n break;\n }\n }\n\n // check if we should skip this tensor\n for (const auto & s : to_skip) {\n if (std::regex_match(name, std::regex(s))) {\n quantize = false;\n break;\n }\n }\n\n // quantize only 2D tensors\n quantize &= (n_dims == 2);\n\n if (quantize) {\n if (ttype != GGML_TYPE_F32 && ttype != GGML_TYPE_F16) {\n fprintf(stderr, \"%s: unsupported ttype %d (%s) for integer quantization\\n\", __func__, ttype, ggml_type_name((ggml_type) ttype));\n return false;\n }\n\n if (ttype == GGML_TYPE_F16) {\n data_f16.resize(nelements);\n finp.read(reinterpret_cast(data_f16.data()), nelements * sizeof(ggml_fp16_t));\n data_f32.resize(nelements);\n for (int i = 0; i < nelements; ++i) {\n data_f32[i] = ggml_fp16_to_fp32(data_f16[i]);\n }\n } else {\n data_f32.resize(nelements);\n finp.read(reinterpret_cast(data_f32.data()), nelements * sizeof(float));\n }\n\n ttype = qtype;\n } else {\n const int bpe = (ttype == 0) ? sizeof(float) : sizeof(uint16_t);\n\n data_u8.resize(nelements*bpe);\n finp.read(reinterpret_cast(data_u8.data()), nelements * bpe);\n }\n\n fout.write(reinterpret_cast(&n_dims), sizeof(n_dims));\n fout.write(reinterpret_cast(&length), sizeof(length));\n fout.write(reinterpret_cast(&ttype), sizeof(ttype));\n for (int i = 0; i < n_dims; ++i) {\n fout.write(reinterpret_cast(&ne[i]), sizeof(ne[i]));\n }\n fout.write(&name[0], length);\n\n if (quantize) {\n work.resize(nelements); // for quantization\n\n size_t cur_size = 0;\n switch ((ggml_type) ttype) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n {\n cur_size = ggml_quantize_chunk((ggml_type) ttype, data_f32.data(), work.data(), 0, nelements/ne[0], ne[0], nullptr);\n } break;\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n case GGML_TYPE_I8:\n case GGML_TYPE_I16:\n case GGML_TYPE_I32:\n case GGML_TYPE_I64:\n case GGML_TYPE_F64:\n case GGML_TYPE_Q8_1:\n case GGML_TYPE_Q8_K:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ2_S:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_BF16:\n case GGML_TYPE_TQ1_0:\n case GGML_TYPE_TQ2_0:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_NVFP4:\n case GGML_TYPE_COUNT:\n {\n fprintf(stderr, \"%s: unsupported quantization type %d (%s)\\n\", __func__, ttype, ggml_type_name((ggml_type) ttype));\n return false;\n }\n }\n\n fout.write(reinterpret_cast(work.data()), cur_size);\n total_size_new += cur_size;\n\n printf(\"size = %8.2f MB -> %8.2f MB\\n\", nelements * sizeof(float)/1024.0/1024.0, cur_size/1024.0/1024.0);\n } else {\n printf(\"size = %8.3f MB\\n\", data_u8.size()/1024.0/1024.0);\n fout.write(reinterpret_cast(data_u8.data()), data_u8.size());\n total_size_new += data_u8.size();\n }\n\n total_size_org += nelements * sizeof(float);\n }\n\n printf(\"%s: model size = %8.2f MB\\n\", __func__, total_size_org/1024.0/1024.0);\n printf(\"%s: quant size = %8.2f MB | ftype = %d (%s)\\n\", __func__, total_size_new/1024.0/1024.0, ftype, ggml_type_name(qtype));\n\n return true;\n}\n" }, { "path": "examples/common-ggml.h", "content": "#pragma once\n\n#include \"ggml.h\"\n\n#include \n#include \n#include \n\nenum ggml_ftype ggml_parse_ftype(const char * str);\n\nvoid ggml_print_ftypes(FILE * fp = stderr);\n\nbool ggml_common_quantize_0(\n std::ifstream & finp,\n std::ofstream & fout,\n const ggml_ftype ftype,\n const std::vector & to_quant,\n const std::vector & to_skip);\n" }, { "path": "examples/common.cpp", "content": "#define _USE_MATH_DEFINES // for M_PI\n\n#include \"common.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n// Function to check if the next argument exists\nstatic std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) {\n if (i + 1 < argc && argv[i + 1][0] != '-') {\n return argv[++i];\n } else {\n fprintf(stderr, \"error: %s requires one argument.\\n\", flag.c_str());\n gpt_print_usage(argc, argv, params);\n exit(0);\n }\n}\n\nbool gpt_params_parse(int argc, char ** argv, gpt_params & params) {\n for (int i = 1; i < argc; i++) {\n std::string arg = argv[i];\n\n if (arg == \"-s\" || arg == \"--seed\") {\n params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"-t\" || arg == \"--threads\") {\n params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"-p\" || arg == \"--prompt\") {\n params.prompt = get_next_arg(i, argc, argv, arg, params);\n } else if (arg == \"-n\" || arg == \"--n_predict\") {\n params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"-np\" || arg == \"--n_parallel\") {\n params.n_parallel = std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"--top_k\") {\n params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"--top_p\") {\n params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"--temp\") {\n params.temp = std::stof(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"--repeat-last-n\") {\n params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"--repeat-penalty\") {\n params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"-b\" || arg == \"--batch_size\") {\n params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"-c\" || arg == \"--context\") {\n params.n_ctx= std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"-ngl\" || arg == \"--gpu-layers\" || arg == \"--n-gpu-layers\") {\n params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"--ignore-eos\") {\n params.ignore_eos = true;\n } else if (arg == \"-m\" || arg == \"--model\") {\n params.model = get_next_arg(i, argc, argv, arg, params);\n } else if (arg == \"-i\" || arg == \"--interactive\") {\n params.interactive = true;\n } else if (arg == \"-ip\" || arg == \"--interactive-port\") {\n params.interactive = true;\n params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params));\n } else if (arg == \"-h\" || arg == \"--help\") {\n gpt_print_usage(argc, argv, params);\n exit(0);\n } else if (arg == \"-f\" || arg == \"--file\") {\n get_next_arg(i, argc, argv, arg, params);\n std::ifstream file(argv[i]);\n if (!file) {\n fprintf(stderr, \"error: failed to open file '%s'\\n\", argv[i]);\n break;\n }\n std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.prompt));\n if (params.prompt.back() == '\\n') {\n params.prompt.pop_back();\n }\n } else if (arg == \"-tt\" || arg == \"--token_test\") {\n params.token_test = get_next_arg(i, argc, argv, arg, params);\n }\n else {\n fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n gpt_print_usage(argc, argv, params);\n exit(0);\n }\n }\n\n return true;\n}\n\nvoid gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {\n fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n fprintf(stderr, \"\\n\");\n fprintf(stderr, \"options:\\n\");\n fprintf(stderr, \" -h, --help show this help message and exit\\n\");\n fprintf(stderr, \" -s SEED, --seed SEED RNG seed (default: -1)\\n\");\n fprintf(stderr, \" -t N, --threads N number of threads to use during computation (default: %d)\\n\", params.n_threads);\n fprintf(stderr, \" -p PROMPT, --prompt PROMPT\\n\");\n fprintf(stderr, \" prompt to start generation with (default: random)\\n\");\n fprintf(stderr, \" -f FNAME, --file FNAME\\n\");\n fprintf(stderr, \" load prompt from a file\\n\");\n fprintf(stderr, \" -tt TOKEN_TEST, --token_test TOKEN_TEST\\n\");\n fprintf(stderr, \" test tokenization\\n\");\n fprintf(stderr, \" -n N, --n_predict N number of tokens to predict (default: %d)\\n\", params.n_predict);\n fprintf(stderr, \" --top_k N top-k sampling (default: %d)\\n\", params.top_k);\n fprintf(stderr, \" --top_p N top-p sampling (default: %.1f)\\n\", params.top_p);\n fprintf(stderr, \" --temp N temperature (default: %.1f)\\n\", params.temp);\n fprintf(stderr, \" --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\\n\", params.repeat_last_n);\n fprintf(stderr, \" --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\\n\", (double)params.repeat_penalty);\n fprintf(stderr, \" -b N, --batch_size N batch size for prompt processing (default: %d)\\n\", params.n_batch);\n fprintf(stderr, \" -c N, --context N context / KV cache size (default: %d)\\n\", params.n_ctx);\n fprintf(stderr, \" --ignore-eos ignore EOS token during generation\\n\");\n fprintf(stderr, \" -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\\n\", params.n_gpu_layers);\n fprintf(stderr, \" -m FNAME, --model FNAME\\n\");\n fprintf(stderr, \" model path (default: %s)\\n\", params.model.c_str());\n fprintf(stderr, \"\\n\");\n}\n\nstd::string gpt_random_prompt(std::mt19937 & rng) {\n const int r = rng() % 10;\n switch (r) {\n case 0: return \"So\";\n case 1: return \"Once upon a time\";\n case 2: return \"When\";\n case 3: return \"The\";\n case 4: return \"After\";\n case 5: return \"If\";\n case 6: return \"import\";\n case 7: return \"He\";\n case 8: return \"She\";\n case 9: return \"They\";\n }\n\n return \"The\";\n}\n\nstd::string trim(const std::string & s) {\n std::regex e(\"^\\\\s+|\\\\s+$\");\n return std::regex_replace(s, e, \"\");\n}\n\nstd::string replace(const std::string & s, const std::string & from, const std::string & to) {\n std::string result = s;\n size_t pos = 0;\n while ((pos = result.find(from, pos)) != std::string::npos) {\n result.replace(pos, from.length(), to);\n pos += to.length();\n }\n return result;\n}\n\nvoid gpt_vocab::add_special_token(const std::string & token) {\n special_tokens.push_back(token);\n}\n\nstd::map json_parse(const std::string & fname) {\n std::map result;\n\n // read file into string\n std::string json;\n {\n std::ifstream ifs(fname);\n if (!ifs) {\n fprintf(stderr, \"Failed to open %s\\n\", fname.c_str());\n exit(1);\n }\n\n json = std::string((std::istreambuf_iterator(ifs)),\n (std::istreambuf_iterator()));\n }\n\n if (json[0] != '{') {\n return result;\n }\n\n // parse json\n {\n bool has_key = false;\n bool in_token = false;\n\n std::string str_key = \"\";\n std::string str_val = \"\";\n\n int n = json.size();\n for (int i = 1; i < n; ++i) {\n if (!in_token) {\n if (json[i] == ' ') continue;\n if (json[i] == '\"') {\n in_token = true;\n continue;\n }\n } else {\n if (json[i] == '\\\\' && i+1 < n) {\n if (has_key == false) {\n str_key += json[i];\n } else {\n str_val += json[i];\n }\n ++i;\n } else if (json[i] == '\"') {\n if (has_key == false) {\n has_key = true;\n ++i;\n while (json[i] == ' ') ++i;\n ++i; // :\n while (json[i] == ' ') ++i;\n if (json[i] != '\\\"') {\n while (json[i] != ',' && json[i] != '}') {\n str_val += json[i++];\n }\n has_key = false;\n } else {\n in_token = true;\n continue;\n }\n } else {\n has_key = false;\n }\n\n str_key = ::replace(str_key, \"\\\\u0120\", \" \" ); // \\u0120 -> space\n str_key = ::replace(str_key, \"\\\\u010a\", \"\\n\"); // \\u010a -> new line\n str_key = ::replace(str_key, \"\\\\\\\"\", \"\\\"\"); // \\\\\\\" -> \"\n\n try {\n result[str_key] = std::stoi(str_val);\n } catch (...) {\n //fprintf(stderr, \"%s: ignoring key '%s' with value '%s'\\n\", fname.c_str(), str_key.c_str(), str_val.c_str());\n\n }\n str_key = \"\";\n str_val = \"\";\n in_token = false;\n continue;\n }\n if (has_key == false) {\n str_key += json[i];\n } else {\n str_val += json[i];\n }\n }\n }\n }\n\n return result;\n}\n\nvoid gpt_split_words(std::string str, std::vector& words) {\n const std::string pattern = R\"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\\s[:alpha:][:digit:]]+|\\s+(?!\\S)|\\s+)\";\n const std::regex re(pattern);\n std::smatch m;\n\n while (std::regex_search(str, m, re)) {\n for (auto x : m) {\n words.push_back(x);\n }\n str = m.suffix();\n }\n}\n\nstd::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {\n std::vector words;\n\n // first split the text into words\n {\n std::string str = text;\n\n // Generate the subpattern from the special_tokens vector if it's not empty\n if (!vocab.special_tokens.empty()) {\n const std::regex escape(R\"([\\[\\\\\\^\\$\\.\\|\\?\\*\\+\\(\\)\\{\\}])\");\n std::string special_tokens_subpattern;\n for (const auto & token : vocab.special_tokens) {\n if (!special_tokens_subpattern.empty()) {\n special_tokens_subpattern += \"|\";\n }\n special_tokens_subpattern += std::regex_replace(token, escape, R\"(\\$&)\");\n }\n\n std::regex re(special_tokens_subpattern);\n std::smatch m;\n // Split the text by special tokens.\n while (std::regex_search(str, m, re)) {\n // Split the substrings in-between special tokens into words.\n gpt_split_words(m.prefix(), words);\n // Add matched special tokens as words.\n for (auto x : m) {\n words.push_back(x);\n }\n str = m.suffix();\n }\n // Remaining text without special tokens will be handled below.\n }\n\n gpt_split_words(str, words);\n }\n\n // find the longest token that forms each word in words:\n std::vector tokens;\n for (const auto & word : words) {\n for (int i = 0; i < (int) word.size(); ){\n for (int j = word.size() - 1; j >= i; j--){\n auto cand = word.substr(i, j-i+1);\n auto it = vocab.token_to_id.find(cand);\n if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab\n tokens.push_back(it->second);\n i = j + 1;\n break;\n }\n else if (j == i){ // word.substr(i, 1) has no matching\n fprintf(stderr, \"%s: unknown token '%s'\\n\", __func__, word.substr(i, 1).data());\n i++;\n }\n }\n }\n }\n\n return tokens;\n}\n\nstatic std::vector parse_tokens_from_string(const std::string& input, char delimiter) {\n std::vector output;\n std::stringstream ss(input);\n std::string token;\n\n while (std::getline(ss, token, delimiter)) {\n output.push_back(std::stoi(token));\n }\n\n return output;\n}\n\nstatic std::map> extract_tests_from_file(const std::string & fpath_test){\n if (fpath_test.empty()){\n fprintf(stderr, \"%s : No test file found.\\n\", __func__);\n return std::map>();\n }\n\n std::map> tests;\n\n auto fin = std::ifstream(fpath_test, std::ios_base::in);\n const char * delimeter = \" => \";\n const char del_tok = ',';\n std::string line;\n while (std::getline(fin, line)) {\n size_t delimiterPos = line.find(delimeter);\n if (delimiterPos != std::string::npos) {\n std::string text = line.substr(0, delimiterPos);\n std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter));\n tests[text] = parse_tokens_from_string(s_tokens, del_tok);\n }\n }\n return tests;\n}\n\nvoid test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){\n std::map> tests = extract_tests_from_file(fpath_test);\n\n size_t n_fails = 0;\n\n for (const auto & test : tests) {\n std::vector tokens = gpt_tokenize(vocab, test.first);\n\n if (tokens != test.second){\n n_fails++;\n\n // print out failure cases\n fprintf(stderr, \"%s : failed test: '%s'\\n\", __func__, test.first.c_str());\n fprintf(stderr, \"%s : tokens in hf: \", __func__);\n for (const auto & t : test.second) {\n fprintf(stderr, \"%s(%d), \", vocab.id_to_token[t].c_str(), t);\n }\n fprintf(stderr, \"\\n\");\n fprintf(stderr, \"%s : tokens in ggml: \", __func__);\n for (const auto & t : tokens) {\n fprintf(stderr, \"%s(%d), \", vocab.id_to_token[t].c_str(), t);\n }\n fprintf(stderr, \"\\n\");\n }\n }\n\n fprintf(stderr, \"%s : %zu tests failed out of %zu tests.\\n\", __func__, n_fails, tests.size());\n}\n\nbool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {\n printf(\"%s: loading vocab from '%s'\\n\", __func__, fname.c_str());\n\n vocab.token_to_id = ::json_parse(fname);\n\n for (const auto & kv : vocab.token_to_id) {\n vocab.id_to_token[kv.second] = kv.first;\n }\n\n printf(\"%s: vocab size = %d\\n\", __func__, (int) vocab.token_to_id.size());\n\n // print the vocabulary\n //for (auto kv : vocab.token_to_id) {\n // printf(\"'%s' -> %d\\n\", kv.first.data(), kv.second);\n //}\n\n return true;\n}\n\ngpt_vocab::id gpt_sample_top_k_top_p(\n const gpt_vocab & vocab,\n const float * logits,\n int top_k,\n double top_p,\n double temp,\n std::mt19937 & rng) {\n int n_logits = vocab.id_to_token.size();\n\n std::vector> logits_id;\n logits_id.reserve(n_logits);\n\n {\n const double scale = 1.0/temp;\n for (int i = 0; i < n_logits; ++i) {\n logits_id.push_back(std::make_pair(logits[i]*scale, i));\n }\n }\n\n // find the top K tokens\n std::partial_sort(\n logits_id.begin(),\n logits_id.begin() + top_k, logits_id.end(),\n [](const std::pair & a, const std::pair & b) {\n return a.first > b.first;\n });\n\n logits_id.resize(top_k);\n\n double maxl = -INFINITY;\n for (const auto & kv : logits_id) {\n maxl = std::max(maxl, kv.first);\n }\n\n // compute probs for the top K tokens\n std::vector probs;\n probs.reserve(logits_id.size());\n\n double sum = 0.0;\n for (const auto & kv : logits_id) {\n double p = exp(kv.first - maxl);\n probs.push_back(p);\n sum += p;\n }\n\n // normalize the probs\n for (auto & p : probs) {\n p /= sum;\n }\n\n if (top_p < 1.0f) {\n double cumsum = 0.0f;\n for (int i = 0; i < top_k; i++) {\n cumsum += probs[i];\n if (cumsum >= top_p) {\n top_k = i + 1;\n probs.resize(top_k);\n logits_id.resize(top_k);\n break;\n }\n }\n\n cumsum = 1.0/cumsum;\n for (int i = 0; i < (int) probs.size(); i++) {\n probs[i] *= cumsum;\n }\n }\n\n //printf(\"\\n\");\n //for (int i = 0; i < (int) probs.size(); i++) {\n // printf(\"%d: '%s' %f\\n\", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);\n //}\n //exit(0);\n\n std::discrete_distribution<> dist(probs.begin(), probs.end());\n int idx = dist(rng);\n\n return logits_id[idx].second;\n}\n\ngpt_vocab::id gpt_sample_top_k_top_p_repeat(\n const gpt_vocab & vocab,\n const float * logits,\n const int32_t * last_n_tokens_data,\n size_t last_n_tokens_data_size,\n int top_k,\n double top_p,\n double temp,\n int repeat_last_n,\n float repeat_penalty,\n std::mt19937 & rng) {\n\n int n_logits = vocab.id_to_token.size();\n\n const auto * plogits = logits;\n\n const auto last_n_tokens = std::vector(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size);\n\n if (temp <= 0) {\n // select the token with the highest logit directly\n float max_logit = plogits[0];\n gpt_vocab::id max_id = 0;\n\n for (int i = 1; i < n_logits; ++i) {\n if (plogits[i] > max_logit) {\n max_logit = plogits[i];\n max_id = i;\n }\n }\n return max_id;\n }\n\n\n std::vector> logits_id;\n logits_id.reserve(n_logits);\n\n {\n const float scale = 1.0f/temp;\n for (int i = 0; i < n_logits; ++i) {\n // repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858)\n // credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main\n if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) {\n // if score < 0 then repetition penalty has to multiplied to reduce the previous token probability\n if (plogits[i] < 0.0f) {\n logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i));\n } else {\n logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i));\n }\n } else {\n logits_id.push_back(std::make_pair(plogits[i]*scale, i));\n }\n }\n }\n\n // find the top K tokens\n std::partial_sort(\n logits_id.begin(),\n logits_id.begin() + top_k, logits_id.end(),\n [](const std::pair & a, const std::pair & b) {\n return a.first > b.first;\n });\n\n logits_id.resize(top_k);\n\n double maxl = -INFINITY;\n for (const auto & kv : logits_id) {\n maxl = std::max(maxl, kv.first);\n }\n\n // compute probs for the top K tokens\n std::vector probs;\n probs.reserve(logits_id.size());\n\n double sum = 0.0;\n for (const auto & kv : logits_id) {\n double p = exp(kv.first - maxl);\n probs.push_back(p);\n sum += p;\n }\n\n // normalize the probs\n for (auto & p : probs) {\n p /= sum;\n }\n\n if (top_p < 1.0f) {\n double cumsum = 0.0f;\n for (int i = 0; i < top_k; i++) {\n cumsum += probs[i];\n if (cumsum >= top_p) {\n top_k = i + 1;\n probs.resize(top_k);\n logits_id.resize(top_k);\n break;\n }\n }\n\n cumsum = 1.0/cumsum;\n for (int i = 0; i < (int) probs.size(); i++) {\n probs[i] *= cumsum;\n }\n }\n\n// printf(\"\\n\");\n// for (int i = 0; i < (int) probs.size(); i++) {\n// for (int i = 0; i < 10; i++) {\n// printf(\"%d: '%s' %f\\n\", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);\n// }\n\n std::discrete_distribution<> dist(probs.begin(), probs.end());\n int idx = dist(rng);\n\n return logits_id[idx].second;\n\n}\n\nvoid high_pass_filter(std::vector & data, float cutoff, float sample_rate) {\n const float rc = 1.0f / (2.0f * M_PI * cutoff);\n const float dt = 1.0f / sample_rate;\n const float alpha = dt / (rc + dt);\n\n float y = data[0];\n\n for (size_t i = 1; i < data.size(); i++) {\n y = alpha * (y + data[i] - data[i - 1]);\n data[i] = y;\n }\n}\n\nbool vad_simple(std::vector & pcmf32, int sample_rate, int last_ms, float vad_thold, float freq_thold, bool verbose) {\n const int n_samples = pcmf32.size();\n const int n_samples_last = (sample_rate * last_ms) / 1000;\n\n if (n_samples_last >= n_samples) {\n // not enough samples - assume no speech\n return false;\n }\n\n if (freq_thold > 0.0f) {\n high_pass_filter(pcmf32, freq_thold, sample_rate);\n }\n\n float energy_all = 0.0f;\n float energy_last = 0.0f;\n\n for (int i = 0; i < n_samples; i++) {\n energy_all += fabsf(pcmf32[i]);\n\n if (i >= n_samples - n_samples_last) {\n energy_last += fabsf(pcmf32[i]);\n }\n }\n\n energy_all /= n_samples;\n energy_last /= n_samples_last;\n\n if (verbose) {\n fprintf(stderr, \"%s: energy_all: %f, energy_last: %f, vad_thold: %f, freq_thold: %f\\n\", __func__, energy_all, energy_last, vad_thold, freq_thold);\n }\n\n if (energy_last > vad_thold*energy_all) {\n return false;\n }\n\n return true;\n}\n\nfloat similarity(const std::string & s0, const std::string & s1) {\n const size_t len0 = s0.size() + 1;\n const size_t len1 = s1.size() + 1;\n\n std::vector col(len1, 0);\n std::vector prevCol(len1, 0);\n\n for (size_t i = 0; i < len1; i++) {\n prevCol[i] = i;\n }\n\n for (size_t i = 0; i < len0; i++) {\n col[0] = i;\n for (size_t j = 1; j < len1; j++) {\n col[j] = std::min(std::min(1 + col[j - 1], 1 + prevCol[j]), prevCol[j - 1] + (i > 0 && s0[i - 1] == s1[j - 1] ? 0 : 1));\n }\n col.swap(prevCol);\n }\n\n const float dist = prevCol[len1 - 1];\n\n return 1.0f - (dist / std::max(s0.size(), s1.size()));\n}\n\nbool is_file_exist(const char * filename) {\n std::ifstream infile(filename);\n return infile.good();\n}\n" }, { "path": "examples/common.h", "content": "// Various helper functions and utilities\n\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n//\n// GPT CLI argument parsing\n//\n\nstruct gpt_params {\n int32_t seed = -1; // RNG seed\n int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());\n int32_t n_predict = 200; // new tokens to predict\n int32_t n_parallel = 1; // number of parallel streams\n int32_t n_batch = 32; // batch size for prompt processing\n int32_t n_ctx = 2048; // context size (this is the KV cache max size)\n int32_t n_gpu_layers = 0; // number of layers to offlload to the GPU\n\n bool ignore_eos = false; // ignore EOS token when generating text\n\n // sampling parameters\n int32_t top_k = 40;\n float top_p = 0.9f;\n float temp = 0.9f;\n int32_t repeat_last_n = 64;\n float repeat_penalty = 1.00f;\n\n std::string model = \"models/gpt-2-117M/ggml-model.bin\"; // model path\n std::string prompt = \"\";\n std::string token_test = \"\";\n\n bool interactive = false;\n int32_t interactive_port = -1;\n};\n\nbool gpt_params_parse(int argc, char ** argv, gpt_params & params);\n\nvoid gpt_print_usage(int argc, char ** argv, const gpt_params & params);\n\nstd::string gpt_random_prompt(std::mt19937 & rng);\n\n//\n// Vocab utils\n//\n\nstd::string trim(const std::string & s);\n\nstd::string replace(\n const std::string & s,\n const std::string & from,\n const std::string & to);\n\nstruct gpt_vocab {\n using id = int32_t;\n using token = std::string;\n\n std::map token_to_id;\n std::map id_to_token;\n std::vector special_tokens;\n\n void add_special_token(const std::string & token);\n};\n\n// poor-man's JSON parsing\nstd::map json_parse(const std::string & fname);\n\nstd::string convert_to_utf8(const std::wstring & input);\n\nstd::wstring convert_to_wstring(const std::string & input);\n\nvoid gpt_split_words(std::string str, std::vector& words);\n\n// split text into tokens\n//\n// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53\n//\n// Regex (Python):\n// r\"\"\"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+\"\"\"\n//\n// Regex (C++):\n// R\"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\\s[:alpha:][:digit:]]+|\\s+(?!\\S)|\\s+)\"\n//\nstd::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text);\n\n// test outputs of gpt_tokenize\n//\n// - compare with tokens generated by the huggingface tokenizer\n// - test cases are chosen based on the model's main language (under 'prompt' directory)\n// - if all sentences are tokenized identically, print 'All tests passed.'\n// - otherwise, print sentence, huggingface tokens, ggml tokens\n//\nvoid test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test);\n\n// load the tokens from encoder.json\nbool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);\n\n// sample next token given probabilities for each embedding\n//\n// - consider only the top K tokens\n// - from them, consider only the top tokens with cumulative probability > P\n//\n// TODO: not sure if this implementation is correct\n// TODO: temperature is not implemented\n//\ngpt_vocab::id gpt_sample_top_k_top_p(\n const gpt_vocab & vocab,\n const float * logits,\n int top_k,\n double top_p,\n double temp,\n std::mt19937 & rng);\n\ngpt_vocab::id gpt_sample_top_k_top_p_repeat(\n const gpt_vocab & vocab,\n const float * logits,\n const int32_t * last_n_tokens_data,\n size_t last_n_tokens_data_size,\n int top_k,\n double top_p,\n double temp,\n int repeat_last_n,\n float repeat_penalty,\n std::mt19937 & rng);\n\n//\n// Audio utils\n//\n\n// Write PCM data into WAV audio file\nclass wav_writer {\nprivate:\n std::ofstream file;\n uint32_t dataSize = 0;\n std::string wav_filename;\n\n bool write_header(const uint32_t sample_rate,\n const uint16_t bits_per_sample,\n const uint16_t channels) {\n\n file.write(\"RIFF\", 4);\n file.write(\"\\0\\0\\0\\0\", 4); // Placeholder for file size\n file.write(\"WAVE\", 4);\n file.write(\"fmt \", 4);\n\n const uint32_t sub_chunk_size = 16;\n const uint16_t audio_format = 1; // PCM format\n const uint32_t byte_rate = sample_rate * channels * bits_per_sample / 8;\n const uint16_t block_align = channels * bits_per_sample / 8;\n\n file.write(reinterpret_cast(&sub_chunk_size), 4);\n file.write(reinterpret_cast(&audio_format), 2);\n file.write(reinterpret_cast(&channels), 2);\n file.write(reinterpret_cast(&sample_rate), 4);\n file.write(reinterpret_cast(&byte_rate), 4);\n file.write(reinterpret_cast(&block_align), 2);\n file.write(reinterpret_cast(&bits_per_sample), 2);\n file.write(\"data\", 4);\n file.write(\"\\0\\0\\0\\0\", 4); // Placeholder for data size\n\n return true;\n }\n\n // It is assumed that PCM data is normalized to a range from -1 to 1\n bool write_audio(const float * data, size_t length) {\n for (size_t i = 0; i < length; ++i) {\n const int16_t intSample = int16_t(data[i] * 32767);\n file.write(reinterpret_cast(&intSample), sizeof(int16_t));\n dataSize += sizeof(int16_t);\n }\n if (file.is_open()) {\n file.seekp(4, std::ios::beg);\n uint32_t fileSize = 36 + dataSize;\n file.write(reinterpret_cast(&fileSize), 4);\n file.seekp(40, std::ios::beg);\n file.write(reinterpret_cast(&dataSize), 4);\n file.seekp(0, std::ios::end);\n }\n return true;\n }\n\n bool open_wav(const std::string & filename) {\n if (filename != wav_filename) {\n if (file.is_open()) {\n file.close();\n }\n }\n if (!file.is_open()) {\n file.open(filename, std::ios::binary);\n wav_filename = filename;\n dataSize = 0;\n }\n return file.is_open();\n }\n\npublic:\n bool open(const std::string & filename,\n const uint32_t sample_rate,\n const uint16_t bits_per_sample,\n const uint16_t channels) {\n\n if (open_wav(filename)) {\n write_header(sample_rate, bits_per_sample, channels);\n } else {\n return false;\n }\n\n return true;\n }\n\n bool close() {\n file.close();\n return true;\n }\n\n bool write(const float * data, size_t length) {\n return write_audio(data, length);\n }\n\n ~wav_writer() {\n if (file.is_open()) {\n file.close();\n }\n }\n};\n\n\n// Apply a high-pass frequency filter to PCM audio\n// Suppresses frequencies below cutoff Hz\nvoid high_pass_filter(\n std::vector & data,\n float cutoff,\n float sample_rate);\n\n// Basic voice activity detection (VAD) using audio energy adaptive threshold\nbool vad_simple(\n std::vector & pcmf32,\n int sample_rate,\n int last_ms,\n float vad_thold,\n float freq_thold,\n bool verbose);\n\n// compute similarity between two strings using Levenshtein distance\nfloat similarity(const std::string & s0, const std::string & s1);\n\n//\n// Terminal utils\n//\n\n#define SQR(X) ((X) * (X))\n#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40\n\n/**\n * Quantizes 24-bit RGB to xterm256 code range [16,256).\n */\nstatic int rgb2xterm256(int r, int g, int b) {\n unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377};\n int av, ir, ig, ib, il, qr, qg, qb, ql;\n av = r * .299 + g * .587 + b * .114 + .5;\n ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8;\n qr = cube[(ir = UNCUBE(r))];\n qg = cube[(ig = UNCUBE(g))];\n qb = cube[(ib = UNCUBE(b))];\n if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <=\n SQR(ql - r) + SQR(ql - g) + SQR(ql - b))\n return ir * 36 + ig * 6 + ib + 020;\n return il + 0350;\n}\n\nstatic std::string set_xterm256_foreground(int r, int g, int b) {\n int x = rgb2xterm256(r, g, b);\n std::ostringstream oss;\n oss << \"\\033[38;5;\" << x << \"m\";\n return oss.str();\n}\n\n// Lowest is red, middle is yellow, highest is green. Color scheme from\n// Paul Tol; it is colorblind friendly https://sronpersonalpages.nl/~pault\nconst std::vector k_colors = {\n set_xterm256_foreground(220, 5, 12),\n set_xterm256_foreground(232, 96, 28),\n set_xterm256_foreground(241, 147, 45),\n set_xterm256_foreground(246, 193, 65),\n set_xterm256_foreground(247, 240, 86),\n set_xterm256_foreground(144, 201, 135),\n set_xterm256_foreground( 78, 178, 101),\n};\n\n// ANSI formatting codes\nstatic std::string set_inverse() {\n return \"\\033[7m\";\n}\n\nstatic std::string set_underline() {\n return \"\\033[4m\";\n}\n\nstatic std::string set_dim() {\n return \"\\033[2m\";\n}\n\n// Style scheme for different confidence levels\nconst std::vector k_styles = {\n set_inverse(), // Low confidence - inverse (highlighted)\n set_underline(), // Medium confidence - underlined\n set_dim(), // High confidence - dim\n};\n\n//\n// Other utils\n//\n\n// check if file exists using ifstream\nbool is_file_exist(const char * filename);\n" }, { "path": "examples/gpt-2/CMakeLists.txt", "content": "#\n# gpt-2\n\nset(TEST_TARGET gpt-2-ctx)\nadd_executable(${TEST_TARGET} main-ctx.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)\n\nset(TEST_TARGET gpt-2-alloc)\nadd_executable(${TEST_TARGET} main-alloc.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)\n\nset(TEST_TARGET gpt-2-backend)\nadd_executable(${TEST_TARGET} main-backend.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)\n\nset(TEST_TARGET gpt-2-sched)\nadd_executable(${TEST_TARGET} main-sched.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)\n\n#\n# gpt-2-quantize\n\nset(TEST_TARGET gpt-2-quantize)\nadd_executable(${TEST_TARGET} quantize.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)\n\n#\n# gpt-2-batched\n\nset(TEST_TARGET gpt-2-batched)\nadd_executable(${TEST_TARGET} main-batched.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)\n" }, { "path": "examples/gpt-2/README.md", "content": "# gpt-2\n\nThis is a C++ example running GPT-2 inference using the [ggml](https://github.com/ggerganov/ggml) library.\n\nThe program runs on the CPU - no video card is required.\n\nThe [Cerebras-GPT](https://huggingface.co/cerebras) models are also supported.\n\nThe example supports the following GPT-2 models:\n\n| Model | Description | Disk Size |\n| --- | --- | --- |\n| 117M | Small model | 240 MB |\n| 345M | Medium model | 680 MB |\n| 774M | Large model | 1.5 GB |\n| 1558M | XL model | 3.0 GB |\n\nSample performance on MacBook M1 Pro:\n\n| Model | Size | Time / Token |\n| --- | --- | --- |\n| GPT-2 | 117M | 5 ms |\n| GPT-2 | 345M | 12 ms |\n| GPT-2 | 774M | 23 ms |\n| GPT-2 | 1558M | 42 ms |\n\n*TODO: add tables for Cerebras-GPT models*\n\nSample output:\n\n```bash\n$ ./bin/gpt-2 -h\nusage: ./bin/gpt-2 [options]\n\noptions:\n -h, --help show this help message and exit\n -s SEED, --seed SEED RNG seed (default: -1)\n -t N, --threads N number of threads to use during computation (default: 8)\n -p PROMPT, --prompt PROMPT\n prompt to start generation with (default: random)\n -n N, --n_predict N number of tokens to predict (default: 200)\n --top_k N top-k sampling (default: 40)\n --top_p N top-p sampling (default: 0.9)\n --temp N temperature (default: 1.0)\n -b N, --batch_size N batch size for prompt processing (default: 8)\n -m FNAME, --model FNAME\n model path (default: models/gpt-2-117M/ggml-model.bin)\n\n$ ./bin/gpt-2\ngpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin'\ngpt2_model_load: n_vocab = 50257\ngpt2_model_load: n_ctx = 1024\ngpt2_model_load: n_embd = 768\ngpt2_model_load: n_head = 12\ngpt2_model_load: n_layer = 12\ngpt2_model_load: f16 = 1\ngpt2_model_load: ggml ctx size = 311.12 MB\ngpt2_model_load: memory size = 72.00 MB, n_mem = 12288\ngpt2_model_load: model size = 239.08 MB\nmain: number of tokens in prompt = 1\n\nSo this is going to be the end of the line for us.\n\nIf the Dolphins continue to do their business, it's possible that the team could make a bid to bring in new defensive coordinator Scott Linehan.\n\nLinehan's job is a little daunting, but he's a great coach and an excellent coach. I don't believe we're going to make the playoffs.\n\nWe're going to have to work hard to keep our heads down and get ready to go.<|endoftext|>\n\nmain: mem per token = 2048612 bytes\nmain: load time = 106.32 ms\nmain: sample time = 7.10 ms\nmain: predict time = 506.40 ms / 5.06 ms per token\nmain: total time = 629.84 ms\n```\n\n## Downloading and converting the original models (GPT-2)\n\nYou can download the original model files using the [download-model.sh](download-model.sh) Bash script. The models are\nin Tensorflow format, so in order to use them with ggml, you need to convert them to appropriate format. This is done\nvia the [convert-ckpt-to-ggml.py](convert-ckpt-to-ggml.py) python script.\n\nHere is the entire process for the GPT-2 117M model (download from official site + conversion):\n\n```bash\ncd ggml/build\n../examples/gpt-2/download-model.sh 117M\n\nDownloading model 117M ...\nmodels/gpt-2-117M/checkpoint 100%[=============================>] 77 --.-KB/s in 0s\nmodels/gpt-2-117M/encoder.json 100%[=============================>] 1018K 1.20MB/s in 0.8s\nmodels/gpt-2-117M/hparams.json 100%[=============================>] 90 --.-KB/s in 0s\nmodels/gpt-2-117M/model.ckpt.data-00000-of-00001 100%[=============================>] 474.70M 1.21MB/s in 8m 39s\nmodels/gpt-2-117M/model.ckpt.index 100%[=============================>] 5.09K --.-KB/s in 0s\nmodels/gpt-2-117M/model.ckpt.meta 100%[=============================>] 460.11K 806KB/s in 0.6s\nmodels/gpt-2-117M/vocab.bpe 100%[=============================>] 445.62K 799KB/s in 0.6s\nDone! Model '117M' saved in 'models/gpt-2-117M/'\n\nRun the convert-ckpt-to-ggml.py script to convert the model to ggml format.\n\n python /Users/john/ggml/examples/gpt-2/convert-ckpt-to-ggml.py models/gpt-2-117M/ 1\n\n```\n\nThis conversion requires that you have python and Tensorflow installed on your computer. Still, if you want to avoid\nthis, you can download the already converted ggml models as described below.\n\n## Downloading and converting the original models (Cerebras-GPT)\n\nClone the respective repository from here: https://huggingface.co/cerebras\n\nUse the [convert-cerebras-to-ggml.py](convert-cerebras-to-ggml.py) script to convert the model to `ggml` format:\n\n```bash\ncd ggml/build\ngit clone https://huggingface.co/cerebras/Cerebras-GPT-111M models/\npython ../examples/gpt-2/convert-cerebras-to-ggml.py models/Cerebras-GPT-111M/\n\n```\n\n## Downloading the ggml model directly (GPT-2)\n\nFor convenience, I will be hosting the converted ggml model files in order to make it easier to run the examples. This\nway, you can directly download a single binary file and start using it. No python or Tensorflow is required.\n\nHere is how to get the 117M ggml model:\n\n```bash\ncd ggml/build\n../examples/gpt-2/download-ggml-model.sh 117M\n\nDownloading ggml model 117M ...\nmodels/gpt-2-117M/ggml-model.bin 100%[===============================>] 239.58M 8.52MB/s in 28s\nDone! Model '117M' saved in 'models/gpt-2-117M/ggml-model.bin'\nYou can now use it like this:\n\n $ ./bin/gpt-2 -m models/gpt-2-117M/ggml-model.bin -p \"This is an example\"\n\n```\n\nAt some point, I might decide to stop hosting these models. So in that case, simply revert to the manual process above.\n\n## Quantizing the models\n\nYou can also try to quantize the `ggml` models via 4-bit integer quantization.\nKeep in mind that for smaller models, this will render them completely useless.\nYou generally want to quantize larger models.\n\n```bash\n# quantize GPT-2 F16 to Q4_0 (faster but less precise)\n./bin/gpt-2-quantize models/gpt-2-1558M/ggml-model-f16.bin models/gpt-2-1558M/ggml-model-q4_0.bin 2\n./bin/gpt-2 -m models/gpt-2-1558M/ggml-model-q4_0.bin -p \"This is an example\"\n\n# quantize Cerebras F16 to Q4_1 (slower but more precise)\n./bin/gpt-2-quantize models/Cerebras-GPT-6.7B/ggml-model-f16.bin models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin 3\n./bin/gpt-2 -m models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin -p \"This is an example\"\n\n```\n\n## Batched generation example\n\nYou can try the batched generation from a given prompt using the gpt-2-batched binary.\n\nSample output:\n\n```bash\n$ gpt-2-batched -np 5 -m models/gpt-2-117M/ggml-model.bin -p \"Hello my name is\" -n 50\n\nmain: seed = 1697037431\ngpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin'\ngpt2_model_load: n_vocab = 50257\ngpt2_model_load: n_ctx = 1024\ngpt2_model_load: n_embd = 768\ngpt2_model_load: n_head = 12\ngpt2_model_load: n_layer = 12\ngpt2_model_load: ftype = 1\ngpt2_model_load: qntvr = 0\ngpt2_model_load: ggml tensor size = 320 bytes\ngpt2_model_load: backend buffer size = 312.72 MB\nggml_init_cublas: found 1 CUDA devices:\n Device 0: NVIDIA GeForce GTX 1660, compute capability 7.5\ngpt2_model_load: using CPU backend\ngpt2_model_load: memory size = 72.00 MB, n_mem = 12288\ngpt2_model_load: model size = 239.08 MB\nextract_tests_from_file : No test file found.\ntest_gpt_tokenizer : 0 tests failed out of 0 tests.\nmain: compute buffer size: 3.26 MB\n\n\nmain: generating 5 sequences ...\nmain: prompt: 'Hello my name is'\nmain: number of tokens in prompt = 4, first 8 tokens: 15496 616 1438 318\n\n\nsequence 0:\n\nHello my name is John. You can call me any way you want, if you want, but for my very first date, I will be on the phone with you. We're both in our early 20s, but I feel like it's all\n\nsequence 1:\n\nHello my name is Robert, and I want to say that we're proud to have your company here on the world's largest platform for sharing your stories with us. This is a huge opportunity for our community. We have hundreds of people on this team and\n\nsequence 2:\n\nHello my name is Jack. I'm the one who created you.\n\nJack is a boy with a big smile and a big heart. He is a handsome guy. He loves the outdoors and loves the people he meets. He wants to be a\n\nsequence 3:\n\nHello my name is John. I am a Canadian citizen with a large number of family in Quebec and I am interested in studying. My aim is to take up a post in the Journal of the International Academy of Sciences of Canada which I am currently finishing.\n\nsequence 4:\n\nHello my name is Dan. I am an entrepreneur. I am a great father. I am a great husband. I am a great husband. I am a great dad. And I am a great husband.\n\nI love my life. I love\n\n\n\nmain: load time = 880.80 ms\nmain: sample time = 91.43 ms\nmain: predict time = 2518.29 ms\nmain: total time = 3544.32 ms\n```\n" }, { "path": "examples/gpt-2/convert-cerebras-to-ggml.py", "content": "# Convert Cerebras models to ggml format\n#\n# ref: https://www.cerebras.net/blog/cerebras-gpt-a-family-of-open-compute-efficient-large-language-models/\n#\n\nimport sys\nimport struct\nimport json\nimport torch\nimport numpy as np\nimport re\n\nfrom transformers import AutoModelForCausalLM\n\n# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py\ndef bytes_to_unicode():\n \"\"\"\n Returns list of utf-8 byte and a corresponding list of unicode strings.\n The reversible bpe codes work on unicode strings.\n This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.\n When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.\n This is a signficant percentage of your normal, say, 32K bpe vocab.\n To avoid that, we want lookup tables between utf-8 bytes and unicode strings.\n And avoids mapping to whitespace/control characters the bpe code barfs on.\n \"\"\"\n bs = list(range(ord(\"!\"), ord(\"~\")+1))+list(range(ord(\"¡\"), ord(\"¬\")+1))+list(range(ord(\"®\"), ord(\"ÿ\")+1))\n cs = bs[:]\n n = 0\n for b in range(2**8):\n if b not in bs:\n bs.append(b)\n cs.append(2**8+n)\n n += 1\n cs = [chr(n) for n in cs]\n return dict(zip(bs, cs))\n\nif len(sys.argv) < 2:\n print(\"Usage: convert-cerebras-to-ggml.py dir-model [use-f32]\\n\")\n sys.exit(1)\n\n# output in the same directory as the model\ndir_model = sys.argv[1]\nfname_out = sys.argv[1] + \"/ggml-model-f16.bin\"\n\nwith open(dir_model + \"/vocab.json\", \"r\", encoding=\"utf-8\") as f:\n encoder = json.load(f)\n\nwith open(dir_model + \"/config.json\", \"r\", encoding=\"utf-8\") as f:\n hparams = json.load(f)\n\n# use 16-bit or 32-bit floats\nuse_f16 = True\nif len(sys.argv) > 2:\n use_f16 = False\n fname_out = sys.argv[1] + \"/ggml-model-f32.bin\"\n\nmodel = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True)\n#print (model)\n\nlist_vars = model.state_dict()\n#print (list_vars)\n\nprint(hparams)\n\nfout = open(fname_out, \"wb\")\n\nfout.write(struct.pack(\"i\", 0x67676d6c)) # magic: ggml in hex\nfout.write(struct.pack(\"i\", hparams[\"vocab_size\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_positions\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_embd\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_head\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_layer\"]))\nfout.write(struct.pack(\"i\", use_f16))\n\nbyte_encoder = bytes_to_unicode()\nbyte_decoder = {v:k for k, v in byte_encoder.items()}\n\nfout.write(struct.pack(\"i\", len(encoder)))\n\nfor key in encoder:\n text = bytearray([byte_decoder[c] for c in key])\n fout.write(struct.pack(\"i\", len(text)))\n fout.write(text)\n\nfor name in list_vars.keys():\n data = list_vars[name].squeeze().numpy()\n print(\"Processing variable: \" + name + \" with shape: \", data.shape)\n\n # rename headers to keep compatibility\n if name == \"transformer.ln_f.weight\":\n name = \"model/ln_f/g\"\n elif name == \"transformer.ln_f.bias\":\n name = \"model/ln_f/b\"\n elif name == \"transformer.wte.weight\":\n name = \"model/wte\"\n elif name == \"transformer.wpe.weight\":\n name = \"model/wpe\"\n elif name == \"lm_head.weight\":\n name = \"model/lm_head\"\n elif re.match(r\"transformer.h\\.\\d+\\.ln_1\\.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/ln_1/g\"\n elif re.match(r\"transformer.h\\.\\d+\\.ln_1\\.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/ln_1/b\"\n elif re.match(r\"transformer.h\\.\\d+\\.attn\\.c_attn\\.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/attn/c_attn/w\"\n elif re.match(r\"transformer.h\\.\\d+\\.attn\\.c_attn\\.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/attn/c_attn/b\"\n elif re.match(r\"transformer.h\\.\\d+\\.attn\\.c_proj\\.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/attn/c_proj/w\"\n elif re.match(r\"transformer.h.\\d+.attn.c_proj.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/attn/c_proj/b\"\n elif re.match(r\"transformer.h.\\d+.ln_2.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/ln_2/g\"\n elif re.match(r\"transformer.h.\\d+.ln_2.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/ln_2/b\"\n elif re.match(r\"transformer.h.\\d+.mlp.c_fc.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/mlp/c_fc/w\"\n elif re.match(r\"transformer.h.\\d+.mlp.c_fc.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/mlp/c_fc/b\"\n elif re.match(r\"transformer.h.\\d+.mlp.c_proj.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/mlp/c_proj/w\"\n elif re.match(r\"transformer.h.\\d+.mlp.c_proj.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/mlp/c_proj/b\"\n else:\n print(\"Unrecognized variable name. %s\", name)\n\n # we don't need these\n if name.endswith(\"attn.masked_bias\") or name.endswith(\".attn.bias\"):\n print(\" Skipping variable: \" + name)\n continue\n\n n_dims = len(data.shape);\n\n # ftype == 0 -> float32, ftype == 1 -> float16\n ftype = 0;\n if use_f16:\n if (name == \"model/wte\" or name == \"model/lm_head\" or name[-2:] == \"/g\" or name[-2:] == \"/w\") and n_dims == 2:\n print(\" Converting to float16\")\n data = data.astype(np.float16)\n ftype = 1\n else:\n print(\" Converting to float32\")\n data = data.astype(np.float32)\n ftype = 0\n\n # for efficiency - transpose the projection matrices\n # \"model/h.*/attn/c_attn/w\"\n # \"model/h.*/attn/c_proj/w\"\n # \"model/h.*/mlp/c_fc/w\"\n # \"model/h.*/mlp/c_proj/w\"\n if name[-14:] == \"/attn/c_attn/w\" or \\\n name[-14:] == \"/attn/c_proj/w\" or \\\n name[-11:] == \"/mlp/c_fc/w\" or \\\n name[-13:] == \"/mlp/c_proj/w\":\n print(\" Transposing\")\n data = data.transpose()\n\n # header\n str = name.encode('utf-8')\n fout.write(struct.pack(\"iii\", n_dims, len(str), ftype))\n for i in range(n_dims):\n fout.write(struct.pack(\"i\", data.shape[n_dims - 1 - i]))\n fout.write(str);\n\n # data\n data.tofile(fout)\n\nfout.close()\n\nprint(\"Done. Output file: \" + fname_out)\nprint(\"\")\n" }, { "path": "examples/gpt-2/convert-ckpt-to-ggml.py", "content": "# Convert a model checkpoint to a ggml compatible file\n#\n# Load the model using TensorFlow.\n# Iterate over all variables and write them to a binary file.\n#\n# For each variable, write the following:\n# - Number of dimensions (int)\n# - Name length (int)\n# - Dimensions (int[n_dims])\n# - Name (char[name_length])\n# - Data (float[n_dims])\n#\n# By default, the bigger matrices are converted to 16-bit floats.\n# This can be disabled by adding the \"use-f32\" CLI argument.\n#\n# At the start of the ggml file we write the model parameters\n# and vocabulary.\n#\n\nimport sys\nimport json\nimport struct\nimport numpy as np\nimport tensorflow as tf\n\n# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py\ndef bytes_to_unicode():\n \"\"\"\n Returns list of utf-8 byte and a corresponding list of unicode strings.\n The reversible bpe codes work on unicode strings.\n This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.\n When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.\n This is a signficant percentage of your normal, say, 32K bpe vocab.\n To avoid that, we want lookup tables between utf-8 bytes and unicode strings.\n And avoids mapping to whitespace/control characters the bpe code barfs on.\n \"\"\"\n bs = list(range(ord(\"!\"), ord(\"~\")+1))+list(range(ord(\"¡\"), ord(\"¬\")+1))+list(range(ord(\"®\"), ord(\"ÿ\")+1))\n cs = bs[:]\n n = 0\n for b in range(2**8):\n if b not in bs:\n bs.append(b)\n cs.append(2**8+n)\n n += 1\n cs = [chr(n) for n in cs]\n return dict(zip(bs, cs))\n\n# helper method to convert a numpy array to different float types\ndef convert_to_ftype(data, ftype):\n # fp16\n if ftype == 1:\n return data.astype(np.float16)\n\n assert False, \"Invalid ftype: \" + str(ftype)\n\nif len(sys.argv) < 3:\n print(\"Usage: convert-ckpt-to-ggml.py dir-model ftype\\n\")\n print(\" ftype == 0 -> float32\")\n print(\" ftype == 1 -> float16\")\n sys.exit(1)\n\n# output in the same directory as the model\ndir_model = sys.argv[1]\nfname_out = sys.argv[1] + \"/ggml-model.bin\"\n\nwith open(dir_model + \"/encoder.json\", \"r\", encoding=\"utf-8\") as f:\n encoder = json.load(f)\n\nwith open(dir_model + \"/hparams.json\", \"r\", encoding=\"utf-8\") as f:\n hparams = json.load(f)\n\n# possible data types\n# ftype == 0 -> float32\n# ftype == 1 -> float16\n#\n# map from ftype to string\nftype_str = [\"f32\", \"f16\"]\n\nftype = 1\nif len(sys.argv) > 2:\n ftype = int(sys.argv[2])\n if ftype < 0 or ftype > 1:\n print(\"Invalid ftype: \" + str(ftype))\n sys.exit(1)\n fname_out = sys.argv[1] + \"/ggml-model-\" + ftype_str[ftype] + \".bin\"\n\nlist_vars = tf.train.list_variables(dir_model)\n\nfout = open(fname_out, \"wb\")\n\nfout.write(struct.pack(\"i\", 0x67676d6c)) # magic: ggml in hex\nfout.write(struct.pack(\"i\", hparams[\"n_vocab\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_ctx\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_embd\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_head\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_layer\"]))\nfout.write(struct.pack(\"i\", ftype))\n\nbyte_encoder = bytes_to_unicode()\nbyte_decoder = {v:k for k, v in byte_encoder.items()}\n\nfout.write(struct.pack(\"i\", len(encoder)))\n\nfor key in encoder:\n text = bytearray([byte_decoder[c] for c in key])\n fout.write(struct.pack(\"i\", len(text)))\n fout.write(text)\n\nfor name, shape in list_vars:\n print(\"Processing variable: \" + name + \" with shape: \", shape)\n\n data = tf.train.load_variable(dir_model, name).squeeze()\n n_dims = len(data.shape);\n\n # for efficiency - transpose the projection matrices\n # \"model/h.*/attn/c_attn/w\"\n # \"model/h.*/attn/c_proj/w\"\n # \"model/h.*/mlp/c_fc/w\"\n # \"model/h.*/mlp/c_proj/w\"\n if name[-14:] == \"/attn/c_attn/w\" or \\\n name[-14:] == \"/attn/c_proj/w\" or \\\n name[-11:] == \"/mlp/c_fc/w\" or \\\n name[-13:] == \"/mlp/c_proj/w\":\n print(\" Transposing\")\n data = data.transpose()\n\n dshape = data.shape\n\n ftype_cur = 0\n if ftype != 0:\n # match name:\n # \"model/wte\"\n # \"model/h.*/attn/c_attn/w\"\n # \"model/h.*/attn/c_proj/w\"\n # \"model/h.*/mlp/c_fc/w\"\n # \"model/h.*/mlp/c_proj/w\"\n if name == \"model/wte\" or name[-2:] == \"/w\":\n print(\" Converting to \" + ftype_str[ftype])\n data = convert_to_ftype(data, ftype)\n ftype_cur = ftype\n else:\n print(\" Converting to float32\")\n data = data.astype(np.float32)\n ftype_cur = 0\n\n # header\n str = name.encode('utf-8')\n fout.write(struct.pack(\"iii\", n_dims, len(str), ftype_cur))\n for i in range(n_dims):\n fout.write(struct.pack(\"i\", dshape[n_dims - 1 - i]))\n fout.write(str);\n\n # data\n data.tofile(fout)\n\nfout.close()\n\nprint(\"Done. Output file: \" + fname_out)\nprint(\"\")\n" }, { "path": "examples/gpt-2/convert-h5-to-ggml.py", "content": "# Convert GPT-2 h5 transformer model to ggml format\n#\n# Load the model using GPT2Model.\n# Iterate over all variables and write them to a binary file.\n#\n# For each variable, write the following:\n# - Number of dimensions (int)\n# - Name length (int)\n# - Dimensions (int[n_dims])\n# - Name (char[name_length])\n# - Data (float[n_dims])\n#\n# By default, the bigger matrices are converted to 16-bit floats.\n# This can be disabled by adding the \"use-f32\" CLI argument.\n#\n# At the start of the ggml file we write the model parameters\n# and vocabulary.\n#\n\nimport sys\nimport struct\nimport json\nimport numpy as np\nimport re\n\nfrom transformers import GPT2Model\n\n# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py\ndef bytes_to_unicode():\n \"\"\"\n Returns list of utf-8 byte and a corresponding list of unicode strings.\n The reversible bpe codes work on unicode strings.\n This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.\n When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.\n This is a signficant percentage of your normal, say, 32K bpe vocab.\n To avoid that, we want lookup tables between utf-8 bytes and unicode strings.\n And avoids mapping to whitespace/control characters the bpe code barfs on.\n \"\"\"\n bs = list(range(ord(\"!\"), ord(\"~\")+1))+list(range(ord(\"¡\"), ord(\"¬\")+1))+list(range(ord(\"®\"), ord(\"ÿ\")+1))\n cs = bs[:]\n n = 0\n for b in range(2**8):\n if b not in bs:\n bs.append(b)\n cs.append(2**8+n)\n n += 1\n cs = [chr(n) for n in cs]\n return dict(zip(bs, cs))\n\nif len(sys.argv) < 2:\n print(\"Usage: convert-h5-to-ggml.py dir-model [use-f32]\\n\")\n sys.exit(1)\n\n# output in the same directory as the model\ndir_model = sys.argv[1]\nfname_out = sys.argv[1] + \"/ggml-model.bin\"\n\nwith open(dir_model + \"/vocab.json\", \"r\", encoding=\"utf-8\") as f:\n encoder = json.load(f)\n\nwith open(dir_model + \"/added_tokens.json\", \"r\", encoding=\"utf-8\") as f:\n encoder_added = json.load(f)\n\nwith open(dir_model + \"/config.json\", \"r\", encoding=\"utf-8\") as f:\n hparams = json.load(f)\n\n# use 16-bit or 32-bit floats\nuse_f16 = True\nif len(sys.argv) > 2:\n use_f16 = False\n fname_out = sys.argv[1] + \"/ggml-model-f32.bin\"\n\nmodel = GPT2Model.from_pretrained(dir_model, low_cpu_mem_usage=True)\n#print (model)\n\nlist_vars = model.state_dict()\n#print (list_vars)\n\nfout = open(fname_out, \"wb\")\n\nfout.write(struct.pack(\"i\", 0x67676d6c)) # magic: ggml in hex\nfout.write(struct.pack(\"i\", hparams[\"vocab_size\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_positions\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_embd\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_head\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_layer\"]))\n#fout.write(struct.pack(\"i\", hparams[\"rotary_dim\"]))\nfout.write(struct.pack(\"i\", use_f16))\n\nbyte_encoder = bytes_to_unicode()\nbyte_decoder = {v:k for k, v in byte_encoder.items()}\n\nfout.write(struct.pack(\"i\", len(encoder) + len(encoder_added)))\n\nfor key in encoder:\n text = bytearray([byte_decoder[c] for c in key])\n fout.write(struct.pack(\"i\", len(text)))\n fout.write(text)\n\nfor key in encoder_added:\n text = bytearray([byte_decoder[c] for c in key])\n fout.write(struct.pack(\"i\", len(text)))\n fout.write(text)\n\nfor name in list_vars.keys():\n data = list_vars[name].squeeze().numpy()\n print(\"Processing variable: \" + name + \" with shape: \", data.shape)\n\n # we don't need these\n if name.endswith(\"attn.masked_bias\") or name.endswith(\".attn.bias\"):\n print(\" Skipping variable: \" + name)\n continue\n\n n_dims = len(data.shape);\n\n # ftype == 0 -> float32, ftype == 1 -> float16\n ftype = 0;\n if use_f16:\n if name[-7:] == \".weight\" and n_dims == 2:\n print(\" Converting to float16\")\n data = data.astype(np.float16)\n ftype = 1\n else:\n print(\" Converting to float32\")\n data = data.astype(np.float32)\n ftype = 0\n\n # for efficiency - transpose these matrices:\n # \"transformer.h.*.mlp.c_proj.weight\n if name.endswith(\".mlp.c_proj.weight\"):\n print(\" Transposing\")\n data = data.transpose()\n\n # rename headers to keep compatibility\n if name == \"ln_f.weight\":\n name = \"model/ln_f/g\"\n elif name == \"ln_f.bias\":\n name = \"model/ln_f/b\"\n elif name == \"wte.weight\":\n name = \"model/wte\"\n elif name == \"wpe.weight\":\n name = \"model/wpe\"\n elif re.match(r\"h\\.\\d+\\.ln_1\\.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/ln_1/g\"\n elif re.match(r\"h\\.\\d+\\.ln_1\\.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/ln_1/b\"\n elif re.match(r\"h\\.\\d+\\.attn\\.c_attn\\.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/attn/c_attn/w\"\n elif re.match(r\"h\\.\\d+\\.attn\\.c_attn\\.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/attn/c_attn/b\"\n elif re.match(r\"h\\.\\d+\\.attn\\.c_proj\\.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/attn/c_proj/w\"\n elif re.match(r\"h.\\d+.attn.c_proj.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/attn/c_proj/b\"\n elif re.match(r\"h.\\d+.ln_2.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/ln_2/g\"\n elif re.match(r\"h.\\d+.ln_2.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/ln_2/b\"\n elif re.match(r\"h.\\d+.mlp.c_fc.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/mlp/c_fc/w\"\n elif re.match(r\"h.\\d+.mlp.c_fc.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/mlp/c_fc/b\"\n elif re.match(r\"h.\\d+.mlp.c_proj.weight\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/mlp/c_proj/w\"\n elif re.match(r\"h.\\d+.mlp.c_proj.bias\", name):\n i = re.findall(\"\\d+\", name)[0]\n name = f\"model/h{i}/mlp/c_proj/b\"\n else:\n print(\"Unrecognized variable name. %s\", name)\n\n str = name.encode('utf-8')\n\n fout.write(struct.pack(\"iii\", n_dims, len(str), ftype))\n for i in range(n_dims):\n fout.write(struct.pack(\"i\", data.shape[n_dims - 1 - i]))\n fout.write(str);\n\n # data\n data.tofile(fout)\n\nfout.close()\n\nprint(\"Done. Output file: \" + fname_out)\nprint(\"\")\n" }, { "path": "examples/gpt-2/download-ggml-model.sh", "content": "#!/bin/bash\n\n# This script downloads GPT-2 model files that have already been converted to ggml format.\n# This way you don't have to convert them yourself.\n#\n# If you want to download the original GPT-2 model files, use the \"download-model.sh\" script instead.\n\n#src=\"https://ggml.ggerganov.com\"\n#pfx=\"ggml-model-gpt-2\"\n\nsrc=\"https://huggingface.co/ggerganov/ggml\"\npfx=\"resolve/main/ggml-model-gpt-2\"\n\nggml_path=$(dirname $(realpath $0))\n\n# GPT-2 models\nmodels=( \"117M\" \"345M\" \"774M\" \"1558M\" )\n\n# list available models\nfunction list_models {\n printf \"\\n\"\n printf \" Available models:\"\n for model in \"${models[@]}\"; do\n printf \" $model\"\n done\n printf \"\\n\\n\"\n}\n\nif [ \"$#\" -ne 1 ]; then\n printf \"Usage: $0 \\n\"\n list_models\n\n exit 1\nfi\n\nmodel=$1\n\nif [[ ! \" ${models[@]} \" =~ \" ${model} \" ]]; then\n printf \"Invalid model: $model\\n\"\n list_models\n\n exit 1\nfi\n\n# download ggml model\n\nprintf \"Downloading ggml model $model ...\\n\"\n\nmkdir -p models/gpt-2-$model\n\nif [ -x \"$(command -v wget)\" ]; then\n wget --quiet --show-progress -O models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin\nelif [ -x \"$(command -v curl)\" ]; then\n curl -L --output models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin\nelse\n printf \"Either wget or curl is required to download models.\\n\"\n exit 1\nfi\n\nif [ $? -ne 0 ]; then\n printf \"Failed to download ggml model $model \\n\"\n printf \"Please try again later or download the original GPT-2 model files and convert them yourself.\\n\"\n exit 1\nfi\n\nprintf \"Done! Model '$model' saved in 'models/gpt-2-$model/ggml-model.bin'\\n\"\nprintf \"You can now use it like this:\\n\\n\"\nprintf \" $ ./bin/gpt-2 -m models/gpt-2-$model/ggml-model.bin -p \\\"This is an example\\\"\\n\"\nprintf \"\\n\"\n" }, { "path": "examples/gpt-2/download-model.sh", "content": "#!/bin/bash\n\nggml_path=$(dirname $(realpath $0))\n\n# GPT-2 models\nmodels=( \"117M\" \"345M\" \"774M\" \"1558M\" )\n\n# list available models\nfunction list_models {\n printf \"\\n\"\n printf \" Available models:\"\n for model in \"${models[@]}\"; do\n printf \" $model\"\n done\n printf \"\\n\\n\"\n}\n\nif [ \"$#\" -ne 1 ]; then\n printf \"Usage: $0 \\n\"\n list_models\n\n exit 1\nfi\n\nmodel=$1\n\nif [[ ! \" ${models[@]} \" =~ \" ${model} \" ]]; then\n printf \"Invalid model: $model\\n\"\n list_models\n\n exit 1\nfi\n\n# download model\n\nprintf \"Downloading model $model ...\\n\"\n\nmkdir -p models/gpt-2-$model\n\nfor file in checkpoint encoder.json hparams.json model.ckpt.data-00000-of-00001 model.ckpt.index model.ckpt.meta vocab.bpe; do\n wget --quiet --show-progress -O models/gpt-2-$model/$file https://openaipublic.blob.core.windows.net/gpt-2/models/$model/$file\ndone\n\nprintf \"Done! Model '$model' saved in 'models/gpt-2-$model/'\\n\\n\"\nprintf \"Run the convert-ckpt-to-ggml.py script to convert the model to ggml format.\\n\"\nprintf \"\\n\"\nprintf \" python $ggml_path/convert-ckpt-to-ggml.py models/gpt-2-$model/\\n\"\nprintf \"\\n\"\n" }, { "path": "examples/gpt-2/main-alloc.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n\n#include \"common.h\"\n#include \"common-ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\n// default hparams (GPT-2 117M)\nstruct gpt2_hparams {\n int32_t n_vocab = 50257;\n int32_t n_ctx = 1024;\n int32_t n_embd = 768;\n int32_t n_head = 12;\n int32_t n_layer = 12;\n int32_t ftype = 1;\n float eps = 1e-5f;\n};\n\nstruct gpt2_layer {\n // normalization\n struct ggml_tensor * ln_1_g;\n struct ggml_tensor * ln_1_b;\n\n struct ggml_tensor * ln_2_g;\n struct ggml_tensor * ln_2_b;\n\n // attention\n struct ggml_tensor * c_attn_attn_w;\n struct ggml_tensor * c_attn_attn_b;\n\n struct ggml_tensor * c_attn_proj_w;\n struct ggml_tensor * c_attn_proj_b;\n\n // mlp\n struct ggml_tensor * c_mlp_fc_w;\n struct ggml_tensor * c_mlp_fc_b;\n\n struct ggml_tensor * c_mlp_proj_w;\n struct ggml_tensor * c_mlp_proj_b;\n};\n\nstruct gpt2_model {\n gpt2_hparams hparams;\n\n // normalization\n struct ggml_tensor * ln_f_g;\n struct ggml_tensor * ln_f_b;\n\n struct ggml_tensor * wte; // token embedding\n struct ggml_tensor * wpe; // position embedding\n struct ggml_tensor * lm_head; // language model head\n\n std::vector layers;\n\n // key + value memory\n struct ggml_tensor * memory_k;\n struct ggml_tensor * memory_v;\n\n //\n struct ggml_context * ctx_w;\n std::map tensors;\n};\n\n// load the model's weights from a file\nbool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) {\n printf(\"%s: loading model from '%s'\\n\", __func__, fname.c_str());\n\n auto fin = std::ifstream(fname, std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"%s: failed to open '%s'\\n\", __func__, fname.c_str());\n return false;\n }\n\n // verify magic\n {\n uint32_t magic;\n fin.read((char *) &magic, sizeof(magic));\n if (magic != GGML_FILE_MAGIC) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, fname.c_str());\n return false;\n }\n }\n\n // load hparams\n {\n auto & hparams = model.hparams;\n\n fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));\n fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));\n\n const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;\n\n printf(\"%s: n_vocab = %d\\n\", __func__, hparams.n_vocab);\n printf(\"%s: n_ctx = %d\\n\", __func__, hparams.n_ctx);\n printf(\"%s: n_embd = %d\\n\", __func__, hparams.n_embd);\n printf(\"%s: n_head = %d\\n\", __func__, hparams.n_head);\n printf(\"%s: n_layer = %d\\n\", __func__, hparams.n_layer);\n printf(\"%s: ftype = %d\\n\", __func__, hparams.ftype);\n printf(\"%s: qntvr = %d\\n\", __func__, qntvr);\n\n hparams.ftype %= GGML_QNT_VERSION_FACTOR;\n }\n\n // load vocab\n {\n int32_t n_vocab = 0;\n fin.read((char *) &n_vocab, sizeof(n_vocab));\n\n if (n_vocab != model.hparams.n_vocab) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad vocab size %d != %d)\\n\",\n __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);\n return false;\n }\n\n std::string word;\n std::vector buf(128);\n\n for (int i = 0; i < n_vocab; i++) {\n uint32_t len;\n fin.read((char *) &len, sizeof(len));\n\n buf.resize(len);\n fin.read((char *) buf.data(), len);\n word.assign(buf.data(), len);\n\n vocab.token_to_id[word] = i;\n vocab.id_to_token[i] = word;\n }\n }\n\n // for the big tensors, we have the option to store the data in 16-bit floats or quantized\n // in order to save memory and also to speed up the computation\n ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));\n if (wtype == GGML_TYPE_COUNT) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad ftype value %d)\\n\",\n __func__, fname.c_str(), model.hparams.ftype);\n return false;\n }\n\n auto & ctx = model.ctx_w;\n\n size_t ctx_size = 0;\n\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_vocab = hparams.n_vocab;\n\n ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g\n ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b\n\n ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte\n ctx_size += ggml_row_size(GGML_TYPE_F32 , n_ctx*n_embd); // wpe\n ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head\n\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b\n\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b\n\n ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k\n ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v\n\n ctx_size += (6 + 12*n_layer)*512; // object overhead\n\n printf(\"%s: ggml tensor size = %d bytes\\n\", __func__, (int) sizeof(ggml_tensor));\n printf(\"%s: ggml ctx size = %6.2f MB\\n\", __func__, ctx_size/(1024.0*1024.0));\n }\n\n // create the ggml context\n {\n struct ggml_init_params params = {\n /*.mem_size =*/ ctx_size,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ false,\n };\n\n model.ctx_w = ggml_init(params);\n if (!model.ctx_w) {\n fprintf(stderr, \"%s: ggml_init() failed\\n\", __func__);\n return false;\n }\n }\n\n // prepare memory for the weights\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_vocab = hparams.n_vocab;\n\n model.layers.resize(n_layer);\n\n model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);\n model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n\n // map by name\n model.tensors[\"model/ln_f/g\"] = model.ln_f_g;\n model.tensors[\"model/ln_f/b\"] = model.ln_f_b;\n\n model.tensors[\"model/wte\"] = model.wte;\n model.tensors[\"model/wpe\"] = model.wpe;\n model.tensors[\"model/lm_head\"] = model.lm_head;\n\n for (int i = 0; i < n_layer; ++i) {\n auto & layer = model.layers[i];\n\n layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);\n layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);\n\n layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);\n layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);\n layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);\n\n layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);\n layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n // map by name\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/g\"] = layer.ln_1_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/b\"] = layer.ln_1_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/g\"] = layer.ln_2_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/b\"] = layer.ln_2_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/w\"] = layer.c_attn_attn_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/b\"] = layer.c_attn_attn_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/w\"] = layer.c_attn_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/b\"] = layer.c_attn_proj_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/w\"] = layer.c_mlp_fc_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/b\"] = layer.c_mlp_fc_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/w\"] = layer.c_mlp_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/b\"] = layer.c_mlp_proj_b;\n }\n }\n\n // key + value memory\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n\n const int n_mem = n_layer*n_ctx;\n const int n_elements = n_embd*n_mem;\n\n model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n\n const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);\n\n printf(\"%s: memory size = %8.2f MB, n_mem = %d\\n\", __func__, memory_size/1024.0/1024.0, n_mem);\n }\n\n // load weights\n {\n size_t total_size = 0;\n\n bool has_lm_head = false;\n\n while (true) {\n int32_t n_dims;\n int32_t length;\n int32_t ttype;\n\n fin.read(reinterpret_cast(&n_dims), sizeof(n_dims));\n fin.read(reinterpret_cast(&length), sizeof(length));\n fin.read(reinterpret_cast(&ttype), sizeof(ttype));\n\n if (fin.eof()) {\n break;\n }\n\n int32_t nelements = 1;\n int32_t ne[2] = { 1, 1 };\n for (int i = 0; i < n_dims; ++i) {\n fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i]));\n nelements *= ne[i];\n }\n\n std::string name(length, 0);\n fin.read(&name[0], length);\n\n if (model.tensors.find(name) == model.tensors.end()) {\n fprintf(stderr, \"%s: unknown tensor '%s' in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n auto tensor = model.tensors[name];\n if (ggml_nelements(tensor) != nelements) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {\n fprintf(stderr, \"%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\\n\",\n __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);\n return false;\n }\n\n // for debugging\n if (0) {\n printf(\"%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\\n\", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));\n }\n\n const size_t bpe = ggml_type_size(ggml_type(ttype));\n\n if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\\n\",\n __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);\n return false;\n }\n\n fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor));\n\n // GPT-2 models share the WTE tensor as the LM head\n if (name == \"model/wte\" && has_lm_head == false) {\n memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor));\n }\n\n if (name == \"model/lm_head\") {\n has_lm_head = true;\n }\n\n total_size += ggml_nbytes(tensor);\n }\n\n printf(\"%s: model size = %8.2f MB\\n\", __func__, total_size/1024.0/1024.0);\n }\n\n fin.close();\n\n return true;\n}\n\n// build the computation graph\nstruct ggml_cgraph * gpt2_graph(\n const gpt2_model & model,\n const int n_past,\n const int n_tokens) {\n const int N = n_tokens;\n\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_head = hparams.n_head;\n\n // since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data\n static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();\n static std::vector buf(buf_size);\n\n struct ggml_init_params params = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ buf.data(),\n /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()\n };\n\n struct ggml_context * ctx = ggml_init(params);\n\n struct ggml_cgraph * gf = ggml_new_graph(ctx);\n\n struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);\n // at this point, the tensor data is not allocated yet and cannot be set\n // we will find the tensor after the graph is allocated by its name, and set the data then\n ggml_set_name(embd, \"embd\");\n // setting a tensor as an input will ensure that it is allocated at the beginning of the graph\n // this is important to ensure that the input tensors are not overwritten before they are used\n ggml_set_input(embd);\n\n struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);\n ggml_set_name(position, \"position\");\n ggml_set_input(position);\n\n // wte + wpe\n struct ggml_tensor * inpL =\n ggml_add(ctx,\n ggml_get_rows(ctx, model.wte, embd),\n ggml_get_rows(ctx, model.wpe, position));\n\n for (int il = 0; il < n_layer; ++il) {\n struct ggml_tensor * cur;\n\n // norm\n {\n // [ 768, N]\n cur = ggml_norm(ctx, inpL, hparams.eps);\n\n // cur = ln_1_g*cur + ln_1_b\n // [ 768, N]\n cur = ggml_add(ctx,\n ggml_mul(ctx,\n ggml_repeat(ctx, model.layers[il].ln_1_g, cur),\n cur),\n ggml_repeat(ctx, model.layers[il].ln_1_b, cur));\n }\n\n // attn\n // [2304, 768] - model.layers[il].c_attn_attn_w\n // [2304, 1] - model.layers[il].c_attn_attn_b\n // [ 768, N] - cur (in)\n // [2304, N] - cur (out)\n //\n // cur = attn_w*cur + attn_b\n // [2304, N]\n {\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_attn_attn_w,\n cur);\n\n cur = ggml_add(ctx,\n ggml_repeat(ctx, model.layers[il].c_attn_attn_b, cur),\n cur);\n }\n\n // self-attention\n {\n struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);\n struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);\n struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);\n\n // store key and value to memory\n if (N >= 1) {\n struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));\n struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));\n\n ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));\n ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));\n }\n\n // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)\n // [64, N, 12]\n struct ggml_tensor * Q =\n ggml_permute(ctx,\n ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N),\n 0, 2, 1, 3);\n\n // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)\n // [64, n_past + N, 12]\n struct ggml_tensor * K =\n ggml_permute(ctx,\n ggml_reshape_3d(ctx,\n ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),\n n_embd/n_head, n_head, n_past + N),\n 0, 2, 1, 3);\n\n // GG: flash attention\n //struct ggml_tensor * V =\n // ggml_cpy(ctx0,\n // ggml_permute(ctx0,\n // ggml_reshape_3d(ctx0,\n // ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),\n // n_embd/n_head, n_head, n_past + N),\n // 1, 2, 0, 3),\n // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));\n\n //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);\n\n // K * Q\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);\n\n // KQ_scaled = KQ / sqrt(n_embd/n_head)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_scaled =\n ggml_scale(ctx,\n KQ,\n 1.0f/sqrtf(float(n_embd)/n_head));\n\n // KQ_masked = mask_past(KQ_scaled)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past);\n\n // KQ = soft_max(KQ_masked)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);\n\n // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()\n // [n_past + N, 64, 12]\n struct ggml_tensor * V_trans =\n ggml_cont_3d(ctx,\n ggml_permute(ctx,\n ggml_reshape_3d(ctx,\n ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),\n n_embd/n_head, n_head, n_past + N),\n 1, 2, 0, 3),\n n_past + N, n_embd/n_head, n_head);\n\n // KQV = transpose(V) * KQ_soft_max\n // [64, N, 12]\n struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);\n\n // KQV_merged = KQV.permute(0, 2, 1, 3)\n // [64, 12, N]\n struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);\n\n // cur = KQV_merged.contiguous().view(n_embd, N)\n // [768, N]\n cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N);\n }\n\n // projection\n // [ 768, 768] - model.layers[il].c_attn_proj_w\n // [ 768, 1] - model.layers[il].c_attn_proj_b\n // [ 768, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n {\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_attn_proj_w,\n cur);\n\n cur = ggml_add(ctx,\n ggml_repeat(ctx, model.layers[il].c_attn_proj_b, cur),\n cur);\n }\n\n // add the input\n cur = ggml_add(ctx, cur, inpL);\n\n struct ggml_tensor * inpFF = cur;\n\n // feed-forward network\n {\n // norm\n {\n cur = ggml_norm(ctx, inpFF, hparams.eps);\n\n // cur = ln_2_g*cur + ln_2_b\n // [ 768, N]\n cur = ggml_add(ctx,\n ggml_mul(ctx,\n ggml_repeat(ctx, model.layers[il].ln_2_g, cur),\n cur),\n ggml_repeat(ctx, model.layers[il].ln_2_b, cur));\n }\n\n // fully connected\n // [3072, 768] - model.layers[il].c_mlp_fc_w\n // [3072, 1] - model.layers[il].c_mlp_fc_b\n // [ 768, N] - cur (in)\n // [3072, N] - cur (out)\n //\n // cur = fc_w*cur + fc_b\n // [3072, N]\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_mlp_fc_w,\n cur);\n\n cur = ggml_add(ctx,\n ggml_repeat(ctx, model.layers[il].c_mlp_fc_b, cur),\n cur);\n\n // GELU activation\n // [3072, N]\n cur = ggml_gelu(ctx, cur);\n\n // projection\n // [ 768, 3072] - model.layers[il].c_mlp_proj_w\n // [ 768, 1] - model.layers[il].c_mlp_proj_b\n // [3072, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_mlp_proj_w,\n cur);\n\n cur = ggml_add(ctx,\n ggml_repeat(ctx, model.layers[il].c_mlp_proj_b, cur),\n cur);\n }\n\n // input for next layer\n inpL = ggml_add(ctx, cur, inpFF);\n }\n\n // norm\n {\n // [ 768, N]\n inpL = ggml_norm(ctx, inpL, hparams.eps);\n\n // inpL = ln_f_g*inpL + ln_f_b\n // [ 768, N]\n inpL = ggml_add(ctx,\n ggml_mul(ctx,\n ggml_repeat(ctx, model.ln_f_g, inpL),\n inpL),\n ggml_repeat(ctx, model.ln_f_b, inpL));\n }\n\n // inpL = WTE * inpL\n // [ 768, 50257] - model.lm_head\n // [ 768, N] - inpL\n inpL = ggml_mul_mat(ctx, model.lm_head, inpL);\n ggml_set_name(inpL, \"logits\");\n // setting a tensor as the output will ensure that it is not overwritten by subsequent operations\n ggml_set_output(inpL);\n\n // logits -> probs\n //inpL = ggml_soft_max(ctx0, inpL);\n\n ggml_build_forward_expand(gf, inpL);\n\n ggml_free(ctx);\n\n return gf;\n}\n\n// evaluate the transformer\n//\n// - model: the model\n// - allocr: ggml_gallocr to use to allocate the compute buffer\n// - n_threads: number of threads to use\n// - n_past: the context size so far\n// - embd_inp: the embeddings of the tokens in the context\n// - embd_w: the predicted logits for the next token\n//\nbool gpt2_eval(\n const gpt2_model & model,\n ggml_gallocr_t allocr,\n const int n_threads,\n const int n_past,\n const std::vector & embd_inp,\n std::vector & embd_w) {\n const int N = embd_inp.size();\n\n const auto & hparams = model.hparams;\n\n const int n_vocab = hparams.n_vocab;\n\n struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size());\n\n // allocate the graph tensors\n ggml_gallocr_alloc_graph(allocr, gf);\n\n // set the graph inputs\n struct ggml_tensor * embd = ggml_graph_get_tensor(gf, \"embd\");\n memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));\n\n struct ggml_tensor * position = ggml_graph_get_tensor(gf, \"position\");\n for (int i = 0; i < N; ++i) {\n ((int32_t *) position->data)[i] = n_past + i;\n }\n\n // run the computation\n struct ggml_cplan plan = ggml_graph_plan(gf, n_threads, nullptr);\n static std::vector work_buffer;\n work_buffer.resize(plan.work_size);\n plan.work_data = work_buffer.data();\n ggml_graph_compute(gf, &plan);\n\n //if (n_past%100 == 0) {\n // ggml_graph_print (&gf);\n // ggml_graph_dump_dot(&gf, NULL, \"gpt-2.dot\");\n //}\n\n // get the graph outputs\n struct ggml_tensor * logits = ggml_graph_get_tensor(gf, \"logits\");\n\n //embd_w.resize(n_vocab*N);\n //memcpy(embd_w.data(), ggml_get_data(logits), sizeof(float)*n_vocab*N);\n\n // return result just for the last token\n embd_w.resize(n_vocab);\n memcpy(embd_w.data(), (float *) ggml_get_data(logits) + (n_vocab*(N-1)), sizeof(float)*n_vocab);\n\n return true;\n}\n\nint main(int argc, char ** argv) {\n ggml_time_init();\n\n const int64_t t_main_start_us = ggml_time_us();\n\n gpt_params params;\n params.model = \"models/gpt-2-117M/ggml-model.bin\";\n\n if (gpt_params_parse(argc, argv, params) == false) {\n return 1;\n }\n\n if (params.seed < 0) {\n params.seed = time(NULL);\n }\n\n printf(\"%s: seed = %d\\n\", __func__, params.seed);\n\n std::mt19937 rng(params.seed);\n if (params.prompt.empty()) {\n params.prompt = gpt_random_prompt(rng);\n }\n\n int64_t t_load_us = 0;\n\n gpt_vocab vocab;\n gpt2_model model;\n\n // load the model\n {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_model_load(params.model, model, vocab)) {\n fprintf(stderr, \"%s: failed to load model from '%s'\\n\", __func__, params.model.c_str());\n return 1;\n }\n\n t_load_us = ggml_time_us() - t_start_us;\n\n test_gpt_tokenizer(vocab, params.token_test);\n }\n\n ggml_gallocr_t allocr = NULL;\n // allocate the compute buffer\n {\n allocr = ggml_gallocr_new(ggml_backend_cpu_buffer_type());\n\n // create the worst case graph for memory usage estimation\n int n_tokens = std::min(model.hparams.n_ctx, params.n_batch);\n int n_past = model.hparams.n_ctx - n_tokens;\n struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens);\n\n // pre-allocate the compute buffer for the worst case (optional)\n ggml_gallocr_reserve(allocr, gf);\n size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);\n fprintf(stderr, \"%s: compute buffer size: %.2f MB\\n\", __func__, mem_size/1024.0/1024.0);\n }\n\n int n_past = 0;\n\n int64_t t_sample_us = 0;\n int64_t t_predict_us = 0;\n\n std::vector logits;\n\n // tokenize the prompt\n std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt);\n\n params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());\n\n printf(\"%s: prompt: '%s'\\n\", __func__, params.prompt.c_str());\n printf(\"%s: number of tokens in prompt = %zu, first 8 tokens: \", __func__, embd_inp.size());\n for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {\n printf(\"%d \", embd_inp[i]);\n }\n printf(\"\\n\\n\");\n\n // submit the input prompt token-by-token\n // this reduces the memory usage during inference, at the cost of a bit of speed at the beginning\n std::vector embd;\n\n for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {\n // predict\n if (embd.size() > 0) {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) {\n printf(\"Failed to predict\\n\");\n return 1;\n }\n\n t_predict_us += ggml_time_us() - t_start_us;\n }\n\n n_past += embd.size();\n embd.clear();\n\n if (i >= embd_inp.size()) {\n // sample next token\n const int top_k = params.top_k;\n const float top_p = params.top_p;\n const float temp = params.temp;\n\n const int n_vocab = model.hparams.n_vocab;\n\n gpt_vocab::id id = 0;\n\n {\n const int64_t t_start_sample_us = ggml_time_us();\n\n id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);\n\n t_sample_us += ggml_time_us() - t_start_sample_us;\n }\n\n // add it to the context\n embd.push_back(id);\n } else {\n // if here, it means we are still processing the input prompt\n for (size_t k = i; k < embd_inp.size(); k++) {\n embd.push_back(embd_inp[k]);\n if (int32_t(embd.size()) >= params.n_batch) {\n break;\n }\n }\n i += embd.size() - 1;\n }\n\n // display text\n for (auto id : embd) {\n printf(\"%s\", vocab.id_to_token[id].c_str());\n }\n fflush(stdout);\n\n // end of text token\n if (embd.back() == 50256) {\n break;\n }\n }\n\n // report timing\n {\n const int64_t t_main_end_us = ggml_time_us();\n\n printf(\"\\n\\n\");\n printf(\"%s: load time = %8.2f ms\\n\", __func__, t_load_us/1000.0f);\n printf(\"%s: sample time = %8.2f ms\\n\", __func__, t_sample_us/1000.0f);\n printf(\"%s: predict time = %8.2f ms / %.2f ms per token\\n\", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);\n printf(\"%s: total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n }\n\n ggml_free(model.ctx_w);\n\n return 0;\n}\n" }, { "path": "examples/gpt-2/main-backend.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n\n#ifdef GGML_USE_CUDA\n#include \"ggml-cuda.h\"\n#endif\n\n#ifdef GGML_USE_METAL\n#include \"ggml-metal.h\"\n#endif\n\n#include \"common.h\"\n#include \"common-ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\n#define GPT2_MAX_NODES 4096\n\nstatic void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {\n (void) level;\n (void) user_data;\n fputs(text, stderr);\n fflush(stderr);\n}\n\n// default hparams (GPT-2 117M)\nstruct gpt2_hparams {\n int32_t n_vocab = 50257;\n int32_t n_ctx = 1024;\n int32_t n_embd = 768;\n int32_t n_head = 12;\n int32_t n_layer = 12;\n int32_t ftype = 1;\n float eps = 1e-5f;\n};\n\nstruct gpt2_layer {\n // normalization\n struct ggml_tensor * ln_1_g;\n struct ggml_tensor * ln_1_b;\n\n struct ggml_tensor * ln_2_g;\n struct ggml_tensor * ln_2_b;\n\n // attention\n struct ggml_tensor * c_attn_attn_w;\n struct ggml_tensor * c_attn_attn_b;\n\n struct ggml_tensor * c_attn_proj_w;\n struct ggml_tensor * c_attn_proj_b;\n\n // mlp\n struct ggml_tensor * c_mlp_fc_w;\n struct ggml_tensor * c_mlp_fc_b;\n\n struct ggml_tensor * c_mlp_proj_w;\n struct ggml_tensor * c_mlp_proj_b;\n};\n\nstruct gpt2_model {\n gpt2_hparams hparams;\n\n // normalization\n struct ggml_tensor * ln_f_g;\n struct ggml_tensor * ln_f_b;\n\n struct ggml_tensor * wte; // token embedding\n struct ggml_tensor * wpe; // position embedding\n struct ggml_tensor * lm_head; // language model head\n\n std::vector layers;\n\n // key + value memory\n struct ggml_tensor * memory_k;\n struct ggml_tensor * memory_v;\n\n //\n struct ggml_context * ctx_w;\n struct ggml_context * ctx_kv;\n\n ggml_backend_t backend = NULL;\n\n ggml_backend_buffer_t buffer_w;\n ggml_backend_buffer_t buffer_kv;\n\n std::map tensors;\n};\n\n// load the model's weights from a file\nbool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab, int n_ctx, int n_gpu_layers) {\n printf(\"%s: loading model from '%s'\\n\", __func__, fname.c_str());\n\n auto fin = std::ifstream(fname, std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"%s: failed to open '%s'\\n\", __func__, fname.c_str());\n return false;\n }\n\n // verify magic\n {\n uint32_t magic;\n fin.read((char *) &magic, sizeof(magic));\n if (magic != GGML_FILE_MAGIC) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, fname.c_str());\n return false;\n }\n }\n\n // load hparams\n {\n auto & hparams = model.hparams;\n\n fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));\n fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));\n\n const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;\n\n printf(\"%s: n_vocab = %d\\n\", __func__, hparams.n_vocab);\n printf(\"%s: n_ctx = %d\\n\", __func__, hparams.n_ctx);\n printf(\"%s: n_embd = %d\\n\", __func__, hparams.n_embd);\n printf(\"%s: n_head = %d\\n\", __func__, hparams.n_head);\n printf(\"%s: n_layer = %d\\n\", __func__, hparams.n_layer);\n printf(\"%s: ftype = %d\\n\", __func__, hparams.ftype);\n printf(\"%s: qntvr = %d\\n\", __func__, qntvr);\n\n hparams.ftype %= GGML_QNT_VERSION_FACTOR;\n }\n\n // load vocab\n {\n int32_t n_vocab = 0;\n fin.read((char *) &n_vocab, sizeof(n_vocab));\n\n if (n_vocab != model.hparams.n_vocab) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad vocab size %d != %d)\\n\",\n __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);\n return false;\n }\n\n std::string word;\n std::vector buf(128);\n\n for (int i = 0; i < n_vocab; i++) {\n uint32_t len;\n fin.read((char *) &len, sizeof(len));\n\n buf.resize(len);\n fin.read((char *) buf.data(), len);\n word.assign(buf.data(), len);\n\n vocab.token_to_id[word] = i;\n vocab.id_to_token[i] = word;\n }\n }\n\n // for the big tensors, we have the option to store the data in 16-bit floats or quantized\n // in order to save memory and also to speed up the computation\n ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));\n if (wtype == GGML_TYPE_COUNT) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad ftype value %d)\\n\",\n __func__, fname.c_str(), model.hparams.ftype);\n return false;\n }\n\n ggml_log_set(ggml_log_callback_default, nullptr);\n\n auto & ctx = model.ctx_w;\n\n // create the ggml context\n {\n size_t n_tensors = 2 + 6 + 12*model.hparams.n_layer;\n struct ggml_init_params params = {\n /*.mem_size =*/ ggml_tensor_overhead() * n_tensors,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n\n ctx = ggml_init(params);\n if (!ctx) {\n fprintf(stderr, \"%s: ggml_init() failed\\n\", __func__);\n return false;\n }\n }\n\n // initialize the backend\n#ifdef GGML_USE_CUDA\n if (n_gpu_layers > 0) {\n fprintf(stderr, \"%s: using CUDA backend\\n\", __func__);\n model.backend = ggml_backend_cuda_init(0);\n if (!model.backend) {\n fprintf(stderr, \"%s: ggml_backend_cuda_init() failed\\n\", __func__);\n }\n }\n#endif\n\n#ifdef GGML_USE_METAL\n if (n_gpu_layers > 0) {\n fprintf(stderr, \"%s: using Metal backend\\n\", __func__);\n model.backend = ggml_backend_metal_init();\n if (!model.backend) {\n fprintf(stderr, \"%s: ggml_backend_metal_init() failed\\n\", __func__);\n }\n }\n#endif\n\n if (!model.backend) {\n // fallback to CPU backend\n fprintf(stderr, \"%s: using CPU backend\\n\", __func__);\n model.backend = ggml_backend_cpu_init();\n }\n\n if (!model.backend) {\n fprintf(stderr, \"%s: ggml_backend_cpu_init() failed\\n\", __func__);\n return false;\n }\n\n // create the tensors for the model\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_vocab = hparams.n_vocab;\n\n model.layers.resize(n_layer);\n\n model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);\n model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n\n // map by name\n model.tensors[\"model/ln_f/g\"] = model.ln_f_g;\n model.tensors[\"model/ln_f/b\"] = model.ln_f_b;\n\n model.tensors[\"model/wte\"] = model.wte;\n model.tensors[\"model/wpe\"] = model.wpe;\n model.tensors[\"model/lm_head\"] = model.lm_head;\n\n for (int i = 0; i < n_layer; ++i) {\n auto & layer = model.layers[i];\n\n layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);\n layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);\n\n layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);\n layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);\n layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);\n\n layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);\n layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n // map by name\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/g\"] = layer.ln_1_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/b\"] = layer.ln_1_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/g\"] = layer.ln_2_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/b\"] = layer.ln_2_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/w\"] = layer.c_attn_attn_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/b\"] = layer.c_attn_attn_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/w\"] = layer.c_attn_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/b\"] = layer.c_attn_proj_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/w\"] = layer.c_mlp_fc_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/b\"] = layer.c_mlp_fc_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/w\"] = layer.c_mlp_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/b\"] = layer.c_mlp_proj_b;\n }\n }\n\n // allocate the model tensors in a backend buffer\n model.buffer_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend);\n\n printf(\"%s: ggml tensor size = %d bytes\\n\", __func__, (int) sizeof(ggml_tensor));\n printf(\"%s: backend buffer size = %6.2f MB\\n\", __func__, ggml_backend_buffer_get_size(model.buffer_w)/(1024.0*1024.0));\n\n // override the default training context with the user-provided\n model.hparams.n_ctx = n_ctx;\n\n // key + value memory\n {\n auto * ctx = model.ctx_kv;\n\n // create the ggml context\n {\n size_t n_tensors = 2;\n struct ggml_init_params params = {\n /*.mem_size =*/ ggml_tensor_overhead() * n_tensors,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n\n ctx = ggml_init(params);\n if (!ctx) {\n fprintf(stderr, \"%s: ggml_init() failed\\n\", __func__);\n return false;\n }\n }\n\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n\n const int n_mem = n_layer*n_ctx;\n const int n_elements = n_embd*n_mem;\n\n // k and v here can also be GGML_TYPE_F16 to save memory and speed up the computation\n // if backend supports it\n model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n\n // allocate the KV memory in a backend buffer\n model.buffer_kv = ggml_backend_alloc_ctx_tensors(ctx, model.backend);\n\n const size_t memory_size = ggml_backend_buffer_get_size(model.buffer_kv);\n printf(\"%s: memory size = %8.2f MB, n_mem = %d\\n\", __func__, memory_size/1024.0/1024.0, n_mem);\n }\n\n // load weights\n {\n size_t total_size = 0;\n\n bool has_lm_head = false;\n\n std::vector read_buf;\n\n while (true) {\n int32_t n_dims;\n int32_t length;\n int32_t ttype;\n\n fin.read(reinterpret_cast(&n_dims), sizeof(n_dims));\n fin.read(reinterpret_cast(&length), sizeof(length));\n fin.read(reinterpret_cast(&ttype), sizeof(ttype));\n\n if (fin.eof()) {\n break;\n }\n\n int32_t nelements = 1;\n int32_t ne[2] = { 1, 1 };\n for (int i = 0; i < n_dims; ++i) {\n fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i]));\n nelements *= ne[i];\n }\n\n std::string name(length, 0);\n fin.read(&name[0], length);\n\n if (model.tensors.find(name) == model.tensors.end()) {\n fprintf(stderr, \"%s: unknown tensor '%s' in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n auto tensor = model.tensors[name];\n ggml_set_name(tensor, name.c_str());\n if (ggml_nelements(tensor) != nelements) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {\n fprintf(stderr, \"%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\\n\",\n __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);\n return false;\n }\n\n // for debugging\n if (0) {\n printf(\"%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\\n\", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));\n }\n\n const size_t bpe = ggml_type_size(ggml_type(ttype));\n\n if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\\n\",\n __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);\n return false;\n }\n\n if (ggml_backend_buffer_is_host(model.buffer_w)) {\n // for some backends such as CPU and Metal, the tensor data is in system memory and we can read directly into it\n fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor));\n } else {\n // read into a temporary buffer first, then copy to device memory\n read_buf.resize(ggml_nbytes(tensor));\n fin.read(read_buf.data(), ggml_nbytes(tensor));\n ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor));\n }\n\n // GPT-2 models share the WTE tensor as the LM head\n if (name == \"model/wte\" && has_lm_head == false) {\n //ggml_backend_tensor_copy(tensor, model.lm_head);\n model.lm_head = tensor;\n }\n\n if (name == \"model/lm_head\") {\n has_lm_head = true;\n }\n\n total_size += ggml_nbytes(tensor);\n }\n\n printf(\"%s: model size = %8.2f MB\\n\", __func__, total_size/1024.0/1024.0);\n }\n\n fin.close();\n\n return true;\n}\n\n// build the computation graph\nstruct ggml_cgraph * gpt2_graph(\n const gpt2_model & model,\n const int n_past,\n const int n_tokens) {\n const int N = n_tokens;\n\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_head = hparams.n_head;\n\n // since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data\n static size_t buf_size = ggml_tensor_overhead()*GPT2_MAX_NODES + ggml_graph_overhead_custom(GPT2_MAX_NODES, false);\n static std::vector buf(buf_size);\n\n struct ggml_init_params params = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ buf.data(),\n /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()\n };\n\n struct ggml_context * ctx = ggml_init(params);\n\n struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, GPT2_MAX_NODES, false);\n\n struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);\n // at this point, the tensor data is not allocated yet and cannot be set\n // we will find the tensor after the graph is allocated by its name, and set the data then\n ggml_set_name(embd, \"embd\");\n // setting a tensor as an input will ensure that it is allocated at the beginning of the graph\n // this is important to ensure that the input tensors are not overwritten before they are used\n ggml_set_input(embd);\n\n struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);\n ggml_set_name(position, \"position\");\n ggml_set_input(position);\n\n // wte + wpe\n struct ggml_tensor * inpL =\n ggml_add(ctx,\n ggml_get_rows(ctx, model.wte, embd),\n ggml_get_rows(ctx, model.wpe, position));\n\n for (int il = 0; il < n_layer; ++il) {\n struct ggml_tensor * cur;\n\n // norm\n {\n // [ 768, N]\n cur = ggml_norm(ctx, inpL, hparams.eps);\n\n // cur = ln_1_g*cur + ln_1_b\n // [ 768, N]\n cur = ggml_add(ctx,\n ggml_mul(ctx,\n cur,\n model.layers[il].ln_1_g),\n model.layers[il].ln_1_b);\n }\n\n // attn\n // [2304, 768] - model.layers[il].c_attn_attn_w\n // [2304, 1] - model.layers[il].c_attn_attn_b\n // [ 768, N] - cur (in)\n // [2304, N] - cur (out)\n //\n // cur = attn_w*cur + attn_b\n // [2304, N]\n {\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_attn_attn_w,\n cur);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_attn_attn_b);\n }\n\n // self-attention\n {\n struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);\n struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);\n struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);\n\n // store key and value to memory\n if (N >= 1) {\n struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));\n struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));\n\n ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));\n ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));\n }\n\n // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)\n // [64, N, 12]\n struct ggml_tensor * Q =\n ggml_permute(ctx,\n ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N),\n 0, 2, 1, 3);\n\n // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)\n // [64, n_past + N, 12]\n struct ggml_tensor * K =\n ggml_permute(ctx,\n ggml_reshape_3d(ctx,\n ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),\n n_embd/n_head, n_head, n_past + N),\n 0, 2, 1, 3);\n\n // GG: flash attention\n //struct ggml_tensor * V =\n // ggml_cpy(ctx0,\n // ggml_permute(ctx0,\n // ggml_reshape_3d(ctx0,\n // ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),\n // n_embd/n_head, n_head, n_past + N),\n // 1, 2, 0, 3),\n // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));\n\n //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);\n\n // K * Q\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);\n\n // KQ_scaled = KQ / sqrt(n_embd/n_head)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_scaled =\n ggml_scale(ctx,\n KQ,\n 1.0f/sqrtf(float(n_embd)/n_head));\n\n // KQ_masked = mask_past(KQ_scaled)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past);\n\n // KQ = soft_max(KQ_masked)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);\n\n // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()\n // [n_past + N, 64, 12]\n struct ggml_tensor * V_trans =\n ggml_cont_3d(ctx,\n ggml_permute(ctx,\n ggml_reshape_3d(ctx,\n ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),\n n_embd/n_head, n_head, n_past + N),\n 1, 2, 0, 3),\n n_past + N, n_embd/n_head, n_head);\n\n // KQV = transpose(V) * KQ_soft_max\n // [64, N, 12]\n struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);\n\n // KQV_merged = KQV.permute(0, 2, 1, 3)\n // [64, 12, N]\n struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);\n\n // cur = KQV_merged.contiguous().view(n_embd, N)\n // [768, N]\n cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N);\n }\n\n // projection\n // [ 768, 768] - model.layers[il].c_attn_proj_w\n // [ 768, 1] - model.layers[il].c_attn_proj_b\n // [ 768, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n {\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_attn_proj_w,\n cur);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_attn_proj_b);\n }\n\n // add the input\n cur = ggml_add(ctx, cur, inpL);\n\n struct ggml_tensor * inpFF = cur;\n\n // feed-forward network\n {\n // norm\n {\n cur = ggml_norm(ctx, inpFF, hparams.eps);\n\n // cur = ln_2_g*cur + ln_2_b\n // [ 768, N]\n cur = ggml_add(ctx,\n ggml_mul(ctx,\n cur,\n model.layers[il].ln_2_g),\n model.layers[il].ln_2_b);\n }\n\n // fully connected\n // [3072, 768] - model.layers[il].c_mlp_fc_w\n // [3072, 1] - model.layers[il].c_mlp_fc_b\n // [ 768, N] - cur (in)\n // [3072, N] - cur (out)\n //\n // cur = fc_w*cur + fc_b\n // [3072, N]\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_mlp_fc_w,\n cur);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_mlp_fc_b);\n\n // GELU activation\n // [3072, N]\n cur = ggml_gelu(ctx, cur);\n\n // projection\n // [ 768, 3072] - model.layers[il].c_mlp_proj_w\n // [ 768, 1] - model.layers[il].c_mlp_proj_b\n // [3072, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_mlp_proj_w,\n cur);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_mlp_proj_b);\n }\n\n // input for next layer\n inpL = ggml_add(ctx, cur, inpFF);\n }\n\n // norm\n {\n // [ 768, N]\n inpL = ggml_norm(ctx, inpL, hparams.eps);\n\n // inpL = ln_f_g*inpL + ln_f_b\n // [ 768, N]\n inpL = ggml_add(ctx,\n ggml_mul(ctx,\n inpL,\n model.ln_f_g),\n model.ln_f_b);\n }\n\n // inpL = WTE * inpL\n // [ 768, 50257] - model.lm_head\n // [ 768, N] - inpL\n inpL = ggml_mul_mat(ctx, model.lm_head, inpL);\n ggml_set_name(inpL, \"logits\");\n // setting a tensor as the output will ensure that it is not overwritten by subsequent operations\n ggml_set_output(inpL);\n\n // logits -> probs\n //inpL = ggml_soft_max(ctx0, inpL);\n\n ggml_build_forward_expand(gf, inpL);\n\n ggml_free(ctx);\n\n return gf;\n}\n\n// evaluate the transformer\n//\n// - model: the model\n// - allocr: ggml_gallocr to use to allocate the compute buffer\n// - n_threads: number of threads to use\n// - n_past: the context size so far\n// - embd_inp: the embeddings of the tokens in the context\n// - embd_w: the predicted logits for the next token\n//\nbool gpt2_eval(\n const gpt2_model & model,\n ggml_gallocr_t allocr,\n const int n_threads,\n const int n_past,\n const std::vector & embd_inp,\n std::vector & embd_w) {\n const int N = embd_inp.size();\n\n const auto & hparams = model.hparams;\n\n const int n_vocab = hparams.n_vocab;\n\n struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size());\n\n // allocate the graph tensors\n ggml_gallocr_alloc_graph(allocr, gf);\n\n // set the graph inputs\n struct ggml_tensor * embd = ggml_graph_get_tensor(gf, \"embd\");\n ggml_backend_tensor_set(embd, embd_inp.data(), 0, N*ggml_element_size(embd));\n\n struct ggml_tensor * position = ggml_graph_get_tensor(gf, \"position\");\n for (int i = 0; i < N; ++i) {\n int32_t v = n_past + i;\n ggml_backend_tensor_set(position, &v, i*sizeof(int32_t), sizeof(v));\n }\n\n // set backend options\n if (ggml_backend_is_cpu(model.backend)) {\n ggml_backend_cpu_set_n_threads(model.backend, n_threads);\n }\n\n // run the computation\n ggml_backend_graph_compute(model.backend, gf);\n\n //if (n_past%100 == 0) {\n // ggml_graph_print (&gf);\n // ggml_graph_dump_dot(&gf, NULL, \"gpt-2.dot\");\n //}\n\n // get the graph outputs\n struct ggml_tensor * logits = ggml_graph_get_tensor(gf, \"logits\");\n\n //embd_w.resize(n_vocab*N);\n //ggml_backend_tensor_get(logits, embd_w.data(), 0, sizeof(float)*n_vocab*N);\n\n // return result just for the last token\n embd_w.resize(n_vocab);\n ggml_backend_tensor_get(logits, embd_w.data(), (n_vocab*(N-1))*sizeof(float), sizeof(float)*n_vocab);\n\n return true;\n}\n\nint main(int argc, char ** argv) {\n ggml_time_init();\n\n const int64_t t_main_start_us = ggml_time_us();\n\n gpt_params params;\n params.model = \"models/gpt-2-117M/ggml-model.bin\";\n\n if (gpt_params_parse(argc, argv, params) == false) {\n return 1;\n }\n\n if (params.seed < 0) {\n params.seed = time(NULL);\n }\n\n printf(\"%s: seed = %d\\n\", __func__, params.seed);\n\n std::mt19937 rng(params.seed);\n if (params.prompt.empty()) {\n params.prompt = gpt_random_prompt(rng);\n }\n\n int64_t t_load_us = 0;\n\n gpt_vocab vocab;\n gpt2_model model;\n\n // load the model\n {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_model_load(params.model, model, vocab, params.n_ctx, params.n_gpu_layers)) {\n fprintf(stderr, \"%s: failed to load model from '%s'\\n\", __func__, params.model.c_str());\n return 1;\n }\n\n t_load_us = ggml_time_us() - t_start_us;\n\n test_gpt_tokenizer(vocab, params.token_test);\n }\n\n ggml_gallocr_t allocr = NULL;\n // allocate the compute buffer\n {\n // create a graph allocator with the backend's default buffer type\n allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));\n\n // create the worst case graph for memory usage estimation\n int n_tokens = std::min(model.hparams.n_ctx, params.n_batch);\n int n_past = model.hparams.n_ctx - n_tokens;\n struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens);\n\n // pre-allocate the compute buffer for the worst case (optional)\n ggml_gallocr_reserve(allocr, gf);\n size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);\n fprintf(stderr, \"%s: compute buffer size: %.2f MB\\n\", __func__, mem_size/1024.0/1024.0);\n }\n\n int n_past = 0;\n\n int64_t t_sample_us = 0;\n int64_t t_predict_us = 0;\n\n std::vector logits;\n\n // tokenize the prompt\n std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt);\n\n params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());\n\n printf(\"%s: prompt: '%s'\\n\", __func__, params.prompt.c_str());\n printf(\"%s: number of tokens in prompt = %zu, first 8 tokens: \", __func__, embd_inp.size());\n for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {\n printf(\"%d \", embd_inp[i]);\n }\n printf(\"\\n\\n\");\n\n // submit the input prompt token-by-token\n // this reduces the memory usage during inference, at the cost of a bit of speed at the beginning\n std::vector embd;\n\n for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {\n // predict\n if (embd.size() > 0) {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) {\n printf(\"Failed to predict\\n\");\n return 1;\n }\n\n t_predict_us += ggml_time_us() - t_start_us;\n }\n\n n_past += embd.size();\n embd.clear();\n\n if (i >= embd_inp.size()) {\n // sample next token\n const int top_k = params.top_k;\n const float top_p = params.top_p;\n const float temp = params.temp;\n\n const int n_vocab = model.hparams.n_vocab;\n\n gpt_vocab::id id = 0;\n\n {\n const int64_t t_start_sample_us = ggml_time_us();\n\n id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);\n\n t_sample_us += ggml_time_us() - t_start_sample_us;\n }\n\n // add it to the context\n embd.push_back(id);\n } else {\n // if here, it means we are still processing the input prompt\n for (size_t k = i; k < embd_inp.size(); k++) {\n embd.push_back(embd_inp[k]);\n if (int32_t(embd.size()) >= params.n_batch) {\n break;\n }\n }\n i += embd.size() - 1;\n }\n\n // display text\n for (auto id : embd) {\n printf(\"%s\", vocab.id_to_token[id].c_str());\n }\n fflush(stdout);\n\n // end of text token\n if (!params.ignore_eos && embd.back() == 50256) {\n break;\n }\n }\n\n // report timing\n {\n const int64_t t_main_end_us = ggml_time_us();\n\n printf(\"\\n\\n\");\n printf(\"%s: load time = %8.2f ms\\n\", __func__, t_load_us/1000.0f);\n printf(\"%s: sample time = %8.2f ms\\n\", __func__, t_sample_us/1000.0f);\n printf(\"%s: predict time = %8.2f ms / %.2f ms per token\\n\", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);\n printf(\"%s: total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n }\n\n ggml_free(model.ctx_w);\n\n ggml_gallocr_free(allocr);\n ggml_backend_buffer_free(model.buffer_w);\n ggml_backend_buffer_free(model.buffer_kv);\n ggml_backend_free(model.backend);\n\n return 0;\n}\n" }, { "path": "examples/gpt-2/main-batched.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n\n#ifdef GGML_USE_CUDA\n#include \"ggml-cuda.h\"\n#endif\n\n#ifdef GGML_USE_METAL\n#include \"ggml-metal.h\"\n#endif\n\n#include \"common.h\"\n#include \"common-ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\n#define GPT2_MAX_NODES 4096\n\nstatic void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {\n (void) level;\n (void) user_data;\n fputs(text, stderr);\n fflush(stderr);\n}\n\ntypedef int32_t gpt2_pos;\ntypedef int32_t gpt2_seq_id;\n\n// default hparams (GPT-2 117M)\nstruct gpt2_hparams {\n int32_t n_vocab = 50257;\n int32_t n_ctx = 1024;\n int32_t n_embd = 768;\n int32_t n_head = 12;\n int32_t n_layer = 12;\n int32_t ftype = 1;\n float eps = 1e-5f;\n};\n\nstruct gpt2_layer {\n // normalization\n struct ggml_tensor * ln_1_g;\n struct ggml_tensor * ln_1_b;\n\n struct ggml_tensor * ln_2_g;\n struct ggml_tensor * ln_2_b;\n\n // attention\n struct ggml_tensor * c_attn_attn_w;\n struct ggml_tensor * c_attn_attn_b;\n\n struct ggml_tensor * c_attn_proj_w;\n struct ggml_tensor * c_attn_proj_b;\n\n // mlp\n struct ggml_tensor * c_mlp_fc_w;\n struct ggml_tensor * c_mlp_fc_b;\n\n struct ggml_tensor * c_mlp_proj_w;\n struct ggml_tensor * c_mlp_proj_b;\n};\n\nstruct gpt2_kv_cell {\n gpt2_pos pos = -1;\n gpt2_pos delta = 0;\n\n std::set seq_id;\n\n bool has_seq_id(const gpt2_seq_id & id) const {\n return seq_id.find(id) != seq_id.end();\n }\n};\n\nstruct gpt2_kv_cache {\n // key + value memory\n struct ggml_tensor * k;\n struct ggml_tensor * v;\n //\n\n uint32_t head = 0;\n uint32_t size = 0;\n\n // computed before each graph build\n uint32_t n = 0;\n\n std::vector cells;\n\n ggml_backend_buffer_t buffer;\n};\n\nstruct gpt2_model {\n gpt2_hparams hparams;\n\n // normalization\n struct ggml_tensor * ln_f_g;\n struct ggml_tensor * ln_f_b;\n\n struct ggml_tensor * wte; // token embedding\n struct ggml_tensor * wpe; // position embedding\n struct ggml_tensor * lm_head; // language model head\n\n std::vector layers;\n\n gpt2_kv_cache kv_cache;\n\n struct ggml_context * ctx_w;\n\n ggml_backend_t backend = NULL;\n\n ggml_backend_buffer_t buffer_w;\n\n std::map tensors;\n};\n\n// Input data for gpt2_decode\n// A gpt2_batch object can contain input about one or many sequences\n// The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens\n//\n// - token : the token ids of the input (used when embd is NULL)\n// - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)\n// - pos : the positions of the respective token in the sequence\n// - seq_id : the sequence to which the respective token belongs\n// - logits : if zero, the logits for the respective token will not be output\n//\nstruct gpt2_batch {\n int32_t n_tokens = -1;\n\n gpt_vocab::id * token = {};\n float * embd = {};\n gpt2_pos * pos = {};\n gpt2_seq_id * seq_id = {};\n int8_t * logits = {};\n};\n\n// load the model's weights from a file\nbool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab, int n_ctx, int n_gpu_layers) {\n printf(\"%s: loading model from '%s'\\n\", __func__, fname.c_str());\n\n auto fin = std::ifstream(fname, std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"%s: failed to open '%s'\\n\", __func__, fname.c_str());\n return false;\n }\n\n // verify magic\n {\n uint32_t magic;\n fin.read((char *) &magic, sizeof(magic));\n if (magic != GGML_FILE_MAGIC) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, fname.c_str());\n return false;\n }\n }\n\n // load hparams\n {\n auto & hparams = model.hparams;\n\n fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));\n fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));\n\n const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;\n\n printf(\"%s: n_vocab = %d\\n\", __func__, hparams.n_vocab);\n printf(\"%s: n_ctx = %d\\n\", __func__, hparams.n_ctx);\n printf(\"%s: n_embd = %d\\n\", __func__, hparams.n_embd);\n printf(\"%s: n_head = %d\\n\", __func__, hparams.n_head);\n printf(\"%s: n_layer = %d\\n\", __func__, hparams.n_layer);\n printf(\"%s: ftype = %d\\n\", __func__, hparams.ftype);\n printf(\"%s: qntvr = %d\\n\", __func__, qntvr);\n\n hparams.ftype %= GGML_QNT_VERSION_FACTOR;\n }\n\n // load vocab\n {\n int32_t n_vocab = 0;\n fin.read((char *) &n_vocab, sizeof(n_vocab));\n\n if (n_vocab != model.hparams.n_vocab) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad vocab size %d != %d)\\n\",\n __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);\n return false;\n }\n\n std::string word;\n std::vector buf(128);\n\n for (int i = 0; i < n_vocab; i++) {\n uint32_t len;\n fin.read((char *) &len, sizeof(len));\n\n buf.resize(len);\n fin.read((char *) buf.data(), len);\n word.assign(buf.data(), len);\n\n vocab.token_to_id[word] = i;\n vocab.id_to_token[i] = word;\n }\n }\n\n // for the big tensors, we have the option to store the data in 16-bit floats or quantized\n // in order to save memory and also to speed up the computation\n ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));\n if (wtype == GGML_TYPE_COUNT) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad ftype value %d)\\n\",\n __func__, fname.c_str(), model.hparams.ftype);\n return false;\n }\n\n auto & ctx = model.ctx_w;\n\n size_t buffer_size = 0;\n\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_vocab = hparams.n_vocab;\n\n buffer_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g\n buffer_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b\n\n buffer_size += ggml_row_size(wtype, n_vocab*n_embd); // wte\n buffer_size += ggml_row_size(GGML_TYPE_F32, n_ctx*n_embd); // wpe\n buffer_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head\n\n buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g\n buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b\n\n buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g\n buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b\n\n buffer_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w\n buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b\n\n buffer_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w\n buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b\n\n buffer_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w\n buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b\n\n buffer_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w\n buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b\n\n buffer_size += (6 + 12*n_layer)*128; // alignment overhead\n\n printf(\"%s: ggml tensor size = %d bytes\\n\", __func__, (int) sizeof(ggml_tensor));\n printf(\"%s: backend buffer size = %6.2f MB\\n\", __func__, buffer_size/(1024.0*1024.0));\n }\n\n ggml_log_set(ggml_log_callback_default, nullptr);\n\n // create the ggml context\n {\n size_t n_tensors = 2 + 6 + 12*model.hparams.n_layer;\n struct ggml_init_params params = {\n /*.mem_size =*/ ggml_tensor_overhead() * n_tensors,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n\n model.ctx_w = ggml_init(params);\n if (!model.ctx_w) {\n fprintf(stderr, \"%s: ggml_init() failed\\n\", __func__);\n return false;\n }\n }\n\n // initialize the backend\n#ifdef GGML_USE_CUDA\n if (n_gpu_layers > 0) {\n fprintf(stderr, \"%s: using CUDA backend\\n\", __func__);\n model.backend = ggml_backend_cuda_init(0);\n if (!model.backend) {\n fprintf(stderr, \"%s: ggml_backend_cuda_init() failed\\n\", __func__);\n }\n }\n#endif\n\n#ifdef GGML_USE_METAL\n if (n_gpu_layers > 0) {\n fprintf(stderr, \"%s: using Metal backend\\n\", __func__);\n model.backend = ggml_backend_metal_init();\n if (!model.backend) {\n fprintf(stderr, \"%s: ggml_backend_metal_init() failed\\n\", __func__);\n }\n }\n#endif\n\n if (!model.backend) {\n // fallback to CPU backend\n fprintf(stderr, \"%s: using CPU backend\\n\", __func__);\n model.backend = ggml_backend_cpu_init();\n }\n\n if (!model.backend) {\n fprintf(stderr, \"%s: ggml_backend_cpu_init() failed\\n\", __func__);\n return false;\n }\n\n // allocate weights buffer\n model.buffer_w = ggml_backend_alloc_buffer(model.backend, buffer_size);\n\n // prepare memory for the weights\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_vocab = hparams.n_vocab;\n\n model.layers.resize(n_layer);\n\n model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);\n model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n\n // map by name\n model.tensors[\"model/ln_f/g\"] = model.ln_f_g;\n model.tensors[\"model/ln_f/b\"] = model.ln_f_b;\n\n model.tensors[\"model/wte\"] = model.wte;\n model.tensors[\"model/wpe\"] = model.wpe;\n model.tensors[\"model/lm_head\"] = model.lm_head;\n\n for (int i = 0; i < n_layer; ++i) {\n auto & layer = model.layers[i];\n\n layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);\n layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);\n\n layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);\n layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);\n layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);\n\n layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);\n layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n // map by name\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/g\"] = layer.ln_1_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/b\"] = layer.ln_1_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/g\"] = layer.ln_2_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/b\"] = layer.ln_2_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/w\"] = layer.c_attn_attn_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/b\"] = layer.c_attn_attn_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/w\"] = layer.c_attn_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/b\"] = layer.c_attn_proj_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/w\"] = layer.c_mlp_fc_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/b\"] = layer.c_mlp_fc_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/w\"] = layer.c_mlp_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/b\"] = layer.c_mlp_proj_b;\n }\n }\n\n // override the default training context with the user-provided\n model.hparams.n_ctx = n_ctx;\n\n // key + value memory\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n\n const int n_mem = n_layer*n_ctx;\n const int n_elements = n_embd*n_mem;\n\n model.kv_cache.k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n model.kv_cache.v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n\n model.kv_cache.head = 0;\n model.kv_cache.size = n_ctx;\n\n model.kv_cache.cells.resize(n_ctx);\n\n const size_t memory_size = ggml_nbytes(model.kv_cache.k) + ggml_nbytes(model.kv_cache.v);\n\n printf(\"%s: memory size = %8.2f MB, n_mem = %d\\n\", __func__, memory_size/1024.0/1024.0, n_mem);\n\n // create a backend buffer (can be in host or device memory)\n model.kv_cache.buffer = ggml_backend_alloc_buffer(model.backend, memory_size + 256);\n\n // allocate the tensors into the backend buffer\n {\n ggml_tallocr alloc = ggml_tallocr_new(model.kv_cache.buffer);\n\n // this updates the pointers in the tensors to point to the correct location in the buffer\n // this is necessary since the ggml_context is .no_alloc == true\n // note that the buffer can actually be a device buffer, depending on the backend\n ggml_tallocr_alloc(&alloc, model.kv_cache.k);\n ggml_tallocr_alloc(&alloc, model.kv_cache.v);\n }\n }\n\n // load weights\n {\n ggml_tallocr alloc = ggml_tallocr_new(model.buffer_w);\n\n size_t total_size = 0;\n\n bool has_lm_head = false;\n\n std::vector read_buf;\n\n while (true) {\n int32_t n_dims;\n int32_t length;\n int32_t ttype;\n\n fin.read(reinterpret_cast(&n_dims), sizeof(n_dims));\n fin.read(reinterpret_cast(&length), sizeof(length));\n fin.read(reinterpret_cast(&ttype), sizeof(ttype));\n\n if (fin.eof()) {\n break;\n }\n\n int32_t nelements = 1;\n int32_t ne[2] = { 1, 1 };\n for (int i = 0; i < n_dims; ++i) {\n fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i]));\n nelements *= ne[i];\n }\n\n std::string name(length, 0);\n fin.read(&name[0], length);\n\n if (model.tensors.find(name) == model.tensors.end()) {\n fprintf(stderr, \"%s: unknown tensor '%s' in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n auto tensor = model.tensors[name];\n ggml_set_name(tensor, name.c_str());\n if (ggml_nelements(tensor) != nelements) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {\n fprintf(stderr, \"%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\\n\",\n __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);\n return false;\n }\n\n // for debugging\n if (0) {\n printf(\"%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\\n\", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));\n }\n\n const size_t bpe = ggml_type_size(ggml_type(ttype));\n\n if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\\n\",\n __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);\n return false;\n }\n\n ggml_tallocr_alloc(&alloc, tensor);\n\n if (ggml_backend_is_cpu (model.backend)\n#ifdef GGML_USE_METAL\n || ggml_backend_is_metal(model.backend)\n#endif\n ) {\n // for the CPU and Metal backend, we can read directly into the tensor\n fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor));\n } else {\n // read into a temporary buffer first, then copy to device memory\n read_buf.resize(ggml_nbytes(tensor));\n fin.read(read_buf.data(), ggml_nbytes(tensor));\n ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor));\n }\n\n // GPT-2 models share the WTE tensor as the LM head\n if (name == \"model/wte\" && has_lm_head == false) {\n //ggml_tallocr_alloc(alloc, model.lm_head);\n //ggml_backend_tensor_copy(tensor, model.lm_head);\n model.lm_head = tensor;\n }\n\n if (name == \"model/lm_head\") {\n has_lm_head = true;\n }\n\n total_size += ggml_nbytes(tensor);\n }\n\n printf(\"%s: model size = %8.2f MB\\n\", __func__, total_size/1024.0/1024.0);\n }\n\n fin.close();\n\n return true;\n}\n\n// build the computation graph\nstruct ggml_cgraph * gpt2_graph(\n const gpt2_model & model,\n const gpt2_batch & batch,\n bool measure) {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_head = hparams.n_head;\n\n const auto & kv_cache = model.kv_cache;\n\n const int32_t n_tokens = batch.n_tokens;\n const int32_t n_kv = measure ? n_ctx : kv_cache.n;\n const int32_t kv_head = measure ? n_ctx - n_tokens : kv_cache.head;\n\n // since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data\n static size_t buf_size = ggml_tensor_overhead()*GPT2_MAX_NODES + ggml_graph_overhead_custom(GPT2_MAX_NODES, false);\n static std::vector buf(buf_size);\n\n struct ggml_init_params params = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ buf.data(),\n /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()\n };\n\n struct ggml_context * ctx = ggml_init(params);\n\n struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, GPT2_MAX_NODES, false);\n\n struct ggml_tensor * inpL;\n if (batch.token) {\n struct ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_tokens);\n ggml_set_name(inp_tokens, \"inp_tokens\");\n ggml_set_input(inp_tokens);\n\n struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_tokens);\n ggml_set_name(position, \"position\");\n ggml_set_input(position);\n\n // wte + wpe\n inpL =\n ggml_add(ctx,\n ggml_get_rows(ctx, model.wte, inp_tokens),\n ggml_get_rows(ctx, model.wpe, position));\n } else {\n GGML_ASSERT(batch.embd);\n\n inpL = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens);\n ggml_set_name(inpL, \"embd\");\n ggml_set_input(inpL);\n }\n\n // KQ_mask (mask for 1 head, it will be broadcasted to all heads)\n struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_kv, n_tokens, 1);\n ggml_set_name(KQ_mask, \"KQ_mask\");\n ggml_set_input(KQ_mask);\n\n\n for (int il = 0; il < n_layer; ++il) {\n struct ggml_tensor * cur;\n\n // norm\n {\n // [ 768, N]\n cur = ggml_norm(ctx, inpL, hparams.eps);\n\n // cur = ln_1_g*cur + ln_1_b\n // [ 768, N]\n cur = ggml_add(ctx,\n ggml_mul(ctx,\n cur,\n model.layers[il].ln_1_g),\n model.layers[il].ln_1_b);\n }\n\n // attn\n // [2304, 768] - model.layers[il].c_attn_attn_w\n // [2304, 1] - model.layers[il].c_attn_attn_b\n // [ 768, n_tokens] - cur (in)\n // [2304, n_tokens] - cur (out)\n //\n // cur = attn_w*cur + attn_b\n // [2304, n_tokens]\n {\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_attn_attn_w,\n cur);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_attn_attn_b);\n }\n\n // self-attention\n {\n struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*n_embd);\n struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, n_tokens, cur->nb[1], 1*sizeof(float)*n_embd);\n struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, n_tokens, cur->nb[1], 2*sizeof(float)*n_embd);\n\n // store key and value to memory\n if (n_tokens >= 1) {\n struct ggml_tensor * k = ggml_view_1d(ctx, model.kv_cache.k, n_tokens*n_embd, (ggml_element_size(model.kv_cache.k)*n_embd)*(il*n_ctx + kv_head));\n struct ggml_tensor * v = ggml_view_1d(ctx, model.kv_cache.v, n_tokens*n_embd, (ggml_element_size(model.kv_cache.v)*n_embd)*(il*n_ctx + kv_head));\n\n ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));\n ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));\n }\n\n // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)\n // [64, N, 12]\n struct ggml_tensor * Q =\n ggml_permute(ctx,\n ggml_cont_3d(ctx,\n Qcur,\n n_embd/n_head, n_head, n_tokens),\n 0, 2, 1, 3);\n\n // K = Kmem.view(n_embd/n_head, n_head, n_kv).permute(0, 2, 1, 3)\n // [64, n_kv, 12]\n struct ggml_tensor * K =\n ggml_permute(ctx,\n ggml_reshape_3d(ctx,\n ggml_view_1d(ctx, model.kv_cache.k, n_kv*n_embd, il*n_ctx*ggml_element_size(model.kv_cache.k)*n_embd),\n n_embd/n_head, n_head, n_kv),\n 0, 2, 1, 3);\n\n // GG: flash attention\n //struct ggml_tensor * V =\n // ggml_cpy(ctx0,\n // ggml_permute(ctx0,\n // ggml_reshape_3d(ctx0,\n // ggml_view_1d(ctx0, model.kv_cache.v, n_kv*n_embd, il*n_ctx*ggml_element_size(model.kv_cache.v)*n_embd),\n // n_embd/n_head, n_head, n_kv),\n // 1, 2, 0, 3),\n // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_embd/n_head, n_head));\n\n //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);\n\n // K * Q\n // [n_kv, n_tokens, 12]\n struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);\n\n // KQ_scaled = KQ / sqrt(n_embd/n_head)\n // [n_kv, n_tokens, 12]\n struct ggml_tensor * KQ_scaled =\n ggml_scale(ctx,\n KQ,\n 1.0f/sqrtf(float(n_embd)/n_head));\n\n // KQ_masked = mask_past(KQ_scaled)\n // [n_kv, n_tokens, 12]\n struct ggml_tensor * KQ_masked = ggml_add(ctx, KQ_scaled, KQ_mask);\n\n // KQ = soft_max(KQ_masked)\n // [n_kv, N, 12]\n struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);\n\n // V_trans = Vmem.view(n_embd/n_head, n_head, n_kv).permute(1, 2, 0, 3).contiguous()\n // [n_kv, 64, 12]\n struct ggml_tensor * V_trans =\n ggml_cont_3d(ctx,\n ggml_permute(ctx,\n ggml_reshape_3d(ctx,\n ggml_view_1d(ctx, model.kv_cache.v, n_kv*n_embd, il*n_ctx*ggml_element_size(model.kv_cache.v)*n_embd),\n n_embd/n_head, n_head, n_kv),\n 1, 2, 0, 3),\n n_kv, n_embd/n_head, n_head);\n\n // KQV = transpose(V) * KQ_soft_max\n // [64, n_tokens, 12]\n struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);\n\n // KQV_merged = KQV.permute(0, 2, 1, 3)\n // [64, 12, n_tokens]\n struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);\n\n // cur = KQV_merged.contiguous().view(n_embd, N)\n // [768, n_tokens]\n cur = ggml_cont_2d(ctx, KQV_merged, n_embd, n_tokens);\n }\n\n // projection\n // [ 768, 768] - model.layers[il].c_attn_proj_w\n // [ 768, 1] - model.layers[il].c_attn_proj_b\n // [ 768, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n {\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_attn_proj_w,\n cur);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_attn_proj_b);\n }\n\n // add the input\n cur = ggml_add(ctx, cur, inpL);\n\n struct ggml_tensor * inpFF = cur;\n\n // feed-forward network\n {\n // norm\n {\n cur = ggml_norm(ctx, inpFF, hparams.eps);\n\n // cur = ln_2_g*cur + ln_2_b\n // [ 768, N]\n cur = ggml_add(ctx,\n ggml_mul(ctx,\n cur,\n model.layers[il].ln_2_g),\n model.layers[il].ln_2_b);\n }\n\n // fully connected\n // [3072, 768] - model.layers[il].c_mlp_fc_w\n // [3072, 1] - model.layers[il].c_mlp_fc_b\n // [ 768, N] - cur (in)\n // [3072, N] - cur (out)\n //\n // cur = fc_w*cur + fc_b\n // [3072, N]\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_mlp_fc_w,\n cur);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_mlp_fc_b);\n\n // GELU activation\n // [3072, N]\n cur = ggml_gelu(ctx, cur);\n\n // projection\n // [ 768, 3072] - model.layers[il].c_mlp_proj_w\n // [ 768, 1] - model.layers[il].c_mlp_proj_b\n // [3072, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_mlp_proj_w,\n cur);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_mlp_proj_b);\n }\n\n // input for next layer\n inpL = ggml_add(ctx, cur, inpFF);\n }\n\n // norm\n {\n // [ 768, N]\n inpL = ggml_norm(ctx, inpL, hparams.eps);\n\n // inpL = ln_f_g*inpL + ln_f_b\n // [ 768, N]\n inpL = ggml_add(ctx,\n ggml_mul(ctx,\n inpL,\n model.ln_f_g),\n model.ln_f_b);\n }\n\n // inpL = WTE * inpL\n // [ 768, 50257] - model.lm_head\n // [ 768, N] - inpL\n inpL = ggml_mul_mat(ctx, model.lm_head, inpL);\n\n // logits -> probs\n //inpL = ggml_soft_max(ctx0, inpL);\n\n ggml_build_forward_expand(gf, inpL);\n\n ggml_free(ctx);\n\n return gf;\n}\n\nstatic void gpt2_kv_cache_seq_cp(\n struct gpt2_kv_cache & cache,\n gpt2_seq_id seq_id_src,\n gpt2_seq_id seq_id_dst,\n gpt2_pos p0,\n gpt2_pos p1) {\n if (p0 < 0) p0 = 0;\n if (p1 < 0) p1 = std::numeric_limits::max();\n\n for (uint32_t i = 0; i < cache.size; ++i) {\n if (cache.cells[i].has_seq_id(seq_id_src) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) {\n cache.cells[i].seq_id.insert(seq_id_dst);\n }\n }\n}\n\nstruct gpt2_batch gpt2_batch_init(int32_t n_tokens, int32_t embd) {\n gpt2_batch batch;\n\n if (embd) {\n batch.embd = (float *) malloc(sizeof(float) * n_tokens * embd);\n } else {\n batch.token = (gpt_vocab::id *) malloc(sizeof(gpt_vocab::id) * n_tokens);\n }\n\n batch.pos = (gpt2_pos *) malloc(sizeof(gpt2_pos) * n_tokens);\n batch.seq_id = (gpt2_seq_id *) malloc(sizeof(gpt2_seq_id) * n_tokens);\n batch.logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens);\n\n return batch;\n}\n\nvoid gpt2_batch_free(struct gpt2_batch batch) {\n if (batch.token) free(batch.token);\n if (batch.embd) free(batch.embd);\n if (batch.pos) free(batch.pos);\n if (batch.seq_id) free(batch.seq_id);\n if (batch.logits) free(batch.logits);\n}\n\n// Positive return values does not mean a fatal error, but rather a warning.\n// 0 - success\n// < 0 - error\nint gpt2_decode(\n struct gpt2_model & model,\n ggml_gallocr_t allocr,\n struct gpt2_batch batch,\n int n_threads,\n std::vector & logits) {\n const int32_t n_tokens = batch.n_tokens;\n const auto & hparams = model.hparams;\n const int n_vocab = hparams.n_vocab;\n\n if (n_tokens == 0) {\n printf(\"%s: n_tokens == 0\", __func__);\n return -1;\n }\n\n GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd));\n\n auto & cache = model.kv_cache;\n\n for (int i = 0; i < n_tokens; i++) {\n cache.cells[cache.head + i].pos = batch.pos[i];\n cache.cells[cache.head + i].seq_id.insert(batch.seq_id[i]);\n }\n\n cache.n = cache.head + n_tokens;\n\n struct ggml_cgraph * gf = gpt2_graph(model, batch, false);\n\n // allocate tensors\n ggml_gallocr_alloc_graph(allocr, gf);\n\n // set the graph inputs\n if (batch.token) {\n struct ggml_tensor * inp_tokens = ggml_graph_get_tensor(gf, \"inp_tokens\");\n ggml_backend_tensor_set(inp_tokens, batch.token, 0, n_tokens*ggml_element_size(inp_tokens));\n\n struct ggml_tensor * position = ggml_graph_get_tensor(gf, \"position\");\n for (int i = 0; i < n_tokens; ++i) {\n int32_t v = batch.pos[i];\n ggml_backend_tensor_set(position, &v, i*sizeof(int32_t), sizeof(v));\n }\n } else {\n struct ggml_tensor * embd = ggml_graph_get_tensor(gf, \"embd\");\n ggml_backend_tensor_set(embd, batch.embd, 0, n_tokens * hparams.n_embd * ggml_element_size(embd));\n }\n\n {\n struct ggml_tensor * KQ_mask = ggml_graph_get_tensor(gf, \"KQ_mask\");\n const auto & kv_cache = model.kv_cache;\n const int32_t n_tokens = batch.n_tokens;\n const int32_t n_kv = kv_cache.n;\n\n std::vector data_buf(n_kv*n_tokens);\n const float neg_inf_v = -INFINITY;\n\n for (int h = 0; h < 1; ++h) {\n int h_offset = h*(n_kv*n_tokens);\n for (int j = 0; j < n_tokens; ++j) {\n const gpt2_pos pos = batch.pos[j];\n const gpt2_seq_id seq_id = batch.seq_id[j];\n\n for (int i = 0; i < n_kv; ++i) {\n if (!kv_cache.cells[i].has_seq_id(seq_id) || kv_cache.cells[i].pos > pos) {\n data_buf[h_offset + j*n_kv + i] = neg_inf_v;\n }\n }\n }\n }\n\n ggml_backend_tensor_set(KQ_mask, data_buf.data(), 0, data_buf.size() * sizeof(float));\n }\n\n // run the computation\n if (ggml_backend_is_cpu(model.backend)) {\n ggml_backend_cpu_set_n_threads(model.backend, n_threads);\n }\n ggml_backend_graph_compute(model.backend, gf);\n\n //if (n_past%100 == 0) {\n // ggml_graph_print (&gf);\n // ggml_graph_dump_dot(&gf, NULL, \"gpt-2.dot\");\n //}\n\n // in this case, the output tensor is the last one in the graph\n struct ggml_tensor * inpL = ggml_graph_node(gf, -1);\n\n if (batch.logits) {\n // return logits for all tokens\n logits.resize(n_vocab*n_tokens);\n for (int32_t i = 0; i < n_tokens; i++) {\n if (batch.logits[i] == 0) {\n continue;\n }\n ggml_backend_tensor_get(inpL, logits.data() + n_vocab*i, n_vocab*i*sizeof(float), sizeof(float)*n_vocab);\n }\n } else {\n // return result just for the last token\n logits.resize(n_vocab);\n ggml_backend_tensor_get(inpL, logits.data(), (n_vocab*(n_tokens-1))*sizeof(float), sizeof(float)*n_vocab);\n }\n\n // update the kv ring buffer\n cache.head += n_tokens;\n\n // ensure kv cache head points to a valid index.\n if (cache.head >= cache.size) {\n printf(\"%s: cache.head >= cache.size\\n\", __func__);\n return -2;\n }\n\n return 0;\n}\n\nint main(int argc, char ** argv) {\n ggml_time_init();\n\n const int64_t t_main_start_us = ggml_time_us();\n\n gpt_params params;\n\n if (gpt_params_parse(argc, argv, params) == false) {\n return 1;\n }\n\n if (params.seed < 0) {\n params.seed = time(NULL);\n }\n\n printf(\"%s: seed = %d\\n\", __func__, params.seed);\n\n std::mt19937 rng(params.seed);\n if (params.prompt.empty()) {\n params.prompt = gpt_random_prompt(rng);\n }\n\n int64_t t_load_us = 0;\n\n gpt_vocab vocab;\n gpt2_model model;\n\n // load the model\n {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_model_load(params.model, model, vocab, params.n_ctx, params.n_gpu_layers)) {\n fprintf(stderr, \"%s: failed to load model from '%s'\\n\", __func__, params.model.c_str());\n return 1;\n }\n\n t_load_us = ggml_time_us() - t_start_us;\n\n test_gpt_tokenizer(vocab, params.token_test);\n }\n\n // tokenize the prompt\n std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt);\n\n const int n_parallel = params.n_parallel;\n const int n_batch_max = std::max(embd_inp.size(), (size_t)n_parallel);\n\n // create a gpt2_batch\n // we use this object to submit token data for decoding\n gpt2_batch batch = gpt2_batch_init(n_batch_max, 0);\n\n // prepare required memory and allocate the compute buffer\n ggml_gallocr_t allocr = NULL;\n {\n // create an allocator to measure the memory usage\n allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));\n\n // create the worst case graph for memory usage estimation\n batch.n_tokens = n_batch_max;\n struct ggml_cgraph * gf = gpt2_graph(model, batch, true);\n\n // pre-allocate the compute buffer for the worst case (optional)\n ggml_gallocr_reserve(allocr, gf);\n size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);\n fprintf(stderr, \"%s: compute buffer size: %.2f MB\\n\", __func__, mem_size/1024.0/1024.0);\n }\n\n int64_t t_sample_us = 0;\n int64_t t_predict_us = 0;\n\n std::vector logits;\n\n // evaluate the initial prompt\n batch.n_tokens = embd_inp.size();\n\n for (int32_t i = 0; i < batch.n_tokens; i++) {\n batch.token[i] = embd_inp[i];\n batch.pos[i] = i;\n batch.seq_id[i] = 0;\n batch.logits[i] = false;\n }\n\n // gpt2_decode will output logits only for the last token of the prompt\n batch.logits[batch.n_tokens - 1] = true;\n\n if (gpt2_decode(model, allocr, batch, params.n_threads, logits) != 0) {\n printf(\"%s: gpt2_decode() failed\\n\", __func__);\n return 1;\n }\n\n // assign the system KV cache to all parallel sequences\n // this way, the parallel sequences will \"reuse\" the prompt tokens without having to copy them\n for (int32_t i = 1; i < n_parallel; ++i) {\n gpt2_kv_cache_seq_cp(model.kv_cache, 0, i, 0, batch.n_tokens);\n }\n\n if (n_parallel > 1) {\n printf(\"\\n\\n%s: generating %d sequences ...\\n\", __func__, n_parallel);\n }\n\n params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());\n\n printf(\"%s: prompt: '%s'\\n\", __func__, params.prompt.c_str());\n printf(\"%s: number of tokens in prompt = %zu, first 8 tokens: \", __func__, embd_inp.size());\n for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {\n printf(\"%d \", embd_inp[i]);\n }\n printf(\"\\n\\n\");\n\n std::vector streams(n_parallel);\n\n // remember the batch index of the last token for each parallel sequence\n // we need this to determine which logits to sample from\n std::vector i_batch(n_parallel, batch.n_tokens - 1);\n\n int n_cur = batch.n_tokens;\n int n_len = batch.n_tokens + params.n_predict;\n int n_decoded = 0;\n\n const int n_vocab = model.hparams.n_vocab;\n const int top_k = params.top_k;\n const float top_p = params.top_p;\n const float temp = params.temp;\n\n while (n_cur < n_len) {\n batch.n_tokens = 0;\n\n for (int32_t i = 0; i < n_parallel; ++i) {\n if (i_batch[i] < 0) {\n // the stream has already finished\n continue;\n }\n\n auto * logits_i = logits.data() + i_batch[i]*n_vocab;\n\n gpt_vocab::id id = 0;\n {\n const int64_t t_start_sample_us = ggml_time_us();\n\n id = gpt_sample_top_k_top_p(vocab, logits_i, top_k, top_p, temp, rng);\n\n t_sample_us += ggml_time_us() - t_start_sample_us;\n }\n\n // is it an end of stream? -> mark the stream as finished\n if ((!params.ignore_eos && id == 50256) || n_cur == n_len - 1) {\n i_batch[i] = -1;\n printf(\"\\n\");\n if (n_parallel > 1) {\n printf(\"%s: stream %d finished at n_cur = %d\", __func__, i, n_cur);\n }\n\n continue;\n }\n\n auto& token = vocab.id_to_token[id];\n if (n_parallel == 1) {\n printf(\"%s\", token.c_str());\n fflush(stdout);\n }\n\n streams[i] += token;\n\n // push this new token for next evaluation\n batch.token [batch.n_tokens] = id;\n batch.pos [batch.n_tokens] = n_cur;\n batch.seq_id[batch.n_tokens] = i;\n batch.logits[batch.n_tokens] = true;\n\n i_batch[i] = batch.n_tokens;\n\n batch.n_tokens += 1;\n\n n_decoded += 1;\n }\n\n // all streams are finished\n if (batch.n_tokens == 0) {\n break;\n }\n\n n_cur += 1;\n\n {\n const int64_t t_start_us = ggml_time_us();\n\n // evaluate the current batch with the transformer model\n int ret_code = gpt2_decode(model, allocr, batch, params.n_threads, logits);\n if (ret_code != 0) {\n fprintf(stderr, \"%s : failed to eval, return code %d\\n\", __func__, ret_code);\n return 1;\n }\n\n t_predict_us += ggml_time_us() - t_start_us;\n }\n }\n\n if (n_parallel > 1) {\n printf(\"\\n\");\n\n for (int32_t i = 0; i < n_parallel; ++i) {\n printf(\"sequence %d:\\n\\n%s%s\\n\\n\", i, params.prompt.c_str(), streams[i].c_str());\n }\n }\n\n // report timing\n {\n const int64_t t_main_end_us = ggml_time_us();\n\n printf(\"\\n\\n\");\n printf(\"%s: n_decoded = %8d\\n\", __func__, n_decoded);\n printf(\"%s: load time = %8.2f ms\\n\", __func__, t_load_us/1000.0f);\n printf(\"%s: sample time = %8.2f ms\\n\", __func__, t_sample_us/1000.0f);\n printf(\"%s: predict time = %8.2f ms\\n\", __func__, t_predict_us/1000.0f);\n printf(\"%s: total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n }\n\n gpt2_batch_free(batch);\n ggml_free(model.ctx_w);\n\n ggml_gallocr_free(allocr);\n ggml_backend_buffer_free(model.buffer_w);\n ggml_backend_buffer_free(model.kv_cache.buffer);\n ggml_backend_free(model.backend);\n\n return 0;\n}\n" }, { "path": "examples/gpt-2/main-ctx.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n#include \"common.h\"\n#include \"common-ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\n// default hparams (GPT-2 117M)\nstruct gpt2_hparams {\n int32_t n_vocab = 50257;\n int32_t n_ctx = 1024;\n int32_t n_embd = 768;\n int32_t n_head = 12;\n int32_t n_layer = 12;\n int32_t ftype = 1;\n float eps = 1e-5f;\n};\n\nstruct gpt2_layer {\n // normalization\n struct ggml_tensor * ln_1_g;\n struct ggml_tensor * ln_1_b;\n\n struct ggml_tensor * ln_2_g;\n struct ggml_tensor * ln_2_b;\n\n // attention\n struct ggml_tensor * c_attn_attn_w;\n struct ggml_tensor * c_attn_attn_b;\n\n struct ggml_tensor * c_attn_proj_w;\n struct ggml_tensor * c_attn_proj_b;\n\n // mlp\n struct ggml_tensor * c_mlp_fc_w;\n struct ggml_tensor * c_mlp_fc_b;\n\n struct ggml_tensor * c_mlp_proj_w;\n struct ggml_tensor * c_mlp_proj_b;\n};\n\nstruct gpt2_model {\n gpt2_hparams hparams;\n\n // normalization\n struct ggml_tensor * ln_f_g;\n struct ggml_tensor * ln_f_b;\n\n struct ggml_tensor * wte; // token embedding\n struct ggml_tensor * wpe; // position embedding\n struct ggml_tensor * lm_head; // language model head\n\n std::vector layers;\n\n // key + value memory\n struct ggml_tensor * memory_k;\n struct ggml_tensor * memory_v;\n\n //\n struct ggml_context * ctx_w;\n std::map tensors;\n};\n\n// load the model's weights from a file\nbool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) {\n printf(\"%s: loading model from '%s'\\n\", __func__, fname.c_str());\n\n auto fin = std::ifstream(fname, std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"%s: failed to open '%s'\\n\", __func__, fname.c_str());\n return false;\n }\n\n // verify magic\n {\n uint32_t magic;\n fin.read((char *) &magic, sizeof(magic));\n if (magic != GGML_FILE_MAGIC) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, fname.c_str());\n return false;\n }\n }\n\n // load hparams\n {\n auto & hparams = model.hparams;\n\n fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));\n fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));\n\n const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;\n\n printf(\"%s: n_vocab = %d\\n\", __func__, hparams.n_vocab);\n printf(\"%s: n_ctx = %d\\n\", __func__, hparams.n_ctx);\n printf(\"%s: n_embd = %d\\n\", __func__, hparams.n_embd);\n printf(\"%s: n_head = %d\\n\", __func__, hparams.n_head);\n printf(\"%s: n_layer = %d\\n\", __func__, hparams.n_layer);\n printf(\"%s: ftype = %d\\n\", __func__, hparams.ftype);\n printf(\"%s: qntvr = %d\\n\", __func__, qntvr);\n\n hparams.ftype %= GGML_QNT_VERSION_FACTOR;\n }\n\n // load vocab\n {\n int32_t n_vocab = 0;\n fin.read((char *) &n_vocab, sizeof(n_vocab));\n\n if (n_vocab != model.hparams.n_vocab) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad vocab size %d != %d)\\n\",\n __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);\n return false;\n }\n\n std::string word;\n std::vector buf(128);\n\n for (int i = 0; i < n_vocab; i++) {\n uint32_t len;\n fin.read((char *) &len, sizeof(len));\n\n buf.resize(len);\n fin.read((char *) buf.data(), len);\n word.assign(buf.data(), len);\n\n vocab.token_to_id[word] = i;\n vocab.id_to_token[i] = word;\n }\n }\n\n // for the big tensors, we have the option to store the data in 16-bit floats or quantized\n // in order to save memory and also to speed up the computation\n ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));\n if (wtype == GGML_TYPE_COUNT) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad ftype value %d)\\n\",\n __func__, fname.c_str(), model.hparams.ftype);\n return false;\n }\n\n auto & ctx = model.ctx_w;\n\n size_t ctx_size = 0;\n\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_vocab = hparams.n_vocab;\n\n ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g\n ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b\n\n ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte\n ctx_size += ggml_row_size(GGML_TYPE_F32, n_ctx*n_embd); // wpe\n ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head\n\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b\n\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b\n\n ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k\n ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v\n\n ctx_size += (6 + 12*n_layer)*512; // object overhead\n\n printf(\"%s: ggml tensor size = %d bytes\\n\", __func__, (int) sizeof(ggml_tensor));\n printf(\"%s: ggml ctx size = %6.2f MB\\n\", __func__, ctx_size/(1024.0*1024.0));\n }\n\n // create the ggml context\n {\n struct ggml_init_params params = {\n /*.mem_size =*/ ctx_size,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ false,\n };\n\n model.ctx_w = ggml_init(params);\n if (!model.ctx_w) {\n fprintf(stderr, \"%s: ggml_init() failed\\n\", __func__);\n return false;\n }\n }\n\n // prepare memory for the weights\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_vocab = hparams.n_vocab;\n\n model.layers.resize(n_layer);\n\n model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);\n model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n\n // map by name\n model.tensors[\"model/ln_f/g\"] = model.ln_f_g;\n model.tensors[\"model/ln_f/b\"] = model.ln_f_b;\n\n model.tensors[\"model/wte\"] = model.wte;\n model.tensors[\"model/wpe\"] = model.wpe;\n model.tensors[\"model/lm_head\"] = model.lm_head;\n\n for (int i = 0; i < n_layer; ++i) {\n auto & layer = model.layers[i];\n\n layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);\n layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);\n\n layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);\n layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);\n layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);\n\n layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);\n layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n // map by name\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/g\"] = layer.ln_1_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/b\"] = layer.ln_1_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/g\"] = layer.ln_2_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/b\"] = layer.ln_2_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/w\"] = layer.c_attn_attn_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/b\"] = layer.c_attn_attn_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/w\"] = layer.c_attn_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/b\"] = layer.c_attn_proj_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/w\"] = layer.c_mlp_fc_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/b\"] = layer.c_mlp_fc_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/w\"] = layer.c_mlp_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/b\"] = layer.c_mlp_proj_b;\n }\n }\n\n // key + value memory\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n\n const int n_mem = n_layer*n_ctx;\n const int n_elements = n_embd*n_mem;\n\n model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n\n const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);\n\n printf(\"%s: memory size = %8.2f MB, n_mem = %d\\n\", __func__, memory_size/1024.0/1024.0, n_mem);\n }\n\n // load weights\n {\n size_t total_size = 0;\n\n bool has_lm_head = false;\n\n while (true) {\n int32_t n_dims;\n int32_t length;\n int32_t ttype;\n\n fin.read(reinterpret_cast(&n_dims), sizeof(n_dims));\n fin.read(reinterpret_cast(&length), sizeof(length));\n fin.read(reinterpret_cast(&ttype), sizeof(ttype));\n\n if (fin.eof()) {\n break;\n }\n\n int32_t nelements = 1;\n int32_t ne[2] = { 1, 1 };\n for (int i = 0; i < n_dims; ++i) {\n fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i]));\n nelements *= ne[i];\n }\n\n std::string name(length, 0);\n fin.read(&name[0], length);\n\n if (model.tensors.find(name) == model.tensors.end()) {\n fprintf(stderr, \"%s: unknown tensor '%s' in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n auto tensor = model.tensors[name];\n if (ggml_nelements(tensor) != nelements) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {\n fprintf(stderr, \"%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\\n\",\n __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);\n return false;\n }\n\n // for debugging\n if (0) {\n printf(\"%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\\n\", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));\n }\n\n const size_t bpe = ggml_type_size(ggml_type(ttype));\n\n if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\\n\",\n __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);\n return false;\n }\n\n fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor));\n\n // GPT-2 models share the WTE tensor as the LM head\n if (name == \"model/wte\" && has_lm_head == false) {\n memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor));\n }\n\n if (name == \"model/lm_head\") {\n has_lm_head = true;\n }\n\n total_size += ggml_nbytes(tensor);\n }\n\n printf(\"%s: model size = %8.2f MB\\n\", __func__, total_size/1024.0/1024.0);\n }\n\n fin.close();\n\n return true;\n}\n\n// evaluate the transformer\n//\n// - model: the model\n// - n_threads: number of threads to use\n// - n_past: the context size so far\n// - embd_inp: the embeddings of the tokens in the context\n// - embd_w: the predicted logits for the next token\n//\nbool gpt2_eval(\n const gpt2_model & model,\n const int n_threads,\n const int n_past,\n const std::vector & embd_inp,\n std::vector & embd_w,\n size_t & mem_per_token) {\n const int N = embd_inp.size();\n\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_head = hparams.n_head;\n const int n_vocab = hparams.n_vocab;\n\n static size_t buf_size = 256u*1024*1024;\n static void * buf = malloc(buf_size);\n\n if (mem_per_token > 0 && mem_per_token*N > buf_size) {\n const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead\n //printf(\"\\n%s: reallocating buffer from %zu to %zu bytes\\n\", __func__, buf_size, buf_size_new);\n\n // reallocate\n buf_size = buf_size_new;\n buf = realloc(buf, buf_size);\n if (buf == nullptr) {\n fprintf(stderr, \"%s: failed to allocate %zu bytes\\n\", __func__, buf_size);\n return false;\n }\n }\n\n struct ggml_init_params params = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ buf,\n /*.no_alloc =*/ false,\n };\n\n struct ggml_context * ctx0 = ggml_init(params);\n struct ggml_cgraph * gf = ggml_new_graph(ctx0);\n\n struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);\n memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));\n\n struct ggml_tensor * position = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);\n for (int i = 0; i < N; ++i) {\n ((int32_t *) position->data)[i] = n_past + i;\n }\n\n // wte + wpe\n struct ggml_tensor * inpL =\n ggml_add(ctx0,\n ggml_get_rows(ctx0, model.wte, embd),\n ggml_get_rows(ctx0, model.wpe, position));\n\n for (int il = 0; il < n_layer; ++il) {\n struct ggml_tensor * cur;\n\n // norm\n {\n // [ 768, N]\n cur = ggml_norm(ctx0, inpL, hparams.eps);\n\n // cur = ln_1_g*cur + ln_1_b\n // [ 768, N]\n cur = ggml_add(ctx0,\n ggml_mul(ctx0,\n ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),\n cur),\n ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));\n }\n\n // attn\n // [2304, 768] - model.layers[il].c_attn_attn_w\n // [2304, 1] - model.layers[il].c_attn_attn_b\n // [ 768, N] - cur (in)\n // [2304, N] - cur (out)\n //\n // cur = attn_w*cur + attn_b\n // [2304, N]\n {\n cur = ggml_mul_mat(ctx0,\n model.layers[il].c_attn_attn_w,\n cur);\n\n cur = ggml_add(ctx0,\n ggml_repeat(ctx0, model.layers[il].c_attn_attn_b, cur),\n cur);\n }\n\n // self-attention\n {\n struct ggml_tensor * Qcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);\n struct ggml_tensor * Kcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);\n struct ggml_tensor * Vcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);\n\n // store key and value to memory\n if (N >= 1) {\n struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));\n struct ggml_tensor * v = ggml_view_1d(ctx0, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));\n\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));\n }\n\n // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)\n // [64, N, 12]\n struct ggml_tensor * Q =\n ggml_permute(ctx0,\n ggml_cpy(ctx0,\n Qcur,\n ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),\n 0, 2, 1, 3);\n\n // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)\n // [64, n_past + N, 12]\n struct ggml_tensor * K =\n ggml_permute(ctx0,\n ggml_reshape_3d(ctx0,\n ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),\n n_embd/n_head, n_head, n_past + N),\n 0, 2, 1, 3);\n\n // GG: flash attention\n //struct ggml_tensor * V =\n // ggml_cpy(ctx0,\n // ggml_permute(ctx0,\n // ggml_reshape_3d(ctx0,\n // ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),\n // n_embd/n_head, n_head, n_past + N),\n // 1, 2, 0, 3),\n // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));\n\n //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);\n\n // K * Q\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);\n\n // KQ_scaled = KQ / sqrt(n_embd/n_head)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));\n\n // KQ_masked = mask_past(KQ_scaled)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);\n\n // KQ = soft_max(KQ_masked)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);\n\n // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()\n // [n_past + N, 64, 12]\n struct ggml_tensor * V_trans =\n ggml_cpy(ctx0,\n ggml_permute(ctx0,\n ggml_reshape_3d(ctx0,\n ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),\n n_embd/n_head, n_head, n_past + N),\n 1, 2, 0, 3),\n ggml_new_tensor_3d(ctx0, model.memory_v->type, n_past + N, n_embd/n_head, n_head));\n\n // KQV = transpose(V) * KQ_soft_max\n // [64, N, 12]\n struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);\n\n // KQV_merged = KQV.permute(0, 2, 1, 3)\n // [64, 12, N]\n struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);\n\n // cur = KQV_merged.contiguous().view(n_embd, N)\n // [768, N]\n cur = ggml_cpy(ctx0,\n KQV_merged,\n ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));\n }\n\n // projection\n // [ 768, 768] - model.layers[il].c_attn_proj_w\n // [ 768, 1] - model.layers[il].c_attn_proj_b\n // [ 768, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n {\n cur = ggml_mul_mat(ctx0,\n model.layers[il].c_attn_proj_w,\n cur);\n\n cur = ggml_add(ctx0,\n ggml_repeat(ctx0, model.layers[il].c_attn_proj_b, cur),\n cur);\n }\n\n // add the input\n cur = ggml_add(ctx0, cur, inpL);\n\n struct ggml_tensor * inpFF = cur;\n\n // feed-forward network\n {\n // norm\n {\n cur = ggml_norm(ctx0, inpFF, hparams.eps);\n\n // cur = ln_2_g*cur + ln_2_b\n // [ 768, N]\n cur = ggml_add(ctx0,\n ggml_mul(ctx0,\n ggml_repeat(ctx0, model.layers[il].ln_2_g, cur),\n cur),\n ggml_repeat(ctx0, model.layers[il].ln_2_b, cur));\n }\n\n // fully connected\n // [3072, 768] - model.layers[il].c_mlp_fc_w\n // [3072, 1] - model.layers[il].c_mlp_fc_b\n // [ 768, N] - cur (in)\n // [3072, N] - cur (out)\n //\n // cur = fc_w*cur + fc_b\n // [3072, N]\n cur = ggml_mul_mat(ctx0,\n model.layers[il].c_mlp_fc_w,\n cur);\n\n cur = ggml_add(ctx0,\n ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),\n cur);\n\n // GELU activation\n // [3072, N]\n cur = ggml_gelu(ctx0, cur);\n\n // projection\n // [ 768, 3072] - model.layers[il].c_mlp_proj_w\n // [ 768, 1] - model.layers[il].c_mlp_proj_b\n // [3072, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n cur = ggml_mul_mat(ctx0,\n model.layers[il].c_mlp_proj_w,\n cur);\n\n cur = ggml_add(ctx0,\n ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),\n cur);\n }\n\n // input for next layer\n inpL = ggml_add(ctx0, cur, inpFF);\n }\n\n // norm\n {\n // [ 768, N]\n inpL = ggml_norm(ctx0, inpL, hparams.eps);\n\n // inpL = ln_f_g*inpL + ln_f_b\n // [ 768, N]\n inpL = ggml_add(ctx0,\n ggml_mul(ctx0,\n ggml_repeat(ctx0, model.ln_f_g, inpL),\n inpL),\n ggml_repeat(ctx0, model.ln_f_b, inpL));\n }\n\n // inpL = WTE * inpL\n // [ 768, 50257] - model.lm_head\n // [ 768, N] - inpL\n inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);\n\n // logits -> probs\n //inpL = ggml_soft_max_inplace(ctx0, inpL);\n\n // run the computation\n ggml_build_forward_expand(gf, inpL);\n ggml_graph_compute_with_ctx(ctx0, gf, n_threads);\n\n //if (n_past%100 == 0) {\n // ggml_graph_print (&gf);\n // ggml_graph_dump_dot(&gf, NULL, \"gpt-2.dot\");\n //}\n\n //embd_w.resize(n_vocab*N);\n //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);\n\n // return result just for the last token\n embd_w.resize(n_vocab);\n memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);\n\n if (mem_per_token == 0) {\n mem_per_token = ggml_used_mem(ctx0)/N;\n }\n //printf(\"used_mem = %zu\\n\", ggml_used_mem(ctx0));\n\n ggml_free(ctx0);\n\n return true;\n}\n\nint main(int argc, char ** argv) {\n ggml_time_init();\n\n const int64_t t_main_start_us = ggml_time_us();\n\n gpt_params params;\n params.model = \"models/gpt-2-117M/ggml-model.bin\";\n\n if (gpt_params_parse(argc, argv, params) == false) {\n return 1;\n }\n\n if (params.seed < 0) {\n params.seed = time(NULL);\n }\n\n printf(\"%s: seed = %d\\n\", __func__, params.seed);\n\n std::mt19937 rng(params.seed);\n if (params.prompt.empty()) {\n params.prompt = gpt_random_prompt(rng);\n }\n\n int64_t t_load_us = 0;\n\n gpt_vocab vocab;\n gpt2_model model;\n\n // load the model\n {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_model_load(params.model, model, vocab)) {\n fprintf(stderr, \"%s: failed to load model from '%s'\\n\", __func__, params.model.c_str());\n return 1;\n }\n\n t_load_us = ggml_time_us() - t_start_us;\n\n test_gpt_tokenizer(vocab, params.token_test);\n }\n\n int n_past = 0;\n\n int64_t t_sample_us = 0;\n int64_t t_predict_us = 0;\n\n std::vector logits;\n\n // tokenize the prompt\n std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt);\n\n params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());\n\n printf(\"%s: prompt: '%s'\\n\", __func__, params.prompt.c_str());\n printf(\"%s: number of tokens in prompt = %zu, first 8 tokens: \", __func__, embd_inp.size());\n for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {\n printf(\"%d \", embd_inp[i]);\n }\n printf(\"\\n\\n\");\n\n // submit the input prompt token-by-token\n // this reduces the memory usage during inference, at the cost of a bit of speed at the beginning\n std::vector embd;\n\n // determine the required inference memory per token:\n size_t mem_per_token = 0;\n gpt2_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);\n\n for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {\n // predict\n if (embd.size() > 0) {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {\n printf(\"Failed to predict\\n\");\n return 1;\n }\n\n t_predict_us += ggml_time_us() - t_start_us;\n }\n\n n_past += embd.size();\n embd.clear();\n\n if (i >= embd_inp.size()) {\n // sample next token\n const int top_k = params.top_k;\n const float top_p = params.top_p;\n const float temp = params.temp;\n\n const int n_vocab = model.hparams.n_vocab;\n\n gpt_vocab::id id = 0;\n\n {\n const int64_t t_start_sample_us = ggml_time_us();\n\n id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);\n\n t_sample_us += ggml_time_us() - t_start_sample_us;\n }\n\n // add it to the context\n embd.push_back(id);\n } else {\n // if here, it means we are still processing the input prompt\n for (size_t k = i; k < embd_inp.size(); k++) {\n embd.push_back(embd_inp[k]);\n if (int32_t(embd.size()) >= params.n_batch) {\n break;\n }\n }\n i += embd.size() - 1;\n }\n\n // display text\n for (auto id : embd) {\n printf(\"%s\", vocab.id_to_token[id].c_str());\n }\n fflush(stdout);\n\n // end of text token\n if (embd.back() == 50256) {\n break;\n }\n }\n\n // report timing\n {\n const int64_t t_main_end_us = ggml_time_us();\n\n printf(\"\\n\\n\");\n printf(\"%s: mem per token = %8zu bytes\\n\", __func__, mem_per_token);\n printf(\"%s: load time = %8.2f ms\\n\", __func__, t_load_us/1000.0f);\n printf(\"%s: sample time = %8.2f ms\\n\", __func__, t_sample_us/1000.0f);\n printf(\"%s: predict time = %8.2f ms / %.2f ms per token\\n\", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);\n printf(\"%s: total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n }\n\n ggml_free(model.ctx_w);\n\n return 0;\n}\n" }, { "path": "examples/gpt-2/main-sched.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n\n#ifdef GGML_USE_CUDA\n#include \"ggml-cuda.h\"\n#endif\n\n#ifdef GGML_USE_METAL\n#include \"ggml-metal.h\"\n#endif\n\n#ifdef GGML_USE_BLAS\n#include \"ggml-blas.h\"\n#endif\n\n#include \"common.h\"\n#include \"common-ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\n#define GPT2_MAX_NODES 4096\n\nstatic void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {\n (void) level;\n (void) user_data;\n fputs(text, stderr);\n fflush(stderr);\n}\n\n// default hparams (GPT-2 117M)\nstruct gpt2_hparams {\n int32_t n_vocab = 50257;\n int32_t n_ctx = 1024;\n int32_t n_embd = 768;\n int32_t n_head = 12;\n int32_t n_layer = 12;\n int32_t ftype = 1;\n float eps = 1e-5f;\n};\n\nstruct gpt2_layer {\n // normalization\n struct ggml_tensor * ln_1_g;\n struct ggml_tensor * ln_1_b;\n\n struct ggml_tensor * ln_2_g;\n struct ggml_tensor * ln_2_b;\n\n // attention\n struct ggml_tensor * c_attn_attn_w;\n struct ggml_tensor * c_attn_attn_b;\n\n struct ggml_tensor * c_attn_proj_w;\n struct ggml_tensor * c_attn_proj_b;\n\n // mlp\n struct ggml_tensor * c_mlp_fc_w;\n struct ggml_tensor * c_mlp_fc_b;\n\n struct ggml_tensor * c_mlp_proj_w;\n struct ggml_tensor * c_mlp_proj_b;\n};\n\nstruct gpt2_model {\n gpt2_hparams hparams;\n\n // normalization\n struct ggml_tensor * ln_f_g;\n struct ggml_tensor * ln_f_b;\n\n struct ggml_tensor * wte; // tkoen embedding\n struct ggml_tensor * wpe; // position embedding\n struct ggml_tensor * lm_head; // language model head\n\n std::vector layers;\n\n // key + value memory\n struct ggml_tensor * memory_k;\n struct ggml_tensor * memory_v;\n\n //\n struct ggml_context * ctx_w;\n\n std::vector backends;\n std::vector buffers_w;\n ggml_backend_buffer_t buffer_kv;\n ggml_backend_buffer_t buffer_input;\n\n std::map tensors;\n\n // inputs/constants\n struct ggml_tensor * embd;\n struct ggml_tensor * position;\n};\n\nvoid init_backends(gpt2_model & model, const gpt_params & params) {\n ggml_backend_t gpu_backend = NULL;\n\n ggml_log_set(ggml_log_callback_default, nullptr);\n\n // initialize the backends\n#ifdef GGML_USE_CUDA\n if (params.n_gpu_layers > 0) {\n fprintf(stderr, \"%s: using CUDA backend\\n\", __func__);\n gpu_backend = ggml_backend_cuda_init(0);\n if (!gpu_backend) {\n fprintf(stderr, \"%s: ggml_backend_cuda_init() failed\\n\", __func__);\n }\n }\n#endif\n\n#ifdef GGML_USE_METAL\n if (params.n_gpu_layers > 0) {\n fprintf(stderr, \"%s: using Metal backend\\n\", __func__);\n gpu_backend = ggml_backend_metal_init();\n if (!gpu_backend) {\n fprintf(stderr, \"%s: ggml_backend_metal_init() failed\\n\", __func__);\n }\n }\n#endif\n if (gpu_backend) {\n model.backends.push_back(gpu_backend);\n }\n\n#ifdef GGML_USE_BLAS\n ggml_backend_t blas_backend = ggml_backend_blas_init();\n if (!blas_backend) {\n fprintf(stderr, \"%s: failed to initialize BLAS backend\\n\", __func__);\n } else {\n ggml_backend_blas_set_n_threads(blas_backend, params.n_threads);\n model.backends.push_back(blas_backend);\n }\n#endif\n\n // always add the CPU backend as a fallback\n ggml_backend_t cpu_backend = ggml_backend_cpu_init();\n ggml_backend_cpu_set_n_threads(cpu_backend, params.n_threads);\n model.backends.push_back(cpu_backend);\n}\n\n// load the model's weights from a file\nbool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab, const gpt_params & params) {\n printf(\"%s: loading model from '%s'\\n\", __func__, fname.c_str());\n\n auto fin = std::ifstream(fname, std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"%s: failed to open '%s'\\n\", __func__, fname.c_str());\n return false;\n }\n\n // verify magic\n {\n uint32_t magic;\n fin.read((char *) &magic, sizeof(magic));\n if (magic != GGML_FILE_MAGIC) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, fname.c_str());\n return false;\n }\n }\n\n // load hparams\n {\n auto & hparams = model.hparams;\n\n fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));\n fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));\n\n const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;\n\n printf(\"%s: n_vocab = %d\\n\", __func__, hparams.n_vocab);\n printf(\"%s: n_ctx = %d\\n\", __func__, hparams.n_ctx);\n printf(\"%s: n_embd = %d\\n\", __func__, hparams.n_embd);\n printf(\"%s: n_head = %d\\n\", __func__, hparams.n_head);\n printf(\"%s: n_layer = %d\\n\", __func__, hparams.n_layer);\n printf(\"%s: ftype = %d\\n\", __func__, hparams.ftype);\n printf(\"%s: qntvr = %d\\n\", __func__, qntvr);\n\n hparams.ftype %= GGML_QNT_VERSION_FACTOR;\n }\n\n // load vocab\n {\n int32_t n_vocab = 0;\n fin.read((char *) &n_vocab, sizeof(n_vocab));\n\n if (n_vocab != model.hparams.n_vocab) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad vocab size %d != %d)\\n\",\n __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);\n return false;\n }\n\n std::string word;\n std::vector buf(128);\n\n for (int i = 0; i < n_vocab; i++) {\n uint32_t len;\n fin.read((char *) &len, sizeof(len));\n\n buf.resize(len);\n fin.read((char *) buf.data(), len);\n word.assign(buf.data(), len);\n\n vocab.token_to_id[word] = i;\n vocab.id_to_token[i] = word;\n }\n }\n\n // for the big tensors, we have the option to store the data in 16-bit floats or quantized\n // in order to save memory and also to speed up the computation\n ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));\n if (wtype == GGML_TYPE_COUNT) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad ftype value %d)\\n\",\n __func__, fname.c_str(), model.hparams.ftype);\n return false;\n }\n\n auto & ctx = model.ctx_w;\n\n // create the ggml context\n {\n size_t n_tensors = 3 /* input */ + 2 /* kv */ + 6 + 12*model.hparams.n_layer;\n struct ggml_init_params params = {\n /*.mem_size =*/ ggml_tensor_overhead() * n_tensors,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n\n model.ctx_w = ggml_init(params);\n if (!model.ctx_w) {\n fprintf(stderr, \"%s: ggml_init() failed\\n\", __func__);\n return false;\n }\n }\n\n // create tensors for the weights\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_vocab = hparams.n_vocab;\n\n model.layers.resize(n_layer);\n\n model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);\n model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n\n // map by name\n model.tensors[\"model/ln_f/g\"] = model.ln_f_g;\n model.tensors[\"model/ln_f/b\"] = model.ln_f_b;\n\n model.tensors[\"model/wte\"] = model.wte;\n model.tensors[\"model/wpe\"] = model.wpe;\n model.tensors[\"model/lm_head\"] = model.lm_head;\n\n for (int i = 0; i < n_layer; ++i) {\n auto & layer = model.layers[i];\n\n layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);\n layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);\n\n layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);\n layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);\n layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);\n\n layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);\n layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n // map by name\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/g\"] = layer.ln_1_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_1/b\"] = layer.ln_1_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/g\"] = layer.ln_2_g;\n model.tensors[\"model/h\" + std::to_string(i) + \"/ln_2/b\"] = layer.ln_2_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/w\"] = layer.c_attn_attn_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_attn/b\"] = layer.c_attn_attn_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/w\"] = layer.c_attn_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/attn/c_proj/b\"] = layer.c_attn_proj_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/w\"] = layer.c_mlp_fc_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_fc/b\"] = layer.c_mlp_fc_b;\n\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/w\"] = layer.c_mlp_proj_w;\n model.tensors[\"model/h\" + std::to_string(i) + \"/mlp/c_proj/b\"] = layer.c_mlp_proj_b;\n }\n }\n\n // assign tensors to backends\n init_backends(model, params);\n ggml_backend_t backend_gpu = model.backends.front();\n ggml_backend_t backend_cpu = model.backends.back();\n std::map tensor_backends;\n {\n const int i_gpu_first_layer = model.hparams.n_layer - params.n_gpu_layers;\n for (auto it : model.tensors) {\n const std::string & name = it.first;\n // input tensors\n if (name == \"model/wte\" || name == \"model/wpe\") {\n if (params.n_gpu_layers > model.hparams.n_layer) {\n tensor_backends[name] = backend_gpu;\n } else {\n tensor_backends[name] = backend_cpu;\n }\n }\n // output tensors\n if (name == \"model/ln_f/g\" || name == \"model/ln_f/b\" || name == \"model/lm_head\") {\n if (params.n_gpu_layers > 0) {\n tensor_backends[name] = backend_gpu;\n } else {\n tensor_backends[name] = backend_cpu;\n }\n }\n // layer tensors\n if (name.substr(0, 7) == \"model/h\") {\n // parse layer number\n int layer = std::stoi(name.substr(7, 2));\n if (layer >= i_gpu_first_layer) {\n tensor_backends[name] = backend_gpu;\n } else {\n tensor_backends[name] = backend_cpu;\n }\n }\n }\n }\n\n // allocate buffers\n std::map backend_buffers;\n for (auto backend : model.backends) {\n // compute the size of the buffer\n size_t size = 0;\n for (auto it : model.tensors) {\n if (tensor_backends[it.first] == backend) {\n size += ggml_nbytes(it.second) + 512;\n }\n }\n if (size > 0) {\n printf(\"%s: %8s buffer size = %8.2f MB\\n\", __func__, ggml_backend_name(backend), size/1024.0/1024.0);\n // allocate the buffer\n ggml_backend_buffer_t buffer = ggml_backend_alloc_buffer(backend, size);\n ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);\n model.buffers_w.push_back(buffer);\n\n // create an allocator for the buffer to allocate the tensors\n auto alloc = ggml_tallocr_new(buffer);\n backend_buffers.insert(std::make_pair(backend, std::move(alloc)));\n } else {\n model.buffers_w.push_back(NULL);\n }\n }\n\n // allocate key + value memory\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n\n const int n_mem = n_layer*n_ctx;\n const int n_elements = n_embd*n_mem;\n\n model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);\n\n ggml_set_name(model.memory_k, \"model/memory_k\");\n ggml_set_name(model.memory_v, \"model/memory_v\");\n\n const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);\n\n printf(\"%s: memory size = %8.2f MB, n_mem = %d\\n\", __func__, memory_size/1024.0/1024.0, n_mem);\n\n // create a backend buffer (can be in host or device memory)\n ggml_backend_t backend_kv = params.n_gpu_layers >= hparams.n_layer/2 ? backend_gpu : backend_cpu;\n printf(\"%s: backend_kv = %s\\n\", __func__, ggml_backend_name(backend_kv));\n model.buffer_kv = ggml_backend_alloc_buffer(backend_kv, memory_size + 512*2);\n\n // allocate the tensors into the backend buffer\n {\n ggml_tallocr alloc = ggml_tallocr_new(model.buffer_kv);\n\n // this updates the pointers in the tensors to point to the correct location in the buffer\n // this is necessary since the ggml_context is .no_alloc == true\n // note that the buffer can actually be a device buffer, depending on the backend\n ggml_tallocr_alloc(&alloc, model.memory_k);\n ggml_tallocr_alloc(&alloc, model.memory_v);\n }\n }\n\n // load weights\n {\n size_t total_size = 0;\n\n bool has_lm_head = false;\n\n std::vector read_buf;\n\n while (true) {\n int32_t n_dims;\n int32_t length;\n int32_t ttype;\n\n fin.read(reinterpret_cast(&n_dims), sizeof(n_dims));\n fin.read(reinterpret_cast(&length), sizeof(length));\n fin.read(reinterpret_cast(&ttype), sizeof(ttype));\n\n if (fin.eof()) {\n break;\n }\n\n int32_t nelements = 1;\n int32_t ne[2] = { 1, 1 };\n for (int i = 0; i < n_dims; ++i) {\n fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i]));\n nelements *= ne[i];\n }\n\n std::string name(length, 0);\n fin.read(&name[0], length);\n\n if (model.tensors.find(name) == model.tensors.end()) {\n fprintf(stderr, \"%s: unknown tensor '%s' in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n auto tensor = model.tensors[name];\n ggml_set_name(tensor, name.c_str());\n if (ggml_nelements(tensor) != nelements) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {\n fprintf(stderr, \"%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\\n\",\n __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);\n return false;\n }\n\n // for debugging\n if (0) {\n printf(\"%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\\n\", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));\n }\n\n const size_t bpe = ggml_type_size(ggml_type(ttype));\n\n if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\\n\",\n __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);\n return false;\n }\n\n // allocate the tensor\n ggml_backend_t backend = tensor_backends[name];\n ggml_tallocr * alloc = &backend_buffers.find(backend)->second;\n ggml_tallocr_alloc(alloc, tensor);\n //printf(\"%s: [%5.5s] %s\\n\", __func__, ggml_backend_name(backend), name.c_str());\n\n if (ggml_backend_is_cpu(backend)\n#ifdef GGML_USE_METAL\n || ggml_backend_is_metal(backend)\n#endif\n ) {\n // for the CPU and Metal backend, we can read directly into the tensor\n fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor));\n } else {\n // read into a temporary buffer first, then copy to device memory\n read_buf.resize(ggml_nbytes(tensor));\n fin.read(read_buf.data(), ggml_nbytes(tensor));\n ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor));\n }\n\n // GPT-2 models share the WTE tensor as the LM head\n if (name == \"model/wte\" && has_lm_head == false) {\n ggml_tallocr * alloc_head = &backend_buffers.find(tensor_backends[\"model/lm_head\"])->second;\n ggml_tallocr_alloc(alloc_head, model.lm_head);\n //printf(\"%s: [%5.5s] %s (copied)\\n\", __func__, ggml_backend_name(tensor_backends[\"model/lm_head\"]), \"model/lm_head\");\n ggml_backend_tensor_copy(tensor, model.lm_head);\n total_size += ggml_nbytes(model.lm_head);\n }\n\n if (name == \"model/lm_head\") {\n has_lm_head = true;\n }\n\n total_size += ggml_nbytes(tensor);\n }\n printf(\"%s: model size = %8.2f MB\\n\", __func__, total_size/1024.0/1024.0);\n }\n\n fin.close();\n\n // allocate input tensors\n {\n model.embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, model.hparams.n_ctx);\n model.position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, model.hparams.n_ctx);\n\n ggml_set_name(model.embd, \"in/embd\");\n ggml_set_name(model.position, \"in/position\");\n\n // add input tensors to cpu backend\n size_t input_size = ggml_nbytes(model.embd) + ggml_nbytes(model.position);\n\n // FIXME: use cpu backend after sched impl\n ggml_backend_t backend_input = params.n_gpu_layers >= model.hparams.n_layer ? backend_gpu : backend_cpu;\n model.buffer_input = ggml_backend_alloc_buffer(backend_input, input_size + 512*3);\n printf(\"%s: backend_in = %s (%zu bytes)\\n\", __func__, ggml_backend_name(backend_input), input_size);\n\n // allocate the tensors into the backend buffer\n ggml_tallocr alloc = ggml_tallocr_new(model.buffer_input);\n ggml_tallocr_alloc(&alloc, model.embd);\n ggml_tallocr_alloc(&alloc, model.position);\n }\n\n return true;\n}\n\n// build the computation graph\nstruct ggml_cgraph * gpt2_graph(\n const gpt2_model & model,\n const int n_past,\n const std::vector & embd_inp) {\n const int N = embd_inp.size();\n\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_head = hparams.n_head;\n\n // since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data\n static size_t buf_size = ggml_tensor_overhead()*GPT2_MAX_NODES + ggml_graph_overhead_custom(GPT2_MAX_NODES, false);\n static std::vector buf(buf_size);\n\n struct ggml_init_params params = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ buf.data(),\n /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()\n };\n\n struct ggml_context * ctx = ggml_init(params);\n\n struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, GPT2_MAX_NODES, false);\n\n struct ggml_tensor * embd = ggml_view_1d(ctx, model.embd, N, 0);\n\n // set inputs\n // TODO: move to gpt2_eval\n ggml_backend_tensor_set(model.embd, embd_inp.data(), 0, N*ggml_element_size(embd));\n\n struct ggml_tensor * position = ggml_view_1d(ctx, model.position, N, 0);\n for (int i = 0; i < N; ++i) {\n int32_t v = n_past + i;\n ggml_backend_tensor_set(model.position, &v, i*sizeof(int32_t), sizeof(v));\n }\n\n const float KQ_scale = 1.0f/sqrtf(float(model.hparams.n_embd)/model.hparams.n_head);\n\n // wte + wpe\n struct ggml_tensor * inpL =\n ggml_add(ctx,\n ggml_get_rows(ctx, model.wte, embd),\n ggml_get_rows(ctx, model.wpe, position));\n ggml_set_name(inpL, \"inpL\");\n ggml_set_name(inpL->src[0], \"wte\");\n ggml_set_name(inpL->src[1], \"wpe\");\n\n for (int il = 0; il < n_layer; ++il) {\n struct ggml_tensor * cur;\n\n // norm\n {\n // [ 768, N]\n cur = ggml_norm(ctx, inpL, hparams.eps);\n ggml_format_name(cur, \"l%d.norm\", il);\n\n // cur = ln_1_g*cur + ln_1_b\n // [ 768, N]\n cur = ggml_add(ctx,\n ggml_mul(ctx,\n cur,\n model.layers[il].ln_1_g),\n model.layers[il].ln_1_b);\n ggml_format_name(cur, \"l%d.ln_1_b\", il);\n ggml_format_name(cur->src[0], \"l%d.ln_1_g\", il);\n }\n\n // attn\n // [2304, 768] - model.layers[il].c_attn_attn_w\n // [2304, 1] - model.layers[il].c_attn_attn_b\n // [ 768, N] - cur (in)\n // [2304, N] - cur (out)\n //\n // cur = attn_w*cur + attn_b\n // [2304, N]\n {\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_attn_attn_w,\n cur);\n ggml_format_name(cur, \"l%d.attn_w\", il);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_attn_attn_b);\n ggml_format_name(cur, \"l%d.attn_b\", il);\n }\n\n // self-attention\n {\n struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);\n struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);\n struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);\n\n ggml_format_name(Qcur, \"l%d.Qcur\", il);\n ggml_format_name(Kcur, \"l%d.Kcur\", il);\n ggml_format_name(Vcur, \"l%d.Vcur\", il);\n\n // store key and value to memory\n if (N >= 1) {\n struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));\n struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));\n\n ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));\n ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));\n }\n\n // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)\n // [64, N, 12]\n struct ggml_tensor * Q =\n ggml_permute(ctx,\n ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N),\n 0, 2, 1, 3);\n ggml_format_name(Q, \"l%d.Q\", il);\n\n // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)\n // [64, n_past + N, 12]\n struct ggml_tensor * K =\n ggml_permute(ctx,\n ggml_reshape_3d(ctx,\n ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),\n n_embd/n_head, n_head, n_past + N),\n 0, 2, 1, 3);\n ggml_format_name(K, \"l%d.K\", il);\n\n // GG: flash attention\n //struct ggml_tensor * V =\n // ggml_cpy(ctx0,\n // ggml_permute(ctx0,\n // ggml_reshape_3d(ctx0,\n // ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),\n // n_embd/n_head, n_head, n_past + N),\n // 1, 2, 0, 3),\n // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));\n\n //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);\n\n // K * Q\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);\n ggml_format_name(KQ, \"l%d.KQ\", il);\n\n // KQ_scaled = KQ / sqrt(n_embd/n_head)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_scaled = ggml_scale(ctx, KQ, KQ_scale);\n ggml_format_name(KQ_scaled, \"l%d.KQ_scaled\", il);\n\n // KQ_masked = mask_past(KQ_scaled)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past);\n ggml_format_name(KQ_masked, \"l%d.KQ_masked\", il);\n\n // KQ = soft_max(KQ_masked)\n // [n_past + N, N, 12]\n struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);\n ggml_format_name(KQ_soft_max, \"l%d.KQ_soft_max\", il);\n\n // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()\n // [n_past + N, 64, 12]\n struct ggml_tensor * V_trans =\n ggml_cont_3d(ctx,\n ggml_permute(ctx,\n ggml_reshape_3d(ctx,\n ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),\n n_embd/n_head, n_head, n_past + N),\n 1, 2, 0, 3),\n n_past + N, n_embd/n_head, n_head);\n\n // KQV = transpose(V) * KQ_soft_max\n // [64, N, 12]\n struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);\n ggml_format_name(KQV, \"l%d.KQV\", il);\n\n // KQV_merged = KQV.permute(0, 2, 1, 3)\n // [64, 12, N]\n struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);\n ggml_format_name(KQV_merged, \"l%d.KQV_merged\", il);\n\n // cur = KQV_merged.contiguous().view(n_embd, N)\n // [768, N]\n cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N);\n ggml_format_name(cur, \"l%d.KQV_merged_contiguous\", il);\n }\n\n // projection\n // [ 768, 768] - model.layers[il].c_attn_proj_w\n // [ 768, 1] - model.layers[il].c_attn_proj_b\n // [ 768, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n {\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_attn_proj_w,\n cur);\n ggml_format_name(cur, \"l%d.attn_proj_w\", il);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_attn_proj_b);\n ggml_format_name(cur, \"l%d.attn_proj_b\", il);\n }\n\n // add the input\n cur = ggml_add(ctx, cur, inpL);\n ggml_format_name(cur, \"l%d.add\", il);\n\n struct ggml_tensor * inpFF = cur;\n\n // feed-forward network\n {\n // norm\n {\n cur = ggml_norm(ctx, inpFF, hparams.eps);\n ggml_format_name(cur, \"l%d.FFnorm\", il);\n\n // cur = ln_2_g*cur + ln_2_b\n // [ 768, N]\n cur = ggml_add(ctx,\n ggml_mul(ctx,\n cur,\n model.layers[il].ln_2_g),\n model.layers[il].ln_2_b);\n ggml_format_name(cur, \"l%d.ln_2_b\", il);\n ggml_format_name(cur->src[0], \"l%d.ln_2_g\", il);\n }\n\n // fully connected\n // [3072, 768] - model.layers[il].c_mlp_fc_w\n // [3072, 1] - model.layers[il].c_mlp_fc_b\n // [ 768, N] - cur (in)\n // [3072, N] - cur (out)\n //\n // cur = fc_w*cur + fc_b\n // [3072, N]\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_mlp_fc_w,\n cur);\n ggml_format_name(cur, \"l%d.mlp_fc_w\", il);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_mlp_fc_b);\n ggml_format_name(cur, \"l%d.mlp_fc_b\", il);\n\n // GELU activation\n // [3072, N]\n cur = ggml_gelu(ctx, cur);\n ggml_format_name(cur, \"l%d.gelu\", il);\n\n // projection\n // [ 768, 3072] - model.layers[il].c_mlp_proj_w\n // [ 768, 1] - model.layers[il].c_mlp_proj_b\n // [3072, N] - cur (in)\n // [ 768, N] - cur (out)\n //\n // cur = proj_w*cur + proj_b\n // [768, N]\n cur = ggml_mul_mat(ctx,\n model.layers[il].c_mlp_proj_w,\n cur);\n ggml_format_name(cur, \"l%d.mlp_proj_w\", il);\n\n cur = ggml_add(ctx,\n cur,\n model.layers[il].c_mlp_proj_b);\n ggml_format_name(cur, \"l%d.mlp_proj_b\", il);\n }\n\n // input for next layer\n inpL = ggml_add(ctx, cur, inpFF);\n ggml_format_name(inpL, \"l%d.add2\", il);\n }\n\n // norm\n {\n // [ 768, N]\n inpL = ggml_norm(ctx, inpL, hparams.eps);\n ggml_format_name(inpL, \"out_norm\");\n\n // inpL = ln_f_g*inpL + ln_f_b\n // [ 768, N]\n inpL = ggml_add(ctx,\n ggml_mul(ctx,\n inpL,\n model.ln_f_g),\n model.ln_f_b);\n ggml_format_name(inpL, \"out_ln_f_b\");\n ggml_format_name(inpL->src[0], \"out_ln_f_g\");\n }\n\n // inpL = WTE * inpL\n // [ 768, 50257] - model.lm_head\n // [ 768, N] - inpL\n inpL = ggml_mul_mat(ctx, model.lm_head, inpL);\n ggml_format_name(inpL, \"out_lm_head\");\n\n // logits -> probs\n //inpL = ggml_soft_max(ctx0, inpL);\n\n ggml_build_forward_expand(gf, inpL);\n\n ggml_free(ctx);\n\n return gf;\n}\n\n// evaluate the transformer\n//\n// - model: the model\n// - sched: the backend scheduler\n// - n_past: the context size so far\n// - embd_inp: the embeddings of the tokens in the context\n// - embd_w: the predicted logits for the next token\n//\nbool gpt2_eval(\n const gpt2_model & model,\n ggml_backend_sched_t sched,\n const int n_past,\n const std::vector & embd_inp,\n std::vector & embd_w) {\n const int N = embd_inp.size();\n\n const auto & hparams = model.hparams;\n\n const int n_vocab = hparams.n_vocab;\n\n struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp);\n\n // run the computation\n ggml_backend_sched_reset(sched);\n ggml_backend_sched_graph_compute(sched, gf);\n\n //if (n_past%100 == 0) {\n // ggml_graph_print (&gf);\n // ggml_graph_dump_dot(&gf, NULL, \"gpt-2.dot\");\n //}\n\n // in this case, the output tensor is the last one in the graph\n struct ggml_tensor * inpL = ggml_graph_node(gf, -1);\n\n //embd_w.resize(n_vocab*N);\n //ggml_backend_tensor_get(inpL, embd_w.data(), 0, sizeof(float)*n_vocab*N);\n\n // return result just for the last token\n embd_w.resize(n_vocab);\n ggml_backend_tensor_get(inpL, embd_w.data(), (n_vocab*(N-1))*sizeof(float), sizeof(float)*n_vocab);\n\n return true;\n}\n\nint main(int argc, char ** argv) {\n ggml_time_init();\n\n const int64_t t_main_start_us = ggml_time_us();\n\n gpt_params params;\n params.model = \"models/gpt-2-117M/ggml-model.bin\";\n\n if (gpt_params_parse(argc, argv, params) == false) {\n return 1;\n }\n\n if (params.seed < 0) {\n params.seed = time(NULL);\n }\n\n printf(\"%s: seed = %d\\n\", __func__, params.seed);\n\n std::mt19937 rng(params.seed);\n if (params.prompt.empty()) {\n params.prompt = gpt_random_prompt(rng);\n }\n\n int64_t t_load_us = 0;\n\n gpt_vocab vocab;\n gpt2_model model;\n\n // load the model\n {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_model_load(params.model, model, vocab, params)) {\n fprintf(stderr, \"%s: failed to load model from '%s'\\n\", __func__, params.model.c_str());\n return 1;\n }\n\n t_load_us = ggml_time_us() - t_start_us;\n\n test_gpt_tokenizer(vocab, params.token_test);\n }\n\n // create the backend scheduler\n // the scheduler handles the allocation of the compute buffers and the scheduling of the computation between the different backends\n ggml_backend_sched_t sched;\n {\n // initialize the scheduler\n sched = ggml_backend_sched_new(model.backends.data(), NULL, model.backends.size(), GPT2_MAX_NODES, false, true);\n\n // create the worst case graph for memory usage estimation\n int n_tokens = std::min(model.hparams.n_ctx, params.n_batch);\n int n_past = model.hparams.n_ctx - n_tokens;\n struct ggml_cgraph * gf = gpt2_graph(model, n_past, std::vector(n_tokens, 0));\n\n ggml_backend_sched_reserve(sched, gf);\n\n\n // compute the required memory\n size_t mem_size = 0;\n for (size_t i = 0; i < model.backends.size(); i++) {\n size_t size = ggml_backend_sched_get_buffer_size(sched, model.backends[i]);\n if (size > 0) {\n mem_size += size;\n printf(\"%s: %8s compute buffer size = %8.2f MB\\n\", __func__, ggml_backend_name(model.backends[i]), size/1024.0/1024.0);\n //printf(\"%s: %8s compute buffer size = %zu bytes\\n\", __func__, ggml_backend_name(model.backends[i]), size);\n }\n }\n\n printf(\"%s: total compute buffer size: %.2f MB\\n\", __func__, mem_size/1024.0/1024.0);\n }\n\n int n_past = 0;\n\n int64_t t_sample_us = 0;\n int64_t t_predict_us = 0;\n\n std::vector logits;\n\n // tokenize the prompt\n std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt);\n\n params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());\n\n printf(\"%s: prompt: '%s'\\n\", __func__, params.prompt.c_str());\n printf(\"%s: number of tokens in prompt = %zu, first 8 tokens: \", __func__, embd_inp.size());\n for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {\n printf(\"%d \", embd_inp[i]);\n }\n printf(\"\\n\\n\");\n\n // submit the input prompt token-by-token\n // this reduces the memory usage during inference, at the cost of a bit of speed at the beginning\n std::vector embd;\n\n for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {\n // predict\n if (embd.size() > 0) {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_eval(model, sched, n_past, embd, logits)) {\n printf(\"Failed to predict\\n\");\n return 1;\n }\n\n t_predict_us += ggml_time_us() - t_start_us;\n }\n\n n_past += embd.size();\n embd.clear();\n\n if (i >= embd_inp.size()) {\n // sample next token\n const int top_k = params.top_k;\n const float top_p = params.top_p;\n const float temp = params.temp;\n\n const int n_vocab = model.hparams.n_vocab;\n\n gpt_vocab::id id = 0;\n\n {\n const int64_t t_start_sample_us = ggml_time_us();\n\n id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);\n\n t_sample_us += ggml_time_us() - t_start_sample_us;\n }\n\n // add it to the context\n embd.push_back(id);\n } else {\n // if here, it means we are still processing the input prompt\n for (size_t k = i; k < embd_inp.size(); k++) {\n embd.push_back(embd_inp[k]);\n if (int32_t(embd.size()) >= params.n_batch) {\n break;\n }\n }\n i += embd.size() - 1;\n }\n\n // display text\n for (auto id : embd) {\n printf(\"%s\", vocab.id_to_token[id].c_str());\n }\n fflush(stdout);\n\n // end of text token\n if (embd.back() == 50256) {\n break;\n }\n }\n\n // report timing\n {\n const int64_t t_main_end_us = ggml_time_us();\n\n printf(\"\\n\\n\");\n printf(\"%s: load time = %8.2f ms\\n\", __func__, t_load_us/1000.0f);\n printf(\"%s: sample time = %8.2f ms\\n\", __func__, t_sample_us/1000.0f);\n printf(\"%s: predict time = %8.2f ms / %.2f ms per token\\n\", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);\n printf(\"%s: total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n }\n\n ggml_free(model.ctx_w);\n\n ggml_backend_sched_free(sched);\n ggml_backend_buffer_free(model.buffer_kv);\n for (auto buf : model.buffers_w) {\n ggml_backend_buffer_free(buf);\n }\n for (auto backend : model.backends) {\n ggml_backend_free(backend);\n }\n\n return 0;\n}\n" }, { "path": "examples/gpt-2/quantize.cpp", "content": "#include \"ggml.h\"\n\n#include \"common.h\"\n#include \"common-ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n// default hparams (GPT-2 117M)\nstruct gpt2_hparams {\n int32_t n_vocab = 50257;\n int32_t n_ctx = 1024;\n int32_t n_embd = 768;\n int32_t n_head = 12;\n int32_t n_layer = 12;\n int32_t ftype = 1;\n};\n\n// quantize a model\nbool gpt2_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {\n gpt_vocab vocab;\n\n printf(\"%s: loading model from '%s'\\n\", __func__, fname_inp.c_str());\n\n auto finp = std::ifstream(fname_inp, std::ios::binary);\n if (!finp) {\n fprintf(stderr, \"%s: failed to open '%s' for reading\\n\", __func__, fname_inp.c_str());\n return false;\n }\n\n auto fout = std::ofstream(fname_out, std::ios::binary);\n if (!fout) {\n fprintf(stderr, \"%s: failed to open '%s' for writing\\n\", __func__, fname_out.c_str());\n return false;\n }\n\n // verify magic\n {\n uint32_t magic;\n finp.read((char *) &magic, sizeof(magic));\n if (magic != GGML_FILE_MAGIC) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, fname_inp.c_str());\n return false;\n }\n\n fout.write((char *) &magic, sizeof(magic));\n }\n\n gpt2_hparams hparams;\n\n // load hparams\n {\n finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n finp.read((char *) &hparams.n_head, sizeof(hparams.n_head));\n finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n finp.read((char *) &hparams.ftype, sizeof(hparams.ftype));\n\n const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;\n const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;\n\n printf(\"%s: n_vocab = %d\\n\", __func__, hparams.n_vocab);\n printf(\"%s: n_ctx = %d\\n\", __func__, hparams.n_ctx);\n printf(\"%s: n_embd = %d\\n\", __func__, hparams.n_embd);\n printf(\"%s: n_head = %d\\n\", __func__, hparams.n_head);\n printf(\"%s: n_layer = %d\\n\", __func__, hparams.n_layer);\n printf(\"%s: ftype (src) = %d\\n\", __func__, hparams.ftype);\n printf(\"%s: qntvr (src) = %d\\n\", __func__, qntvr_src);\n printf(\"%s: ftype (dst) = %d\\n\", __func__, ftype_dst);\n printf(\"%s: qntvr (dst) = %d\\n\", __func__, GGML_QNT_VERSION);\n\n fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n fout.write((char *) &hparams.n_head, sizeof(hparams.n_head));\n fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n fout.write((char *) &ftype_dst, sizeof(ftype_dst));\n }\n\n // load vocab\n {\n int32_t n_vocab = 0;\n finp.read ((char *) &n_vocab, sizeof(n_vocab));\n fout.write((char *) &n_vocab, sizeof(n_vocab));\n\n if (n_vocab != hparams.n_vocab) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad vocab size %d != %d)\\n\",\n __func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);\n return false;\n }\n\n std::string word;\n for (int i = 0; i < n_vocab; i++) {\n uint32_t len;\n finp.read ((char *) &len, sizeof(len));\n fout.write((char *) &len, sizeof(len));\n\n word.resize(len);\n finp.read ((char *) word.data(), len);\n fout.write((char *) word.data(), len);\n\n vocab.token_to_id[word] = i;\n vocab.id_to_token[i] = word;\n }\n }\n\n // regexes of tensor names to be quantized\n const std::vector to_quant = {\n \"model/wte\",\n \"model/lm_head\",\n \"model/h.*/attn/c_attn/w\",\n \"model/h.*/attn/c_proj/w\",\n \"model/h.*/mlp/c_fc/w\",\n \"model/h.*/mlp/c_proj/w\",\n };\n\n if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) {\n fprintf(stderr, \"%s: failed to quantize model '%s'\\n\", __func__, fname_inp.c_str());\n return false;\n }\n\n finp.close();\n fout.close();\n\n return true;\n}\n\n// usage:\n// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type\n//\nint main(int argc, char ** argv) {\n if (argc != 4) {\n fprintf(stderr, \"usage: %s model-f32.bin model-quant.bin type\\n\", argv[0]);\n ggml_print_ftypes(stderr);\n return 1;\n }\n\n // needed to initialize f16 tables\n {\n struct ggml_init_params params = { 0, NULL, false };\n struct ggml_context * ctx = ggml_init(params);\n ggml_free(ctx);\n }\n\n const std::string fname_inp = argv[1];\n const std::string fname_out = argv[2];\n\n const ggml_ftype ftype = ggml_parse_ftype(argv[3]);\n\n const int64_t t_main_start_us = ggml_time_us();\n\n int64_t t_quantize_us = 0;\n\n // load the model\n {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gpt2_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {\n fprintf(stderr, \"%s: failed to quantize model from '%s'\\n\", __func__, fname_inp.c_str());\n return 1;\n }\n\n t_quantize_us = ggml_time_us() - t_start_us;\n }\n\n // report timing\n {\n const int64_t t_main_end_us = ggml_time_us();\n\n printf(\"\\n\");\n printf(\"%s: quantize time = %8.2f ms\\n\", __func__, t_quantize_us/1000.0f);\n printf(\"%s: total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n }\n\n return 0;\n}\n" }, { "path": "examples/gpt-j/CMakeLists.txt", "content": "#\n# gpt-j\n\nset(TEST_TARGET gpt-j)\nadd_executable(${TEST_TARGET} main.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)\n\n#\n# gpt-j-quantize\n\nset(TEST_TARGET gpt-j-quantize)\nadd_executable(${TEST_TARGET} quantize.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)\n" }, { "path": "examples/gpt-j/README.md", "content": "# gpt-j\n\nLocal GPT-J inference on your computer using C/C++\n\nNo video card required. You just need to have 16 GB of RAM.\n\n## Motivation\n\nThe GPT-J 6B model is the open-source alternative to OpenAI's GPT-3. It's basically a neural network that allows you to\ngenerate coherent, human-like text given a certain context (prompt).\n\nThe GPT-J model is quite big - the compact version of the model uses 16-bit floating point representation of the weights\nand is still 12 GB big. This means that in order to run inference on your computer, you would need to have a video card\nwith at least 12 GB of video RAM. Alternatively, you can try to run the python implementations on the CPU, but that\nwould probably not be very efficient as they are primarily optimized for running on a GPU (or at least this is my guess -\nI don't have much experience with python).\n\nI wanted to try and run the model on my MacBook, so I decided to implement the model inference from scratch using my own\ncustom build tensor library. The tensor library (called [ggml](https://github.com/ggerganov/ggml), written in C) is in\nearly development stage, but it already allows me to run the GPT-J model.\n\nOn my 32GB MacBook M1 Pro, I achieve an inference speed of about `125 ms/token` or about ~6 words per second (1 word\ntypically consists of 1 or 2 tokens).\n\nHere is a sample run with prompt `int main(int argc, char ** argv) {`:\n\n```bash\n$ time ./bin/gpt-j -p \"int main(int argc, char ** argv) {\"\n\ngptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ...\ngptj_model_load: n_vocab = 50400\ngptj_model_load: n_ctx = 2048\ngptj_model_load: n_embd = 4096\ngptj_model_load: n_head = 16\ngptj_model_load: n_layer = 28\ngptj_model_load: n_rot = 64\ngptj_model_load: f16 = 1\ngptj_model_load: ggml ctx size = 13334.86 MB\ngptj_model_load: memory_size = 1792.00 MB, n_mem = 57344\ngptj_model_load: ................................... done\ngptj_model_load: model size = 11542.79 MB / num tensors = 285\nmain: number of tokens in prompt = 13\n\nint main(int argc, char ** argv) {\n (void)argc;\n (void)argv;\n\n {\n struct sockaddr_in addr;\n int addrlen;\n char * ip = \"192.168.1.4\";\n int i;\n\n if ( (addrlen = sizeof(addr)) == -1 )\n return -1;\n\n for (i = 0; i < 10; ++i) {\n addr.sin_family = AF_INET;\n addr.sin_addr.s_addr = inet_addr(ip);\n\nmain: mem per token = 16430420 bytes\nmain: load time = 6211.48 ms\nmain: sample time = 13.74 ms\nmain: predict time = 26420.34 ms / 124.62 ms per token\nmain: total time = 33035.37 ms\n\nreal\t0m33.171s\nuser\t3m32.269s\nsys 0m3.686s\n\n$\n```\n\nIt took ~6.2 seconds to load the model to memory. After that, it took ~26.4 seconds to generate 200 tokens of what\nlooks like to be the beginning of a networking program in C. Pretty cool!\n\nHere is another run, just for fun:\n\n```bash\ntime ./bin/gpt-j -n 500 -t 8 -p \"Ask HN: Inherited the worst code and tech team I have ever seen. How to fix it?\n\"\n\ngptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ...\ngptj_model_load: n_vocab = 50400\ngptj_model_load: n_ctx = 2048\ngptj_model_load: n_embd = 4096\ngptj_model_load: n_head = 16\ngptj_model_load: n_layer = 28\ngptj_model_load: n_rot = 64\ngptj_model_load: f16 = 1\ngptj_model_load: ggml ctx size = 13334.86 MB\ngptj_model_load: memory_size = 1792.00 MB, n_mem = 57344\ngptj_model_load: ................................... done\ngptj_model_load: model size = 11542.79 MB / num tensors = 285\nmain: number of tokens in prompt = 24\n\nAsk HN: Inherited the worst code and tech team I have ever seen. How to fix it?\n\nI've inherited a team with some very strange and un-documented practices, one of them is that they use an old custom\napplication with a very slow tech stack written in Python that the team doesn't want to touch but also doesn't want to\nthrow away as it has some \"legacy\" code in it.\n\nThe problem is, the tech stack is very very slow.\n\nThey have a single web server on a VM that is slow.\nThe server is a little bit busy (not very busy though) and they have a lot of processes (30+ that are constantly being\nspawned by the application)\nThey have an application that is single threaded and was written in Python and the team don't want to touch this, and\nthe application is very slow.\n\nMy task as a new member of the team is to fix this.\n\nI'm a senior dev on the team (3 years on the project) and have been told that I will take the lead on this task. I know\nnext to nothing about Python. So here is what I have so far.\n\nWhat I have done is I've been trying to debug the processes with the \"ps\" command. This way I can see what is running\nand where. From what I see, the application spawns 10 processes a minute and some of them are used for nothing.\n\nI have also started to look for the code. The application source is not in GitHub or any other repository, it is only on\nour internal GitLab.\n\nWhat I've found so far:\n\nThe application uses a custom SQLAlchemy implementation to interact with the data. I've looked at the source, it looks\nlike an object cache or something like that. But from what I've seen, the cache gets full every 20 minutes and then gets\ncleared with a special command.\n\nAnother strange thing is that the application creates a file for every entry in the database (even if the entry already\nexists). I've looked at the file to see if it contains something, but it seems to be a JSON file with lots of records.\n\nThe other strange thing is that I can only find the database tables in the GitLab repository and not the code. So I\ncan't really understand how the application is supposed to interact with the database.\n\nI also found a \"log\" directory, but the code is encrypted with AES. From what I've found, it is in\n\nmain: mem per token = 16430420 bytes\nmain: load time = 3900.10 ms\nmain: sample time = 32.58 ms\nmain: predict time = 68049.91 ms / 130.11 ms per token\nmain: total time = 73020.05 ms\n\nreal\t1m13.156s\nuser\t9m1.328s\nsys. 0m7.103s\n```\n\n## Implementation details\n\nThe high level implementation of the model is contained in the [main.cpp](main.cpp) file. The core computations are\nperformed by the [ggml](https://github.com/ggerganov/ggml/blob/master/include/ggml.h) library.\n\n\n#### Matrix multiplication\n\nThe most performance critical part of the implementation is of course the matrix multiplication routine. 99% of the time\nis spent here, so it was important to optimize this as much as possible.\n\nOn Arm64, I utilize the 128-bit NEON intrinsics for 16-bit floating point operations:\n\nhttps://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/src/ggml.c#L187-L243\n\nThese instructions allow each core to operate simultaneously on 64 16-bit floats. I'm no expert in SIMD, but after quite\nsome trials this was the most efficient code for dot product of a row and column that I could come up with. Combined\nwith the parallel computation on 8 CPU threads, I believe I'm close to the maximum performance that one could possibly\nget on the M1 CPU. Still, I'm curious to know if there is a more efficient way to implement this.\n\n\n#### Attempt to use the M1 GPU\n\nOne interesting property of the GPT-J transformer architecture is that it allows you to perform part of the inference in\nparallel - i.e. the Feed-forward network can be computed in parallel to the Self-attention layer:\n\nhttps://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/examples/gpt-j/main.cpp#L507-L531\n\nSo I thought why not try and bring in the M1 GPU to compute half of the neural network in parallel to the CPU and\npotentially gain some extra performance. Thanks to the M1's shared memory model, it was relatively easy to offload part\nof the computation to the GPU using Apple's [Metal Performance\nShaders](https://developer.apple.com/documentation/metalperformanceshaders). The GPU shares the host memory, so there is\nno need to copy the data back and forth as you would normally do with Cuda or OpenCL. The weight matrices are directly\navailable to be used by the GPU.\n\nHowever, to my surprise, using MPS together with the CPU did not lead to any performance improvement at all. My\nconclusion was that the 8-thread NEON CPU computation is already saturating the memory bandwidth of the M1 and since\nthe CPU and the GPU on the MacBook are sharing that bandwidth, it does not help to offload the computation to the GPU.\nAnother observation was that the MPS GPU matrix multiplication using 16-bit floats had the same performance as the\n8-thread NEON CPU implementation. Again, I explain this with a saturated memory channel. But of course, my explanation\ncould be totally wrong and somehow the implementation wasn't utilizing the resources correctly.\n\nIn the end, I decided to not use MPS or the GPU all together.\n\n### Zero memory allocations\n\nAnother property of my implementation is that it does not perform any memory allocations once the model is loaded into\nmemory. All required memory is allocated at the start of the program with a single `malloc` (technically 2 calls, but\nthat is not important).\n\n## Usage\n\nIf you want to give this a try and you are on Linux or Mac OS, simply follow these instructions:\n\n```bash\n# Download the ggml-compatible GPT-J 6B model (requires 12GB disk space)\n../examples/gpt-j/download-ggml-model.sh 6B\n\n# Run the inference (requires 16GB of CPU RAM)\n./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin -p \"This is an example\"\n\n# Input prompt through pipe and run the inference.\necho \"This is an example\" > prompt.txt\ncat prompt.txt | ./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin\n```\n\nTo run the `gpt-j` tool, you need the 12GB `ggml-model.bin` file which contains the GPT-J model in\n[ggml](https://github.com/ggerganov/ggml) compatible format. In the instructions above, the binary file\nis downloaded from my repository on Hugging Face using the [download-ggml-model.sh](download-ggml-model.sh) script.\nYou can also, download the file manually from this link:\n\nhttps://huggingface.co/ggerganov/ggml/tree/main\n\n---\n\nAlternatively, if you don't want to download the 12GB ggml model file, you can perform the conversion yourself using\npython.\n\nFirst, you need to download the full GPT-J model from here: https://huggingface.co/EleutherAI/gpt-j-6B\n\nNote that the full model is quite big - about 72 GB. After you download it, you need to convert it to ggml format using\nthe [convert-h5-to-ggml.py](convert-h5-to-ggml.py) script. This will generate the `ggml-model.bin` file, which you can\nthen use with the `gpt-j` program.\n\n\n## GPT-2\n\nI also implemented a tool for CPU inference using the smaller GPT-2 models. They have worse quality compared to GPT-J,\nbut are much faster to execute.\n\nFor example, the Small GPT-2 model is only 240 MB big and the inference speed on my MacBook is about 200 tokens/sec.\n\nFor more details, checkout the GPT-2 example here: [gpt-2](https://github.com/ggerganov/ggml/tree/master/examples/gpt-2)\n" }, { "path": "examples/gpt-j/convert-h5-to-ggml.py", "content": "# Convert GPT-J-6B h5 transformer model to ggml format\n#\n# Load the model using GPTJForCausalLM.\n# Iterate over all variables and write them to a binary file.\n#\n# For each variable, write the following:\n# - Number of dimensions (int)\n# - Name length (int)\n# - Dimensions (int[n_dims])\n# - Name (char[name_length])\n# - Data (float[n_dims])\n#\n# By default, the bigger matrices are converted to 16-bit floats.\n# This can be disabled by adding the \"use-f32\" CLI argument.\n#\n# At the start of the ggml file we write the model parameters\n# and vocabulary.\n#\n\nimport sys\nimport struct\nimport json\nimport torch\nimport numpy as np\n\nfrom transformers import GPTJForCausalLM\n\n# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py\ndef bytes_to_unicode():\n \"\"\"\n Returns list of utf-8 byte and a corresponding list of unicode strings.\n The reversible bpe codes work on unicode strings.\n This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.\n When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.\n This is a signficant percentage of your normal, say, 32K bpe vocab.\n To avoid that, we want lookup tables between utf-8 bytes and unicode strings.\n And avoids mapping to whitespace/control characters the bpe code barfs on.\n \"\"\"\n bs = list(range(ord(\"!\"), ord(\"~\")+1))+list(range(ord(\"¡\"), ord(\"¬\")+1))+list(range(ord(\"®\"), ord(\"ÿ\")+1))\n cs = bs[:]\n n = 0\n for b in range(2**8):\n if b not in bs:\n bs.append(b)\n cs.append(2**8+n)\n n += 1\n cs = [chr(n) for n in cs]\n return dict(zip(bs, cs))\n\nif len(sys.argv) < 3:\n print(\"Usage: convert-h5-to-ggml.py dir-model [use-f32]\\n\")\n print(\" ftype == 0 -> float32\")\n print(\" ftype == 1 -> float16\")\n sys.exit(1)\n\n# output in the same directory as the model\ndir_model = sys.argv[1]\nfname_out = sys.argv[1] + \"/ggml-model.bin\"\n\nwith open(dir_model + \"/vocab.json\", \"r\", encoding=\"utf-8\") as f:\n encoder = json.load(f)\n\nwith open(dir_model + \"/added_tokens.json\", \"r\", encoding=\"utf-8\") as f:\n encoder_added = json.load(f)\n\nwith open(dir_model + \"/config.json\", \"r\", encoding=\"utf-8\") as f:\n hparams = json.load(f)\n\n# possible data types\n# ftype == 0 -> float32\n# ftype == 1 -> float16\n#\n# map from ftype to string\nftype_str = [\"f32\", \"f16\"]\n\nftype = 1\nif len(sys.argv) > 2:\n ftype = int(sys.argv[2])\n if ftype < 0 or ftype > 1:\n print(\"Invalid ftype: \" + str(ftype))\n sys.exit(1)\n fname_out = sys.argv[1] + \"/ggml-model-\" + ftype_str[ftype] + \".bin\"\n\n\nmodel = GPTJForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True)\n#print (model)\n\nlist_vars = model.state_dict()\n#print (list_vars)\n\nfout = open(fname_out, \"wb\")\n\nfout.write(struct.pack(\"i\", 0x67676d6c)) # magic: ggml in hex\nfout.write(struct.pack(\"i\", hparams[\"vocab_size\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_positions\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_embd\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_head\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_layer\"]))\nfout.write(struct.pack(\"i\", hparams[\"rotary_dim\"]))\nfout.write(struct.pack(\"i\", ftype))\n\nbyte_encoder = bytes_to_unicode()\nbyte_decoder = {v:k for k, v in byte_encoder.items()}\n\nfout.write(struct.pack(\"i\", len(encoder) + len(encoder_added)))\n\nfor key in encoder:\n text = bytearray([byte_decoder[c] for c in key])\n fout.write(struct.pack(\"i\", len(text)))\n fout.write(text)\n\nfor key in encoder_added:\n text = bytearray([byte_decoder[c] for c in key])\n fout.write(struct.pack(\"i\", len(text)))\n fout.write(text)\n\nfor name in list_vars.keys():\n data = list_vars[name].squeeze().numpy()\n print(\"Processing variable: \" + name + \" with shape: \", data.shape)\n\n # we don't need these\n if name.endswith(\"attn.masked_bias\") or name.endswith(\".attn.bias\"):\n print(\" Skipping variable: \" + name)\n continue\n\n n_dims = len(data.shape);\n\n # ftype == 0 -> float32, ftype == 1 -> float16\n ftype_cur = 0;\n if ftype != 0:\n if name[-7:] == \".weight\" and n_dims == 2:\n print(\" Converting to float16\")\n data = data.astype(np.float16)\n ftype_cur = 1\n else:\n print(\" Converting to float32\")\n data = data.astype(np.float32)\n ftype_cur = 0\n else:\n if data.dtype != np.float32:\n print(\" Converting to float32\")\n data = data.astype(np.float32)\n ftype_cur = 0\n\n # for efficiency - transpose these matrices:\n # (note - with latest ggml this is no longer more efficient, so disabling it)\n # \"transformer.h.*.mlp.fc_in.weight\"\n # \"transformer.h.*.attn.out_proj.weight\"\n # \"transformer.h.*.attn.q_proj.weight\"\n # \"transformer.h.*.attn.k_proj.weight\"\n # \"transformer.h.*.attn.v_proj.weight\"\n #if name.endswith(\".mlp.fc_in.weight\") or \\\n # name.endswith(\".attn.out_proj.weight\") or \\\n # name.endswith(\".attn.q_proj.weight\") or \\\n # name.endswith(\".attn.k_proj.weight\") or \\\n # name.endswith(\".attn.v_proj.weight\"):\n # print(\" Transposing\")\n # data = data.transpose()\n\n # header\n str = name.encode('utf-8')\n fout.write(struct.pack(\"iii\", n_dims, len(str), ftype_cur))\n for i in range(n_dims):\n fout.write(struct.pack(\"i\", data.shape[n_dims - 1 - i]))\n fout.write(str);\n\n # data\n data.tofile(fout)\n\nfout.close()\n\nprint(\"Done. Output file: \" + fname_out)\nprint(\"\")\n" }, { "path": "examples/gpt-j/download-ggml-model.sh", "content": "#!/bin/bash\n\n# This script downloads GPT-J model files that have already been converted to ggml format.\n# This way you don't have to convert them yourself.\n#\n# If you want to download the original GPT-J model files, use the \"download-model.sh\" script instead.\n\n#src=\"https://ggml.ggerganov.com\"\n#pfx=\"ggml-model-gpt-j\"\n\nsrc=\"https://huggingface.co/ggerganov/ggml\"\npfx=\"resolve/main/ggml-model-gpt-j\"\n\nggml_path=$(dirname $(realpath $0))\n\n# GPT-J models\nmodels=( \"6B\" )\n\n# list available models\nfunction list_models {\n printf \"\\n\"\n printf \" Available models:\"\n for model in \"${models[@]}\"; do\n printf \" $model\"\n done\n printf \"\\n\\n\"\n}\n\nif [ \"$#\" -ne 1 ]; then\n printf \"Usage: $0 \\n\"\n list_models\n\n exit 1\nfi\n\nmodel=$1\n\nif [[ ! \" ${models[@]} \" =~ \" ${model} \" ]]; then\n printf \"Invalid model: $model\\n\"\n list_models\n\n exit 1\nfi\n\n# download ggml model\n\nprintf \"Downloading ggml model $model ...\\n\"\n\nmkdir -p models/gpt-j-$model\n\nif [ -x \"$(command -v wget)\" ]; then\n wget --quiet --show-progress -O models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin\nelif [ -x \"$(command -v curl)\" ]; then\n curl -L --output models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin\nelse\n printf \"Either wget or curl is required to download models.\\n\"\n exit 1\nfi\n\nif [ $? -ne 0 ]; then\n printf \"Failed to download ggml model $model \\n\"\n printf \"Please try again later or download the original GPT-J model files and convert them yourself.\\n\"\n exit 1\nfi\n\nprintf \"Done! Model '$model' saved in 'models/gpt-j-$model/ggml-model.bin'\\n\"\nprintf \"You can now use it like this:\\n\\n\"\nprintf \" $ ./bin/gpt-j -m models/gpt-j-$model/ggml-model.bin -p \\\"This is an example\\\"\\n\"\nprintf \"\\n\"\n" }, { "path": "examples/gpt-j/download-model.sh", "content": "#!/bin/bash\n\nprintf \"To obtain the GPT-J 6B model files, please visit: https://huggingface.co/EleutherAI/gpt-j-6B\\n\\n\"\n\nprintf \"The model is very big. For example, the reposirory above is 72GB in size.\\n\"\nprintf \"If you are sure that you want to clone it, simply run the following command:\\n\\n\"\n\nprintf \" $ git clone https://huggingface.co/EleutherAI/gpt-j-6B models/gpt-j-6B\\n\\n\"\n\nprintf \"Alternatively, use the 'download-ggml-model.sh' script to download a 12GB ggml version of the model.\\n\"\nprintf \"This version is enough to run inference using the ggml library.\\n\\n\"\n" }, { "path": "examples/gpt-j/main.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n\n#include \"common.h\"\n#include \"common-ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\n\n// default hparams (GPT-J 6B)\nstruct gptj_hparams {\n int32_t n_vocab = 50400;\n int32_t n_ctx = 2048;\n int32_t n_embd = 4096;\n int32_t n_head = 16;\n int32_t n_layer = 28;\n int32_t n_rot = 64;\n int32_t ftype = 1;\n float eps = 1e-5f;\n};\n\nstruct gptj_layer {\n // normalization\n struct ggml_tensor * ln_1_g;\n struct ggml_tensor * ln_1_b;\n\n // attention\n struct ggml_tensor * c_attn_q_proj_w;\n struct ggml_tensor * c_attn_k_proj_w;\n struct ggml_tensor * c_attn_v_proj_w;\n\n struct ggml_tensor * c_attn_proj_w;\n\n // ff\n struct ggml_tensor * c_mlp_fc_w;\n struct ggml_tensor * c_mlp_fc_b;\n\n struct ggml_tensor * c_mlp_proj_w;\n struct ggml_tensor * c_mlp_proj_b;\n};\n\nstruct gptj_model {\n gptj_hparams hparams;\n\n // normalization\n struct ggml_tensor * ln_f_g;\n struct ggml_tensor * ln_f_b;\n\n struct ggml_tensor * wte; // token embedding\n\n struct ggml_tensor * lmh_g; // language model head\n struct ggml_tensor * lmh_b; // language model bias\n\n std::vector layers;\n\n // key + value memory\n struct ggml_tensor * memory_k;\n struct ggml_tensor * memory_v;\n\n //\n struct ggml_context * ctx;\n std::map tensors;\n};\n\n// load the model's weights from a file\nbool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {\n printf(\"%s: loading model from '%s' - please wait ...\\n\", __func__, fname.c_str());\n\n auto fin = std::ifstream(fname, std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"%s: failed to open '%s'\\n\", __func__, fname.c_str());\n return false;\n }\n\n // verify magic\n {\n uint32_t magic;\n fin.read((char *) &magic, sizeof(magic));\n if (magic != GGML_FILE_MAGIC) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, fname.c_str());\n return false;\n }\n }\n\n // load hparams\n {\n auto & hparams = model.hparams;\n\n fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));\n fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n fin.read((char *) &hparams.n_rot, sizeof(hparams.n_rot));\n fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));\n\n const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;\n\n printf(\"%s: n_vocab = %d\\n\", __func__, hparams.n_vocab);\n printf(\"%s: n_ctx = %d\\n\", __func__, hparams.n_ctx);\n printf(\"%s: n_embd = %d\\n\", __func__, hparams.n_embd);\n printf(\"%s: n_head = %d\\n\", __func__, hparams.n_head);\n printf(\"%s: n_layer = %d\\n\", __func__, hparams.n_layer);\n printf(\"%s: n_rot = %d\\n\", __func__, hparams.n_rot);\n printf(\"%s: ftype = %d\\n\", __func__, hparams.ftype);\n printf(\"%s: qntvr = %d\\n\", __func__, qntvr);\n\n hparams.ftype %= GGML_QNT_VERSION_FACTOR;\n }\n\n // load vocab\n {\n int32_t n_vocab = 0;\n fin.read((char *) &n_vocab, sizeof(n_vocab));\n\n if (n_vocab != model.hparams.n_vocab) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad vocab size %d != %d)\\n\",\n __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);\n return false;\n }\n\n std::string word;\n std::vector buf(128);\n\n for (int i = 0; i < n_vocab; i++) {\n uint32_t len;\n fin.read((char *) &len, sizeof(len));\n\n buf.resize(len);\n fin.read((char *) buf.data(), len);\n word.assign(buf.data(), len);\n\n vocab.token_to_id[word] = i;\n vocab.id_to_token[i] = word;\n }\n }\n\n // for the big tensors, we have the option to store the data in 16-bit floats or quantized\n // in order to save memory and also to speed up the computation\n ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));\n if (wtype == GGML_TYPE_COUNT) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad ftype value %d)\\n\",\n __func__, fname.c_str(), model.hparams.ftype);\n return false;\n }\n\n auto & ctx = model.ctx;\n\n size_t ctx_size = 0;\n\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_vocab = hparams.n_vocab;\n\n ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g\n ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b\n\n ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // wte\n\n ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // lmh_g\n ctx_size += ggml_row_size(GGML_TYPE_F32, n_vocab); // lmh_b\n\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_q_proj_w\n ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_k_proj_w\n ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_v_proj_w\n\n ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w\n\n ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b\n\n ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w\n ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_mlp_proj_b\n\n ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_k\n ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_v\n\n ctx_size += (5 + 10*n_layer)*512; // object overhead\n\n printf(\"%s: ggml ctx size = %6.2f MB\\n\", __func__, ctx_size/(1024.0*1024.0));\n }\n\n // create the ggml context\n {\n struct ggml_init_params params = {\n /*.mem_size =*/ ctx_size,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ false,\n };\n\n model.ctx = ggml_init(params);\n if (!model.ctx) {\n fprintf(stderr, \"%s: ggml_init() failed\\n\", __func__);\n return false;\n }\n }\n\n // prepare memory for the weights\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_vocab = hparams.n_vocab;\n\n model.layers.resize(n_layer);\n\n model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n\n model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n model.lmh_g = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);\n model.lmh_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);\n\n // map by name\n model.tensors[\"transformer.wte.weight\"] = model.wte;\n\n model.tensors[\"transformer.ln_f.weight\"] = model.ln_f_g;\n model.tensors[\"transformer.ln_f.bias\"] = model.ln_f_b;\n\n model.tensors[\"lm_head.weight\"] = model.lmh_g;\n model.tensors[\"lm_head.bias\"] = model.lmh_b;\n\n for (int i = 0; i < n_layer; ++i) {\n auto & layer = model.layers[i];\n\n layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);\n layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);\n layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);\n\n layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);\n\n layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);\n layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);\n\n layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);\n layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);\n\n // map by name\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".ln_1.weight\"] = layer.ln_1_g;\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".ln_1.bias\"] = layer.ln_1_b;\n\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".attn.q_proj.weight\"] = layer.c_attn_q_proj_w;\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".attn.k_proj.weight\"] = layer.c_attn_k_proj_w;\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".attn.v_proj.weight\"] = layer.c_attn_v_proj_w;\n\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".attn.out_proj.weight\"] = layer.c_attn_proj_w;\n\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".mlp.fc_in.weight\"] = layer.c_mlp_fc_w;\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".mlp.fc_in.bias\"] = layer.c_mlp_fc_b;\n\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".mlp.fc_out.weight\"] = layer.c_mlp_proj_w;\n model.tensors[\"transformer.h.\" + std::to_string(i) + \".mlp.fc_out.bias\"] = layer.c_mlp_proj_b;\n }\n }\n\n // key + value memory\n {\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n\n const int n_mem = n_layer*n_ctx;\n const int n_elements = n_embd*n_mem;\n\n model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements);\n model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements);\n\n const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);\n\n printf(\"%s: memory_size = %8.2f MB, n_mem = %d\\n\", __func__, memory_size/1024.0/1024.0, n_mem);\n }\n\n // load weights\n {\n int n_tensors = 0;\n size_t total_size = 0;\n\n printf(\"%s: \", __func__);\n\n while (true) {\n int32_t n_dims;\n int32_t length;\n int32_t ttype;\n\n fin.read(reinterpret_cast(&n_dims), sizeof(n_dims));\n fin.read(reinterpret_cast(&length), sizeof(length));\n fin.read(reinterpret_cast(&ttype), sizeof(ttype));\n\n if (fin.eof()) {\n break;\n }\n\n int32_t nelements = 1;\n int32_t ne[2] = { 1, 1 };\n for (int i = 0; i < n_dims; ++i) {\n fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i]));\n nelements *= ne[i];\n }\n\n std::string name(length, 0);\n fin.read(&name[0], length);\n\n if (model.tensors.find(name) == model.tensors.end()) {\n fprintf(stderr, \"%s: unknown tensor '%s' in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n auto tensor = model.tensors[name];\n if (ggml_nelements(tensor) != nelements) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file\\n\", __func__, name.c_str());\n return false;\n }\n\n if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {\n fprintf(stderr, \"%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\\n\",\n __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);\n return false;\n }\n\n // for debugging\n if (0) {\n printf(\"%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\\n\", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));\n }\n\n const size_t bpe = ggml_type_size(ggml_type(ttype));\n\n if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\\n\",\n __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);\n return false;\n }\n\n fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor));\n\n //printf(\"%42s - [%5d, %5d], type = %6s, %6.2f MB\\n\", name.c_str(), ne[0], ne[1], ttype == 0 ? \"float\" : \"f16\", ggml_nbytes(tensor)/1024.0/1024.0);\n total_size += ggml_nbytes(tensor);\n if (++n_tensors % 8 == 0) {\n printf(\".\");\n fflush(stdout);\n }\n }\n\n printf(\" done\\n\");\n\n printf(\"%s: model size = %8.2f MB / num tensors = %d\\n\", __func__, total_size/1024.0/1024.0, n_tensors);\n }\n\n fin.close();\n\n return true;\n}\n\n// evaluate the transformer\n//\n// - model: the model\n// - n_threads: number of threads to use\n// - n_past: the context size so far\n// - embd_inp: the embeddings of the tokens in the context\n// - embd_w: the predicted logits for the next token\n//\n// The GPT-J model requires about 16MB of memory per input token.\n//\nbool gptj_eval(\n const gptj_model & model,\n const int n_threads,\n const int n_past,\n const std::vector & embd_inp,\n std::vector & embd_w,\n size_t & mem_per_token) {\n const int N = embd_inp.size();\n\n const auto & hparams = model.hparams;\n\n const int n_embd = hparams.n_embd;\n const int n_layer = hparams.n_layer;\n const int n_ctx = hparams.n_ctx;\n const int n_head = hparams.n_head;\n const int n_vocab = hparams.n_vocab;\n const int n_rot = hparams.n_rot;\n\n static size_t buf_size = 256u*1024*1024;\n static void * buf = malloc(buf_size);\n\n if (mem_per_token > 0 && mem_per_token*N > buf_size) {\n const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead\n //printf(\"\\n%s: reallocating buffer from %zu to %zu bytes\\n\", __func__, buf_size, buf_size_new);\n\n // reallocate\n buf_size = buf_size_new;\n buf = realloc(buf, buf_size);\n if (buf == nullptr) {\n fprintf(stderr, \"%s: failed to allocate %zu bytes\\n\", __func__, buf_size);\n return false;\n }\n }\n\n struct ggml_init_params params = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ buf,\n /*.no_alloc =*/ false,\n };\n\n struct ggml_context * ctx0 = ggml_init(params);\n struct ggml_cgraph * gf = ggml_new_graph(ctx0);\n\n // KQ_pos - contains the positions\n struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);\n int * data = (int *) KQ_pos->data;\n for (int i = 0; i < N; ++i) {\n data[i] = n_past + i;\n }\n\n struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);\n memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));\n\n // wte\n struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);\n\n for (int il = 0; il < n_layer; ++il) {\n struct ggml_tensor * cur;\n\n // norm\n {\n cur = ggml_norm(ctx0, inpL, hparams.eps);\n\n // cur = ln_1_g*cur + ln_1_b\n cur = ggml_add(ctx0,\n ggml_mul(ctx0,\n ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),\n cur),\n ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));\n }\n\n struct ggml_tensor * inpSA = cur;\n\n // self-attention\n {\n struct ggml_tensor * Qcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0);\n struct ggml_tensor * Kcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0);\n\n // store key and value to memory\n {\n struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));\n\n struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));\n struct ggml_tensor * v = ggml_view_2d(ctx0, model.memory_v, N, n_embd,\n ( n_ctx)*ggml_element_size(model.memory_v),\n (il*n_ctx)*ggml_element_size(model.memory_v)*n_embd + n_past*ggml_element_size(model.memory_v));\n\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));\n }\n\n // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)\n struct ggml_tensor * Q =\n ggml_permute(ctx0,\n Qcur,\n 0, 2, 1, 3);\n\n // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)\n struct ggml_tensor * K =\n ggml_permute(ctx0,\n ggml_reshape_3d(ctx0,\n ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),\n n_embd/n_head, n_head, n_past + N),\n 0, 2, 1, 3);\n\n // K * Q\n struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);\n\n // KQ_scaled = KQ / sqrt(n_embd/n_head)\n struct ggml_tensor * KQ_scaled =\n ggml_scale_inplace(ctx0,\n KQ,\n 1.0f/sqrt(float(n_embd)/n_head));\n\n // KQ_masked = mask_past(KQ_scaled)\n struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);\n\n // KQ = soft_max(KQ_masked)\n struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);\n\n // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()\n struct ggml_tensor * V =\n ggml_view_3d(ctx0, model.memory_v,\n n_past + N, n_embd/n_head, n_head,\n n_ctx*ggml_element_size(model.memory_v),\n n_ctx*ggml_element_size(model.memory_v)*n_embd/n_head,\n il*n_ctx*ggml_element_size(model.memory_v)*n_embd);\n\n // KQV = transpose(V) * KQ_soft_max\n struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);\n\n // KQV_merged = KQV.permute(0, 2, 1, 3)\n struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);\n\n // cur = KQV_merged.contiguous().view(n_embd, N)\n cur = ggml_cpy(ctx0,\n KQV_merged,\n ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));\n\n // projection (no bias)\n cur = ggml_mul_mat(ctx0,\n model.layers[il].c_attn_proj_w,\n cur);\n }\n\n struct ggml_tensor * inpFF = cur;\n\n // feed-forward network\n // this is independent of the self-attention result, so it could be done in parallel to the self-attention\n {\n // note here we pass inpSA instead of cur\n cur = ggml_mul_mat(ctx0,\n model.layers[il].c_mlp_fc_w,\n inpSA);\n\n cur = ggml_add(ctx0,\n ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),\n cur);\n\n // GELU activation\n cur = ggml_gelu(ctx0, cur);\n\n // projection\n // cur = proj_w*cur + proj_b\n cur = ggml_mul_mat(ctx0,\n model.layers[il].c_mlp_proj_w,\n cur);\n\n cur = ggml_add(ctx0,\n ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),\n cur);\n }\n\n // self-attention + FF\n cur = ggml_add(ctx0, cur, inpFF);\n\n // input for next layer\n inpL = ggml_add(ctx0, cur, inpL);\n }\n\n // norm\n {\n inpL = ggml_norm(ctx0, inpL, hparams.eps);\n\n // inpL = ln_f_g*inpL + ln_f_b\n inpL = ggml_add(ctx0,\n ggml_mul(ctx0,\n ggml_repeat(ctx0, model.ln_f_g, inpL),\n inpL),\n ggml_repeat(ctx0, model.ln_f_b, inpL));\n }\n\n // lm_head\n {\n inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);\n\n inpL = ggml_add(ctx0,\n ggml_repeat(ctx0, model.lmh_b, inpL),\n inpL);\n }\n\n // logits -> probs\n //inpL = ggml_soft_max_inplace(ctx0, inpL);\n\n // run the computation\n ggml_build_forward_expand(gf, inpL);\n ggml_graph_compute_with_ctx(ctx0, gf, n_threads);\n\n //if (n_past%100 == 0) {\n // ggml_graph_print (&gf);\n // ggml_graph_dump_dot(&gf, NULL, \"gpt-j.dot\");\n //}\n\n //embd_w.resize(n_vocab*N);\n //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);\n\n // return result for just the last token\n embd_w.resize(n_vocab);\n memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);\n\n if (mem_per_token == 0) {\n mem_per_token = ggml_used_mem(ctx0)/N;\n }\n //printf(\"used_mem = %zu\\n\", ggml_used_mem(ctx0));\n\n ggml_free(ctx0);\n\n return true;\n}\n\nint main(int argc, char ** argv) {\n ggml_time_init();\n\n const int64_t t_main_start_us = ggml_time_us();\n\n gpt_params params;\n params.model = \"models/gpt-j-6B/ggml-model.bin\";\n\n if (gpt_params_parse(argc, argv, params) == false) {\n return 1;\n }\n\n if (params.seed < 0) {\n params.seed = time(NULL);\n }\n\n printf(\"%s: seed = %d\\n\", __func__, params.seed);\n\n std::mt19937 rng(params.seed);\n if (params.prompt.empty()) {\n params.prompt = gpt_random_prompt(rng);\n }\n\n int64_t t_load_us = 0;\n\n gpt_vocab vocab;\n gptj_model model;\n\n // load the model\n {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gptj_model_load(params.model, model, vocab)) {\n fprintf(stderr, \"%s: failed to load model from '%s'\\n\", __func__, params.model.c_str());\n return 1;\n }\n\n t_load_us = ggml_time_us() - t_start_us;\n\n test_gpt_tokenizer(vocab, params.token_test);\n }\n\n int n_past = 0;\n\n int64_t t_sample_us = 0;\n int64_t t_predict_us = 0;\n\n std::vector logits;\n\n // tokenize the prompt\n std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt);\n\n params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());\n\n printf(\"%s: number of tokens in prompt = %zu\\n\", __func__, embd_inp.size());\n printf(\"\\n\");\n\n std::vector embd;\n\n // determine the required inference memory per token:\n size_t mem_per_token = 0;\n gptj_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);\n\n for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {\n // predict\n if (embd.size() > 0) {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gptj_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {\n printf(\"Failed to predict\\n\");\n return 1;\n }\n\n t_predict_us += ggml_time_us() - t_start_us;\n }\n\n n_past += embd.size();\n embd.clear();\n\n if (i >= embd_inp.size()) {\n // sample next token\n const int top_k = params.top_k;\n const float top_p = params.top_p;\n const float temp = params.temp;\n\n const int n_vocab = model.hparams.n_vocab;\n\n gpt_vocab::id id = 0;\n\n {\n const int64_t t_start_sample_us = ggml_time_us();\n\n id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);\n\n t_sample_us += ggml_time_us() - t_start_sample_us;\n }\n\n // add it to the context\n embd.push_back(id);\n } else {\n // if here, it means we are still processing the input prompt\n for (size_t k = i; k < embd_inp.size(); k++) {\n embd.push_back(embd_inp[k]);\n if (int32_t(embd.size()) > params.n_batch) {\n break;\n }\n }\n i += embd.size() - 1;\n }\n\n // display text\n for (auto id : embd) {\n printf(\"%s\", vocab.id_to_token[id].c_str());\n }\n fflush(stdout);\n\n // end of text token\n if (embd.back() == 50256) {\n break;\n }\n }\n\n // report timing\n {\n const int64_t t_main_end_us = ggml_time_us();\n\n printf(\"\\n\\n\");\n printf(\"%s: mem per token = %8zu bytes\\n\", __func__, mem_per_token);\n printf(\"%s: load time = %8.2f ms\\n\", __func__, t_load_us/1000.0f);\n printf(\"%s: sample time = %8.2f ms\\n\", __func__, t_sample_us/1000.0f);\n printf(\"%s: predict time = %8.2f ms / %.2f ms per token\\n\", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);\n printf(\"%s: total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n }\n\n ggml_free(model.ctx);\n\n return 0;\n}\n" }, { "path": "examples/gpt-j/quantize.cpp", "content": "#include \"ggml.h\"\n\n#include \"common.h\"\n#include \"common-ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n// default hparams (GPT-J 6B)\nstruct gptj_hparams {\n int32_t n_vocab = 50400;\n int32_t n_ctx = 2048;\n int32_t n_embd = 4096;\n int32_t n_head = 16;\n int32_t n_layer = 28;\n int32_t n_rot = 64;\n int32_t ftype = 1;\n};\n\n// quantize a model\nbool gptj_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {\n gpt_vocab vocab;\n\n printf(\"%s: loading model from '%s'\\n\", __func__, fname_inp.c_str());\n\n auto finp = std::ifstream(fname_inp, std::ios::binary);\n if (!finp) {\n fprintf(stderr, \"%s: failed to open '%s' for reading\\n\", __func__, fname_inp.c_str());\n return false;\n }\n\n auto fout = std::ofstream(fname_out, std::ios::binary);\n if (!fout) {\n fprintf(stderr, \"%s: failed to open '%s' for writing\\n\", __func__, fname_out.c_str());\n return false;\n }\n\n // verify magic\n {\n uint32_t magic;\n finp.read((char *) &magic, sizeof(magic));\n if (magic != GGML_FILE_MAGIC) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, fname_inp.c_str());\n return false;\n }\n\n fout.write((char *) &magic, sizeof(magic));\n }\n\n gptj_hparams hparams;\n\n // load hparams\n {\n finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n finp.read((char *) &hparams.n_head, sizeof(hparams.n_head));\n finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n finp.read((char *) &hparams.n_rot, sizeof(hparams.n_rot));\n finp.read((char *) &hparams.ftype, sizeof(hparams.ftype));\n\n const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;\n const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;\n\n printf(\"%s: n_vocab = %d\\n\", __func__, hparams.n_vocab);\n printf(\"%s: n_ctx = %d\\n\", __func__, hparams.n_ctx);\n printf(\"%s: n_embd = %d\\n\", __func__, hparams.n_embd);\n printf(\"%s: n_head = %d\\n\", __func__, hparams.n_head);\n printf(\"%s: n_layer = %d\\n\", __func__, hparams.n_layer);\n printf(\"%s: ftype (src) = %d\\n\", __func__, hparams.ftype);\n printf(\"%s: qntvr (src) = %d\\n\", __func__, qntvr_src);\n printf(\"%s: ftype (dst) = %d\\n\", __func__, ftype_dst);\n printf(\"%s: qntvr (dst) = %d\\n\", __func__, GGML_QNT_VERSION);\n\n fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));\n fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));\n fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd));\n fout.write((char *) &hparams.n_head, sizeof(hparams.n_head));\n fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer));\n fout.write((char *) &hparams.n_rot, sizeof(hparams.n_rot));\n fout.write((char *) &ftype_dst, sizeof(ftype_dst));\n }\n\n // load vocab\n {\n int32_t n_vocab = 0;\n finp.read ((char *) &n_vocab, sizeof(n_vocab));\n fout.write((char *) &n_vocab, sizeof(n_vocab));\n\n if (n_vocab != hparams.n_vocab) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad vocab size %d != %d)\\n\",\n __func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);\n return false;\n }\n\n std::string word;\n for (int i = 0; i < n_vocab; i++) {\n uint32_t len;\n finp.read ((char *) &len, sizeof(len));\n fout.write((char *) &len, sizeof(len));\n\n word.resize(len);\n finp.read ((char *) word.data(), len);\n fout.write((char *) word.data(), len);\n\n vocab.token_to_id[word] = i;\n vocab.id_to_token[i] = word;\n }\n }\n\n // regexes of tensor names to be quantized\n const std::vector to_quant = {\n \".*weight\",\n };\n\n if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) {\n fprintf(stderr, \"%s: failed to quantize model '%s'\\n\", __func__, fname_inp.c_str());\n return false;\n }\n\n finp.close();\n fout.close();\n\n return true;\n}\n\n// usage:\n// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type\n//\nint main(int argc, char ** argv) {\n if (argc != 4) {\n fprintf(stderr, \"usage: %s model-f32.bin model-quant.bin type\\n\", argv[0]);\n ggml_print_ftypes(stderr);\n return 1;\n }\n\n // needed to initialize f16 tables\n {\n struct ggml_init_params params = { 0, NULL, false };\n struct ggml_context * ctx = ggml_init(params);\n ggml_free(ctx);\n }\n\n const std::string fname_inp = argv[1];\n const std::string fname_out = argv[2];\n\n const ggml_ftype ftype = ggml_parse_ftype(argv[3]);\n\n const int64_t t_main_start_us = ggml_time_us();\n\n int64_t t_quantize_us = 0;\n\n // load the model\n {\n const int64_t t_start_us = ggml_time_us();\n\n if (!gptj_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {\n fprintf(stderr, \"%s: failed to quantize model from '%s'\\n\", __func__, fname_inp.c_str());\n return 1;\n }\n\n t_quantize_us = ggml_time_us() - t_start_us;\n }\n\n // report timing\n {\n const int64_t t_main_end_us = ggml_time_us();\n\n printf(\"\\n\");\n printf(\"%s: quantize time = %8.2f ms\\n\", __func__, t_quantize_us/1000.0f);\n printf(\"%s: total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n }\n\n return 0;\n}\n" }, { "path": "examples/magika/CMakeLists.txt", "content": "#\n# magika\n\nset(TEST_TARGET magika)\nadd_executable(${TEST_TARGET} main.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)\n\n#\n# For GPU offloading\n\nif (GGML_CUDA)\n add_compile_definitions(GGML_USE_CUDA)\nendif()\n\nif (GGML_METAL)\n add_compile_definitions(GGML_USE_METAL)\nendif()\n" }, { "path": "examples/magika/README.md", "content": "# Google Magika inference\n\nSimple example that shows how to use GGML for inference with the [Google Magika](https://github.com/google/magika) file type detection model.\n\n### Usage\n\n- Obtain the Magika model in H5 format\n - Pinned version: https://github.com/google/magika/blob/4460acb5d3f86807c3b53223229dee2afa50c025/assets_generation/models/standard_v1/model.h5\n- Use `convert.py` to convert the model to gguf format:\n```bash\n $ python examples/magika/convert.py /path/to/model.h5\n```\n- Invoke the program with the model file and a list of files to identify:\n```bash\n $ build/bin/magika model.h5.gguf examples/sam/example.jpg examples/magika/convert.py README.md src/ggml.c /bin/gcc write.exe jfk.wav\n examples/sam/example.jpg : jpeg (100.00%) pptx (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%)\n examples/magika/convert.py : python (99.99%) javascript (0.00%) txt (0.00%) asm (0.00%) scala (0.00%)\n README.md : markdown (100.00%) txt (0.00%) yaml (0.00%) ppt (0.00%) shell (0.00%)\n src/ggml.c : c (99.95%) txt (0.04%) asm (0.01%) yaml (0.00%) html (0.00%)\n /bin/gcc : elf (99.98%) odex (0.02%) pptx (0.00%) smali (0.00%) shell (0.00%)\n write.exe : pebin (100.00%) ppt (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%)\n jfk.wav : wav (100.00%) ppt (0.00%) shell (0.00%) sevenzip (0.00%) scala (0.00%)\n```\n" }, { "path": "examples/magika/convert.py", "content": "import sys\nfrom tensorflow import keras\nimport gguf\n\ndef convert(model_name):\n model = keras.models.load_model(model_name, compile=False)\n gguf_model_name = model_name + \".gguf\"\n gguf_writer = gguf.GGUFWriter(gguf_model_name, \"magika\")\n\n for layer in model.layers:\n # export layers with weights\n if layer.weights:\n for weight in layer.weights:\n print(f\" [{weight.name}] {weight.shape} {weight.dtype}\")\n weight_data = weight.numpy()\n gguf_writer.add_tensor(weight.name, weight_data.T)\n\n\n gguf_writer.write_header_to_file()\n gguf_writer.write_kv_data_to_file()\n gguf_writer.write_tensors_to_file()\n gguf_writer.close()\n print(\"Model converted and saved to '{}'\".format(gguf_model_name))\n\n\nif __name__ == '__main__':\n if len(sys.argv) > 1:\n model_file = sys.argv[1]\n else:\n model_file = \"model.h5\"\n\n convert(model_file)\n" }, { "path": "examples/magika/main.cpp", "content": "#include \"ggml.h\"\n#include \"gguf.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\nstatic const char * magika_labels[] = {\n \"ai\", \"apk\", \"appleplist\", \"asm\", \"asp\",\n \"batch\", \"bmp\", \"bzip\", \"c\", \"cab\",\n \"cat\", \"chm\", \"coff\", \"crx\", \"cs\",\n \"css\", \"csv\", \"deb\", \"dex\", \"dmg\",\n \"doc\", \"docx\", \"elf\", \"emf\", \"eml\",\n \"epub\", \"flac\", \"gif\", \"go\", \"gzip\",\n \"hlp\", \"html\", \"ico\", \"ini\", \"internetshortcut\",\n \"iso\", \"jar\", \"java\", \"javabytecode\", \"javascript\",\n \"jpeg\", \"json\", \"latex\", \"lisp\", \"lnk\",\n \"m3u\", \"macho\", \"makefile\", \"markdown\", \"mht\",\n \"mp3\", \"mp4\", \"mscompress\", \"msi\", \"mum\",\n \"odex\", \"odp\", \"ods\", \"odt\", \"ogg\",\n \"outlook\", \"pcap\", \"pdf\", \"pebin\", \"pem\",\n \"perl\", \"php\", \"png\", \"postscript\", \"powershell\",\n \"ppt\", \"pptx\", \"python\", \"pythonbytecode\", \"rar\",\n \"rdf\", \"rpm\", \"rst\", \"rtf\", \"ruby\",\n \"rust\", \"scala\", \"sevenzip\", \"shell\", \"smali\",\n \"sql\", \"squashfs\", \"svg\", \"swf\", \"symlinktext\",\n \"tar\", \"tga\", \"tiff\", \"torrent\", \"ttf\",\n \"txt\", \"unknown\", \"vba\", \"wav\", \"webm\",\n \"webp\", \"winregistry\", \"wmf\", \"xar\", \"xls\",\n \"xlsb\", \"xlsx\", \"xml\", \"xpi\", \"xz\",\n \"yaml\", \"zip\", \"zlibstream\"\n};\n\nstruct magika_hparams {\n const int block_size = 4096;\n const int beg_size = 512;\n const int mid_size = 512;\n const int end_size = 512;\n const int min_file_size_for_dl = 16;\n const int n_label = 113;\n const float f_norm_eps = 0.001f;\n const int padding_token = 256;\n};\n\nstruct magika_model {\n ~magika_model() {\n ggml_backend_buffer_free(buf_w);\n ggml_backend_free(backend);\n ggml_free(ctx_w);\n }\n\n magika_hparams hparams;\n\n struct ggml_tensor * dense_w;\n struct ggml_tensor * dense_b;\n\n struct ggml_tensor * layer_norm_gamma;\n struct ggml_tensor * layer_norm_beta;\n\n struct ggml_tensor * dense_1_w;\n struct ggml_tensor * dense_1_b;\n\n struct ggml_tensor * dense_2_w;\n struct ggml_tensor * dense_2_b;\n\n struct ggml_tensor * layer_norm_1_gamma;\n struct ggml_tensor * layer_norm_1_beta;\n\n struct ggml_tensor * target_label_w;\n struct ggml_tensor * target_label_b;\n\n ggml_backend_t backend = ggml_backend_cpu_init();\n ggml_backend_buffer_t buf_w = nullptr;\n struct ggml_context * ctx_w = nullptr;\n};\n\nstruct ggml_tensor * checked_get_tensor(struct ggml_context * ctx, const char * name) {\n struct ggml_tensor * tensor = ggml_get_tensor(ctx, name);\n if (!tensor) {\n fprintf(stderr, \"%s: tensor '%s' not found\\n\", __func__, name);\n throw std::runtime_error(\"ggml_get_tensor() failed\");\n }\n return tensor;\n}\n\nbool magika_model_load(const std::string & fname, magika_model & model) {\n auto & ctx = model.ctx_w;\n\n struct gguf_init_params params = {\n /*.no_alloc =*/ true,\n /*.ctx =*/ &ctx,\n };\n\n struct gguf_context * ctx_gguf = gguf_init_from_file(fname.c_str(), params);\n if (!ctx_gguf) {\n fprintf(stderr, \"%s: gguf_init_from_file() failed\\n\", __func__);\n return false;\n }\n\n model.buf_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend);\n if (!model.buf_w) {\n fprintf(stderr, \"%s: ggml_backend_alloc_ctx_tensors() failed\\n\", __func__);\n gguf_free(ctx_gguf);\n return false;\n }\n\n try {\n model.dense_w = checked_get_tensor(ctx, \"dense/kernel:0\");\n model.dense_b = checked_get_tensor(ctx, \"dense/bias:0\");\n\n model.layer_norm_gamma = checked_get_tensor(ctx, \"layer_normalization/gamma:0\");\n model.layer_norm_beta = checked_get_tensor(ctx, \"layer_normalization/beta:0\");\n\n model.dense_1_w = checked_get_tensor(ctx, \"dense_1/kernel:0\");\n model.dense_1_b = checked_get_tensor(ctx, \"dense_1/bias:0\");\n\n model.dense_2_w = checked_get_tensor(ctx, \"dense_2/kernel:0\");\n model.dense_2_b = checked_get_tensor(ctx, \"dense_2/bias:0\");\n\n model.layer_norm_1_gamma = checked_get_tensor(ctx, \"layer_normalization_1/gamma:0\");\n model.layer_norm_1_beta = checked_get_tensor(ctx, \"layer_normalization_1/beta:0\");\n\n model.target_label_w = checked_get_tensor(ctx, \"target_label/kernel:0\");\n model.target_label_b = checked_get_tensor(ctx, \"target_label/bias:0\");\n } catch (const std::exception & e) {\n fprintf(stderr, \"%s: %s\\n\", __func__, e.what());\n gguf_free(ctx_gguf);\n return false;\n }\n\n FILE * f = fopen(fname.c_str(), \"rb\");\n if (!f) {\n fprintf(stderr, \"%s: fopen() failed\\n\", __func__);\n gguf_free(ctx_gguf);\n return false;\n }\n\n const int n_tensors = gguf_get_n_tensors(ctx_gguf);\n\n for (int i = 0; i < n_tensors; i++) {\n const char * name = gguf_get_tensor_name(ctx_gguf, i);\n struct ggml_tensor * tensor = ggml_get_tensor(ctx, name);\n size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i);\n\n //printf(\"%-30s: [%3ld, %3ld, %3ld, %3ld] %s\\n\",\n // name,\n // tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3],\n // ggml_type_name(tensor->type));\n\n std::vector buf(ggml_nbytes(tensor));\n if (fseek(f, offs, SEEK_SET) != 0) {\n fprintf(stderr, \"%s: fseek() failed\\n\", __func__);\n gguf_free(ctx_gguf);\n fclose(f);\n return false;\n }\n\n if (fread(buf.data(), 1, buf.size(), f) != buf.size()) {\n fprintf(stderr, \"%s: fread() failed\\n\", __func__);\n gguf_free(ctx_gguf);\n fclose(f);\n return false;\n }\n\n ggml_backend_tensor_set(tensor, buf.data(), 0, buf.size());\n }\n\n fclose(f);\n\n gguf_free(ctx_gguf);\n\n return true;\n}\n\nstruct ggml_cgraph * magika_graph(\n const magika_model & model,\n const int n_files) {\n\n const auto & hparams = model.hparams;\n\n static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();\n static std::vector buf(buf_size);\n\n struct ggml_init_params params = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ buf.data(),\n /*.no_alloc =*/ true,\n };\n\n struct ggml_context * ctx = ggml_init(params);\n\n struct ggml_cgraph * gf = ggml_new_graph(ctx);\n\n struct ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 257, 1536, n_files); // one-hot\n ggml_set_name(input, \"input\");\n ggml_set_input(input);\n\n struct ggml_tensor * cur;\n\n // dense\n cur = ggml_mul_mat(ctx, model.dense_w, input);\n cur = ggml_add(ctx, cur, model.dense_b); // [128, 1536, n_files]\n cur = ggml_gelu(ctx, cur);\n\n // reshape\n cur = ggml_reshape_3d(ctx, cur, 512, 384, n_files); // [384, 512, n_files]\n cur = ggml_cont(ctx, ggml_transpose(ctx, cur));\n\n // layer normalization\n cur = ggml_norm(ctx, cur, hparams.f_norm_eps);\n cur = ggml_mul(ctx, cur, model.layer_norm_gamma); // [384, 512, n_files]\n cur = ggml_add(ctx, cur, model.layer_norm_beta); // [384, 512, n_files]\n\n // dense_1\n cur = ggml_cont(ctx, ggml_transpose(ctx, cur));\n cur = ggml_mul_mat(ctx, model.dense_1_w, cur);\n cur = ggml_add(ctx, cur, model.dense_1_b); // [256, 384, n_files]\n cur = ggml_gelu(ctx, cur);\n\n // dense_2\n cur = ggml_mul_mat(ctx, model.dense_2_w, cur);\n cur = ggml_add(ctx, cur, model.dense_2_b); // [256, 384, n_files]\n cur = ggml_gelu(ctx, cur);\n\n // global_max_pooling1d\n cur = ggml_cont(ctx, ggml_transpose(ctx, cur)); // [384, 256, n_files]\n cur = ggml_pool_1d(ctx, cur, GGML_OP_POOL_MAX, 384, 384, 0); // [1, 256, n_files]\n cur = ggml_reshape_2d(ctx, cur, 256, n_files); // [256, n_files]\n\n // layer normalization 1\n cur = ggml_norm(ctx, cur, hparams.f_norm_eps);\n cur = ggml_mul(ctx, cur, model.layer_norm_1_gamma); // [256, n_files]\n cur = ggml_add(ctx, cur, model.layer_norm_1_beta); // [256, n_files]\n\n // target_label\n cur = ggml_mul_mat(ctx, model.target_label_w, cur);\n cur = ggml_add(ctx, cur, model.target_label_b); // [n_label, n_files]\n cur = ggml_soft_max(ctx, cur); // [n_label, n_files]\n ggml_set_name(cur, \"target_label_probs\");\n ggml_set_output(cur);\n\n ggml_build_forward_expand(gf, cur);\n\n return gf;\n}\n\nbool magika_eval(\n struct magika_model & model,\n const std::vector & fnames) {\n\n const auto & hparams = model.hparams;\n\n static ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));\n\n struct ggml_cgraph * gf = magika_graph(model, fnames.size());\n\n if (!ggml_gallocr_alloc_graph(alloc, gf)) {\n fprintf(stderr, \"%s: ggml_gallocr_alloc_graph() failed\\n\", __func__);\n return false;\n }\n\n struct ggml_tensor * input = ggml_graph_get_tensor(gf, \"input\");\n\n for (size_t i = 0; i < fnames.size(); i++) {\n FILE * f = fopen(fnames[i].c_str(), \"rb\");\n if (!f) {\n fprintf(stderr, \"%s: fopen() failed\\n\", __func__);\n return false;\n }\n fseek(f, 0, SEEK_END);\n long fsize = ftell(f);\n\n // the buffer is padded with the padding_token if the file is smaller than the block size\n std::vector buf(1536, hparams.padding_token);\n std::vector read_buf(std::max(hparams.beg_size, std::max(hparams.mid_size, hparams.end_size)));\n\n // read beg\n fseek(f, 0, SEEK_SET);\n int n_read = fread(read_buf.data(), 1, hparams.beg_size, f);\n for (int j = 0; j < n_read; j++) {\n // pad at the end\n buf[j] = read_buf[j];\n }\n\n // read mid\n long mid_offs = std::max(0L, (fsize - hparams.mid_size) / 2);\n fseek(f, mid_offs, SEEK_SET);\n n_read = fread(read_buf.data(), 1, hparams.mid_size, f);\n for (int j = 0; j < n_read; j++) {\n // pad at both ends\n long mid_idx = hparams.beg_size + (hparams.mid_size / 2) - n_read / 2 + j;\n buf[mid_idx] = read_buf[j];\n }\n\n // read end\n long end_offs = std::max(0L, fsize - hparams.end_size);\n fseek(f, end_offs, SEEK_SET);\n n_read = fread(read_buf.data(), 1, hparams.end_size, f);\n for (int j = 0; j < n_read; j++) {\n // pad at the beginning\n int end_idx = hparams.beg_size + hparams.mid_size + hparams.end_size - n_read + j;\n buf[end_idx] = read_buf[j];\n }\n\n fclose(f);\n\n const size_t inp_bytes = hparams.beg_size + hparams.mid_size + hparams.end_size;\n\n // convert to one-hot\n std::vector one_hot(257*inp_bytes);\n for (size_t j = 0; j < inp_bytes; j++) {\n one_hot[257*j + buf[j]] = 1.0f;\n }\n\n ggml_backend_tensor_set(input, one_hot.data(), 257*inp_bytes*i*sizeof(float), 257*inp_bytes*sizeof(float));\n }\n\n if (ggml_backend_graph_compute(model.backend, gf) != GGML_STATUS_SUCCESS) {\n fprintf(stderr, \"%s: ggml_backend_graph_compute() failed\\n\", __func__);\n return false;\n }\n\n struct ggml_tensor * target_label_probs = ggml_graph_get_tensor(gf, \"target_label_probs\");\n\n // print probabilities for the top labels of each file\n for (size_t i = 0; i < fnames.size(); i++) {\n std::vector probs(hparams.n_label);\n ggml_backend_tensor_get(target_label_probs, probs.data(), hparams.n_label*i*sizeof(float), hparams.n_label*sizeof(float));\n\n // sort the probabilities\n std::vector idx(hparams.n_label);\n std::iota(idx.begin(), idx.end(), 0);\n std::sort(idx.begin(), idx.end(), [&probs](int i1, int i2) { return probs[i1] > probs[i2]; });\n\n // print the top labels\n const int top_n = 5;\n printf(\"%-30s: \", fnames[i].c_str());\n for (int j = 0; j < top_n; j++) {\n printf(\"%s (%.2f%%) \", magika_labels[idx[j]], probs[idx[j]]*100);\n }\n printf(\"\\n\");\n }\n\n return true;\n}\n\nint main(int argc, const char ** argv) {\n if (argc < 3) {\n fprintf(stderr, \"usage: %s [ ...]\\n\", argv[0]);\n return 1;\n }\n\n const char * model_fname = argv[1];\n std::vector fnames;\n for (int i = 2; i < argc; i++) {\n fnames.push_back(argv[i]);\n }\n\n magika_model model;\n if (!magika_model_load(model_fname, model)) {\n fprintf(stderr, \"magika_model_load() failed\\n\");\n return 1;\n }\n\n magika_eval(model, fnames);\n\n return 0;\n}\n" }, { "path": "examples/mnist/.gitignore", "content": "data/\n*.gguf\n*.ggml\n" }, { "path": "examples/mnist/CMakeLists.txt", "content": "#\n# mnist-common\n\nset(TEST_TARGET mnist-common)\nadd_library(${TEST_TARGET} STATIC mnist-common.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common)\n\n#\n# mnist-eval\n\nset(TEST_TARGET mnist-eval)\nadd_executable(${TEST_TARGET} mnist-eval.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)\n\n#\n# mnist-train\n\nset(TEST_TARGET mnist-train)\nadd_executable(${TEST_TARGET} mnist-train.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)\n\n\n#\n# mnist-wasm\nif (EMSCRIPTEN)\n set(TARGET mnist)\n\n add_executable(${TARGET} mnist-common.cpp)\n target_link_libraries(${TARGET} PRIVATE ggml ggml-cpu)\n\n set_target_properties(${TARGET} PROPERTIES LINK_FLAGS \" \\\n --bind \\\n -s FORCE_FILESYSTEM=1 \\\n -s USE_PTHREADS=1 \\\n -s PTHREAD_POOL_SIZE=10 \\\n -s ASSERTIONS=1 \\\n -s WASM=1 \\\n -s EXPORTED_RUNTIME_METHODS=\\\"['ccall', 'cwrap', 'setValue', 'getValue']\\\" \\\n -s EXPORTED_FUNCTIONS=\\\"['_wasm_eval','_wasm_random_digit','_malloc','_free']\\\" \\\n -s ALLOW_MEMORY_GROWTH=1 \\\n --preload-file ${CMAKE_CURRENT_SOURCE_DIR}/mnist-f32.gguf@/ \\\n --preload-file ${CMAKE_CURRENT_SOURCE_DIR}/t10k-images-idx3-ubyte@/ \\\n \")\n\n # Copy output to web directory\n add_custom_command(\n TARGET ${TARGET} POST_BUILD\n COMMAND ${CMAKE_COMMAND} -E copy\n ${CMAKE_BINARY_DIR}/bin/mnist.js\n ${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.js\n COMMAND ${CMAKE_COMMAND} -E copy\n ${CMAKE_BINARY_DIR}/bin/mnist.wasm\n ${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.wasm\n COMMAND ${CMAKE_COMMAND} -E copy\n ${CMAKE_BINARY_DIR}/bin/mnist.worker.js\n ${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.worker.js\n )\nendif()\n" }, { "path": "examples/mnist/README.md", "content": "# MNIST Examples for GGML\n\nThis directory contains simple examples of how to use GGML for training and inference using the [MNIST dataset](https://yann.lecun.com/exdb/mnist/).\nAll commands listed in this README assume the working directory to be `examples/mnist`.\nPlease note that training in GGML is a work-in-progress and not production ready.\n\n## Obtaining the data\n\nA description of the dataset can be found on [Yann LeCun's website](https://yann.lecun.com/exdb/mnist/).\nWhile it is also in principle possible to download the dataset from this website these downloads are frequently throttled and\nit is recommended to use [HuggingFace](https://huggingface.co/datasets/ylecun/mnist) instead.\nThe dataset will be downloaded automatically when running `mnist-train-fc.py`.\n\n## Fully connected network\n\nFor our first example we will train a fully connected network.\nTo train a fully connected model in PyTorch and save it as a GGUF file, run:\n\n```bash\n$ python3 mnist-train-fc.py mnist-fc-f32.gguf\n\n...\n\nTest loss: 0.066377+-0.010468, Test accuracy: 97.94+-0.14%\n\nModel tensors saved to mnist-fc-f32.gguf:\nfc1.weight (500, 784)\nfc1.bias (500,)\nfc2.weight (10, 500)\nfc2.bias (10,)\n```\n\nThe training script includes an evaluation of the model on the test set.\nTo evaluate the model on the CPU using GGML, run:\n\n```bash\n$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte\n\n________________________________________________________\n________________________________________________________\n________________________________________________________\n________________________________________________________\n__________________________________####__________________\n______________________________########__________________\n__________________________##########____________________\n______________________##############____________________\n____________________######________####__________________\n__________________________________####__________________\n__________________________________####__________________\n________________________________####____________________\n______________________________####______________________\n________________________##########______________________\n______________________########__####____________________\n________________________##__________##__________________\n____________________________________##__________________\n__________________________________##____________________\n__________________________________##____________________\n________________________________##______________________\n____________________________####________________________\n__________##____________######__________________________\n__________##############________________________________\n________________####____________________________________\n________________________________________________________\n________________________________________________________\n________________________________________________________\n________________________________________________________\nggml_cuda_init: GGML_CUDA_FORCE_MMQ: no\nggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no\nggml_cuda_init: found 1 CUDA devices:\n Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes\nmnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend\nmnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):\nmnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)\nmnist_model_init_from_file: loading model weights from 'mnist-fc-f32.gguf'\nmnist_model_init_from_file: model arch is mnist-fc\nmnist_model_init_from_file: successfully loaded weights from mnist-fc-f32.gguf\nmain: loaded model in 109.44 ms\nmnist_model_eval: model evaluation on 10000 images took 76.92 ms, 7.69 us/image\nmain: predicted digit is 3\nmain: test_loss=0.066379+-0.009101\nmain: test_acc=97.94+-0.14%\n```\n\nIn addition to the evaluation on the test set the GGML evaluation also prints a random image from the test set as well as the model prediction for said image.\nTo train a fully connected model on the CPU using GGML run:\n\n``` bash\n$ ../../build/bin/mnist-train mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte\n```\n\nIt can then be evaluated with the same binary as above.\n\n## Convolutional network\n\nTo train a convolutional network using TensorFlow run:\n\n```bash\n$ python3 mnist-train-cnn.py mnist-cnn-f32.gguf\n\n...\n\nTest loss: 0.047947\nTest accuracy: 98.46%\nGGUF model saved to 'mnist-cnn-f32.gguf'\n```\n\nThe saved model can be evaluated on the CPU using the `mnist-eval` binary:\n\n```bash\n$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte\n\n________________________________________________________\n________________________________________________________\n________________________________________________________\n________________________________________________________\n________________________________________________________\n______________________________________##________________\n______________________________________##________________\n______________________________________##________________\n____________________________________##__________________\n__________________________________####__________________\n__________________________________##____________________\n________________________________##______________________\n______________________________##________________________\n____________________________####________________________\n____________________________##__________________________\n__________________________##____________________________\n________________________##______________________________\n______________________##________________________________\n____________________####________________________________\n____________________##__________________________________\n__________________##____________________________________\n________________##______________________________________\n________________________________________________________\n________________________________________________________\n________________________________________________________\n________________________________________________________\n________________________________________________________\n________________________________________________________\nggml_cuda_init: GGML_CUDA_FORCE_MMQ: no\nggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no\nggml_cuda_init: found 1 CUDA devices:\n Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes\nmnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend\nmnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):\nmnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)\nmnist_model_init_from_file: loading model weights from 'mnist-cnn-f32.gguf'\nmnist_model_init_from_file: model arch is mnist-cnn\nmnist_model_init_from_file: successfully loaded weights from mnist-cnn-f32.gguf\nmain: loaded model in 91.99 ms\nmnist_model_eval: model evaluation on 10000 images took 267.61 ms, 26.76 us/image\nmain: predicted digit is 1\nmain: test_loss=0.047955+-0.007029\nmain: test_acc=98.46+-0.12%\n```\n\nLike with the fully connected network the convolutional network can also be trained using GGML:\n\n``` bash\n$ ../../build/bin/mnist-train mnist-cnn mnist-cnn-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte\n```\n\nAs always, the evaluation is done using `mnist-eval` and like with the fully connected network the GGML graph is exported to `mnist-cnn-f32.ggml`.\n\n## Hardware Acceleration\n\nBoth the training and evaluation code is agnostic in terms of hardware as long as the corresponding GGML backend has implemented the necessary operations.\nA specific backend can be selected by appending the above commands with a backend name.\nThe compute graphs then schedule the operations to preferentially use the specified backend.\nNote that if a backend does not implement some of the necessary operations a CPU fallback is used instead which may result in bad performance.\n\n## Web demo\n\nThe evaluation code can be compiled to WebAssembly using [Emscripten](https://emscripten.org/) (may need to re-login to update `$PATH` after installation).\nFirst, copy the GGUF file of either of the trained models to `examples/mnist` and name it `mnist-f32.gguf`.\nCopy the test set to `examples/mnist` and name it `t10k-images-idx3-ubyte`.\nSymlinking these files will *not* work!\nCompile the code like so:\n\n```bash\n$ cd ../../\n$ mkdir -p build-em\n$ emcmake cmake .. -DGGML_BUILD_EXAMPLES=ON \\\n -DCMAKE_C_FLAGS=\"-pthread -matomics -mbulk-memory\" \\\n -DCMAKE_CXX_FLAGS=\"-pthread -matomics -mbulk-memory\"\n$ make mnist\n```\n\nThe compilation output is copied into `examples/mnist/web`.\nTo run it, you need an HTTP server.\nFor example:\n\n``` bash\n$ python3 examples/mnist/server.py\n\nServing directory '/home/danbev/work/ai/ggml/examples/mnist/web' at http://localhost:8000\nApplication context root: http://localhost:8000/\n```\n\nThe web demo can then be accessed via the link printed on the console.\nSimply draw a digit on the canvas and the model will try to predict what it's supposed to be.\nAlternatively, click the \"Random\" button to retrieve a random digit from the test set.\nBe aware that like all neural networks the one we trained is susceptible to distributional shift:\nif the numbers you draw look different than the ones in the training set\n(e.g. because they're not centered) the model will perform comparatively worse.\nAn online demo can be accessed [here](https://mnist.ggerganov.com).\n" }, { "path": "examples/mnist/mnist-common.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-opt.h\"\n\n#include \"mnist-common.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nbool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset) {\n auto fin = std::ifstream(fname, std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"failed to open images file %s\\n\", fname.c_str());\n return false;\n }\n fin.seekg(16);\n\n uint8_t image[MNIST_NINPUT];\n struct ggml_tensor * images = ggml_opt_dataset_data(dataset);\n float * buf = ggml_get_data_f32(images);\n\n GGML_ASSERT(images->ne[0] == MNIST_NINPUT);\n for (int64_t iex = 0; iex < images->ne[1]; ++iex) {\n fin.read((char *) image, sizeof(image));\n\n for (int64_t i = 0; i < MNIST_NINPUT; ++i) {\n buf[iex*MNIST_NINPUT + i] = image[i] / 255.0f; // Normalize to [0, 1]\n }\n }\n\n return true;\n}\n\nvoid mnist_image_print(FILE * stream, ggml_opt_dataset_t dataset, const int iex) {\n struct ggml_tensor * images = ggml_opt_dataset_data(dataset);\n GGML_ASSERT(images->ne[0] == MNIST_NINPUT);\n GGML_ASSERT(iex < images->ne[1]);\n const float * image = ggml_get_data_f32(images) + iex*MNIST_NINPUT;\n\n for (int64_t row = 0; row < MNIST_HW; row++) {\n for (int64_t col = 0; col < MNIST_HW; col++) {\n const int rgb = roundf(255.0f * image[row*MNIST_HW + col]);\n#ifdef _WIN32\n fprintf(stream, \"%s\", rgb >= 220 ? \"##\" : \"__\"); // Represented via text.\n#else\n fprintf(stream, \"\\033[48;2;%d;%d;%dm \\033[0m\", rgb, rgb, rgb); // Represented via colored blocks.\n#endif // _WIN32\n }\n fprintf(stream, \"\\n\");\n }\n}\n\nbool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset) {\n auto fin = std::ifstream(fname, std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"failed to open labels file %s\\n\", fname.c_str());\n return 0;\n }\n fin.seekg(8);\n\n uint8_t label;\n struct ggml_tensor * labels = ggml_opt_dataset_labels(dataset);\n float * buf = ggml_get_data_f32(labels);\n\n GGML_ASSERT(labels->ne[0] == MNIST_NCLASSES);\n for (int64_t iex = 0; iex < labels->ne[1]; ++iex) {\n fin.read((char *) &label, sizeof(label));\n\n for (int64_t i = 0; i < MNIST_NCLASSES; ++i) {\n buf[iex*MNIST_NCLASSES + i] = i == label ? 1.0f : 0.0f;\n }\n }\n\n return true;\n}\n\n// Temporary util function for loading data from GGUF to a backend != CPU until GGML itself provides this functionality:\nbool load_from_gguf(const char * fname, struct ggml_context * ctx_ggml, struct gguf_context * ctx_gguf) {\n FILE * f = ggml_fopen(fname, \"rb\");\n if (!f) {\n return false;\n }\n\n const size_t buf_size = 4*1024*1024;\n void * buf = malloc(buf_size);\n\n const int n_tensors = gguf_get_n_tensors(ctx_gguf);\n for (int i = 0; i < n_tensors; i++) {\n const char * name = gguf_get_tensor_name(ctx_gguf, i);\n\n struct ggml_tensor * tensor = ggml_get_tensor(ctx_ggml, name);\n if (!tensor) {\n continue;\n }\n\n const size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i);\n\n if (fseek(f, offs, SEEK_SET) != 0) {\n fclose(f);\n free(buf);\n return false;\n }\n\n const size_t nbytes = ggml_nbytes(tensor);\n for (size_t pos = 0; pos < nbytes; pos += buf_size) {\n const size_t nbytes_cpy = buf_size < nbytes - pos ? buf_size : nbytes - pos;\n\n if (fread(buf, 1, nbytes_cpy, f) != nbytes_cpy) {\n fclose(f);\n free(buf);\n return false;\n }\n\n ggml_backend_tensor_set(tensor, buf, pos, nbytes_cpy);\n }\n }\n\n fclose(f);\n free(buf);\n return true;\n}\n\nmnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {\n mnist_model model(backend, nbatch_logical, nbatch_physical);\n fprintf(stderr, \"%s: loading model weights from '%s'\\n\", __func__, fname.c_str());\n\n struct gguf_context * ctx;\n {\n struct gguf_init_params params = {\n /*.no_alloc =*/ true,\n /*.ctx =*/ &model.ctx_gguf,\n };\n ctx = gguf_init_from_file(fname.c_str(), params);\n if (!ctx) {\n fprintf(stderr, \"%s: gguf_init_from_file() failed\\n\", __func__);\n exit(1);\n }\n }\n model.arch = gguf_get_val_str(ctx, gguf_find_key(ctx, \"general.architecture\"));\n fprintf(stderr, \"%s: model arch is %s\\n\", __func__, model.arch.c_str());\n\n if (model.arch == \"mnist-fc\") {\n model.fc1_weight = ggml_get_tensor(model.ctx_gguf, \"fc1.weight\");\n GGML_ASSERT(model.fc1_weight->ne[0] == MNIST_NINPUT);\n GGML_ASSERT(model.fc1_weight->ne[1] == MNIST_NHIDDEN);\n GGML_ASSERT(model.fc1_weight->ne[2] == 1);\n GGML_ASSERT(model.fc1_weight->ne[3] == 1);\n\n model.fc1_bias = ggml_get_tensor(model.ctx_gguf, \"fc1.bias\");\n GGML_ASSERT(model.fc1_bias->ne[0] == MNIST_NHIDDEN);\n GGML_ASSERT(model.fc1_bias->ne[1] == 1);\n GGML_ASSERT(model.fc1_bias->ne[2] == 1);\n GGML_ASSERT(model.fc1_bias->ne[3] == 1);\n\n model.fc2_weight = ggml_get_tensor(model.ctx_gguf, \"fc2.weight\");\n GGML_ASSERT(model.fc2_weight->ne[0] == MNIST_NHIDDEN);\n GGML_ASSERT(model.fc2_weight->ne[1] == MNIST_NCLASSES);\n GGML_ASSERT(model.fc2_weight->ne[2] == 1);\n GGML_ASSERT(model.fc2_weight->ne[3] == 1);\n\n model.fc2_bias = ggml_get_tensor(model.ctx_gguf, \"fc2.bias\");\n GGML_ASSERT(model.fc2_bias->ne[0] == MNIST_NCLASSES);\n GGML_ASSERT(model.fc2_bias->ne[1] == 1);\n GGML_ASSERT(model.fc2_bias->ne[2] == 1);\n GGML_ASSERT(model.fc2_bias->ne[3] == 1);\n } else if (model.arch == \"mnist-cnn\") {\n model.conv1_kernel = ggml_get_tensor(model.ctx_gguf, \"conv1.kernel\");\n GGML_ASSERT(model.conv1_kernel->type == GGML_TYPE_F32);\n GGML_ASSERT(model.conv1_kernel->ne[0] == 3);\n GGML_ASSERT(model.conv1_kernel->ne[1] == 3);\n GGML_ASSERT(model.conv1_kernel->ne[2] == 1);\n GGML_ASSERT(model.conv1_kernel->ne[3] == MNIST_CNN_NCB);\n\n model.conv1_bias = ggml_get_tensor(model.ctx_gguf, \"conv1.bias\");\n GGML_ASSERT(model.conv1_bias->type == GGML_TYPE_F32);\n GGML_ASSERT(model.conv1_bias->ne[0] == 1);\n GGML_ASSERT(model.conv1_bias->ne[1] == 1);\n GGML_ASSERT(model.conv1_bias->ne[2] == MNIST_CNN_NCB);\n GGML_ASSERT(model.conv1_bias->ne[3] == 1);\n\n model.conv2_kernel = ggml_get_tensor(model.ctx_gguf, \"conv2.kernel\");\n GGML_ASSERT(model.conv2_kernel->type == GGML_TYPE_F32);\n GGML_ASSERT(model.conv2_kernel->ne[0] == 3);\n GGML_ASSERT(model.conv2_kernel->ne[1] == 3);\n GGML_ASSERT(model.conv2_kernel->ne[2] == MNIST_CNN_NCB);\n GGML_ASSERT(model.conv2_kernel->ne[3] == MNIST_CNN_NCB*2);\n\n model.conv2_bias = ggml_get_tensor(model.ctx_gguf, \"conv2.bias\");\n GGML_ASSERT(model.conv2_bias->type == GGML_TYPE_F32);\n GGML_ASSERT(model.conv2_bias->ne[0] == 1);\n GGML_ASSERT(model.conv2_bias->ne[1] == 1);\n GGML_ASSERT(model.conv2_bias->ne[2] == MNIST_CNN_NCB*2);\n GGML_ASSERT(model.conv2_bias->ne[3] == 1);\n\n model.dense_weight = ggml_get_tensor(model.ctx_gguf, \"dense.weight\");\n GGML_ASSERT(model.dense_weight->type == GGML_TYPE_F32);\n GGML_ASSERT(model.dense_weight->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));\n GGML_ASSERT(model.dense_weight->ne[1] == MNIST_NCLASSES);\n GGML_ASSERT(model.dense_weight->ne[2] == 1);\n GGML_ASSERT(model.dense_weight->ne[3] == 1);\n\n model.dense_bias = ggml_get_tensor(model.ctx_gguf, \"dense.bias\");\n GGML_ASSERT(model.dense_bias->type == GGML_TYPE_F32);\n GGML_ASSERT(model.dense_bias->ne[0] == MNIST_NCLASSES);\n GGML_ASSERT(model.dense_bias->ne[1] == 1);\n GGML_ASSERT(model.dense_bias->ne[2] == 1);\n GGML_ASSERT(model.dense_bias->ne[3] == 1);\n } else {\n fprintf(stderr, \"%s: unknown model arch: %s\\n\", __func__, model.arch.c_str());\n }\n\n model.buf_gguf = ggml_backend_alloc_ctx_tensors(model.ctx_gguf, model.backends[0]);\n\n if(!load_from_gguf(fname.c_str(), model.ctx_gguf, ctx)) {\n fprintf(stderr, \"%s: loading weights from %s failed\\n\", __func__, fname.c_str());\n exit(1);\n }\n\n // The space in ctx_gguf exactly fits the model weights,\n // the images (which also need to be statically allocated) need to be put in a different context.\n\n model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, nbatch_physical);\n\n ggml_set_name(model.images, \"images\");\n ggml_set_input(model.images);\n\n model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);\n\n fprintf(stderr, \"%s: successfully loaded weights from %s\\n\", __func__, fname.c_str());\n return model;\n}\n\nmnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {\n mnist_model model(backend, nbatch_logical, nbatch_physical);\n model.arch = arch;\n\n std::random_device rd{};\n std::mt19937 gen{rd()};\n std::normal_distribution nd{0.0f, 1e-2f};\n std::vector init_tensors;\n\n if (model.arch == \"mnist-fc\") {\n fprintf(stderr, \"%s: initializing random weights for a fully connected model\\n\", __func__);\n\n model.fc1_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NHIDDEN);\n model.fc1_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN);\n model.fc2_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN, MNIST_NCLASSES);\n model.fc2_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);\n\n ggml_set_name(model.fc1_weight, \"fc1.weight\");\n ggml_set_name(model.fc1_bias, \"fc1.bias\");\n ggml_set_name(model.fc2_weight, \"fc2.weight\");\n ggml_set_name(model.fc2_bias, \"fc2.bias\");\n\n init_tensors.push_back(model.fc1_weight);\n init_tensors.push_back(model.fc1_bias);\n init_tensors.push_back(model.fc2_weight);\n init_tensors.push_back(model.fc2_bias);\n } else if (model.arch == \"mnist-cnn\") {\n model.conv1_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, 1, MNIST_CNN_NCB);\n model.conv1_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB);\n model.conv2_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, MNIST_CNN_NCB, MNIST_CNN_NCB*2);\n model.conv2_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB*2);\n model.dense_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), MNIST_NCLASSES);\n model.dense_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);\n\n ggml_set_name(model.conv1_kernel, \"conv1.kernel\");\n ggml_set_name(model.conv1_bias, \"conv1.bias\");\n ggml_set_name(model.conv2_kernel, \"conv2.kernel\");\n ggml_set_name(model.conv2_bias, \"conv2.bias\");\n ggml_set_name(model.dense_weight, \"dense.weight\");\n ggml_set_name(model.dense_bias, \"dense.bias\");\n\n init_tensors.push_back(model.conv1_kernel);\n init_tensors.push_back(model.conv1_bias);\n init_tensors.push_back(model.conv2_kernel);\n init_tensors.push_back(model.conv2_bias);\n init_tensors.push_back(model.dense_weight);\n init_tensors.push_back(model.dense_bias);\n } else {\n fprintf(stderr, \"%s: unknown model arch: %s\\n\", __func__, model.arch.c_str());\n }\n\n model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NBATCH_PHYSICAL);\n ggml_set_name(model.images, \"images\");\n ggml_set_input(model.images);\n\n model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);\n\n for (ggml_tensor * t : init_tensors) {\n GGML_ASSERT(t->type == GGML_TYPE_F32);\n const int64_t ne = ggml_nelements(t);\n std::vector tmp(ne);\n\n for (int64_t i = 0; i < ne; ++i) {\n tmp[i] = nd(gen);\n }\n ggml_backend_tensor_set(t, tmp.data(), 0, ggml_nbytes(t));\n }\n\n return model;\n}\n\nvoid mnist_model_build(mnist_model & model) {\n if (model.arch == \"mnist-fc\") {\n ggml_set_param(model.fc1_weight);\n ggml_set_param(model.fc1_bias);\n ggml_set_param(model.fc2_weight);\n ggml_set_param(model.fc2_bias);\n\n ggml_tensor * fc1 = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,\n ggml_mul_mat(model.ctx_compute, model.fc1_weight, model.images),\n model.fc1_bias));\n model.logits = ggml_add(model.ctx_compute,\n ggml_mul_mat(model.ctx_compute, model.fc2_weight, fc1),\n model.fc2_bias);\n } else if (model.arch == \"mnist-cnn\") {\n ggml_set_param(model.conv1_kernel);\n ggml_set_param(model.conv1_bias);\n ggml_set_param(model.conv2_kernel);\n ggml_set_param(model.conv2_bias);\n ggml_set_param(model.dense_weight);\n ggml_set_param(model.dense_bias);\n\n struct ggml_tensor * images_2D = ggml_reshape_4d(model.ctx_compute, model.images, MNIST_HW, MNIST_HW, 1, model.images->ne[1]);\n\n struct ggml_tensor * conv1_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,\n ggml_conv_2d(model.ctx_compute, model.conv1_kernel, images_2D, 1, 1, 1, 1, 1, 1),\n model.conv1_bias));\n GGML_ASSERT(conv1_out->ne[0] == MNIST_HW);\n GGML_ASSERT(conv1_out->ne[1] == MNIST_HW);\n GGML_ASSERT(conv1_out->ne[2] == MNIST_CNN_NCB);\n GGML_ASSERT(conv1_out->ne[3] == model.nbatch_physical);\n\n struct ggml_tensor * conv2_in = ggml_pool_2d(model.ctx_compute, conv1_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);\n GGML_ASSERT(conv2_in->ne[0] == MNIST_HW/2);\n GGML_ASSERT(conv2_in->ne[1] == MNIST_HW/2);\n GGML_ASSERT(conv2_in->ne[2] == MNIST_CNN_NCB);\n GGML_ASSERT(conv2_in->ne[3] == model.nbatch_physical);\n\n struct ggml_tensor * conv2_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,\n ggml_conv_2d(model.ctx_compute, model.conv2_kernel, conv2_in, 1, 1, 1, 1, 1, 1),\n model.conv2_bias));\n GGML_ASSERT(conv2_out->ne[0] == MNIST_HW/2);\n GGML_ASSERT(conv2_out->ne[1] == MNIST_HW/2);\n GGML_ASSERT(conv2_out->ne[2] == MNIST_CNN_NCB*2);\n GGML_ASSERT(conv2_out->ne[3] == model.nbatch_physical);\n\n struct ggml_tensor * dense_in = ggml_pool_2d(model.ctx_compute, conv2_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);\n GGML_ASSERT(dense_in->ne[0] == MNIST_HW/4);\n GGML_ASSERT(dense_in->ne[1] == MNIST_HW/4);\n GGML_ASSERT(dense_in->ne[2] == MNIST_CNN_NCB*2);\n GGML_ASSERT(dense_in->ne[3] == model.nbatch_physical);\n\n dense_in = ggml_reshape_2d(model.ctx_compute,\n ggml_cont(model.ctx_compute, ggml_permute(model.ctx_compute, dense_in, 1, 2, 0, 3)),\n (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), model.nbatch_physical);\n GGML_ASSERT(dense_in->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));\n GGML_ASSERT(dense_in->ne[1] == model.nbatch_physical);\n GGML_ASSERT(dense_in->ne[2] == 1);\n GGML_ASSERT(dense_in->ne[3] == 1);\n\n model.logits = ggml_add(model.ctx_compute, ggml_mul_mat(model.ctx_compute, model.dense_weight, dense_in), model.dense_bias);\n } else {\n GGML_ASSERT(false);\n }\n\n ggml_set_name(model.logits, \"logits\");\n ggml_set_output(model.logits);\n GGML_ASSERT(model.logits->type == GGML_TYPE_F32);\n GGML_ASSERT(model.logits->ne[0] == MNIST_NCLASSES);\n GGML_ASSERT(model.logits->ne[1] == model.nbatch_physical);\n GGML_ASSERT(model.logits->ne[2] == 1);\n GGML_ASSERT(model.logits->ne[3] == 1);\n}\n\nggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset) {\n ggml_opt_result_t result = ggml_opt_result_init();\n\n ggml_opt_params params = ggml_opt_default_params(model.backend_sched, GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);\n params.ctx_compute = model.ctx_compute;\n params.inputs = model.images;\n params.outputs = model.logits;\n params.build_type = GGML_OPT_BUILD_TYPE_FORWARD;\n ggml_opt_context_t opt_ctx = ggml_opt_init(params);\n\n {\n const int64_t t_start_us = ggml_time_us();\n\n ggml_opt_epoch(opt_ctx, dataset, nullptr, result, /*idata_split =*/ 0, nullptr, nullptr);\n\n const int64_t t_total_us = ggml_time_us() - t_start_us;\n const double t_total_ms = 1e-3*t_total_us;\n const int nex = ggml_opt_dataset_data(dataset)->ne[1];\n fprintf(stderr, \"%s: model evaluation on %d images took %.2lf ms, %.2lf us/image\\n\",\n __func__, nex, t_total_ms, (double) t_total_us/nex);\n }\n\n ggml_opt_free(opt_ctx);\n\n return result;\n}\n\nvoid mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split) {\n ggml_opt_fit(model.backend_sched, model.ctx_compute, model.images, model.logits, dataset,\n GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, GGML_OPT_OPTIMIZER_TYPE_ADAMW, ggml_opt_get_default_optimizer_params, nepoch, model.nbatch_logical, val_split, false);\n}\n\nvoid mnist_model_save(mnist_model & model, const std::string & fname) {\n printf(\"%s: saving model to '%s'\\n\", __func__, fname.c_str());\n\n struct ggml_context * ggml_ctx;\n {\n struct ggml_init_params params = {\n /*.mem_size =*/ 100 * 1024*1024,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ false,\n };\n ggml_ctx = ggml_init(params);\n }\n\n gguf_context * gguf_ctx = gguf_init_empty();\n gguf_set_val_str(gguf_ctx, \"general.architecture\", model.arch.c_str());\n\n std::vector weights;\n if (model.arch == \"mnist-fc\") {\n weights = {model.fc1_weight, model.fc1_bias, model.fc2_weight, model.fc2_bias};\n } else if (model.arch == \"mnist-cnn\") {\n weights = {model.conv1_kernel, model.conv1_bias, model.conv2_kernel, model.conv2_bias, model.dense_weight, model.dense_bias};\n } else {\n GGML_ASSERT(false);\n }\n for (struct ggml_tensor * t : weights) {\n struct ggml_tensor * copy = ggml_dup_tensor(ggml_ctx, t);\n ggml_set_name(copy, t->name);\n ggml_backend_tensor_get(t, copy->data, 0, ggml_nbytes(t));\n gguf_add_tensor(gguf_ctx, copy);\n }\n gguf_write_to_file(gguf_ctx, fname.c_str(), false);\n\n ggml_free(ggml_ctx);\n gguf_free(gguf_ctx);\n}\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nint wasm_eval(uint8_t * digitPtr) {\n std::vector digit(digitPtr, digitPtr + MNIST_NINPUT);\n\n ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, 1, 1);\n struct ggml_tensor * data = ggml_opt_dataset_data(dataset);\n\n float * buf = ggml_get_data_f32(data);\n for (int i = 0; i < MNIST_NINPUT; ++i) {\n buf[i] = digitPtr[i] / 255.0f;\n }\n ggml_set_zero(ggml_opt_dataset_labels(dataset)); // The labels are not needed.\n\n mnist_model model = mnist_model_init_from_file(\"mnist-f32.gguf\", \"CPU\", /*nbatch_logical =*/ 1, /*nbatch_physical =*/ 1);\n mnist_model_build(model);\n ggml_opt_result_t result = mnist_model_eval(model, dataset);\n\n int32_t pred;\n ggml_opt_result_pred(result, &pred);\n\n return pred;\n}\n\nint wasm_random_digit(char * digitPtr) {\n auto fin = std::ifstream(\"t10k-images-idx3-ubyte\", std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"failed to open digits file\\n\");\n return 0;\n }\n srand(time(NULL));\n\n // Seek to a random digit: 16-byte header + 28*28 * (random 0 - 10000)\n fin.seekg(16 + MNIST_NINPUT * (rand() % MNIST_NTEST));\n fin.read(digitPtr, MNIST_NINPUT);\n\n return 1;\n}\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "examples/mnist/mnist-common.h", "content": "#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n#include \"ggml.h\"\n#include \"gguf.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-opt.h\"\n\n#define MNIST_NTRAIN 60000\n#define MNIST_NTEST 10000\n\n// Gradient accumulation can be achieved by setting the logical batch size to a multiple of the physical one.\n// The logical batch size determines how many datapoints are used for a gradient update.\n// The physical batch size determines how many datapoints are processed in parallel, larger values utilize compute better but need more memory.\n#define MNIST_NBATCH_LOGICAL 1000\n#define MNIST_NBATCH_PHYSICAL 500\n\nstatic_assert(MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL == 0, \"MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL != 0\");\nstatic_assert(MNIST_NTRAIN % MNIST_NBATCH_LOGICAL == 0, \"MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0\");\nstatic_assert(MNIST_NTEST % MNIST_NBATCH_LOGICAL == 0, \"MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0\");\n\n#define MNIST_HW 28\n#define MNIST_NINPUT (MNIST_HW*MNIST_HW)\n#define MNIST_NCLASSES 10\n\n#define MNIST_NHIDDEN 500\n\n// NCB = number of channels base\n#define MNIST_CNN_NCB 8\n\nstruct mnist_model {\n std::string arch;\n ggml_backend_sched_t backend_sched;\n std::vector backends;\n const int nbatch_logical;\n const int nbatch_physical;\n\n struct ggml_tensor * images = nullptr;\n struct ggml_tensor * logits = nullptr;\n\n struct ggml_tensor * fc1_weight = nullptr;\n struct ggml_tensor * fc1_bias = nullptr;\n struct ggml_tensor * fc2_weight = nullptr;\n struct ggml_tensor * fc2_bias = nullptr;\n\n struct ggml_tensor * conv1_kernel = nullptr;\n struct ggml_tensor * conv1_bias = nullptr;\n struct ggml_tensor * conv2_kernel = nullptr;\n struct ggml_tensor * conv2_bias = nullptr;\n struct ggml_tensor * dense_weight = nullptr;\n struct ggml_tensor * dense_bias = nullptr;\n\n struct ggml_context * ctx_gguf = nullptr;\n struct ggml_context * ctx_static = nullptr;\n struct ggml_context * ctx_compute = nullptr;\n ggml_backend_buffer_t buf_gguf = nullptr;\n ggml_backend_buffer_t buf_static = nullptr;\n\n mnist_model(const std::string & backend_name, const int nbatch_logical, const int nbatch_physical)\n : nbatch_logical(nbatch_logical), nbatch_physical(nbatch_physical) {\n std::vector devices;\n const int ncores_logical = std::thread::hardware_concurrency();\n const int nthreads = std::min(ncores_logical, (ncores_logical + 4) / 2);\n\n // Add primary backend:\n if (!backend_name.empty()) {\n ggml_backend_dev_t dev = ggml_backend_dev_by_name(backend_name.c_str());\n if (dev == nullptr) {\n fprintf(stderr, \"%s: ERROR: backend %s not found, available:\\n\", __func__, backend_name.c_str());\n for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {\n ggml_backend_dev_t dev_i = ggml_backend_dev_get(i);\n fprintf(stderr, \" - %s (%s)\\n\", ggml_backend_dev_name(dev_i), ggml_backend_dev_description(dev_i));\n }\n exit(1);\n }\n\n ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);\n GGML_ASSERT(backend);\n\n if (ggml_backend_is_cpu(backend)) {\n ggml_backend_cpu_set_n_threads(backend, nthreads);\n }\n\n backends.push_back(backend);\n devices.push_back(dev);\n }\n\n // Add all available backends as fallback.\n // A \"backend\" is a stream on a physical device so there is no problem with adding multiple backends for the same device.\n for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {\n ggml_backend_dev_t dev = ggml_backend_dev_get(i);\n\n ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);\n GGML_ASSERT(backend);\n\n if (ggml_backend_is_cpu(backend)) {\n ggml_backend_cpu_set_n_threads(backend, nthreads);\n }\n\n backends.push_back(backend);\n devices.push_back(dev);\n }\n\n // The order of the backends passed to ggml_backend_sched_new determines which backend is given priority.\n backend_sched = ggml_backend_sched_new(backends.data(), nullptr, backends.size(), GGML_DEFAULT_GRAPH_SIZE, false, true);\n fprintf(stderr, \"%s: using %s (%s) as primary backend\\n\",\n __func__, ggml_backend_name(backends[0]), ggml_backend_dev_description(devices[0]));\n if (backends.size() >= 2) {\n fprintf(stderr, \"%s: unsupported operations will be executed on the following fallback backends (in order of priority):\\n\", __func__);\n for (size_t i = 1; i < backends.size(); ++i) {\n fprintf(stderr, \"%s: - %s (%s)\\n\", __func__, ggml_backend_name(backends[i]), ggml_backend_dev_description(devices[i]));\n }\n }\n\n {\n const size_t size_meta = 1024*ggml_tensor_overhead();\n struct ggml_init_params params = {\n /*.mem_size =*/ size_meta,\n /*.mem_buffer =*/ nullptr,\n /*.no_alloc =*/ true,\n };\n ctx_static = ggml_init(params);\n }\n\n {\n // The compute context needs a total of 3 compute graphs: forward pass + backwards pass (with/without optimizer step).\n const size_t size_meta = GGML_DEFAULT_GRAPH_SIZE*ggml_tensor_overhead() + 3*ggml_graph_overhead();\n struct ggml_init_params params = {\n /*.mem_size =*/ size_meta,\n /*.mem_buffer =*/ nullptr,\n /*.no_alloc =*/ true,\n };\n ctx_compute = ggml_init(params);\n }\n }\n\n ~mnist_model() {\n ggml_free(ctx_gguf);\n ggml_free(ctx_static);\n ggml_free(ctx_compute);\n\n ggml_backend_buffer_free(buf_gguf);\n ggml_backend_buffer_free(buf_static);\n ggml_backend_sched_free(backend_sched);\n for (ggml_backend_t backend : backends) {\n ggml_backend_free(backend);\n }\n }\n};\n\nbool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset);\nvoid mnist_image_print(FILE * f, ggml_opt_dataset_t dataset, const int iex);\nbool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset);\n\nmnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical);\nmnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical);\nvoid mnist_model_build(mnist_model & model);\nggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset);\nvoid mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split);\nvoid mnist_model_save(mnist_model & model, const std::string & fname);\n" }, { "path": "examples/mnist/mnist-eval.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-opt.h\"\n\n#include \"mnist-common.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\nint main(int argc, char ** argv) {\n srand(time(NULL));\n ggml_time_init();\n\n if (argc != 4 && argc != 5) {\n fprintf(stderr, \"Usage: %s mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte [CPU/CUDA0]\\n\", argv[0]);\n exit(1);\n }\n\n ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTEST, MNIST_NBATCH_PHYSICAL);\n\n if (!mnist_image_load(argv[2], dataset)) {\n return 1;\n }\n if (!mnist_label_load(argv[3], dataset)) {\n return 1;\n }\n\n const int iex = rand() % MNIST_NTEST;\n mnist_image_print(stdout, dataset, iex);\n\n const std::string backend = argc >= 5 ? argv[4] : \"\";\n\n const int64_t t_start_us = ggml_time_us();\n mnist_model model = mnist_model_init_from_file(argv[1], backend, MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);\n mnist_model_build(model);\n const int64_t t_load_us = ggml_time_us() - t_start_us;\n fprintf(stdout, \"%s: loaded model in %.2lf ms\\n\", __func__, t_load_us / 1000.0);\n\n ggml_opt_result_t result_eval = mnist_model_eval(model, dataset);\n\n std::vector pred(MNIST_NTEST);\n ggml_opt_result_pred(result_eval, pred.data());\n fprintf(stdout, \"%s: predicted digit is %d\\n\", __func__, pred[iex]);\n\n double loss;\n double loss_unc;\n ggml_opt_result_loss(result_eval, &loss, &loss_unc);\n fprintf(stdout, \"%s: test_loss=%.6lf+-%.6lf\\n\", __func__, loss, loss_unc);\n\n double accuracy;\n double accuracy_unc;\n ggml_opt_result_accuracy(result_eval, &accuracy, &accuracy_unc);\n fprintf(stdout, \"%s: test_acc=%.2lf+-%.2lf%%\\n\", __func__, 100.0*accuracy, 100.0*accuracy_unc);\n\n ggml_opt_result_free(result_eval);\n\n return 0;\n}\n" }, { "path": "examples/mnist/mnist-train-cnn.py", "content": "#!/usr/bin/env python3\nimport sys\nfrom time import time\nimport gguf\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\n\ndef train(model_path):\n # Model / data parameters\n num_classes = 10\n input_shape = (28, 28, 1)\n\n # Load the data and split it between train and test sets\n (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n\n # Scale images to the [0, 1] range\n x_train = x_train.astype(\"float32\") / 255\n x_test = x_test.astype(\"float32\") / 255\n x_train = np.expand_dims(x_train, -1)\n x_test = np.expand_dims(x_test, -1)\n print(\"x_train shape:\", x_train.shape)\n print(x_train.shape[0], \"train samples\")\n print(x_test.shape[0], \"test samples\")\n\n # convert class vectors to binary class matrices\n y_train = keras.utils.to_categorical(y_train, num_classes)\n y_test = keras.utils.to_categorical(y_test, num_classes)\n\n model = keras.Sequential(\n [\n keras.Input(shape=input_shape, dtype=tf.float32),\n layers.Conv2D(8, kernel_size=(3, 3), padding=\"same\", activation=\"relu\", dtype=tf.float32),\n layers.MaxPooling2D(pool_size=(2, 2)),\n layers.Conv2D(16, kernel_size=(3, 3), padding=\"same\", activation=\"relu\", dtype=tf.float32),\n layers.MaxPooling2D(pool_size=(2, 2)),\n layers.Flatten(),\n layers.Dense(num_classes, activation=\"softmax\", dtype=tf.float32),\n ]\n )\n\n model.summary()\n batch_size = 1000\n epochs = 30\n model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n\n t_start = time()\n model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)\n print(f\"Training took {time()-t_start:.2f}s\")\n\n score = model.evaluate(x_test, y_test, verbose=0)\n print(f\"Test loss: {score[0]:.6f}\")\n print(f\"Test accuracy: {100*score[1]:.2f}%\")\n\n gguf_writer = gguf.GGUFWriter(model_path, \"mnist-cnn\")\n\n conv1_kernel = model.layers[0].weights[0].numpy()\n conv1_kernel = np.moveaxis(conv1_kernel, [2, 3], [0, 1])\n gguf_writer.add_tensor(\"conv1.kernel\", conv1_kernel, raw_shape=(8, 1, 3, 3))\n\n conv1_bias = model.layers[0].weights[1].numpy()\n gguf_writer.add_tensor(\"conv1.bias\", conv1_bias, raw_shape=(1, 8, 1, 1))\n\n conv2_kernel = model.layers[2].weights[0].numpy()\n conv2_kernel = np.moveaxis(conv2_kernel, [0, 1, 2, 3], [2, 3, 1, 0])\n gguf_writer.add_tensor(\"conv2.kernel\", conv2_kernel, raw_shape=(16, 8, 3, 3))\n\n conv2_bias = model.layers[2].weights[1].numpy()\n gguf_writer.add_tensor(\"conv2.bias\", conv2_bias, raw_shape=(1, 16, 1, 1))\n\n dense_weight = model.layers[-1].weights[0].numpy()\n dense_weight = dense_weight.transpose()\n gguf_writer.add_tensor(\"dense.weight\", dense_weight, raw_shape=(10, 7*7*16))\n\n dense_bias = model.layers[-1].weights[1].numpy()\n gguf_writer.add_tensor(\"dense.bias\", dense_bias)\n\n gguf_writer.write_header_to_file()\n gguf_writer.write_kv_data_to_file()\n gguf_writer.write_tensors_to_file()\n gguf_writer.close()\n print(f\"GGUF model saved to '{model_path}'\")\n\n\nif __name__ == '__main__':\n if len(sys.argv) != 2:\n print(f\"Usage: {sys.argv[0]} \")\n sys.exit(1)\n train(sys.argv[1])\n" }, { "path": "examples/mnist/mnist-train-fc.py", "content": "import gguf\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torchvision.datasets as dsets\nimport torchvision.transforms as transforms\nfrom torch.autograd import Variable\n\nimport sys\nfrom time import time\n\ninput_size = 784 # img_size = (28,28) ---> 28*28=784 in total\nhidden_size = 500 # number of nodes at hidden layer\nnum_classes = 10 # number of output classes discrete range [0,9]\nnum_epochs = 30 # number of times which the entire dataset is passed throughout the model\nbatch_size = 1000 # the size of input data used for one iteration\nlr = 1e-3 # size of step\n\n\nclass Net(nn.Module):\n def __init__(self, input_size, hidden_size, num_classes):\n super(Net, self).__init__()\n self.fc1 = nn.Linear(input_size, hidden_size)\n self.relu = nn.ReLU()\n self.fc2 = nn.Linear(hidden_size, num_classes)\n\n def forward(self, x):\n out = self.fc1(x)\n out = self.relu(out)\n out = self.fc2(out)\n return out\n\n\ndef train(model_path):\n train_data = dsets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=True)\n test_data = dsets.MNIST(root='./data', train=False, transform=transforms.ToTensor())\n\n assert len(train_data) == 60000\n assert len(test_data) == 10000\n\n kwargs_train_test = dict(batch_size=batch_size, num_workers=4, pin_memory=True)\n train_gen = torch.utils.data.DataLoader(dataset=train_data, shuffle=True, **kwargs_train_test)\n test_gen = torch.utils.data.DataLoader(dataset=test_data, shuffle=False, **kwargs_train_test)\n\n net = Net(input_size, hidden_size, num_classes)\n\n if torch.cuda.is_available():\n net.cuda()\n\n loss_function = nn.CrossEntropyLoss()\n optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n\n t_start = time()\n for epoch in range(num_epochs):\n loss_history = []\n ncorrect = 0\n\n for i, (images, labels) in enumerate(train_gen):\n images = Variable(images.view(-1, 28*28))\n labels = Variable(labels)\n\n if torch.cuda.is_available():\n images = images.cuda()\n labels = labels.cuda()\n\n optimizer.zero_grad()\n outputs = net(images)\n loss = loss_function(outputs, labels)\n\n loss_history.append(loss.cpu().data)\n _, predictions = torch.max(outputs, 1)\n ncorrect += (predictions == labels).sum()\n\n loss.backward()\n optimizer.step()\n\n if (i + 1)*batch_size % 10000 == 0:\n loss_mean = np.mean(loss_history)\n accuracy = ncorrect / ((i + 1) * batch_size)\n print(\n f\"Epoch [{epoch+1:02d}/{num_epochs}], \"\n f\"Step [{(i+1)*batch_size:05d}/{len(train_data)}], \"\n f\"Loss: {loss_mean:.4f}, Accuracy: {100*accuracy:.2f}%\")\n print()\n print(f\"Training took {time()-t_start:.2f}s\")\n\n loss_history = []\n ncorrect = 0\n\n for i, (images, labels) in enumerate(test_gen):\n images = Variable(images.view(-1, 28*28))\n labels = Variable(labels)\n\n if torch.cuda.is_available():\n images = images.cuda()\n labels = labels.cuda()\n\n outputs = net(images)\n loss = loss_function(outputs, labels)\n\n loss_history.append(loss.cpu().data)\n _, predictions = torch.max(outputs, 1)\n ncorrect += (predictions == labels).sum().cpu().numpy()\n\n loss_mean = np.mean(loss_history)\n loss_uncertainty = np.std(loss_history) / np.sqrt(len(loss_history) - 1)\n accuracy_mean = ncorrect / (len(test_gen) * batch_size)\n accuracy_uncertainty = np.sqrt(accuracy_mean * (1.0 - accuracy_mean) / (len(test_gen) * batch_size))\n print()\n print(f\"Test loss: {loss_mean:.6f}+-{loss_uncertainty:.6f}, Test accuracy: {100*accuracy_mean:.2f}+-{100*accuracy_uncertainty:.2f}%\")\n\n gguf_writer = gguf.GGUFWriter(model_path, \"mnist-fc\")\n\n print()\n print(f\"Model tensors saved to {model_path}:\")\n for tensor_name in net.state_dict().keys():\n data = net.state_dict()[tensor_name].squeeze().cpu().numpy()\n print(tensor_name, \"\\t\", data.shape)\n gguf_writer.add_tensor(tensor_name, data)\n\n gguf_writer.write_header_to_file()\n gguf_writer.write_kv_data_to_file()\n gguf_writer.write_tensors_to_file()\n gguf_writer.close()\n\n\nif __name__ == '__main__':\n if len(sys.argv) != 2:\n print(f\"Usage: {sys.argv[0]} \")\n sys.exit(1)\n train(sys.argv[1])\n" }, { "path": "examples/mnist/mnist-train.cpp", "content": "#include \"ggml-opt.h\"\n#include \"mnist-common.h\"\n\n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\nint main(int argc, char ** argv) {\n if (argc != 5 && argc != 6) {\n fprintf(stderr, \"Usage: %s mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte [CPU/CUDA0]\\n\", argv[0]);\n exit(0);\n }\n\n // The MNIST model is so small that the overhead from data shuffling is non-negligible, especially with CUDA.\n // With a shard size of 10 this overhead is greatly reduced at the cost of less shuffling (does not seem to have a significant impact).\n // A batch of 500 images then consists of 50 random shards of size 10 instead of 500 random shards of size 1.\n ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTRAIN, /*ndata_shard =*/ 10);\n\n if (!mnist_image_load(argv[3], dataset)) {\n return 1;\n }\n if (!mnist_label_load(argv[4], dataset)) {\n return 1;\n }\n\n mnist_model model = mnist_model_init_random(argv[1], argc >= 6 ? argv[5] : \"\", MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);\n\n mnist_model_build(model);\n\n mnist_model_train(model, dataset, /*nepoch =*/ 30, /*val_split =*/ 0.05f);\n\n mnist_model_save(model, argv[2]);\n}\n" }, { "path": "examples/mnist/server.py", "content": "import http.server\nimport socketserver\nimport os\nimport sys\n\nDIRECTORY = os.path.abspath(os.path.join(os.path.dirname(__file__), 'web'))\nPORT = 8000\n\nclass CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, directory=DIRECTORY, **kwargs)\n\n def end_headers(self):\n # Add required headers for SharedArrayBuffer\n self.send_header(\"Cross-Origin-Opener-Policy\", \"same-origin\")\n self.send_header(\"Cross-Origin-Embedder-Policy\", \"require-corp\")\n self.send_header(\"Access-Control-Allow-Origin\", \"*\")\n super().end_headers()\n\n# Enable address reuse\nclass CustomServer(socketserver.TCPServer):\n allow_reuse_address = True\n\ntry:\n with CustomServer((\"\", PORT), CustomHTTPRequestHandler) as httpd:\n print(f\"Serving directory '{DIRECTORY}' at http://localhost:{PORT}\")\n print(f\"Application context root: http://localhost:{PORT}/\")\n try:\n httpd.serve_forever()\n except KeyboardInterrupt:\n print(\"\\nServer stopped.\")\n # Force complete exit\n sys.exit(0)\nexcept OSError as e:\n print(f\"Error: {e}\")\n sys.exit(1)\n" }, { "path": "examples/perf-metal/CMakeLists.txt", "content": "#\n# perf-metal\n\nset(TEST_TARGET perf-metal)\nadd_executable(${TEST_TARGET} perf-metal.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml)\n\n" }, { "path": "examples/perf-metal/perf-metal.cpp", "content": "// basic tool to experiment with the Metal backend\n//\n// 1. Get GPU trace of a dummy graph:\n//\n// rm -rf /tmp/perf-metal.gputrace\n// make -j perf-metal && METAL_CAPTURE_ENABLED=1 ./bin/perf-metal\n// open /tmp/perf-metal.gputrace\n//\n// https://github.com/ggerganov/llama.cpp/issues/9507\n//\n\n#include \"ggml.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-metal.h\"\n\n#include \n#include \n#include \n\nint main(int argc, char ** argv) {\n int n_op = 1024;\n int n_iter = 128;\n\n if (argc > 1) {\n n_op = std::atoi(argv[1]);\n }\n\n if (argc > 2) {\n n_iter = std::atoi(argv[2]);\n }\n\n printf(\"%s: n_op = %d, n_iter = %d\\n\", __func__, n_op, n_iter);\n\n const int ne00 = 8;\n const int ne01 = 8;\n const int ne11 = 8;\n\n std::vector data0(ne00*ne01, 1.0f);\n std::vector data1(ne00*ne01, 1.0f/ne00);\n\n ggml_backend_t backend = ggml_backend_metal_init();\n if (!backend) {\n fprintf(stderr, \"%s: ggml_backend_metal_init() failed\\n\", __func__);\n return 1;\n }\n\n const size_t ctx_size = 2 * ggml_tensor_overhead();\n\n struct ggml_init_params params = {\n /*.mem_size =*/ ctx_size,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n struct ggml_context * ctx = ggml_init(params);\n\n struct ggml_tensor * t0 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne01);\n struct ggml_tensor * t1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne11);\n\n ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend);\n\n ggml_backend_tensor_set(t0, data0.data(), 0, ggml_nbytes(t0));\n ggml_backend_tensor_set(t1, data1.data(), 0, ggml_nbytes(t1));\n\n struct ggml_cgraph * gf = NULL;\n\n struct ggml_context * ctx_cgraph = NULL;\n\n // create a dummy compute graph:\n //\n // x = mul_mat(t0, t1)\n // x = x * 1.0f\n // x = mul_mat(x, t1)\n // x = x * 1.0f\n // ... repeat n_op times ...\n //\n {\n struct ggml_init_params params0 = {\n /*.mem_size =*/ 4*n_op*ggml_tensor_overhead() + ggml_graph_overhead(),\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n ctx_cgraph = ggml_init(params0);\n\n gf = ggml_new_graph_custom(ctx_cgraph, 4*n_op, false);\n\n struct ggml_tensor * cur = ggml_mul_mat(ctx_cgraph, t0, t1);\n cur = ggml_scale(ctx_cgraph, cur, 1.0f);\n\n for (int i = 0; i < n_op - 1; i++) {\n cur = ggml_mul_mat(ctx_cgraph, cur, t1);\n cur = ggml_scale(ctx_cgraph, cur, 1.0f);\n }\n\n cur = ggml_scale(ctx_cgraph, cur, 42.0f);\n\n ggml_build_forward_expand(gf, cur);\n }\n\n printf(\"%s: graph nodes = %d\\n\", __func__, ggml_graph_n_nodes(gf));\n\n ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));\n ggml_gallocr_alloc_graph(allocr, gf);\n\n {\n // warm-up\n ggml_backend_graph_compute(backend, gf);\n\n const int64_t t_start = ggml_time_us();\n\n for (int iter = 0; iter < n_iter; iter++) {\n ggml_backend_graph_compute(backend, gf);\n }\n\n const int64_t t_end = ggml_time_us();\n\n // actual trace\n ggml_backend_metal_capture_next_compute(backend);\n ggml_backend_graph_compute(backend, gf);\n //std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace!\n ggml_backend_metal_capture_next_compute(backend);\n ggml_backend_graph_compute(backend, gf);\n //std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace!\n ggml_backend_metal_capture_next_compute(backend);\n ggml_backend_graph_compute(backend, gf);\n\n printf(\"%s: time = %f ms\\n\", __func__, (t_end - t_start) / 1000.0 / n_iter);\n }\n\n {\n struct ggml_tensor * res = ggml_graph_node(gf, -1);\n\n std::vector data(res->ne[0] * res->ne[1], 0.0f);\n\n ggml_backend_tensor_get(res, data.data(), 0, ggml_nbytes(res));\n\n for (int i1 = 0; i1 < res->ne[1]; i1++) {\n for (int i0 = 0; i0 < res->ne[0]; i0++) {\n printf(\"%f \", data[i1*res->ne[0] + i0]);\n }\n printf(\"\\n\");\n }\n }\n\n ggml_free(ctx_cgraph);\n ggml_gallocr_free(allocr);\n ggml_free(ctx);\n ggml_backend_buffer_free(buffer);\n ggml_backend_free(backend);\n\n return 0;\n}\n" }, { "path": "examples/prompts/dolly-v2.txt", "content": "Hello World! => 12092,3645,2\nI can't believe it's already Friday!\" => 42,476,626,2868,352,434,2168,6794,1476\nThe URL for the website is https://www.example.com.\" => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449\n\"She said, 'I love to travel.'\" => 3,2993,753,13,686,42,2389,281,4288,18574\n'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464\n\"Let's meet at 2:30 p.m. in the park.\" => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449\nThe book costs $19.99 => 510,1984,4815,370,746,15,1525\n\"John's favorite color is blue.\" => 3,8732,434,7583,3295,310,4797,449\nTh@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2\nC@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32\nW0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2\nH0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32\nI l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15\nWh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32\nThe cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15\nI need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15\nThe sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15\nShe is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15\nWe went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15\nHe plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15\nThey are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15\nThe flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15\nI enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15\nWe need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15\nThe dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15\nShe is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15\nHe is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15\nThe children are playing in the playground. => 510,2151,403,4882,275,253,41008,15\nI'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15\nThe coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15\nThey are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15\nShe sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15\nWe should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15\nHe is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15\nThe rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15\nI enjoy watching romantic movies. => 42,4264,7487,18109,11321,15\nThey are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15\nShe dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15\nHe is an excellent basketball player. => 1328,310,271,7126,14648,4760,15\nThe baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15\nI need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15\nThey are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15\nShe is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15\nWe should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15\nThe car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15\nHe loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15\nThe birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15\nI want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15\nThey are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15\nShe is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15\nWe are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15\nThe cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15\nHe is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15\nThe students are studying for their exams. => 510,3484,403,12392,323,616,34666,15\nI enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15\nThey are renovating their house. => 3726,403,30074,839,616,2419,15\nShe is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15\nWe should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15\nThe traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15\nHe is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15\nThe baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15\nI need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15\nThey are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15\nShe is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15\nWe are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15\nThe cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15\nHe is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15\nThe students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15\nI enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15\nThey are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15\nShe is taking photographs of nature. => 2993,310,3192,15928,273,3753,15\nWe should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15\nThe traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15\nHe is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15\nThe baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15\nI need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15\nThey are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15\nShe is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15\nWe are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15\nThe coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15\nHe is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15\nThe students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15\nI enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15\nThey are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15\nShe is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15\nWe should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15\nThe traffic is congest => 510,7137,310,25801\nThe sun is shining brightly today. => 510,5101,310,28115,43925,3063,15\nI enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15\nShe plays the piano beautifully. => 2993,7120,253,18542,27839,15\nThe cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15\nI love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15\nHe always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15\nThe flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15\nShe danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15\nThe dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15\nWe went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15\nHe speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15\nThe train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15\nShe cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15\n" }, { "path": "examples/prompts/gpt-2-chinese.txt", "content": "请问洗手间在哪里？ => 6435,7309,3819,2797,7313,1762,1525,7027,8043\n" }, { "path": "examples/prompts/gpt-2.txt", "content": "Hello World! => 15496,2159,0\nI can't believe it's already Friday!\" => 40,460,470,1975,340,338,1541,3217,2474\nThe URL for the website is https://www.example.com.\" => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526\n\"She said, 'I love to travel.'\" => 1,3347,531,11,705,40,1842,284,3067,11496\n'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637\n\"Let's meet at 2:30 p.m. in the park.\" => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526\nThe book costs $19.99 => 464,1492,3484,720,1129,13,2079\n\"John's favorite color is blue.\" => 1,7554,338,4004,3124,318,4171,526\nTh@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0\nC@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30\nW0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0\nH0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30\nI l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13\nWh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30\nThe cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13\nI need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13\nThe sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13\nShe is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13\nWe went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13\nHe plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13\nThey are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13\nThe flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13\nI enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13\nWe need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13\nThe dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13\nShe is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13\nHe is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13\nThe children are playing in the playground. => 464,1751,389,2712,287,262,24817,13\nI'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13\nThe coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13\nThey are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13\nShe sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13\nWe should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13\nHe is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13\nThe rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13\nI enjoy watching romantic movies. => 40,2883,4964,14348,6918,13\nThey are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13\nShe dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13\nHe is an excellent basketball player. => 1544,318,281,6275,9669,2137,13\nThe baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13\nI need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13\nThey are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13\nShe is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13\nWe should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13\nThe car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13\nHe loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13\nThe birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13\nI want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13\nThey are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13\nShe is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13\nWe are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13\nThe cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13\nHe is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13\nThe students are studying for their exams. => 464,2444,389,11065,329,511,26420,13\nI enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13\nThey are renovating their house. => 2990,389,24317,803,511,2156,13\nShe is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13\nWe should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13\nThe traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13\nHe is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13\nThe baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13\nI need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13\nThey are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13\nShe is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13\nWe are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13\nThe cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13\nHe is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13\nThe students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13\nI enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13\nThey are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13\nShe is taking photographs of nature. => 3347,318,2263,12566,286,3450,13\nWe should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13\nThe traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13\nHe is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13\nThe baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13\nI need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13\nThey are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13\nShe is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13\nWe are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13\nThe coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13\nHe is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13\nThe students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13\nI enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13\nThey are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13\nShe is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13\nWe should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13\nThe traffic is congest => 464,4979,318,22791\nThe sun is shining brightly today. => 464,4252,318,22751,35254,1909,13\nI enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13\nShe plays the piano beautifully. => 3347,5341,262,19132,21104,13\nThe cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13\nI love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13\nHe always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13\nThe flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13\nShe danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13\nThe dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13\nWe went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13\nHe speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13\nThe train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13\nShe cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13\n" }, { "path": "examples/prompts/gpt-j.txt", "content": "Hello World! => 15496,2159,0\nI can't believe it's already Friday!\" => 40,460,470,1975,340,338,1541,3217,2474\nThe URL for the website is https://www.example.com.\" => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526\n\"She said, 'I love to travel.'\" => 1,3347,531,11,705,40,1842,284,3067,11496\n'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637\n\"Let's meet at 2:30 p.m. in the park.\" => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526\nThe book costs $19.99 => 464,1492,3484,720,1129,13,2079\n\"John's favorite color is blue.\" => 1,7554,338,4004,3124,318,4171,526\nTh@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0\nC@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30\nW0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0\nH0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30\nI l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13\nWh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30\nThe cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13\nI need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13\nThe sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13\nShe is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13\nWe went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13\nHe plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13\nThey are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13\nThe flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13\nI enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13\nWe need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13\nThe dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13\nShe is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13\nHe is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13\nThe children are playing in the playground. => 464,1751,389,2712,287,262,24817,13\nI'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13\nThe coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13\nThey are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13\nShe sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13\nWe should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13\nHe is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13\nThe rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13\nI enjoy watching romantic movies. => 40,2883,4964,14348,6918,13\nThey are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13\nShe dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13\nHe is an excellent basketball player. => 1544,318,281,6275,9669,2137,13\nThe baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13\nI need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13\nThey are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13\nShe is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13\nWe should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13\nThe car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13\nHe loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13\nThe birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13\nI want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13\nThey are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13\nShe is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13\nWe are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13\nThe cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13\nHe is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13\nThe students are studying for their exams. => 464,2444,389,11065,329,511,26420,13\nI enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13\nThey are renovating their house. => 2990,389,24317,803,511,2156,13\nShe is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13\nWe should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13\nThe traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13\nHe is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13\nThe baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13\nI need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13\nThey are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13\nShe is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13\nWe are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13\nThe cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13\nHe is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13\nThe students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13\nI enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13\nThey are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13\nShe is taking photographs of nature. => 3347,318,2263,12566,286,3450,13\nWe should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13\nThe traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13\nHe is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13\nThe baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13\nI need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13\nThey are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13\nShe is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13\nWe are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13\nThe coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13\nHe is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13\nThe students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13\nI enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13\nThey are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13\nShe is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13\nWe should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13\nThe traffic is congest => 464,4979,318,22791\nThe sun is shining brightly today. => 464,4252,318,22751,35254,1909,13\nI enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13\nShe plays the piano beautifully. => 3347,5341,262,19132,21104,13\nThe cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13\nI love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13\nHe always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13\nThe flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13\nShe danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13\nThe dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13\nWe went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13\nHe speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13\nThe train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13\nShe cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13\n" }, { "path": "examples/prompts/gpt-neox-japanese.txt", "content": "明日の天気はどうですか。 => 263,7353,268,18461,271,1722,18405,265\n" }, { "path": "examples/prompts/gpt-neox.txt", "content": "Hello World! => 12092,3645,2\nI can't believe it's already Friday!\" => 42,476,626,2868,352,434,2168,6794,1476\nThe URL for the website is https://www.example.com.\" => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449\n\"She said, 'I love to travel.'\" => 3,2993,753,13,686,42,2389,281,4288,18574\n'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464\n\"Let's meet at 2:30 p.m. in the park.\" => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449\nThe book costs $19.99 => 510,1984,4815,370,746,15,1525\n\"John's favorite color is blue.\" => 3,8732,434,7583,3295,310,4797,449\nTh@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2\nC@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32\nW0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2\nH0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32\nI l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15\nWh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32\nThe cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15\nI need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15\nThe sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15\nShe is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15\nWe went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15\nHe plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15\nThey are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15\nThe flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15\nI enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15\nWe need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15\nThe dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15\nShe is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15\nHe is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15\nThe children are playing in the playground. => 510,2151,403,4882,275,253,41008,15\nI'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15\nThe coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15\nThey are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15\nShe sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15\nWe should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15\nHe is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15\nThe rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15\nI enjoy watching romantic movies. => 42,4264,7487,18109,11321,15\nThey are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15\nShe dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15\nHe is an excellent basketball player. => 1328,310,271,7126,14648,4760,15\nThe baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15\nI need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15\nThey are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15\nShe is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15\nWe should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15\nThe car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15\nHe loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15\nThe birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15\nI want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15\nThey are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15\nShe is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15\nWe are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15\nThe cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15\nHe is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15\nThe students are studying for their exams. => 510,3484,403,12392,323,616,34666,15\nI enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15\nThey are renovating their house. => 3726,403,30074,839,616,2419,15\nShe is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15\nWe should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15\nThe traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15\nHe is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15\nThe baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15\nI need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15\nThey are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15\nShe is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15\nWe are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15\nThe cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15\nHe is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15\nThe students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15\nI enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15\nThey are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15\nShe is taking photographs of nature. => 2993,310,3192,15928,273,3753,15\nWe should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15\nThe traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15\nHe is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15\nThe baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15\nI need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15\nThey are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15\nShe is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15\nWe are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15\nThe coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15\nHe is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15\nThe students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15\nI enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15\nThey are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15\nShe is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15\nWe should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15\nThe traffic is congest => 510,7137,310,25801\nThe sun is shining brightly today. => 510,5101,310,28115,43925,3063,15\nI enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15\nShe plays the piano beautifully. => 2993,7120,253,18542,27839,15\nThe cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15\nI love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15\nHe always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15\nThe flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15\nShe danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15\nThe dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15\nWe went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15\nHe speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15\nThe train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15\nShe cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15\n" }, { "path": "examples/prompts/polyglot-ko.txt", "content": "이것은 테스트 이다. => 12271,296,6474,28037,17\n걱정할 필요 없다. => 18311,482,1062,550,267,17\n버그는 언젠가 고쳐진다. => 6904,272,8575,10381,1765,17\n" }, { "path": "examples/prompts/replit.txt", "content": "Hello World! => 6466,147,2317,350\nI can't believe it's already Friday!\" => 286,512,172,185,13392,393,172,155,3239,147,29249,8537\nThe URL for the website is https://www.example.com.\" => 505,5635,250,170,11745,235,147,303,262,552,148,811,148,241,148,161\n\"She said, 'I love to travel.'\" => 161,10386,4089,150,206,286,8440,194,147,12363,148,172,161\n'The temperature is 25.5°C.' => 172,505,147,9502,235,147,20022,8516,228,148,172\n\"Let's meet at 2:30 p.m. in the park.\" => 161,8997,172,155,17120,536,147,162,5245,147,207,148,204,148,219,170,147,17664,148,161\nThe book costs $19.99 => 505,147,2277,17494,236,166,11824\n\"John's favorite color is blue.\" => 161,7475,172,155,147,11105,147,349,235,17046,148,161\nTh@nk y0u f0r y0ur h3lp! => 6309,240,9019,147,237,159,247,147,202,159,223,147,237,159,2458,147,226,171,3899,350\nC@n I g3t a c0ffee, pl3@se? => 228,240,211,398,147,267,171,185,216,147,196,159,13360,163,150,147,1287,171,240,155,163,272\nW0w! Th@t's @m@zing! => 450,159,274,350,147,6309,240,185,172,155,268,204,240,301,248,350\nH0w 4re y0u t0d@y? => 304,159,274,320,440,147,237,159,247,147,185,159,182,240,237,272\nI l0ve t0 tr@vel @r0und the w0rld. => 286,997,159,1290,147,185,159,147,490,240,3893,268,223,159,3981,170,147,274,159,223,2833,148\nWh@t's y0ur f@v0rite m0vie? => 450,226,240,185,172,155,147,237,159,2458,147,202,240,252,159,5961,163,147,204,159,24373,272\nThe cat is sleeping on the mat. => 505,147,1604,235,147,3987,248,347,170,147,1297,148\nI need to buy some groceries for dinner. => 286,1645,194,147,8068,1499,147,10022,1037,10023,250,147,182,2749,148\nThe sun is shining brightly in the sky. => 505,147,5852,235,147,7304,2967,147,215,649,391,219,170,147,7310,148\nShe is reading a book in the park. => 10386,235,9838,216,147,2277,219,170,147,17664,148\nWe went for a walk on the beach yesterday. => 3250,10825,250,216,147,8156,347,170,294,5371,147,28830,148\nHe plays the guitar like a pro. => 5301,7084,155,170,147,4604,2214,1425,216,3474,148\nThey are going to the movies tonight. => 18815,429,6552,194,170,147,15877,194,7907,148\nThe flowers are blooming in the garden. => 505,147,22953,155,429,147,10411,2799,248,219,170,147,22140,148\nI enjoy listening to classical music. => 286,23162,15876,248,194,239,4251,147,7395,148\nWe need to buy groceries for the week. => 3250,1645,194,147,8068,147,10022,1037,10023,250,170,9238,148\nThe dog is chasing its tail in circles. => 505,147,6540,235,147,196,916,248,1602,147,5129,219,147,4095,155,148\nShe is wearing a beautiful red dress. => 10386,235,147,16427,248,216,147,23447,147,1160,147,14592,148\nHe is a talented actor in Hollywood. => 5301,235,216,147,29750,246,147,5112,219,147,16924,391,10477,148\nThe children are playing in the playground. => 505,7934,429,7084,248,219,170,7084,12055,148\nI'm going to visit my grandparents this weekend. => 286,172,204,6552,194,9939,1247,147,11806,12019,291,9238,314,148\nThe coffee tastes bitter without sugar. => 505,147,21526,147,20931,155,5145,1430,1988,147,28759,148\nThey are planning a surprise party for her. => 18815,429,147,23661,216,147,29240,147,7344,250,1869,148\nShe sings like an angel on stage. => 10386,147,155,6502,1425,426,147,26028,347,12685,148\nWe should take a vacation to relax. => 3250,936,4654,216,147,15388,946,194,1998,2744,148\nHe is studying medicine at the university. => 5301,235,7959,248,147,20742,1668,536,170,147,8025,148\nThe rain is pouring heavily outside. => 505,147,6885,235,5306,248,1189,5451,391,8096,148\nI enjoy watching romantic movies. => 286,23162,147,3355,248,147,26080,4140,147,15877,148\nThey are celebrating their anniversary today. => 18815,429,147,30000,5841,1669,147,24734,5464,1770,13386,148\nShe dances gracefully to the music. => 10386,147,182,1626,155,147,267,8771,8001,194,170,147,7395,148\nHe is an excellent basketball player. => 5301,235,426,147,12300,675,185,147,26646,5132,6294,148\nThe baby is sleeping soundly in the crib. => 505,147,23597,235,147,3987,248,12642,391,219,170,147,7696,215,148\nI need to finish my homework before dinner. => 286,1645,194,147,6717,1247,147,1071,2722,2643,147,182,2749,148\nThey are organizing a charity event next month. => 18815,429,147,16442,248,216,1054,1511,1663,2399,12821,148\nShe is cooking a delicious meal for us. => 10386,235,147,20453,248,216,3936,23455,147,26658,250,147,539,148\nWe should go hiking in the mountains. => 3250,936,4242,147,2254,5357,219,170,147,204,18028,155,148\nThe car broke down on the way to work. => 505,7553,147,510,10036,4288,347,170,3699,194,1916,148\nHe loves playing video games in his free time. => 5301,8440,155,7084,248,8722,147,11281,219,1439,4002,801,148\nThe birds are chirping in the trees. => 505,147,13043,155,429,147,3904,223,4639,219,170,5311,155,148\nI want to learn how to play the piano. => 286,1857,194,14167,2496,194,7084,170,147,207,23635,148\nThey are building a new shopping mall in the city. => 18815,429,11038,216,277,147,22184,147,204,609,219,170,147,2416,148\nShe is writing a novel in her spare time. => 10386,235,3242,216,147,25814,219,1869,6772,2382,801,148\nWe are going to the zoo this Saturday. => 3250,429,6552,194,170,147,25101,291,147,31426,148\nThe cake looks delicious with chocolate frosting. => 505,147,24422,16303,3936,23455,312,147,5619,533,2239,147,202,3973,3431,148\nHe is a talented painter who sells his artwork. => 5301,235,216,147,29750,246,147,9226,279,2888,13004,155,1439,12234,2722,148\nThe students are studying for their exams. => 505,15707,429,7959,248,250,1669,147,12398,155,148\nI enjoy swimming in the ocean. => 286,23162,147,4729,8528,248,219,170,147,26193,148\nThey are renovating their house. => 18815,429,991,10724,3643,1669,13788,148\nShe is practicing yoga to stay healthy. => 10386,235,147,18453,248,147,5063,1186,194,15344,147,28550,148\nWe should plant flowers in the garden. => 3250,936,147,9212,147,22953,155,219,170,147,22140,148\nThe traffic is heavy during rush hour. => 505,147,11097,235,147,22232,4340,147,22319,147,5686,148\nHe is a skilled chef who creates amazing dishes. => 5301,235,216,147,8891,246,9784,202,2888,13720,147,28880,147,23852,383,148\nThe baby is crawling on the floor. => 505,147,23597,235,147,22120,248,347,170,147,5895,148\nI need to buy a new pair of shoes. => 286,1645,194,147,8068,216,277,12632,210,147,155,21953,155,148\nThey are going on a road trip across the country. => 18815,429,6552,347,216,147,6362,147,11395,9762,170,11305,148\nShe is playing the piano beautifully. => 10386,235,7084,248,170,147,207,23635,147,23447,391,148\nWe are going to a concert tomorrow night. => 3250,429,6552,194,216,1710,4391,29524,12716,148\nThe cake tastes delicious with vanilla frosting. => 505,147,24422,147,20931,155,3936,23455,312,5535,7476,147,202,3973,3431,148\nHe is a dedicated teacher who inspires his students. => 5301,235,216,326,8298,3460,147,9675,2888,147,28801,155,1439,15707,148\nThe students are participating in a science fair. => 505,15707,429,147,30961,3643,219,216,147,10587,147,7636,148\nI enjoy hiking in the mountains. => 286,23162,147,2254,5357,219,170,147,204,18028,155,148\nThey are organizing a beach cleanup next weekend. => 18815,429,147,16442,248,216,294,5371,147,10401,2399,9238,314,148\nShe is taking photographs of nature. => 10386,235,147,12345,147,4709,1547,155,210,147,211,8603,148\nWe should try a new restaurant in town. => 3250,936,147,746,216,277,147,11007,219,147,10200,148\nThe traffic is moving slowly on the highway. => 505,147,11097,235,147,8601,147,9880,391,347,170,5976,3330,148\nHe is a talented singer with a beautiful voice. => 5301,235,216,147,29750,246,147,155,248,279,312,216,147,23447,147,9316,148\nThe baby is laughing and giggling. => 505,147,23597,235,147,23066,248,221,147,2341,3631,2869,148\nI need to do laundry and wash my clothes. => 286,1645,194,543,960,3981,2154,221,147,27589,1247,147,22141,383,148\nThey are planning a trip to Europe. => 18815,429,147,23661,216,147,11395,194,13131,148\nShe is learning how to play the guitar. => 10386,235,11754,2496,194,7084,170,147,4604,2214,148\nWe are going to a museum this Sunday. => 3250,429,6552,194,216,147,204,433,1177,291,147,29111,148\nThe coffee smells amazing in the morning. => 505,147,21526,31454,155,147,28880,219,170,20701,148\nHe is a hardworking farmer who grows crops. => 5301,235,216,8524,14992,147,16679,279,2888,147,6044,155,147,8650,155,148\nThe students are presenting their research projects. => 505,15707,429,5130,248,1669,13217,14235,148\nI enjoy playing soccer with my friends. => 286,23162,7084,248,147,9351,5318,312,1247,147,5347,155,148\nThey are volunteering at a local shelter. => 18815,429,147,5238,7478,163,12798,536,216,2491,2905,1359,279,148\nShe is practicing martial arts for self-defense. => 10386,235,147,18453,248,147,3261,185,4381,12234,155,250,623,153,29896,148\nWe should try a new recipe for dinner. => 3250,936,147,746,216,277,147,9851,250,147,182,2749,148\nThe traffic is congest => 505,147,11097,235,1710,14169\nThe sun is shining brightly today. => 505,147,5852,235,147,7304,2967,147,215,649,391,13386,148\nI enjoy reading books in my free time. => 286,23162,9838,147,9670,219,1247,4002,801,148\nShe plays the piano beautifully. => 10386,7084,155,170,147,207,23635,147,23447,391,148\nThe cat chased the mouse around the room. => 505,147,1604,147,196,916,246,170,12551,6890,170,9654,148\nI love eating pizza with extra cheese. => 286,8440,147,163,3643,147,207,8403,312,8230,9784,383,163,148\nHe always wears a hat wherever he goes. => 5301,5418,147,16427,155,216,147,4879,2171,2433,1189,16177,148\nThe flowers in the garden are blooming. => 505,147,22953,155,219,170,147,22140,429,147,10411,2799,248,148\nShe danced gracefully on the stage. => 10386,13378,12408,147,267,8771,8001,347,170,12685,148\nThe dog barked loudly in the park. => 505,147,6540,147,973,293,246,147,30182,391,219,170,147,17664,148\nWe went swimming in the ocean yesterday. => 3250,10825,147,4729,8528,248,219,170,147,26193,147,28830,148\nHe speaks fluent French and Spanish. => 5301,147,13285,155,147,21677,147,254,17590,221,147,31519,148\nThe train arrived at the station on time. => 505,147,872,147,20712,182,536,170,147,7184,347,801,148\nShe cooked a delicious meal for her family. => 10386,147,20453,246,216,3936,23455,147,26658,250,1869,147,2002,148\n" }, { "path": "examples/prompts/starcoder.txt", "content": "Hello World! => 8279,10896,19\nI can't believe it's already Friday!\" => 59,883,1330,13710,561,1182,3425,506,25674,11555\nThe URL for the website is https://www.example.com.\" => 1318,3834,436,322,9575,438,1678,555,1499,32,2763,32,508,3107\n\"She said, 'I love to travel.'\" => 20,25387,9884,30,330,59,14290,372,25283,29329\n'The temperature is 25.5°C.' => 25,1318,13587,438,225,36,39,32,39,23767,53,4564\n\"Let's meet at 2:30 p.m. in the park.\" => 20,9809,1182,18450,821,225,36,44,37,34,298,32,95,32,328,322,880,93,3107\nThe book costs $19.99 => 1318,7618,25950,398,35,43,32,43,43\n\"John's favorite color is blue.\" => 20,19693,1182,27448,1963,438,10087,3107\nTh@nk y0u f0r y0ur h3lp! => 1027,50,19877,533,34,103,296,34,100,533,34,305,420,37,1915,19\nC@n I g3t a c0ffee, pl3@se? => 53,50,96,439,485,37,102,312,281,34,21298,30,1278,37,50,277,49\nW0w! Th@t's @m@zing! => 73,34,105,19,947,50,102,1182,477,95,50,26768,19\nH0w 4re y0u t0d@y? => 58,34,105,225,38,268,533,34,103,273,34,86,50,107,49\nI l0ve t0 tr@vel @r0und the w0rld. => 59,456,34,587,273,34,554,50,1203,477,100,34,642,322,341,34,100,1381,32\nWh@t's y0ur f@v0rite m0vie? => 2444,50,102,1182,533,34,305,296,50,104,34,1049,345,34,104,1075,49\nThe cat is sleeping on the mat. => 1318,10501,438,9368,299,544,322,2491,32\nI need to buy some groceries for dinner. => 59,1849,372,16968,1629,20234,85,6958,436,343,3369,32\nThe sun is shining brightly in the sky. => 1318,15323,438,787,19068,38231,631,328,322,26718,32\nShe is reading a book in the park. => 25387,438,9175,312,7618,328,322,880,93,32\nWe went for a walk on the beach yesterday. => 3122,14236,436,312,13503,544,322,526,867,39485,32\nHe plays the guitar like a pro. => 1331,41271,322,3932,19931,2124,312,534,32\nThey are going to the movies tonight. => 31805,884,6783,372,322,27889,26076,694,32\nThe flowers are blooming in the garden. => 1318,7290,483,884,323,18466,299,328,322,485,22461,32\nI enjoy listening to classical music. => 59,31567,20498,372,443,1578,17522,32\nWe need to buy groceries for the week. => 3122,1849,372,16968,20234,85,6958,436,322,8209,32\nThe dog is chasing its tail in circles. => 1318,27435,438,663,9949,2819,13203,328,46428,32\nShe is wearing a beautiful red dress. => 25387,438,996,6992,312,36493,3346,343,714,32\nHe is a talented actor in Hollywood. => 1331,438,312,273,9556,318,16038,328,48228,631,21118,32\nThe children are playing in the playground. => 1318,5713,884,19788,328,322,4654,1749,32\nI'm going to visit my grandparents this weekend. => 59,3464,6783,372,7725,1672,33162,19277,458,40618,32\nThe coffee tastes bitter without sugar. => 1318,36917,273,633,307,3493,391,2876,309,18628,32\nThey are planning a surprise party for her. => 31805,884,26116,312,6178,9251,15270,436,7791,32\nShe sings like an angel on stage. => 25387,309,2052,2124,600,600,17691,544,10019,32\nWe should take a vacation to relax. => 3122,1395,4818,312,29164,367,372,41972,32\nHe is studying medicine at the university. => 1331,438,14866,299,32388,482,821,322,707,9190,32\nThe rain is pouring heavily outside. => 1318,36987,438,9202,299,46003,2801,11127,32\nI enjoy watching romantic movies. => 59,31567,37652,26045,7268,27889,32\nThey are celebrating their anniversary today. => 31805,884,48278,839,1741,3623,23921,5810,672,11610,32\nShe dances gracefully to the music. => 25387,343,3151,31376,4938,372,322,17522,32\nHe is an excellent basketball player. => 1331,438,600,39203,48400,11653,4362,32\nThe baby is sleeping soundly in the crib. => 1318,323,17156,438,9368,299,9934,631,328,322,281,7972,32\nI need to finish my homework before dinner. => 59,1849,372,11361,1672,6765,1007,2670,343,3369,32\nThey are organizing a charity event next month. => 31805,884,10558,6183,312,1351,543,1692,2354,6811,32\nShe is cooking a delicious meal for us. => 25387,438,23682,299,312,409,406,2406,597,279,436,1770,32\nWe should go hiking in the mountains. => 3122,1395,1983,420,1546,299,328,322,10874,1907,32\nThe car broke down on the way to work. => 1318,6346,43289,2835,544,322,3352,372,1389,32\nHe loves playing video games in his free time. => 1331,598,4954,19788,6027,19705,328,6697,3741,1133,32\nThe birds are chirping in the trees. => 1318,8424,3210,884,663,476,7075,328,322,23453,32\nI want to learn how to play the piano. => 59,2637,372,7350,2624,372,4654,322,298,25757,32\nThey are building a new shopping mall in the city. => 31805,884,9038,312,537,40692,345,464,328,322,11297,32\nShe is writing a novel in her spare time. => 25387,438,4127,312,32913,328,7791,1869,586,1133,32\nWe are going to the zoo this Saturday. => 3122,884,6783,372,322,1288,604,458,358,30288,32\nThe cake looks delicious with chocolate frosting. => 1318,281,1062,7780,409,406,2406,623,10408,27589,296,20932,299,32\nHe is a talented painter who sells his artwork. => 1331,438,312,273,9556,318,42300,6560,10800,101,6697,5549,1007,32\nThe students are studying for their exams. => 1318,16512,884,14866,299,436,3623,538,1462,32\nI enjoy swimming in the ocean. => 59,31567,2535,449,6714,328,322,337,18857,32\nThey are renovating their house. => 31805,884,316,15007,1741,3623,17075,32\nShe is practicing yoga to stay healthy. => 25387,438,11808,11636,533,40067,372,20005,44538,32\nWe should plant flowers in the garden. => 3122,1395,26795,7290,483,328,322,485,22461,32\nThe traffic is heavy during rush hour. => 1318,16391,438,32389,5929,540,1372,12021,32\nHe is a skilled chef who creates amazing dishes. => 1331,438,312,3001,12088,44051,6560,9585,36986,1214,4279,32\nThe baby is crawling on the floor. => 1318,323,17156,438,281,1294,2920,544,322,17648,32\nI need to buy a new pair of shoes. => 59,1849,372,16968,312,537,6092,432,787,37764,32\nThey are going on a road trip across the country. => 31805,884,6783,544,312,24122,19337,10160,322,10769,32\nShe is playing the piano beautifully. => 25387,438,19788,322,298,25757,526,4846,325,514,107,32\nWe are going to a concert tomorrow night. => 3122,884,6783,372,312,457,6989,31841,19212,32\nThe cake tastes delicious with vanilla frosting. => 1318,281,1062,273,633,307,409,406,2406,623,44653,296,20932,299,32\nHe is a dedicated teacher who inspires his students. => 1331,438,312,23112,30877,6560,26194,8017,6697,16512,32\nThe students are participating in a science fair. => 1318,16512,884,24623,1741,328,312,27536,19375,32\nI enjoy hiking in the mountains. => 59,31567,420,1546,299,328,322,10874,1907,32\nThey are organizing a beach cleanup next weekend. => 31805,884,10558,6183,312,526,867,13144,2354,40618,32\nShe is taking photographs of nature. => 25387,438,15137,15110,23626,432,24406,32\nWe should try a new restaurant in town. => 3122,1395,1596,312,537,43719,328,38212,32\nThe traffic is moving slowly on the highway. => 1318,16391,438,14089,12899,631,544,322,3857,3073,32\nHe is a talented singer with a beautiful voice. => 1331,438,312,273,9556,318,309,10118,623,312,36493,20309,32\nThe baby is laughing and giggling. => 1318,323,17156,438,2317,2943,299,461,485,365,36088,32\nI need to do laundry and wash my clothes. => 59,1849,372,745,2317,642,994,461,341,917,1672,7375,46948,32\nThey are planning a trip to Europe. => 31805,884,26116,312,19337,372,27268,32\nShe is learning how to play the guitar. => 25387,438,9608,2624,372,4654,322,3932,19931,32\nWe are going to a museum this Sunday. => 3122,884,6783,372,312,345,539,378,458,358,28036,32\nThe coffee smells amazing in the morning. => 1318,36917,309,42153,101,36986,328,322,33768,32\nHe is a hardworking farmer who grows crops. => 1331,438,312,6784,13578,9019,2302,6560,485,2138,25170,1069,32\nThe students are presenting their research projects. => 1318,16512,884,5024,299,3623,13234,8528,32\nI enjoy playing soccer with my friends. => 59,31567,19788,22682,10035,623,1672,22523,32\nThey are volunteering at a local shelter. => 31805,884,3920,45585,8637,821,312,2196,309,2542,391,32\nShe is practicing martial arts for self-defense. => 25387,438,11808,11636,345,502,564,5549,101,436,630,31,43694,32\nWe should try a new recipe for dinner. => 3122,1395,1596,312,537,15233,436,343,3369,32\nThe traffic is congest => 1318,16391,438,457,2776\nThe sun is shining brightly today. => 1318,15323,438,787,19068,38231,631,11610,32\nI enjoy reading books in my free time. => 59,31567,9175,21739,328,1672,3741,1133,32\nShe plays the piano beautifully. => 25387,41271,322,298,25757,526,4846,325,514,107,32\nThe cat chased the mouse around the room. => 1318,10501,663,16109,322,8459,6835,322,8355,32\nI love eating pizza with extra cheese. => 59,14290,484,1741,47630,623,6717,8277,30315,32\nHe always wears a hat wherever he goes. => 1331,5182,996,4177,312,25793,2154,424,938,13107,32\nThe flowers in the garden are blooming. => 1318,7290,483,328,322,485,22461,884,323,18466,299,32\nShe danced gracefully on the stage. => 25387,343,6087,31376,4938,544,322,10019,32\nThe dog barked loudly in the park. => 1318,27435,323,1087,318,598,836,631,328,322,880,93,32\nWe went swimming in the ocean yesterday. => 3122,14236,2535,449,6714,328,322,337,18857,39485,32\nHe speaks fluent French and Spanish. => 1331,24498,101,38055,43652,461,14911,1708,32\nThe train arrived at the station on time. => 1318,5683,2099,32114,821,322,18662,544,1133,32\nShe cooked a delicious meal for her family. => 25387,23682,318,312,409,406,2406,597,279,436,7791,13872,32\n" }, { "path": "examples/prompts/test-cases.txt", "content": "# test case format\n# : \n\nEnglish: Hello World!\nEnglish: I can't believe it's already Friday!\"\nEnglish: The URL for the website is https://www.example.com.\"\nEnglish: \"She said, 'I love to travel.'\"\nEnglish: 'The temperature is 25.5°C.'\nEnglish: \"Let's meet at 2:30 p.m. in the park.\"\nEnglish: The book costs $19.99\nEnglish: \"John's favorite color is blue.\"\nEnglish: Th@nk y0u f0r y0ur h3lp!\nEnglish: C@n I g3t a c0ffee, pl3@se?\nEnglish: W0w! Th@t's @m@zing!\nEnglish: H0w 4re y0u t0d@y?\nEnglish: I l0ve t0 tr@vel @r0und the w0rld.\nEnglish: Wh@t's y0ur f@v0rite m0vie?\nEnglish: The cat is sleeping on the mat.\nEnglish: I need to buy some groceries for dinner.\nEnglish: The sun is shining brightly in the sky.\nEnglish: She is reading a book in the park.\nEnglish: We went for a walk on the beach yesterday.\nEnglish: He plays the guitar like a pro.\nEnglish: They are going to the movies tonight.\nEnglish: The flowers are blooming in the garden.\nEnglish: I enjoy listening to classical music.\nEnglish: We need to buy groceries for the week.\nEnglish: The dog is chasing its tail in circles.\nEnglish: She is wearing a beautiful red dress.\nEnglish: He is a talented actor in Hollywood.\nEnglish: The children are playing in the playground.\nEnglish: I'm going to visit my grandparents this weekend.\nEnglish: The coffee tastes bitter without sugar.\nEnglish: They are planning a surprise party for her.\nEnglish: She sings like an angel on stage.\nEnglish: We should take a vacation to relax.\nEnglish: He is studying medicine at the university.\nEnglish: The rain is pouring heavily outside.\nEnglish: I enjoy watching romantic movies.\nEnglish: They are celebrating their anniversary today.\nEnglish: She dances gracefully to the music.\nEnglish: He is an excellent basketball player.\nEnglish: The baby is sleeping soundly in the crib.\nEnglish: I need to finish my homework before dinner.\nEnglish: They are organizing a charity event next month.\nEnglish: She is cooking a delicious meal for us.\nEnglish: We should go hiking in the mountains.\nEnglish: The car broke down on the way to work.\nEnglish: He loves playing video games in his free time.\nEnglish: The birds are chirping in the trees.\nEnglish: I want to learn how to play the piano.\nEnglish: They are building a new shopping mall in the city.\nEnglish: She is writing a novel in her spare time.\nEnglish: We are going to the zoo this Saturday.\nEnglish: The cake looks delicious with chocolate frosting.\nEnglish: He is a talented painter who sells his artwork.\nEnglish: The students are studying for their exams.\nEnglish: I enjoy swimming in the ocean.\nEnglish: They are renovating their house.\nEnglish: She is practicing yoga to stay healthy.\nEnglish: We should plant flowers in the garden.\nEnglish: The traffic is heavy during rush hour.\nEnglish: He is a skilled chef who creates amazing dishes.\nEnglish: The baby is crawling on the floor.\nEnglish: I need to buy a new pair of shoes.\nEnglish: They are going on a road trip across the country.\nEnglish: She is playing the piano beautifully.\nEnglish: We are going to a concert tomorrow night.\nEnglish: The cake tastes delicious with vanilla frosting.\nEnglish: He is a dedicated teacher who inspires his students.\nEnglish: The students are participating in a science fair.\nEnglish: I enjoy hiking in the mountains.\nEnglish: They are organizing a beach cleanup next weekend.\nEnglish: She is taking photographs of nature.\nEnglish: We should try a new restaurant in town.\nEnglish: The traffic is moving slowly on the highway.\nEnglish: He is a talented singer with a beautiful voice.\nEnglish: The baby is laughing and giggling.\nEnglish: I need to do laundry and wash my clothes.\nEnglish: They are planning a trip to Europe.\nEnglish: She is learning how to play the guitar.\nEnglish: We are going to a museum this Sunday.\nEnglish: The coffee smells amazing in the morning.\nEnglish: He is a hardworking farmer who grows crops.\nEnglish: The students are presenting their research projects.\nEnglish: I enjoy playing soccer with my friends.\nEnglish: They are volunteering at a local shelter.\nEnglish: She is practicing martial arts for self-defense.\nEnglish: We should try a new recipe for dinner.\nEnglish: The traffic is congest\nEnglish: The sun is shining brightly today.\nEnglish: I enjoy reading books in my free time.\nEnglish: She plays the piano beautifully.\nEnglish: The cat chased the mouse around the room.\nEnglish: I love eating pizza with extra cheese.\nEnglish: He always wears a hat wherever he goes.\nEnglish: The flowers in the garden are blooming.\nEnglish: She danced gracefully on the stage.\nEnglish: The dog barked loudly in the park.\nEnglish: We went swimming in the ocean yesterday.\nEnglish: He speaks fluent French and Spanish.\nEnglish: The train arrived at the station on time.\nEnglish: She cooked a delicious meal for her family.\nKorean: 이것은 테스트 이다.\nKorean: 걱정할 필요 없다.\nKorean: 버그는 언젠가 고쳐진다.\nJapanese: 明日の天気はどうですか。\nChinese: 请问洗手间在哪里？\nEmoji: I'm feeling 😄 today!\nUnicode: ◑ ▢ ▣ ◱" }, { "path": "examples/prompts/tokenize_huggingface.py", "content": "import os\nfrom transformers import AutoTokenizer\n\nos.environ['TOKENIZERS_PARALLELISM'] = \"false\"\n\nlist_repo_hf = [\"databricks/dolly-v2-3b\", # dolly-v2 (3b, 7b, 12b models share the same tokenizer)\n \"gpt2\", # gpt-2 (gpt2-xl, gpt2-large share the same tokenizer)\n \"uer/gpt2-chinese-cluecorpussmall\", # gpt-2-chinese\n \"EleutherAI/gpt-j-6b\", # gpt-j\n \"EleutherAI/gpt-neox-20b\", # gpt-neox\n \"EleutherAI/polyglot-ko-1.3b\", # gpt-neox (polyglot-ko 5.8b and 12.8b share the same tokenizer\")\n \"rinna/japanese-gpt-neox-3.6b\", # gpt-neox\n # mpt-7b (uses gpt-neox-20b tokenizer)\n \"replit/replit-code-v1-3b\", # replit\n \"bigcode/starcoder\", # starcoder (huggingface-cli login required)\n \"openai/whisper-tiny\" # whisper (base, large, large-v2 share the same tokenizer)\n ]\n\nrepo2ggml = {\"databricks/dolly-v2-3b\" : \"dolly-v2\",\n \"gpt2\" : \"gpt-2\",\n \"uer/gpt2-chinese-cluecorpussmall\" : \"gpt-2-chinese\",\n \"EleutherAI/gpt-j-6b\" : \"gpt-j\",\n \"EleutherAI/gpt-neox-20b\" : \"gpt-neox\",\n \"EleutherAI/polyglot-ko-1.3b\" : \"polyglot-ko\",\n \"rinna/japanese-gpt-neox-3.6b\" : \"gpt-neox-japanese\",\n \"replit/replit-code-v1-3b\" : \"replit\",\n \"bigcode/starcoder\" : \"starcoder\",\n \"openai/whisper-tiny\" : \"whisper\"}\n\nrepo2language = {\"databricks/dolly-v2-3b\" : \"english\",\n \"gpt2\" : \"english\",\n \"uer/gpt2-chinese-cluecorpussmall\" : \"chinese\",\n \"EleutherAI/gpt-j-6b\" : \"english\",\n \"EleutherAI/gpt-neox-20b\" : \"english\",\n \"EleutherAI/polyglot-ko-1.3b\" : \"korean\",\n \"rinna/japanese-gpt-neox-3.6b\" : \"japanese\",\n \"replit/replit-code-v1-3b\" : \"english\",\n \"bigcode/starcoder\" : \"english\",\n \"openai/whisper-tiny\" : \"english\"}\n\ndelimeter = \": \"\ntest_sentences = []\nwith open(\"test-cases.txt\", \"r\") as f:\n lines = [l.rstrip() for l in f.readlines()]\n for l in lines:\n if delimeter in l:\n language = l[:l.index(delimeter)]\n sentence = l[l.index(delimeter) + len(delimeter):]\n test_sentences.append((language.lower(), sentence))\n\nfor repo in list_repo_hf:\n\n target_language = repo2language[repo]\n\n tokenizer = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)\n\n tokens_hf = []\n for language, sentence in test_sentences:\n if language == target_language:\n tokens = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentence))\n tokens_hf.append((sentence, tokens))\n\n save_txt = repo2ggml[repo] + \".txt\"\n with open(save_txt, \"w\") as f:\n f.writelines([sentence + \" => \" + \",\".join(str(t) for t in tokens) + \"\\n\" for sentence, tokens in tokens_hf])\n" }, { "path": "examples/prompts/whisper.txt", "content": "Hello World! => 15947,3937,0\nI can't believe it's already Friday!\" => 40,393,380,1697,309,311,1217,6984,2963\nThe URL for the website is https://www.example.com.\" => 2278,12905,337,220,3322,3144,307,34426,21492,17919,13,3121,335,781,13,1112,889\n\"She said, 'I love to travel.'\" => 1,9526,848,11,922,40,959,220,1353,220,17227,779,28763\n'The temperature is 25.5°C.' => 6,2278,220,18275,610,1503,307,3552,13,20,11782,34,4443\n\"Let's meet at 2:30 p.m. in the park.\" => 1,8373,311,1677,412,568,25,3446,280,13,76,13,294,220,3322,3884,889\nThe book costs $19.99 => 2278,1446,5497,1848,3405,13,8494\n\"John's favorite color is blue.\" => 1,16938,311,2954,2017,307,3344,889\nTh@nk y0u f0r y0ur h3lp! => 2434,31,77,74,288,15,84,283,15,81,288,15,374,276,18,75,79,0\nC@n I g3t a c0ffee, pl3@se? => 34,31,77,286,290,18,83,257,269,15,4617,11,499,18,31,405,30\nW0w! Th@t's @m@zing! => 54,15,86,0,334,31,83,311,10428,76,31,8781,0\nH0w 4re y0u t0d@y? => 39,15,86,1017,265,288,15,84,220,83,15,67,31,88,30\nI l0ve t0 tr@vel @r0und the w0rld. => 40,287,15,303,220,83,15,220,6903,31,779,10428,81,15,997,220,3322,261,15,81,348,13\nWh@t's y0ur f@v0rite m0vie? => 2471,31,83,311,288,15,374,283,31,85,15,35002,275,15,12702,30\nThe cat is sleeping on the mat. => 2278,3857,307,8296,322,220,3322,3803,13\nI need to buy some groceries for dinner. => 40,643,220,1353,2256,512,31391,337,6148,13\nThe sun is shining brightly in the sky. => 2278,3295,307,18269,47418,294,220,3322,5443,13\nShe is reading a book in the park. => 9526,307,3760,257,1446,294,220,3322,3884,13\nWe went for a walk on the beach yesterday. => 4360,1437,337,257,1792,322,220,3322,7534,5186,13\nHe plays the guitar like a pro. => 5205,5749,220,3322,7531,411,257,447,13\nThey are going to the movies tonight. => 8829,366,516,220,1353,220,3322,6233,220,1756,397,13\nThe flowers are blooming in the garden. => 2278,8085,366,45294,294,220,3322,7431,13\nI enjoy listening to classical music. => 40,2103,4764,220,1353,13735,1318,13\nWe need to buy groceries for the week. => 4360,643,220,1353,2256,31391,337,220,3322,1243,13\nThe dog is chasing its tail in circles. => 2278,3000,307,17876,1080,220,14430,294,13040,13\nShe is wearing a beautiful red dress. => 9526,307,4769,257,2238,2182,5231,13\nHe is a talented actor in Hollywood. => 5205,307,257,220,32831,6003,8747,294,11628,13\nThe children are playing in the playground. => 2278,2227,366,2433,294,220,3322,24646,13\nI'm going to visit my grandparents this weekend. => 40,478,516,220,1353,3441,452,21876,220,11176,6711,13\nThe coffee tastes bitter without sugar. => 2278,4982,220,83,40246,13871,1553,5076,13\nThey are planning a surprise party for her. => 8829,366,5038,257,6365,3595,337,720,13\nShe sings like an angel on stage. => 9526,23250,411,364,14250,322,3233,13\nWe should take a vacation to relax. => 4360,820,220,27612,257,12830,220,1353,5789,13\nHe is studying medicine at the university. => 5205,307,7601,7195,412,220,3322,5454,13\nThe rain is pouring heavily outside. => 2278,4830,307,20450,10950,2380,13\nI enjoy watching romantic movies. => 40,2103,1976,13590,6233,13\nThey are celebrating their anniversary today. => 8829,366,15252,220,3322,347,12962,220,83,378,320,13\nShe dances gracefully to the music. => 9526,28322,10042,2277,220,1353,220,3322,1318,13\nHe is an excellent basketball player. => 5205,307,364,7103,11767,4256,13\nThe baby is sleeping soundly in the crib. => 2278,3186,307,8296,1626,356,294,220,3322,47163,13\nI need to finish my homework before dinner. => 40,643,220,1353,2413,452,14578,949,6148,13\nThey are organizing a charity event next month. => 8829,366,17608,257,16863,2280,958,1618,13\nShe is cooking a delicious meal for us. => 9526,307,6361,257,4809,6791,337,505,13\nWe should go hiking in the mountains. => 4360,820,352,23784,294,220,3322,10233,13\nThe car broke down on the way to work. => 2278,1032,6902,760,322,220,3322,636,220,1353,589,13\nHe loves playing video games in his free time. => 5205,6752,2433,960,2813,294,702,1737,220,3766,13\nThe birds are chirping in the trees. => 2278,9009,366,36682,294,220,3322,220,3599,279,13\nI want to learn how to play the piano. => 40,528,220,1353,1466,577,220,1353,862,220,3322,9211,13\nThey are building a new shopping mall in the city. => 8829,366,2390,257,777,8688,16026,294,220,3322,2307,13\nShe is writing a novel in her spare time. => 9526,307,3579,257,7613,294,720,13798,220,3766,13\nWe are going to the zoo this Saturday. => 4360,366,516,220,1353,220,3322,25347,220,11176,8803,13\nThe cake looks delicious with chocolate frosting. => 2278,5908,1542,4809,365,6215,37048,13\nHe is a talented painter who sells his artwork. => 5205,307,257,220,32831,6003,26619,567,20897,702,15829,13\nThe students are studying for their exams. => 2278,1731,366,7601,337,220,3322,347,20514,13\nI enjoy swimming in the ocean. => 40,2103,11989,294,220,3322,7810,13\nThey are renovating their house. => 8829,366,18845,990,220,3322,347,1782,13\nShe is practicing yoga to stay healthy. => 9526,307,11350,15128,220,1353,1754,4627,13\nWe should plant flowers in the garden. => 4360,820,3709,8085,294,220,3322,7431,13\nThe traffic is heavy during rush hour. => 2278,220,17227,3341,307,4676,1830,9300,1773,13\nHe is a skilled chef who creates amazing dishes. => 5205,307,257,19690,10530,567,7829,2243,10814,13\nThe baby is crawling on the floor. => 2278,3186,307,32979,322,220,3322,4123,13\nI need to buy a new pair of shoes. => 40,643,220,1353,2256,257,777,6119,295,6654,13\nThey are going on a road trip across the country. => 8829,366,516,322,257,3060,220,83,8400,2108,220,3322,1941,13\nShe is playing the piano beautifully. => 9526,307,2433,220,3322,9211,16525,13\nWe are going to a concert tomorrow night. => 4360,366,516,220,1353,257,8543,220,83,298,3162,1818,13\nThe cake tastes delicious with vanilla frosting. => 2278,5908,220,83,40246,4809,365,17528,37048,13\nHe is a dedicated teacher who inspires his students. => 5205,307,257,8374,220,975,4062,567,32566,702,1731,13\nThe students are participating in a science fair. => 2278,1731,366,13950,294,257,3497,3143,13\nI enjoy hiking in the mountains. => 40,2103,23784,294,220,3322,10233,13\nThey are organizing a beach cleanup next weekend. => 8829,366,17608,257,7534,40991,958,6711,13\nShe is taking photographs of nature. => 9526,307,220,48625,17649,295,3687,13\nWe should try a new restaurant in town. => 4360,820,220,83,627,257,777,6383,294,220,30401,13\nThe traffic is moving slowly on the highway. => 2278,220,17227,3341,307,2684,5692,322,220,3322,17205,13\nHe is a talented singer with a beautiful voice. => 5205,307,257,220,32831,6003,11564,365,257,2238,3177,13\nThe baby is laughing and giggling. => 2278,3186,307,5059,293,290,24542,13\nI need to do laundry and wash my clothes. => 40,643,220,1353,360,19811,293,5675,452,5534,13\nThey are planning a trip to Europe. => 8829,366,5038,257,220,83,8400,220,1353,3315,13\nShe is learning how to play the guitar. => 9526,307,2539,577,220,1353,862,220,3322,7531,13\nWe are going to a museum this Sunday. => 4360,366,516,220,1353,257,8441,220,11176,7776,13\nThe coffee smells amazing in the morning. => 2278,4982,10036,2243,294,220,3322,2446,13\nHe is a hardworking farmer who grows crops. => 5205,307,257,1152,22475,17891,567,13156,16829,13\nThe students are presenting their research projects. => 2278,1731,366,15578,220,3322,347,2132,4455,13\nI enjoy playing soccer with my friends. => 40,2103,2433,15469,365,452,1855,13\nThey are volunteering at a local shelter. => 8829,366,33237,412,257,2654,13341,13\nShe is practicing martial arts for self-defense. => 9526,307,11350,20755,8609,337,2698,12,49268,13\nWe should try a new recipe for dinner. => 4360,820,220,83,627,257,777,6782,337,6148,13\nThe traffic is congest => 2278,220,17227,3341,307,31871\nThe sun is shining brightly today. => 2278,3295,307,18269,47418,220,83,378,320,13\nI enjoy reading books in my free time. => 40,2103,3760,3642,294,452,1737,220,3766,13\nShe plays the piano beautifully. => 9526,5749,220,3322,9211,16525,13\nThe cat chased the mouse around the room. => 2278,3857,33091,220,3322,9719,926,220,3322,1808,13\nI love eating pizza with extra cheese. => 40,959,3936,8298,365,2857,5399,13\nHe always wears a hat wherever he goes. => 5205,1009,20877,257,2385,8660,415,1709,13\nThe flowers in the garden are blooming. => 2278,8085,294,220,3322,7431,366,45294,13\nShe danced gracefully on the stage. => 9526,32909,10042,2277,322,220,3322,3233,13\nThe dog barked loudly in the park. => 2278,3000,16202,292,22958,294,220,3322,3884,13\nWe went swimming in the ocean yesterday. => 4360,1437,11989,294,220,3322,7810,5186,13\nHe speaks fluent French and Spanish. => 5205,10789,40799,5522,293,8058,13\nThe train arrived at the station on time. => 2278,220,83,7146,6678,412,220,3322,5214,322,220,3766,13\nShe cooked a delicious meal for her family. => 9526,9267,257,4809,6791,337,720,1605,13\n" }, { "path": "examples/python/README.md", "content": "# Simple autogenerated Python bindings for ggml\n\nThis folder contains:\n\n- Scripts to generate full Python bindings from ggml headers (+ stubs for autocompletion in IDEs)\n- Some barebones utils (see [ggml/utils.py](./ggml/utils.py)):\n - `ggml.utils.init` builds a context that's freed automatically when the pointer gets GC'd\n - `ggml.utils.copy` **copies between same-shaped tensors (numpy or ggml), w/ automatic (de/re)quantization**\n - `ggml.utils.numpy` returns a numpy view over a ggml tensor; if it's quantized, it returns a copy (requires `allow_copy=True`)\n- Very basic examples (anyone wants to port [llama2.c](https://github.com/karpathy/llama2.c)?)\n\nProvided you set `GGML_LIBRARY=.../path/to/libggml_shared.so` (see instructions below), it's trivial to do some operations on quantized tensors:\n\n```python\n# Make sure libllama.so is in your [DY]LD_LIBRARY_PATH, or set GGML_LIBRARY=.../libggml_shared.so\n\nfrom ggml import lib, ffi\nfrom ggml.utils import init, copy, numpy\nimport numpy as np\n\nctx = init(mem_size=12*1024*1024)\nn = 256\nn_threads = 4\n\na = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)\nb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # Can't both be quantized\nsum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too!\n\ngf = ffi.new('struct ggml_cgraph*')\nlib.ggml_build_forward_expand(gf, sum)\n\ncopy(np.array([i for i in range(n)], np.float32), a)\ncopy(np.array([i*100 for i in range(n)], np.float32), b)\n\nlib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)\n\nprint(numpy(a, allow_copy=True))\n# 0. 1.0439453 2.0878906 3.131836 4.1757812 5.2197266. ...\nprint(numpy(b))\n# 0. 100. 200. 300. 400. 500. ...\nprint(numpy(sum, allow_copy=True))\n# 0. 105.4375 210.875 316.3125 421.75 527.1875 ...\n```\n\n### Prerequisites\n\nYou'll need a shared library of ggml to use the bindings.\n\n#### Build libggml_shared.so or libllama.so\n\nAs of this writing the best is to use [ggerganov/llama.cpp](https://github.com/ggerganov/llama.cpp)'s generated `libggml_shared.so` or `libllama.so`, which you can build as follows:\n\n```bash\ngit clone https://github.com/ggerganov/llama.cpp\n# On a CUDA-enabled system add -DLLAMA_CUDA=1\n# On a Mac add -DLLAMA_METAL=1\ncmake llama.cpp \\\n -B llama_build \\\n -DCMAKE_C_FLAGS=-Ofast \\\n -DLLAMA_NATIVE=1 \\\n -DLLAMA_LTO=1 \\\n -DBUILD_SHARED_LIBS=1 \\\n -DLLAMA_MPI=1 \\\n -DLLAMA_BUILD_TESTS=0 \\\n -DLLAMA_BUILD_EXAMPLES=0\n( cd llama_build && make -j )\n\n# On Mac, this will be libggml_shared.dylib instead\nexport GGML_LIBRARY=$PWD/llama_build/libggml_shared.so\n# Alternatively, you can just copy it to your system's lib dir, e.g /usr/local/lib\n```\n\n#### (Optional) Regenerate the bindings and stubs\n\nIf you added or changed any signatures of the C API, you'll want to regenerate the bindings ([ggml/cffi.py](./ggml/cffi.py)) and stubs ([ggml/__init__.pyi](./ggml/__init__.pyi)).\n\nLuckily it's a one-liner using [regenerate.py](./regenerate.py):\n\n```bash\npip install -q cffi\n\npython regenerate.py\n```\n\nBy default it assumes `llama.cpp` was cloned in ../../../llama.cpp (alongside the ggml folder). You can override this with:\n\n```bash\nC_INCLUDE_DIR=$LLAMA_CPP_DIR python regenerate.py\n```\n\nYou can also edit [api.h](./api.h) to control which files should be included in the generated bindings (defaults to `llama.cpp/ggml*.h`)\n\nIn fact, if you wanted to only generate bindings for the current version of the `ggml` repo itself (instead of `llama.cpp`; you'd loose support for k-quants), you could run:\n\n```bash\nAPI=../../include/ggml.h python regenerate.py\n```\n\n## Develop\n\nRun tests:\n\n```bash\npytest\n```\n\n### Alternatives\n\nThis example's goal is to showcase [cffi](https://cffi.readthedocs.io/)-generated bindings that are trivial to use and update, but there are already alternatives in the wild:\n\n- https://github.com/abetlen/ggml-python: these bindings seem to be hand-written and use [ctypes](https://docs.python.org/3/library/ctypes.html). It has [high-quality API reference docs](https://ggml-python.readthedocs.io/en/latest/api-reference/#ggml.ggml) that can be used with these bindings too, but it doesn't expose Metal, CUDA, MPI or OpenCL calls, doesn't support transparent (de/re)quantization like this example does (see [ggml.utils](./ggml/utils.py) module), and won't pick up your local changes.\n\n- https://github.com/abetlen/llama-cpp-python: these expose the C++ `llama.cpp` interface, which this example cannot easily be extended to support (`cffi` only generates bindings of C libraries)\n\n- [pybind11](https://github.com/pybind/pybind11) and [nanobind](https://github.com/wjakob/nanobind) are two alternatives to cffi that support binding C++ libraries, but it doesn't seem either of them have an automatic generator (writing bindings is rather time-consuming).\n" }, { "path": "examples/python/api.h", "content": "/*\n List here all the headers you want to expose in the Python bindings,\n then run `python regenerate.py` (see details in README.md)\n*/\n\n#include \"ggml.h\"\n#include \"ggml-metal.h\"\n#include \"ggml-opencl.h\"\n\n// Headers below are currently only present in the llama.cpp repository, comment them out if you don't have them.\n#include \"k_quants.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-cuda.h\"\n#include \"ggml-mpi.h\"" }, { "path": "examples/python/example_add_quant.py", "content": "from ggml import lib, ffi\nfrom ggml.utils import init, copy, numpy\nimport numpy as np\n\nctx = init(mem_size=12*1024*1024) # automatically freed when pointer is GC'd\nn = 256\nn_threads = 4\n\na = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)\nb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # can't both be quantized\nsum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too!\n\n# See cffi's doc on how to allocate native memory: it's very simple!\n# https://cffi.readthedocs.io/en/latest/ref.html#ffi-interface\ngf = ffi.new('struct ggml_cgraph*')\nlib.ggml_build_forward_expand(gf, sum)\n\ncopy(np.array([i for i in range(n)], np.float32), a)\ncopy(np.array([i*100 for i in range(n)], np.float32), b)\n\nlib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)\n\nprint(numpy(a, allow_copy=True))\nprint(numpy(b))\nprint(numpy(sum, allow_copy=True))" }, { "path": "examples/python/example_test_all_quants.py", "content": "from ggml import ffi, lib\nfrom ggml.utils import init, numpy, copy\nimport numpy as np\nfrom math import pi, cos, sin, ceil\n\nimport matplotlib.pyplot as plt\n\nctx = init(mem_size=100*1024*1024) # Will be auto-GC'd\nn = 256\n\norig = np.array([\n [\n cos(j * 2 * pi / n) * (sin(i * 2 * pi / n))\n for j in range(n)\n ]\n for i in range(n)\n], np.float32)\norig_tensor = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, n, n)\ncopy(orig, orig_tensor)\n\nquants = [\n type for type in range(lib.GGML_TYPE_COUNT)\n if lib.ggml_is_quantized(type) and\n type not in [lib.GGML_TYPE_Q8_1, lib.GGML_TYPE_Q8_K] # Apparently not supported\n]\n# quants = [lib.GGML_TYPE_Q2_K] # Test a single one\n\ndef get_name(type):\n name = lib.ggml_type_name(type)\n return ffi.string(name).decode('utf-8') if name else '?'\n\nquants.sort(key=get_name)\nquants.insert(0, None)\nprint(quants)\n\nncols=4\nnrows = ceil(len(quants) / ncols)\n\nplt.figure(figsize=(ncols * 5, nrows * 5), layout='tight')\n\nfor i, type in enumerate(quants):\n plt.subplot(nrows, ncols, i + 1)\n try:\n if type == None:\n plt.title('Original')\n plt.imshow(orig)\n else:\n quantized_tensor = lib.ggml_new_tensor_2d(ctx, type, n, n)\n copy(orig_tensor, quantized_tensor)\n quantized = numpy(quantized_tensor, allow_copy=True)\n d = quantized - orig\n results = {\n \"l2\": np.linalg.norm(d, 2),\n \"linf\": np.linalg.norm(d, np.inf),\n \"compression\":\n round(lib.ggml_nbytes(orig_tensor) /\n lib.ggml_nbytes(quantized_tensor), 1)\n }\n name = get_name(type)\n print(f'{name}: {results}')\n\n plt.title(f'{name} ({results[\"compression\"]}x smaller)')\n plt.imshow(quantized, interpolation='nearest')\n \n except Exception as e:\n print(f'Error: {e}')\n\nplt.show()" }, { "path": "examples/python/ggml/__init__.py", "content": "\"\"\"\n Python bindings for the ggml library.\n\n Usage example:\n\n from ggml import lib, ffi\n from ggml.utils import init, copy, numpy\n import numpy as np\n\n ctx = init(mem_size=10*1024*1024)\n n = 1024\n n_threads = 4\n\n a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)\n b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)\n sum = lib.ggml_add(ctx, a, b)\n\n gf = ffi.new('struct ggml_cgraph*')\n lib.ggml_build_forward_expand(gf, sum)\n\n copy(np.array([i for i in range(n)], np.float32), a)\n copy(np.array([i*100 for i in range(n)], np.float32), b)\n lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)\n\n print(numpy(sum, allow_copy=True))\n\n See https://cffi.readthedocs.io/en/latest/cdef.html for more on cffi.\n\"\"\"\n\ntry:\n from ggml.cffi import ffi as ffi\nexcept ImportError as e:\n raise ImportError(f\"Couldn't find ggml bindings ({e}). Run `python regenerate.py` or check your PYTHONPATH.\")\n\nimport os, platform\n\n__exact_library = os.environ.get(\"GGML_LIBRARY\")\nif __exact_library:\n __candidates = [__exact_library]\nelif platform.system() == \"Windows\":\n __candidates = [\"ggml_shared.dll\", \"llama.dll\"]\nelse:\n __candidates = [\"libggml_shared.so\", \"libllama.so\"]\n if platform.system() == \"Darwin\":\n __candidates += [\"libggml_shared.dylib\", \"libllama.dylib\"]\n\nfor i, name in enumerate(__candidates):\n try:\n # This is where all the functions, enums and constants are defined\n lib = ffi.dlopen(name)\n except OSError:\n if i < len(__candidates) - 1:\n continue\n raise OSError(f\"Couldn't find ggml's shared library (tried names: {__candidates}). Add its directory to DYLD_LIBRARY_PATH (on Mac) or LD_LIBRARY_PATH, or define GGML_LIBRARY.\")\n\n# This contains the cffi helpers such as new, cast, string, etc.\n# https://cffi.readthedocs.io/en/latest/ref.html#ffi-interface\nffi = ffi\n" }, { "path": "examples/python/ggml/__init__.pyi", "content": "# auto-generated file\nimport ggml.ffi as ffi\nimport numpy as np\nclass lib:\n @property\n def GGML_BACKEND_CPU(self) -> int: ...\n @property\n def GGML_BACKEND_GPU(self) -> int: ...\n @property\n def GGML_BACKEND_GPU_SPLIT(self) -> int: ...\n @property\n def GGML_FTYPE_ALL_F32(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_F16(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q2_K(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q3_K(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q4_0(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q4_1(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q4_1_SOME_F16(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q4_K(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q5_0(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q5_1(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q5_K(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q6_K(self) -> int: ...\n @property\n def GGML_FTYPE_MOSTLY_Q8_0(self) -> int: ...\n @property\n def GGML_FTYPE_UNKNOWN(self) -> int: ...\n @property\n def GGML_LINESEARCH_BACKTRACKING_ARMIJO(self) -> int: ...\n @property\n def GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE(self) -> int: ...\n @property\n def GGML_LINESEARCH_BACKTRACKING_WOLFE(self) -> int: ...\n @property\n def GGML_LINESEARCH_DEFAULT(self) -> int: ...\n @property\n def GGML_LINESEARCH_FAIL(self) -> int: ...\n @property\n def GGML_LINESEARCH_INVALID_PARAMETERS(self) -> int: ...\n @property\n def GGML_LINESEARCH_MAXIMUM_ITERATIONS(self) -> int: ...\n @property\n def GGML_LINESEARCH_MAXIMUM_STEP(self) -> int: ...\n @property\n def GGML_LINESEARCH_MINIMUM_STEP(self) -> int: ...\n @property\n def GGML_OBJECT_GRAPH(self) -> int: ...\n @property\n def GGML_OBJECT_TENSOR(self) -> int: ...\n @property\n def GGML_OBJECT_WORK_BUFFER(self) -> int: ...\n @property\n def GGML_OPT_ADAM(self) -> int: ...\n @property\n def GGML_OPT_DID_NOT_CONVERGE(self) -> int: ...\n @property\n def GGML_OPT_FAIL(self) -> int: ...\n @property\n def GGML_OPT_INVALID_WOLFE(self) -> int: ...\n @property\n def GGML_OPT_LBFGS(self) -> int: ...\n @property\n def GGML_OPT_NO_CONTEXT(self) -> int: ...\n @property\n def GGML_OPT_OK(self) -> int: ...\n @property\n def GGML_OP_ACC(self) -> int: ...\n @property\n def GGML_OP_ADD(self) -> int: ...\n @property\n def GGML_OP_ADD1(self) -> int: ...\n @property\n def GGML_OP_ALIBI(self) -> int: ...\n @property\n def GGML_OP_ARGMAX(self) -> int: ...\n @property\n def GGML_OP_CLAMP(self) -> int: ...\n @property\n def GGML_OP_CONT(self) -> int: ...\n @property\n def GGML_OP_CONV_1D(self) -> int: ...\n @property\n def GGML_OP_CONV_2D(self) -> int: ...\n @property\n def GGML_OP_COUNT(self) -> int: ...\n @property\n def GGML_OP_CPY(self) -> int: ...\n @property\n def GGML_OP_CROSS_ENTROPY_LOSS(self) -> int: ...\n @property\n def GGML_OP_CROSS_ENTROPY_LOSS_BACK(self) -> int: ...\n @property\n def GGML_OP_DIAG(self) -> int: ...\n @property\n def GGML_OP_DIAG_MASK_INF(self) -> int: ...\n @property\n def GGML_OP_DIAG_MASK_ZERO(self) -> int: ...\n @property\n def GGML_OP_DIV(self) -> int: ...\n @property\n def GGML_OP_DUP(self) -> int: ...\n @property\n def GGML_OP_FLASH_ATTN(self) -> int: ...\n @property\n def GGML_OP_FLASH_ATTN_BACK(self) -> int: ...\n @property\n def GGML_OP_FLASH_FF(self) -> int: ...\n @property\n def GGML_OP_GET_ROWS(self) -> int: ...\n @property\n def GGML_OP_GET_ROWS_BACK(self) -> int: ...\n @property\n def GGML_OP_LOG(self) -> int: ...\n @property\n def GGML_OP_MAP_BINARY(self) -> int: ...\n @property\n def GGML_OP_MAP_CUSTOM1(self) -> int: ...\n @property\n def GGML_OP_MAP_CUSTOM1_F32(self) -> int: ...\n @property\n def GGML_OP_MAP_CUSTOM2(self) -> int: ...\n @property\n def GGML_OP_MAP_CUSTOM2_F32(self) -> int: ...\n @property\n def GGML_OP_MAP_CUSTOM3(self) -> int: ...\n @property\n def GGML_OP_MAP_CUSTOM3_F32(self) -> int: ...\n @property\n def GGML_OP_MAP_UNARY(self) -> int: ...\n @property\n def GGML_OP_MEAN(self) -> int: ...\n @property\n def GGML_OP_MUL(self) -> int: ...\n @property\n def GGML_OP_MUL_MAT(self) -> int: ...\n @property\n def GGML_OP_NONE(self) -> int: ...\n @property\n def GGML_OP_NORM(self) -> int: ...\n @property\n def GGML_OP_OUT_PROD(self) -> int: ...\n @property\n def GGML_OP_PERMUTE(self) -> int: ...\n @property\n def GGML_OP_POOL_1D(self) -> int: ...\n @property\n def GGML_OP_POOL_2D(self) -> int: ...\n @property\n def GGML_OP_POOL_AVG(self) -> int: ...\n @property\n def GGML_OP_POOL_COUNT(self) -> int: ...\n @property\n def GGML_OP_POOL_MAX(self) -> int: ...\n @property\n def GGML_OP_REPEAT(self) -> int: ...\n @property\n def GGML_OP_REPEAT_BACK(self) -> int: ...\n @property\n def GGML_OP_RESHAPE(self) -> int: ...\n @property\n def GGML_OP_RMS_NORM(self) -> int: ...\n @property\n def GGML_OP_RMS_NORM_BACK(self) -> int: ...\n @property\n def GGML_OP_ROPE(self) -> int: ...\n @property\n def GGML_OP_ROPE_BACK(self) -> int: ...\n @property\n def GGML_OP_SCALE(self) -> int: ...\n @property\n def GGML_OP_SET(self) -> int: ...\n @property\n def GGML_OP_SILU_BACK(self) -> int: ...\n @property\n def GGML_OP_SOFT_MAX(self) -> int: ...\n @property\n def GGML_OP_SOFT_MAX_BACK(self) -> int: ...\n @property\n def GGML_OP_SQR(self) -> int: ...\n @property\n def GGML_OP_SQRT(self) -> int: ...\n @property\n def GGML_OP_SUB(self) -> int: ...\n @property\n def GGML_OP_SUM(self) -> int: ...\n @property\n def GGML_OP_SUM_ROWS(self) -> int: ...\n @property\n def GGML_OP_TRANSPOSE(self) -> int: ...\n @property\n def GGML_OP_UNARY(self) -> int: ...\n @property\n def GGML_OP_VIEW(self) -> int: ...\n @property\n def GGML_OP_WIN_PART(self) -> int: ...\n @property\n def GGML_OP_WIN_UNPART(self) -> int: ...\n @property\n def GGML_TASK_COMPUTE(self) -> int: ...\n @property\n def GGML_TASK_FINALIZE(self) -> int: ...\n @property\n def GGML_TASK_INIT(self) -> int: ...\n @property\n def GGML_TYPE_COUNT(self) -> int: ...\n @property\n def GGML_TYPE_F16(self) -> int: ...\n @property\n def GGML_TYPE_F32(self) -> int: ...\n @property\n def GGML_TYPE_I16(self) -> int: ...\n @property\n def GGML_TYPE_I32(self) -> int: ...\n @property\n def GGML_TYPE_I8(self) -> int: ...\n @property\n def GGML_TYPE_Q2_K(self) -> int: ...\n @property\n def GGML_TYPE_Q3_K(self) -> int: ...\n @property\n def GGML_TYPE_Q4_0(self) -> int: ...\n @property\n def GGML_TYPE_Q4_1(self) -> int: ...\n @property\n def GGML_TYPE_Q4_K(self) -> int: ...\n @property\n def GGML_TYPE_Q5_0(self) -> int: ...\n @property\n def GGML_TYPE_Q5_1(self) -> int: ...\n @property\n def GGML_TYPE_Q5_K(self) -> int: ...\n @property\n def GGML_TYPE_Q6_K(self) -> int: ...\n @property\n def GGML_TYPE_Q8_0(self) -> int: ...\n @property\n def GGML_TYPE_Q8_1(self) -> int: ...\n @property\n def GGML_TYPE_Q8_K(self) -> int: ...\n @property\n def GGML_UNARY_OP_ABS(self) -> int: ...\n @property\n def GGML_UNARY_OP_ELU(self) -> int: ...\n @property\n def GGML_UNARY_OP_GELU(self) -> int: ...\n @property\n def GGML_UNARY_OP_GELU_QUICK(self) -> int: ...\n @property\n def GGML_UNARY_OP_NEG(self) -> int: ...\n @property\n def GGML_UNARY_OP_RELU(self) -> int: ...\n @property\n def GGML_UNARY_OP_SGN(self) -> int: ...\n @property\n def GGML_UNARY_OP_SILU(self) -> int: ...\n @property\n def GGML_UNARY_OP_STEP(self) -> int: ...\n @property\n def GGML_UNARY_OP_TANH(self) -> int: ...\n @property\n def GGUF_TYPE_ARRAY(self) -> int: ...\n @property\n def GGUF_TYPE_BOOL(self) -> int: ...\n @property\n def GGUF_TYPE_COUNT(self) -> int: ...\n @property\n def GGUF_TYPE_FLOAT32(self) -> int: ...\n @property\n def GGUF_TYPE_INT16(self) -> int: ...\n @property\n def GGUF_TYPE_INT32(self) -> int: ...\n @property\n def GGUF_TYPE_INT8(self) -> int: ...\n @property\n def GGUF_TYPE_STRING(self) -> int: ...\n @property\n def GGUF_TYPE_UINT16(self) -> int: ...\n @property\n def GGUF_TYPE_UINT32(self) -> int: ...\n @property\n def GGUF_TYPE_UINT8(self) -> int: ...\n def abort_callback(data: ffi.CData) -> bool:\n \"\"\"\n abort ggml_graph_compute when true\n\n bool (*abort_callback)(void * data);\n \"\"\"\n ...\n def dequantize_row_q2_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"\n Dequantization\n\n void dequantize_row_q2_K(const block_q2_K * restrict x, float * restrict y, int k);\n \"\"\"\n ...\n def dequantize_row_q3_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int k);\"\"\"\n ...\n def dequantize_row_q4_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void dequantize_row_q4_K(const block_q4_K * restrict x, float * restrict y, int k);\"\"\"\n ...\n def dequantize_row_q5_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int k);\"\"\"\n ...\n def dequantize_row_q6_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int k);\"\"\"\n ...\n def dequantize_row_q8_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int k);\"\"\"\n ...\n def ggml_abs(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_abs(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_abs_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_abs_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_acc(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_acc(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t nb2,\n size_t nb3,\n size_t offset);\n \"\"\"\n ...\n def ggml_acc_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_acc_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t nb2,\n size_t nb3,\n size_t offset);\n \"\"\"\n ...\n def ggml_add(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_add(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_add1(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_add1(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_add1_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_add1_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_add_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_add_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_alibi(ctx: ffi.CData, a: ffi.CData, n_past: int, n_head: int, bias_max: float) -> ffi.CData:\n \"\"\"\n alibi position embedding\n in-place, returns view(a)\n\n struct ggml_tensor * ggml_alibi(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past,\n int n_head,\n float bias_max);\n \"\"\"\n ...\n def ggml_allocr_alloc(alloc: ffi.CData, tensor: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_allocr_alloc_graph(alloc: ffi.CData, graph: ffi.CData) -> int:\n \"\"\"GGML_API size_t ggml_allocr_alloc_graph(struct ggml_allocr * alloc, struct ggml_cgraph * graph);\"\"\"\n ...\n def ggml_allocr_free(alloc: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_allocr_free(struct ggml_allocr * alloc);\"\"\"\n ...\n def ggml_allocr_is_measure(alloc: ffi.CData) -> bool:\n \"\"\"GGML_API bool ggml_allocr_is_measure(struct ggml_allocr * alloc);\"\"\"\n ...\n def ggml_allocr_new(data: ffi.CData, size: int, alignment: int) -> ffi.CData:\n \"\"\"GGML_API struct ggml_allocr * ggml_allocr_new(void * data, size_t size, size_t alignment);\"\"\"\n ...\n def ggml_allocr_new_measure(alignment: int) -> ffi.CData:\n \"\"\"GGML_API struct ggml_allocr * ggml_allocr_new_measure(size_t alignment);\"\"\"\n ...\n def ggml_allocr_reset(alloc: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_allocr_reset(struct ggml_allocr * alloc);\"\"\"\n ...\n def ggml_allocr_set_parse_seq(alloc: ffi.CData, list: ffi.CData, n: int) -> None:\n \"\"\"\n tell the allocator to parse nodes following the order described in the list\n you should call this if your graph are optimized to execute out-of-order\n\n GGML_API void ggml_allocr_set_parse_seq(struct ggml_allocr * alloc, int * list, int n);\n \"\"\"\n ...\n def ggml_are_same_shape(t0: ffi.CData, t1: ffi.CData) -> bool:\n \"\"\" GGML_API bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1);\"\"\"\n ...\n def ggml_argmax(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n argmax along rows\n\n GGML_API struct ggml_tensor * ggml_argmax(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_blck_size(type: int) -> int:\n \"\"\" GGML_API int ggml_blck_size (enum ggml_type type);\"\"\"\n ...\n def ggml_build_backward(ctx: ffi.CData, gf: ffi.CData, keep: bool) -> ffi.CData:\n \"\"\" GGML_API struct ggml_cgraph ggml_build_backward(struct ggml_context * ctx, struct ggml_cgraph * gf, bool keep);\"\"\"\n ...\n def ggml_build_forward(tensor: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct ggml_cgraph ggml_build_forward (struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_build_forward_ctx(ctx: ffi.CData, tensor: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct ggml_cgraph * ggml_build_forward_ctx(struct ggml_context * ctx, struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_build_forward_expand(cgraph: ffi.CData, tensor: ffi.CData) -> None:\n \"\"\" GGML_API void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_cl_can_mul_mat(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData) -> bool:\n \"\"\"bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);\"\"\"\n ...\n def ggml_cl_free_data(tensor: ffi.CData) -> None:\n \"\"\"void ggml_cl_free_data(const struct ggml_tensor* tensor);\"\"\"\n ...\n def ggml_cl_host_free(ptr: ffi.CData) -> None:\n \"\"\"void ggml_cl_host_free(void * ptr);\"\"\"\n ...\n def ggml_cl_host_malloc(size: int) -> ffi.CData:\n \"\"\"void * ggml_cl_host_malloc(size_t size);\"\"\"\n ...\n def ggml_cl_init() -> None:\n \"\"\"void ggml_cl_init(void);\"\"\"\n ...\n def ggml_cl_mul(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData) -> None:\n \"\"\"void ggml_cl_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);\"\"\"\n ...\n def ggml_cl_mul_mat(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData, wdata: ffi.CData, wsize: int) -> None:\n \"\"\"void ggml_cl_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize);\"\"\"\n ...\n def ggml_cl_mul_mat_get_wsize(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData) -> int:\n \"\"\"size_t ggml_cl_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);\"\"\"\n ...\n def ggml_cl_transform_tensor(data: ffi.CData, tensor: ffi.CData) -> None:\n \"\"\"void ggml_cl_transform_tensor(void * data, struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_clamp(ctx: ffi.CData, a: ffi.CData, min: float, max: float) -> ffi.CData:\n \"\"\"\n clamp\n in-place, returns view(a)\n\n struct ggml_tensor * ggml_clamp(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float min,\n float max);\n \"\"\"\n ...\n def ggml_cont(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n make contiguous\n\n GGML_API struct ggml_tensor * ggml_cont(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_conv_1d(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, s0: int, p0: int, d0: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_conv_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int s0, // stride\n int p0, // padding\n int d0); // dilation\n \"\"\"\n ...\n def ggml_conv_1d_ph(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, s: int, d: int) -> ffi.CData:\n \"\"\"\n conv_1d with padding = half\n alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)\n\n GGML_API struct ggml_tensor * ggml_conv_1d_ph(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int s,\n int d);\n \"\"\"\n ...\n def ggml_conv_2d(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, s0: int, s1: int, p0: int, p1: int, d0: int, d1: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_conv_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int s0,\n int s1,\n int p0,\n int p1,\n int d0,\n int d1);\n \"\"\"\n ...\n def ggml_cpu_has_arm_fma() -> int:\n \"\"\" GGML_API int ggml_cpu_has_arm_fma (void);\"\"\"\n ...\n def ggml_cpu_has_avx() -> int:\n \"\"\" GGML_API int ggml_cpu_has_avx (void);\"\"\"\n ...\n def ggml_cpu_has_avx2() -> int:\n \"\"\" GGML_API int ggml_cpu_has_avx2 (void);\"\"\"\n ...\n def ggml_cpu_has_avx512() -> int:\n \"\"\" GGML_API int ggml_cpu_has_avx512 (void);\"\"\"\n ...\n def ggml_cpu_has_avx512_vbmi() -> int:\n \"\"\" GGML_API int ggml_cpu_has_avx512_vbmi(void);\"\"\"\n ...\n def ggml_cpu_has_avx512_vnni() -> int:\n \"\"\" GGML_API int ggml_cpu_has_avx512_vnni(void);\"\"\"\n ...\n def ggml_cpu_has_blas() -> int:\n \"\"\" GGML_API int ggml_cpu_has_blas (void);\"\"\"\n ...\n def ggml_cpu_has_clblast() -> int:\n \"\"\" GGML_API int ggml_cpu_has_clblast (void);\"\"\"\n ...\n def ggml_cpu_has_cuda() -> int:\n \"\"\" GGML_API int ggml_cpu_has_cuda (void);\"\"\"\n ...\n def ggml_cpu_has_f16c() -> int:\n \"\"\" GGML_API int ggml_cpu_has_f16c (void);\"\"\"\n ...\n def ggml_cpu_has_fma() -> int:\n \"\"\" GGML_API int ggml_cpu_has_fma (void);\"\"\"\n ...\n def ggml_cpu_has_fp16_va() -> int:\n \"\"\" GGML_API int ggml_cpu_has_fp16_va (void);\"\"\"\n ...\n def ggml_cpu_has_gpublas() -> int:\n \"\"\" GGML_API int ggml_cpu_has_gpublas (void);\"\"\"\n ...\n def ggml_cpu_has_neon() -> int:\n \"\"\" GGML_API int ggml_cpu_has_neon (void);\"\"\"\n ...\n def ggml_cpu_has_sse3() -> int:\n \"\"\" GGML_API int ggml_cpu_has_sse3 (void);\"\"\"\n ...\n def ggml_cpu_has_vsx() -> int:\n \"\"\" GGML_API int ggml_cpu_has_vsx (void);\"\"\"\n ...\n def ggml_cpu_has_wasm_simd() -> int:\n \"\"\" GGML_API int ggml_cpu_has_wasm_simd (void);\"\"\"\n ...\n def ggml_cpy(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n a -> b, return view(b)\n\n GGML_API struct ggml_tensor * ggml_cpy(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_cross_entropy_loss(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_cross_entropy_loss(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_cross_entropy_loss_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c);\n \"\"\"\n ...\n def ggml_cuda_assign_buffers(tensor: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_cuda_assign_buffers(struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_cuda_assign_buffers_force_inplace(tensor: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_cuda_assign_buffers_no_scratch(tensor: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_cuda_can_mul_mat(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData) -> bool:\n \"\"\"GGML_API bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);\"\"\"\n ...\n def ggml_cuda_compute_forward(params: ffi.CData, tensor: ffi.CData) -> bool:\n \"\"\"GGML_API bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_cuda_free_data(tensor: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_cuda_free_data(struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_cuda_free_scratch() -> None:\n \"\"\"GGML_API void ggml_cuda_free_scratch(void);\"\"\"\n ...\n def ggml_cuda_get_device_count() -> int:\n \"\"\"GGML_API int ggml_cuda_get_device_count(void);\"\"\"\n ...\n def ggml_cuda_get_device_description(device: int, description: ffi.CData, description_size: int) -> None:\n \"\"\"GGML_API void ggml_cuda_get_device_description(int device, char * description, size_t description_size);\"\"\"\n ...\n def ggml_cuda_host_free(ptr: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_cuda_host_free(void * ptr);\"\"\"\n ...\n def ggml_cuda_host_malloc(size: int) -> ffi.CData:\n \"\"\"GGML_API void * ggml_cuda_host_malloc(size_t size);\"\"\"\n ...\n def ggml_cuda_set_main_device(main_device: int) -> None:\n \"\"\"GGML_API void ggml_cuda_set_main_device(int main_device);\"\"\"\n ...\n def ggml_cuda_set_mul_mat_q(mul_mat_q: bool) -> None:\n \"\"\"GGML_API void ggml_cuda_set_mul_mat_q(bool mul_mat_q);\"\"\"\n ...\n def ggml_cuda_set_scratch_size(scratch_size: int) -> None:\n \"\"\"GGML_API void ggml_cuda_set_scratch_size(size_t scratch_size);\"\"\"\n ...\n def ggml_cuda_set_tensor_split(tensor_split: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_cuda_set_tensor_split(const float * tensor_split);\"\"\"\n ...\n def ggml_cuda_transform_tensor(data: ffi.CData, tensor: ffi.CData) -> None:\n \"\"\"GGML_API void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_cycles() -> int:\n \"\"\" GGML_API int64_t ggml_cycles(void);\"\"\"\n ...\n def ggml_cycles_per_ms() -> int:\n \"\"\" GGML_API int64_t ggml_cycles_per_ms(void);\"\"\"\n ...\n def ggml_diag(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_diag(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_diag_mask_inf(ctx: ffi.CData, a: ffi.CData, n_past: int) -> ffi.CData:\n \"\"\"\n set elements above the diagonal to -INF\n\n GGML_API struct ggml_tensor * ggml_diag_mask_inf(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past);\n \"\"\"\n ...\n def ggml_diag_mask_inf_inplace(ctx: ffi.CData, a: ffi.CData, n_past: int) -> ffi.CData:\n \"\"\"\n in-place, returns view(a)\n\n GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past);\n \"\"\"\n ...\n def ggml_diag_mask_zero(ctx: ffi.CData, a: ffi.CData, n_past: int) -> ffi.CData:\n \"\"\"\n set elements above the diagonal to 0\n\n GGML_API struct ggml_tensor * ggml_diag_mask_zero(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past);\n \"\"\"\n ...\n def ggml_diag_mask_zero_inplace(ctx: ffi.CData, a: ffi.CData, n_past: int) -> ffi.CData:\n \"\"\"\n in-place, returns view(a)\n\n GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past);\n \"\"\"\n ...\n def ggml_div(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_div(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_div_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_div_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_dup(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_dup(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_dup_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n in-place, returns view(a)\n\n GGML_API struct ggml_tensor * ggml_dup_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_dup_tensor(ctx: ffi.CData, src: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);\"\"\"\n ...\n def ggml_element_size(tensor: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_elu(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_elu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_elu_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_elu_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_flash_attn(ctx: ffi.CData, q: ffi.CData, k: ffi.CData, v: ffi.CData, masked: bool) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_flash_attn(\n struct ggml_context * ctx,\n struct ggml_tensor * q,\n struct ggml_tensor * k,\n struct ggml_tensor * v,\n bool masked);\n \"\"\"\n ...\n def ggml_flash_attn_back(ctx: ffi.CData, q: ffi.CData, k: ffi.CData, v: ffi.CData, d: ffi.CData, masked: bool) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_flash_attn_back(\n struct ggml_context * ctx,\n struct ggml_tensor * q,\n struct ggml_tensor * k,\n struct ggml_tensor * v,\n struct ggml_tensor * d,\n bool masked);\n \"\"\"\n ...\n def ggml_flash_ff(ctx: ffi.CData, a: ffi.CData, b0: ffi.CData, b1: ffi.CData, c0: ffi.CData, c1: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_flash_ff(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b0,\n struct ggml_tensor * b1,\n struct ggml_tensor * c0,\n struct ggml_tensor * c1);\n \"\"\"\n ...\n def ggml_format_name(tensor: ffi.CData, fmt: ffi.CData, *args2) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);\"\"\"\n ...\n def ggml_fp16_to_fp32(x: np.float16) -> float:\n \"\"\"\n convert FP16 <-> FP32\n\n GGML_API float ggml_fp16_to_fp32(ggml_fp16_t x);\n \"\"\"\n ...\n def ggml_fp16_to_fp32_row(x: ffi.CData, y: ffi.CData, n: int) -> None:\n \"\"\" GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int n);\"\"\"\n ...\n def ggml_fp32_to_fp16(x: float) -> np.float16:\n \"\"\" GGML_API ggml_fp16_t ggml_fp32_to_fp16(float x);\"\"\"\n ...\n def ggml_fp32_to_fp16_row(x: ffi.CData, y: ffi.CData, n: int) -> None:\n \"\"\" GGML_API void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int n);\"\"\"\n ...\n def ggml_free(ctx: ffi.CData) -> None:\n \"\"\" GGML_API void ggml_free(struct ggml_context * ctx);\"\"\"\n ...\n def ggml_ftype_to_ggml_type(ftype: int) -> int:\n \"\"\"\n TODO: temporary until model loading of ggml examples is refactored\n\n GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);\n \"\"\"\n ...\n def ggml_gelu(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n TODO: double-check this computation is correct\n\n GGML_API struct ggml_tensor * ggml_gelu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_gelu_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_gelu_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_gelu_quick(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_gelu_quick(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_gelu_quick_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_get_data(tensor: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_get_data_f32(tensor: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_get_f32_1d(tensor: ffi.CData, i: int) -> float:\n \"\"\" GGML_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i);\"\"\"\n ...\n def ggml_get_i32_1d(tensor: ffi.CData, i: int) -> int:\n \"\"\" GGML_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i);\"\"\"\n ...\n def ggml_get_max_tensor_size(ctx: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);\"\"\"\n ...\n def ggml_get_mem_buffer(ctx: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);\"\"\"\n ...\n def ggml_get_mem_size(ctx: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);\"\"\"\n ...\n def ggml_get_name(tensor: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_get_no_alloc(ctx: ffi.CData) -> bool:\n \"\"\" GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);\"\"\"\n ...\n def ggml_get_rows(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_get_rows(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_get_rows_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_get_rows_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c);\n \"\"\"\n ...\n def ggml_get_tensor(ctx: ffi.CData, name: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);\"\"\"\n ...\n def ggml_get_unary_op(tensor: ffi.CData) -> int:\n \"\"\" GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_graph_compute(cgraph: ffi.CData, cplan: ffi.CData) -> int:\n \"\"\" GGML_API int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);\"\"\"\n ...\n def ggml_graph_compute_with_ctx(ctx: ffi.CData, cgraph: ffi.CData, n_threads: int) -> None:\n \"\"\"\n same as ggml_graph_compute() but the work data is allocated as a part of the context\n note: the drawback of this API is that you must have ensured that the context has enough memory for the work data\n\n GGML_API void ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);\n \"\"\"\n ...\n def ggml_graph_dump_dot(gb: ffi.CData, gf: ffi.CData, filename: ffi.CData) -> None:\n \"\"\"\n dump the graph into a file using the dot format\n\n GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename);\n \"\"\"\n ...\n def ggml_graph_get_tensor(cgraph: ffi.CData, name: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name);\"\"\"\n ...\n def ggml_graph_overhead() -> int:\n \"\"\" GGML_API size_t ggml_graph_overhead(void);\"\"\"\n ...\n def ggml_graph_plan(cgraph: ffi.CData, n_threads: int) -> ffi.CData:\n \"\"\"\n ggml_graph_plan() has to be called before ggml_graph_compute()\n when plan.work_size > 0, caller must allocate memory for plan.work_data\n\n GGML_API struct ggml_cplan ggml_graph_plan (struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/);\n \"\"\"\n ...\n def ggml_graph_print(cgraph: ffi.CData) -> None:\n \"\"\"\n print info and performance information for the graph\n\n GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);\n \"\"\"\n ...\n def ggml_graph_reset(cgraph: ffi.CData) -> None:\n \"\"\" GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph);\"\"\"\n ...\n def ggml_init(params: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct ggml_context * ggml_init(struct ggml_init_params params);\"\"\"\n ...\n def ggml_init_cuda() -> None:\n \"\"\"GGML_API void ggml_init_cuda(void);\"\"\"\n ...\n def ggml_internal_get_type_traits(type: int) -> ffi.CData:\n \"\"\" ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type);\"\"\"\n ...\n def ggml_is_contiguous(tensor: ffi.CData) -> bool:\n \"\"\" GGML_API bool ggml_is_contiguous(const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_is_numa() -> bool:\n \"\"\" GGML_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node\"\"\"\n ...\n def ggml_is_permuted(tensor: ffi.CData) -> bool:\n \"\"\" GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_is_quantized(type: int) -> bool:\n \"\"\" GGML_API bool ggml_is_quantized(enum ggml_type type);\"\"\"\n ...\n def ggml_is_transposed(tensor: ffi.CData) -> bool:\n \"\"\" GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_log(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_log(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_log_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_log_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_map_binary_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n ggml_binary_op_f32_t fun),\n \"use ggml_map_custom2 instead\");\n \"\"\"\n ...\n def ggml_map_binary_inplace_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_inplace_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n ggml_binary_op_f32_t fun),\n \"use ggml_map_custom2_inplace instead\");\n \"\"\"\n ...\n def ggml_map_custom1(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_map_custom1(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n ggml_custom1_op_t fun,\n int n_tasks,\n void * userdata);\n \"\"\"\n ...\n def ggml_map_custom1_f32(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n ggml_custom1_op_f32_t fun),\n \"use ggml_map_custom1 instead\");\n \"\"\"\n ...\n def ggml_map_custom1_inplace(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_map_custom1_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n ggml_custom1_op_t fun,\n int n_tasks,\n void * userdata);\n \"\"\"\n ...\n def ggml_map_custom1_inplace_f32(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_inplace_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n ggml_custom1_op_f32_t fun),\n \"use ggml_map_custom1_inplace instead\");\n \"\"\"\n ...\n def ggml_map_custom2(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_map_custom2(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n ggml_custom2_op_t fun,\n int n_tasks,\n void * userdata);\n \"\"\"\n ...\n def ggml_map_custom2_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n ggml_custom2_op_f32_t fun),\n \"use ggml_map_custom2 instead\");\n \"\"\"\n ...\n def ggml_map_custom2_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_map_custom2_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n ggml_custom2_op_t fun,\n int n_tasks,\n void * userdata);\n \"\"\"\n ...\n def ggml_map_custom2_inplace_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_inplace_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n ggml_custom2_op_f32_t fun),\n \"use ggml_map_custom2_inplace instead\");\n \"\"\"\n ...\n def ggml_map_custom3(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_map_custom3(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n ggml_custom3_op_t fun,\n int n_tasks,\n void * userdata);\n \"\"\"\n ...\n def ggml_map_custom3_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n ggml_custom3_op_f32_t fun),\n \"use ggml_map_custom3 instead\");\n \"\"\"\n ...\n def ggml_map_custom3_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_map_custom3_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n ggml_custom3_op_t fun,\n int n_tasks,\n void * userdata);\n \"\"\"\n ...\n def ggml_map_custom3_inplace_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_inplace_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n ggml_custom3_op_f32_t fun),\n \"use ggml_map_custom3_inplace instead\");\n \"\"\"\n ...\n def ggml_map_unary_f32(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n ggml_unary_op_f32_t fun),\n \"use ggml_map_custom1 instead\");\n \"\"\"\n ...\n def ggml_map_unary_inplace_f32(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_inplace_f32(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n ggml_unary_op_f32_t fun),\n \"use ggml_map_custom1_inplace instead\");\n \"\"\"\n ...\n def ggml_mean(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n mean along rows\n\n GGML_API struct ggml_tensor * ggml_mean(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_metal_add_buffer(ctx: ffi.CData, name: ffi.CData, data: ffi.CData, size: int, max_size: int) -> bool:\n \"\"\"\n creates a mapping between a host memory buffer and a device memory buffer\n - make sure to map all buffers used in the graph before calling ggml_metal_graph_compute\n - the mapping is used during computation to determine the arguments of the compute kernels\n - you don't need to keep the host memory buffer allocated as it is never accessed by Metal\n - max_size specifies the maximum size of a tensor and is used to create shared views such\n that it is guaranteed that the tensor will fit in at least one of the views\n\n\n bool ggml_metal_add_buffer(\n struct ggml_metal_context * ctx,\n const char * name,\n void * data,\n size_t size,\n size_t max_size);\n \"\"\"\n ...\n def ggml_metal_free(ctx: ffi.CData) -> None:\n \"\"\"void ggml_metal_free(struct ggml_metal_context * ctx);\"\"\"\n ...\n def ggml_metal_get_concur_list(ctx: ffi.CData) -> ffi.CData:\n \"\"\"\n output the concur_list for ggml_alloc\n\n int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx);\n \"\"\"\n ...\n def ggml_metal_get_tensor(ctx: ffi.CData, t: ffi.CData) -> None:\n \"\"\"\n get data from the device into host memory\n\n void ggml_metal_get_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t);\n \"\"\"\n ...\n def ggml_metal_graph_compute(ctx: ffi.CData, gf: ffi.CData) -> None:\n \"\"\"\n same as ggml_graph_compute but uses Metal\n creates gf->n_threads command buffers in parallel\n\n void ggml_metal_graph_compute(struct ggml_metal_context * ctx, struct ggml_cgraph * gf);\n \"\"\"\n ...\n def ggml_metal_graph_find_concurrency(ctx: ffi.CData, gf: ffi.CData, check_mem: bool) -> None:\n \"\"\"\n try to find operations that can be run concurrently in the graph\n you should run it again if the topology of your graph changes\n\n void ggml_metal_graph_find_concurrency(struct ggml_metal_context * ctx, struct ggml_cgraph * gf, bool check_mem);\n \"\"\"\n ...\n def ggml_metal_host_free(data: ffi.CData) -> None:\n \"\"\"void ggml_metal_host_free (void * data);\"\"\"\n ...\n def ggml_metal_host_malloc(n: int) -> ffi.CData:\n \"\"\"void * ggml_metal_host_malloc(size_t n);\"\"\"\n ...\n def ggml_metal_if_optimized(ctx: ffi.CData) -> int:\n \"\"\"\n if the graph has been optimized for concurrently dispatch, return length of the concur_list if optimized\n\n int ggml_metal_if_optimized(struct ggml_metal_context * ctx);\n \"\"\"\n ...\n def ggml_metal_init(n_cb: int) -> ffi.CData:\n \"\"\"\n number of command buffers to use\n\n struct ggml_metal_context * ggml_metal_init(int n_cb);\n \"\"\"\n ...\n def ggml_metal_set_n_cb(ctx: ffi.CData, n_cb: int) -> None:\n \"\"\"\n set the number of command buffers to use\n\n void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb);\n \"\"\"\n ...\n def ggml_metal_set_tensor(ctx: ffi.CData, t: ffi.CData) -> None:\n \"\"\"\n set data from host memory into the device\n\n void ggml_metal_set_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t);\n \"\"\"\n ...\n def ggml_mpi_backend_free() -> None:\n \"\"\"void ggml_mpi_backend_free(void);\"\"\"\n ...\n def ggml_mpi_backend_init() -> None:\n \"\"\"void ggml_mpi_backend_init(void);\"\"\"\n ...\n def ggml_mpi_eval_init(ctx_mpi: ffi.CData, n_tokens: ffi.CData, n_past: ffi.CData, n_threads: ffi.CData) -> None:\n \"\"\"\n void ggml_mpi_eval_init(\n struct ggml_mpi_context * ctx_mpi,\n int * n_tokens,\n int * n_past,\n int * n_threads);\n \"\"\"\n ...\n def ggml_mpi_free(ctx: ffi.CData) -> None:\n \"\"\"void ggml_mpi_free(struct ggml_mpi_context * ctx);\"\"\"\n ...\n def ggml_mpi_graph_compute_post(ctx_mpi: ffi.CData, gf: ffi.CData, n_layers: int) -> None:\n \"\"\"\n void ggml_mpi_graph_compute_post(\n struct ggml_mpi_context * ctx_mpi,\n struct ggml_cgraph * gf,\n int n_layers);\n \"\"\"\n ...\n def ggml_mpi_graph_compute_pre(ctx_mpi: ffi.CData, gf: ffi.CData, n_layers: int) -> None:\n \"\"\"\n void ggml_mpi_graph_compute_pre(\n struct ggml_mpi_context * ctx_mpi,\n struct ggml_cgraph * gf,\n int n_layers);\n \"\"\"\n ...\n def ggml_mpi_init() -> ffi.CData:\n \"\"\"struct ggml_mpi_context * ggml_mpi_init(void);\"\"\"\n ...\n def ggml_mpi_rank(ctx: ffi.CData) -> int:\n \"\"\"int ggml_mpi_rank(struct ggml_mpi_context * ctx);\"\"\"\n ...\n def ggml_mul(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_mul(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_mul_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_mul_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_mul_mat(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n A: n columns, m rows\n B: n columns, p rows (i.e. we transpose it internally)\n result is m columns, p rows\n\n GGML_API struct ggml_tensor * ggml_mul_mat(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_nbytes(tensor: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_nbytes_pad(tensor: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN\"\"\"\n ...\n def ggml_nbytes_split(tensor: ffi.CData, nrows_split: int) -> int:\n \"\"\" GGML_API size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split);\"\"\"\n ...\n def ggml_neg(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_neg(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_neg_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_neg_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_nelements(tensor: ffi.CData) -> int:\n \"\"\" GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_new_f32(ctx: ffi.CData, value: float) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value);\"\"\"\n ...\n def ggml_new_graph(ctx: ffi.CData) -> ffi.CData:\n \"\"\"\n graph allocation in a context\n\n GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx);\n \"\"\"\n ...\n def ggml_new_i32(ctx: ffi.CData, value: int) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value);\"\"\"\n ...\n def ggml_new_tensor(ctx: ffi.CData, type: int, n_dims: int, ne: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_new_tensor(\n struct ggml_context * ctx,\n enum ggml_type type,\n int n_dims,\n const int64_t *ne);\n \"\"\"\n ...\n def ggml_new_tensor_1d(ctx: ffi.CData, type: int, ne0: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_new_tensor_1d(\n struct ggml_context * ctx,\n enum ggml_type type,\n int64_t ne0);\n \"\"\"\n ...\n def ggml_new_tensor_2d(ctx: ffi.CData, type: int, ne0: int, ne1: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_new_tensor_2d(\n struct ggml_context * ctx,\n enum ggml_type type,\n int64_t ne0,\n int64_t ne1);\n \"\"\"\n ...\n def ggml_new_tensor_3d(ctx: ffi.CData, type: int, ne0: int, ne1: int, ne2: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_new_tensor_3d(\n struct ggml_context * ctx,\n enum ggml_type type,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2);\n \"\"\"\n ...\n def ggml_new_tensor_4d(ctx: ffi.CData, type: int, ne0: int, ne1: int, ne2: int, ne3: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_new_tensor_4d(\n struct ggml_context * ctx,\n enum ggml_type type,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3);\n \"\"\"\n ...\n def ggml_norm(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n normalize along rows\n TODO: eps is hardcoded to 1e-5 for now\n\n GGML_API struct ggml_tensor * ggml_norm(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_norm_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_norm_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_nrows(tensor: ffi.CData) -> int:\n \"\"\" GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_numa_init() -> None:\n \"\"\" GGML_API void ggml_numa_init(void); // call once for better performance on NUMA systems\"\"\"\n ...\n def ggml_op_name(op: int) -> ffi.CData:\n \"\"\" GGML_API const char * ggml_op_name (enum ggml_op op);\"\"\"\n ...\n def ggml_op_symbol(op: int) -> ffi.CData:\n \"\"\" GGML_API const char * ggml_op_symbol(enum ggml_op op);\"\"\"\n ...\n def ggml_opt(ctx: ffi.CData, params: ffi.CData, f: ffi.CData) -> int:\n \"\"\"\n optimize the function defined by the tensor f\n\n GGML_API enum ggml_opt_result ggml_opt(\n struct ggml_context * ctx,\n struct ggml_opt_params params,\n struct ggml_tensor * f);\n \"\"\"\n ...\n def ggml_opt_default_params(type: int) -> ffi.CData:\n \"\"\" GGML_API struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type);\"\"\"\n ...\n def ggml_opt_init(ctx: ffi.CData, opt: ffi.CData, params: ffi.CData, nx: int) -> None:\n \"\"\"\n initialize optimizer context\n\n GGML_API void ggml_opt_init(\n struct ggml_context * ctx,\n struct ggml_opt_context * opt,\n struct ggml_opt_params params,\n int64_t nx);\n \"\"\"\n ...\n def ggml_opt_resume(ctx: ffi.CData, opt: ffi.CData, f: ffi.CData) -> int:\n \"\"\"\n continue optimizing the function defined by the tensor f\n\n GGML_API enum ggml_opt_result ggml_opt_resume(\n struct ggml_context * ctx,\n struct ggml_opt_context * opt,\n struct ggml_tensor * f);\n \"\"\"\n ...\n def ggml_opt_resume_g(ctx: ffi.CData, opt: ffi.CData, f: ffi.CData, gf: ffi.CData, gb: ffi.CData) -> int:\n \"\"\"\n continue optimizing the function defined by the tensor f\n\n GGML_API enum ggml_opt_result ggml_opt_resume_g(\n struct ggml_context * ctx,\n struct ggml_opt_context * opt,\n struct ggml_tensor * f,\n struct ggml_cgraph * gf,\n struct ggml_cgraph * gb);\n \"\"\"\n ...\n def ggml_out_prod(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n A: m columns, n rows,\n B: p columns, n rows,\n result is m columns, p rows\n\n GGML_API struct ggml_tensor * ggml_out_prod(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_permute(ctx: ffi.CData, a: ffi.CData, axis0: int, axis1: int, axis2: int, axis3: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_permute(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int axis0,\n int axis1,\n int axis2,\n int axis3);\n \"\"\"\n ...\n def ggml_pool_1d(ctx: ffi.CData, a: ffi.CData, op: int, k0: int, s0: int, p0: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_pool_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_op_pool op,\n int k0, // kernel size\n int s0, // stride\n int p0); // padding\n \"\"\"\n ...\n def ggml_pool_2d(ctx: ffi.CData, a: ffi.CData, op: int, k0: int, k1: int, s0: int, s1: int, p0: int, p1: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_pool_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_op_pool op,\n int k0,\n int k1,\n int s0,\n int s1,\n int p0,\n int p1);\n \"\"\"\n ...\n def ggml_print_object(obj: ffi.CData) -> None:\n \"\"\" GGML_API void ggml_print_object (const struct ggml_object * obj);\"\"\"\n ...\n def ggml_print_objects(ctx: ffi.CData) -> None:\n \"\"\" GGML_API void ggml_print_objects(const struct ggml_context * ctx);\"\"\"\n ...\n def ggml_quantize_chunk(type: int, src: ffi.CData, dst: ffi.CData, start: int, n: int, hist: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist);\"\"\"\n ...\n def ggml_quantize_q2_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\"\n Quantization with histogram collection\n\n size_t ggml_quantize_q2_K(const float * src, void * dst, int n, int k, int64_t * hist);\n \"\"\"\n ...\n def ggml_quantize_q3_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\"size_t ggml_quantize_q3_K(const float * src, void * dst, int n, int k, int64_t * hist);\"\"\"\n ...\n def ggml_quantize_q4_0(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist);\"\"\"\n ...\n def ggml_quantize_q4_1(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist);\"\"\"\n ...\n def ggml_quantize_q4_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\"size_t ggml_quantize_q4_K(const float * src, void * dst, int n, int k, int64_t * hist);\"\"\"\n ...\n def ggml_quantize_q5_0(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_quantize_q5_0(const float * src, void * dst, int n, int k, int64_t * hist);\"\"\"\n ...\n def ggml_quantize_q5_1(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_quantize_q5_1(const float * src, void * dst, int n, int k, int64_t * hist);\"\"\"\n ...\n def ggml_quantize_q5_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\"size_t ggml_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist);\"\"\"\n ...\n def ggml_quantize_q6_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\"size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist);\"\"\"\n ...\n def ggml_quantize_q8_0(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * hist);\"\"\"\n ...\n def ggml_relu(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_relu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_relu_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_relu_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_repeat(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n if a is the same shape as b, and a is not parameter, return a\n otherwise, return a new tensor: repeat(a) to fit in b\n\n GGML_API struct ggml_tensor * ggml_repeat(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_repeat_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_repeat_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_reshape(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n return view(a), b specifies the new shape\n TODO: when we start computing gradient, make a copy instead of view\n\n GGML_API struct ggml_tensor * ggml_reshape(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_reshape_1d(ctx: ffi.CData, a: ffi.CData, ne0: int) -> ffi.CData:\n \"\"\"\n return view(a)\n TODO: when we start computing gradient, make a copy instead of view\n\n GGML_API struct ggml_tensor * ggml_reshape_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0);\n \"\"\"\n ...\n def ggml_reshape_2d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_reshape_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1);\n \"\"\"\n ...\n def ggml_reshape_3d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, ne2: int) -> ffi.CData:\n \"\"\"\n return view(a)\n TODO: when we start computing gradient, make a copy instead of view\n\n GGML_API struct ggml_tensor * ggml_reshape_3d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2);\n \"\"\"\n ...\n def ggml_reshape_4d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, ne2: int, ne3: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_reshape_4d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3);\n \"\"\"\n ...\n def ggml_rms_norm(ctx: ffi.CData, a: ffi.CData, eps: float) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_rms_norm(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float eps);\n \"\"\"\n ...\n def ggml_rms_norm_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n a - x\n b - dy\n TODO: update with configurable eps\n\n GGML_API struct ggml_tensor * ggml_rms_norm_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_rms_norm_inplace(ctx: ffi.CData, a: ffi.CData, eps: float) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_rms_norm_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float eps);\n \"\"\"\n ...\n def ggml_rope(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int) -> ffi.CData:\n \"\"\"\n rotary position embedding\n if mode & 1 == 1, skip n_past elements\n if mode & 2 == 1, GPT-NeoX style\n if mode & 4 == 1, ChatGLM style\n TODO: avoid creating a new tensor every time\n\n GGML_API struct ggml_tensor * ggml_rope(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past,\n int n_dims,\n int mode,\n int n_ctx);\n \"\"\"\n ...\n def ggml_rope_back(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int) -> ffi.CData:\n \"\"\"\n rotary position embedding backward, i.e compute dx from dy\n a - dy\n\n GGML_API struct ggml_tensor * ggml_rope_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past,\n int n_dims,\n int mode,\n int n_ctx);\n \"\"\"\n ...\n def ggml_rope_custom(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int, freq_base: float, freq_scale: float) -> ffi.CData:\n \"\"\"\n custom RoPE\n\n GGML_API struct ggml_tensor * ggml_rope_custom(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past,\n int n_dims,\n int mode,\n int n_ctx,\n float freq_base,\n float freq_scale);\n \"\"\"\n ...\n def ggml_rope_custom_inplace(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int, freq_base: float, freq_scale: float) -> ffi.CData:\n \"\"\"\n in-place, returns view(a)\n\n GGML_API struct ggml_tensor * ggml_rope_custom_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past,\n int n_dims,\n int mode,\n int n_ctx,\n float freq_base,\n float freq_scale);\n \"\"\"\n ...\n def ggml_rope_inplace(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int) -> ffi.CData:\n \"\"\"\n in-place, returns view(a)\n\n GGML_API struct ggml_tensor * ggml_rope_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past,\n int n_dims,\n int mode,\n int n_ctx);\n \"\"\"\n ...\n def ggml_scale(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_scale(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_scale_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n in-place, returns view(a)\n\n GGML_API struct ggml_tensor * ggml_scale_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_set(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData:\n \"\"\"\n b -> view(a,offset,nb1,nb2,3), return modified a\n\n GGML_API struct ggml_tensor * ggml_set(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t nb2,\n size_t nb3,\n size_t offset);\n \"\"\"\n ...\n def ggml_set_1d(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, offset: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_set_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t offset);\n \"\"\"\n ...\n def ggml_set_1d_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, offset: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_set_1d_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t offset);\n \"\"\"\n ...\n def ggml_set_2d(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, offset: int) -> ffi.CData:\n \"\"\"\n b -> view(a,offset,nb1,nb2,3), return modified a\n\n GGML_API struct ggml_tensor * ggml_set_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t offset);\n \"\"\"\n ...\n def ggml_set_2d_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, offset: int) -> ffi.CData:\n \"\"\"\n b -> view(a,offset,nb1,nb2,3), return view(a)\n\n GGML_API struct ggml_tensor * ggml_set_2d_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t offset);\n \"\"\"\n ...\n def ggml_set_f32(tensor: ffi.CData, value: float) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);\"\"\"\n ...\n def ggml_set_f32_1d(tensor: ffi.CData, i: int, value: float) -> None:\n \"\"\" GGML_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value);\"\"\"\n ...\n def ggml_set_i32(tensor: ffi.CData, value: int) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);\"\"\"\n ...\n def ggml_set_i32_1d(tensor: ffi.CData, i: int, value: int) -> None:\n \"\"\" GGML_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value);\"\"\"\n ...\n def ggml_set_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData:\n \"\"\"\n b -> view(a,offset,nb1,nb2,3), return view(a)\n\n GGML_API struct ggml_tensor * ggml_set_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t nb2,\n size_t nb3,\n size_t offset);\n \"\"\"\n ...\n def ggml_set_name(tensor: ffi.CData, name: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);\"\"\"\n ...\n def ggml_set_no_alloc(ctx: ffi.CData, no_alloc: bool) -> None:\n \"\"\" GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);\"\"\"\n ...\n def ggml_set_param(ctx: ffi.CData, tensor: ffi.CData) -> None:\n \"\"\"\n GGML_API void ggml_set_param(\n struct ggml_context * ctx,\n struct ggml_tensor * tensor);\n \"\"\"\n ...\n def ggml_set_scratch(ctx: ffi.CData, scratch: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_set_scratch (struct ggml_context * ctx, struct ggml_scratch scratch);\"\"\"\n ...\n def ggml_set_zero(tensor: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);\"\"\"\n ...\n def ggml_sgn(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_sgn(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_sgn_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_sgn_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_silu(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_silu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_silu_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n a - x\n b - dy\n\n GGML_API struct ggml_tensor * ggml_silu_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_silu_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_silu_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_soft_max(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_soft_max(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_soft_max_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_soft_max_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_soft_max_back_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n in-place, returns view(a)\n\n GGML_API struct ggml_tensor * ggml_soft_max_back_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_soft_max_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n in-place, returns view(a)\n\n GGML_API struct ggml_tensor * ggml_soft_max_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_sqr(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_sqr(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_sqr_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_sqr_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_sqrt(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_sqrt(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_sqrt_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_sqrt_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_step(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_step(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_step_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_step_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_sub(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_sub(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_sub_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_sub_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n \"\"\"\n ...\n def ggml_sum(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n return scalar\n\n GGML_API struct ggml_tensor * ggml_sum(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_sum_rows(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]\n\n GGML_API struct ggml_tensor * ggml_sum_rows(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_tanh(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_tanh(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_tanh_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_tanh_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_tensor_overhead() -> int:\n \"\"\"\n use this to compute the memory overhead of a tensor\n\n GGML_API size_t ggml_tensor_overhead(void);\n \"\"\"\n ...\n def ggml_time_init() -> None:\n \"\"\" GGML_API void ggml_time_init(void); // call this once at the beginning of the program\"\"\"\n ...\n def ggml_time_ms() -> int:\n \"\"\" GGML_API int64_t ggml_time_ms(void);\"\"\"\n ...\n def ggml_time_us() -> int:\n \"\"\" GGML_API int64_t ggml_time_us(void);\"\"\"\n ...\n def ggml_transpose(ctx: ffi.CData, a: ffi.CData) -> ffi.CData:\n \"\"\"\n alias for ggml_permute(ctx, a, 1, 0, 2, 3)\n\n GGML_API struct ggml_tensor * ggml_transpose(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n \"\"\"\n ...\n def ggml_type_name(type: int) -> ffi.CData:\n \"\"\" GGML_API const char * ggml_type_name(enum ggml_type type);\"\"\"\n ...\n def ggml_type_size(type: int) -> int:\n \"\"\" GGML_API size_t ggml_type_size (enum ggml_type type); // size in bytes for all elements in a block\"\"\"\n ...\n def ggml_type_sizef(type: int) -> float:\n \"\"\" GGML_API float ggml_type_sizef(enum ggml_type type); // ggml_type_size()/ggml_blck_size() as float\"\"\"\n ...\n def ggml_unary(ctx: ffi.CData, a: ffi.CData, op: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_unary(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_unary_op op);\n \"\"\"\n ...\n def ggml_unary_inplace(ctx: ffi.CData, a: ffi.CData, op: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_unary_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_unary_op op);\n \"\"\"\n ...\n def ggml_used_mem(ctx: ffi.CData) -> int:\n \"\"\" GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);\"\"\"\n ...\n def ggml_vec_dot_q2_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None:\n \"\"\"\n Dot product\n\n void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);\n \"\"\"\n ...\n def ggml_vec_dot_q3_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None:\n \"\"\"void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);\"\"\"\n ...\n def ggml_vec_dot_q4_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None:\n \"\"\"void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);\"\"\"\n ...\n def ggml_vec_dot_q5_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None:\n \"\"\"void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);\"\"\"\n ...\n def ggml_vec_dot_q6_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None:\n \"\"\"void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);\"\"\"\n ...\n def ggml_view_1d(ctx: ffi.CData, a: ffi.CData, ne0: int, offset: int) -> ffi.CData:\n \"\"\"\n offset in bytes\n\n GGML_API struct ggml_tensor * ggml_view_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n size_t offset);\n \"\"\"\n ...\n def ggml_view_2d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, nb1: int, offset: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_view_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n size_t nb1, // row stride in bytes\n size_t offset);\n \"\"\"\n ...\n def ggml_view_3d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, ne2: int, nb1: int, nb2: int, offset: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_view_3d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n size_t nb1, // row stride in bytes\n size_t nb2, // slice stride in bytes\n size_t offset);\n \"\"\"\n ...\n def ggml_view_4d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, ne2: int, ne3: int, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData:\n \"\"\"\n GGML_API struct ggml_tensor * ggml_view_4d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3,\n size_t nb1, // row stride in bytes\n size_t nb2, // slice stride in bytes\n size_t nb3,\n size_t offset);\n \"\"\"\n ...\n def ggml_view_tensor(ctx: ffi.CData, src: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, const struct ggml_tensor * src);\"\"\"\n ...\n def ggml_win_part(ctx: ffi.CData, a: ffi.CData, w: int) -> ffi.CData:\n \"\"\"\n partition into non-overlapping windows with padding if needed\n example:\n a: 768 64 64 1\n w: 14\n res: 768 14 14 25\n used in sam\n\n GGML_API struct ggml_tensor * ggml_win_part(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int w);\n \"\"\"\n ...\n def ggml_win_unpart(ctx: ffi.CData, a: ffi.CData, w0: int, h0: int, w: int) -> ffi.CData:\n \"\"\"\n reverse of ggml_win_part\n used in sam\n\n GGML_API struct ggml_tensor * ggml_win_unpart(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int w0,\n int h0,\n int w);\n \"\"\"\n ...\n def gguf_add_tensor(ctx: ffi.CData, tensor: ffi.CData) -> None:\n \"\"\"\n manage tensor info\n\n GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);\n \"\"\"\n ...\n def gguf_find_key(ctx: ffi.CData, key: ffi.CData) -> int:\n \"\"\" GGML_API int gguf_find_key(struct gguf_context * ctx, const char * key);\"\"\"\n ...\n def gguf_find_tensor(ctx: ffi.CData, name: ffi.CData) -> int:\n \"\"\" GGML_API int gguf_find_tensor (struct gguf_context * ctx, const char * name);\"\"\"\n ...\n def gguf_free(ctx: ffi.CData) -> None:\n \"\"\" GGML_API void gguf_free(struct gguf_context * ctx);\"\"\"\n ...\n def gguf_get_alignment(ctx: ffi.CData) -> int:\n \"\"\" GGML_API size_t gguf_get_alignment (struct gguf_context * ctx);\"\"\"\n ...\n def gguf_get_arr_data(ctx: ffi.CData, i: int) -> ffi.CData:\n \"\"\" GGML_API const void * gguf_get_arr_data(struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_arr_n(ctx: ffi.CData, i: int) -> int:\n \"\"\" GGML_API int gguf_get_arr_n (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_arr_str(ctx: ffi.CData, key_id: int, i: int) -> ffi.CData:\n \"\"\" GGML_API const char * gguf_get_arr_str (struct gguf_context * ctx, int key_id, int i);\"\"\"\n ...\n def gguf_get_arr_type(ctx: ffi.CData, i: int) -> int:\n \"\"\" GGML_API enum gguf_type gguf_get_arr_type(struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_data(ctx: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API void * gguf_get_data (struct gguf_context * ctx);\"\"\"\n ...\n def gguf_get_data_offset(ctx: ffi.CData) -> int:\n \"\"\" GGML_API size_t gguf_get_data_offset(struct gguf_context * ctx);\"\"\"\n ...\n def gguf_get_key(ctx: ffi.CData, i: int) -> ffi.CData:\n \"\"\" GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_kv_type(ctx: ffi.CData, i: int) -> int:\n \"\"\" GGML_API enum gguf_type gguf_get_kv_type (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_meta_data(ctx: ffi.CData, data: ffi.CData) -> None:\n \"\"\" GGML_API void gguf_get_meta_data(struct gguf_context * ctx, void * data);\"\"\"\n ...\n def gguf_get_meta_size(ctx: ffi.CData) -> int:\n \"\"\"\n get the size in bytes of the meta data (header, kv pairs, tensor info) including padding\n\n GGML_API size_t gguf_get_meta_size(struct gguf_context * ctx);\n \"\"\"\n ...\n def gguf_get_n_kv(ctx: ffi.CData) -> int:\n \"\"\" GGML_API int gguf_get_n_kv(struct gguf_context * ctx);\"\"\"\n ...\n def gguf_get_n_tensors(ctx: ffi.CData) -> int:\n \"\"\" GGML_API int gguf_get_n_tensors (struct gguf_context * ctx);\"\"\"\n ...\n def gguf_get_tensor_name(ctx: ffi.CData, i: int) -> ffi.CData:\n \"\"\" GGML_API char * gguf_get_tensor_name (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_tensor_offset(ctx: ffi.CData, i: int) -> int:\n \"\"\" GGML_API size_t gguf_get_tensor_offset(struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_val_bool(ctx: ffi.CData, i: int) -> bool:\n \"\"\" GGML_API bool gguf_get_val_bool(struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_val_f32(ctx: ffi.CData, i: int) -> float:\n \"\"\" GGML_API float gguf_get_val_f32 (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_val_i16(ctx: ffi.CData, i: int) -> int:\n \"\"\" GGML_API int16_t gguf_get_val_i16 (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_val_i32(ctx: ffi.CData, i: int) -> int:\n \"\"\" GGML_API int32_t gguf_get_val_i32 (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_val_i8(ctx: ffi.CData, i: int) -> int:\n \"\"\" GGML_API int8_t gguf_get_val_i8 (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_val_str(ctx: ffi.CData, i: int) -> ffi.CData:\n \"\"\" GGML_API const char * gguf_get_val_str (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_val_u16(ctx: ffi.CData, i: int) -> int:\n \"\"\" GGML_API uint16_t gguf_get_val_u16 (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_val_u32(ctx: ffi.CData, i: int) -> int:\n \"\"\" GGML_API uint32_t gguf_get_val_u32 (struct gguf_context * ctx, int i);\"\"\"\n ...\n def gguf_get_val_u8(ctx: ffi.CData, i: int) -> int:\n \"\"\"\n results are undefined if the wrong type is used for the key\n\n GGML_API uint8_t gguf_get_val_u8 (struct gguf_context * ctx, int i);\n \"\"\"\n ...\n def gguf_get_version(ctx: ffi.CData) -> int:\n \"\"\" GGML_API int gguf_get_version (struct gguf_context * ctx);\"\"\"\n ...\n def gguf_init_empty() -> ffi.CData:\n \"\"\" GGML_API struct gguf_context * gguf_init_empty(void);\"\"\"\n ...\n def gguf_init_from_file(fname: ffi.CData, params: ffi.CData) -> ffi.CData:\n \"\"\" GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);\"\"\"\n ...\n def gguf_set_arr_data(ctx: ffi.CData, key: ffi.CData, type: int, data: ffi.CData, n: int) -> None:\n \"\"\" GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, int n);\"\"\"\n ...\n def gguf_set_arr_str(ctx: ffi.CData, key: ffi.CData, data: ffi.CData, n: int) -> None:\n \"\"\" GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, int n);\"\"\"\n ...\n def gguf_set_kv(ctx: ffi.CData, src: ffi.CData) -> None:\n \"\"\"\n set or add KV pairs from another context\n\n GGML_API void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src);\n \"\"\"\n ...\n def gguf_set_tensor_data(ctx: ffi.CData, name: ffi.CData, data: ffi.CData, size: int) -> None:\n \"\"\" GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size);\"\"\"\n ...\n def gguf_set_tensor_type(ctx: ffi.CData, name: ffi.CData, type: int) -> None:\n \"\"\" GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);\"\"\"\n ...\n def gguf_set_val_bool(ctx: ffi.CData, key: ffi.CData, val: bool) -> None:\n \"\"\" GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val);\"\"\"\n ...\n def gguf_set_val_f32(ctx: ffi.CData, key: ffi.CData, val: float) -> None:\n \"\"\" GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val);\"\"\"\n ...\n def gguf_set_val_i16(ctx: ffi.CData, key: ffi.CData, val: int) -> None:\n \"\"\" GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val);\"\"\"\n ...\n def gguf_set_val_i32(ctx: ffi.CData, key: ffi.CData, val: int) -> None:\n \"\"\" GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val);\"\"\"\n ...\n def gguf_set_val_i8(ctx: ffi.CData, key: ffi.CData, val: int) -> None:\n \"\"\" GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val);\"\"\"\n ...\n def gguf_set_val_str(ctx: ffi.CData, key: ffi.CData, val: ffi.CData) -> None:\n \"\"\" GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);\"\"\"\n ...\n def gguf_set_val_u16(ctx: ffi.CData, key: ffi.CData, val: int) -> None:\n \"\"\" GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);\"\"\"\n ...\n def gguf_set_val_u32(ctx: ffi.CData, key: ffi.CData, val: int) -> None:\n \"\"\" GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);\"\"\"\n ...\n def gguf_set_val_u8(ctx: ffi.CData, key: ffi.CData, val: int) -> None:\n \"\"\"\n overrides existing values or adds a new one\n\n GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val);\n \"\"\"\n ...\n def gguf_type_name(type: int) -> ffi.CData:\n \"\"\" GGML_API const char * gguf_type_name(enum gguf_type type);\"\"\"\n ...\n def gguf_write_to_file(ctx: ffi.CData, fname: ffi.CData, only_meta: bool) -> None:\n \"\"\"\n write the entire context to a binary file\n\n GGML_API void gguf_write_to_file(struct gguf_context * ctx, const char * fname, bool only_meta);\n \"\"\"\n ...\n def quantize_row_q2_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q2_K(const float * restrict x, void * restrict y, int k);\"\"\"\n ...\n def quantize_row_q2_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"\n Quantization\n\n void quantize_row_q2_K_reference(const float * restrict x, block_q2_K * restrict y, int k);\n \"\"\"\n ...\n def quantize_row_q3_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q3_K(const float * restrict x, void * restrict y, int k);\"\"\"\n ...\n def quantize_row_q3_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict y, int k);\"\"\"\n ...\n def quantize_row_q4_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q4_K(const float * restrict x, void * restrict y, int k);\"\"\"\n ...\n def quantize_row_q4_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict y, int k);\"\"\"\n ...\n def quantize_row_q5_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q5_K(const float * restrict x, void * restrict y, int k);\"\"\"\n ...\n def quantize_row_q5_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k);\"\"\"\n ...\n def quantize_row_q6_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q6_K(const float * restrict x, void * restrict y, int k);\"\"\"\n ...\n def quantize_row_q6_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k);\"\"\"\n ...\n def quantize_row_q8_K(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q8_K(const float * restrict x, void * restrict y, int k);\"\"\"\n ...\n def quantize_row_q8_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None:\n \"\"\"void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k);\"\"\"\n ...\n" }, { "path": "examples/python/ggml/cffi.py", "content": "# auto-generated file\nimport _cffi_backend\n\nffi = _cffi_backend.FFI('ggml.cffi',\n _version = 0x2601,\n _types = b'\\x00\\x00\\xB6\\x0D\\x00\\x00\\x09\\x0B\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x04\\x2F\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x04\\x31\\x03\\x00\\x04\\x3D\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x04\\x32\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x04\\x34\\x03\\x00\\x03\\xFE\\x03\\x00\\x04\\x53\\x03\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x04\\x3D\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x04\\x3E\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x00\\x10\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\xB6\\x0D\\x00\\x00\\x00\\x0F\\x00\\x02\\xD0\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x0F\\x0D\\x00\\x00\\x04\\x0B\\x00\\x00\\x00\\x0F\\x00\\x00\\x0F\\x0D\\x00\\x00\\x01\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x0F\\x0D\\x00\\x00\\x0B\\x0B\\x00\\x00\\x00\\x0F\\x00\\x00\\x0F\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x0F\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x0F\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x16\\x0D\\x00\\x00\\x0B\\x11\\x00\\x04\\x38\\x03\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x16\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x44\\x11\\x00\\x00\\x08\\x11\\x00\\x04\\x30\\x03\\x00\\x00\\x4B\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x16\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x20\\x09\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x01\\x0D\\x00\\x00\\x01\\x0B\\x00\\x00\\x00\\x0F\\x00\\x01\\x14\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x34\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x02\\x7E\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\xF4\\x0D\\x00\\x00\\x01\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\xF4\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\xF4\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\xF4\\x0D\\x00\\x00\\x06\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x18\\x0D\\x00\\x00\\x01\\x11\\x00\\x00\\x00\\x0F\\x00\\x02\\xE9\\x0D\\x00\\x00\\x0E\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x22\\x0D\\x00\\x00\\x01\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x22\\x0D\\x00\\x00\\x4B\\x11\\x00\\x04\\x33\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x22\\x0D\\x00\\x00\\x0E\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x22\\x0D\\x00\\x04\\x35\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x22\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x22\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x22\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x22\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x22\\x0D\\x00\\x00\\x00\\x0F\\x00\\x00\\xDB\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\xDB\\x0D\\x00\\x00\\x00\\x0F\\x00\\x03\\xB0\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x03\\xB5\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x04\\x0D\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x04\\x0D\\x00\\x00\\x10\\x11\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x4B\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x4B\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x30\\x0D\\x00\\x00\\x0F\\x11\\x00\\x00\\x0B\\x03\\x00\\x00\\xB0\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x30\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x4B\\x11\\x00\\x00\\x01\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x30\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x0B\\x0D\\x00\\x00\\x1B\\x09\\x00\\x00\\x00\\x0F\\x00\\x04\\x33\\x0D\\x00\\x00\\x4B\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x0E\\x0D\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x7F\\x0D\\x00\\x00\\x00\\x0F\\x00\\x00\\x50\\x0D\\x00\\x00\\x07\\x0B\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x4B\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x01\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\xDB\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x01\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x01\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x01\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x01\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x0D\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x05\\x0B\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x01\\x01\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0A\\x0B\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0D\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0D\\x01\\x00\\x00\\x0D\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x0D\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x0D\\x01\\x00\\x00\\x0D\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0B\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x01\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x01\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x03\\x5C\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x03\\x62\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x10\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x02\\xD8\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x03\\x4F\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x08\\x11\\x00\\x03\\x54\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x10\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x02\\xD3\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x03\\x44\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x03\\x48\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x10\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x01\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x0D\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x08\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x21\\x0D\\x00\\x00\\x0F\\x11\\x00\\x00\\x24\\x09\\x00\\x00\\x00\\x0F\\x00\\x00\\x21\\x0D\\x00\\x00\\x00\\x0F\\x00\\x03\\xBA\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x03\\xBF\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x01\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x01\\x11\\x00\\x00\\xF4\\x03\\x00\\x00\\x10\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\xDB\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x02\\x35\\x11\\x00\\x00\\x10\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x02\\x39\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x04\\x11\\x00\\x00\\x4B\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x21\\x09\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x04\\x32\\x03\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x11\\x0D\\x00\\x00\\x00\\x0F\\x00\\x00\\x6C\\x0D\\x00\\x00\\x0D\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x6C\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x00\\x10\\x0D\\x00\\x02\\x4B\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x10\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x10\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x00\\x0F\\x00\\x00\\x10\\x0D\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x02\\xE1\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x01\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x03\\xF8\\x03\\x00\\x00\\xF4\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x03\\xF9\\x03\\x00\\x02\\x7E\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x03\\xFA\\x03\\x00\\x02\\x7E\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x03\\xFB\\x03\\x00\\x02\\x7E\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x03\\xFC\\x03\\x00\\x02\\x7E\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x03\\xFD\\x03\\x00\\x02\\x7E\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0F\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x0F\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x35\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x35\\x11\\x00\\x03\\xF8\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x35\\x11\\x00\\x03\\xF9\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x35\\x11\\x00\\x03\\xFA\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x35\\x11\\x00\\x03\\xFB\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x35\\x11\\x00\\x03\\xFC\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x35\\x11\\x00\\x03\\xFD\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x35\\x11\\x00\\x00\\x6C\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x35\\x11\\x00\\x00\\x10\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x07\\x01\\x00\\x03\\xFE\\x03\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x07\\x01\\x00\\x02\\x7E\\x11\\x00\\x02\\x35\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x07\\x01\\x00\\x02\\x7E\\x11\\x00\\x02\\x35\\x11\\x00\\x02\\x35\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x07\\x01\\x00\\x02\\x7E\\x11\\x00\\x04\\x53\\x03\\x00\\x02\\xE1\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x04\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x04\\x11\\x00\\x00\\x22\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x04\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x4B\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x4B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x04\\x30\\x03\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\xF8\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\xF8\\x11\\x00\\x02\\xF8\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x01\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x4B\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x44\\x11\\x00\\x00\\x50\\x11\\x00\\x00\\x0B\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0B\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\x4B\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0E\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0E\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0E\\x11\\x00\\x00\\x4B\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0E\\x11\\x00\\x00\\x4B\\x11\\x00\\x00\\x01\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0E\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x7F\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x7F\\x11\\x00\\x02\\xE9\\x11\\x00\\x02\\xE9\\x11\\x00\\x02\\xE9\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x7F\\x11\\x00\\x00\\x4B\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x04\\x37\\x03\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x10\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x10\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x08\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x10\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x0D\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x15\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x10\\x11\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x01\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x0F\\x03\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x01\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x34\\x11\\x00\\x02\\xE1\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x0D\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x05\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x03\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x04\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x08\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x00\\x06\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x0F\\x11\\x00\\x02\\xE1\\x11\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x15\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x21\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x21\\x11\\x00\\x00\\x10\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x0A\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x6C\\x03\\x00\\x02\\x7E\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x10\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x10\\x11\\x00\\x00\\x08\\x11\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x02\\xE1\\x11\\x00\\x02\\x7E\\x11\\x00\\x00\\x07\\x01\\x00\\x00\\x00\\x0F\\x00\\x04\\x53\\x0D\\x00\\x00\\x00\\x0F\\x00\\x00\\x24\\x03\\x00\\x00\\x0D\\x09\\x00\\x00\\x0E\\x09\\x00\\x00\\x0F\\x09\\x00\\x00\\x10\\x09\\x00\\x00\\x11\\x09\\x00\\x00\\x12\\x09\\x00\\x00\\x13\\x09\\x00\\x00\\x14\\x09\\x00\\x00\\x04\\x09\\x00\\x00\\x05\\x09\\x00\\x00\\x06\\x09\\x00\\x00\\x07\\x09\\x00\\x00\\x08\\x09\\x00\\x00\\x09\\x09\\x00\\x00\\x0A\\x09\\x00\\x00\\x02\\x01\\x00\\x03\\xFE\\x05\\x00\\x00\\x00\\x80\\x00\\x03\\xFE\\x05\\x00\\x00\\x00\\x10\\x00\\x03\\xFE\\x05\\x00\\x00\\x00\\xC0\\x00\\x03\\xFE\\x05\\x00\\x00\\x00\\x25\\x00\\x03\\xFE\\x05\\x00\\x00\\x00\\x28\\x00\\x03\\xFE\\x05\\x00\\x00\\x00\\x04\\x00\\x03\\xFE\\x05\\x00\\x00\\x00\\x38\\x00\\x03\\xFE\\x05\\x00\\x00\\x00\\x40\\x00\\x03\\xFE\\x05\\x00\\x00\\x1F\\xF0\\x00\\x03\\xFE\\x05\\x00\\x00\\x00\\x08\\x00\\x00\\x00\\x0B\\x00\\x00\\x02\\x0B\\x00\\x00\\x03\\x0B\\x00\\x00\\x06\\x0B\\x00\\x00\\x08\\x0B\\x00\\x00\\x0B\\x09\\x00\\x00\\x22\\x05\\x00\\x00\\x10\\x00\\x00\\x00\\x22\\x05\\x00\\x00\\x00\\x08\\x00\\x00\\x0F\\x01\\x00\\x00\\xDB\\x05\\x00\\x00\\x00\\x04\\x00\\x00\\x09\\x01\\x00\\x03\\xB0\\x05\\x00\\x00\\x00\\x10\\x00\\x03\\xB5\\x05\\x00\\x00\\x00\\x10\\x00\\x03\\xB5\\x05\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x09\\x00\\x00\\x01\\x09\\x00\\x00\\x02\\x09\\x00\\x00\\x03\\x09\\x00\\x04\\x2C\\x03\\x00\\x00\\x0C\\x09\\x00\\x04\\x2E\\x03\\x00\\x00\\x15\\x09\\x00\\x00\\x16\\x09\\x00\\x00\\x17\\x09\\x00\\x00\\x18\\x09\\x00\\x00\\x19\\x09\\x00\\x00\\x1A\\x09\\x00\\x00\\x1C\\x09\\x00\\x00\\x1D\\x09\\x00\\x04\\x37\\x03\\x00\\x00\\x1E\\x09\\x00\\x00\\x1F\\x09\\x00\\x00\\x08\\x05\\x00\\x00\\x10\\x00\\x00\\x00\\x08\\x05\\x00\\x00\\x00\\x06\\x00\\x00\\x22\\x09\\x00\\x00\\x23\\x09\\x00\\x03\\xBA\\x03\\x00\\x03\\xBA\\x05\\x00\\x00\\x00\\x80\\x00\\x03\\xBA\\x05\\x00\\x00\\x00\\x0C\\x00\\x03\\xBA\\x05\\x00\\x00\\x00\\x10\\x00\\x03\\xBA\\x05\\x00\\x00\\x00\\x20\\x00\\x03\\xBA\\x05\\x00\\x00\\x00\\x40\\x00\\x00\\x0C\\x01\\x00\\x00\\x11\\x05\\x00\\x00\\x00\\x04\\x00\\x00\\x10\\x05\\x00\\x00\\x20\\x51\\x00\\x02\\xC6\\x03\\x00\\x02\\xDE\\x03\\x00\\x03\\xE0\\x03\\x00\\x03\\xE7\\x03\\x00\\x00\\x00\\x01',\n _globals = (b'\\xFF\\xFF\\xFF\\x0BGGML_BACKEND_CPU',0,b'\\xFF\\xFF\\xFF\\x0BGGML_BACKEND_GPU',10,b'\\xFF\\xFF\\xFF\\x0BGGML_BACKEND_GPU_SPLIT',20,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_ALL_F32',0,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_F16',1,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q2_K',10,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q3_K',11,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q4_0',2,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q4_1',3,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q4_1_SOME_F16',4,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q4_K',12,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q5_0',8,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q5_1',9,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q5_K',13,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q6_K',14,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_MOSTLY_Q8_0',7,b'\\xFF\\xFF\\xFF\\x0BGGML_FTYPE_UNKNOWN',-1,b'\\xFF\\xFF\\xFF\\x1FGGML_GRAPH_SIZE',164520,b'\\xFF\\xFF\\xFF\\x0BGGML_LINESEARCH_BACKTRACKING_ARMIJO',0,b'\\xFF\\xFF\\xFF\\x0BGGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE',2,b'\\xFF\\xFF\\xFF\\x0BGGML_LINESEARCH_BACKTRACKING_WOLFE',1,b'\\xFF\\xFF\\xFF\\x0BGGML_LINESEARCH_DEFAULT',1,b'\\xFF\\xFF\\xFF\\x0BGGML_LINESEARCH_FAIL',-128,b'\\xFF\\xFF\\xFF\\x0BGGML_LINESEARCH_INVALID_PARAMETERS',-124,b'\\xFF\\xFF\\xFF\\x0BGGML_LINESEARCH_MAXIMUM_ITERATIONS',-125,b'\\xFF\\xFF\\xFF\\x0BGGML_LINESEARCH_MAXIMUM_STEP',-126,b'\\xFF\\xFF\\xFF\\x0BGGML_LINESEARCH_MINIMUM_STEP',-127,b'\\xFF\\xFF\\xFF\\x0BGGML_OBJECT_GRAPH',1,b'\\xFF\\xFF\\xFF\\x1FGGML_OBJECT_SIZE',32,b'\\xFF\\xFF\\xFF\\x0BGGML_OBJECT_TENSOR',0,b'\\xFF\\xFF\\xFF\\x0BGGML_OBJECT_WORK_BUFFER',2,b'\\xFF\\xFF\\xFF\\x0BGGML_OPT_ADAM',0,b'\\xFF\\xFF\\xFF\\x0BGGML_OPT_DID_NOT_CONVERGE',1,b'\\xFF\\xFF\\xFF\\x0BGGML_OPT_FAIL',4,b'\\xFF\\xFF\\xFF\\x0BGGML_OPT_INVALID_WOLFE',3,b'\\xFF\\xFF\\xFF\\x0BGGML_OPT_LBFGS',1,b'\\xFF\\xFF\\xFF\\x0BGGML_OPT_NO_CONTEXT',2,b'\\xFF\\xFF\\xFF\\x0BGGML_OPT_OK',0,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_ACC',4,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_ADD',2,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_ADD1',3,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_ALIBI',40,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_ARGMAX',14,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_CLAMP',41,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_CONT',26,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_CONV_1D',42,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_CONV_2D',43,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_COUNT',62,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_CPY',25,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_CROSS_ENTROPY_LOSS',60,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_CROSS_ENTROPY_LOSS_BACK',61,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_DIAG',33,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_DIAG_MASK_INF',34,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_DIAG_MASK_ZERO',35,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_DIV',7,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_DUP',1,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_FLASH_ATTN',46,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_FLASH_ATTN_BACK',48,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_FLASH_FF',47,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_GET_ROWS',31,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_GET_ROWS_BACK',32,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_LOG',10,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MAP_BINARY',53,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MAP_CUSTOM1',57,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MAP_CUSTOM1_F32',54,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MAP_CUSTOM2',58,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MAP_CUSTOM2_F32',55,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MAP_CUSTOM3',59,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MAP_CUSTOM3_F32',56,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MAP_UNARY',52,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MEAN',13,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MUL',6,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_MUL_MAT',21,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_NONE',0,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_NORM',18,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_OUT_PROD',22,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_PERMUTE',29,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_POOL_1D',44,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_POOL_2D',45,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_POOL_AVG',1,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_POOL_COUNT',2,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_POOL_MAX',0,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_REPEAT',15,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_REPEAT_BACK',16,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_RESHAPE',27,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_RMS_NORM',19,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_RMS_NORM_BACK',20,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_ROPE',38,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_ROPE_BACK',39,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SCALE',23,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SET',24,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SILU_BACK',17,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SOFT_MAX',36,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SOFT_MAX_BACK',37,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SQR',8,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SQRT',9,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SUB',5,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SUM',11,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_SUM_ROWS',12,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_TRANSPOSE',30,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_UNARY',51,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_VIEW',28,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_WIN_PART',49,b'\\xFF\\xFF\\xFF\\x0BGGML_OP_WIN_UNPART',50,b'\\xFF\\xFF\\xFF\\x0BGGML_TASK_COMPUTE',1,b'\\xFF\\xFF\\xFF\\x0BGGML_TASK_FINALIZE',2,b'\\xFF\\xFF\\xFF\\x0BGGML_TASK_INIT',0,b'\\xFF\\xFF\\xFF\\x1FGGML_TENSOR_SIZE',288,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_COUNT',19,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_F16',1,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_F32',0,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_I16',17,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_I32',18,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_I8',16,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q2_K',10,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q3_K',11,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q4_0',2,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q4_1',3,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q4_K',12,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q5_0',6,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q5_1',7,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q5_K',13,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q6_K',14,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q8_0',8,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q8_1',9,b'\\xFF\\xFF\\xFF\\x0BGGML_TYPE_Q8_K',15,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_ABS',0,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_ELU',5,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_GELU',7,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_GELU_QUICK',8,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_NEG',2,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_RELU',6,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_SGN',1,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_SILU',9,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_STEP',3,b'\\xFF\\xFF\\xFF\\x0BGGML_UNARY_OP_TANH',4,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_ARRAY',9,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_BOOL',7,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_COUNT',10,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_FLOAT32',6,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_INT16',3,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_INT32',5,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_INT8',1,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_STRING',8,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_UINT16',2,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_UINT32',4,b'\\xFF\\xFF\\xFF\\x0BGGUF_TYPE_UINT8',0,b'\\x00\\x02\\x9A\\x23__assert_rtn',0,b'\\x00\\x02\\x7C\\x23dequantize_row_q2_K',0,b'\\x00\\x02\\x81\\x23dequantize_row_q3_K',0,b'\\x00\\x02\\x86\\x23dequantize_row_q4_K',0,b'\\x00\\x02\\x8B\\x23dequantize_row_q5_K',0,b'\\x00\\x02\\x90\\x23dequantize_row_q6_K',0,b'\\x00\\x02\\x95\\x23dequantize_row_q8_K',0,b'\\x00\\x00\\xFA\\x23ggml_abs',0,b'\\x00\\x00\\xFA\\x23ggml_abs_inplace',0,b'\\x00\\x01\\xDD\\x23ggml_acc',0,b'\\x00\\x01\\xDD\\x23ggml_acc_inplace',0,b'\\x00\\x01\\x84\\x23ggml_add',0,b'\\x00\\x01\\x84\\x23ggml_add1',0,b'\\x00\\x01\\x84\\x23ggml_add1_inplace',0,b'\\x00\\x01\\x84\\x23ggml_add_inplace',0,b'\\x00\\x01\\x26\\x23ggml_alibi',0,b'\\x00\\x02\\xEC\\x23ggml_allocr_alloc',0,b'\\x00\\x02\\x42\\x23ggml_allocr_alloc_graph',0,b'\\x00\\x02\\xE4\\x23ggml_allocr_free',0,b'\\x00\\x00\\x03\\x23ggml_allocr_is_measure',0,b'\\x00\\x00\\xA2\\x23ggml_allocr_new',0,b'\\x00\\x00\\x9F\\x23ggml_allocr_new_measure',0,b'\\x00\\x02\\xE4\\x23ggml_allocr_reset',0,b'\\x00\\x02\\xE7\\x23ggml_allocr_set_parse_seq',0,b'\\x00\\x00\\x17\\x23ggml_are_same_shape',0,b'\\x00\\x00\\xFA\\x23ggml_argmax',0,b'\\x00\\x00\\x74\\x23ggml_blck_size',0,b'\\x00\\x00\\xB3\\x23ggml_build_backward',0,b'\\x00\\x00\\xB8\\x23ggml_build_forward',0,b'\\x00\\x00\\xAA\\x23ggml_build_forward_ctx',0,b'\\x00\\x02\\xF3\\x23ggml_build_forward_expand',0,b'\\x00\\x00\\x1B\\x23ggml_cl_can_mul_mat',0,b'\\x00\\x03\\x6B\\x23ggml_cl_free_data',0,b'\\x00\\x03\\xE0\\x23ggml_cl_host_free',0,b'\\x00\\x02\\x72\\x23ggml_cl_host_malloc',0,b'\\x00\\x03\\xEC\\x23ggml_cl_init',0,b'\\x00\\x03\\x78\\x23ggml_cl_mul',0,b'\\x00\\x03\\x7D\\x23ggml_cl_mul_mat',0,b'\\x00\\x02\\x54\\x23ggml_cl_mul_mat_get_wsize',0,b'\\x00\\x03\\xE3\\x23ggml_cl_transform_tensor',0,b'\\x00\\x01\\x1B\\x23ggml_clamp',0,b'\\x00\\x00\\xFA\\x23ggml_cont',0,b'\\x00\\x01\\x90\\x23ggml_conv_1d',0,b'\\x00\\x01\\x89\\x23ggml_conv_1d_ph',0,b'\\x00\\x01\\x98\\x23ggml_conv_2d',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_arm_fma',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_avx',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_avx2',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_avx512',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_avx512_vbmi',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_avx512_vnni',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_blas',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_clblast',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_cublas',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_f16c',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_fma',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_fp16_va',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_gpublas',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_neon',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_sse3',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_vsx',0,b'\\x00\\x00\\x90\\x23ggml_cpu_has_wasm_simd',0,b'\\x00\\x01\\x84\\x23ggml_cpy',0,b'\\x00\\x01\\x84\\x23ggml_cross_entropy_loss',0,b'\\x00\\x01\\xA3\\x23ggml_cross_entropy_loss_back',0,b'\\x00\\x03\\x41\\x23ggml_cuda_assign_buffers',0,b'\\x00\\x03\\x41\\x23ggml_cuda_assign_buffers_force_inplace',0,b'\\x00\\x03\\x41\\x23ggml_cuda_assign_buffers_no_scratch',0,b'\\x00\\x00\\x1B\\x23ggml_cuda_can_mul_mat',0,b'\\x00\\x00\\x06\\x23ggml_cuda_compute_forward',0,b'\\x00\\x03\\x41\\x23ggml_cuda_free_data',0,b'\\x00\\x03\\xEC\\x23ggml_cuda_free_scratch',0,b'\\x00\\x00\\x90\\x23ggml_cuda_get_device_count',0,b'\\x00\\x02\\xCE\\x23ggml_cuda_get_device_description',0,b'\\x00\\x03\\xE0\\x23ggml_cuda_host_free',0,b'\\x00\\x02\\x72\\x23ggml_cuda_host_malloc',0,b'\\x00\\x02\\xCB\\x23ggml_cuda_set_main_device',0,b'\\x00\\x02\\x79\\x23ggml_cuda_set_mul_mat_q',0,b'\\x00\\x03\\xD8\\x23ggml_cuda_set_scratch_size',0,b'\\x00\\x02\\xA0\\x23ggml_cuda_set_tensor_split',0,b'\\x00\\x03\\xE3\\x23ggml_cuda_transform_tensor',0,b'\\x00\\x00\\x95\\x23ggml_cycles',0,b'\\x00\\x00\\x95\\x23ggml_cycles_per_ms',0,b'\\x00\\x00\\xFA\\x23ggml_diag',0,b'\\x00\\x01\\x21\\x23ggml_diag_mask_inf',0,b'\\x00\\x01\\x21\\x23ggml_diag_mask_inf_inplace',0,b'\\x00\\x01\\x21\\x23ggml_diag_mask_zero',0,b'\\x00\\x01\\x21\\x23ggml_diag_mask_zero_inplace',0,b'\\x00\\x01\\x84\\x23ggml_div',0,b'\\x00\\x01\\x84\\x23ggml_div_inplace',0,b'\\x00\\x00\\xFA\\x23ggml_dup',0,b'\\x00\\x00\\xFA\\x23ggml_dup_inplace',0,b'\\x00\\x02\\x0B\\x23ggml_dup_tensor',0,b'\\x00\\x02\\x4D\\x23ggml_element_size',0,b'\\x00\\x00\\xFA\\x23ggml_elu',0,b'\\x00\\x00\\xFA\\x23ggml_elu_inplace',0,b'\\x00\\x01\\xA9\\x23ggml_flash_attn',0,b'\\x00\\x01\\xB0\\x23ggml_flash_attn_back',0,b'\\x00\\x01\\xB8\\x23ggml_flash_ff',0,b'\\x00\\x02\\x16\\x23ggml_format_name',0,b'\\x00\\x00\\x6B\\x23ggml_fp16_to_fp32',0,b'\\x00\\x03\\xDB\\x23ggml_fp16_to_fp32_row',0,b'\\x00\\x02\\x62\\x23ggml_fp32_to_fp16',0,b'\\x00\\x02\\xC1\\x23ggml_fp32_to_fp16_row',0,b'\\x00\\x03\\x03\\x23ggml_free',0,b'\\x00\\x00\\x53\\x23ggml_ftype_to_ggml_type',0,b'\\x00\\x00\\xFA\\x23ggml_gelu',0,b'\\x00\\x00\\xFA\\x23ggml_gelu_inplace',0,b'\\x00\\x00\\xFA\\x23ggml_gelu_quick',0,b'\\x00\\x00\\xFA\\x23ggml_gelu_quick_inplace',0,b'\\x00\\x02\\x6C\\x23ggml_get_data',0,b'\\x00\\x00\\x5D\\x23ggml_get_data_f32',0,b'\\x00\\x00\\x63\\x23ggml_get_f32_1d',0,b'\\x00\\x00\\x81\\x23ggml_get_i32_1d',0,b'\\x00\\x02\\x4A\\x23ggml_get_max_tensor_size',0,b'\\x00\\x02\\x69\\x23ggml_get_mem_buffer',0,b'\\x00\\x02\\x4A\\x23ggml_get_mem_size',0,b'\\x00\\x00\\x36\\x23ggml_get_name',0,b'\\x00\\x00\\x0A\\x23ggml_get_no_alloc',0,b'\\x00\\x01\\x84\\x23ggml_get_rows',0,b'\\x00\\x01\\xA3\\x23ggml_get_rows_back',0,b'\\x00\\x00\\xCE\\x23ggml_get_tensor',0,b'\\x00\\x00\\x56\\x23ggml_get_unary_op',0,b'\\x00\\x00\\x77\\x23ggml_graph_compute',0,b'\\x00\\x03\\x0A\\x23ggml_graph_compute_with_ctx',0,b'\\x00\\x02\\xFE\\x23ggml_graph_dump_dot',0,b'\\x00\\x02\\xFA\\x23ggml_graph_export',0,b'\\x00\\x00\\xCA\\x23ggml_graph_get_tensor',0,b'\\x00\\x00\\xAE\\x23ggml_graph_import',0,b'\\x00\\x02\\x60\\x23ggml_graph_overhead',0,b'\\x00\\x00\\xBE\\x23ggml_graph_plan',0,b'\\x00\\x02\\xF7\\x23ggml_graph_print',0,b'\\x00\\x02\\xF0\\x23ggml_graph_reset',0,b'\\x00\\x00\\xBB\\x23ggml_init',0,b'\\x00\\x03\\xEC\\x23ggml_init_cublas',0,b'\\x00\\x00\\x6E\\x23ggml_internal_get_type_traits',0,b'\\x00\\x00\\x14\\x23ggml_is_contiguous',0,b'\\x00\\x00\\x27\\x23ggml_is_numa',0,b'\\x00\\x00\\x14\\x23ggml_is_permuted',0,b'\\x00\\x00\\x00\\x23ggml_is_quantized',0,b'\\x00\\x00\\x14\\x23ggml_is_transposed',0,b'\\x00\\x00\\xFA\\x23ggml_log',0,b'\\x00\\x00\\xFA\\x23ggml_log_inplace',0,b'\\x00\\x01\\xE6\\x23ggml_map_binary_f32',0,b'\\x00\\x01\\xE6\\x23ggml_map_binary_inplace_f32',0,b'\\x00\\x02\\x04\\x23ggml_map_custom1',0,b'\\x00\\x01\\xFF\\x23ggml_map_custom1_f32',0,b'\\x00\\x02\\x04\\x23ggml_map_custom1_inplace',0,b'\\x00\\x01\\xFF\\x23ggml_map_custom1_inplace_f32',0,b'\\x00\\x01\\xF2\\x23ggml_map_custom2',0,b'\\x00\\x01\\xEC\\x23ggml_map_custom2_f32',0,b'\\x00\\x01\\xF2\\x23ggml_map_custom2_inplace',0,b'\\x00\\x01\\xEC\\x23ggml_map_custom2_inplace_f32',0,b'\\x00\\x01\\xC7\\x23ggml_map_custom3',0,b'\\x00\\x01\\xC0\\x23ggml_map_custom3_f32',0,b'\\x00\\x01\\xC7\\x23ggml_map_custom3_inplace',0,b'\\x00\\x01\\xC0\\x23ggml_map_custom3_inplace_f32',0,b'\\x00\\x01\\xFA\\x23ggml_map_unary_f32',0,b'\\x00\\x01\\xFA\\x23ggml_map_unary_inplace_f32',0,b'\\x00\\x00\\xFA\\x23ggml_mean',0,b'\\x00\\x00\\x0D\\x23ggml_metal_add_buffer',0,b'\\x00\\x03\\x1C\\x23ggml_metal_free',0,b'\\x00\\x00\\x71\\x23ggml_metal_get_concur_list',0,b'\\x00\\x03\\x2C\\x23ggml_metal_get_tensor',0,b'\\x00\\x03\\x23\\x23ggml_metal_graph_compute',0,b'\\x00\\x03\\x27\\x23ggml_metal_graph_find_concurrency',0,b'\\x00\\x03\\xE0\\x23ggml_metal_host_free',0,b'\\x00\\x02\\x72\\x23ggml_metal_host_malloc',0,b'\\x00\\x00\\x7B\\x23ggml_metal_if_optimized',0,b'\\x00\\x00\\xC2\\x23ggml_metal_init',0,b'\\x00\\x03\\x1F\\x23ggml_metal_set_n_cb',0,b'\\x00\\x03\\x2C\\x23ggml_metal_set_tensor',0,b'\\x00\\x03\\xEC\\x23ggml_mpi_backend_free',0,b'\\x00\\x03\\xEC\\x23ggml_mpi_backend_init',0,b'\\x00\\x03\\x33\\x23ggml_mpi_eval_init',0,b'\\x00\\x03\\x30\\x23ggml_mpi_free',0,b'\\x00\\x03\\x39\\x23ggml_mpi_graph_compute_post',0,b'\\x00\\x03\\x39\\x23ggml_mpi_graph_compute_pre',0,b'\\x00\\x00\\xC5\\x23ggml_mpi_init',0,b'\\x00\\x00\\x7E\\x23ggml_mpi_rank',0,b'\\x00\\x01\\x84\\x23ggml_mul',0,b'\\x00\\x01\\x84\\x23ggml_mul_inplace',0,b'\\x00\\x01\\x84\\x23ggml_mul_mat',0,b'\\x00\\x02\\x4D\\x23ggml_nbytes',0,b'\\x00\\x02\\x4D\\x23ggml_nbytes_pad',0,b'\\x00\\x02\\x50\\x23ggml_nbytes_split',0,b'\\x00\\x00\\xFA\\x23ggml_neg',0,b'\\x00\\x00\\xFA\\x23ggml_neg_inplace',0,b'\\x00\\x00\\x92\\x23ggml_nelements',0,b'\\x00\\x00\\xF2\\x23ggml_new_f32',0,b'\\x00\\x00\\xA7\\x23ggml_new_graph',0,b'\\x00\\x00\\xF6\\x23ggml_new_i32',0,b'\\x00\\x00\\xD2\\x23ggml_new_tensor',0,b'\\x00\\x00\\xD8\\x23ggml_new_tensor_1d',0,b'\\x00\\x00\\xDD\\x23ggml_new_tensor_2d',0,b'\\x00\\x00\\xE3\\x23ggml_new_tensor_3d',0,b'\\x00\\x00\\xEA\\x23ggml_new_tensor_4d',0,b'\\x00\\x00\\xFA\\x23ggml_norm',0,b'\\x00\\x00\\xFA\\x23ggml_norm_inplace',0,b'\\x00\\x00\\x92\\x23ggml_nrows',0,b'\\x00\\x03\\xEC\\x23ggml_numa_init',0,b'\\x00\\x00\\x2D\\x23ggml_op_name',0,b'\\x00\\x00\\x2D\\x23ggml_op_symbol',0,b'\\x00\\x00\\x4E\\x23ggml_opt',0,b'\\x00\\x00\\xC7\\x23ggml_opt_default_params',0,b'\\x00\\x03\\x0F\\x23ggml_opt_init',0,b'\\x00\\x00\\x42\\x23ggml_opt_resume',0,b'\\x00\\x00\\x47\\x23ggml_opt_resume_g',0,b'\\x00\\x01\\x84\\x23ggml_out_prod',0,b'\\x00\\x01\\x34\\x23ggml_permute',0,b'\\x00\\x00\\xFE\\x23ggml_pool_1d',0,b'\\x00\\x01\\x06\\x23ggml_pool_2d',0,b'\\x00\\x03\\x3E\\x23ggml_print_object',0,b'\\x00\\x03\\x19\\x23ggml_print_objects',0,b'\\x00\\x02\\x33\\x23ggml_quantize_chunk',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q2_K',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q3_K',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q4_0',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q4_1',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q4_K',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q5_0',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q5_1',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q5_K',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q6_K',0,b'\\x00\\x02\\x3B\\x23ggml_quantize_q8_0',0,b'\\x00\\x00\\xFA\\x23ggml_relu',0,b'\\x00\\x00\\xFA\\x23ggml_relu_inplace',0,b'\\x00\\x01\\x84\\x23ggml_repeat',0,b'\\x00\\x01\\x84\\x23ggml_repeat_back',0,b'\\x00\\x01\\x84\\x23ggml_reshape',0,b'\\x00\\x01\\x46\\x23ggml_reshape_1d',0,b'\\x00\\x01\\x4B\\x23ggml_reshape_2d',0,b'\\x00\\x01\\x51\\x23ggml_reshape_3d',0,b'\\x00\\x01\\x58\\x23ggml_reshape_4d',0,b'\\x00\\x01\\x16\\x23ggml_rms_norm',0,b'\\x00\\x01\\x84\\x23ggml_rms_norm_back',0,b'\\x00\\x01\\x16\\x23ggml_rms_norm_inplace',0,b'\\x00\\x01\\x34\\x23ggml_rope',0,b'\\x00\\x01\\x34\\x23ggml_rope_back',0,b'\\x00\\x01\\x3C\\x23ggml_rope_custom',0,b'\\x00\\x01\\x3C\\x23ggml_rope_custom_inplace',0,b'\\x00\\x01\\x34\\x23ggml_rope_inplace',0,b'\\x00\\x01\\x84\\x23ggml_scale',0,b'\\x00\\x01\\x84\\x23ggml_scale_inplace',0,b'\\x00\\x01\\xDD\\x23ggml_set',0,b'\\x00\\x01\\xD0\\x23ggml_set_1d',0,b'\\x00\\x01\\xD0\\x23ggml_set_1d_inplace',0,b'\\x00\\x01\\xD6\\x23ggml_set_2d',0,b'\\x00\\x01\\xD6\\x23ggml_set_2d_inplace',0,b'\\x00\\x02\\x1A\\x23ggml_set_f32',0,b'\\x00\\x03\\x6E\\x23ggml_set_f32_1d',0,b'\\x00\\x02\\x1E\\x23ggml_set_i32',0,b'\\x00\\x03\\x73\\x23ggml_set_i32_1d',0,b'\\x00\\x01\\xDD\\x23ggml_set_inplace',0,b'\\x00\\x02\\x12\\x23ggml_set_name',0,b'\\x00\\x03\\x06\\x23ggml_set_no_alloc',0,b'\\x00\\x03\\x15\\x23ggml_set_param',0,b'\\x00\\x02\\x46\\x23ggml_set_scratch',0,b'\\x00\\x02\\x0F\\x23ggml_set_zero',0,b'\\x00\\x00\\xFA\\x23ggml_sgn',0,b'\\x00\\x00\\xFA\\x23ggml_sgn_inplace',0,b'\\x00\\x00\\xFA\\x23ggml_silu',0,b'\\x00\\x01\\x84\\x23ggml_silu_back',0,b'\\x00\\x00\\xFA\\x23ggml_silu_inplace',0,b'\\x00\\x00\\xFA\\x23ggml_soft_max',0,b'\\x00\\x01\\x84\\x23ggml_soft_max_back',0,b'\\x00\\x01\\x84\\x23ggml_soft_max_back_inplace',0,b'\\x00\\x00\\xFA\\x23ggml_soft_max_inplace',0,b'\\x00\\x00\\xFA\\x23ggml_sqr',0,b'\\x00\\x00\\xFA\\x23ggml_sqr_inplace',0,b'\\x00\\x00\\xFA\\x23ggml_sqrt',0,b'\\x00\\x00\\xFA\\x23ggml_sqrt_inplace',0,b'\\x00\\x00\\xFA\\x23ggml_step',0,b'\\x00\\x00\\xFA\\x23ggml_step_inplace',0,b'\\x00\\x01\\x84\\x23ggml_sub',0,b'\\x00\\x01\\x84\\x23ggml_sub_inplace',0,b'\\x00\\x00\\xFA\\x23ggml_sum',0,b'\\x00\\x00\\xFA\\x23ggml_sum_rows',0,b'\\x00\\x00\\xFA\\x23ggml_tanh',0,b'\\x00\\x00\\xFA\\x23ggml_tanh_inplace',0,b'\\x00\\x02\\x60\\x23ggml_tensor_overhead',0,b'\\x00\\x03\\xEC\\x23ggml_time_init',0,b'\\x00\\x00\\x95\\x23ggml_time_ms',0,b'\\x00\\x00\\x95\\x23ggml_time_us',0,b'\\x00\\x00\\xFA\\x23ggml_transpose',0,b'\\x00\\x00\\x30\\x23ggml_type_name',0,b'\\x00\\x02\\x30\\x23ggml_type_size',0,b'\\x00\\x00\\x60\\x23ggml_type_sizef',0,b'\\x00\\x01\\x11\\x23ggml_unary',0,b'\\x00\\x01\\x11\\x23ggml_unary_inplace',0,b'\\x00\\x02\\x4A\\x23ggml_used_mem',0,b'\\x00\\x02\\xDE\\x23ggml_vec_dot_q2_K_q8_K',0,b'\\x00\\x02\\xDE\\x23ggml_vec_dot_q3_K_q8_K',0,b'\\x00\\x02\\xDE\\x23ggml_vec_dot_q4_K_q8_K',0,b'\\x00\\x02\\xDE\\x23ggml_vec_dot_q5_K_q8_K',0,b'\\x00\\x02\\xDE\\x23ggml_vec_dot_q6_K_q8_K',0,b'\\x00\\x01\\x7E\\x23ggml_view_1d',0,b'\\x00\\x01\\x76\\x23ggml_view_2d',0,b'\\x00\\x01\\x6C\\x23ggml_view_3d',0,b'\\x00\\x01\\x60\\x23ggml_view_4d',0,b'\\x00\\x02\\x0B\\x23ggml_view_tensor',0,b'\\x00\\x01\\x21\\x23ggml_win_part',0,b'\\x00\\x01\\x2D\\x23ggml_win_unpart',0,b'\\x00\\x03\\xCC\\x23gguf_add_tensor',0,b'\\x00\\x00\\x88\\x23gguf_find_key',0,b'\\x00\\x00\\x88\\x23gguf_find_tensor',0,b'\\x00\\x03\\x84\\x23gguf_free',0,b'\\x00\\x02\\x59\\x23gguf_get_alignment',0,b'\\x00\\x02\\x75\\x23gguf_get_arr_data',0,b'\\x00\\x00\\x8C\\x23gguf_get_arr_n',0,b'\\x00\\x00\\x3D\\x23gguf_get_arr_str',0,b'\\x00\\x00\\x59\\x23gguf_get_arr_type',0,b'\\x00\\x02\\x6F\\x23gguf_get_data',0,b'\\x00\\x02\\x59\\x23gguf_get_data_offset',0,b'\\x00\\x00\\x39\\x23gguf_get_key',0,b'\\x00\\x00\\x59\\x23gguf_get_kv_type',0,b'\\x00\\x03\\xD4\\x23gguf_get_meta_data',0,b'\\x00\\x02\\x59\\x23gguf_get_meta_size',0,b'\\x00\\x00\\x85\\x23gguf_get_n_kv',0,b'\\x00\\x00\\x85\\x23gguf_get_n_tensors',0,b'\\x00\\x00\\x29\\x23gguf_get_tensor_name',0,b'\\x00\\x02\\x5C\\x23gguf_get_tensor_offset',0,b'\\x00\\x00\\x20\\x23gguf_get_val_bool',0,b'\\x00\\x00\\x67\\x23gguf_get_val_f32',0,b'\\x00\\x00\\x97\\x23gguf_get_val_i16',0,b'\\x00\\x00\\x8C\\x23gguf_get_val_i32',0,b'\\x00\\x00\\x9B\\x23gguf_get_val_i8',0,b'\\x00\\x00\\x39\\x23gguf_get_val_str',0,b'\\x00\\x02\\x65\\x23gguf_get_val_u16',0,b'\\x00\\x02\\x2C\\x23gguf_get_val_u32',0,b'\\x00\\x02\\x28\\x23gguf_get_val_u8',0,b'\\x00\\x00\\x85\\x23gguf_get_version',0,b'\\x00\\x02\\x26\\x23gguf_init_empty',0,b'\\x00\\x02\\x22\\x23gguf_init_from_file',0,b'\\x00\\x03\\x9C\\x23gguf_set_arr_data',0,b'\\x00\\x03\\x8C\\x23gguf_set_arr_str',0,b'\\x00\\x03\\xD0\\x23gguf_set_kv',0,b'\\x00\\x03\\xC6\\x23gguf_set_tensor_data',0,b'\\x00\\x03\\x97\\x23gguf_set_tensor_type',0,b'\\x00\\x03\\x87\\x23gguf_set_val_bool',0,b'\\x00\\x03\\xA3\\x23gguf_set_val_f32',0,b'\\x00\\x03\\xAD\\x23gguf_set_val_i16',0,b'\\x00\\x03\\xA8\\x23gguf_set_val_i32',0,b'\\x00\\x03\\xB2\\x23gguf_set_val_i8',0,b'\\x00\\x03\\x92\\x23gguf_set_val_str',0,b'\\x00\\x03\\xC1\\x23gguf_set_val_u16',0,b'\\x00\\x03\\xBC\\x23gguf_set_val_u32',0,b'\\x00\\x03\\xB7\\x23gguf_set_val_u8',0,b'\\x00\\x00\\x33\\x23gguf_type_name',0,b'\\x00\\x03\\x87\\x23gguf_write_to_file',0,b'\\x00\\x02\\xC6\\x23quantize_row_q2_K',0,b'\\x00\\x02\\xA3\\x23quantize_row_q2_K_reference',0,b'\\x00\\x02\\xC6\\x23quantize_row_q3_K',0,b'\\x00\\x02\\xA8\\x23quantize_row_q3_K_reference',0,b'\\x00\\x02\\xC6\\x23quantize_row_q4_K',0,b'\\x00\\x02\\xAD\\x23quantize_row_q4_K_reference',0,b'\\x00\\x02\\xC6\\x23quantize_row_q5_K',0,b'\\x00\\x02\\xB2\\x23quantize_row_q5_K_reference',0,b'\\x00\\x02\\xC6\\x23quantize_row_q6_K',0,b'\\x00\\x02\\xB7\\x23quantize_row_q6_K_reference',0,b'\\x00\\x02\\xC6\\x23quantize_row_q8_K',0,b'\\x00\\x02\\xBC\\x23quantize_row_q8_K_reference',0),\n _struct_unions = ((b'\\x00\\x00\\x04\\x27\\x00\\x00\\x00\\x02$1',b'\\x00\\x00\\x22\\x11n_iter',b'\\x00\\x00\\xF4\\x11sched',b'\\x00\\x00\\xF4\\x11decay',b'\\x00\\x00\\xF4\\x11alpha',b'\\x00\\x00\\xF4\\x11beta1',b'\\x00\\x00\\xF4\\x11beta2',b'\\x00\\x00\\xF4\\x11eps',b'\\x00\\x00\\xF4\\x11eps_f',b'\\x00\\x00\\xF4\\x11eps_g'),(b'\\x00\\x00\\x04\\x28\\x00\\x00\\x00\\x02$2',b'\\x00\\x00\\x22\\x11m',b'\\x00\\x00\\x22\\x11n_iter',b'\\x00\\x00\\x22\\x11max_linesearch',b'\\x00\\x00\\xF4\\x11eps',b'\\x00\\x00\\xF4\\x11ftol',b'\\x00\\x00\\xF4\\x11wolfe',b'\\x00\\x00\\xF4\\x11min_step',b'\\x00\\x00\\xF4\\x11max_step',b'\\x00\\x04\\x14\\x11linesearch'),(b'\\x00\\x00\\x04\\x29\\x00\\x00\\x00\\x02$3',b'\\x00\\x00\\x08\\x11x',b'\\x00\\x00\\x08\\x11g1',b'\\x00\\x00\\x08\\x11g2',b'\\x00\\x00\\x08\\x11m',b'\\x00\\x00\\x08\\x11v',b'\\x00\\x00\\x08\\x11mh',b'\\x00\\x00\\x08\\x11vh',b'\\x00\\x00\\x08\\x11pf',b'\\x00\\x00\\xF4\\x11fx_best',b'\\x00\\x00\\xF4\\x11fx_prev',b'\\x00\\x00\\x22\\x11n_no_improvement'),(b'\\x00\\x00\\x04\\x2A\\x00\\x00\\x00\\x02$4',b'\\x00\\x00\\x08\\x11x',b'\\x00\\x00\\x08\\x11xp',b'\\x00\\x00\\x08\\x11g',b'\\x00\\x00\\x08\\x11gp',b'\\x00\\x00\\x08\\x11d',b'\\x00\\x00\\x08\\x11pf',b'\\x00\\x00\\x08\\x11lmal',b'\\x00\\x00\\x08\\x11lmys',b'\\x00\\x00\\x08\\x11lms',b'\\x00\\x00\\x08\\x11lmy',b'\\x00\\x00\\xF4\\x11fx_best',b'\\x00\\x00\\xF4\\x11step',b'\\x00\\x00\\x22\\x11j',b'\\x00\\x00\\x22\\x11k',b'\\x00\\x00\\x22\\x11end',b'\\x00\\x00\\x22\\x11n_no_improvement'),(b'\\x00\\x00\\x03\\xF7\\x00\\x00\\x00\\x03$__mbstate_t',b'\\x00\\x03\\xFF\\x11__mbstate8',b'\\x00\\x00\\xDB\\x11_mbstateL'),(b'\\x00\\x00\\x03\\xF8\\x00\\x00\\x00\\x02$block_q2_K',b'\\x00\\x04\\x44\\x11scales',b'\\x00\\x04\\x48\\x11qs',b'\\x00\\x00\\x6C\\x11d',b'\\x00\\x00\\x6C\\x11dmin'),(b'\\x00\\x00\\x03\\xF9\\x00\\x00\\x00\\x02$block_q3_K',b'\\x00\\x04\\x46\\x11hmask',b'\\x00\\x04\\x48\\x11qs',b'\\x00\\x04\\x42\\x11scales',b'\\x00\\x00\\x6C\\x11d'),(b'\\x00\\x00\\x03\\xFA\\x00\\x00\\x00\\x02$block_q4_K',b'\\x00\\x00\\x6C\\x11d',b'\\x00\\x00\\x6C\\x11dmin',b'\\x00\\x04\\x42\\x11scales',b'\\x00\\x04\\x40\\x11qs'),(b'\\x00\\x00\\x03\\xFB\\x00\\x00\\x00\\x02$block_q5_K',b'\\x00\\x00\\x6C\\x11d',b'\\x00\\x00\\x6C\\x11dmin',b'\\x00\\x04\\x42\\x11scales',b'\\x00\\x04\\x46\\x11qh',b'\\x00\\x04\\x40\\x11qs'),(b'\\x00\\x00\\x03\\xFC\\x00\\x00\\x00\\x02$block_q6_K',b'\\x00\\x04\\x40\\x11ql',b'\\x00\\x04\\x48\\x11qh',b'\\x00\\x04\\x23\\x11scales',b'\\x00\\x00\\x6C\\x11d'),(b'\\x00\\x00\\x03\\xFD\\x00\\x00\\x00\\x02$block_q8_K',b'\\x00\\x00\\xF4\\x11d',b'\\x00\\x04\\x25\\x11qs',b'\\x00\\x04\\x21\\x11bsums'),(b'\\x00\\x00\\x04\\x18\\x00\\x00\\x00\\x02$ggml_type_traits_t',b'\\x00\\x00\\x0F\\x11type_name',b'\\x00\\x00\\x22\\x11blck_size',b'\\x00\\x00\\x11\\x11type_size',b'\\x00\\x00\\xB6\\x11is_quantized',b'\\x00\\x04\\x52\\x11to_float',b'\\x00\\x04\\x4F\\x11from_float',b'\\x00\\x04\\x4F\\x11from_float_reference',b'\\x00\\x04\\x50\\x11vec_dot',b'\\x00\\x00\\x01\\x11vec_dot_type'),(b'\\x00\\x00\\x04\\x2C\\x00\\x00\\x00\\x02__darwin_pthread_handler_rec',b'\\x00\\x04\\x51\\x11__routine',b'\\x00\\x00\\x10\\x11__arg',b'\\x00\\x04\\x2B\\x11__next'),(b'\\x00\\x00\\x03\\xEF\\x00\\x00\\x00\\x02_opaque_pthread_attr_t',b'\\x00\\x04\\x20\\x11__sig',b'\\x00\\x04\\x0B\\x11__opaque'),(b'\\x00\\x00\\x03\\xF0\\x00\\x00\\x00\\x02_opaque_pthread_cond_t',b'\\x00\\x04\\x20\\x11__sig',b'\\x00\\x04\\x07\\x11__opaque'),(b'\\x00\\x00\\x03\\xF1\\x00\\x00\\x00\\x02_opaque_pthread_condattr_t',b'\\x00\\x04\\x20\\x11__sig',b'\\x00\\x04\\x11\\x11__opaque'),(b'\\x00\\x00\\x03\\xF2\\x00\\x00\\x00\\x02_opaque_pthread_mutex_t',b'\\x00\\x04\\x20\\x11__sig',b'\\x00\\x04\\x0B\\x11__opaque'),(b'\\x00\\x00\\x03\\xF3\\x00\\x00\\x00\\x02_opaque_pthread_mutexattr_t',b'\\x00\\x04\\x20\\x11__sig',b'\\x00\\x04\\x11\\x11__opaque'),(b'\\x00\\x00\\x03\\xF4\\x00\\x00\\x00\\x02_opaque_pthread_once_t',b'\\x00\\x04\\x20\\x11__sig',b'\\x00\\x04\\x11\\x11__opaque'),(b'\\x00\\x00\\x03\\xF5\\x00\\x00\\x00\\x02_opaque_pthread_rwlock_t',b'\\x00\\x04\\x20\\x11__sig',b'\\x00\\x04\\x03\\x11__opaque'),(b'\\x00\\x00\\x03\\xF6\\x00\\x00\\x00\\x02_opaque_pthread_rwlockattr_t',b'\\x00\\x04\\x20\\x11__sig',b'\\x00\\x04\\x01\\x11__opaque'),(b'\\x00\\x00\\x04\\x2E\\x00\\x00\\x00\\x02_opaque_pthread_t',b'\\x00\\x04\\x20\\x11__sig',b'\\x00\\x04\\x2B\\x11__cleanup_stack',b'\\x00\\x04\\x0F\\x11__opaque'),(b'\\x00\\x00\\x04\\x2F\\x00\\x00\\x00\\x10ggml_allocr',),(b'\\x00\\x00\\x04\\x30\\x00\\x00\\x00\\x02ggml_cgraph',b'\\x00\\x00\\x22\\x11n_nodes',b'\\x00\\x00\\x22\\x11n_leafs',b'\\x00\\x04\\x39\\x11nodes',b'\\x00\\x04\\x39\\x11grads',b'\\x00\\x04\\x39\\x11leafs',b'\\x00\\x04\\x4D\\x11visited_hash_table',b'\\x00\\x00\\x22\\x11perf_runs',b'\\x00\\x00\\xDB\\x11perf_cycles',b'\\x00\\x00\\xDB\\x11perf_time_us'),(b'\\x00\\x00\\x04\\x31\\x00\\x00\\x00\\x02ggml_compute_params',b'\\x00\\x04\\x17\\x11type',b'\\x00\\x00\\x22\\x11ith',b'\\x00\\x00\\x22\\x11nth',b'\\x00\\x00\\x11\\x11wsize',b'\\x00\\x00\\x10\\x11wdata'),(b'\\x00\\x00\\x04\\x32\\x00\\x00\\x00\\x10ggml_context',),(b'\\x00\\x00\\x04\\x33\\x00\\x00\\x00\\x02ggml_cplan',b'\\x00\\x00\\x11\\x11work_size',b'\\x00\\x04\\x3F\\x11work_data',b'\\x00\\x00\\x22\\x11n_threads',b'\\x00\\x04\\x19\\x11n_tasks',b'\\x00\\x03\\xEE\\x11abort_callback',b'\\x00\\x00\\x10\\x11abort_callback_data'),(b'\\x00\\x00\\x00\\xBC\\x00\\x00\\x00\\x02ggml_init_params',b'\\x00\\x00\\x11\\x11mem_size',b'\\x00\\x00\\x10\\x11mem_buffer',b'\\x00\\x00\\xB6\\x11no_alloc'),(b'\\x00\\x00\\x04\\x34\\x00\\x00\\x00\\x10ggml_metal_context',),(b'\\x00\\x00\\x04\\x35\\x00\\x00\\x00\\x10ggml_mpi_context',),(b'\\x00\\x00\\x04\\x37\\x00\\x00\\x00\\x02ggml_object',b'\\x00\\x00\\x11\\x11offs',b'\\x00\\x00\\x11\\x11size',b'\\x00\\x04\\x36\\x11next',b'\\x00\\x04\\x15\\x11type',b'\\x00\\x04\\x09\\x11padding'),(b'\\x00\\x00\\x04\\x38\\x00\\x00\\x00\\x02ggml_opt_context',b'\\x00\\x00\\x0B\\x11ctx',b'\\x00\\x00\\x50\\x11params',b'\\x00\\x00\\x22\\x11iter',b'\\x00\\x00\\xDB\\x11nx',b'\\x00\\x00\\xB6\\x11just_initialized',b'\\x00\\x04\\x29\\x11adam',b'\\x00\\x04\\x2A\\x11lbfgs'),(b'\\x00\\x00\\x00\\x50\\x00\\x00\\x00\\x02ggml_opt_params',b'\\x00\\x00\\xC8\\x11type',b'\\x00\\x00\\x22\\x11n_threads',b'\\x00\\x00\\x22\\x11past',b'\\x00\\x00\\xF4\\x11delta',b'\\x00\\x00\\x22\\x11max_no_improvement',b'\\x00\\x00\\xB6\\x11print_forward_graph',b'\\x00\\x00\\xB6\\x11print_backward_graph',b'\\x00\\x04\\x27\\x11adam',b'\\x00\\x04\\x28\\x11lbfgs'),(b'\\x00\\x00\\x02\\x48\\x00\\x00\\x00\\x02ggml_scratch',b'\\x00\\x00\\x11\\x11offs',b'\\x00\\x00\\x11\\x11size',b'\\x00\\x00\\x10\\x11data'),(b'\\x00\\x00\\x04\\x3D\\x00\\x00\\x00\\x02ggml_tensor',b'\\x00\\x00\\x01\\x11type',b'\\x00\\x04\\x13\\x11backend',b'\\x00\\x00\\x22\\x11n_dims',b'\\x00\\x04\\x1E\\x11ne',b'\\x00\\x04\\x4B\\x11nb',b'\\x00\\x00\\x2E\\x11op',b'\\x00\\x04\\x1B\\x11op_params',b'\\x00\\x00\\xB6\\x11is_param',b'\\x00\\x00\\x08\\x11grad',b'\\x00\\x04\\x3B\\x11src',b'\\x00\\x00\\x22\\x11perf_runs',b'\\x00\\x00\\xDB\\x11perf_cycles',b'\\x00\\x00\\xDB\\x11perf_time_us',b'\\x00\\x00\\x10\\x11data',b'\\x00\\x04\\x0D\\x11name',b'\\x00\\x00\\x10\\x11extra',b'\\x00\\x04\\x09\\x11padding'),(b'\\x00\\x00\\x04\\x3E\\x00\\x00\\x00\\x10gguf_context',),(b'\\x00\\x00\\x02\\x24\\x00\\x00\\x00\\x02gguf_init_params',b'\\x00\\x00\\xB6\\x11no_alloc',b'\\x00\\x00\\xB0\\x11ctx')),\n _enums = (b'\\x00\\x00\\x04\\x13\\x00\\x00\\x00\\x16ggml_backend\\x00GGML_BACKEND_CPU,GGML_BACKEND_GPU,GGML_BACKEND_GPU_SPLIT',b'\\x00\\x00\\x00\\x54\\x00\\x00\\x00\\x15ggml_ftype\\x00GGML_FTYPE_UNKNOWN,GGML_FTYPE_ALL_F32,GGML_FTYPE_MOSTLY_F16,GGML_FTYPE_MOSTLY_Q4_0,GGML_FTYPE_MOSTLY_Q4_1,GGML_FTYPE_MOSTLY_Q4_1_SOME_F16,GGML_FTYPE_MOSTLY_Q8_0,GGML_FTYPE_MOSTLY_Q5_0,GGML_FTYPE_MOSTLY_Q5_1,GGML_FTYPE_MOSTLY_Q2_K,GGML_FTYPE_MOSTLY_Q3_K,GGML_FTYPE_MOSTLY_Q4_K,GGML_FTYPE_MOSTLY_Q5_K,GGML_FTYPE_MOSTLY_Q6_K',b'\\x00\\x00\\x04\\x14\\x00\\x00\\x00\\x16ggml_linesearch\\x00GGML_LINESEARCH_DEFAULT,GGML_LINESEARCH_BACKTRACKING_ARMIJO,GGML_LINESEARCH_BACKTRACKING_WOLFE,GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE',b'\\x00\\x00\\x04\\x15\\x00\\x00\\x00\\x16ggml_object_type\\x00GGML_OBJECT_TENSOR,GGML_OBJECT_GRAPH,GGML_OBJECT_WORK_BUFFER',b'\\x00\\x00\\x00\\x2E\\x00\\x00\\x00\\x16ggml_op\\x00GGML_OP_NONE,GGML_OP_DUP,GGML_OP_ADD,GGML_OP_ADD1,GGML_OP_ACC,GGML_OP_SUB,GGML_OP_MUL,GGML_OP_DIV,GGML_OP_SQR,GGML_OP_SQRT,GGML_OP_LOG,GGML_OP_SUM,GGML_OP_SUM_ROWS,GGML_OP_MEAN,GGML_OP_ARGMAX,GGML_OP_REPEAT,GGML_OP_REPEAT_BACK,GGML_OP_SILU_BACK,GGML_OP_NORM,GGML_OP_RMS_NORM,GGML_OP_RMS_NORM_BACK,GGML_OP_MUL_MAT,GGML_OP_OUT_PROD,GGML_OP_SCALE,GGML_OP_SET,GGML_OP_CPY,GGML_OP_CONT,GGML_OP_RESHAPE,GGML_OP_VIEW,GGML_OP_PERMUTE,GGML_OP_TRANSPOSE,GGML_OP_GET_ROWS,GGML_OP_GET_ROWS_BACK,GGML_OP_DIAG,GGML_OP_DIAG_MASK_INF,GGML_OP_DIAG_MASK_ZERO,GGML_OP_SOFT_MAX,GGML_OP_SOFT_MAX_BACK,GGML_OP_ROPE,GGML_OP_ROPE_BACK,GGML_OP_ALIBI,GGML_OP_CLAMP,GGML_OP_CONV_1D,GGML_OP_CONV_2D,GGML_OP_POOL_1D,GGML_OP_POOL_2D,GGML_OP_FLASH_ATTN,GGML_OP_FLASH_FF,GGML_OP_FLASH_ATTN_BACK,GGML_OP_WIN_PART,GGML_OP_WIN_UNPART,GGML_OP_UNARY,GGML_OP_MAP_UNARY,GGML_OP_MAP_BINARY,GGML_OP_MAP_CUSTOM1_F32,GGML_OP_MAP_CUSTOM2_F32,GGML_OP_MAP_CUSTOM3_F32,GGML_OP_MAP_CUSTOM1,GGML_OP_MAP_CUSTOM2,GGML_OP_MAP_CUSTOM3,GGML_OP_CROSS_ENTROPY_LOSS,GGML_OP_CROSS_ENTROPY_LOSS_BACK,GGML_OP_COUNT',b'\\x00\\x00\\x01\\x01\\x00\\x00\\x00\\x16ggml_op_pool\\x00GGML_OP_POOL_MAX,GGML_OP_POOL_AVG,GGML_OP_POOL_COUNT',b'\\x00\\x00\\x04\\x16\\x00\\x00\\x00\\x15ggml_opt_result\\x00GGML_OPT_OK,GGML_OPT_DID_NOT_CONVERGE,GGML_OPT_NO_CONTEXT,GGML_OPT_INVALID_WOLFE,GGML_OPT_FAIL,GGML_LINESEARCH_FAIL,GGML_LINESEARCH_MINIMUM_STEP,GGML_LINESEARCH_MAXIMUM_STEP,GGML_LINESEARCH_MAXIMUM_ITERATIONS,GGML_LINESEARCH_INVALID_PARAMETERS',b'\\x00\\x00\\x00\\xC8\\x00\\x00\\x00\\x16ggml_opt_type\\x00GGML_OPT_ADAM,GGML_OPT_LBFGS',b'\\x00\\x00\\x04\\x17\\x00\\x00\\x00\\x16ggml_task_type\\x00GGML_TASK_INIT,GGML_TASK_COMPUTE,GGML_TASK_FINALIZE',b'\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x16ggml_type\\x00GGML_TYPE_F32,GGML_TYPE_F16,GGML_TYPE_Q4_0,GGML_TYPE_Q4_1,GGML_TYPE_Q5_0,GGML_TYPE_Q5_1,GGML_TYPE_Q8_0,GGML_TYPE_Q8_1,GGML_TYPE_Q2_K,GGML_TYPE_Q3_K,GGML_TYPE_Q4_K,GGML_TYPE_Q5_K,GGML_TYPE_Q6_K,GGML_TYPE_Q8_K,GGML_TYPE_I8,GGML_TYPE_I16,GGML_TYPE_I32,GGML_TYPE_COUNT',b'\\x00\\x00\\x01\\x14\\x00\\x00\\x00\\x16ggml_unary_op\\x00GGML_UNARY_OP_ABS,GGML_UNARY_OP_SGN,GGML_UNARY_OP_NEG,GGML_UNARY_OP_STEP,GGML_UNARY_OP_TANH,GGML_UNARY_OP_ELU,GGML_UNARY_OP_RELU,GGML_UNARY_OP_GELU,GGML_UNARY_OP_GELU_QUICK,GGML_UNARY_OP_SILU',b'\\x00\\x00\\x00\\x34\\x00\\x00\\x00\\x16gguf_type\\x00GGUF_TYPE_UINT8,GGUF_TYPE_INT8,GGUF_TYPE_UINT16,GGUF_TYPE_INT16,GGUF_TYPE_UINT32,GGUF_TYPE_INT32,GGUF_TYPE_FLOAT32,GGUF_TYPE_BOOL,GGUF_TYPE_STRING,GGUF_TYPE_ARRAY,GGUF_TYPE_COUNT'),\n _typenames = (b'\\x00\\x00\\x00\\xDB__darwin_blkcnt_t',b'\\x00\\x00\\x00\\x22__darwin_blksize_t',b'\\x00\\x00\\x00\\x11__darwin_clock_t',b'\\x00\\x00\\x00\\x22__darwin_ct_rune_t',b'\\x00\\x00\\x00\\x22__darwin_dev_t',b'\\x00\\x00\\x03\\xBF__darwin_fsblkcnt_t',b'\\x00\\x00\\x03\\xBF__darwin_fsfilcnt_t',b'\\x00\\x00\\x03\\xBF__darwin_gid_t',b'\\x00\\x00\\x03\\xBF__darwin_id_t',b'\\x00\\x00\\x04\\x4A__darwin_ino64_t',b'\\x00\\x00\\x04\\x4A__darwin_ino_t',b'\\x00\\x00\\x04\\x20__darwin_intptr_t',b'\\x00\\x00\\x03\\xBF__darwin_mach_port_name_t',b'\\x00\\x00\\x03\\xBF__darwin_mach_port_t',b'\\x00\\x00\\x03\\xF7__darwin_mbstate_t',b'\\x00\\x00\\x00\\x6C__darwin_mode_t',b'\\x00\\x00\\x03\\xBF__darwin_natural_t',b'\\x00\\x00\\x00\\xDB__darwin_off_t',b'\\x00\\x00\\x00\\x22__darwin_pid_t',b'\\x00\\x00\\x03\\xEF__darwin_pthread_attr_t',b'\\x00\\x00\\x03\\xF0__darwin_pthread_cond_t',b'\\x00\\x00\\x03\\xF1__darwin_pthread_condattr_t',b'\\x00\\x00\\x00\\x11__darwin_pthread_key_t',b'\\x00\\x00\\x03\\xF2__darwin_pthread_mutex_t',b'\\x00\\x00\\x03\\xF3__darwin_pthread_mutexattr_t',b'\\x00\\x00\\x03\\xF4__darwin_pthread_once_t',b'\\x00\\x00\\x03\\xF5__darwin_pthread_rwlock_t',b'\\x00\\x00\\x03\\xF6__darwin_pthread_rwlockattr_t',b'\\x00\\x00\\x04\\x2D__darwin_pthread_t',b'\\x00\\x00\\x04\\x20__darwin_ptrdiff_t',b'\\x00\\x00\\x00\\x22__darwin_rune_t',b'\\x00\\x00\\x03\\xBF__darwin_sigset_t',b'\\x00\\x00\\x00\\x11__darwin_size_t',b'\\x00\\x00\\x03\\xBF__darwin_socklen_t',b'\\x00\\x00\\x04\\x20__darwin_ssize_t',b'\\x00\\x00\\x00\\x22__darwin_suseconds_t',b'\\x00\\x00\\x04\\x20__darwin_time_t',b'\\x00\\x00\\x03\\xBF__darwin_uid_t',b'\\x00\\x00\\x03\\xBF__darwin_useconds_t',b'\\x00\\x00\\x04\\x05__darwin_uuid_string_t',b'\\x00\\x00\\x04\\x44__darwin_uuid_t',b'\\x00\\x00\\x00\\x22__darwin_wchar_t',b'\\x00\\x00\\x00\\x22__darwin_wint_t',b'\\x00\\x00\\x03\\xB0__int16_t',b'\\x00\\x00\\x00\\x22__int32_t',b'\\x00\\x00\\x00\\xDB__int64_t',b'\\x00\\x00\\x03\\xB5__int8_t',b'\\x00\\x00\\x03\\xF7__mbstate_t',b'\\x00\\x00\\x00\\x6C__uint16_t',b'\\x00\\x00\\x03\\xBF__uint32_t',b'\\x00\\x00\\x04\\x4A__uint64_t',b'\\x00\\x00\\x03\\xBA__uint8_t',b'\\x00\\x00\\x03\\xF8block_q2_K',b'\\x00\\x00\\x03\\xF9block_q3_K',b'\\x00\\x00\\x03\\xFAblock_q4_K',b'\\x00\\x00\\x03\\xFBblock_q5_K',b'\\x00\\x00\\x03\\xFCblock_q6_K',b'\\x00\\x00\\x03\\xFDblock_q8_K',b'\\x00\\x00\\x01\\xEAggml_binary_op_f32_t',b'\\x00\\x00\\x02\\x02ggml_custom1_op_f32_t',b'\\x00\\x00\\x02\\x07ggml_custom1_op_t',b'\\x00\\x00\\x01\\xF0ggml_custom2_op_f32_t',b'\\x00\\x00\\x01\\xF6ggml_custom2_op_t',b'\\x00\\x00\\x01\\xC5ggml_custom3_op_f32_t',b'\\x00\\x00\\x01\\xCCggml_custom3_op_t',b'\\x00\\x00\\x00\\x6Cggml_fp16_t',b'\\x00\\x00\\x04\\x4Fggml_from_float_t',b'\\x00\\x00\\x04\\x52ggml_to_float_t',b'\\x00\\x00\\x04\\x18ggml_type_traits_t',b'\\x00\\x00\\x01\\xFDggml_unary_op_f32_t',b'\\x00\\x00\\x04\\x50ggml_vec_dot_t',b'\\x00\\x00\\x03\\xB0int16_t',b'\\x00\\x00\\x00\\x22int32_t',b'\\x00\\x00\\x00\\xDBint64_t',b'\\x00\\x00\\x03\\xB5int8_t',b'\\x00\\x00\\x03\\xB0int_fast16_t',b'\\x00\\x00\\x00\\x22int_fast32_t',b'\\x00\\x00\\x00\\xDBint_fast64_t',b'\\x00\\x00\\x03\\xB5int_fast8_t',b'\\x00\\x00\\x03\\xB0int_least16_t',b'\\x00\\x00\\x00\\x22int_least32_t',b'\\x00\\x00\\x00\\xDBint_least64_t',b'\\x00\\x00\\x03\\xB5int_least8_t',b'\\x00\\x00\\x04\\x20intmax_t',b'\\x00\\x00\\x04\\x20intptr_t',b'\\x00\\x00\\x04\\x1Dmax_align_t',b'\\x00\\x00\\x04\\x20ptrdiff_t',b'\\x00\\x00\\x00\\xDBregister_t',b'\\x00\\x00\\x00\\x11rsize_t',b'\\x00\\x00\\x00\\x11size_t',b'\\x00\\x00\\x04\\x4Asyscall_arg_t',b'\\x00\\x00\\x00\\x6Cu_int16_t',b'\\x00\\x00\\x03\\xBFu_int32_t',b'\\x00\\x00\\x04\\x4Au_int64_t',b'\\x00\\x00\\x03\\xBAu_int8_t',b'\\x00\\x00\\x00\\x6Cuint16_t',b'\\x00\\x00\\x03\\xBFuint32_t',b'\\x00\\x00\\x04\\x4Auint64_t',b'\\x00\\x00\\x03\\xBAuint8_t',b'\\x00\\x00\\x00\\x6Cuint_fast16_t',b'\\x00\\x00\\x03\\xBFuint_fast32_t',b'\\x00\\x00\\x04\\x4Auint_fast64_t',b'\\x00\\x00\\x03\\xBAuint_fast8_t',b'\\x00\\x00\\x00\\x6Cuint_least16_t',b'\\x00\\x00\\x03\\xBFuint_least32_t',b'\\x00\\x00\\x04\\x4Auint_least64_t',b'\\x00\\x00\\x03\\xBAuint_least8_t',b'\\x00\\x00\\x00\\x11uintmax_t',b'\\x00\\x00\\x00\\x11uintptr_t',b'\\x00\\x00\\x04\\x4Auser_addr_t',b'\\x00\\x00\\x00\\xDBuser_long_t',b'\\x00\\x00\\x00\\xDBuser_off_t',b'\\x00\\x00\\x04\\x4Auser_size_t',b'\\x00\\x00\\x00\\xDBuser_ssize_t',b'\\x00\\x00\\x00\\xDBuser_time_t',b'\\x00\\x00\\x04\\x4Auser_ulong_t',b'\\x00\\x00\\x00\\x22wchar_t'),\n)\n" }, { "path": "examples/python/ggml/ffi/__init__.pyi", "content": "# Phony stubs.\n\nclass CData:\n pass\n\nclass CType:\n pass" }, { "path": "examples/python/ggml/utils.py", "content": "\"\"\"\n Common helpers for working with ggml + numpy\n\"\"\"\nfrom ggml import ffi, lib\nfrom typing import Union, Optional\nimport numpy as np\n\ndef init(mem_size: int, mem_buffer: ffi.CData = ffi.NULL, no_alloc: bool = False) -> ffi.CData:\n \"\"\"\n Initialize a ggml context, which will be freed automatically when the pointer is garbage collected.\n \"\"\"\n params = ffi.new('struct ggml_init_params*')\n params.mem_size = mem_size\n params.mem_buffer = mem_buffer\n params.no_alloc = no_alloc\n return ffi.gc(lib.ggml_init(params[0]), lib.ggml_free)\n \nTensorLike = Union[ffi.CData, np.ndarray]\n\ndef copy(from_tensor: TensorLike, to_tensor: TensorLike, allow_requantize: bool = True):\n \"\"\"\n Copy the contents of one tensor to another, doing any necessary (de/re)quantization transparently.\n Works across numpy & ggml tensors, but they must have the same shape (and be contiguous).\n\n Parameters\n ----------\n from_tensor : TensorLike\n The tensor to copy from (a numpy array or possibly-quantized ggml tensor)\n to_tensor : TensorLike\n The tensor to copy to (a numpy array or possibly-quantized ggml tensor)\n allow_requantize : bool\n If False, will throw an error if requantization is required (i.e. both from_tensor\n and to_tensor are quantized with different quantization types)\n \"\"\"\n if id(from_tensor) == id(to_tensor):\n return\n \n __expect_same_layout(\"source\", from_tensor, \"destination\", to_tensor)\n __check_shape_consistent_with_type(from_tensor)\n __check_shape_consistent_with_type(to_tensor)\n\n from_type = __get_type(from_tensor)\n to_type = __get_type(to_tensor)\n\n if from_type == to_type:\n ffi.memmove(__get_data(to_tensor), __get_data(from_tensor), __get_nbytes(from_tensor))\n else:\n assert allow_requantize or not lib.ggml_is_quantized(from_type) or not lib.ggml_is_quantized(to_type), \\\n f\"Requantizing from {__type_name(from_type)} to {__type_name(to_type)} is disabled. Force with allow_requantize=True\"\n \n __set_floats(to_tensor, __get_floats(from_tensor))\n\ndef numpy(tensor: ffi.CData, allow_copy: Union[bool, np.ndarray] = False, allow_requantize=False) -> np.ndarray:\n \"\"\"\n Convert a ggml tensor to a numpy array.\n If the tensor isn't quantized, the returned numpy array will be a view over its data.\n \n If it is quantized (and allow_copy is True), the copy will involve dequantization and the returned array will\n be a copy of the original tensor (any changes to the numpy array won't then be reflected back to the tensor).\n\n Parameters\n ----------\n tensor : ffi.CData\n The tensor to convert to a numpy array\n allow_copy : bool or np.ndarray\n If False, will throw an error if the tensor is quantized (since dequantization requires extra memory).\n If True, will dequantize the tensor and return a copy of the data in a new float32 numpy array.\n If an np.ndarray, will copy the data into the given array (which must be the same shape as the tensor) when dequantization is needed\n allow_requantize : bool\n If allow_copy is a tensor with a different quantization type than the source tensor, will throw an error unless allow_requantize is True.\n \"\"\"\n shape = __get_shape(tensor)\n\n if lib.ggml_is_quantized(tensor.type):\n if allow_copy == False:\n raise ValueError(f\"{__describe(tensor)} is quantized, conversion to numpy requires a copy (pass allow_copy=True; changes to the numpy array won't affect the original).\")\n elif isinstance(allow_copy, np.ndarray):\n __expect_same_layout(\"source tensor\", tensor, \"dequantization output tensor\", allow_copy)\n destination = allow_copy\n else:\n destination = np.empty(shape, dtype=np.float32)\n\n copy(tensor, destination, allow_requantize=allow_requantize)\n return destination\n else:\n dtype = __type_to_dtype(tensor.type)\n if not dtype:\n raise NotImplementedError(f'Cannot convert {__describe(tensor)} to numpy')\n\n assert __is_contiguous(tensor), f\"Cannot convert {__describe(tensor)} to numpy (support contiguous tensors only)\"\n nbytes = lib.ggml_nelements(tensor) * lib.ggml_type_size(tensor.type)\n array = np.frombuffer(ffi.buffer(lib.ggml_get_data(tensor), nbytes), dtype=dtype)\n array.shape = shape\n return array\n\ndef __type_name(type: int) -> str:\n name = lib.ggml_type_name(type)\n return ffi.string(name).decode('utf-8') if name else None\n\n__k_quant_types = set([\n lib.GGML_TYPE_Q2_K,\n lib.GGML_TYPE_Q3_K,\n lib.GGML_TYPE_Q4_K,\n lib.GGML_TYPE_Q5_K,\n lib.GGML_TYPE_Q6_K,\n lib.GGML_TYPE_Q8_K,\n])\n\n__type_to_dtype_dict = {\n lib.GGML_TYPE_I8: np.int8,\n lib.GGML_TYPE_I16: np.int16,\n lib.GGML_TYPE_I32: np.int32,\n lib.GGML_TYPE_F16: np.float16,\n lib.GGML_TYPE_F32: np.float32,\n}\n\ndef __type_to_dtype(type: int) -> Optional[np.dtype]: return __type_to_dtype_dict.get(type)\ndef __dtype_to_type(dtype: np.dtype):\n if dtype == np.float32: return lib.GGML_TYPE_F32\n elif dtype == np.float16: return lib.GGML_TYPE_F16\n elif dtype == np.int32: return lib.GGML_TYPE_I32\n elif dtype == np.int16: return lib.GGML_TYPE_I16\n elif dtype == np.int8: return lib.GGML_TYPE_I8\n else: raise ValueError(f\"Unsupported dtype: {dtype}\")\n\ndef __describe(tensor: ffi.CType): return f'Tensor[{__type_name(__get_type(tensor))}, {__get_shape(tensor)}]'\ndef __get_type(tensor: TensorLike): return __dtype_to_type(tensor.dtype) if isinstance(tensor, np.ndarray) else tensor.type\ndef __get_shape(x: TensorLike): return x.shape if isinstance(x, np.ndarray) else tuple([x.ne[i] for i in range(x.n_dims)])\ndef __get_strides(x: TensorLike): return x.strides if isinstance(x, np.ndarray) else tuple([x.nb[i] for i in range(x.n_dims)])\ndef __get_data(x: TensorLike) -> ffi.CData: return ffi.from_buffer(x) if isinstance(x, np.ndarray) else lib.ggml_get_data(x)\ndef __get_nbytes(tensor: TensorLike): return tensor.nbytes if isinstance(tensor, np.ndarray) else lib.ggml_nbytes(tensor)\ndef __get_nelements(tensor: TensorLike): return tensor.size if isinstance(tensor, np.ndarray) else lib.ggml_nelements(tensor)\ndef __is_contiguous(tensor: TensorLike): return tensor.flags['C_CONTIGUOUS'] if isinstance(tensor, np.ndarray) else lib.ggml_is_contiguous(tensor)\n\ndef __get_floats(tensor: TensorLike) -> ffi.CData:\n data, type = __get_data(tensor), __get_type(tensor)\n if type == lib.GGML_TYPE_F32:\n return ffi.cast('float*', data)\n else:\n nelements = __get_nelements(tensor)\n floats = ffi.new('float[]', nelements)\n if type == lib.GGML_TYPE_F16:\n lib.ggml_fp16_to_fp32_row(ffi.cast('uint16_t*', data), floats, nelements)\n elif lib.ggml_is_quantized(type):\n qtype = lib.ggml_internal_get_type_traits(type)\n assert qtype.to_float, f\"Type {__type_name(type)} is not supported by ggml\"\n qtype.to_float(data, floats, nelements)\n else:\n raise NotImplementedError(f'Cannot read floats from {__describe(tensor)}')\n return floats\n\ndef __set_floats(tensor: TensorLike, f32_data: ffi.CData) -> None:\n data, type, nbytes = __get_data(tensor), __get_type(tensor), __get_nbytes(tensor)\n if type == lib.GGML_TYPE_F32:\n ffi.memmove(data, f32_data, nbytes)\n else:\n nelements = __get_nelements(tensor)\n if type == lib.GGML_TYPE_F16:\n lib.ggml_fp32_to_fp16_row(f32_data, ffi.cast('uint16_t*', data), nelements)\n elif lib.ggml_is_quantized(type):\n qtype = lib.ggml_internal_get_type_traits(type)\n assert qtype.from_float, f\"Type {__type_name(type)} is not supported by ggml\"\n qtype.from_float(f32_data, data, nelements)\n else:\n raise NotImplementedError(f'Cannot write floats to {__describe(tensor)}')\n\ndef __expect_same_layout(name1: str, tensor1: TensorLike, name2: str, tensor2: TensorLike):\n shape1, shape2 = __get_shape(tensor1), __get_shape(tensor2)\n assert shape1 == shape2, f\"Shape mismatch: {name1} has {shape1} but {name2} has {shape2}\"\n assert __is_contiguous(tensor1) and __is_contiguous(tensor2), f\"Only contiguous tensors are supported (got {name1} with strides {__get_strides(tensor1)} and {name2} with strides {__get_strides(tensor2)})\"\n\ndef __check_shape_consistent_with_type(tensor: TensorLike):\n type = __get_type(tensor)\n if not lib.ggml_is_quantized(type):\n return\n shape = __get_shape(tensor)\n\n block_size = lib.ggml_blck_size(type)\n assert not (block_size == 0 and type in __k_quant_types), f\"Can't quantize, native library was not compiled with USE_K_QUANTS!\"\n assert block_size > 0, f\"Invalid block size {block_size} for type {__type_name(type)}\"\n for i, d in enumerate(shape):\n assert d % block_size == 0, f\"Dimension {i} of {__describe(tensor)} is not divisible by {block_size}, required for quantization.\"\n" }, { "path": "examples/python/regenerate.py", "content": "# Generates bindings for the ggml library.\n#\n# cffi requires prior C preprocessing of the headers, and it uses pycparser which chokes on a couple of things\n# so we help it a bit (e.g. replace sizeof expressions with their value, remove exotic syntax found in Darwin headers).\nimport os, sys, re, subprocess\nimport cffi\nfrom stubs import generate_stubs\n\nAPI = os.environ.get('API', 'api.h')\nCC = os.environ.get('CC') or 'gcc'\nC_INCLUDE_DIR = os.environ.get('C_INCLUDE_DIR', '../../../llama.cpp')\nCPPFLAGS = [\n \"-I\", C_INCLUDE_DIR,\n '-D__fp16=uint16_t', # pycparser doesn't support __fp16\n '-D__attribute__(x)=',\n '-D_Static_assert(x, m)=',\n] + [x for x in os.environ.get('CPPFLAGS', '').split(' ') if x != '']\n\ntry: header = subprocess.run([CC, \"-E\", *CPPFLAGS, API], capture_output=True, text=True, check=True).stdout\nexcept subprocess.CalledProcessError as e: print(f'{e.stderr}\\n{e}', file=sys.stderr); raise\n\nheader = '\\n'.join([l for l in header.split('\\n') if '__darwin_va_list' not in l]) # pycparser hates this\n\n# Replace constant size expressions w/ their value (compile & run a mini exe for each, because why not).\n# First, extract anyting *inside* square brackets and anything that looks like a sizeof call.\nfor expr in set(re.findall(f'(?<=\\\\[)[^\\\\]]+(?=])|sizeof\\\\s*\\\\([^()]+\\\\)', header)):\n if re.match(r'^(\\d+|\\s*)$', expr): continue # skip constants and empty bracket contents\n subprocess.run([CC, \"-o\", \"eval_size_expr\", *CPPFLAGS, \"-x\", \"c\", \"-\"], text=True, check=True,\n input=f'''#include \n #include \"{API}\"\n int main() {{ printf(\"%lu\", (size_t)({expr})); }}''')\n size = subprocess.run([\"./eval_size_expr\"], capture_output=True, text=True, check=True).stdout\n print(f'Computed constexpr {expr} = {size}')\n header = header.replace(expr, size)\n\nffibuilder = cffi.FFI()\nffibuilder.cdef(header)\nffibuilder.set_source(f'ggml.cffi', None) # we're not compiling a native extension, as this quickly gets hairy\nffibuilder.compile(verbose=True)\n\nwith open(\"ggml/__init__.pyi\", \"wt\") as f:\n f.write(generate_stubs(header))" }, { "path": "examples/python/stubs.py", "content": "\"\"\"\n This generates .pyi stubs for the cffi Python bindings generated by regenerate.py\n\"\"\"\nimport sys, re, itertools\nsys.path.extend(['.', '..']) # for pycparser\n\nfrom pycparser import c_ast, parse_file, CParser\nimport pycparser.plyparser\nfrom pycparser.c_ast import PtrDecl, TypeDecl, FuncDecl, EllipsisParam, IdentifierType, Struct, Enum, Typedef\nfrom typing import Tuple\n\n__c_type_to_python_type = {\n 'void': 'None', '_Bool': 'bool',\n 'char': 'int', 'short': 'int', 'int': 'int', 'long': 'int',\n 'ptrdiff_t': 'int', 'size_t': 'int',\n 'int8_t': 'int', 'uint8_t': 'int',\n 'int16_t': 'int', 'uint16_t': 'int',\n 'int32_t': 'int', 'uint32_t': 'int',\n 'int64_t': 'int', 'uint64_t': 'int',\n 'float': 'float', 'double': 'float',\n 'ggml_fp16_t': 'np.float16',\n}\n\ndef format_type(t: TypeDecl):\n if isinstance(t, PtrDecl) or isinstance(t, Struct):\n return 'ffi.CData'\n if isinstance(t, Enum):\n return 'int'\n if isinstance(t, TypeDecl):\n return format_type(t.type)\n if isinstance(t, IdentifierType):\n assert len(t.names) == 1, f'Expected a single name, got {t.names}'\n return __c_type_to_python_type.get(t.names[0]) or 'ffi.CData'\n return t.name\n\nclass PythonStubFuncDeclVisitor(c_ast.NodeVisitor):\n def __init__(self):\n self.sigs = {}\n self.sources = {}\n\n def get_source_snippet_lines(self, coord: pycparser.plyparser.Coord) -> Tuple[list[str], list[str]]:\n if coord.file not in self.sources:\n with open(coord.file, 'rt') as f:\n self.sources[coord.file] = f.readlines()\n source_lines = self.sources[coord.file]\n ncomment_lines = len(list(itertools.takewhile(lambda i: re.search(r'^\\s*(//|/\\*)', source_lines[i]), range(coord.line - 2, -1, -1))))\n comment_lines = [l.strip() for l in source_lines[coord.line - 1 - ncomment_lines:coord.line - 1]]\n decl_lines = []\n for line in source_lines[coord.line - 1:]:\n decl_lines.append(line.rstrip())\n if (';' in line) or ('{' in line): break\n return (comment_lines, decl_lines)\n\n def visit_Enum(self, node: Enum):\n if node.values is not None:\n for e in node.values.enumerators:\n self.sigs[e.name] = f' @property\\n def {e.name}(self) -> int: ...'\n\n def visit_Typedef(self, node: Typedef):\n pass\n\n def visit_FuncDecl(self, node: FuncDecl):\n ret_type = node.type\n is_ptr = False\n while isinstance(ret_type, PtrDecl):\n ret_type = ret_type.type\n is_ptr = True\n\n fun_name = ret_type.declname\n if fun_name.startswith('__'):\n return\n\n args = []\n argnames = []\n def gen_name(stem):\n i = 1\n while True:\n new_name = stem if i == 1 else f'{stem}{i}'\n if new_name not in argnames: return new_name\n i += 1\n\n for a in node.args.params:\n if isinstance(a, EllipsisParam):\n arg_name = gen_name('args')\n argnames.append(arg_name)\n args.append('*' + gen_name('args'))\n elif format_type(a.type) == 'None':\n continue\n else:\n arg_name = a.name or gen_name('arg')\n argnames.append(arg_name)\n args.append(f'{arg_name}: {format_type(a.type)}')\n\n ret = format_type(ret_type if not is_ptr else node.type)\n\n comment_lines, decl_lines = self.get_source_snippet_lines(node.coord)\n\n lines = [f' def {fun_name}({\", \".join(args)}) -> {ret}:']\n if len(comment_lines) == 0 and len(decl_lines) == 1:\n lines += [f' \"\"\"{decl_lines[0]}\"\"\"']\n else:\n lines += [' \"\"\"']\n lines += [f' {c.lstrip(\"/* \")}' for c in comment_lines]\n if len(comment_lines) > 0:\n lines += ['']\n lines += [f' {d}' for d in decl_lines]\n lines += [' \"\"\"']\n lines += [' ...']\n self.sigs[fun_name] = '\\n'.join(lines)\n\ndef generate_stubs(header: str):\n \"\"\"\n Generates a .pyi Python stub file for the GGML API using C header files.\n \"\"\"\n\n v = PythonStubFuncDeclVisitor()\n v.visit(CParser().parse(header, \"\"))\n\n keys = list(v.sigs.keys())\n keys.sort()\n\n return '\\n'.join([\n '# auto-generated file',\n 'import ggml.ffi as ffi',\n 'import numpy as np',\n 'class lib:',\n *[v.sigs[k] for k in keys]\n ])\n" }, { "path": "examples/python/test_tensor.py", "content": "import pytest\nfrom pytest import raises\n\nfrom ggml import lib, ffi\nfrom ggml.utils import init, copy, numpy\nimport numpy as np\nimport numpy.testing as npt\n\n@pytest.fixture()\ndef ctx():\n print(\"setup\")\n yield init(mem_size=10*1024*1024)\n print(\"teardown\")\n\nclass TestNumPy:\n \n # Single element\n\n def test_set_get_single_i32(self, ctx):\n i = lib.ggml_new_i32(ctx, 42)\n assert lib.ggml_get_i32_1d(i, 0) == 42\n assert numpy(i) == np.array([42], dtype=np.int32)\n\n def test_set_get_single_f32(self, ctx):\n i = lib.ggml_new_f32(ctx, 4.2)\n \n epsilon = 0.000001 # Not sure why so large a difference??\n pytest.approx(lib.ggml_get_f32_1d(i, 0), 4.2, epsilon)\n pytest.approx(numpy(i), np.array([4.2], dtype=np.float32), epsilon)\n\n def _test_copy_np_to_ggml(self, a: np.ndarray, t: ffi.CData):\n a2 = a.copy() # Clone original\n copy(a, t)\n npt.assert_array_equal(numpy(t), a2)\n\n # I32\n\n def test_copy_np_to_ggml_1d_i32(self, ctx):\n t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_I32, 10)\n a = np.arange(10, dtype=np.int32)\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_2d_i32(self, ctx):\n t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_I32, 2, 3)\n a = np.arange(2 * 3, dtype=np.int32).reshape((2, 3))\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_3d_i32(self, ctx):\n t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_I32, 2, 3, 4)\n a = np.arange(2 * 3 * 4, dtype=np.int32).reshape((2, 3, 4))\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_4d_i32(self, ctx):\n t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_I32, 2, 3, 4, 5)\n a = np.arange(2 * 3 * 4 * 5, dtype=np.int32).reshape((2, 3, 4, 5))\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_4d_n_i32(self, ctx):\n dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash\n pdims = ffi.new('int64_t[]', len(dims))\n for i, d in enumerate(dims): pdims[i] = d\n t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_I32, len(dims), pdims)\n a = np.arange(np.prod(dims), dtype=np.int32).reshape(tuple(pdims))\n self._test_copy_np_to_ggml(a, t)\n\n # F32\n\n def test_copy_np_to_ggml_1d_f32(self, ctx):\n t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)\n a = np.arange(10, dtype=np.float32)\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_2d_f32(self, ctx):\n t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, 2, 3)\n a = np.arange(2 * 3, dtype=np.float32).reshape((2, 3))\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_3d_f32(self, ctx):\n t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F32, 2, 3, 4)\n a = np.arange(2 * 3 * 4, dtype=np.float32).reshape((2, 3, 4))\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_4d_f32(self, ctx):\n t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F32, 2, 3, 4, 5)\n a = np.arange(2 * 3 * 4 * 5, dtype=np.float32).reshape((2, 3, 4, 5))\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_4d_n_f32(self, ctx):\n dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash\n pdims = ffi.new('int64_t[]', len(dims))\n for i, d in enumerate(dims): pdims[i] = d\n t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_F32, len(dims), pdims)\n a = np.arange(np.prod(dims), dtype=np.float32).reshape(tuple(pdims))\n self._test_copy_np_to_ggml(a, t)\n\n # F16\n\n def test_copy_np_to_ggml_1d_f16(self, ctx):\n t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F16, 10)\n a = np.arange(10, dtype=np.float16)\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_2d_f16(self, ctx):\n t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F16, 2, 3)\n a = np.arange(2 * 3, dtype=np.float16).reshape((2, 3))\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_3d_f16(self, ctx):\n t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F16, 2, 3, 4)\n a = np.arange(2 * 3 * 4, dtype=np.float16).reshape((2, 3, 4))\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_4d_f16(self, ctx):\n t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F16, 2, 3, 4, 5)\n a = np.arange(2 * 3 * 4 * 5, dtype=np.float16).reshape((2, 3, 4, 5))\n self._test_copy_np_to_ggml(a, t)\n\n def test_copy_np_to_ggml_4d_n_f16(self, ctx):\n dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash\n pdims = ffi.new('int64_t[]', len(dims))\n for i, d in enumerate(dims): pdims[i] = d\n t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_F16, len(dims), pdims)\n a = np.arange(np.prod(dims), dtype=np.float16).reshape(tuple(pdims))\n self._test_copy_np_to_ggml(a, t)\n\n # Mismatching shapes\n\n def test_copy_mismatching_shapes_1d(self, ctx):\n t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)\n a = np.arange(10, dtype=np.float32)\n copy(a, t) # OK\n \n a = a.reshape((5, 2))\n with raises(AssertionError): copy(a, t)\n with raises(AssertionError): copy(t, a)\n \n def test_copy_mismatching_shapes_2d(self, ctx):\n t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, 2, 3)\n a = np.arange(6, dtype=np.float32)\n copy(a.reshape((2, 3)), t) # OK\n \n a = a.reshape((3, 2))\n with raises(AssertionError): copy(a, t)\n with raises(AssertionError): copy(t, a)\n\n def test_copy_mismatching_shapes_3d(self, ctx):\n t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F32, 2, 3, 4)\n a = np.arange(24, dtype=np.float32)\n copy(a.reshape((2, 3, 4)), t) # OK\n \n a = a.reshape((2, 4, 3))\n with raises(AssertionError): copy(a, t)\n with raises(AssertionError): copy(t, a)\n\n def test_copy_mismatching_shapes_4d(self, ctx):\n t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F32, 2, 3, 4, 5)\n a = np.arange(24*5, dtype=np.float32)\n copy(a.reshape((2, 3, 4, 5)), t) # OK\n \n a = a.reshape((2, 3, 5, 4))\n with raises(AssertionError): copy(a, t)\n with raises(AssertionError): copy(t, a)\n\n def test_copy_f16_to_f32(self, ctx):\n t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 1)\n a = np.array([123.45], dtype=np.float16)\n copy(a, t)\n np.testing.assert_allclose(lib.ggml_get_f32_1d(t, 0), 123.45, rtol=1e-3)\n\n def test_copy_f32_to_f16(self, ctx):\n t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F16, 1)\n a = np.array([123.45], dtype=np.float32)\n copy(a, t)\n np.testing.assert_allclose(lib.ggml_get_f32_1d(t, 0), 123.45, rtol=1e-3)\n\n def test_copy_f16_to_Q5_K(self, ctx):\n n = 256\n t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)\n a = np.arange(n, dtype=np.float16)\n copy(a, t)\n np.testing.assert_allclose(a, numpy(t, allow_copy=True), rtol=0.05)\n\n def test_copy_Q5_K_to_f16(self, ctx):\n n = 256\n t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)\n copy(np.arange(n, dtype=np.float32), t)\n a = np.arange(n, dtype=np.float16)\n copy(t, a)\n np.testing.assert_allclose(a, numpy(t, allow_copy=True), rtol=0.05)\n\n def test_copy_i16_f32_mismatching_types(self, ctx):\n t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 1)\n a = np.arange(1, dtype=np.int16)\n with raises(NotImplementedError): copy(a, t)\n with raises(NotImplementedError): copy(t, a)\n\nclass TestTensorCopy:\n\n def test_copy_self(self, ctx):\n t = lib.ggml_new_i32(ctx, 42)\n copy(t, t)\n assert lib.ggml_get_i32_1d(t, 0) == 42\n\n def test_copy_1d(self, ctx):\n t1 = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)\n t2 = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)\n a = np.arange(10, dtype=np.float32)\n copy(a, t1)\n copy(t1, t2)\n assert np.allclose(a, numpy(t2))\n assert np.allclose(numpy(t1), numpy(t2))\n\nclass TestGraph:\n\n def test_add(self, ctx):\n n = 256\n ta = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)\n tb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)\n tsum = lib.ggml_add(ctx, ta, tb)\n assert tsum.type == lib.GGML_TYPE_F32\n\n gf = ffi.new('struct ggml_cgraph*')\n lib.ggml_build_forward_expand(gf, tsum)\n\n a = np.arange(0, n, dtype=np.float32)\n b = np.arange(n, 0, -1, dtype=np.float32)\n copy(a, ta)\n copy(b, tb)\n\n lib.ggml_graph_compute_with_ctx(ctx, gf, 1)\n\n assert np.allclose(numpy(tsum, allow_copy=True), a + b)\n\nclass TestQuantization:\n\n def test_quantized_add(self, ctx):\n n = 256\n ta = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)\n tb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)\n tsum = lib.ggml_add(ctx, ta, tb)\n assert tsum.type == lib.GGML_TYPE_Q5_K\n\n gf = ffi.new('struct ggml_cgraph*')\n lib.ggml_build_forward_expand(gf, tsum)\n\n a = np.arange(0, n, dtype=np.float32)\n b = np.arange(n, 0, -1, dtype=np.float32)\n copy(a, ta)\n copy(b, tb)\n\n lib.ggml_graph_compute_with_ctx(ctx, gf, 1)\n\n unquantized_sum = a + b\n sum = numpy(tsum, allow_copy=True)\n\n diff = np.linalg.norm(unquantized_sum - sum, np.inf)\n assert diff > 4\n assert diff < 5\n" }, { "path": "examples/sam/CMakeLists.txt", "content": "#\n# sam\n\nset(TEST_TARGET sam)\nadd_executable(${TEST_TARGET} sam.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common)\n\n#\n# sam-quantize\n\n#set(TEST_TARGET sam-quantize)\n#add_executable(${TEST_TARGET} quantize.cpp)\n#target_link_libraries(${TEST_TARGET} PRIVATE ggml common)\n" }, { "path": "examples/sam/README.md", "content": "# SAM.cpp\n\nInference of Meta's [Segment Anything Model](https://github.com/facebookresearch/segment-anything/) in pure C/C++\n\n## Description\n\nThe example currently supports only the [ViT-B SAM model checkpoint](https://huggingface.co/facebook/sam-vit-base).\n\n## Next steps\n\n- [X] Reduce memory usage by utilizing the new ggml-alloc\n- [X] Remove redundant graph nodes\n- [ ] Make inference faster\n- [X] Fix the difference in output masks compared to the PyTorch implementation\n- [X] Filter masks based on stability score\n- [ ] Add support for user input\n- [ ] Support F16 for heavy F32 ops\n- [ ] Test quantization\n- [X] Support bigger model checkpoints\n- [ ] GPU support\n\n## Quick start\nSetup Python and build examples according to main README.\n\n```bash\n# Download PTH model\nwget -P examples/sam/ https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth\n\n# Convert PTH model to ggml\npython examples/sam/convert-pth-to-ggml.py examples/sam/sam_vit_b_01ec64.pth examples/sam/ 1\n\n# run inference\n./bin/sam -t 16 -i ../examples/sam/example.jpg -m ../examples/sam/ggml-model-f16.bin\n```\n\n## Downloading and converting the model checkpoints\n\nYou can download a [model checkpoint](https://github.com/facebookresearch/segment-anything/tree/main#model-checkpoints) and convert it to `ggml` format using the script `convert-pth-to-ggml.py`:\n\n## Example output on M2 Ultra\n```\n $ ▶ make -j sam && time ./bin/sam -t 8 -i img.jpg\n[ 28%] Built target common\n[ 71%] Built target ggml\n[100%] Built target sam\nmain: seed = 1693224265\nmain: loaded image 'img.jpg' (680 x 453)\nsam_image_preprocess: scale = 0.664062\nmain: preprocessed image (1024 x 1024)\nsam_model_load: loading model from 'models/sam-vit-b/ggml-model-f16.bin' - please wait ...\nsam_model_load: n_enc_state = 768\nsam_model_load: n_enc_layer = 12\nsam_model_load: n_enc_head = 12\nsam_model_load: n_enc_out_chans = 256\nsam_model_load: n_pt_embd = 4\nsam_model_load: ftype = 1\nsam_model_load: qntvr = 0\noperator(): ggml ctx size = 202.32 MB\nsam_model_load: ...................................... done\nsam_model_load: model size = 185.05 MB / num tensors = 304\nembd_img\ndims: 64 64 256 1 f32\nFirst & Last 10 elements:\n-0.05117 -0.06408 -0.07154 -0.06991 -0.07212 -0.07690 -0.07508 -0.07281 -0.07383 -0.06779\n0.01589 0.01775 0.02250 0.01675 0.01766 0.01661 0.01811 0.02051 0.02103 0.03382\nsum: 12736.272313\n\nSkipping mask 0 with iou 0.705935 below threshold 0.880000\nSkipping mask 1 with iou 0.762136 below threshold 0.880000\nMask 2: iou = 0.947081, stability_score = 0.955437, bbox (371, 436), (144, 168)\n\n\nmain: load time = 51.28 ms\nmain: total time = 2047.49 ms\n\nreal\t0m2.068s\nuser\t0m16.343s\nsys\t0m0.214s\n```\n\nInput point is (414.375, 162.796875) (currently hardcoded)\n\nInput image:\n\n![llamas](https://user-images.githubusercontent.com/8558655/261301565-37b7bf4b-bf91-40cf-8ec1-1532316e1612.jpg)\n\nOutput mask (mask_out_2.png in build folder):\n\n![mask_glasses](https://user-images.githubusercontent.com/8558655/263706800-47eeea30-1457-4c87-938b-8f11536c5aa7.png)\n\n## References\n\n- [ggml](https://github.com/ggerganov/ggml)\n- [SAM](https://segment-anything.com/)\n- [SAM demo](https://segment-anything.com/demo)\n" }, { "path": "examples/sam/convert-pth-to-ggml.py", "content": "# Convert a SAM model checkpoint to a ggml compatible file\n#\n\nimport sys\nimport torch\nimport struct\nimport numpy as np\n\nif len(sys.argv) < 3:\n print(\"Usage: convert-pth-to-ggml.py file-model dir-output [ftype]\\n\")\n print(\" ftype == 0 -> float32\")\n print(\" ftype == 1 -> float16\")\n sys.exit(1)\n\n# output in the same directory as the model\nfname_model = sys.argv[1]\ndir_out = sys.argv[2]\nfname_out = dir_out + \"/ggml-model.bin\"\n\n# possible data types\n# ftype == 0 -> float32\n# ftype == 1 -> float16\n#\n# map from ftype to string\nftype_str = [\"f32\", \"f16\"]\n\nftype = 1\nif len(sys.argv) > 3:\n ftype = int(sys.argv[3])\n\nif ftype < 0 or ftype > 1:\n print(\"Invalid ftype: \" + str(ftype))\n sys.exit(1)\n\nfname_out = fname_out.replace(\".bin\", \"-\" + ftype_str[ftype] + \".bin\")\n\n# Default params are set to sam_vit_b checkpoint\nn_enc_state = 768\nn_enc_layers = 12\nn_enc_heads = 12\nn_enc_out_chans = 256\nn_pt_embd = 4\n\nmodel = torch.load(fname_model, map_location=\"cpu\")\nfor k, v in model.items():\n print(k, v.shape)\n if k == \"image_encoder.blocks.0.norm1.weight\":\n n_enc_state = v.shape[0]\n\nif n_enc_state == 1024: # sam_vit_l\n n_enc_layers = 24\n n_enc_heads = 16\nelif n_enc_state == 1280: # sam_vit_h\n n_enc_layers = 32\n n_enc_heads = 16\n\nhparams = {\n \"n_enc_state\": n_enc_state,\n \"n_enc_layers\": n_enc_layers,\n \"n_enc_heads\": n_enc_heads,\n \"n_enc_out_chans\": n_enc_out_chans,\n \"n_pt_embd\": n_pt_embd,\n}\n\nprint(hparams)\n\nfor k, v in model.items():\n print(k, v.shape)\n\n#exit()\n#code.interact(local=locals())\n\nfout = open(fname_out, \"wb\")\n\nfout.write(struct.pack(\"i\", 0x67676d6c)) # magic: ggml in hex\nfout.write(struct.pack(\"i\", hparams[\"n_enc_state\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_enc_layers\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_enc_heads\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_enc_out_chans\"]))\nfout.write(struct.pack(\"i\", hparams[\"n_pt_embd\"]))\nfout.write(struct.pack(\"i\", ftype))\n\nfor k, v in model.items():\n name = k\n shape = v.shape\n\n if name[:19] == \"prompt_encoder.mask\":\n continue\n\n print(\"Processing variable: \" + name + \" with shape: \", shape, \" and type: \", v.dtype)\n\n #data = tf.train.load_variable(dir_model, name).squeeze()\n #data = v.numpy().squeeze()\n data = v.numpy()\n n_dims = len(data.shape)\n\n # for efficiency - transpose some matrices\n # \"model/h.*/attn/c_attn/w\"\n # \"model/h.*/attn/c_proj/w\"\n # \"model/h.*/mlp/c_fc/w\"\n # \"model/h.*/mlp/c_proj/w\"\n #if name[-14:] == \"/attn/c_attn/w\" or \\\n # name[-14:] == \"/attn/c_proj/w\" or \\\n # name[-11:] == \"/mlp/c_fc/w\" or \\\n # name[-13:] == \"/mlp/c_proj/w\":\n # print(\" Transposing\")\n # data = data.transpose()\n\n dshape = data.shape\n\n # default type is fp16\n ftype_cur = 1\n if ftype == 0 or n_dims == 1 or \\\n name == \"image_encoder.pos_embed\" or \\\n name.startswith(\"prompt_encoder\") or \\\n name.startswith(\"mask_decoder.iou_token\") or \\\n name.startswith(\"mask_decoder.mask_tokens\"):\n print(\" Converting to float32\")\n data = data.astype(np.float32)\n ftype_cur = 0\n else:\n print(\" Converting to float16\")\n data = data.astype(np.float16)\n\n # reshape the 1D bias into a 4D tensor so we can use ggml_repeat\n # keep it in F32 since the data is small\n if name == \"image_encoder.patch_embed.proj.bias\":\n data = data.reshape(1, data.shape[0], 1, 1)\n n_dims = len(data.shape)\n dshape = data.shape\n\n print(\" New shape: \", dshape)\n\n # header\n str = name.encode('utf-8')\n fout.write(struct.pack(\"iii\", n_dims, len(str), ftype_cur))\n for i in range(n_dims):\n fout.write(struct.pack(\"i\", dshape[n_dims - 1 - i]))\n fout.write(str)\n\n # data\n data.tofile(fout)\n\nfout.close()\n\nprint(\"Done. Output file: \" + fname_out)\nprint(\"\")\n" }, { "path": "examples/sam/sam.cpp", "content": "#define _USE_MATH_DEFINES // for M_PI\n#define _CRT_SECURE_NO_DEPRECATE // Disables ridiculous \"unsafe\" warnigns on Windows\n\n#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n#define STB_IMAGE_IMPLEMENTATION\n#include \"stb_image.h\"\n#define STB_IMAGE_WRITE_IMPLEMENTATION\n#include \"stb_image_write.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\n// default hparams (ViT-B SAM)\nstruct sam_hparams {\n int32_t n_enc_state = 768;\n int32_t n_enc_layer = 12;\n int32_t n_enc_head = 12;\n int32_t n_enc_out_chans = 256;\n int32_t n_pt_embd = 4;\n int32_t n_dec_heads = 8;\n int32_t ftype = 1;\n float mask_threshold = 0.f;\n float iou_threshold = 0.88f;\n float stability_score_threshold = 0.95f;\n float stability_score_offset = 1.0f;\n float eps = 1e-6f;\n float eps_decoder_transformer = 1e-5f;\n\n int32_t n_enc_head_dim() const { return n_enc_state / n_enc_head; }\n int32_t n_img_size() const { return 1024; }\n int32_t n_window_size() const { return 14; }\n int32_t n_patch_size() const { return 16; }\n int32_t n_img_embd() const { return n_img_size() / n_patch_size(); }\n\n std::vector global_attn_indices() const {\n switch (n_enc_state) {\n case 768: return { 2, 5, 8, 11 };\n case 1024: return { 5, 11, 17, 23 };\n case 1280: return { 7, 15, 23, 31 };\n default:\n {\n fprintf(stderr, \"%s: unsupported n_enc_state = %d\\n\", __func__, n_enc_state);\n } break;\n };\n\n return {};\n }\n\n bool is_global_attn(int32_t layer) const {\n const auto indices = global_attn_indices();\n\n for (const auto & idx : indices) {\n if (layer == idx) {\n return true;\n }\n }\n\n return false;\n }\n};\n\nstruct sam_layer_enc {\n struct ggml_tensor * norm1_w;\n struct ggml_tensor * norm1_b;\n\n struct ggml_tensor * rel_pos_w;\n struct ggml_tensor * rel_pos_h;\n\n struct ggml_tensor * qkv_w;\n struct ggml_tensor * qkv_b;\n\n struct ggml_tensor * proj_w;\n struct ggml_tensor * proj_b;\n\n struct ggml_tensor * norm2_w;\n struct ggml_tensor * norm2_b;\n\n struct ggml_tensor * mlp_lin1_w;\n struct ggml_tensor * mlp_lin1_b;\n\n struct ggml_tensor * mlp_lin2_w;\n struct ggml_tensor * mlp_lin2_b;\n};\n\nstruct sam_encoder_image {\n struct ggml_tensor * pe;\n\n struct ggml_tensor * proj_w;\n struct ggml_tensor * proj_b;\n\n struct ggml_tensor * neck_conv_0;\n struct ggml_tensor * neck_norm_0_w;\n struct ggml_tensor * neck_norm_0_b;\n struct ggml_tensor * neck_conv_1;\n struct ggml_tensor * neck_norm_1_w;\n struct ggml_tensor * neck_norm_1_b;\n\n std::vector layers;\n};\n\nstruct sam_encoder_prompt {\n struct ggml_tensor * pe;\n\n struct ggml_tensor * not_a_pt_embd_w;\n std::vector pt_embd;\n\n struct ggml_tensor * no_mask_embd_w;\n //std::vector mask_down_w;\n //std::vector mask_down_b;\n};\n\nstruct sam_layer_dec_transformer_attn {\n // q_proj\n struct ggml_tensor * q_w;\n struct ggml_tensor * q_b;\n\n // k_proj\n struct ggml_tensor * k_w;\n struct ggml_tensor * k_b;\n\n // v_proj\n struct ggml_tensor * v_w;\n struct ggml_tensor * v_b;\n\n // out_proj\n struct ggml_tensor * out_w;\n struct ggml_tensor * out_b;\n};\n\nstruct sam_layer_dec_transformer {\n sam_layer_dec_transformer_attn self_attn;\n\n // norm1\n struct ggml_tensor * norm1_w;\n struct ggml_tensor * norm1_b;\n\n sam_layer_dec_transformer_attn cross_attn_token_to_img;\n\n // norm2\n struct ggml_tensor * norm2_w;\n struct ggml_tensor * norm2_b;\n\n // mlp.lin1\n struct ggml_tensor * mlp_lin1_w;\n struct ggml_tensor * mlp_lin1_b;\n\n // mlp.lin2\n struct ggml_tensor * mlp_lin2_w;\n struct ggml_tensor * mlp_lin2_b;\n\n // norm3\n struct ggml_tensor * norm3_w;\n struct ggml_tensor * norm3_b;\n\n // norm4\n struct ggml_tensor * norm4_w;\n struct ggml_tensor * norm4_b;\n\n sam_layer_dec_transformer_attn cross_attn_img_to_token;\n};\n\nstruct sam_layer_dec_output_hypernet_mlps {\n // mlps_*.layers.0\n struct ggml_tensor * w_0;\n struct ggml_tensor * b_0;\n\n // mlps_*.layers.1\n struct ggml_tensor * w_1;\n struct ggml_tensor * b_1;\n\n // mlps_*.layers.2\n struct ggml_tensor * w_2;\n struct ggml_tensor * b_2;\n};\n\nstruct sam_decoder_mask {\n std::vector transformer_layers;\n\n // trasnformer.final_attn_token_to_image\n sam_layer_dec_transformer_attn transformer_final_attn_token_to_img;\n\n // transformer.norm_final\n struct ggml_tensor * transformer_norm_final_w;\n struct ggml_tensor * transformer_norm_final_b;\n\n // output_upscaling.0\n struct ggml_tensor * output_upscaling_0_w;\n struct ggml_tensor * output_upscaling_0_b;\n\n // output_upscaling.1\n struct ggml_tensor * output_upscaling_1_w;\n struct ggml_tensor * output_upscaling_1_b;\n\n // output_upscaling.3\n struct ggml_tensor * output_upscaling_3_w;\n struct ggml_tensor * output_upscaling_3_b;\n\n // output_hypernetworks_mlps\n std::vector output_hypernet_mlps;\n\n // iou_prediction_head.0\n struct ggml_tensor * iou_prediction_head_0_w;\n struct ggml_tensor * iou_prediction_head_0_b;\n\n // iou_prediction_head.1\n struct ggml_tensor * iou_prediction_head_1_w;\n struct ggml_tensor * iou_prediction_head_1_b;\n\n // iou_prediction_head.2\n struct ggml_tensor * iou_prediction_head_2_w;\n struct ggml_tensor * iou_prediction_head_2_b;\n\n // iou_token.weight\n struct ggml_tensor * iou_token_w;\n\n // mask_tokens.weight\n struct ggml_tensor * mask_tokens_w;\n};\n\n\nstruct sam_state {\n struct ggml_tensor * embd_img;\n\n struct ggml_tensor * low_res_masks;\n struct ggml_tensor * iou_predictions;\n\n //struct ggml_tensor * tmp_save = {};\n\n struct ggml_context * ctx;\n\n // buffer for `ggml_graph_plan.work_data`\n std::vector work_buffer;\n // buffers to evaluate the model\n std::vector buf_compute_img_enc;\n\n std::vector buf_compute_fast;\n\n ggml_gallocr_t allocr = {};\n};\n\n// void save_tensor(sam_state& state, struct ggml_tensor * t, struct ggml_cgraph * gf) {\n// if (!state.tmp_save) {\n// state.tmp_save = ggml_new_tensor(state.ctx, t->type, t->n_dims, t->ne);\n// }\n// struct ggml_tensor * tmp0 = ggml_cpy(state.ctx, t, state.tmp_save);\n// ggml_build_forward_expand(gf, tmp0);\n// }\n\nstruct sam_model {\n sam_hparams hparams;\n\n sam_encoder_image enc_img;\n sam_encoder_prompt enc_prompt;\n sam_decoder_mask dec;\n\n //\n struct ggml_context * ctx;\n std::map tensors;\n};\n\nstruct sam_point {\n float x;\n float y;\n};\n\nstruct sam_box {\n float x1;\n float y1;\n float x2;\n float y2;\n};\n\n// RGB uint8 image\nstruct sam_image_u8 {\n int nx;\n int ny;\n\n std::vector data;\n};\n\n// RGB float32 image\n// Memory layout: RGBRGBRGB...\nstruct sam_image_f32 {\n int nx;\n int ny;\n\n std::vector data;\n};\n\nenum sam_prompt_type {\n SAM_PROMPT_TYPE_POINT = 0,\n SAM_PROMPT_TYPE_BOX = 1,\n};\n\nstruct sam_prompt {\n sam_prompt_type prompt_type = SAM_PROMPT_TYPE_POINT;\n sam_point pt = { 414.375f, 162.796875f, };\n sam_box box = { 368.0f, 144.0f, 441.0f, 173.0f };\n};\n\nstruct sam_params {\n int32_t seed = -1; // RNG seed\n int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());\n\n std::string model = \"models/sam-vit-b/ggml-model-f16.bin\"; // model path\n std::string fname_inp = \"img.jpg\";\n std::string fname_out = \"img.out\";\n float mask_threshold = 0.f;\n float iou_threshold = 0.88f;\n float stability_score_threshold = 0.95f;\n float stability_score_offset = 1.0f;\n float eps = 1e-6f;\n float eps_decoder_transformer = 1e-5f;\n\n sam_prompt prompt;\n bool multimask_output = true;\n};\n\nvoid print_t_f32(const char* title, struct ggml_tensor * t, int n = 10) {\n printf(\"%s\\n\", title);\n float * data = (float *)t->data;\n printf(\"dims: % \" PRId64 \" % \" PRId64 \" % \" PRId64 \" % \" PRId64 \" f32\\n\", t->ne[0], t->ne[1], t->ne[2], t->ne[3]);\n printf(\"First & Last %d elements:\\n\", n);\n for (int i = 0; i < std::min((int) (t->ne[0]*t->ne[1]), n); i++) {\n printf(\"%.5f \", data[i]);\n if (i != 0 && i % t->ne[0] == 0) {\n printf(\"\\n\");\n }\n }\n printf(\"\\n\");\n for (int i = 0; i < std::min((int) (t->ne[0]*t->ne[1]), n); i++) {\n printf(\"%.5f \", data[ggml_nelements(t) - n + i]);\n if ((ggml_nelements(t) - n + i) % t->ne[0] == 0) {\n printf(\"\\n\");\n }\n }\n printf(\"\\n\");\n double sum = 0.0;\n for (int i = 0; i < ggml_nelements(t); i++) {\n sum += data[i];\n }\n printf(\"sum: %f\\n\\n\", sum);\n}\n\nstatic void ggml_disconnect_node_from_graph(ggml_tensor * t) {\n t->op = GGML_OP_NONE;\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n t->src[i] = NULL;\n }\n}\n\nstatic void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) {\n struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, nullptr);\n\n if (plan.work_size > 0) {\n buf.resize(plan.work_size);\n plan.work_data = buf.data();\n }\n\n ggml_graph_compute(graph, &plan);\n}\n\nstatic void ggml_sam_sin(struct ggml_tensor * dst , const struct ggml_tensor * src, int ith, int nth, void * userdata) {\n GGML_ASSERT(userdata == NULL);\n GGML_ASSERT(ggml_are_same_shape(dst, src));\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_is_contiguous(src));\n\n const float * src_data = ggml_get_data_f32(src);\n float * dst_data = ggml_get_data_f32(dst);\n\n const int ne = (int)ggml_nelements(dst);\n const int dr = (ne + nth - 1) / nth;\n const int ie0 = dr * ith;\n const int ie1 = std::min(ie0 + dr, ne);\n\n for (int i = ie0; i < ie1; ++i) {\n dst_data[i] = sinf(src_data[i]);\n }\n}\n\nstatic void ggml_sam_cos(struct ggml_tensor * dst , const struct ggml_tensor * src, int ith, int nth, void * userdata) {\n GGML_ASSERT(userdata == NULL);\n GGML_ASSERT(ggml_are_same_shape(dst, src));\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_is_contiguous(src));\n\n const float * src_data = ggml_get_data_f32(src);\n float * dst_data = ggml_get_data_f32(dst);\n\n const int ne = (int)ggml_nelements(dst);\n const int dr = (ne + nth - 1) / nth;\n const int ie0 = dr * ith;\n const int ie1 = std::min(ie0 + dr, ne);\n\n for (int i = ie0; i < ie1; ++i) {\n dst_data[i] = cosf(src_data[i]);\n }\n}\n\nbool sam_image_load_from_file(const std::string & fname, sam_image_u8 & img) {\n int nx, ny, nc;\n auto data = stbi_load(fname.c_str(), &nx, &ny, &nc, 3);\n if (!data) {\n fprintf(stderr, \"%s: failed to load '%s'\\n\", __func__, fname.c_str());\n return false;\n }\n\n img.nx = nx;\n img.ny = ny;\n img.data.resize(nx * ny * 3);\n memcpy(img.data.data(), data, nx * ny * 3);\n\n stbi_image_free(data);\n\n return true;\n}\n\n// ref: https://github.com/facebookresearch/segment-anything/blob/efeab7296ab579d4a261e554eca80faf6b33924a/segment_anything/modeling/sam.py#L164\n// resize largest dimension to 1024\n// normalize: x = (x - mean) / std\n// mean = [123.675, 116.28, 103.53]\n// std = [58.395, 57.12, 57.375]\n// TODO: why are these hardcoded !?\n// pad to 1024x1024\n// TODO: for some reason, this is not numerically identical to pytorch's interpolation\nbool sam_image_preprocess(const sam_image_u8 & img, sam_image_f32 & res) {\n const int nx = img.nx;\n const int ny = img.ny;\n\n const int nx2 = 1024;\n const int ny2 = 1024;\n\n res.nx = nx2;\n res.ny = ny2;\n res.data.resize(3*nx2*ny2);\n\n const float scale = std::max(nx, ny) / 1024.0f;\n\n fprintf(stderr, \"%s: scale = %f\\n\", __func__, scale);\n\n const int nx3 = int(nx/scale + 0.5f);\n const int ny3 = int(ny/scale + 0.5f);\n\n const float m3[3] = { 123.675f, 116.280f, 103.530f };\n const float s3[3] = { 58.395f, 57.120f, 57.375f };\n\n for (int y = 0; y < ny3; y++) {\n for (int x = 0; x < nx3; x++) {\n for (int c = 0; c < 3; c++) {\n // linear interpolation\n const float sx = (x + 0.5f)*scale - 0.5f;\n const float sy = (y + 0.5f)*scale - 0.5f;\n\n const int x0 = std::max(0, (int) std::floor(sx));\n const int y0 = std::max(0, (int) std::floor(sy));\n\n const int x1 = std::min(x0 + 1, nx - 1);\n const int y1 = std::min(y0 + 1, ny - 1);\n\n const float dx = sx - x0;\n const float dy = sy - y0;\n\n const int j00 = 3*(y0*nx + x0) + c;\n const int j01 = 3*(y0*nx + x1) + c;\n const int j10 = 3*(y1*nx + x0) + c;\n const int j11 = 3*(y1*nx + x1) + c;\n\n const float v00 = img.data[j00];\n const float v01 = img.data[j01];\n const float v10 = img.data[j10];\n const float v11 = img.data[j11];\n\n const float v0 = v00*(1.0f - dx) + v01*dx;\n const float v1 = v10*(1.0f - dx) + v11*dx;\n\n const float v = v0*(1.0f - dy) + v1*dy;\n\n const uint8_t v2 = std::min(std::max(std::round(v), 0.0f), 255.0f);\n\n const int i = 3*(y*nx3 + x) + c;\n\n res.data[i] = (float(v2) - m3[c]) / s3[c];\n }\n }\n }\n\n return true;\n}\n\n// load the model's weights from a file\nbool sam_model_load(const sam_params & params, sam_model & model) {\n fprintf(stderr, \"%s: loading model from '%s' - please wait ...\\n\", __func__, params.model.c_str());\n\n auto fin = std::ifstream(params.model, std::ios::binary);\n if (!fin) {\n fprintf(stderr, \"%s: failed to open '%s'\\n\", __func__, params.model.c_str());\n return false;\n }\n\n // verify magic\n {\n uint32_t magic;\n fin.read((char *) &magic, sizeof(magic));\n if (magic != 0x67676d6c) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, params.model.c_str());\n return false;\n }\n }\n\n // load hparams\n {\n // Override defaults with user choices\n model.hparams.mask_threshold = params.mask_threshold;\n model.hparams.iou_threshold = params.iou_threshold;\n model.hparams.stability_score_threshold = params.stability_score_threshold;\n model.hparams.stability_score_offset = params.stability_score_offset;\n model.hparams.eps = params.eps;\n model.hparams.eps_decoder_transformer = params.eps_decoder_transformer;\n\n auto & hparams = model.hparams;\n\n fin.read((char *) &hparams.n_enc_state, sizeof(hparams.n_enc_state));\n fin.read((char *) &hparams.n_enc_layer, sizeof(hparams.n_enc_layer));\n fin.read((char *) &hparams.n_enc_head, sizeof(hparams.n_enc_head));\n fin.read((char *) &hparams.n_enc_out_chans, sizeof(hparams.n_enc_out_chans));\n fin.read((char *) &hparams.n_pt_embd, sizeof(hparams.n_pt_embd));\n fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));\n\n const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;\n\n printf(\"%s: n_enc_state = %d\\n\", __func__, hparams.n_enc_state);\n printf(\"%s: n_enc_layer = %d\\n\", __func__, hparams.n_enc_layer);\n printf(\"%s: n_enc_head = %d\\n\", __func__, hparams.n_enc_head);\n printf(\"%s: n_enc_out_chans = %d\\n\", __func__, hparams.n_enc_out_chans);\n printf(\"%s: n_pt_embd = %d\\n\", __func__, hparams.n_pt_embd);\n printf(\"%s: ftype = %d\\n\", __func__, hparams.ftype);\n printf(\"%s: qntvr = %d\\n\", __func__, qntvr);\n\n hparams.ftype %= GGML_QNT_VERSION_FACTOR;\n\n }\n\n // for the big tensors, we have the option to store the data in 16-bit floats or quantized\n // in order to save memory and also to speed up the computation\n ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));\n if (wtype == GGML_TYPE_COUNT) {\n fprintf(stderr, \"%s: invalid model file '%s' (bad ftype value %d)\\n\",\n __func__, params.model.c_str(), model.hparams.ftype);\n return false;\n }\n\n auto & ctx = model.ctx;\n\n const size_t ctx_size = [&]() {\n size_t ctx_size = 0;\n\n const auto & hparams = model.hparams;\n\n const int32_t n_enc_state = hparams.n_enc_state;\n const int32_t n_enc_layer = hparams.n_enc_layer;\n const int32_t n_enc_head_dim = hparams.n_enc_head_dim();\n const int32_t n_enc_out_chans = hparams.n_enc_out_chans;\n const int32_t n_pt_embd = hparams.n_pt_embd;\n\n const int32_t n_enc_layer_local = hparams.global_attn_indices().size();\n const int32_t n_enc_layer_global = n_enc_layer - n_enc_layer_local;\n\n const int32_t n_img_embd = hparams.n_img_embd();\n const int32_t n_window_size = hparams.n_window_size();\n const int32_t n_patch_size = hparams.n_patch_size();\n\n // image encoder\n {\n ctx_size += n_enc_state*n_img_embd*n_img_embd*ggml_type_size(GGML_TYPE_F32);\n\n ctx_size += n_enc_state*3*n_patch_size*n_patch_size*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_enc_state*ggml_type_size(GGML_TYPE_F32);\n\n ctx_size += n_enc_state*n_enc_out_chans*1*1*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_enc_out_chans*n_enc_out_chans*3*3*ggml_type_size(GGML_TYPE_F16);\n\n ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32);\n ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32);\n\n ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32);\n ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32);\n }\n\n // image encoder layers\n {\n ctx_size += n_enc_layer*n_enc_state*ggml_type_size(GGML_TYPE_F32);\n ctx_size += n_enc_layer*n_enc_state*ggml_type_size(GGML_TYPE_F32);\n\n ctx_size += n_enc_layer_global*n_enc_head_dim*(2*n_img_embd - 1)*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_enc_layer_global*n_enc_head_dim*(2*n_img_embd - 1)*ggml_type_size(GGML_TYPE_F16);\n\n ctx_size += n_enc_layer_local*n_enc_head_dim*(2*n_window_size - 1)*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_enc_layer_local*n_enc_head_dim*(2*n_window_size - 1)*ggml_type_size(GGML_TYPE_F16);\n\n ctx_size += n_enc_layer*3*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_enc_layer*3*n_enc_state* ggml_type_size(GGML_TYPE_F32);\n\n ctx_size += n_enc_layer*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_enc_layer*n_enc_state* ggml_type_size(GGML_TYPE_F32);\n\n ctx_size += n_enc_layer*n_enc_state*ggml_type_size(GGML_TYPE_F32);\n ctx_size += n_enc_layer*n_enc_state*ggml_type_size(GGML_TYPE_F32);\n\n ctx_size += n_enc_layer*4*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_enc_layer*4*n_enc_state* ggml_type_size(GGML_TYPE_F32);\n\n ctx_size += n_enc_layer*4*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_enc_layer*4*n_enc_state* ggml_type_size(GGML_TYPE_F32);\n }\n\n ctx_size += (8 + 14*n_enc_layer)*ggml_tensor_overhead();\n\n // prompt encoder\n {\n ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F16); // 2*(n_enc_out_chans/2)\n\n ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32);\n ctx_size += n_pt_embd*n_enc_out_chans*ggml_type_size(GGML_TYPE_F32);\n }\n\n ctx_size += (2 + n_pt_embd)*ggml_tensor_overhead();\n\n // mask decoder\n {\n //transformer\n {\n const int tfm_layers_count = 2;\n const int qkv_count = 3;\n const int norm_count = 4;\n const int n_hypernet_mpls_count = 4;\n\n // self_attn\n ctx_size += tfm_layers_count*qkv_count*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16);\n ctx_size += tfm_layers_count*qkv_count*n_enc_state* ggml_type_size(GGML_TYPE_F32);\n ctx_size += tfm_layers_count*n_enc_state* ggml_type_size(GGML_TYPE_F32);\n\n // all norms\n ctx_size += tfm_layers_count*norm_count*n_enc_state*ggml_type_size(GGML_TYPE_F32);\n ctx_size += tfm_layers_count*norm_count*n_enc_state*ggml_type_size(GGML_TYPE_F32);\n\n // cross_attn_token_to_img\n ctx_size += tfm_layers_count*qkv_count*n_enc_state*(n_enc_state/2)*ggml_type_size(GGML_TYPE_F16);\n ctx_size += tfm_layers_count*qkv_count*(n_enc_state/2)* ggml_type_size(GGML_TYPE_F32);\n ctx_size += tfm_layers_count*n_enc_state* ggml_type_size(GGML_TYPE_F32);\n\n // mlp\n ctx_size += tfm_layers_count*8*n_enc_out_chans*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16);\n ctx_size += tfm_layers_count*8*n_enc_out_chans* ggml_type_size(GGML_TYPE_F32);\n ctx_size += tfm_layers_count*n_enc_out_chans*8*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16);\n ctx_size += tfm_layers_count*n_enc_out_chans* ggml_type_size(GGML_TYPE_F32);\n\n // cross_attn_img_to_token\n ctx_size += tfm_layers_count*qkv_count*n_enc_state*(n_enc_state/2)*ggml_type_size(GGML_TYPE_F16);\n ctx_size += tfm_layers_count*qkv_count*(n_enc_state/2)* ggml_type_size(GGML_TYPE_F32);\n ctx_size += tfm_layers_count*n_enc_state* ggml_type_size(GGML_TYPE_F32);\n\n // transformer_final_attn_token_to_img\n ctx_size += qkv_count*n_enc_state*(n_enc_state/2)*ggml_type_size(GGML_TYPE_F16);\n ctx_size += qkv_count*(n_enc_state/2)* ggml_type_size(GGML_TYPE_F32);\n ctx_size += n_enc_state* ggml_type_size(GGML_TYPE_F32);\n\n // transformer_norm_final\n ctx_size += norm_count*n_enc_state*ggml_type_size(GGML_TYPE_F32);\n ctx_size += norm_count*n_enc_state*ggml_type_size(GGML_TYPE_F32);\n\n // output_upscaling\n ctx_size += n_enc_out_chans*n_img_embd*2*2*ggml_type_size(GGML_TYPE_F16);\n ctx_size += 3*n_img_embd* ggml_type_size(GGML_TYPE_F32);\n ctx_size += n_enc_out_chans*n_img_embd*(n_img_embd/2)*2*2*ggml_type_size(GGML_TYPE_F16);\n ctx_size += (n_img_embd/2)* ggml_type_size(GGML_TYPE_F32);\n\n // output_hypernetworks_mlps\n ctx_size += n_hypernet_mpls_count*2*n_enc_out_chans*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_hypernet_mpls_count*2*n_enc_out_chans* ggml_type_size(GGML_TYPE_F32);\n ctx_size += n_hypernet_mpls_count*n_enc_out_chans*(n_img_embd/2)*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_hypernet_mpls_count*(n_img_embd/2)* ggml_type_size(GGML_TYPE_F32);\n\n // iou_prediction_head\n ctx_size += 2*n_enc_out_chans*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16);\n ctx_size += 2*n_enc_out_chans* ggml_type_size(GGML_TYPE_F32);\n ctx_size += n_pt_embd*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16);\n ctx_size += n_pt_embd* ggml_type_size(GGML_TYPE_F32);\n\n // iou_token_w\n ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32);\n\n // mask_tokens_w\n ctx_size += n_pt_embd*n_enc_out_chans*ggml_type_size(GGML_TYPE_F32);\n }\n }\n fprintf(stderr, \"%s: ggml ctx size = %6.2f MB\\n\", __func__, ctx_size/(1024.0*1024.0));\n\n return ctx_size;\n }();\n\n // create the ggml context\n {\n struct ggml_init_params params = {\n /*.mem_size =*/ ctx_size,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ false,\n };\n\n ctx = ggml_init(params);\n if (!ctx) {\n fprintf(stderr, \"%s: ggml_init() failed\\n\", __func__);\n return false;\n }\n }\n\n // prepare memory for the weights\n {\n const auto & hparams = model.hparams;\n\n const int32_t n_enc_state = hparams.n_enc_state;\n const int32_t n_enc_layer = hparams.n_enc_layer;\n const int32_t n_enc_head_dim = hparams.n_enc_head_dim();\n const int32_t n_enc_out_chans = hparams.n_enc_out_chans;\n const int32_t n_pt_embd = hparams.n_pt_embd;\n\n const int32_t n_img_embd = hparams.n_img_embd();\n const int32_t n_window_size = hparams.n_window_size();\n const int32_t n_patch_size = hparams.n_patch_size();\n\n model.enc_img.layers.resize(n_enc_layer);\n\n // image encoder\n {\n auto & enc = model.enc_img;\n\n enc.pe = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_enc_state, n_img_embd, n_img_embd, 1);\n\n enc.proj_w = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, n_patch_size, n_patch_size, 3, n_enc_state);\n enc.proj_b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 1, n_enc_state);\n\n enc.neck_conv_0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 1, 1, n_enc_state, n_enc_out_chans);\n enc.neck_conv_1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 3, 3, n_enc_out_chans, n_enc_out_chans);\n\n enc.neck_norm_0_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n enc.neck_norm_0_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n enc.neck_norm_1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n enc.neck_norm_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n model.tensors[\"image_encoder.pos_embed\"] = enc.pe;\n\n model.tensors[\"image_encoder.patch_embed.proj.weight\"] = enc.proj_w;\n model.tensors[\"image_encoder.patch_embed.proj.bias\"] = enc.proj_b;\n\n model.tensors[\"image_encoder.neck.0.weight\"] = enc.neck_conv_0;\n model.tensors[\"image_encoder.neck.2.weight\"] = enc.neck_conv_1;\n\n model.tensors[\"image_encoder.neck.1.weight\"] = enc.neck_norm_0_w;\n model.tensors[\"image_encoder.neck.1.bias\"] = enc.neck_norm_0_b;\n\n model.tensors[\"image_encoder.neck.3.weight\"] = enc.neck_norm_1_w;\n model.tensors[\"image_encoder.neck.3.bias\"] = enc.neck_norm_1_b;\n\n for (int i = 0; i < n_enc_layer; ++i) {\n auto & layer = enc.layers[i];\n\n layer.norm1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state);\n layer.norm1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state);\n\n if (hparams.is_global_attn(i)) {\n layer.rel_pos_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_head_dim, 2*n_img_embd - 1);\n layer.rel_pos_h = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_head_dim, 2*n_img_embd - 1);\n } else {\n layer.rel_pos_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_head_dim, 2*n_window_size - 1);\n layer.rel_pos_h = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_head_dim, 2*n_window_size - 1);\n }\n\n layer.qkv_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_state, 3*n_enc_state);\n layer.qkv_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_enc_state);\n\n layer.proj_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_state, n_enc_state);\n layer.proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state);\n\n layer.norm2_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state);\n layer.norm2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state);\n\n layer.mlp_lin1_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_state, 4*n_enc_state);\n layer.mlp_lin1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_enc_state);\n\n layer.mlp_lin2_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, 4*n_enc_state, n_enc_state);\n layer.mlp_lin2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state);\n\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".norm1.weight\"] = layer.norm1_w;\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".norm1.bias\"] = layer.norm1_b;\n\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".attn.rel_pos_w\"] = layer.rel_pos_w;\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".attn.rel_pos_h\"] = layer.rel_pos_h;\n\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".attn.qkv.weight\"] = layer.qkv_w;\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".attn.qkv.bias\"] = layer.qkv_b;\n\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".attn.proj.weight\"] = layer.proj_w;\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".attn.proj.bias\"] = layer.proj_b;\n\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".norm2.weight\"] = layer.norm2_w;\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".norm2.bias\"] = layer.norm2_b;\n\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".mlp.lin1.weight\"] = layer.mlp_lin1_w;\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".mlp.lin1.bias\"] = layer.mlp_lin1_b;\n\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".mlp.lin2.weight\"] = layer.mlp_lin2_w;\n model.tensors[\"image_encoder.blocks.\" + std::to_string(i) + \".mlp.lin2.bias\"] = layer.mlp_lin2_b;\n }\n }\n\n // prompt encoder\n {\n auto & enc = model.enc_prompt;\n\n enc.pe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_enc_out_chans/2, 2);\n\n enc.not_a_pt_embd_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n enc.no_mask_embd_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n model.tensors[\"prompt_encoder.pe_layer.positional_encoding_gaussian_matrix\"] = enc.pe;\n model.tensors[\"prompt_encoder.not_a_point_embed.weight\"] = enc.not_a_pt_embd_w;\n model.tensors[\"prompt_encoder.no_mask_embed.weight\"] = enc.no_mask_embd_w;\n\n enc.pt_embd.resize(n_pt_embd);\n for (int i = 0; i < n_pt_embd; i++) {\n enc.pt_embd[i] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n model.tensors[\"prompt_encoder.point_embeddings.\" + std::to_string(i) + \".weight\"] = enc.pt_embd[i];\n }\n }\n\n // mask decoder\n {\n auto & dec = model.dec;\n auto & tfm_layers = dec.transformer_layers;\n\n const int tfm_layers_count = 2;\n tfm_layers.resize(tfm_layers_count);\n for (int i = 0; i < tfm_layers_count; ++i) {\n auto& l = tfm_layers[i];\n l.self_attn.q_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans);\n l.self_attn.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n l.self_attn.k_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans);\n l.self_attn.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n l.self_attn.v_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans);\n l.self_attn.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n l.self_attn.out_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans);\n l.self_attn.out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n l.norm1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n l.norm1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n l.cross_attn_token_to_img.q_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2);\n l.cross_attn_token_to_img.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2);\n l.cross_attn_token_to_img.k_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2);\n l.cross_attn_token_to_img.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2);\n l.cross_attn_token_to_img.v_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2);\n l.cross_attn_token_to_img.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2);\n l.cross_attn_token_to_img.out_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans/2, n_enc_out_chans);\n l.cross_attn_token_to_img.out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n l.norm2_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n l.norm2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n l.mlp_lin1_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, 8*n_enc_out_chans);\n l.mlp_lin1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 8*n_enc_out_chans);\n l.mlp_lin2_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, 8*n_enc_out_chans, n_enc_out_chans);\n l.mlp_lin2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n l.norm3_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n l.norm3_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n l.norm4_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n l.norm4_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n l.cross_attn_img_to_token.q_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2);\n l.cross_attn_img_to_token.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2);\n l.cross_attn_img_to_token.k_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2);\n l.cross_attn_img_to_token.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2);\n l.cross_attn_img_to_token.v_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2);\n l.cross_attn_img_to_token.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2);\n l.cross_attn_img_to_token.out_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans/2, n_enc_out_chans);\n l.cross_attn_img_to_token.out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n const auto prefix = \"mask_decoder.transformer.layers.\" + std::to_string(i) + \".\";\n model.tensors[prefix + \"self_attn.q_proj.weight\"] = l.self_attn.q_w;\n model.tensors[prefix + \"self_attn.q_proj.bias\"] = l.self_attn.q_b;\n model.tensors[prefix + \"self_attn.k_proj.weight\"] = l.self_attn.k_w;\n model.tensors[prefix + \"self_attn.k_proj.bias\"] = l.self_attn.k_b;\n model.tensors[prefix + \"self_attn.v_proj.weight\"] = l.self_attn.v_w;\n model.tensors[prefix + \"self_attn.v_proj.bias\"] = l.self_attn.v_b;\n model.tensors[prefix + \"self_attn.out_proj.weight\"] = l.self_attn.out_w;\n model.tensors[prefix + \"self_attn.out_proj.bias\"] = l.self_attn.out_b;\n\n model.tensors[prefix + \"norm1.weight\"] = l.norm1_w;\n model.tensors[prefix + \"norm1.bias\"] = l.norm1_b;\n\n model.tensors[prefix + \"cross_attn_token_to_image.q_proj.weight\"] = l.cross_attn_token_to_img.q_w;\n model.tensors[prefix + \"cross_attn_token_to_image.q_proj.bias\"] = l.cross_attn_token_to_img.q_b;\n model.tensors[prefix + \"cross_attn_token_to_image.k_proj.weight\"] = l.cross_attn_token_to_img.k_w;\n model.tensors[prefix + \"cross_attn_token_to_image.k_proj.bias\"] = l.cross_attn_token_to_img.k_b;\n model.tensors[prefix + \"cross_attn_token_to_image.v_proj.weight\"] = l.cross_attn_token_to_img.v_w;\n model.tensors[prefix + \"cross_attn_token_to_image.v_proj.bias\"] = l.cross_attn_token_to_img.v_b;\n model.tensors[prefix + \"cross_attn_token_to_image.out_proj.weight\"] = l.cross_attn_token_to_img.out_w;\n model.tensors[prefix + \"cross_attn_token_to_image.out_proj.bias\"] = l.cross_attn_token_to_img.out_b;\n\n model.tensors[prefix + \"norm2.weight\"] = l.norm2_w;\n model.tensors[prefix + \"norm2.bias\"] = l.norm2_b;\n\n model.tensors[prefix + \"mlp.lin1.weight\"] = l.mlp_lin1_w;\n model.tensors[prefix + \"mlp.lin1.bias\"] = l.mlp_lin1_b;\n model.tensors[prefix + \"mlp.lin2.weight\"] = l.mlp_lin2_w;\n model.tensors[prefix + \"mlp.lin2.bias\"] = l.mlp_lin2_b;\n\n model.tensors[prefix + \"norm3.weight\"] = l.norm3_w;\n model.tensors[prefix + \"norm3.bias\"] = l.norm3_b;\n model.tensors[prefix + \"norm4.weight\"] = l.norm4_w;\n model.tensors[prefix + \"norm4.bias\"] = l.norm4_b;\n\n model.tensors[prefix + \"cross_attn_image_to_token.q_proj.weight\"] = l.cross_attn_img_to_token.q_w;\n model.tensors[prefix + \"cross_attn_image_to_token.q_proj.bias\"] = l.cross_attn_img_to_token.q_b;\n model.tensors[prefix + \"cross_attn_image_to_token.k_proj.weight\"] = l.cross_attn_img_to_token.k_w;\n model.tensors[prefix + \"cross_attn_image_to_token.k_proj.bias\"] = l.cross_attn_img_to_token.k_b;\n model.tensors[prefix + \"cross_attn_image_to_token.v_proj.weight\"] = l.cross_attn_img_to_token.v_w;\n model.tensors[prefix + \"cross_attn_image_to_token.v_proj.bias\"] = l.cross_attn_img_to_token.v_b;\n model.tensors[prefix + \"cross_attn_image_to_token.out_proj.weight\"] = l.cross_attn_img_to_token.out_w;\n model.tensors[prefix + \"cross_attn_image_to_token.out_proj.bias\"] = l.cross_attn_img_to_token.out_b;\n }\n\n dec.transformer_final_attn_token_to_img.q_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2);\n dec.transformer_final_attn_token_to_img.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2);\n dec.transformer_final_attn_token_to_img.k_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2);\n dec.transformer_final_attn_token_to_img.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2);\n dec.transformer_final_attn_token_to_img.v_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2);\n dec.transformer_final_attn_token_to_img.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2);\n dec.transformer_final_attn_token_to_img.out_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans/2, n_enc_out_chans);\n dec.transformer_final_attn_token_to_img.out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n model.tensors[\"mask_decoder.transformer.final_attn_token_to_image.q_proj.weight\"] = dec.transformer_final_attn_token_to_img.q_w;\n model.tensors[\"mask_decoder.transformer.final_attn_token_to_image.q_proj.bias\"] = dec.transformer_final_attn_token_to_img.q_b;\n model.tensors[\"mask_decoder.transformer.final_attn_token_to_image.k_proj.weight\"] = dec.transformer_final_attn_token_to_img.k_w;\n model.tensors[\"mask_decoder.transformer.final_attn_token_to_image.k_proj.bias\"] = dec.transformer_final_attn_token_to_img.k_b;\n model.tensors[\"mask_decoder.transformer.final_attn_token_to_image.v_proj.weight\"] = dec.transformer_final_attn_token_to_img.v_w;\n model.tensors[\"mask_decoder.transformer.final_attn_token_to_image.v_proj.bias\"] = dec.transformer_final_attn_token_to_img.v_b;\n model.tensors[\"mask_decoder.transformer.final_attn_token_to_image.out_proj.weight\"] = dec.transformer_final_attn_token_to_img.out_w;\n model.tensors[\"mask_decoder.transformer.final_attn_token_to_image.out_proj.bias\"] = dec.transformer_final_attn_token_to_img.out_b;\n\n dec.transformer_norm_final_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n dec.transformer_norm_final_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n\n model.tensors[\"mask_decoder.transformer.norm_final_attn.weight\"] = dec.transformer_norm_final_w;\n model.tensors[\"mask_decoder.transformer.norm_final_attn.bias\"] = dec.transformer_norm_final_b;\n\n dec.output_upscaling_0_w = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 2, 2, n_img_embd, n_enc_out_chans);\n dec.output_upscaling_0_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd);\n dec.output_upscaling_1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd);\n dec.output_upscaling_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd);\n dec.output_upscaling_3_w = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 2, 2, n_img_embd/2, n_img_embd);\n dec.output_upscaling_3_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd/2);\n\n model.tensors[\"mask_decoder.output_upscaling.0.weight\"] = dec.output_upscaling_0_w;\n model.tensors[\"mask_decoder.output_upscaling.0.bias\"] = dec.output_upscaling_0_b;\n model.tensors[\"mask_decoder.output_upscaling.1.weight\"] = dec.output_upscaling_1_w;\n model.tensors[\"mask_decoder.output_upscaling.1.bias\"] = dec.output_upscaling_1_b;\n model.tensors[\"mask_decoder.output_upscaling.3.weight\"] = dec.output_upscaling_3_w;\n model.tensors[\"mask_decoder.output_upscaling.3.bias\"] = dec.output_upscaling_3_b;\n\n const int n_hypernet_mpls_count = 4;\n dec.output_hypernet_mlps.resize(n_hypernet_mpls_count);\n for (int i = 0; i < n_hypernet_mpls_count; ++i) {\n auto& mlp = dec.output_hypernet_mlps[i];\n\n mlp.w_0 = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans);\n mlp.b_0 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n mlp.w_1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans);\n mlp.b_1 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n mlp.w_2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_img_embd/2);\n mlp.b_2 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd/2);\n\n const auto prefix = \"mask_decoder.output_hypernetworks_mlps.\" + std::to_string(i) + \".\";\n model.tensors[prefix + \"layers.0.weight\"] = mlp.w_0;\n model.tensors[prefix + \"layers.0.bias\"] = mlp.b_0;\n model.tensors[prefix + \"layers.1.weight\"] = mlp.w_1;\n model.tensors[prefix + \"layers.1.bias\"] = mlp.b_1;\n model.tensors[prefix + \"layers.2.weight\"] = mlp.w_2;\n model.tensors[prefix + \"layers.2.bias\"] = mlp.b_2;\n }\n\n dec.iou_prediction_head_0_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans);\n dec.iou_prediction_head_0_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n dec.iou_prediction_head_1_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans);\n dec.iou_prediction_head_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans);\n dec.iou_prediction_head_2_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_pt_embd);\n dec.iou_prediction_head_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_pt_embd);\n\n dec.iou_token_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_enc_out_chans, 1);\n dec.mask_tokens_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_enc_out_chans, n_pt_embd);\n\n model.tensors[\"mask_decoder.iou_prediction_head.layers.0.weight\"] = dec.iou_prediction_head_0_w;\n model.tensors[\"mask_decoder.iou_prediction_head.layers.0.bias\"] = dec.iou_prediction_head_0_b;\n model.tensors[\"mask_decoder.iou_prediction_head.layers.1.weight\"] = dec.iou_prediction_head_1_w;\n model.tensors[\"mask_decoder.iou_prediction_head.layers.1.bias\"] = dec.iou_prediction_head_1_b;\n model.tensors[\"mask_decoder.iou_prediction_head.layers.2.weight\"] = dec.iou_prediction_head_2_w;\n model.tensors[\"mask_decoder.iou_prediction_head.layers.2.bias\"] = dec.iou_prediction_head_2_b;\n\n model.tensors[\"mask_decoder.iou_token.weight\"] = dec.iou_token_w;\n model.tensors[\"mask_decoder.mask_tokens.weight\"] = dec.mask_tokens_w;\n }\n }\n\n // load weights\n {\n int n_tensors = 0;\n size_t total_size = 0;\n\n fprintf(stderr, \"%s: \", __func__);\n\n while (true) {\n int32_t n_dims;\n int32_t length;\n int32_t ftype;\n\n fin.read(reinterpret_cast(&n_dims), sizeof(n_dims));\n fin.read(reinterpret_cast(&length), sizeof(length));\n fin.read(reinterpret_cast(&ftype), sizeof(ftype));\n\n if (fin.eof()) {\n break;\n }\n\n int64_t nelements = 1;\n int64_t ne[4] = { 1, 1, 1, 1 };\n for (int i = 0; i < n_dims; ++i) {\n int32_t ne_cur;\n fin.read(reinterpret_cast(&ne_cur), sizeof(ne_cur));\n ne[i] = ne_cur;\n nelements *= ne[i];\n }\n\n std::string name(length, 0);\n fin.read(&name[0], length);\n\n if (model.tensors.find(name.data()) == model.tensors.end()) {\n fprintf(stderr, \"%s: unknown tensor '%s' in model file\\n\", __func__, name.data());\n return false;\n }\n\n auto tensor = model.tensors[name.data()];\n //printf(\"ne0 = %jd, ne1 = %jd, ne2 = %jd, ne3 = %jd\\n\", ne[0], ne[1], ne[2], ne[3]);\n\n if (ggml_nelements(tensor) != nelements) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file: got %d, expected %d\\n\",\n __func__, name.data(), (int) nelements, (int) ggml_nelements(tensor));\n return false;\n }\n\n if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1] || tensor->ne[2] != ne[2] || tensor->ne[3] != ne[3]) {\n fprintf(stderr, \"%s: tensor '%s' has wrong shape in model file: got [%d, %d, %d, %d], expected [%d, %d, %d, %d]\\n\",\n __func__, name.data(),\n (int) ne[0], (int) ne[1], (int) ne[2], (int) ne[3],\n (int) tensor->ne[0], (int) tensor->ne[1], (int) tensor->ne[2], (int) tensor->ne[3]);\n return false;\n }\n\n size_t bpe = 0;\n\n switch (ftype) {\n case 0: bpe = ggml_type_size(GGML_TYPE_F32); break;\n case 1: bpe = ggml_type_size(GGML_TYPE_F16); break;\n case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break;\n case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break;\n default:\n {\n fprintf(stderr, \"%s: unknown ftype %d in model file\\n\", __func__, ftype);\n return false;\n }\n };\n\n if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {\n fprintf(stderr, \"%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\\n\",\n __func__, name.data(), ggml_nbytes(tensor), (size_t) nelements*bpe);\n return false;\n }\n\n fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor));\n\n total_size += ggml_nbytes(tensor);\n if (++n_tensors % 8 == 0) {\n fprintf(stderr, \".\");\n fflush(stdout);\n }\n }\n\n if (n_tensors != int(model.tensors.size())) {\n fprintf(stderr, \"%s: model file has %d tensors, but %d tensors were expected\\n\", __func__, n_tensors, (int) model.tensors.size());\n return false;\n }\n\n fprintf(stderr, \" done\\n\");\n\n fprintf(stderr, \"%s: model size = %8.2f MB / num tensors = %d\\n\", __func__, total_size/1024.0/1024.0, n_tensors);\n }\n\n fin.close();\n\n return true;\n}\n\nstruct ggml_tensor * sam_fill_dense_pe(\n const sam_model & model,\n struct ggml_context * ctx0,\n struct ggml_cgraph * gf,\n sam_state & state) {\n const auto & hparams = model.hparams;\n const auto & enc = model.enc_prompt;\n\n\n const int32_t n_img_embd = hparams.n_img_embd();\n struct ggml_tensor * xy_embed_stacked = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 2, n_img_embd, n_img_embd);\n ggml_set_name(xy_embed_stacked, \"xy_embed_stacked\");\n ggml_set_input(xy_embed_stacked);\n\n struct ggml_tensor * cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, enc.pe)), xy_embed_stacked);\n\n cur = ggml_scale(ctx0, cur, float(2.0*M_PI));\n\n // concat\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L192\n {\n struct ggml_tensor * t_sin = ggml_map_custom1(ctx0, cur, ggml_sam_sin, GGML_N_TASKS_MAX, NULL);\n struct ggml_tensor * t_cos = ggml_map_custom1(ctx0, cur, ggml_sam_cos, GGML_N_TASKS_MAX, NULL);\n\n cur = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, t_sin->ne[0] + t_cos->ne[0], cur->ne[1], cur->ne[2]);\n\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, t_sin, ggml_view_3d(ctx0, cur, t_sin->ne[0], t_sin->ne[1], t_sin->ne[2], cur->nb[1], cur->nb[2], 0)));\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, t_cos, ggml_view_3d(ctx0, cur, t_sin->ne[0], t_sin->ne[1], t_sin->ne[2], cur->nb[1], cur->nb[2], t_sin->nb[1])));\n }\n\n struct ggml_tensor * pe_img_dense = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));\n ggml_build_forward_expand(gf, pe_img_dense);\n\n return pe_img_dense;\n}\n\nstruct ggml_tensor* sam_layer_norm_2d(\n struct ggml_context * ctx0,\n struct ggml_tensor * layer,\n int n_channels,\n struct ggml_tensor * w,\n struct ggml_tensor * b,\n float eps) {\n // LayerNorm2d\n // normalize along channel dimmension\n // TODO: better implementation\n layer = ggml_permute(ctx0,\n ggml_norm(ctx0, ggml_cont(ctx0, ggml_permute(ctx0, layer, 1, 2, 0, 3)), eps),\n 2, 0, 1, 3);\n\n layer = ggml_add(ctx0,\n ggml_mul(ctx0,\n ggml_repeat(ctx0, ggml_reshape_3d(ctx0, w, 1, 1, n_channels), layer),\n layer),\n ggml_repeat(ctx0, ggml_reshape_3d(ctx0, b, 1, 1, n_channels), layer));\n\n return layer;\n}\n\nstruct ggml_cgraph * sam_encode_image(\n const sam_model & model,\n sam_state & state,\n const sam_image_f32 & img) {\n\n const auto & hparams = model.hparams;\n const auto & enc = model.enc_img;\n\n const int32_t n_enc_state = hparams.n_enc_state;\n const int32_t n_enc_layer = hparams.n_enc_layer;\n const int32_t n_enc_head = hparams.n_enc_head;\n const int32_t n_enc_head_dim = hparams.n_enc_head_dim();\n const int32_t n_enc_out_chans = hparams.n_enc_out_chans;\n const int32_t n_img_size = hparams.n_img_size();\n const int32_t n_window_size = hparams.n_window_size();\n\n struct ggml_init_params ggml_params = {\n /*.mem_size =*/ state.buf_compute_img_enc.size(),\n /*.mem_buffer =*/ state.buf_compute_img_enc.data(),\n /*.no_alloc =*/ true, // skip allocating as we use ggml_alloc to allocate exact memory requirements\n };\n\n struct ggml_context * ctx0 = ggml_init(ggml_params);\n struct ggml_cgraph * gf = ggml_new_graph(ctx0);\n\n struct ggml_tensor * inp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_img_size, n_img_size, 3, 1);\n ggml_set_name(inp, \"inp\");\n ggml_set_input(inp);\n\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L392\n struct ggml_tensor * cur = ggml_conv_2d_sk_p0(ctx0, enc.proj_w, inp);\n cur = ggml_add_inplace(ctx0,\n cur,\n ggml_repeat(ctx0, enc.proj_b, cur));\n\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L394\n // keep in F32\n cur = ggml_cont(ctx0,\n ggml_permute(ctx0, cur, 1, 2, 0, 3));\n\n // convert to F16\n //cur = ggml_cpy(ctx0,\n // ggml_permute(ctx0, cur, 1, 2, 0, 3),\n // ggml_new_tensor_3d(ctx0, GGML_TYPE_F16, n_enc_state, n_img_embd, n_img_embd));\n\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L108-L109\n cur = ggml_add_inplace(ctx0, cur, enc.pe);\n\n struct ggml_tensor * inpL = cur;\n\n for (int il = 0; il < n_enc_layer; ++il) {\n const auto & layer = enc.layers[il];\n\n // norm\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L168\n {\n cur = ggml_norm(ctx0, inpL, hparams.eps);\n\n // cur = ln_0_w*cur + ln_0_b\n cur = ggml_mul(ctx0, cur, layer.norm1_w);\n cur = ggml_add_inplace(ctx0, cur, layer.norm1_b);\n }\n\n const int64_t w0 = cur->ne[1];\n const int64_t h0 = cur->ne[2];\n\n if (hparams.is_global_attn(il) == false) {\n // local attention layer - apply window partition\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L169-L172\n cur = ggml_win_part(ctx0, cur, n_window_size);\n }\n\n const int64_t W = cur->ne[1];\n const int64_t H = cur->ne[2];\n\n // self-attention\n {\n cur = ggml_mul_mat(ctx0, layer.qkv_w, cur);\n cur = ggml_add_inplace(ctx0, cur, layer.qkv_b);\n\n // split qkv into separate tensors\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L225-L229\n const int B = cur->ne[3];\n\n cur = ggml_reshape_4d(ctx0, cur, n_enc_state, 3, W*H, B);\n cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 3, 1, 2));\n\n struct ggml_tensor * Q;\n struct ggml_tensor * K;\n struct ggml_tensor * V;\n\n Q = ggml_view_3d (ctx0, cur, n_enc_state, W*H, B, cur->nb[1], cur->nb[2], 0*cur->nb[3]);\n Q = ggml_reshape_4d(ctx0, Q, n_enc_head_dim, n_enc_head, W*H, B);\n Q = ggml_cont (ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3));\n Q = ggml_reshape_3d(ctx0, Q, n_enc_head_dim, W*H, B*n_enc_head);\n\n K = ggml_view_3d (ctx0, cur, n_enc_state, W*H, B, cur->nb[1], cur->nb[2], 1*cur->nb[3]);\n K = ggml_reshape_4d(ctx0, K, n_enc_head_dim, n_enc_head, W*H, B);\n K = ggml_cont (ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3));\n K = ggml_reshape_3d(ctx0, K, n_enc_head_dim, W*H, B*n_enc_head);\n\n V = ggml_view_3d (ctx0, cur, n_enc_state, W*H, B, cur->nb[1], cur->nb[2], 2*cur->nb[3]);\n V = ggml_reshape_4d(ctx0, V, n_enc_head_dim, n_enc_head, W*H, B);\n V = ggml_cont (ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3)); // transposed\n V = ggml_reshape_3d(ctx0, V, W*H, n_enc_head_dim, B*n_enc_head);\n\n struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);\n\n struct ggml_tensor * KQ_scaled =\n ggml_scale_inplace(ctx0,\n KQ,\n 1.0f/sqrtf(n_enc_head_dim));\n\n struct ggml_tensor * rw = ggml_get_rel_pos(ctx0, layer.rel_pos_w, W, W);\n struct ggml_tensor * rh = ggml_get_rel_pos(ctx0, layer.rel_pos_h, H, H);\n\n struct ggml_tensor * q_r = ggml_reshape_4d(ctx0, Q, n_enc_head_dim, W, H, B*n_enc_head);\n\n struct ggml_tensor * rel_w = ggml_cont(ctx0, ggml_permute(ctx0,\n ggml_mul_mat(ctx0,\n rw,\n ggml_cont(ctx0, ggml_permute(ctx0, q_r, 0, 2, 1, 3))),\n 0, 2, 1, 3));\n struct ggml_tensor * rel_h = ggml_mul_mat(ctx0, rh, q_r);\n\n struct ggml_tensor * attn = ggml_add_rel_pos_inplace(ctx0, KQ_scaled, rel_w, rel_h);\n\n struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, attn);\n\n struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);\n\n cur =\n ggml_reshape_4d(ctx0,\n ggml_cont(ctx0,\n ggml_permute(ctx0,\n ggml_reshape_4d(ctx0, KQV, n_enc_head_dim, W*H, n_enc_head, B),\n 0, 2, 1, 3)),\n n_enc_state, W, H, B);\n\n cur = ggml_mul_mat(ctx0, layer.proj_w, cur);\n cur = ggml_add_inplace(ctx0, cur, layer.proj_b);\n }\n\n if (hparams.is_global_attn(il) == false) {\n // local attention layer - reverse window partition\n cur = ggml_win_unpart(ctx0, cur, w0, h0, n_window_size);\n }\n\n cur = ggml_add_inplace(ctx0, cur, inpL);\n\n struct ggml_tensor * inpFF = cur;\n\n // feed-forward network\n {\n // norm\n {\n cur = ggml_norm(ctx0, inpFF, hparams.eps);\n\n // cur = mlp_ln_w*cur + mlp_ln_b\n cur = ggml_mul(ctx0, cur, layer.norm2_w);\n cur = ggml_add_inplace(ctx0, cur, layer.norm2_b);\n }\n\n // fully connected\n cur = ggml_mul_mat(ctx0, layer.mlp_lin1_w, cur);\n cur = ggml_add_inplace(ctx0, cur, layer.mlp_lin1_b);\n\n // GELU activation\n cur = ggml_gelu(ctx0, cur);\n\n // projection\n cur = ggml_mul_mat(ctx0, layer.mlp_lin2_w, cur);\n cur = ggml_add_inplace(ctx0, cur, layer.mlp_lin2_b);\n }\n\n inpL = ggml_add(ctx0, cur, inpFF);\n }\n\n cur = ggml_cont(ctx0, ggml_permute(ctx0, inpL, 2, 0, 1, 3));\n\n cur = ggml_conv_2d_sk_p0(ctx0, enc.neck_conv_0, cur);\n\n cur = sam_layer_norm_2d(ctx0, cur, n_enc_out_chans, enc.neck_norm_0_w, enc.neck_norm_0_b, hparams.eps);\n\n cur = ggml_conv_2d_s1_ph(ctx0, enc.neck_conv_1, cur);\n\n cur = sam_layer_norm_2d(ctx0, cur, n_enc_out_chans, enc.neck_norm_1_w, enc.neck_norm_1_b, hparams.eps);\n\n cur = ggml_cpy(ctx0, cur, state.embd_img);\n\n ggml_build_forward_expand(gf, cur);\n ggml_disconnect_node_from_graph(state.embd_img);\n\n //ggml_graph_print(&gf);\n\n ggml_free(ctx0);\n\n ggml_gallocr_alloc_graph(state.allocr, gf);\n\n {\n struct ggml_tensor * inp = ggml_graph_get_tensor(gf, \"inp\");\n float * data = (float *) ggml_get_data(inp);\n\n const int nx = img.nx;\n const int ny = img.ny;\n const int n = nx*ny;\n\n GGML_ASSERT(nx == n_img_size && ny == n_img_size);\n\n for (int k = 0; k < 3; k++) {\n for (int y = 0; y < ny; y++) {\n for (int x = 0; x < nx; x++) {\n data[k*n + y*nx + x] = img.data[3*(y*nx + x) + k];\n }\n }\n }\n }\n\n return gf;\n}\n\n\nstruct prompt_encoder_result {\n struct ggml_tensor * embd_prompt_sparse = {};\n struct ggml_tensor * embd_prompt_dense = {};\n};\n\nstruct ggml_tensor * sam_prompt_encode_pe_encoding(\n const sam_encoder_prompt & enc,\n struct ggml_context * ctx0,\n struct ggml_cgraph * gf,\n struct ggml_tensor * coords) {\n\n auto * cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, enc.pe)), coords);\n cur = ggml_scale(ctx0, cur, float(2.0*M_PI));\n\n // concat\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L192\n {\n struct ggml_tensor * t_sin = ggml_map_custom1(ctx0, cur, ggml_sam_sin, GGML_N_TASKS_MAX, NULL);\n struct ggml_tensor * t_cos = ggml_map_custom1(ctx0, cur, ggml_sam_cos, GGML_N_TASKS_MAX, NULL);\n\n cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, t_sin->ne[0] + t_cos->ne[0], cur->ne[1]);\n\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, t_sin, ggml_view_2d(ctx0, cur, t_sin->ne[0], t_sin->ne[1], cur->nb[1], 0)));\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, t_cos, ggml_view_2d(ctx0, cur, t_sin->ne[0], t_sin->ne[1], cur->nb[1], t_sin->nb[1])));\n\n }\n return cur;\n\n}\n// encode a prompt\n//\n// - points\n// - boxes\n// - masks\n//\n// TODO: currently just encode a single point for simplicity\n//\nprompt_encoder_result sam_encode_prompt(\n const sam_model & model,\n struct ggml_context * ctx0,\n struct ggml_cgraph * gf,\n sam_state & state,\n const sam_prompt & prompt) {\n\n const auto & hparams = model.hparams;\n const auto & enc = model.enc_prompt;\n\n struct ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 2, 2);\n ggml_set_name(inp, \"prompt_input\");\n ggml_set_input(inp);\n\n auto * embd_prompt_sparse = [&]() -> struct ggml_tensor * {\n switch (prompt.prompt_type) {\n case SAM_PROMPT_TYPE_POINT: {\n // PromptEncoder._embed_points\n auto * pt_embd = sam_prompt_encode_pe_encoding(enc, ctx0, gf, inp);\n\n // overwrite label == -1 with not_a_point_embed.weight\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L86\n // TODO: extend for multiple points\n auto * pt_embd_not = ggml_view_2d(ctx0, pt_embd, pt_embd->ne[0], 1, pt_embd->nb[1], pt_embd->nb[1]);\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, enc.not_a_pt_embd_w, pt_embd_not));\n\n // add point_embeddings[1] to label == 1\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L90\n auto * pt_embd1 = ggml_view_2d(ctx0, pt_embd, pt_embd->ne[0], 1, pt_embd->nb[1], 0);\n ggml_build_forward_expand(gf, ggml_add_inplace(ctx0, pt_embd1, enc.pt_embd[1]));\n\n return pt_embd;\n } break;\n case SAM_PROMPT_TYPE_BOX: {\n // PromptEncoder._embed_boxes\n auto * corner_embd = sam_prompt_encode_pe_encoding(enc, ctx0, gf, inp);\n\n // corner_embd[:, 0, :] += self.point_embeddings[2].weight\n // corner_embd[:, 1, :] += self.point_embeddings[3].weight\n auto * corner0 = ggml_view_2d(\n ctx0, corner_embd, corner_embd->ne[0], 1, corner_embd->nb[1], 0);\n auto * corner1 = ggml_view_2d(\n ctx0, corner_embd, corner_embd->ne[0], 1, corner_embd->nb[1], corner_embd->nb[1]);\n\n ggml_build_forward_expand(gf, ggml_add_inplace(ctx0, corner0, enc.pt_embd[2]));\n ggml_build_forward_expand(gf, ggml_add_inplace(ctx0, corner1, enc.pt_embd[3]));\n\n return corner_embd;\n } break;\n default: {\n fprintf(stderr, \"%s: unsupported prompt type %d\\n\", __func__, prompt.prompt_type);\n return nullptr;\n } break;\n }\n }();\n\n ggml_build_forward_expand(gf, embd_prompt_sparse);\n\n struct ggml_tensor * embd_prompt_dense = ggml_repeat(ctx0,\n ggml_cont(ctx0,\n ggml_view_3d(ctx0, enc.no_mask_embd_w,\n 1, 1, enc.no_mask_embd_w->ne[0], enc.no_mask_embd_w->nb[0], enc.no_mask_embd_w->nb[0], 0)),\n ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hparams.n_img_embd(), hparams.n_img_embd(), hparams.n_enc_out_chans));\n\n ggml_build_forward_expand(gf, embd_prompt_dense);\n\n //printf(\"used_mem = %zu\\n\", ggml_used_mem(ctx0));\n\n prompt_encoder_result res;\n res.embd_prompt_sparse = embd_prompt_sparse;\n res.embd_prompt_dense = embd_prompt_dense;\n return res;\n}\n\nstruct ggml_tensor* sam_decode_mask_transformer_attn(\n const sam_layer_dec_transformer_attn & attn,\n struct ggml_tensor * queries,\n struct ggml_tensor * keys,\n struct ggml_tensor * values,\n struct ggml_context * ctx0,\n const sam_model & model) {\n const auto & hparams = model.hparams;\n const int n_head = hparams.n_dec_heads;\n\n struct ggml_tensor * Qcur = {};\n struct ggml_tensor * Kcur = {};\n struct ggml_tensor * Vcur = {};\n\n Qcur = ggml_mul_mat(ctx0, attn.q_w, queries);\n Qcur = ggml_add_inplace(ctx0, Qcur, attn.q_b);\n\n Kcur = ggml_mul_mat(ctx0, attn.k_w, keys);\n Kcur = ggml_add_inplace(ctx0, Kcur, attn.k_b);\n\n Vcur = ggml_mul_mat(ctx0, attn.v_w, values);\n Vcur = ggml_add_inplace(ctx0, Vcur, attn.v_b);\n\n struct ggml_tensor * Q = {};\n struct ggml_tensor * K = {};\n struct ggml_tensor * V = {};\n\n Q = ggml_reshape_4d(ctx0, Qcur, Qcur->ne[0]/n_head, n_head, Qcur->ne[1], Qcur->ne[2]);\n Q = ggml_cont(ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3));\n\n K = ggml_reshape_4d(ctx0, Kcur, Kcur->ne[0]/n_head, n_head, Kcur->ne[1], Kcur->ne[2]);\n K = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3));\n\n V = ggml_reshape_4d(ctx0, Vcur, Vcur->ne[0]/n_head, n_head, Vcur->ne[1], Vcur->ne[2]);\n V = ggml_cont(ctx0, ggml_permute(ctx0, V, 0, 2, 1, 3));\n\n // Q * K\n struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);\n\n struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, 1.0f/sqrt(float(Q->ne[0])));\n\n struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_scaled);\n\n struct ggml_tensor * KQV = ggml_mul_mat(ctx0, KQ_soft_max, ggml_cont(ctx0, ggml_transpose(ctx0, V)));\n\n struct ggml_tensor * KQV_merged = ggml_cont(ctx0, ggml_transpose(ctx0, KQV));\n KQV_merged = ggml_cont(ctx0, ggml_permute(ctx0, KQV_merged, 0, 2, 1, 3));\n KQV_merged = ggml_reshape_3d(ctx0, KQV_merged, KQV_merged->ne[0]*KQV_merged->ne[1], KQV_merged->ne[2], KQV_merged->ne[3]);\n KQV_merged = ggml_mul_mat(ctx0, attn.out_w, KQV_merged);\n KQV_merged = ggml_add_inplace(ctx0, KQV_merged, attn.out_b);\n\n return KQV_merged;\n}\n\nstruct ggml_tensor * sam_decode_mask_mlp_relu_3(\n struct ggml_tensor * in,\n struct ggml_tensor * w_0,\n struct ggml_tensor * b_0,\n struct ggml_tensor * w_1,\n struct ggml_tensor * b_1,\n struct ggml_tensor * w_2,\n struct ggml_tensor * b_2,\n struct ggml_context * ctx0) {\n\n struct ggml_tensor * cur = {};\n cur = ggml_mul_mat(ctx0, w_0, in);\n cur = ggml_add_inplace(ctx0, cur, b_0);\n\n cur = ggml_relu_inplace(ctx0, cur);\n\n cur = ggml_mul_mat(ctx0, w_1, cur);\n cur = ggml_add_inplace(ctx0, cur, b_1);\n\n cur = ggml_relu_inplace(ctx0, cur);\n\n cur = ggml_mul_mat(ctx0, w_2, cur);\n cur = ggml_add_inplace(ctx0, cur, b_2);\n\n return cur;\n}\n\nbool sam_decode_mask(\n const sam_model & model,\n const prompt_encoder_result & prompt,\n struct ggml_tensor * pe_img,\n struct ggml_context * ctx0,\n struct ggml_cgraph * gf,\n sam_state & state,\n const bool multimask_output) {\n\n const auto & hparams = model.hparams;\n const auto & dec = model.dec;\n const int n_img_embd = hparams.n_img_embd();\n\n struct ggml_tensor * tokens = {};\n {\n // Concatenate output tokens\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L120\n const auto& sparse = prompt.embd_prompt_sparse;\n\n tokens = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, dec.iou_token_w->ne[0], dec.iou_token_w->ne[1] + dec.mask_tokens_w->ne[1] + sparse->ne[1], sparse->ne[2]);\n\n const size_t offsets[3] = { 0, dec.iou_token_w->ne[1]*tokens->nb[1], dec.iou_token_w->ne[1]*tokens->nb[1] + dec.mask_tokens_w->ne[1]*tokens->nb[1] };\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, dec.iou_token_w, ggml_view_2d(ctx0, tokens, tokens->ne[0], dec.iou_token_w->ne[1], tokens->nb[1], offsets[0])));\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, dec.mask_tokens_w, ggml_view_2d(ctx0, tokens, tokens->ne[0], dec.mask_tokens_w->ne[1], tokens->nb[1], offsets[1])));\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, sparse, ggml_view_2d(ctx0, tokens, tokens->ne[0], sparse->ne[1], tokens->nb[1], offsets[2])));\n // TODO: Sparse prompt embeddings can have more than one point\n }\n\n\n struct ggml_tensor * src = {};\n struct ggml_tensor * pos_src = {};\n int srcNE[4] = { 0, 0, 0, 0 };\n {\n // Expand per-image data in the batch direction to be per-mask\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L125\n src = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, state.embd_img->ne[0], state.embd_img->ne[1], state.embd_img->ne[2], tokens->ne[2]);\n\n src = ggml_add(ctx0,\n ggml_repeat(ctx0,\n state.embd_img,\n src),\n prompt.embd_prompt_dense);\n\n srcNE[0] = src->ne[0];\n srcNE[1] = src->ne[1];\n srcNE[2] = src->ne[2];\n srcNE[3] = src->ne[3];\n\n // flatten & permute\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L83\n src = ggml_cont(ctx0, ggml_permute(ctx0,\n ggml_view_3d(ctx0,\n src,\n src->ne[0]*src->ne[1],\n src->ne[2],\n src->ne[3],\n src->nb[2],\n src->nb[3],\n 0),\n 1, 0, 2, 3));\n\n pos_src = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, pe_img->ne[0], pe_img->ne[1], pe_img->ne[2], tokens->ne[2]);\n pos_src = ggml_repeat(ctx0,\n pe_img,\n pos_src);\n\n // flatten & permute\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L83\n pos_src = ggml_cont(ctx0, ggml_permute(ctx0,\n ggml_view_3d(ctx0,\n pos_src,\n pos_src->ne[0]*pos_src->ne[1],\n pos_src->ne[2],\n pos_src->ne[3],\n pos_src->nb[2],\n pos_src->nb[3],\n 0),\n 1, 0, 2, 3));\n }\n\n struct ggml_tensor * queries = tokens;\n struct ggml_tensor * keys = src;\n {\n // Run the transformer\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L62\n for (int i = 0; i < int(model.dec.transformer_layers.size()); ++i) {\n const auto& tfm_layer = model.dec.transformer_layers[i];\n\n // Self attention block\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L154\n const bool skip_first_layer_pe = i == 0;\n if (skip_first_layer_pe) {\n queries = sam_decode_mask_transformer_attn(tfm_layer.self_attn, queries, queries, queries, ctx0, model);\n }\n else {\n struct ggml_tensor * q_0 = ggml_add(ctx0, queries, tokens);\n\n struct ggml_tensor * self_attn = sam_decode_mask_transformer_attn(tfm_layer.self_attn, q_0, q_0, queries, ctx0, model);\n queries = ggml_add(ctx0, queries, self_attn);\n }\n\n queries = ggml_norm(ctx0, queries, hparams.eps_decoder_transformer);\n queries = ggml_add_inplace(ctx0,\n ggml_mul(ctx0, queries, tfm_layer.norm1_w),\n tfm_layer.norm1_b);\n\n // Cross attention block, tokens attending to image embedding\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L163\n struct ggml_tensor * q_1 = ggml_add(ctx0, queries, tokens);\n struct ggml_tensor * k_1 = ggml_add(ctx0, keys, pos_src);\n\n struct ggml_tensor * cross_attn_token_to_img = sam_decode_mask_transformer_attn(tfm_layer.cross_attn_token_to_img, q_1, k_1, keys, ctx0, model);\n\n queries = ggml_add_inplace(ctx0, queries, cross_attn_token_to_img);\n queries = ggml_norm_inplace(ctx0, queries, hparams.eps_decoder_transformer);\n queries = ggml_add_inplace(ctx0,\n ggml_mul(ctx0, queries, tfm_layer.norm2_w),\n tfm_layer.norm2_b);\n\n // MLP block\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L170\n struct ggml_tensor * mlp_out = ggml_mul_mat(ctx0,\n tfm_layer.mlp_lin1_w,\n queries);\n\n mlp_out = ggml_add_inplace(ctx0, mlp_out, tfm_layer.mlp_lin1_b);\n\n // RELU activation\n mlp_out = ggml_relu_inplace(ctx0, mlp_out);\n mlp_out = ggml_mul_mat(ctx0, tfm_layer.mlp_lin2_w, mlp_out);\n\n mlp_out = ggml_add_inplace(ctx0, mlp_out, tfm_layer.mlp_lin2_b);\n\n queries = ggml_add_inplace(ctx0, queries, mlp_out);\n queries = ggml_norm_inplace(ctx0, queries, hparams.eps_decoder_transformer);\n queries = ggml_add_inplace(ctx0,\n ggml_mul(ctx0, queries, tfm_layer.norm3_w),\n tfm_layer.norm3_b);\n\n // Cross attention block, image embedding attending to tokens\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L175\n struct ggml_tensor * q_2 = ggml_add(ctx0, queries, tokens);\n struct ggml_tensor * k_2 = ggml_add(ctx0, keys, pos_src);\n\n struct ggml_tensor * cross_attn_img_to_token = sam_decode_mask_transformer_attn(tfm_layer.cross_attn_img_to_token, k_2, q_2, queries, ctx0, model);\n keys = ggml_add_inplace(ctx0, keys, cross_attn_img_to_token);\n keys = ggml_norm_inplace(ctx0, keys, hparams.eps_decoder_transformer);\n keys = ggml_add_inplace(ctx0,\n ggml_mul(ctx0, keys, tfm_layer.norm4_w),\n tfm_layer.norm4_b);\n }\n\n // Apply the final attention layer from the points to the image\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L99\n struct ggml_tensor * q = ggml_add(ctx0, queries, tokens);\n struct ggml_tensor * k = ggml_add(ctx0, keys, pos_src);\n\n struct ggml_tensor * final_attn_token_to_img = sam_decode_mask_transformer_attn(dec.transformer_final_attn_token_to_img, q, k, keys, ctx0, model);\n\n queries = ggml_add_inplace(ctx0, queries, final_attn_token_to_img);\n queries = ggml_norm_inplace(ctx0, queries, hparams.eps_decoder_transformer);\n queries = ggml_add_inplace(ctx0,\n ggml_mul(ctx0, queries, dec.transformer_norm_final_w),\n dec.transformer_norm_final_b);\n }\n\n\n struct ggml_tensor * iou_pred = ggml_view_2d(ctx0, queries, queries->ne[0], queries->ne[2], queries->nb[2], 0);\n const int num_mask_tokens = 4; // num_multimask_outputs + 1\n struct ggml_tensor * mask_tokens_out = ggml_view_3d(ctx0, queries, queries->ne[0], num_mask_tokens, queries->ne[2], queries->nb[1], num_mask_tokens*queries->nb[1], queries->nb[1]);\n\n // Upscale mask embeddings and predict masks using the mask tokens\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L136\n keys = ggml_cont(ctx0, ggml_transpose(ctx0, keys));\n keys = ggml_view_4d(ctx0, keys, srcNE[0], srcNE[1], srcNE[2], srcNE[3], srcNE[0]*keys->nb[0], keys->nb[1], keys->nb[2], 0);\n // ggml_build_forward_expand(gf, keys);\n struct ggml_tensor * upscaled_embedding = {};\n {\n // ConvTranspose2d\n keys = ggml_conv_transpose_2d_p0(ctx0, dec.output_upscaling_0_w, keys, 2);\n keys = ggml_add_inplace(ctx0, keys, ggml_repeat(ctx0,\n ggml_reshape_3d(ctx0, dec.output_upscaling_0_b, 1, 1, dec.output_upscaling_0_b->ne[0]),\n keys));\n\n keys = sam_layer_norm_2d(ctx0, keys, n_img_embd, dec.output_upscaling_1_w, dec.output_upscaling_1_b, hparams.eps);\n\n // GELU activation\n keys = ggml_gelu_inplace(ctx0, keys);\n\n // ConvTranspose2d\n keys = ggml_conv_transpose_2d_p0(ctx0, dec.output_upscaling_3_w, keys, 2);\n keys = ggml_add_inplace(ctx0, ggml_repeat(ctx0,\n ggml_reshape_3d(ctx0, dec.output_upscaling_3_b, 1, 1, dec.output_upscaling_3_b->ne[0]),\n keys), keys);\n // GELU activation\n keys = ggml_gelu_inplace(ctx0, keys);\n upscaled_embedding = ggml_reshape_3d(ctx0, keys, keys->ne[0]*keys->ne[1], keys->ne[2], keys->ne[3]);\n upscaled_embedding = ggml_cont(ctx0, ggml_transpose(ctx0, upscaled_embedding)); // TODO: Shouldn't be needed\n }\n\n struct ggml_tensor * hyper_in = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_img_embd/2, num_mask_tokens, mask_tokens_out->ne[2]);\n\n for (int i = 0; i < num_mask_tokens; ++i) {\n const auto& mlp = dec.output_hypernet_mlps[i];\n struct ggml_tensor * in = ggml_view_2d(ctx0, mask_tokens_out, mask_tokens_out->ne[0], mask_tokens_out->ne[2], mask_tokens_out->nb[1], i*mask_tokens_out->nb[1]);\n struct ggml_tensor * out = sam_decode_mask_mlp_relu_3(in, mlp.w_0, mlp.b_0, mlp.w_1, mlp.b_1, mlp.w_2, mlp.b_2, ctx0);\n ggml_build_forward_expand(gf, ggml_cpy(ctx0, out, ggml_view_2d(ctx0, hyper_in, hyper_in->ne[0], hyper_in->ne[2], hyper_in->nb[1], i*hyper_in->nb[1])));\n }\n\n struct ggml_tensor * masks = ggml_mul_mat(ctx0, hyper_in, upscaled_embedding);\n masks = ggml_cont(ctx0, ggml_transpose(ctx0, masks)); // TODO: Shouldn't be needed\n masks = ggml_reshape_4d(ctx0, masks, keys->ne[0], keys->ne[1], masks->ne[1], keys->ne[3]);\n\n // Generate mask quality predictions\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L146\n iou_pred = sam_decode_mask_mlp_relu_3(iou_pred, dec.iou_prediction_head_0_w, dec.iou_prediction_head_0_b, dec.iou_prediction_head_1_w, dec.iou_prediction_head_1_b, dec.iou_prediction_head_2_w, dec.iou_prediction_head_2_b, ctx0);\n\n // Select the correct mask or masks for output\n // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L101\n if (multimask_output) {\n iou_pred = ggml_cpy(state.ctx, ggml_view_1d(ctx0, iou_pred, iou_pred->ne[0] - 1, iou_pred->nb[0]), state.iou_predictions);\n masks = ggml_view_4d(ctx0, masks, masks->ne[0], masks->ne[1], masks->ne[2] - 1, masks->ne[3],\n masks->nb[1], masks->nb[2], masks->nb[3], masks->nb[2] /* offset*/);\n masks = ggml_cpy(state.ctx, masks, state.low_res_masks);\n } else {\n iou_pred = ggml_cpy(state.ctx, ggml_view_1d(ctx0, iou_pred, 1, 0), ggml_view_1d(ctx0, state.iou_predictions, 1, 0));\n masks = ggml_view_4d(ctx0, masks, masks->ne[0], masks->ne[1], 1, masks->ne[3],\n masks->nb[1], masks->nb[2], masks->nb[3], 0);\n auto * low_res_mask = ggml_view_4d(ctx0, state.low_res_masks, masks->ne[0], masks->ne[1], 1, masks->ne[3],\n masks->nb[1], masks->nb[2], masks->nb[3], 0);\n masks = ggml_cpy(state.ctx, masks, low_res_mask);\n }\n\n ggml_build_forward_expand(gf, masks);\n ggml_build_forward_expand(gf, iou_pred);\n\n ggml_disconnect_node_from_graph(state.low_res_masks);\n ggml_disconnect_node_from_graph(state.iou_predictions);\n\n return true;\n}\n\nbool sam_write_masks(const sam_hparams& hparams, int nx, int ny, const sam_state & state, const std::string & fname, const bool multimask_output) {\n if (state.low_res_masks->ne[2] == 0) return true;\n if (state.low_res_masks->ne[2] != state.iou_predictions->ne[0]) {\n printf(\"Error: number of masks (%d) does not match number of iou predictions (%d)\\n\", (int)state.low_res_masks->ne[2], (int)state.iou_predictions->ne[0]);\n return false;\n }\n\n const int n_img_size = hparams.n_img_size();\n const float mask_threshold = hparams.mask_threshold;\n const float iou_threshold = hparams.iou_threshold;\n const float stability_score_threshold = hparams.stability_score_threshold;\n const float intersection_threshold = mask_threshold + hparams.stability_score_offset;\n const float union_threshold = mask_threshold - hparams.stability_score_offset;\n\n const int ne0 = state.low_res_masks->ne[0];\n const int ne1 = state.low_res_masks->ne[1];\n const int ne2 = multimask_output ? state.low_res_masks->ne[2] : 1;\n\n // Remove padding and upscale masks to the original image size.\n // ref: https://github.com/facebookresearch/segment-anything/blob/efeab7296ab579d4a261e554eca80faf6b33924a/segment_anything/modeling/sam.py#L140\n\n const float preprocess_scale = std::max(nx, ny) / float(n_img_size);\n const int cropped_nx = int(nx / preprocess_scale + 0.5f);\n const int cropped_ny = int(ny / preprocess_scale + 0.5f);\n\n const float scale_x_1 = (float)ne0 / (float)n_img_size;\n const float scale_y_1 = (float)ne1 / (float)n_img_size;\n\n const float scale_x_2 = float(cropped_nx) / float(nx);\n const float scale_y_2 = float(cropped_ny) / float(ny);\n\n const auto iou_data = (float*)state.iou_predictions->data;\n\n for (int i = 0; i < ne2; ++i) {\n if (iou_threshold > 0.f && iou_data[i] < iou_threshold) {\n printf(\"Skipping mask %d with iou %f below threshold %f\\n\", i, iou_data[i], iou_threshold);\n continue; // Filtering masks with iou below the threshold\n }\n\n std::vector mask_data(n_img_size*n_img_size);\n {\n const float* data = (float *) state.low_res_masks->data + i*ne0*ne1;\n\n for (int iy = 0; iy < n_img_size; ++iy) {\n for (int ix = 0; ix < n_img_size; ++ix) {\n const float sx = std::max(scale_x_1*(ix + 0.5f) - 0.5f, 0.0f);\n const float sy = std::max(scale_y_1*(iy + 0.5f) - 0.5f, 0.0f);\n\n const int x0 = std::max(0, (int)sx);\n const int y0 = std::max(0, (int)sy);\n\n const int x1 = std::min(x0 + 1, ne0 - 1);\n const int y1 = std::min(y0 + 1, ne1 - 1);\n\n const float dx = sx - x0;\n const float dy = sy - y0;\n\n const int j00 = y0*ne0 + x0;\n const int j01 = y0*ne0 + x1;\n const int j10 = y1*ne0 + x0;\n const int j11 = y1*ne0 + x1;\n\n const float v00 = data[j00];\n const float v01 = data[j01];\n const float v10 = data[j10];\n const float v11 = data[j11];\n\n const float v0 = (1-dx)*v00 + dx*v01;\n const float v1 = (1-dx)*v10 + dx*v11;\n\n const float v = (1-dy)*v0 + dy*v1;\n\n mask_data[iy*n_img_size + ix] = v;\n }\n }\n }\n\n int intersections = 0;\n int unions = 0;\n sam_image_u8 res;\n int min_iy = ny;\n int max_iy = 0;\n int min_ix = nx;\n int max_ix = 0;\n {\n const float* data = mask_data.data();\n\n res.nx = nx;\n res.ny = ny;\n res.data.resize(nx*ny);\n\n for (int iy = 0; iy < ny; ++iy) {\n for (int ix = 0; ix < nx; ++ix) {\n const float sx = std::max(scale_x_2*(ix + 0.5f) - 0.5f, 0.0f);\n const float sy = std::max(scale_y_2*(iy + 0.5f) - 0.5f, 0.0f);\n\n const int x0 = std::max(0, (int)sx);\n const int y0 = std::max(0, (int)sy);\n\n const int x1 = std::min(x0 + 1, cropped_nx - 1);\n const int y1 = std::min(y0 + 1, cropped_ny - 1);\n\n const float dx = sx - x0;\n const float dy = sy - y0;\n\n const int j00 = y0*n_img_size + x0;\n const int j01 = y0*n_img_size + x1;\n const int j10 = y1*n_img_size + x0;\n const int j11 = y1*n_img_size + x1;\n\n const float v00 = data[j00];\n const float v01 = data[j01];\n const float v10 = data[j10];\n const float v11 = data[j11];\n\n const float v0 = (1-dx)*v00 + dx*v01;\n const float v1 = (1-dx)*v10 + dx*v11;\n\n const float v = (1-dy)*v0 + dy*v1;\n\n if (v > intersection_threshold) {\n intersections++;\n }\n if (v > union_threshold) {\n unions++;\n }\n if (v > mask_threshold) {\n min_iy = std::min(min_iy, iy);\n max_iy = std::max(max_iy, iy);\n min_ix = std::min(min_ix, ix);\n max_ix = std::max(max_ix, ix);\n\n res.data[iy*nx + ix] = 255;\n }\n }\n }\n }\n\n const float stability_score = float(intersections) / float(unions);\n if (stability_score_threshold > 0.f && stability_score < stability_score_threshold) {\n printf(\"Skipping mask %d with stability score %f below threshold %f\\n\", i, stability_score, stability_score_threshold);\n continue; // Filtering masks with stability score below the threshold\n }\n\n printf(\"Mask %d: iou = %f, stability_score = %f, bbox (%d, %d), (%d, %d)\\n\",\n i, iou_data[i], stability_score, min_ix, max_ix, min_iy, max_iy);\n\n const std::string filename = multimask_output ? fname + std::to_string(i) + \".png\" : fname + \".png\";\n if (!stbi_write_png(filename.c_str(), res.nx, res.ny, 1, res.data.data(), res.nx)) {\n printf(\"%s: failed to write mask %s\\n\", __func__, filename.c_str());\n return false;\n }\n }\n\n\n return true;\n}\n\n\nstruct ggml_cgraph * sam_build_fast_graph(\n const sam_model & model,\n sam_state & state,\n const int nx,\n const int ny,\n const sam_prompt & prompt,\n const bool multimask_output) {\n\n struct ggml_init_params ggml_params = {\n /*.mem_size =*/ state.buf_compute_fast.size(),\n /*.mem_buffer =*/ state.buf_compute_fast.data(),\n /*.no_alloc =*/ true, // skip allocating as we use ggml_alloc to allocate exact memory requirements\n };\n\n struct ggml_context * ctx0 = ggml_init(ggml_params);\n struct ggml_cgraph * gf = ggml_new_graph(ctx0);\n\n prompt_encoder_result enc_res = sam_encode_prompt(model, ctx0, gf, state, prompt);\n if (!enc_res.embd_prompt_sparse || !enc_res.embd_prompt_dense) {\n fprintf(stderr, \"%s: failed to encode prompt\\n\", __func__);\n return {};\n }\n\n struct ggml_tensor * pe_img_dense = sam_fill_dense_pe(model, ctx0, gf, state);\n if (!pe_img_dense) {\n fprintf(stderr, \"%s: failed to get dense positional encoding\\n\", __func__);\n return {};\n }\n\n if (!sam_decode_mask(model, enc_res, pe_img_dense, ctx0, gf, state, multimask_output)) {\n fprintf(stderr, \"%s: failed to decode mask\\n\", __func__);\n return {};\n }\n\n ggml_free(ctx0);\n\n ggml_gallocr_alloc_graph(state.allocr, gf);\n\n struct ggml_tensor * inp = ggml_graph_get_tensor(gf, \"prompt_input\");\n auto * data = (float *) inp->data;\n\n // Transform prompt (point or box)\n {\n // https://github.com/facebookresearch/segment-anything/blob/dca509fe793f601edb92606367a655c15ac00fdf/segment_anything/utils/transforms.py#L33\n // The point scaling here is greatly simplified but mathematically equivalent.\n const auto scale = 1.0F / std::max(nx, ny);\n\n switch (prompt.prompt_type) {\n case SAM_PROMPT_TYPE_POINT: {\n const auto & pt = prompt.pt;\n\n // set the input by converting the [0, 1] coordinates to [-1, 1]\n data[0] = 2.0f*pt.x*scale - 1.0f;\n data[1] = 2.0f*pt.y*scale - 1.0f;\n\n // padding\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L81-L85\n data[2] = 2.0f*(0.0f) - 1.0f;\n data[3] = 2.0f*(0.0f) - 1.0f;\n } break;\n case SAM_PROMPT_TYPE_BOX: {\n const auto & box = prompt.box;\n\n data[0] = 2.0f*box.x1*scale - 1.0f;\n data[1] = 2.0f*box.y1*scale - 1.0f;\n data[2] = 2.0f*box.x2*scale - 1.0f;\n data[3] = 2.0f*box.y2*scale - 1.0f;\n } break;\n }\n }\n\n // from sam_fill_dense_pe\n {\n struct ggml_tensor * xy_embed_stacked = ggml_graph_get_tensor(gf, \"xy_embed_stacked\");\n const int32_t n_img_embd = model.hparams.n_img_embd();\n const float n_img_embd_inv = 1.0f / n_img_embd;\n float * data = (float *) ggml_get_data(xy_embed_stacked);\n for (int i = 0; i < n_img_embd; ++i) {\n const int row = 2*i*n_img_embd;\n const float y_val = 2 * (i + 0.5f) * n_img_embd_inv - 1;\n for (int j = 0; j < n_img_embd; ++j) {\n const float x_val = 2 * (j + 0.5f) * n_img_embd_inv - 1;\n data[row + 2*j + 0] = x_val;\n data[row + 2*j + 1] = y_val;\n }\n }\n }\n\n return gf;\n}\n\nvoid sam_print_usage(int argc, char ** argv, const sam_params & params) {\n fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n fprintf(stderr, \"\\n\");\n fprintf(stderr, \"options:\\n\");\n fprintf(stderr, \" -h, --help show this help message and exit\\n\");\n fprintf(stderr, \" -s SEED, --seed SEED RNG seed (default: -1)\\n\");\n fprintf(stderr, \" -t N, --threads N number of threads to use during computation (default: %d)\\n\", params.n_threads);\n fprintf(stderr, \" -m FNAME, --model FNAME\\n\");\n fprintf(stderr, \" model path (default: %s)\\n\", params.model.c_str());\n fprintf(stderr, \" -i FNAME, --inp FNAME\\n\");\n fprintf(stderr, \" input file (default: %s)\\n\", params.fname_inp.c_str());\n fprintf(stderr, \" -o FNAME, --out FNAME\\n\");\n fprintf(stderr, \" mask file name prefix (default: %s)\\n\", params.fname_out.c_str());\n fprintf(stderr, \" -sm, --single-mask\\n\");\n fprintf(stderr, \" single mask output (default multi mask output)\\n\");\n fprintf(stderr, \"SAM hyperparameters:\\n\");\n fprintf(stderr, \" -mt FLOAT, --mask-threshold\\n\");\n fprintf(stderr, \" mask threshold (default: %f)\\n\", params.mask_threshold);\n fprintf(stderr, \" -it FLOAT, --iou-threshold\\n\");\n fprintf(stderr, \" iou threshold (default: %f)\\n\", params.iou_threshold);\n fprintf(stderr, \" -st FLOAT, --score-threshold\\n\");\n fprintf(stderr, \" score threshold (default: %f)\\n\", params.stability_score_threshold);\n fprintf(stderr, \" -so FLOAT, --score-offset\\n\");\n fprintf(stderr, \" score offset (default: %f)\\n\", params.stability_score_offset);\n fprintf(stderr, \" -e FLOAT, --epsilon\\n\");\n fprintf(stderr, \" epsilon (default: %f)\\n\", params.eps);\n fprintf(stderr, \" -ed FLOAT, --epsilon-decoder-transformer\\n\");\n fprintf(stderr, \" epsilon decoder transformer (default: %f)\\n\", params.eps_decoder_transformer);\n fprintf(stderr, \"SAM prompt:\\n\");\n fprintf(stderr, \" -p [x,y], --point-prompt\\n\");\n fprintf(stderr, \" point to be used as prompt for SAM (default: %f,%f). Must be in a format FLOAT,FLOAT \\n\", params.prompt.pt.x, params.prompt.pt.y);\n fprintf(stderr, \" -b [x1,y1,x2,y2], --box-prompt\\n\");\n fprintf(stderr, \" box to be used as prompt for SAM (default: %f,%f,%f,%f). Must be in a format FLOAT,FLOAT,FLOAT,FLOAT \\n\",\n params.prompt.box.x1, params.prompt.box.y1, params.prompt.box.x2, params.prompt.box.y2);\n fprintf(stderr, \"\\n\");\n}\n\nbool sam_params_parse(int argc, char ** argv, sam_params & params) {\n\n bool use_point_prompt = false;\n bool use_box_prompt = false;\n\n for (int i = 1; i < argc; i++) {\n std::string arg = argv[i];\n\n if (arg == \"-s\" || arg == \"--seed\") {\n params.seed = std::stoi(argv[++i]);\n } else if (arg == \"-t\" || arg == \"--threads\") {\n params.n_threads = std::stoi(argv[++i]);\n } else if (arg == \"-m\" || arg == \"--model\") {\n params.model = argv[++i];\n } else if (arg == \"-i\" || arg == \"--inp\") {\n params.fname_inp = argv[++i];\n } else if (arg == \"-o\" || arg == \"--out\") {\n params.fname_out = argv[++i];\n } else if (arg == \"-sm\" || arg == \"--single-mask\") {\n params.multimask_output = false;\n } else if (arg == \"-mt\" || arg == \"--mask-threshold\") {\n params.mask_threshold = std::stof(argv[++i]);\n } else if (arg == \"-it\" || arg == \"--iou-threshold\") {\n params.iou_threshold = std::stof(argv[++i]);\n } else if (arg == \"-st\" || arg == \"--score-threshold\") {\n params.stability_score_threshold = std::stof(argv[++i]);\n } else if (arg == \"-so\" || arg == \"--score-offset\") {\n params.stability_score_offset = std::stof(argv[++i]);\n } else if (arg == \"-e\" || arg == \"--epsilon\") {\n params.eps = std::stof(argv[++i]);\n } else if (arg == \"-ed\" || arg == \"--epsilon-decoder-transformer\") {\n params.eps_decoder_transformer = std::stof(argv[++i]);\n } else if (arg == \"-p\" || arg == \"--point-prompt\") {\n // TODO multiple points per model invocation\n use_point_prompt = true;\n char* point = argv[++i];\n\n char* coord = strtok(point, \",\");\n if (!coord){\n fprintf(stderr, \"Error while parsing prompt!\\n\");\n exit(1);\n }\n params.prompt.pt.x = std::stof(coord);\n\n coord = strtok(NULL, \",\");\n if (!coord){\n fprintf(stderr, \"Error while parsing prompt!\\n\");\n exit(1);\n }\n params.prompt.pt.y = std::stof(coord);\n } else if (arg == \"-b\" || arg == \"--box-prompt\") {\n use_box_prompt = true;\n char * box_prompt = argv[++i];\n float box_vals[4];\n\n char * val = strtok(box_prompt, \",\");\n if (!val) {\n fprintf(stderr, \"Error while parsing prompt!\\n\");\n exit(1);\n }\n box_vals[0] = std::stof(val);\n\n for (int j = 1; j < 4; ++j) {\n char * val = strtok(NULL, \",\");\n if (!val) {\n fprintf(stderr, \"Error while parsing prompt!\\n\");\n exit(1);\n }\n box_vals[j] = std::stof(val);\n }\n\n params.prompt.box.x1 = box_vals[0];\n params.prompt.box.y1 = box_vals[1];\n params.prompt.box.x2 = box_vals[2];\n params.prompt.box.y2 = box_vals[3];\n } else if (arg == \"-h\" || arg == \"--help\") {\n sam_print_usage(argc, argv, params);\n exit(0);\n } else {\n fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n sam_print_usage(argc, argv, params);\n exit(0);\n }\n }\n\n if (use_box_prompt && use_point_prompt) {\n fprintf(stderr, \"Error: use either point or box prompt, not both.\\n\");\n exit(1);\n }\n\n params.prompt.prompt_type = SAM_PROMPT_TYPE_POINT;\n if (use_box_prompt) {\n params.prompt.prompt_type = SAM_PROMPT_TYPE_BOX;\n }\n\n return true;\n}\n\n\nint main(int argc, char ** argv) {\n const int64_t t_main_start_us = ggml_time_us();\n\n sam_params params;\n params.model = \"models/sam-vit-b/ggml-model-f16.bin\";\n\n sam_model model;\n sam_state state;\n int64_t t_load_us = 0;\n\n if (sam_params_parse(argc, argv, params) == false) {\n return 1;\n }\n\n if (params.seed < 0) {\n params.seed = time(NULL);\n }\n fprintf(stderr, \"%s: seed = %d\\n\", __func__, params.seed);\n\n // load the image\n sam_image_u8 img0;\n if (!sam_image_load_from_file(params.fname_inp, img0)) {\n fprintf(stderr, \"%s: failed to load image from '%s'\\n\", __func__, params.fname_inp.c_str());\n return 1;\n }\n fprintf(stderr, \"%s: loaded image '%s' (%d x %d)\\n\", __func__, params.fname_inp.c_str(), img0.nx, img0.ny);\n\n // preprocess to f32\n sam_image_f32 img1;\n if (!sam_image_preprocess(img0, img1)) {\n fprintf(stderr, \"%s: failed to preprocess image\\n\", __func__);\n return 1;\n }\n fprintf(stderr, \"%s: preprocessed image (%d x %d)\\n\", __func__, img1.nx, img1.ny);\n\n\n // load the model\n {\n const int64_t t_start_us = ggml_time_us();\n\n if (!sam_model_load(params, model)) {\n fprintf(stderr, \"%s: failed to load model from '%s'\\n\", __func__, params.model.c_str());\n return 1;\n }\n\n t_load_us = ggml_time_us() - t_start_us;\n }\n\n {\n static size_t buf_size = 256u*1024*1024;\n\n struct ggml_init_params ggml_params = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ false,\n };\n\n state.ctx = ggml_init(ggml_params);\n\n state.embd_img = ggml_new_tensor_3d(state.ctx, GGML_TYPE_F32,\n model.hparams.n_img_embd(), model.hparams.n_img_embd(), model.hparams.n_enc_out_chans);\n\n state.low_res_masks = ggml_new_tensor_3d(state.ctx, GGML_TYPE_F32,\n model.hparams.n_enc_out_chans, model.hparams.n_enc_out_chans, 3);\n\n state.iou_predictions = ggml_new_tensor_1d(state.ctx, GGML_TYPE_F32, 3);\n }\n\n // Encode image\n {\n state.buf_compute_img_enc.resize(ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead());\n state.allocr = ggml_gallocr_new(ggml_backend_cpu_buffer_type());\n\n struct ggml_cgraph * gf = sam_encode_image(model, state, img1);\n if (!gf) {\n fprintf(stderr, \"%s: failed to encode image\\n\", __func__);\n return 1;\n }\n\n ggml_graph_compute_helper(state.work_buffer, gf, params.n_threads);\n\n // print_t_f32(\"embd_img\", state.embd_img);\n\n ggml_gallocr_free(state.allocr);\n state.allocr = NULL;\n state.work_buffer.clear();\n }\n\n // Encode prompt and decode mask\n {\n state.buf_compute_fast.resize(ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead());\n state.allocr = ggml_gallocr_new(ggml_backend_cpu_buffer_type());\n\n switch (params.prompt.prompt_type) {\n case SAM_PROMPT_TYPE_POINT:\n fprintf(stderr, \"Using point prompt: (%f, %f)\\n\", params.prompt.pt.x, params.prompt.pt.y);\n break;\n case SAM_PROMPT_TYPE_BOX:\n fprintf(stderr, \"Using box prompt: (%f, %f, %f, %f)\\n\",\n params.prompt.box.x1,\n params.prompt.box.y1,\n params.prompt.box.x2,\n params.prompt.box.y2);\n break;\n }\n\n struct ggml_cgraph * gf = sam_build_fast_graph(model, state, img0.nx, img0.ny, params.prompt, params.multimask_output);\n if (!gf) {\n fprintf(stderr, \"%s: failed to build fast graph\\n\", __func__);\n return 1;\n }\n\n ggml_graph_compute_helper(state.work_buffer, gf, params.n_threads);\n\n //print_t_f32(\"iou_predictions\", state.iou_predictions);\n //print_t_f32(\"low_res_masks\", state.low_res_masks);\n ggml_gallocr_free(state.allocr);\n state.allocr = NULL;\n }\n\n if (!sam_write_masks(model.hparams, img0.nx, img0.ny, state, params.fname_out, params.multimask_output)) {\n fprintf(stderr, \"%s: failed to write masks\\n\", __func__);\n return 1;\n }\n\n // report timing\n {\n const int64_t t_main_end_us = ggml_time_us();\n\n fprintf(stderr, \"\\n\\n\");\n fprintf(stderr, \"%s: load time = %8.2f ms\\n\", __func__, t_load_us/1000.0f);\n fprintf(stderr, \"%s: total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n }\n\n ggml_free(model.ctx);\n\n return 0;\n}\n" }, { "path": "examples/simple/CMakeLists.txt", "content": "#\n# simple-ctx\n\nset(TEST_TARGET simple-ctx)\nadd_executable(${TEST_TARGET} simple-ctx.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml)\n\n#\n# simple-backend\n\nset(TEST_TARGET simple-backend)\nadd_executable(${TEST_TARGET} simple-backend.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml)\n\nif (GGML_CUDA)\n add_compile_definitions(GGML_USE_CUDA)\nendif()\n\nif (GGML_METAL)\n add_compile_definitions(GGML_USE_METAL)\nendif()\n" }, { "path": "examples/simple/README.md", "content": "## Simple\n\nThis example simply performs a matrix multiplication, solely for the purpose of demonstrating a basic usage of ggml and backend handling. The code is commented to help understand what each part does.\n\nTraditional matrix multiplication goes like this (multiply row-by-column):\n\n$$\nA \\times B = C\n$$\n\n$$\n\\begin{bmatrix}\n2 & 8 \\\\\n5 & 1 \\\\\n4 & 2 \\\\\n8 & 6 \\\\\n\\end{bmatrix}\n\\times\n\\begin{bmatrix}\n10 & 9 & 5 \\\\\n5 & 9 & 4 \\\\\n\\end{bmatrix}\n\\=\n\\begin{bmatrix}\n60 & 90 & 42 \\\\\n55 & 54 & 29 \\\\\n50 & 54 & 28 \\\\\n110 & 126 & 64 \\\\\n\\end{bmatrix}\n$$\n\nIn `ggml`, we pass the matrix $B$ in transposed form and multiply row-by-row. The result $C$ is also transposed:\n\n$$\nggml\\\\_mul\\\\_mat(A, B^T) = C^T\n$$\n\n$$\nggml\\\\_mul\\\\_mat(\n\\begin{bmatrix}\n2 & 8 \\\\\n5 & 1 \\\\\n4 & 2 \\\\\n8 & 6 \\\\\n\\end{bmatrix}\n,\n\\begin{bmatrix}\n10 & 5 \\\\\n9 & 9 \\\\\n5 & 4 \\\\\n\\end{bmatrix}\n)\n\\=\n\\begin{bmatrix}\n60 & 55 & 50 & 110 \\\\\n90 & 54 & 54 & 126 \\\\\n42 & 29 & 28 & 64 \\\\\n\\end{bmatrix}\n$$\n\nThe `simple-ctx` doesn't support gpu acceleration. `simple-backend` demonstrates how to use other backends like CUDA and Metal.\n" }, { "path": "examples/simple/simple-backend.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\nstatic void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {\n (void) level;\n (void) user_data;\n fputs(text, stderr);\n fflush(stderr);\n}\n\n// This is a simple model with two tensors a and b\nstruct simple_model {\n struct ggml_tensor * a {};\n struct ggml_tensor * b {};\n\n // the backend to perform the computation (CPU, CUDA, METAL)\n ggml_backend_t backend {};\n ggml_backend_t cpu_backend {};\n ggml_backend_sched_t sched {};\n\n // storage for the graph and tensors\n std::vector buf;\n};\n\n// initialize data of matrices to perform matrix multiplication\nconst int rows_A = 4, cols_A = 2;\n\nfloat matrix_A[rows_A * cols_A] = {\n 2, 8,\n 5, 1,\n 4, 2,\n 8, 6\n};\n\nconst int rows_B = 3, cols_B = 2;\n/* Transpose([\n 10, 9, 5,\n 5, 9, 4\n]) 2 rows, 3 cols */\nfloat matrix_B[rows_B * cols_B] = {\n 10, 5,\n 9, 9,\n 5, 4\n};\n\n\n// initialize the tensors of the model in this case two matrices 2x2\nvoid init_model(simple_model & model) {\n ggml_log_set(ggml_log_callback_default, nullptr);\n\n ggml_backend_load_all();\n\n model.backend = ggml_backend_init_best();\n model.cpu_backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);\n\n ggml_backend_t backends[2] = { model.backend, model.cpu_backend };\n model.sched = ggml_backend_sched_new(backends, nullptr, 2, GGML_DEFAULT_GRAPH_SIZE, false, true);\n}\n\n// build the compute graph to perform a matrix multiplication\nstruct ggml_cgraph * build_graph(simple_model& model) {\n size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();\n model.buf.resize(buf_size);\n\n struct ggml_init_params params0 = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ model.buf.data(),\n /*.no_alloc =*/ true, // the tensors will be allocated later\n };\n\n // create a context to build the graph\n struct ggml_context * ctx = ggml_init(params0);\n\n struct ggml_cgraph * gf = ggml_new_graph(ctx);\n\n // create tensors\n model.a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cols_A, rows_A);\n model.b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cols_B, rows_B);\n\n // result = a*b^T\n struct ggml_tensor * result = ggml_mul_mat(ctx, model.a, model.b);\n\n // build operations nodes\n ggml_build_forward_expand(gf, result);\n\n ggml_free(ctx);\n\n return gf;\n}\n\n// compute with backend\nstruct ggml_tensor * compute(simple_model & model, struct ggml_cgraph * gf) {\n ggml_backend_sched_reset(model.sched);\n ggml_backend_sched_alloc_graph(model.sched, gf);\n\n // load data from cpu memory to backend buffer\n ggml_backend_tensor_set(model.a, matrix_A, 0, ggml_nbytes(model.a));\n ggml_backend_tensor_set(model.b, matrix_B, 0, ggml_nbytes(model.b));\n\n // compute the graph\n ggml_backend_sched_graph_compute(model.sched, gf);\n\n // in this case, the output tensor is the last one in the graph\n return ggml_graph_node(gf, -1);\n}\n\nint main(void) {\n ggml_time_init();\n\n simple_model model;\n init_model(model);\n\n struct ggml_cgraph * gf = build_graph(model);\n\n // perform computation\n struct ggml_tensor * result = compute(model, gf);\n\n // create a array to print result\n std::vector out_data(ggml_nelements(result));\n\n // bring the data from the backend memory\n ggml_backend_tensor_get(result, out_data.data(), 0, ggml_nbytes(result));\n\n // expected result:\n // [ 60.00 55.00 50.00 110.00\n // 90.00 54.00 54.00 126.00\n // 42.00 29.00 28.00 64.00 ]\n\n printf(\"mul mat (%d x %d) (transposed result):\\n[\", (int) result->ne[0], (int) result->ne[1]);\n for (int j = 0; j < result->ne[1] /* rows */; j++) {\n if (j > 0) {\n printf(\"\\n\");\n }\n\n for (int i = 0; i < result->ne[0] /* cols */; i++) {\n printf(\" %.2f\", out_data[j * result->ne[0] + i]);\n }\n }\n printf(\" ]\\n\");\n\n // release backend memory and free backend\n ggml_backend_sched_free(model.sched);\n ggml_backend_free(model.backend);\n ggml_backend_free(model.cpu_backend);\n return 0;\n}\n" }, { "path": "examples/simple/simple-ctx.cpp", "content": "#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n// This is a simple model with two tensors a and b\nstruct simple_model {\n struct ggml_tensor * a;\n struct ggml_tensor * b;\n\n // the context to define the tensor information (dimensions, size, memory data)\n struct ggml_context * ctx;\n};\n\n// initialize the tensors of the model in this case two matrices 2x2\nvoid load_model(simple_model & model, float * a, float * b, int rows_A, int cols_A, int rows_B, int cols_B) {\n size_t ctx_size = 0;\n {\n ctx_size += rows_A * cols_A * ggml_type_size(GGML_TYPE_F32); // tensor a\n ctx_size += rows_B * cols_B * ggml_type_size(GGML_TYPE_F32); // tensor b\n ctx_size += 2 * ggml_tensor_overhead(), // tensors\n ctx_size += ggml_graph_overhead(); // compute graph\n ctx_size += 1024; // some overhead\n }\n\n struct ggml_init_params params {\n /*.mem_size =*/ ctx_size,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ false, // NOTE: this should be false when using the legacy API\n };\n\n // create context\n model.ctx = ggml_init(params);\n\n // create tensors\n model.a = ggml_new_tensor_2d(model.ctx, GGML_TYPE_F32, cols_A, rows_A);\n model.b = ggml_new_tensor_2d(model.ctx, GGML_TYPE_F32, cols_B, rows_B);\n\n memcpy(model.a->data, a, ggml_nbytes(model.a));\n memcpy(model.b->data, b, ggml_nbytes(model.b));\n}\n\n// build the compute graph to perform a matrix multiplication\nstruct ggml_cgraph * build_graph(const simple_model& model) {\n struct ggml_cgraph * gf = ggml_new_graph(model.ctx);\n\n // result = a*b^T\n struct ggml_tensor * result = ggml_mul_mat(model.ctx, model.a, model.b);\n\n ggml_build_forward_expand(gf, result);\n return gf;\n}\n\n// compute with backend\nstruct ggml_tensor * compute(const simple_model & model) {\n struct ggml_cgraph * gf = build_graph(model);\n\n int n_threads = 1; // number of threads to perform some operations with multi-threading\n\n ggml_graph_compute_with_ctx(model.ctx, gf, n_threads);\n\n // in this case, the output tensor is the last one in the graph\n return ggml_graph_node(gf, -1);\n}\n\nint main(void) {\n ggml_time_init();\n\n // initialize data of matrices to perform matrix multiplication\n const int rows_A = 4, cols_A = 2;\n\n float matrix_A[rows_A * cols_A] = {\n 2, 8,\n 5, 1,\n 4, 2,\n 8, 6\n };\n\n const int rows_B = 3, cols_B = 2;\n /* Transpose([\n 10, 9, 5,\n 5, 9, 4\n ]) 2 rows, 3 cols */\n float matrix_B[rows_B * cols_B] = {\n 10, 5,\n 9, 9,\n 5, 4\n };\n\n simple_model model;\n load_model(model, matrix_A, matrix_B, rows_A, cols_A, rows_B, cols_B);\n\n // perform computation in cpu\n struct ggml_tensor * result = compute(model);\n\n // get the result data pointer as a float array to print\n std::vector out_data(ggml_nelements(result));\n memcpy(out_data.data(), result->data, ggml_nbytes(result));\n\n // expected result:\n // [ 60.00 55.00 50.00 110.00\n // 90.00 54.00 54.00 126.00\n // 42.00 29.00 28.00 64.00 ]\n\n printf(\"mul mat (%d x %d) (transposed result):\\n[\", (int) result->ne[0], (int) result->ne[1]);\n for (int j = 0; j < result->ne[1] /* rows */; j++) {\n if (j > 0) {\n printf(\"\\n\");\n }\n\n for (int i = 0; i < result->ne[0] /* cols */; i++) {\n printf(\" %.2f\", out_data[j * result->ne[0] + i]);\n }\n }\n printf(\" ]\\n\");\n\n // free memory\n ggml_free(model.ctx);\n return 0;\n}\n" }, { "path": "examples/stb_image.h", "content": "/* stb_image - v2.28 - public domain image loader - http://nothings.org/stb\n no warranty implied; use at your own risk\n\n Do this:\n #define STB_IMAGE_IMPLEMENTATION\n before you include this file in *one* C or C++ file to create the implementation.\n\n // i.e. it should look like this:\n #include ...\n #include ...\n #include ...\n #define STB_IMAGE_IMPLEMENTATION\n #include \"stb_image.h\"\n\n You can #define STBI_ASSERT(x) before the #include to avoid using assert.h.\n And #define STBI_MALLOC, STBI_REALLOC, and STBI_FREE to avoid using malloc,realloc,free\n\n\n QUICK NOTES:\n Primarily of interest to game developers and other people who can\n avoid problematic images and only need the trivial interface\n\n JPEG baseline & progressive (12 bpc/arithmetic not supported, same as stock IJG lib)\n PNG 1/2/4/8/16-bit-per-channel\n\n TGA (not sure what subset, if a subset)\n BMP non-1bpp, non-RLE\n PSD (composited view only, no extra channels, 8/16 bit-per-channel)\n\n GIF (*comp always reports as 4-channel)\n HDR (radiance rgbE format)\n PIC (Softimage PIC)\n PNM (PPM and PGM binary only)\n\n Animated GIF still needs a proper API, but here's one way to do it:\n http://gist.github.com/urraka/685d9a6340b26b830d49\n\n - decode from memory or through FILE (define STBI_NO_STDIO to remove code)\n - decode from arbitrary I/O callbacks\n - SIMD acceleration on x86/x64 (SSE2) and ARM (NEON)\n\n Full documentation under \"DOCUMENTATION\" below.\n\n\nLICENSE\n\n See end of file for license information.\n\nRECENT REVISION HISTORY:\n\n 2.28 (2023-01-29) many error fixes, security errors, just tons of stuff\n 2.27 (2021-07-11) document stbi_info better, 16-bit PNM support, bug fixes\n 2.26 (2020-07-13) many minor fixes\n 2.25 (2020-02-02) fix warnings\n 2.24 (2020-02-02) fix warnings; thread-local failure_reason and flip_vertically\n 2.23 (2019-08-11) fix clang static analysis warning\n 2.22 (2019-03-04) gif fixes, fix warnings\n 2.21 (2019-02-25) fix typo in comment\n 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs\n 2.19 (2018-02-11) fix warning\n 2.18 (2018-01-30) fix warnings\n 2.17 (2018-01-29) bugfix, 1-bit BMP, 16-bitness query, fix warnings\n 2.16 (2017-07-23) all functions have 16-bit variants; optimizations; bugfixes\n 2.15 (2017-03-18) fix png-1,2,4; all Imagenet JPGs; no runtime SSE detection on GCC\n 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs\n 2.13 (2016-12-04) experimental 16-bit API, only for PNG so far; fixes\n 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes\n 2.11 (2016-04-02) 16-bit PNGS; enable SSE2 in non-gcc x64\n RGB-format JPEG; remove white matting in PSD;\n allocate large structures on the stack;\n correct channel count for PNG & BMP\n 2.10 (2016-01-22) avoid warning introduced in 2.09\n 2.09 (2016-01-16) 16-bit TGA; comments in PNM files; STBI_REALLOC_SIZED\n\n See end of file for full revision history.\n\n\n ============================ Contributors =========================\n\n Image formats Extensions, features\n Sean Barrett (jpeg, png, bmp) Jetro Lauha (stbi_info)\n Nicolas Schulz (hdr, psd) Martin \"SpartanJ\" Golini (stbi_info)\n Jonathan Dummer (tga) James \"moose2000\" Brown (iPhone PNG)\n Jean-Marc Lienher (gif) Ben \"Disch\" Wenger (io callbacks)\n Tom Seddon (pic) Omar Cornut (1/2/4-bit PNG)\n Thatcher Ulrich (psd) Nicolas Guillemot (vertical flip)\n Ken Miller (pgm, ppm) Richard Mitton (16-bit PSD)\n github:urraka (animated gif) Junggon Kim (PNM comments)\n Christopher Forseth (animated gif) Daniel Gibson (16-bit TGA)\n socks-the-fox (16-bit PNG)\n Jeremy Sawicki (handle all ImageNet JPGs)\n Optimizations & bugfixes Mikhail Morozov (1-bit BMP)\n Fabian \"ryg\" Giesen Anael Seghezzi (is-16-bit query)\n Arseny Kapoulkine Simon Breuss (16-bit PNM)\n John-Mark Allen\n Carmelo J Fdez-Aguera\n\n Bug & warning fixes\n Marc LeBlanc David Woo Guillaume George Martins Mozeiko\n Christpher Lloyd Jerry Jansson Joseph Thomson Blazej Dariusz Roszkowski\n Phil Jordan Dave Moore Roy Eltham\n Hayaki Saito Nathan Reed Won Chun\n Luke Graham Johan Duparc Nick Verigakis the Horde3D community\n Thomas Ruf Ronny Chevalier github:rlyeh\n Janez Zemva John Bartholomew Michal Cichon github:romigrou\n Jonathan Blow Ken Hamada Tero Hanninen github:svdijk\n Eugene Golushkov Laurent Gomila Cort Stratton github:snagar\n Aruelien Pocheville Sergio Gonzalez Thibault Reuille github:Zelex\n Cass Everitt Ryamond Barbiero github:grim210\n Paul Du Bois Engin Manap Aldo Culquicondor github:sammyhw\n Philipp Wiesemann Dale Weiler Oriol Ferrer Mesia github:phprus\n Josh Tobin Neil Bickford Matthew Gregan github:poppolopoppo\n Julian Raschke Gregory Mullen Christian Floisand github:darealshinji\n Baldur Karlsson Kevin Schmidt JR Smith github:Michaelangel007\n Brad Weinberger Matvey Cherevko github:mosra\n Luca Sas Alexander Veselov Zack Middleton [reserved]\n Ryan C. Gordon [reserved] [reserved]\n DO NOT ADD YOUR NAME HERE\n\n Jacko Dirks\n\n To add your name to the credits, pick a random blank space in the middle and fill it.\n 80% of merge conflicts on stb PRs are due to people adding their name at the end\n of the credits.\n*/\n\n#ifndef STBI_INCLUDE_STB_IMAGE_H\n#define STBI_INCLUDE_STB_IMAGE_H\n\n// DOCUMENTATION\n//\n// Limitations:\n// - no 12-bit-per-channel JPEG\n// - no JPEGs with arithmetic coding\n// - GIF always returns *comp=4\n//\n// Basic usage (see HDR discussion below for HDR usage):\n// int x,y,n;\n// unsigned char *data = stbi_load(filename, &x, &y, &n, 0);\n// // ... process data if not NULL ...\n// // ... x = width, y = height, n = # 8-bit components per pixel ...\n// // ... replace '0' with '1'..'4' to force that many components per pixel\n// // ... but 'n' will always be the number that it would have been if you said 0\n// stbi_image_free(data);\n//\n// Standard parameters:\n// int *x -- outputs image width in pixels\n// int *y -- outputs image height in pixels\n// int *channels_in_file -- outputs # of image components in image file\n// int desired_channels -- if non-zero, # of image components requested in result\n//\n// The return value from an image loader is an 'unsigned char *' which points\n// to the pixel data, or NULL on an allocation failure or if the image is\n// corrupt or invalid. The pixel data consists of *y scanlines of *x pixels,\n// with each pixel consisting of N interleaved 8-bit components; the first\n// pixel pointed to is top-left-most in the image. There is no padding between\n// image scanlines or between pixels, regardless of format. The number of\n// components N is 'desired_channels' if desired_channels is non-zero, or\n// *channels_in_file otherwise. If desired_channels is non-zero,\n// *channels_in_file has the number of components that _would_ have been\n// output otherwise. E.g. if you set desired_channels to 4, you will always\n// get RGBA output, but you can check *channels_in_file to see if it's trivially\n// opaque because e.g. there were only 3 channels in the source image.\n//\n// An output image with N components has the following components interleaved\n// in this order in each pixel:\n//\n// N=#comp components\n// 1 grey\n// 2 grey, alpha\n// 3 red, green, blue\n// 4 red, green, blue, alpha\n//\n// If image loading fails for any reason, the return value will be NULL,\n// and *x, *y, *channels_in_file will be unchanged. The function\n// stbi_failure_reason() can be queried for an extremely brief, end-user\n// unfriendly explanation of why the load failed. Define STBI_NO_FAILURE_STRINGS\n// to avoid compiling these strings at all, and STBI_FAILURE_USERMSG to get slightly\n// more user-friendly ones.\n//\n// Paletted PNG, BMP, GIF, and PIC images are automatically depalettized.\n//\n// To query the width, height and component count of an image without having to\n// decode the full file, you can use the stbi_info family of functions:\n//\n// int x,y,n,ok;\n// ok = stbi_info(filename, &x, &y, &n);\n// // returns ok=1 and sets x, y, n if image is a supported format,\n// // 0 otherwise.\n//\n// Note that stb_image pervasively uses ints in its public API for sizes,\n// including sizes of memory buffers. This is now part of the API and thus\n// hard to change without causing breakage. As a result, the various image\n// loaders all have certain limits on image size; these differ somewhat\n// by format but generally boil down to either just under 2GB or just under\n// 1GB. When the decoded image would be larger than this, stb_image decoding\n// will fail.\n//\n// Additionally, stb_image will reject image files that have any of their\n// dimensions set to a larger value than the configurable STBI_MAX_DIMENSIONS,\n// which defaults to 2**24 = 16777216 pixels. Due to the above memory limit,\n// the only way to have an image with such dimensions load correctly\n// is for it to have a rather extreme aspect ratio. Either way, the\n// assumption here is that such larger images are likely to be malformed\n// or malicious. If you do need to load an image with individual dimensions\n// larger than that, and it still fits in the overall size limit, you can\n// #define STBI_MAX_DIMENSIONS on your own to be something larger.\n//\n// ===========================================================================\n//\n// UNICODE:\n//\n// If compiling for Windows and you wish to use Unicode filenames, compile\n// with\n// #define STBI_WINDOWS_UTF8\n// and pass utf8-encoded filenames. Call stbi_convert_wchar_to_utf8 to convert\n// Windows wchar_t filenames to utf8.\n//\n// ===========================================================================\n//\n// Philosophy\n//\n// stb libraries are designed with the following priorities:\n//\n// 1. easy to use\n// 2. easy to maintain\n// 3. good performance\n//\n// Sometimes I let \"good performance\" creep up in priority over \"easy to maintain\",\n// and for best performance I may provide less-easy-to-use APIs that give higher\n// performance, in addition to the easy-to-use ones. Nevertheless, it's important\n// to keep in mind that from the standpoint of you, a client of this library,\n// all you care about is #1 and #3, and stb libraries DO NOT emphasize #3 above all.\n//\n// Some secondary priorities arise directly from the first two, some of which\n// provide more explicit reasons why performance can't be emphasized.\n//\n// - Portable (\"ease of use\")\n// - Small source code footprint (\"easy to maintain\")\n// - No dependencies (\"ease of use\")\n//\n// ===========================================================================\n//\n// I/O callbacks\n//\n// I/O callbacks allow you to read from arbitrary sources, like packaged\n// files or some other source. Data read from callbacks are processed\n// through a small internal buffer (currently 128 bytes) to try to reduce\n// overhead.\n//\n// The three functions you must define are \"read\" (reads some bytes of data),\n// \"skip\" (skips some bytes of data), \"eof\" (reports if the stream is at the end).\n//\n// ===========================================================================\n//\n// SIMD support\n//\n// The JPEG decoder will try to automatically use SIMD kernels on x86 when\n// supported by the compiler. For ARM Neon support, you must explicitly\n// request it.\n//\n// (The old do-it-yourself SIMD API is no longer supported in the current\n// code.)\n//\n// On x86, SSE2 will automatically be used when available based on a run-time\n// test; if not, the generic C versions are used as a fall-back. On ARM targets,\n// the typical path is to have separate builds for NEON and non-NEON devices\n// (at least this is true for iOS and Android). Therefore, the NEON support is\n// toggled by a build flag: define STBI_NEON to get NEON loops.\n//\n// If for some reason you do not want to use any of SIMD code, or if\n// you have issues compiling it, you can disable it entirely by\n// defining STBI_NO_SIMD.\n//\n// ===========================================================================\n//\n// HDR image support (disable by defining STBI_NO_HDR)\n//\n// stb_image supports loading HDR images in general, and currently the Radiance\n// .HDR file format specifically. You can still load any file through the existing\n// interface; if you attempt to load an HDR file, it will be automatically remapped\n// to LDR, assuming gamma 2.2 and an arbitrary scale factor defaulting to 1;\n// both of these constants can be reconfigured through this interface:\n//\n// stbi_hdr_to_ldr_gamma(2.2f);\n// stbi_hdr_to_ldr_scale(1.0f);\n//\n// (note, do not use _inverse_ constants; stbi_image will invert them\n// appropriately).\n//\n// Additionally, there is a new, parallel interface for loading files as\n// (linear) floats to preserve the full dynamic range:\n//\n// float *data = stbi_loadf(filename, &x, &y, &n, 0);\n//\n// If you load LDR images through this interface, those images will\n// be promoted to floating point values, run through the inverse of\n// constants corresponding to the above:\n//\n// stbi_ldr_to_hdr_scale(1.0f);\n// stbi_ldr_to_hdr_gamma(2.2f);\n//\n// Finally, given a filename (or an open file or memory block--see header\n// file for details) containing image data, you can query for the \"most\n// appropriate\" interface to use (that is, whether the image is HDR or\n// not), using:\n//\n// stbi_is_hdr(char *filename);\n//\n// ===========================================================================\n//\n// iPhone PNG support:\n//\n// We optionally support converting iPhone-formatted PNGs (which store\n// premultiplied BGRA) back to RGB, even though they're internally encoded\n// differently. To enable this conversion, call\n// stbi_convert_iphone_png_to_rgb(1).\n//\n// Call stbi_set_unpremultiply_on_load(1) as well to force a divide per\n// pixel to remove any premultiplied alpha *only* if the image file explicitly\n// says there's premultiplied data (currently only happens in iPhone images,\n// and only if iPhone convert-to-rgb processing is on).\n//\n// ===========================================================================\n//\n// ADDITIONAL CONFIGURATION\n//\n// - You can suppress implementation of any of the decoders to reduce\n// your code footprint by #defining one or more of the following\n// symbols before creating the implementation.\n//\n// STBI_NO_JPEG\n// STBI_NO_PNG\n// STBI_NO_BMP\n// STBI_NO_PSD\n// STBI_NO_TGA\n// STBI_NO_GIF\n// STBI_NO_HDR\n// STBI_NO_PIC\n// STBI_NO_PNM (.ppm and .pgm)\n//\n// - You can request *only* certain decoders and suppress all other ones\n// (this will be more forward-compatible, as addition of new decoders\n// doesn't require you to disable them explicitly):\n//\n// STBI_ONLY_JPEG\n// STBI_ONLY_PNG\n// STBI_ONLY_BMP\n// STBI_ONLY_PSD\n// STBI_ONLY_TGA\n// STBI_ONLY_GIF\n// STBI_ONLY_HDR\n// STBI_ONLY_PIC\n// STBI_ONLY_PNM (.ppm and .pgm)\n//\n// - If you use STBI_NO_PNG (or _ONLY_ without PNG), and you still\n// want the zlib decoder to be available, #define STBI_SUPPORT_ZLIB\n//\n// - If you define STBI_MAX_DIMENSIONS, stb_image will reject images greater\n// than that size (in either width or height) without further processing.\n// This is to let programs in the wild set an upper bound to prevent\n// denial-of-service attacks on untrusted data, as one could generate a\n// valid image of gigantic dimensions and force stb_image to allocate a\n// huge block of memory and spend disproportionate time decoding it. By\n// default this is set to (1 << 24), which is 16777216, but that's still\n// very big.\n\n#ifndef STBI_NO_STDIO\n#include \n#endif // STBI_NO_STDIO\n\n#define STBI_VERSION 1\n\nenum\n{\n STBI_default = 0, // only used for desired_channels\n\n STBI_grey = 1,\n STBI_grey_alpha = 2,\n STBI_rgb = 3,\n STBI_rgb_alpha = 4\n};\n\n#include \ntypedef unsigned char stbi_uc;\ntypedef unsigned short stbi_us;\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#ifndef STBIDEF\n#ifdef STB_IMAGE_STATIC\n#define STBIDEF static\n#else\n#define STBIDEF extern\n#endif\n#endif\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// PRIMARY API - works on images of any type\n//\n\n//\n// load image by filename, open file, or memory buffer\n//\n\ntypedef struct\n{\n int (*read) (void *user,char *data,int size); // fill 'data' with 'size' bytes. return number of bytes actually read\n void (*skip) (void *user,int n); // skip the next 'n' bytes, or 'unget' the last -n bytes if negative\n int (*eof) (void *user); // returns nonzero if we are at end of file/data\n} stbi_io_callbacks;\n\n////////////////////////////////////\n//\n// 8-bits-per-channel interface\n//\n\nSTBIDEF stbi_uc *stbi_load_from_memory (stbi_uc const *buffer, int len , int *x, int *y, int *channels_in_file, int desired_channels);\nSTBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk , void *user, int *x, int *y, int *channels_in_file, int desired_channels);\n\n#ifndef STBI_NO_STDIO\nSTBIDEF stbi_uc *stbi_load (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels);\nSTBIDEF stbi_uc *stbi_load_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels);\n// for stbi_load_from_file, file pointer is left pointing immediately after image\n#endif\n\n#ifndef STBI_NO_GIF\nSTBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp);\n#endif\n\n#ifdef STBI_WINDOWS_UTF8\nSTBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input);\n#endif\n\n////////////////////////////////////\n//\n// 16-bits-per-channel interface\n//\n\nSTBIDEF stbi_us *stbi_load_16_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels);\nSTBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels);\n\n#ifndef STBI_NO_STDIO\nSTBIDEF stbi_us *stbi_load_16 (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels);\nSTBIDEF stbi_us *stbi_load_from_file_16(FILE *f, int *x, int *y, int *channels_in_file, int desired_channels);\n#endif\n\n////////////////////////////////////\n//\n// float-per-channel interface\n//\n#ifndef STBI_NO_LINEAR\n STBIDEF float *stbi_loadf_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels);\n STBIDEF float *stbi_loadf_from_callbacks (stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels);\n\n #ifndef STBI_NO_STDIO\n STBIDEF float *stbi_loadf (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels);\n STBIDEF float *stbi_loadf_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels);\n #endif\n#endif\n\n#ifndef STBI_NO_HDR\n STBIDEF void stbi_hdr_to_ldr_gamma(float gamma);\n STBIDEF void stbi_hdr_to_ldr_scale(float scale);\n#endif // STBI_NO_HDR\n\n#ifndef STBI_NO_LINEAR\n STBIDEF void stbi_ldr_to_hdr_gamma(float gamma);\n STBIDEF void stbi_ldr_to_hdr_scale(float scale);\n#endif // STBI_NO_LINEAR\n\n// stbi_is_hdr is always defined, but always returns false if STBI_NO_HDR\nSTBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user);\nSTBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len);\n#ifndef STBI_NO_STDIO\nSTBIDEF int stbi_is_hdr (char const *filename);\nSTBIDEF int stbi_is_hdr_from_file(FILE *f);\n#endif // STBI_NO_STDIO\n\n\n// get a VERY brief reason for failure\n// on most compilers (and ALL modern mainstream compilers) this is threadsafe\nSTBIDEF const char *stbi_failure_reason (void);\n\n// free the loaded image -- this is just free()\nSTBIDEF void stbi_image_free (void *retval_from_stbi_load);\n\n// get image dimensions & components without fully decoding\nSTBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp);\nSTBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp);\nSTBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len);\nSTBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *clbk, void *user);\n\n#ifndef STBI_NO_STDIO\nSTBIDEF int stbi_info (char const *filename, int *x, int *y, int *comp);\nSTBIDEF int stbi_info_from_file (FILE *f, int *x, int *y, int *comp);\nSTBIDEF int stbi_is_16_bit (char const *filename);\nSTBIDEF int stbi_is_16_bit_from_file(FILE *f);\n#endif\n\n\n\n// for image formats that explicitly notate that they have premultiplied alpha,\n// we just return the colors as stored in the file. set this flag to force\n// unpremultiplication. results are undefined if the unpremultiply overflow.\nSTBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply);\n\n// indicate whether we should process iphone images back to canonical format,\n// or just pass them through \"as-is\"\nSTBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert);\n\n// flip the image vertically, so the first pixel in the output array is the bottom left\nSTBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip);\n\n// as above, but only applies to images loaded on the thread that calls the function\n// this function is only available if your compiler supports thread-local variables;\n// calling it will fail to link if your compiler doesn't\nSTBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply);\nSTBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert);\nSTBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip);\n\n// ZLIB client - used by PNG, available for other purposes\n\nSTBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen);\nSTBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header);\nSTBIDEF char *stbi_zlib_decode_malloc(const char *buffer, int len, int *outlen);\nSTBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, const char *ibuffer, int ilen);\n\nSTBIDEF char *stbi_zlib_decode_noheader_malloc(const char *buffer, int len, int *outlen);\nSTBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen);\n\n\n#ifdef __cplusplus\n}\n#endif\n\n//\n//\n//// end header file /////////////////////////////////////////////////////\n#endif // STBI_INCLUDE_STB_IMAGE_H\n\n#ifdef STB_IMAGE_IMPLEMENTATION\n\n#if defined(STBI_ONLY_JPEG) || defined(STBI_ONLY_PNG) || defined(STBI_ONLY_BMP) \\\n || defined(STBI_ONLY_TGA) || defined(STBI_ONLY_GIF) || defined(STBI_ONLY_PSD) \\\n || defined(STBI_ONLY_HDR) || defined(STBI_ONLY_PIC) || defined(STBI_ONLY_PNM) \\\n || defined(STBI_ONLY_ZLIB)\n #ifndef STBI_ONLY_JPEG\n #define STBI_NO_JPEG\n #endif\n #ifndef STBI_ONLY_PNG\n #define STBI_NO_PNG\n #endif\n #ifndef STBI_ONLY_BMP\n #define STBI_NO_BMP\n #endif\n #ifndef STBI_ONLY_PSD\n #define STBI_NO_PSD\n #endif\n #ifndef STBI_ONLY_TGA\n #define STBI_NO_TGA\n #endif\n #ifndef STBI_ONLY_GIF\n #define STBI_NO_GIF\n #endif\n #ifndef STBI_ONLY_HDR\n #define STBI_NO_HDR\n #endif\n #ifndef STBI_ONLY_PIC\n #define STBI_NO_PIC\n #endif\n #ifndef STBI_ONLY_PNM\n #define STBI_NO_PNM\n #endif\n#endif\n\n#if defined(STBI_NO_PNG) && !defined(STBI_SUPPORT_ZLIB) && !defined(STBI_NO_ZLIB)\n#define STBI_NO_ZLIB\n#endif\n\n\n#include \n#include // ptrdiff_t on osx\n#include \n#include \n#include \n\n#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR)\n#include // ldexp, pow\n#endif\n\n#ifndef STBI_NO_STDIO\n#include \n#endif\n\n#ifndef STBI_ASSERT\n#include \n#define STBI_ASSERT(x) assert(x)\n#endif\n\n#ifdef __cplusplus\n#define STBI_EXTERN extern \"C\"\n#else\n#define STBI_EXTERN extern\n#endif\n\n\n#ifndef _MSC_VER\n #ifdef __cplusplus\n #define stbi_inline inline\n #else\n #define stbi_inline\n #endif\n#else\n #define stbi_inline __forceinline\n#endif\n\n#ifndef STBI_NO_THREAD_LOCALS\n #if defined(__cplusplus) && __cplusplus >= 201103L\n #define STBI_THREAD_LOCAL thread_local\n #elif defined(__GNUC__) && __GNUC__ < 5\n #define STBI_THREAD_LOCAL __thread\n #elif defined(_MSC_VER)\n #define STBI_THREAD_LOCAL __declspec(thread)\n #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__)\n #define STBI_THREAD_LOCAL _Thread_local\n #endif\n\n #ifndef STBI_THREAD_LOCAL\n #if defined(__GNUC__)\n #define STBI_THREAD_LOCAL __thread\n #endif\n #endif\n#endif\n\n#if defined(_MSC_VER) || defined(__SYMBIAN32__)\ntypedef unsigned short stbi__uint16;\ntypedef signed short stbi__int16;\ntypedef unsigned int stbi__uint32;\ntypedef signed int stbi__int32;\n#else\n#include \ntypedef uint16_t stbi__uint16;\ntypedef int16_t stbi__int16;\ntypedef uint32_t stbi__uint32;\ntypedef int32_t stbi__int32;\n#endif\n\n// should produce compiler error if size is wrong\ntypedef unsigned char validate_uint32[sizeof(stbi__uint32)==4 ? 1 : -1];\n\n#ifdef _MSC_VER\n#define STBI_NOTUSED(v) (void)(v)\n#else\n#define STBI_NOTUSED(v) (void)sizeof(v)\n#endif\n\n#ifdef _MSC_VER\n#define STBI_HAS_LROTL\n#endif\n\n#ifdef STBI_HAS_LROTL\n #define stbi_lrot(x,y) _lrotl(x,y)\n#else\n #define stbi_lrot(x,y) (((x) << (y)) | ((x) >> (-(y) & 31)))\n#endif\n\n#if defined(STBI_MALLOC) && defined(STBI_FREE) && (defined(STBI_REALLOC) || defined(STBI_REALLOC_SIZED))\n// ok\n#elif !defined(STBI_MALLOC) && !defined(STBI_FREE) && !defined(STBI_REALLOC) && !defined(STBI_REALLOC_SIZED)\n// ok\n#else\n#error \"Must define all or none of STBI_MALLOC, STBI_FREE, and STBI_REALLOC (or STBI_REALLOC_SIZED).\"\n#endif\n\n#ifndef STBI_MALLOC\n#define STBI_MALLOC(sz) malloc(sz)\n#define STBI_REALLOC(p,newsz) realloc(p,newsz)\n#define STBI_FREE(p) free(p)\n#endif\n\n#ifndef STBI_REALLOC_SIZED\n#define STBI_REALLOC_SIZED(p,oldsz,newsz) STBI_REALLOC(p,newsz)\n#endif\n\n// x86/x64 detection\n#if defined(__x86_64__) || defined(_M_X64)\n#define STBI__X64_TARGET\n#elif defined(__i386) || defined(_M_IX86)\n#define STBI__X86_TARGET\n#endif\n\n#if defined(__GNUC__) && defined(STBI__X86_TARGET) && !defined(__SSE2__) && !defined(STBI_NO_SIMD)\n// gcc doesn't support sse2 intrinsics unless you compile with -msse2,\n// which in turn means it gets to use SSE2 everywhere. This is unfortunate,\n// but previous attempts to provide the SSE2 functions with runtime\n// detection caused numerous issues. The way architecture extensions are\n// exposed in GCC/Clang is, sadly, not really suited for one-file libs.\n// New behavior: if compiled with -msse2, we use SSE2 without any\n// detection; if not, we don't use it at all.\n#define STBI_NO_SIMD\n#endif\n\n#if defined(__MINGW32__) && defined(STBI__X86_TARGET) && !defined(STBI_MINGW_ENABLE_SSE2) && !defined(STBI_NO_SIMD)\n// Note that __MINGW32__ doesn't actually mean 32-bit, so we have to avoid STBI__X64_TARGET\n//\n// 32-bit MinGW wants ESP to be 16-byte aligned, but this is not in the\n// Windows ABI and VC++ as well as Windows DLLs don't maintain that invariant.\n// As a result, enabling SSE2 on 32-bit MinGW is dangerous when not\n// simultaneously enabling \"-mstackrealign\".\n//\n// See https://github.com/nothings/stb/issues/81 for more information.\n//\n// So default to no SSE2 on 32-bit MinGW. If you've read this far and added\n// -mstackrealign to your build settings, feel free to #define STBI_MINGW_ENABLE_SSE2.\n#define STBI_NO_SIMD\n#endif\n\n#if !defined(STBI_NO_SIMD) && (defined(STBI__X86_TARGET) || defined(STBI__X64_TARGET))\n#define STBI_SSE2\n#include \n\n#ifdef _MSC_VER\n\n#if _MSC_VER >= 1400 // not VC6\n#include // __cpuid\nstatic int stbi__cpuid3(void)\n{\n int info[4];\n __cpuid(info,1);\n return info[3];\n}\n#else\nstatic int stbi__cpuid3(void)\n{\n int res;\n __asm {\n mov eax,1\n cpuid\n mov res,edx\n }\n return res;\n}\n#endif\n\n#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name\n\n#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2)\nstatic int stbi__sse2_available(void)\n{\n int info3 = stbi__cpuid3();\n return ((info3 >> 26) & 1) != 0;\n}\n#endif\n\n#else // assume GCC-style if not VC++\n#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16)))\n\n#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2)\nstatic int stbi__sse2_available(void)\n{\n // If we're even attempting to compile this on GCC/Clang, that means\n // -msse2 is on, which means the compiler is allowed to use SSE2\n // instructions at will, and so are we.\n return 1;\n}\n#endif\n\n#endif\n#endif\n\n// ARM NEON\n#if defined(STBI_NO_SIMD) && defined(STBI_NEON)\n#undef STBI_NEON\n#endif\n\n#ifdef STBI_NEON\n#include \n#ifdef _MSC_VER\n#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name\n#else\n#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16)))\n#endif\n#endif\n\n#ifndef STBI_SIMD_ALIGN\n#define STBI_SIMD_ALIGN(type, name) type name\n#endif\n\n#ifndef STBI_MAX_DIMENSIONS\n#define STBI_MAX_DIMENSIONS (1 << 24)\n#endif\n\n///////////////////////////////////////////////\n//\n// stbi__context struct and start_xxx functions\n\n// stbi__context structure is our basic context used by all images, so it\n// contains all the IO context, plus some basic image information\ntypedef struct\n{\n stbi__uint32 img_x, img_y;\n int img_n, img_out_n;\n\n stbi_io_callbacks io;\n void *io_user_data;\n\n int read_from_callbacks;\n int buflen;\n stbi_uc buffer_start[128];\n int callback_already_read;\n\n stbi_uc *img_buffer, *img_buffer_end;\n stbi_uc *img_buffer_original, *img_buffer_original_end;\n} stbi__context;\n\n\nstatic void stbi__refill_buffer(stbi__context *s);\n\n// initialize a memory-decode context\nstatic void stbi__start_mem(stbi__context *s, stbi_uc const *buffer, int len)\n{\n s->io.read = NULL;\n s->read_from_callbacks = 0;\n s->callback_already_read = 0;\n s->img_buffer = s->img_buffer_original = (stbi_uc *) buffer;\n s->img_buffer_end = s->img_buffer_original_end = (stbi_uc *) buffer+len;\n}\n\n// initialize a callback-based context\nstatic void stbi__start_callbacks(stbi__context *s, stbi_io_callbacks *c, void *user)\n{\n s->io = *c;\n s->io_user_data = user;\n s->buflen = sizeof(s->buffer_start);\n s->read_from_callbacks = 1;\n s->callback_already_read = 0;\n s->img_buffer = s->img_buffer_original = s->buffer_start;\n stbi__refill_buffer(s);\n s->img_buffer_original_end = s->img_buffer_end;\n}\n\n#ifndef STBI_NO_STDIO\n\nstatic int stbi__stdio_read(void *user, char *data, int size)\n{\n return (int) fread(data,1,size,(FILE*) user);\n}\n\nstatic void stbi__stdio_skip(void *user, int n)\n{\n int ch;\n fseek((FILE*) user, n, SEEK_CUR);\n ch = fgetc((FILE*) user); /* have to read a byte to reset feof()'s flag */\n if (ch != EOF) {\n ungetc(ch, (FILE *) user); /* push byte back onto stream if valid. */\n }\n}\n\nstatic int stbi__stdio_eof(void *user)\n{\n return feof((FILE*) user) || ferror((FILE *) user);\n}\n\nstatic stbi_io_callbacks stbi__stdio_callbacks =\n{\n stbi__stdio_read,\n stbi__stdio_skip,\n stbi__stdio_eof,\n};\n\nstatic void stbi__start_file(stbi__context *s, FILE *f)\n{\n stbi__start_callbacks(s, &stbi__stdio_callbacks, (void *) f);\n}\n\n//static void stop_file(stbi__context *s) { }\n\n#endif // !STBI_NO_STDIO\n\nstatic void stbi__rewind(stbi__context *s)\n{\n // conceptually rewind SHOULD rewind to the beginning of the stream,\n // but we just rewind to the beginning of the initial buffer, because\n // we only use it after doing 'test', which only ever looks at at most 92 bytes\n s->img_buffer = s->img_buffer_original;\n s->img_buffer_end = s->img_buffer_original_end;\n}\n\nenum\n{\n STBI_ORDER_RGB,\n STBI_ORDER_BGR\n};\n\ntypedef struct\n{\n int bits_per_channel;\n int num_channels;\n int channel_order;\n} stbi__result_info;\n\n#ifndef STBI_NO_JPEG\nstatic int stbi__jpeg_test(stbi__context *s);\nstatic void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);\nstatic int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp);\n#endif\n\n#ifndef STBI_NO_PNG\nstatic int stbi__png_test(stbi__context *s);\nstatic void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);\nstatic int stbi__png_info(stbi__context *s, int *x, int *y, int *comp);\nstatic int stbi__png_is16(stbi__context *s);\n#endif\n\n#ifndef STBI_NO_BMP\nstatic int stbi__bmp_test(stbi__context *s);\nstatic void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);\nstatic int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp);\n#endif\n\n#ifndef STBI_NO_TGA\nstatic int stbi__tga_test(stbi__context *s);\nstatic void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);\nstatic int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp);\n#endif\n\n#ifndef STBI_NO_PSD\nstatic int stbi__psd_test(stbi__context *s);\nstatic void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc);\nstatic int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp);\nstatic int stbi__psd_is16(stbi__context *s);\n#endif\n\n#ifndef STBI_NO_HDR\nstatic int stbi__hdr_test(stbi__context *s);\nstatic float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);\nstatic int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp);\n#endif\n\n#ifndef STBI_NO_PIC\nstatic int stbi__pic_test(stbi__context *s);\nstatic void *stbi__pic_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);\nstatic int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp);\n#endif\n\n#ifndef STBI_NO_GIF\nstatic int stbi__gif_test(stbi__context *s);\nstatic void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);\nstatic void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp);\nstatic int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp);\n#endif\n\n#ifndef STBI_NO_PNM\nstatic int stbi__pnm_test(stbi__context *s);\nstatic void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);\nstatic int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp);\nstatic int stbi__pnm_is16(stbi__context *s);\n#endif\n\nstatic\n#ifdef STBI_THREAD_LOCAL\nSTBI_THREAD_LOCAL\n#endif\nconst char *stbi__g_failure_reason;\n\nSTBIDEF const char *stbi_failure_reason(void)\n{\n return stbi__g_failure_reason;\n}\n\n#ifndef STBI_NO_FAILURE_STRINGS\nstatic int stbi__err(const char *str)\n{\n stbi__g_failure_reason = str;\n return 0;\n}\n#endif\n\nstatic void *stbi__malloc(size_t size)\n{\n return STBI_MALLOC(size);\n}\n\n// stb_image uses ints pervasively, including for offset calculations.\n// therefore the largest decoded image size we can support with the\n// current code, even on 64-bit targets, is INT_MAX. this is not a\n// significant limitation for the intended use case.\n//\n// we do, however, need to make sure our size calculations don't\n// overflow. hence a few helper functions for size calculations that\n// multiply integers together, making sure that they're non-negative\n// and no overflow occurs.\n\n// return 1 if the sum is valid, 0 on overflow.\n// negative terms are considered invalid.\nstatic int stbi__addsizes_valid(int a, int b)\n{\n if (b < 0) return 0;\n // now 0 <= b <= INT_MAX, hence also\n // 0 <= INT_MAX - b <= INTMAX.\n // And \"a + b <= INT_MAX\" (which might overflow) is the\n // same as a <= INT_MAX - b (no overflow)\n return a <= INT_MAX - b;\n}\n\n// returns 1 if the product is valid, 0 on overflow.\n// negative factors are considered invalid.\nstatic int stbi__mul2sizes_valid(int a, int b)\n{\n if (a < 0 || b < 0) return 0;\n if (b == 0) return 1; // mul-by-0 is always safe\n // portable way to check for no overflows in a*b\n return a <= INT_MAX/b;\n}\n\n#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR)\n// returns 1 if \"a*b + add\" has no negative terms/factors and doesn't overflow\nstatic int stbi__mad2sizes_valid(int a, int b, int add)\n{\n return stbi__mul2sizes_valid(a, b) && stbi__addsizes_valid(a*b, add);\n}\n#endif\n\n// returns 1 if \"a*b*c + add\" has no negative terms/factors and doesn't overflow\nstatic int stbi__mad3sizes_valid(int a, int b, int c, int add)\n{\n return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) &&\n stbi__addsizes_valid(a*b*c, add);\n}\n\n// returns 1 if \"a*b*c*d + add\" has no negative terms/factors and doesn't overflow\n#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM)\nstatic int stbi__mad4sizes_valid(int a, int b, int c, int d, int add)\n{\n return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) &&\n stbi__mul2sizes_valid(a*b*c, d) && stbi__addsizes_valid(a*b*c*d, add);\n}\n#endif\n\n#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR)\n// mallocs with size overflow checking\nstatic void *stbi__malloc_mad2(int a, int b, int add)\n{\n if (!stbi__mad2sizes_valid(a, b, add)) return NULL;\n return stbi__malloc(a*b + add);\n}\n#endif\n\nstatic void *stbi__malloc_mad3(int a, int b, int c, int add)\n{\n if (!stbi__mad3sizes_valid(a, b, c, add)) return NULL;\n return stbi__malloc(a*b*c + add);\n}\n\n#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM)\nstatic void *stbi__malloc_mad4(int a, int b, int c, int d, int add)\n{\n if (!stbi__mad4sizes_valid(a, b, c, d, add)) return NULL;\n return stbi__malloc(a*b*c*d + add);\n}\n#endif\n\n// returns 1 if the sum of two signed ints is valid (between -2^31 and 2^31-1 inclusive), 0 on overflow.\nstatic int stbi__addints_valid(int a, int b)\n{\n if ((a >= 0) != (b >= 0)) return 1; // a and b have different signs, so no overflow\n if (a < 0 && b < 0) return a >= INT_MIN - b; // same as a + b >= INT_MIN; INT_MIN - b cannot overflow since b < 0.\n return a <= INT_MAX - b;\n}\n\n// returns 1 if the product of two signed shorts is valid, 0 on overflow.\nstatic int stbi__mul2shorts_valid(short a, short b)\n{\n if (b == 0 || b == -1) return 1; // multiplication by 0 is always 0; check for -1 so SHRT_MIN/b doesn't overflow\n if ((a >= 0) == (b >= 0)) return a <= SHRT_MAX/b; // product is positive, so similar to mul2sizes_valid\n if (b < 0) return a <= SHRT_MIN / b; // same as a * b >= SHRT_MIN\n return a >= SHRT_MIN / b;\n}\n\n// stbi__err - error\n// stbi__errpf - error returning pointer to float\n// stbi__errpuc - error returning pointer to unsigned char\n\n#ifdef STBI_NO_FAILURE_STRINGS\n #define stbi__err(x,y) 0\n#elif defined(STBI_FAILURE_USERMSG)\n #define stbi__err(x,y) stbi__err(y)\n#else\n #define stbi__err(x,y) stbi__err(x)\n#endif\n\n#define stbi__errpf(x,y) ((float *)(size_t) (stbi__err(x,y)?NULL:NULL))\n#define stbi__errpuc(x,y) ((unsigned char *)(size_t) (stbi__err(x,y)?NULL:NULL))\n\nSTBIDEF void stbi_image_free(void *retval_from_stbi_load)\n{\n STBI_FREE(retval_from_stbi_load);\n}\n\n#ifndef STBI_NO_LINEAR\nstatic float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp);\n#endif\n\n#ifndef STBI_NO_HDR\nstatic stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp);\n#endif\n\nstatic int stbi__vertically_flip_on_load_global = 0;\n\nSTBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip)\n{\n stbi__vertically_flip_on_load_global = flag_true_if_should_flip;\n}\n\n#ifndef STBI_THREAD_LOCAL\n#define stbi__vertically_flip_on_load stbi__vertically_flip_on_load_global\n#else\nstatic STBI_THREAD_LOCAL int stbi__vertically_flip_on_load_local, stbi__vertically_flip_on_load_set;\n\nSTBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip)\n{\n stbi__vertically_flip_on_load_local = flag_true_if_should_flip;\n stbi__vertically_flip_on_load_set = 1;\n}\n\n#define stbi__vertically_flip_on_load (stbi__vertically_flip_on_load_set \\\n ? stbi__vertically_flip_on_load_local \\\n : stbi__vertically_flip_on_load_global)\n#endif // STBI_THREAD_LOCAL\n\nstatic void *stbi__load_main(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc)\n{\n memset(ri, 0, sizeof(*ri)); // make sure it's initialized if we add new fields\n ri->bits_per_channel = 8; // default is 8 so most paths don't have to be changed\n ri->channel_order = STBI_ORDER_RGB; // all current input & output are this, but this is here so we can add BGR order\n ri->num_channels = 0;\n\n // test the formats with a very explicit header first (at least a FOURCC\n // or distinctive magic number first)\n #ifndef STBI_NO_PNG\n if (stbi__png_test(s)) return stbi__png_load(s,x,y,comp,req_comp, ri);\n #endif\n #ifndef STBI_NO_BMP\n if (stbi__bmp_test(s)) return stbi__bmp_load(s,x,y,comp,req_comp, ri);\n #endif\n #ifndef STBI_NO_GIF\n if (stbi__gif_test(s)) return stbi__gif_load(s,x,y,comp,req_comp, ri);\n #endif\n #ifndef STBI_NO_PSD\n if (stbi__psd_test(s)) return stbi__psd_load(s,x,y,comp,req_comp, ri, bpc);\n #else\n STBI_NOTUSED(bpc);\n #endif\n #ifndef STBI_NO_PIC\n if (stbi__pic_test(s)) return stbi__pic_load(s,x,y,comp,req_comp, ri);\n #endif\n\n // then the formats that can end up attempting to load with just 1 or 2\n // bytes matching expectations; these are prone to false positives, so\n // try them later\n #ifndef STBI_NO_JPEG\n if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp, ri);\n #endif\n #ifndef STBI_NO_PNM\n if (stbi__pnm_test(s)) return stbi__pnm_load(s,x,y,comp,req_comp, ri);\n #endif\n\n #ifndef STBI_NO_HDR\n if (stbi__hdr_test(s)) {\n float *hdr = stbi__hdr_load(s, x,y,comp,req_comp, ri);\n return stbi__hdr_to_ldr(hdr, *x, *y, req_comp ? req_comp : *comp);\n }\n #endif\n\n #ifndef STBI_NO_TGA\n // test tga last because it's a crappy test!\n if (stbi__tga_test(s))\n return stbi__tga_load(s,x,y,comp,req_comp, ri);\n #endif\n\n return stbi__errpuc(\"unknown image type\", \"Image not of any known type, or corrupt\");\n}\n\nstatic stbi_uc *stbi__convert_16_to_8(stbi__uint16 *orig, int w, int h, int channels)\n{\n int i;\n int img_len = w * h * channels;\n stbi_uc *reduced;\n\n reduced = (stbi_uc *) stbi__malloc(img_len);\n if (reduced == NULL) return stbi__errpuc(\"outofmem\", \"Out of memory\");\n\n for (i = 0; i < img_len; ++i)\n reduced[i] = (stbi_uc)((orig[i] >> 8) & 0xFF); // top half of each byte is sufficient approx of 16->8 bit scaling\n\n STBI_FREE(orig);\n return reduced;\n}\n\nstatic stbi__uint16 *stbi__convert_8_to_16(stbi_uc *orig, int w, int h, int channels)\n{\n int i;\n int img_len = w * h * channels;\n stbi__uint16 *enlarged;\n\n enlarged = (stbi__uint16 *) stbi__malloc(img_len*2);\n if (enlarged == NULL) return (stbi__uint16 *) stbi__errpuc(\"outofmem\", \"Out of memory\");\n\n for (i = 0; i < img_len; ++i)\n enlarged[i] = (stbi__uint16)((orig[i] << 8) + orig[i]); // replicate to high and low byte, maps 0->0, 255->0xffff\n\n STBI_FREE(orig);\n return enlarged;\n}\n\nstatic void stbi__vertical_flip(void *image, int w, int h, int bytes_per_pixel)\n{\n int row;\n size_t bytes_per_row = (size_t)w * bytes_per_pixel;\n stbi_uc temp[2048];\n stbi_uc *bytes = (stbi_uc *)image;\n\n for (row = 0; row < (h>>1); row++) {\n stbi_uc *row0 = bytes + row*bytes_per_row;\n stbi_uc *row1 = bytes + (h - row - 1)*bytes_per_row;\n // swap row0 with row1\n size_t bytes_left = bytes_per_row;\n while (bytes_left) {\n size_t bytes_copy = (bytes_left < sizeof(temp)) ? bytes_left : sizeof(temp);\n memcpy(temp, row0, bytes_copy);\n memcpy(row0, row1, bytes_copy);\n memcpy(row1, temp, bytes_copy);\n row0 += bytes_copy;\n row1 += bytes_copy;\n bytes_left -= bytes_copy;\n }\n }\n}\n\n#ifndef STBI_NO_GIF\nstatic void stbi__vertical_flip_slices(void *image, int w, int h, int z, int bytes_per_pixel)\n{\n int slice;\n int slice_size = w * h * bytes_per_pixel;\n\n stbi_uc *bytes = (stbi_uc *)image;\n for (slice = 0; slice < z; ++slice) {\n stbi__vertical_flip(bytes, w, h, bytes_per_pixel);\n bytes += slice_size;\n }\n}\n#endif\n\nstatic unsigned char *stbi__load_and_postprocess_8bit(stbi__context *s, int *x, int *y, int *comp, int req_comp)\n{\n stbi__result_info ri;\n void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 8);\n\n if (result == NULL)\n return NULL;\n\n // it is the responsibility of the loaders to make sure we get either 8 or 16 bit.\n STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16);\n\n if (ri.bits_per_channel != 8) {\n result = stbi__convert_16_to_8((stbi__uint16 *) result, *x, *y, req_comp == 0 ? *comp : req_comp);\n ri.bits_per_channel = 8;\n }\n\n // @TODO: move stbi__convert_format to here\n\n if (stbi__vertically_flip_on_load) {\n int channels = req_comp ? req_comp : *comp;\n stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi_uc));\n }\n\n return (unsigned char *) result;\n}\n\nstatic stbi__uint16 *stbi__load_and_postprocess_16bit(stbi__context *s, int *x, int *y, int *comp, int req_comp)\n{\n stbi__result_info ri;\n void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 16);\n\n if (result == NULL)\n return NULL;\n\n // it is the responsibility of the loaders to make sure we get either 8 or 16 bit.\n STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16);\n\n if (ri.bits_per_channel != 16) {\n result = stbi__convert_8_to_16((stbi_uc *) result, *x, *y, req_comp == 0 ? *comp : req_comp);\n ri.bits_per_channel = 16;\n }\n\n // @TODO: move stbi__convert_format16 to here\n // @TODO: special case RGB-to-Y (and RGBA-to-YA) for 8-bit-to-16-bit case to keep more precision\n\n if (stbi__vertically_flip_on_load) {\n int channels = req_comp ? req_comp : *comp;\n stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi__uint16));\n }\n\n return (stbi__uint16 *) result;\n}\n\n#if !defined(STBI_NO_HDR) && !defined(STBI_NO_LINEAR)\nstatic void stbi__float_postprocess(float *result, int *x, int *y, int *comp, int req_comp)\n{\n if (stbi__vertically_flip_on_load && result != NULL) {\n int channels = req_comp ? req_comp : *comp;\n stbi__vertical_flip(result, *x, *y, channels * sizeof(float));\n }\n}\n#endif\n\n#ifndef STBI_NO_STDIO\n\n#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8)\nSTBI_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide);\nSTBI_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default);\n#endif\n\n#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8)\nSTBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input)\n{\n\treturn WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL);\n}\n#endif\n\nstatic FILE *stbi__fopen(char const *filename, char const *mode)\n{\n FILE *f;\n#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8)\n wchar_t wMode[64];\n wchar_t wFilename[1024];\n\tif (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename)))\n return 0;\n\n\tif (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode)))\n return 0;\n\n#if defined(_MSC_VER) && _MSC_VER >= 1400\n\tif (0 != _wfopen_s(&f, wFilename, wMode))\n\t\tf = 0;\n#else\n f = _wfopen(wFilename, wMode);\n#endif\n\n#elif defined(_MSC_VER) && _MSC_VER >= 1400\n if (0 != fopen_s(&f, filename, mode))\n f=0;\n#else\n f = fopen(filename, mode);\n#endif\n return f;\n}\n\n\nSTBIDEF stbi_uc *stbi_load(char const *filename, int *x, int *y, int *comp, int req_comp)\n{\n FILE *f = stbi__fopen(filename, \"rb\");\n unsigned char *result;\n if (!f) return stbi__errpuc(\"can't fopen\", \"Unable to open file\");\n result = stbi_load_from_file(f,x,y,comp,req_comp);\n fclose(f);\n return result;\n}\n\nSTBIDEF stbi_uc *stbi_load_from_file(FILE *f, int *x, int *y, int *comp, int req_comp)\n{\n unsigned char *result;\n stbi__context s;\n stbi__start_file(&s,f);\n result = stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp);\n if (result) {\n // need to 'unget' all the characters in the IO buffer\n fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR);\n }\n return result;\n}\n\nSTBIDEF stbi__uint16 *stbi_load_from_file_16(FILE *f, int *x, int *y, int *comp, int req_comp)\n{\n stbi__uint16 *result;\n stbi__context s;\n stbi__start_file(&s,f);\n result = stbi__load_and_postprocess_16bit(&s,x,y,comp,req_comp);\n if (result) {\n // need to 'unget' all the characters in the IO buffer\n fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR);\n }\n return result;\n}\n\nSTBIDEF stbi_us *stbi_load_16(char const *filename, int *x, int *y, int *comp, int req_comp)\n{\n FILE *f = stbi__fopen(filename, \"rb\");\n stbi__uint16 *result;\n if (!f) return (stbi_us *) stbi__errpuc(\"can't fopen\", \"Unable to open file\");\n result = stbi_load_from_file_16(f,x,y,comp,req_comp);\n fclose(f);\n return result;\n}\n\n\n#endif //!STBI_NO_STDIO\n\nSTBIDEF stbi_us *stbi_load_16_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels)\n{\n stbi__context s;\n stbi__start_mem(&s,buffer,len);\n return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels);\n}\n\nSTBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels)\n{\n stbi__context s;\n stbi__start_callbacks(&s, (stbi_io_callbacks *)clbk, user);\n return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels);\n}\n\nSTBIDEF stbi_uc *stbi_load_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp)\n{\n stbi__context s;\n stbi__start_mem(&s,buffer,len);\n return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp);\n}\n\nSTBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp)\n{\n stbi__context s;\n stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user);\n return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp);\n}\n\n#ifndef STBI_NO_GIF\nSTBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp)\n{\n unsigned char *result;\n stbi__context s;\n stbi__start_mem(&s,buffer,len);\n\n result = (unsigned char*) stbi__load_gif_main(&s, delays, x, y, z, comp, req_comp);\n if (stbi__vertically_flip_on_load) {\n stbi__vertical_flip_slices( result, *x, *y, *z, *comp );\n }\n\n return result;\n}\n#endif\n\n#ifndef STBI_NO_LINEAR\nstatic float *stbi__loadf_main(stbi__context *s, int *x, int *y, int *comp, int req_comp)\n{\n unsigned char *data;\n #ifndef STBI_NO_HDR\n if (stbi__hdr_test(s)) {\n stbi__result_info ri;\n float *hdr_data = stbi__hdr_load(s,x,y,comp,req_comp, &ri);\n if (hdr_data)\n stbi__float_postprocess(hdr_data,x,y,comp,req_comp);\n return hdr_data;\n }\n #endif\n data = stbi__load_and_postprocess_8bit(s, x, y, comp, req_comp);\n if (data)\n return stbi__ldr_to_hdr(data, *x, *y, req_comp ? req_comp : *comp);\n return stbi__errpf(\"unknown image type\", \"Image not of any known type, or corrupt\");\n}\n\nSTBIDEF float *stbi_loadf_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp)\n{\n stbi__context s;\n stbi__start_mem(&s,buffer,len);\n return stbi__loadf_main(&s,x,y,comp,req_comp);\n}\n\nSTBIDEF float *stbi_loadf_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp)\n{\n stbi__context s;\n stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user);\n return stbi__loadf_main(&s,x,y,comp,req_comp);\n}\n\n#ifndef STBI_NO_STDIO\nSTBIDEF float *stbi_loadf(char const *filename, int *x, int *y, int *comp, int req_comp)\n{\n float *result;\n FILE *f = stbi__fopen(filename, \"rb\");\n if (!f) return stbi__errpf(\"can't fopen\", \"Unable to open file\");\n result = stbi_loadf_from_file(f,x,y,comp,req_comp);\n fclose(f);\n return result;\n}\n\nSTBIDEF float *stbi_loadf_from_file(FILE *f, int *x, int *y, int *comp, int req_comp)\n{\n stbi__context s;\n stbi__start_file(&s,f);\n return stbi__loadf_main(&s,x,y,comp,req_comp);\n}\n#endif // !STBI_NO_STDIO\n\n#endif // !STBI_NO_LINEAR\n\n// these is-hdr-or-not is defined independent of whether STBI_NO_LINEAR is\n// defined, for API simplicity; if STBI_NO_LINEAR is defined, it always\n// reports false!\n\nSTBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len)\n{\n #ifndef STBI_NO_HDR\n stbi__context s;\n stbi__start_mem(&s,buffer,len);\n return stbi__hdr_test(&s);\n #else\n STBI_NOTUSED(buffer);\n STBI_NOTUSED(len);\n return 0;\n #endif\n}\n\n#ifndef STBI_NO_STDIO\nSTBIDEF int stbi_is_hdr (char const *filename)\n{\n FILE *f = stbi__fopen(filename, \"rb\");\n int result=0;\n if (f) {\n result = stbi_is_hdr_from_file(f);\n fclose(f);\n }\n return result;\n}\n\nSTBIDEF int stbi_is_hdr_from_file(FILE *f)\n{\n #ifndef STBI_NO_HDR\n long pos = ftell(f);\n int res;\n stbi__context s;\n stbi__start_file(&s,f);\n res = stbi__hdr_test(&s);\n fseek(f, pos, SEEK_SET);\n return res;\n #else\n STBI_NOTUSED(f);\n return 0;\n #endif\n}\n#endif // !STBI_NO_STDIO\n\nSTBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user)\n{\n #ifndef STBI_NO_HDR\n stbi__context s;\n stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user);\n return stbi__hdr_test(&s);\n #else\n STBI_NOTUSED(clbk);\n STBI_NOTUSED(user);\n return 0;\n #endif\n}\n\n#ifndef STBI_NO_LINEAR\nstatic float stbi__l2h_gamma=2.2f, stbi__l2h_scale=1.0f;\n\nSTBIDEF void stbi_ldr_to_hdr_gamma(float gamma) { stbi__l2h_gamma = gamma; }\nSTBIDEF void stbi_ldr_to_hdr_scale(float scale) { stbi__l2h_scale = scale; }\n#endif\n\nstatic float stbi__h2l_gamma_i=1.0f/2.2f, stbi__h2l_scale_i=1.0f;\n\nSTBIDEF void stbi_hdr_to_ldr_gamma(float gamma) { stbi__h2l_gamma_i = 1/gamma; }\nSTBIDEF void stbi_hdr_to_ldr_scale(float scale) { stbi__h2l_scale_i = 1/scale; }\n\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// Common code used by all image loaders\n//\n\nenum\n{\n STBI__SCAN_load=0,\n STBI__SCAN_type,\n STBI__SCAN_header\n};\n\nstatic void stbi__refill_buffer(stbi__context *s)\n{\n int n = (s->io.read)(s->io_user_data,(char*)s->buffer_start,s->buflen);\n s->callback_already_read += (int) (s->img_buffer - s->img_buffer_original);\n if (n == 0) {\n // at end of file, treat same as if from memory, but need to handle case\n // where s->img_buffer isn't pointing to safe memory, e.g. 0-byte file\n s->read_from_callbacks = 0;\n s->img_buffer = s->buffer_start;\n s->img_buffer_end = s->buffer_start+1;\n *s->img_buffer = 0;\n } else {\n s->img_buffer = s->buffer_start;\n s->img_buffer_end = s->buffer_start + n;\n }\n}\n\nstbi_inline static stbi_uc stbi__get8(stbi__context *s)\n{\n if (s->img_buffer < s->img_buffer_end)\n return *s->img_buffer++;\n if (s->read_from_callbacks) {\n stbi__refill_buffer(s);\n return *s->img_buffer++;\n }\n return 0;\n}\n\n#if defined(STBI_NO_JPEG) && defined(STBI_NO_HDR) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM)\n// nothing\n#else\nstbi_inline static int stbi__at_eof(stbi__context *s)\n{\n if (s->io.read) {\n if (!(s->io.eof)(s->io_user_data)) return 0;\n // if feof() is true, check if buffer = end\n // special case: we've only got the special 0 character at the end\n if (s->read_from_callbacks == 0) return 1;\n }\n\n return s->img_buffer >= s->img_buffer_end;\n}\n#endif\n\n#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC)\n// nothing\n#else\nstatic void stbi__skip(stbi__context *s, int n)\n{\n if (n == 0) return; // already there!\n if (n < 0) {\n s->img_buffer = s->img_buffer_end;\n return;\n }\n if (s->io.read) {\n int blen = (int) (s->img_buffer_end - s->img_buffer);\n if (blen < n) {\n s->img_buffer = s->img_buffer_end;\n (s->io.skip)(s->io_user_data, n - blen);\n return;\n }\n }\n s->img_buffer += n;\n}\n#endif\n\n#if defined(STBI_NO_PNG) && defined(STBI_NO_TGA) && defined(STBI_NO_HDR) && defined(STBI_NO_PNM)\n// nothing\n#else\nstatic int stbi__getn(stbi__context *s, stbi_uc *buffer, int n)\n{\n if (s->io.read) {\n int blen = (int) (s->img_buffer_end - s->img_buffer);\n if (blen < n) {\n int res, count;\n\n memcpy(buffer, s->img_buffer, blen);\n\n count = (s->io.read)(s->io_user_data, (char*) buffer + blen, n - blen);\n res = (count == (n-blen));\n s->img_buffer = s->img_buffer_end;\n return res;\n }\n }\n\n if (s->img_buffer+n <= s->img_buffer_end) {\n memcpy(buffer, s->img_buffer, n);\n s->img_buffer += n;\n return 1;\n } else\n return 0;\n}\n#endif\n\n#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC)\n// nothing\n#else\nstatic int stbi__get16be(stbi__context *s)\n{\n int z = stbi__get8(s);\n return (z << 8) + stbi__get8(s);\n}\n#endif\n\n#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC)\n// nothing\n#else\nstatic stbi__uint32 stbi__get32be(stbi__context *s)\n{\n stbi__uint32 z = stbi__get16be(s);\n return (z << 16) + stbi__get16be(s);\n}\n#endif\n\n#if defined(STBI_NO_BMP) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF)\n// nothing\n#else\nstatic int stbi__get16le(stbi__context *s)\n{\n int z = stbi__get8(s);\n return z + (stbi__get8(s) << 8);\n}\n#endif\n\n#ifndef STBI_NO_BMP\nstatic stbi__uint32 stbi__get32le(stbi__context *s)\n{\n stbi__uint32 z = stbi__get16le(s);\n z += (stbi__uint32)stbi__get16le(s) << 16;\n return z;\n}\n#endif\n\n#define STBI__BYTECAST(x) ((stbi_uc) ((x) & 255)) // truncate int to byte without warnings\n\n#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM)\n// nothing\n#else\n//////////////////////////////////////////////////////////////////////////////\n//\n// generic converter from built-in img_n to req_comp\n// individual types do this automatically as much as possible (e.g. jpeg\n// does all cases internally since it needs to colorspace convert anyway,\n// and it never has alpha, so very few cases ). png can automatically\n// interleave an alpha=255 channel, but falls back to this for other cases\n//\n// assume data buffer is malloced, so malloc a new one and free that one\n// only failure mode is malloc failing\n\nstatic stbi_uc stbi__compute_y(int r, int g, int b)\n{\n return (stbi_uc) (((r*77) + (g*150) + (29*b)) >> 8);\n}\n#endif\n\n#if defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM)\n// nothing\n#else\nstatic unsigned char *stbi__convert_format(unsigned char *data, int img_n, int req_comp, unsigned int x, unsigned int y)\n{\n int i,j;\n unsigned char *good;\n\n if (req_comp == img_n) return data;\n STBI_ASSERT(req_comp >= 1 && req_comp <= 4);\n\n good = (unsigned char *) stbi__malloc_mad3(req_comp, x, y, 0);\n if (good == NULL) {\n STBI_FREE(data);\n return stbi__errpuc(\"outofmem\", \"Out of memory\");\n }\n\n for (j=0; j < (int) y; ++j) {\n unsigned char *src = data + j * x * img_n ;\n unsigned char *dest = good + j * x * req_comp;\n\n #define STBI__COMBO(a,b) ((a)*8+(b))\n #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b)\n // convert source image with img_n components to one with req_comp components;\n // avoid switch per pixel, so use switch per scanline and massive macros\n switch (STBI__COMBO(img_n, req_comp)) {\n STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=255; } break;\n STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break;\n STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=255; } break;\n STBI__CASE(2,1) { dest[0]=src[0]; } break;\n STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break;\n STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break;\n STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=255; } break;\n STBI__CASE(3,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break;\n STBI__CASE(3,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = 255; } break;\n STBI__CASE(4,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break;\n STBI__CASE(4,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = src[3]; } break;\n STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break;\n default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return stbi__errpuc(\"unsupported\", \"Unsupported format conversion\");\n }\n #undef STBI__CASE\n }\n\n STBI_FREE(data);\n return good;\n}\n#endif\n\n#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD)\n// nothing\n#else\nstatic stbi__uint16 stbi__compute_y_16(int r, int g, int b)\n{\n return (stbi__uint16) (((r*77) + (g*150) + (29*b)) >> 8);\n}\n#endif\n\n#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD)\n// nothing\n#else\nstatic stbi__uint16 *stbi__convert_format16(stbi__uint16 *data, int img_n, int req_comp, unsigned int x, unsigned int y)\n{\n int i,j;\n stbi__uint16 *good;\n\n if (req_comp == img_n) return data;\n STBI_ASSERT(req_comp >= 1 && req_comp <= 4);\n\n good = (stbi__uint16 *) stbi__malloc(req_comp * x * y * 2);\n if (good == NULL) {\n STBI_FREE(data);\n return (stbi__uint16 *) stbi__errpuc(\"outofmem\", \"Out of memory\");\n }\n\n for (j=0; j < (int) y; ++j) {\n stbi__uint16 *src = data + j * x * img_n ;\n stbi__uint16 *dest = good + j * x * req_comp;\n\n #define STBI__COMBO(a,b) ((a)*8+(b))\n #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b)\n // convert source image with img_n components to one with req_comp components;\n // avoid switch per pixel, so use switch per scanline and massive macros\n switch (STBI__COMBO(img_n, req_comp)) {\n STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=0xffff; } break;\n STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break;\n STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=0xffff; } break;\n STBI__CASE(2,1) { dest[0]=src[0]; } break;\n STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break;\n STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break;\n STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=0xffff; } break;\n STBI__CASE(3,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break;\n STBI__CASE(3,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = 0xffff; } break;\n STBI__CASE(4,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break;\n STBI__CASE(4,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = src[3]; } break;\n STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break;\n default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return (stbi__uint16*) stbi__errpuc(\"unsupported\", \"Unsupported format conversion\");\n }\n #undef STBI__CASE\n }\n\n STBI_FREE(data);\n return good;\n}\n#endif\n\n#ifndef STBI_NO_LINEAR\nstatic float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp)\n{\n int i,k,n;\n float *output;\n if (!data) return NULL;\n output = (float *) stbi__malloc_mad4(x, y, comp, sizeof(float), 0);\n if (output == NULL) { STBI_FREE(data); return stbi__errpf(\"outofmem\", \"Out of memory\"); }\n // compute number of non-alpha components\n if (comp & 1) n = comp; else n = comp-1;\n for (i=0; i < x*y; ++i) {\n for (k=0; k < n; ++k) {\n output[i*comp + k] = (float) (pow(data[i*comp+k]/255.0f, stbi__l2h_gamma) * stbi__l2h_scale);\n }\n }\n if (n < comp) {\n for (i=0; i < x*y; ++i) {\n output[i*comp + n] = data[i*comp + n]/255.0f;\n }\n }\n STBI_FREE(data);\n return output;\n}\n#endif\n\n#ifndef STBI_NO_HDR\n#define stbi__float2int(x) ((int) (x))\nstatic stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp)\n{\n int i,k,n;\n stbi_uc *output;\n if (!data) return NULL;\n output = (stbi_uc *) stbi__malloc_mad3(x, y, comp, 0);\n if (output == NULL) { STBI_FREE(data); return stbi__errpuc(\"outofmem\", \"Out of memory\"); }\n // compute number of non-alpha components\n if (comp & 1) n = comp; else n = comp-1;\n for (i=0; i < x*y; ++i) {\n for (k=0; k < n; ++k) {\n float z = (float) pow(data[i*comp+k]*stbi__h2l_scale_i, stbi__h2l_gamma_i) * 255 + 0.5f;\n if (z < 0) z = 0;\n if (z > 255) z = 255;\n output[i*comp + k] = (stbi_uc) stbi__float2int(z);\n }\n if (k < comp) {\n float z = data[i*comp+k] * 255 + 0.5f;\n if (z < 0) z = 0;\n if (z > 255) z = 255;\n output[i*comp + k] = (stbi_uc) stbi__float2int(z);\n }\n }\n STBI_FREE(data);\n return output;\n}\n#endif\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// \"baseline\" JPEG/JFIF decoder\n//\n// simple implementation\n// - doesn't support delayed output of y-dimension\n// - simple interface (only one output format: 8-bit interleaved RGB)\n// - doesn't try to recover corrupt jpegs\n// - doesn't allow partial loading, loading multiple at once\n// - still fast on x86 (copying globals into locals doesn't help x86)\n// - allocates lots of intermediate memory (full size of all components)\n// - non-interleaved case requires this anyway\n// - allows good upsampling (see next)\n// high-quality\n// - upsampled channels are bilinearly interpolated, even across blocks\n// - quality integer IDCT derived from IJG's 'slow'\n// performance\n// - fast huffman; reasonable integer IDCT\n// - some SIMD kernels for common paths on targets with SSE2/NEON\n// - uses a lot of intermediate memory, could cache poorly\n\n#ifndef STBI_NO_JPEG\n\n// huffman decoding acceleration\n#define FAST_BITS 9 // larger handles more cases; smaller stomps less cache\n\ntypedef struct\n{\n stbi_uc fast[1 << FAST_BITS];\n // weirdly, repacking this into AoS is a 10% speed loss, instead of a win\n stbi__uint16 code[256];\n stbi_uc values[256];\n stbi_uc size[257];\n unsigned int maxcode[18];\n int delta[17]; // old 'firstsymbol' - old 'firstcode'\n} stbi__huffman;\n\ntypedef struct\n{\n stbi__context *s;\n stbi__huffman huff_dc[4];\n stbi__huffman huff_ac[4];\n stbi__uint16 dequant[4][64];\n stbi__int16 fast_ac[4][1 << FAST_BITS];\n\n// sizes for components, interleaved MCUs\n int img_h_max, img_v_max;\n int img_mcu_x, img_mcu_y;\n int img_mcu_w, img_mcu_h;\n\n// definition of jpeg image component\n struct\n {\n int id;\n int h,v;\n int tq;\n int hd,ha;\n int dc_pred;\n\n int x,y,w2,h2;\n stbi_uc *data;\n void *raw_data, *raw_coeff;\n stbi_uc *linebuf;\n short *coeff; // progressive only\n int coeff_w, coeff_h; // number of 8x8 coefficient blocks\n } img_comp[4];\n\n stbi__uint32 code_buffer; // jpeg entropy-coded buffer\n int code_bits; // number of valid bits\n unsigned char marker; // marker seen while filling entropy buffer\n int nomore; // flag if we saw a marker so must stop\n\n int progressive;\n int spec_start;\n int spec_end;\n int succ_high;\n int succ_low;\n int eob_run;\n int jfif;\n int app14_color_transform; // Adobe APP14 tag\n int rgb;\n\n int scan_n, order[4];\n int restart_interval, todo;\n\n// kernels\n void (*idct_block_kernel)(stbi_uc *out, int out_stride, short data[64]);\n void (*YCbCr_to_RGB_kernel)(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step);\n stbi_uc *(*resample_row_hv_2_kernel)(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs);\n} stbi__jpeg;\n\nstatic int stbi__build_huffman(stbi__huffman *h, int *count)\n{\n int i,j,k=0;\n unsigned int code;\n // build size list for each symbol (from JPEG spec)\n for (i=0; i < 16; ++i) {\n for (j=0; j < count[i]; ++j) {\n h->size[k++] = (stbi_uc) (i+1);\n if(k >= 257) return stbi__err(\"bad size list\",\"Corrupt JPEG\");\n }\n }\n h->size[k] = 0;\n\n // compute actual symbols (from jpeg spec)\n code = 0;\n k = 0;\n for(j=1; j <= 16; ++j) {\n // compute delta to add to code to compute symbol id\n h->delta[j] = k - code;\n if (h->size[k] == j) {\n while (h->size[k] == j)\n h->code[k++] = (stbi__uint16) (code++);\n if (code-1 >= (1u << j)) return stbi__err(\"bad code lengths\",\"Corrupt JPEG\");\n }\n // compute largest code + 1 for this size, preshifted as needed later\n h->maxcode[j] = code << (16-j);\n code <<= 1;\n }\n h->maxcode[j] = 0xffffffff;\n\n // build non-spec acceleration table; 255 is flag for not-accelerated\n memset(h->fast, 255, 1 << FAST_BITS);\n for (i=0; i < k; ++i) {\n int s = h->size[i];\n if (s <= FAST_BITS) {\n int c = h->code[i] << (FAST_BITS-s);\n int m = 1 << (FAST_BITS-s);\n for (j=0; j < m; ++j) {\n h->fast[c+j] = (stbi_uc) i;\n }\n }\n }\n return 1;\n}\n\n// build a table that decodes both magnitude and value of small ACs in\n// one go.\nstatic void stbi__build_fast_ac(stbi__int16 *fast_ac, stbi__huffman *h)\n{\n int i;\n for (i=0; i < (1 << FAST_BITS); ++i) {\n stbi_uc fast = h->fast[i];\n fast_ac[i] = 0;\n if (fast < 255) {\n int rs = h->values[fast];\n int run = (rs >> 4) & 15;\n int magbits = rs & 15;\n int len = h->size[fast];\n\n if (magbits && len + magbits <= FAST_BITS) {\n // magnitude code followed by receive_extend code\n int k = ((i << len) & ((1 << FAST_BITS) - 1)) >> (FAST_BITS - magbits);\n int m = 1 << (magbits - 1);\n if (k < m) k += (~0U << magbits) + 1;\n // if the result is small enough, we can fit it in fast_ac table\n if (k >= -128 && k <= 127)\n fast_ac[i] = (stbi__int16) ((k * 256) + (run * 16) + (len + magbits));\n }\n }\n }\n}\n\nstatic void stbi__grow_buffer_unsafe(stbi__jpeg *j)\n{\n do {\n unsigned int b = j->nomore ? 0 : stbi__get8(j->s);\n if (b == 0xff) {\n int c = stbi__get8(j->s);\n while (c == 0xff) c = stbi__get8(j->s); // consume fill bytes\n if (c != 0) {\n j->marker = (unsigned char) c;\n j->nomore = 1;\n return;\n }\n }\n j->code_buffer |= b << (24 - j->code_bits);\n j->code_bits += 8;\n } while (j->code_bits <= 24);\n}\n\n// (1 << n) - 1\nstatic const stbi__uint32 stbi__bmask[17]={0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535};\n\n// decode a jpeg huffman value from the bitstream\nstbi_inline static int stbi__jpeg_huff_decode(stbi__jpeg *j, stbi__huffman *h)\n{\n unsigned int temp;\n int c,k;\n\n if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);\n\n // look at the top FAST_BITS and determine what symbol ID it is,\n // if the code is <= FAST_BITS\n c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1);\n k = h->fast[c];\n if (k < 255) {\n int s = h->size[k];\n if (s > j->code_bits)\n return -1;\n j->code_buffer <<= s;\n j->code_bits -= s;\n return h->values[k];\n }\n\n // naive test is to shift the code_buffer down so k bits are\n // valid, then test against maxcode. To speed this up, we've\n // preshifted maxcode left so that it has (16-k) 0s at the\n // end; in other words, regardless of the number of bits, it\n // wants to be compared against something shifted to have 16;\n // that way we don't need to shift inside the loop.\n temp = j->code_buffer >> 16;\n for (k=FAST_BITS+1 ; ; ++k)\n if (temp < h->maxcode[k])\n break;\n if (k == 17) {\n // error! code not found\n j->code_bits -= 16;\n return -1;\n }\n\n if (k > j->code_bits)\n return -1;\n\n // convert the huffman code to the symbol id\n c = ((j->code_buffer >> (32 - k)) & stbi__bmask[k]) + h->delta[k];\n if(c < 0 || c >= 256) // symbol id out of bounds!\n return -1;\n STBI_ASSERT((((j->code_buffer) >> (32 - h->size[c])) & stbi__bmask[h->size[c]]) == h->code[c]);\n\n // convert the id to a symbol\n j->code_bits -= k;\n j->code_buffer <<= k;\n return h->values[c];\n}\n\n// bias[n] = (-1<code_bits < n) stbi__grow_buffer_unsafe(j);\n if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing\n\n sgn = j->code_buffer >> 31; // sign bit always in MSB; 0 if MSB clear (positive), 1 if MSB set (negative)\n k = stbi_lrot(j->code_buffer, n);\n j->code_buffer = k & ~stbi__bmask[n];\n k &= stbi__bmask[n];\n j->code_bits -= n;\n return k + (stbi__jbias[n] & (sgn - 1));\n}\n\n// get some unsigned bits\nstbi_inline static int stbi__jpeg_get_bits(stbi__jpeg *j, int n)\n{\n unsigned int k;\n if (j->code_bits < n) stbi__grow_buffer_unsafe(j);\n if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing\n k = stbi_lrot(j->code_buffer, n);\n j->code_buffer = k & ~stbi__bmask[n];\n k &= stbi__bmask[n];\n j->code_bits -= n;\n return k;\n}\n\nstbi_inline static int stbi__jpeg_get_bit(stbi__jpeg *j)\n{\n unsigned int k;\n if (j->code_bits < 1) stbi__grow_buffer_unsafe(j);\n if (j->code_bits < 1) return 0; // ran out of bits from stream, return 0s intead of continuing\n k = j->code_buffer;\n j->code_buffer <<= 1;\n --j->code_bits;\n return k & 0x80000000;\n}\n\n// given a value that's at position X in the zigzag stream,\n// where does it appear in the 8x8 matrix coded as row-major?\nstatic const stbi_uc stbi__jpeg_dezigzag[64+15] =\n{\n 0, 1, 8, 16, 9, 2, 3, 10,\n 17, 24, 32, 25, 18, 11, 4, 5,\n 12, 19, 26, 33, 40, 48, 41, 34,\n 27, 20, 13, 6, 7, 14, 21, 28,\n 35, 42, 49, 56, 57, 50, 43, 36,\n 29, 22, 15, 23, 30, 37, 44, 51,\n 58, 59, 52, 45, 38, 31, 39, 46,\n 53, 60, 61, 54, 47, 55, 62, 63,\n // let corrupt input sample past end\n 63, 63, 63, 63, 63, 63, 63, 63,\n 63, 63, 63, 63, 63, 63, 63\n};\n\n// decode one 64-entry block--\nstatic int stbi__jpeg_decode_block(stbi__jpeg *j, short data[64], stbi__huffman *hdc, stbi__huffman *hac, stbi__int16 *fac, int b, stbi__uint16 *dequant)\n{\n int diff,dc,k;\n int t;\n\n if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);\n t = stbi__jpeg_huff_decode(j, hdc);\n if (t < 0 || t > 15) return stbi__err(\"bad huffman code\",\"Corrupt JPEG\");\n\n // 0 all the ac values now so we can do it 32-bits at a time\n memset(data,0,64*sizeof(data[0]));\n\n diff = t ? stbi__extend_receive(j, t) : 0;\n if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err(\"bad delta\",\"Corrupt JPEG\");\n dc = j->img_comp[b].dc_pred + diff;\n j->img_comp[b].dc_pred = dc;\n if (!stbi__mul2shorts_valid(dc, dequant[0])) return stbi__err(\"can't merge dc and ac\", \"Corrupt JPEG\");\n data[0] = (short) (dc * dequant[0]);\n\n // decode AC components, see JPEG spec\n k = 1;\n do {\n unsigned int zig;\n int c,r,s;\n if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);\n c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1);\n r = fac[c];\n if (r) { // fast-AC path\n k += (r >> 4) & 15; // run\n s = r & 15; // combined length\n if (s > j->code_bits) return stbi__err(\"bad huffman code\", \"Combined length longer than code bits available\");\n j->code_buffer <<= s;\n j->code_bits -= s;\n // decode into unzigzag'd location\n zig = stbi__jpeg_dezigzag[k++];\n data[zig] = (short) ((r >> 8) * dequant[zig]);\n } else {\n int rs = stbi__jpeg_huff_decode(j, hac);\n if (rs < 0) return stbi__err(\"bad huffman code\",\"Corrupt JPEG\");\n s = rs & 15;\n r = rs >> 4;\n if (s == 0) {\n if (rs != 0xf0) break; // end block\n k += 16;\n } else {\n k += r;\n // decode into unzigzag'd location\n zig = stbi__jpeg_dezigzag[k++];\n data[zig] = (short) (stbi__extend_receive(j,s) * dequant[zig]);\n }\n }\n } while (k < 64);\n return 1;\n}\n\nstatic int stbi__jpeg_decode_block_prog_dc(stbi__jpeg *j, short data[64], stbi__huffman *hdc, int b)\n{\n int diff,dc;\n int t;\n if (j->spec_end != 0) return stbi__err(\"can't merge dc and ac\", \"Corrupt JPEG\");\n\n if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);\n\n if (j->succ_high == 0) {\n // first scan for DC coefficient, must be first\n memset(data,0,64*sizeof(data[0])); // 0 all the ac values now\n t = stbi__jpeg_huff_decode(j, hdc);\n if (t < 0 || t > 15) return stbi__err(\"can't merge dc and ac\", \"Corrupt JPEG\");\n diff = t ? stbi__extend_receive(j, t) : 0;\n\n if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err(\"bad delta\", \"Corrupt JPEG\");\n dc = j->img_comp[b].dc_pred + diff;\n j->img_comp[b].dc_pred = dc;\n if (!stbi__mul2shorts_valid(dc, 1 << j->succ_low)) return stbi__err(\"can't merge dc and ac\", \"Corrupt JPEG\");\n data[0] = (short) (dc * (1 << j->succ_low));\n } else {\n // refinement scan for DC coefficient\n if (stbi__jpeg_get_bit(j))\n data[0] += (short) (1 << j->succ_low);\n }\n return 1;\n}\n\n// @OPTIMIZE: store non-zigzagged during the decode passes,\n// and only de-zigzag when dequantizing\nstatic int stbi__jpeg_decode_block_prog_ac(stbi__jpeg *j, short data[64], stbi__huffman *hac, stbi__int16 *fac)\n{\n int k;\n if (j->spec_start == 0) return stbi__err(\"can't merge dc and ac\", \"Corrupt JPEG\");\n\n if (j->succ_high == 0) {\n int shift = j->succ_low;\n\n if (j->eob_run) {\n --j->eob_run;\n return 1;\n }\n\n k = j->spec_start;\n do {\n unsigned int zig;\n int c,r,s;\n if (j->code_bits < 16) stbi__grow_buffer_unsafe(j);\n c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1);\n r = fac[c];\n if (r) { // fast-AC path\n k += (r >> 4) & 15; // run\n s = r & 15; // combined length\n if (s > j->code_bits) return stbi__err(\"bad huffman code\", \"Combined length longer than code bits available\");\n j->code_buffer <<= s;\n j->code_bits -= s;\n zig = stbi__jpeg_dezigzag[k++];\n data[zig] = (short) ((r >> 8) * (1 << shift));\n } else {\n int rs = stbi__jpeg_huff_decode(j, hac);\n if (rs < 0) return stbi__err(\"bad huffman code\",\"Corrupt JPEG\");\n s = rs & 15;\n r = rs >> 4;\n if (s == 0) {\n if (r < 15) {\n j->eob_run = (1 << r);\n if (r)\n j->eob_run += stbi__jpeg_get_bits(j, r);\n --j->eob_run;\n break;\n }\n k += 16;\n } else {\n k += r;\n zig = stbi__jpeg_dezigzag[k++];\n data[zig] = (short) (stbi__extend_receive(j,s) * (1 << shift));\n }\n }\n } while (k <= j->spec_end);\n } else {\n // refinement scan for these AC coefficients\n\n short bit = (short) (1 << j->succ_low);\n\n if (j->eob_run) {\n --j->eob_run;\n for (k = j->spec_start; k <= j->spec_end; ++k) {\n short *p = &data[stbi__jpeg_dezigzag[k]];\n if (*p != 0)\n if (stbi__jpeg_get_bit(j))\n if ((*p & bit)==0) {\n if (*p > 0)\n *p += bit;\n else\n *p -= bit;\n }\n }\n } else {\n k = j->spec_start;\n do {\n int r,s;\n int rs = stbi__jpeg_huff_decode(j, hac); // @OPTIMIZE see if we can use the fast path here, advance-by-r is so slow, eh\n if (rs < 0) return stbi__err(\"bad huffman code\",\"Corrupt JPEG\");\n s = rs & 15;\n r = rs >> 4;\n if (s == 0) {\n if (r < 15) {\n j->eob_run = (1 << r) - 1;\n if (r)\n j->eob_run += stbi__jpeg_get_bits(j, r);\n r = 64; // force end of block\n } else {\n // r=15 s=0 should write 16 0s, so we just do\n // a run of 15 0s and then write s (which is 0),\n // so we don't have to do anything special here\n }\n } else {\n if (s != 1) return stbi__err(\"bad huffman code\", \"Corrupt JPEG\");\n // sign bit\n if (stbi__jpeg_get_bit(j))\n s = bit;\n else\n s = -bit;\n }\n\n // advance by r\n while (k <= j->spec_end) {\n short *p = &data[stbi__jpeg_dezigzag[k++]];\n if (*p != 0) {\n if (stbi__jpeg_get_bit(j))\n if ((*p & bit)==0) {\n if (*p > 0)\n *p += bit;\n else\n *p -= bit;\n }\n } else {\n if (r == 0) {\n *p = (short) s;\n break;\n }\n --r;\n }\n }\n } while (k <= j->spec_end);\n }\n }\n return 1;\n}\n\n// take a -128..127 value and stbi__clamp it and convert to 0..255\nstbi_inline static stbi_uc stbi__clamp(int x)\n{\n // trick to use a single test to catch both cases\n if ((unsigned int) x > 255) {\n if (x < 0) return 0;\n if (x > 255) return 255;\n }\n return (stbi_uc) x;\n}\n\n#define stbi__f2f(x) ((int) (((x) * 4096 + 0.5)))\n#define stbi__fsh(x) ((x) * 4096)\n\n// derived from jidctint -- DCT_ISLOW\n#define STBI__IDCT_1D(s0,s1,s2,s3,s4,s5,s6,s7) \\\n int t0,t1,t2,t3,p1,p2,p3,p4,p5,x0,x1,x2,x3; \\\n p2 = s2; \\\n p3 = s6; \\\n p1 = (p2+p3) * stbi__f2f(0.5411961f); \\\n t2 = p1 + p3*stbi__f2f(-1.847759065f); \\\n t3 = p1 + p2*stbi__f2f( 0.765366865f); \\\n p2 = s0; \\\n p3 = s4; \\\n t0 = stbi__fsh(p2+p3); \\\n t1 = stbi__fsh(p2-p3); \\\n x0 = t0+t3; \\\n x3 = t0-t3; \\\n x1 = t1+t2; \\\n x2 = t1-t2; \\\n t0 = s7; \\\n t1 = s5; \\\n t2 = s3; \\\n t3 = s1; \\\n p3 = t0+t2; \\\n p4 = t1+t3; \\\n p1 = t0+t3; \\\n p2 = t1+t2; \\\n p5 = (p3+p4)*stbi__f2f( 1.175875602f); \\\n t0 = t0*stbi__f2f( 0.298631336f); \\\n t1 = t1*stbi__f2f( 2.053119869f); \\\n t2 = t2*stbi__f2f( 3.072711026f); \\\n t3 = t3*stbi__f2f( 1.501321110f); \\\n p1 = p5 + p1*stbi__f2f(-0.899976223f); \\\n p2 = p5 + p2*stbi__f2f(-2.562915447f); \\\n p3 = p3*stbi__f2f(-1.961570560f); \\\n p4 = p4*stbi__f2f(-0.390180644f); \\\n t3 += p1+p4; \\\n t2 += p2+p3; \\\n t1 += p2+p4; \\\n t0 += p1+p3;\n\nstatic void stbi__idct_block(stbi_uc *out, int out_stride, short data[64])\n{\n int i,val[64],*v=val;\n stbi_uc *o;\n short *d = data;\n\n // columns\n for (i=0; i < 8; ++i,++d, ++v) {\n // if all zeroes, shortcut -- this avoids dequantizing 0s and IDCTing\n if (d[ 8]==0 && d[16]==0 && d[24]==0 && d[32]==0\n && d[40]==0 && d[48]==0 && d[56]==0) {\n // no shortcut 0 seconds\n // (1|2|3|4|5|6|7)==0 0 seconds\n // all separate -0.047 seconds\n // 1 && 2|3 && 4|5 && 6|7: -0.047 seconds\n int dcterm = d[0]*4;\n v[0] = v[8] = v[16] = v[24] = v[32] = v[40] = v[48] = v[56] = dcterm;\n } else {\n STBI__IDCT_1D(d[ 0],d[ 8],d[16],d[24],d[32],d[40],d[48],d[56])\n // constants scaled things up by 1<<12; let's bring them back\n // down, but keep 2 extra bits of precision\n x0 += 512; x1 += 512; x2 += 512; x3 += 512;\n v[ 0] = (x0+t3) >> 10;\n v[56] = (x0-t3) >> 10;\n v[ 8] = (x1+t2) >> 10;\n v[48] = (x1-t2) >> 10;\n v[16] = (x2+t1) >> 10;\n v[40] = (x2-t1) >> 10;\n v[24] = (x3+t0) >> 10;\n v[32] = (x3-t0) >> 10;\n }\n }\n\n for (i=0, v=val, o=out; i < 8; ++i,v+=8,o+=out_stride) {\n // no fast case since the first 1D IDCT spread components out\n STBI__IDCT_1D(v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7])\n // constants scaled things up by 1<<12, plus we had 1<<2 from first\n // loop, plus horizontal and vertical each scale by sqrt(8) so together\n // we've got an extra 1<<3, so 1<<17 total we need to remove.\n // so we want to round that, which means adding 0.5 * 1<<17,\n // aka 65536. Also, we'll end up with -128 to 127 that we want\n // to encode as 0..255 by adding 128, so we'll add that before the shift\n x0 += 65536 + (128<<17);\n x1 += 65536 + (128<<17);\n x2 += 65536 + (128<<17);\n x3 += 65536 + (128<<17);\n // tried computing the shifts into temps, or'ing the temps to see\n // if any were out of range, but that was slower\n o[0] = stbi__clamp((x0+t3) >> 17);\n o[7] = stbi__clamp((x0-t3) >> 17);\n o[1] = stbi__clamp((x1+t2) >> 17);\n o[6] = stbi__clamp((x1-t2) >> 17);\n o[2] = stbi__clamp((x2+t1) >> 17);\n o[5] = stbi__clamp((x2-t1) >> 17);\n o[3] = stbi__clamp((x3+t0) >> 17);\n o[4] = stbi__clamp((x3-t0) >> 17);\n }\n}\n\n#ifdef STBI_SSE2\n// sse2 integer IDCT. not the fastest possible implementation but it\n// produces bit-identical results to the generic C version so it's\n// fully \"transparent\".\nstatic void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64])\n{\n // This is constructed to match our regular (generic) integer IDCT exactly.\n __m128i row0, row1, row2, row3, row4, row5, row6, row7;\n __m128i tmp;\n\n // dot product constant: even elems=x, odd elems=y\n #define dct_const(x,y) _mm_setr_epi16((x),(y),(x),(y),(x),(y),(x),(y))\n\n // out(0) = c0[even]*x + c0[odd]*y (c0, x, y 16-bit, out 32-bit)\n // out(1) = c1[even]*x + c1[odd]*y\n #define dct_rot(out0,out1, x,y,c0,c1) \\\n __m128i c0##lo = _mm_unpacklo_epi16((x),(y)); \\\n __m128i c0##hi = _mm_unpackhi_epi16((x),(y)); \\\n __m128i out0##_l = _mm_madd_epi16(c0##lo, c0); \\\n __m128i out0##_h = _mm_madd_epi16(c0##hi, c0); \\\n __m128i out1##_l = _mm_madd_epi16(c0##lo, c1); \\\n __m128i out1##_h = _mm_madd_epi16(c0##hi, c1)\n\n // out = in << 12 (in 16-bit, out 32-bit)\n #define dct_widen(out, in) \\\n __m128i out##_l = _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), (in)), 4); \\\n __m128i out##_h = _mm_srai_epi32(_mm_unpackhi_epi16(_mm_setzero_si128(), (in)), 4)\n\n // wide add\n #define dct_wadd(out, a, b) \\\n __m128i out##_l = _mm_add_epi32(a##_l, b##_l); \\\n __m128i out##_h = _mm_add_epi32(a##_h, b##_h)\n\n // wide sub\n #define dct_wsub(out, a, b) \\\n __m128i out##_l = _mm_sub_epi32(a##_l, b##_l); \\\n __m128i out##_h = _mm_sub_epi32(a##_h, b##_h)\n\n // butterfly a/b, add bias, then shift by \"s\" and pack\n #define dct_bfly32o(out0, out1, a,b,bias,s) \\\n { \\\n __m128i abiased_l = _mm_add_epi32(a##_l, bias); \\\n __m128i abiased_h = _mm_add_epi32(a##_h, bias); \\\n dct_wadd(sum, abiased, b); \\\n dct_wsub(dif, abiased, b); \\\n out0 = _mm_packs_epi32(_mm_srai_epi32(sum_l, s), _mm_srai_epi32(sum_h, s)); \\\n out1 = _mm_packs_epi32(_mm_srai_epi32(dif_l, s), _mm_srai_epi32(dif_h, s)); \\\n }\n\n // 8-bit interleave step (for transposes)\n #define dct_interleave8(a, b) \\\n tmp = a; \\\n a = _mm_unpacklo_epi8(a, b); \\\n b = _mm_unpackhi_epi8(tmp, b)\n\n // 16-bit interleave step (for transposes)\n #define dct_interleave16(a, b) \\\n tmp = a; \\\n a = _mm_unpacklo_epi16(a, b); \\\n b = _mm_unpackhi_epi16(tmp, b)\n\n #define dct_pass(bias,shift) \\\n { \\\n /* even part */ \\\n dct_rot(t2e,t3e, row2,row6, rot0_0,rot0_1); \\\n __m128i sum04 = _mm_add_epi16(row0, row4); \\\n __m128i dif04 = _mm_sub_epi16(row0, row4); \\\n dct_widen(t0e, sum04); \\\n dct_widen(t1e, dif04); \\\n dct_wadd(x0, t0e, t3e); \\\n dct_wsub(x3, t0e, t3e); \\\n dct_wadd(x1, t1e, t2e); \\\n dct_wsub(x2, t1e, t2e); \\\n /* odd part */ \\\n dct_rot(y0o,y2o, row7,row3, rot2_0,rot2_1); \\\n dct_rot(y1o,y3o, row5,row1, rot3_0,rot3_1); \\\n __m128i sum17 = _mm_add_epi16(row1, row7); \\\n __m128i sum35 = _mm_add_epi16(row3, row5); \\\n dct_rot(y4o,y5o, sum17,sum35, rot1_0,rot1_1); \\\n dct_wadd(x4, y0o, y4o); \\\n dct_wadd(x5, y1o, y5o); \\\n dct_wadd(x6, y2o, y5o); \\\n dct_wadd(x7, y3o, y4o); \\\n dct_bfly32o(row0,row7, x0,x7,bias,shift); \\\n dct_bfly32o(row1,row6, x1,x6,bias,shift); \\\n dct_bfly32o(row2,row5, x2,x5,bias,shift); \\\n dct_bfly32o(row3,row4, x3,x4,bias,shift); \\\n }\n\n __m128i rot0_0 = dct_const(stbi__f2f(0.5411961f), stbi__f2f(0.5411961f) + stbi__f2f(-1.847759065f));\n __m128i rot0_1 = dct_const(stbi__f2f(0.5411961f) + stbi__f2f( 0.765366865f), stbi__f2f(0.5411961f));\n __m128i rot1_0 = dct_const(stbi__f2f(1.175875602f) + stbi__f2f(-0.899976223f), stbi__f2f(1.175875602f));\n __m128i rot1_1 = dct_const(stbi__f2f(1.175875602f), stbi__f2f(1.175875602f) + stbi__f2f(-2.562915447f));\n __m128i rot2_0 = dct_const(stbi__f2f(-1.961570560f) + stbi__f2f( 0.298631336f), stbi__f2f(-1.961570560f));\n __m128i rot2_1 = dct_const(stbi__f2f(-1.961570560f), stbi__f2f(-1.961570560f) + stbi__f2f( 3.072711026f));\n __m128i rot3_0 = dct_const(stbi__f2f(-0.390180644f) + stbi__f2f( 2.053119869f), stbi__f2f(-0.390180644f));\n __m128i rot3_1 = dct_const(stbi__f2f(-0.390180644f), stbi__f2f(-0.390180644f) + stbi__f2f( 1.501321110f));\n\n // rounding biases in column/row passes, see stbi__idct_block for explanation.\n __m128i bias_0 = _mm_set1_epi32(512);\n __m128i bias_1 = _mm_set1_epi32(65536 + (128<<17));\n\n // load\n row0 = _mm_load_si128((const __m128i *) (data + 0*8));\n row1 = _mm_load_si128((const __m128i *) (data + 1*8));\n row2 = _mm_load_si128((const __m128i *) (data + 2*8));\n row3 = _mm_load_si128((const __m128i *) (data + 3*8));\n row4 = _mm_load_si128((const __m128i *) (data + 4*8));\n row5 = _mm_load_si128((const __m128i *) (data + 5*8));\n row6 = _mm_load_si128((const __m128i *) (data + 6*8));\n row7 = _mm_load_si128((const __m128i *) (data + 7*8));\n\n // column pass\n dct_pass(bias_0, 10);\n\n {\n // 16bit 8x8 transpose pass 1\n dct_interleave16(row0, row4);\n dct_interleave16(row1, row5);\n dct_interleave16(row2, row6);\n dct_interleave16(row3, row7);\n\n // transpose pass 2\n dct_interleave16(row0, row2);\n dct_interleave16(row1, row3);\n dct_interleave16(row4, row6);\n dct_interleave16(row5, row7);\n\n // transpose pass 3\n dct_interleave16(row0, row1);\n dct_interleave16(row2, row3);\n dct_interleave16(row4, row5);\n dct_interleave16(row6, row7);\n }\n\n // row pass\n dct_pass(bias_1, 17);\n\n {\n // pack\n __m128i p0 = _mm_packus_epi16(row0, row1); // a0a1a2a3...a7b0b1b2b3...b7\n __m128i p1 = _mm_packus_epi16(row2, row3);\n __m128i p2 = _mm_packus_epi16(row4, row5);\n __m128i p3 = _mm_packus_epi16(row6, row7);\n\n // 8bit 8x8 transpose pass 1\n dct_interleave8(p0, p2); // a0e0a1e1...\n dct_interleave8(p1, p3); // c0g0c1g1...\n\n // transpose pass 2\n dct_interleave8(p0, p1); // a0c0e0g0...\n dct_interleave8(p2, p3); // b0d0f0h0...\n\n // transpose pass 3\n dct_interleave8(p0, p2); // a0b0c0d0...\n dct_interleave8(p1, p3); // a4b4c4d4...\n\n // store\n _mm_storel_epi64((__m128i *) out, p0); out += out_stride;\n _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p0, 0x4e)); out += out_stride;\n _mm_storel_epi64((__m128i *) out, p2); out += out_stride;\n _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p2, 0x4e)); out += out_stride;\n _mm_storel_epi64((__m128i *) out, p1); out += out_stride;\n _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p1, 0x4e)); out += out_stride;\n _mm_storel_epi64((__m128i *) out, p3); out += out_stride;\n _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p3, 0x4e));\n }\n\n#undef dct_const\n#undef dct_rot\n#undef dct_widen\n#undef dct_wadd\n#undef dct_wsub\n#undef dct_bfly32o\n#undef dct_interleave8\n#undef dct_interleave16\n#undef dct_pass\n}\n\n#endif // STBI_SSE2\n\n#ifdef STBI_NEON\n\n// NEON integer IDCT. should produce bit-identical\n// results to the generic C version.\nstatic void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64])\n{\n int16x8_t row0, row1, row2, row3, row4, row5, row6, row7;\n\n int16x4_t rot0_0 = vdup_n_s16(stbi__f2f(0.5411961f));\n int16x4_t rot0_1 = vdup_n_s16(stbi__f2f(-1.847759065f));\n int16x4_t rot0_2 = vdup_n_s16(stbi__f2f( 0.765366865f));\n int16x4_t rot1_0 = vdup_n_s16(stbi__f2f( 1.175875602f));\n int16x4_t rot1_1 = vdup_n_s16(stbi__f2f(-0.899976223f));\n int16x4_t rot1_2 = vdup_n_s16(stbi__f2f(-2.562915447f));\n int16x4_t rot2_0 = vdup_n_s16(stbi__f2f(-1.961570560f));\n int16x4_t rot2_1 = vdup_n_s16(stbi__f2f(-0.390180644f));\n int16x4_t rot3_0 = vdup_n_s16(stbi__f2f( 0.298631336f));\n int16x4_t rot3_1 = vdup_n_s16(stbi__f2f( 2.053119869f));\n int16x4_t rot3_2 = vdup_n_s16(stbi__f2f( 3.072711026f));\n int16x4_t rot3_3 = vdup_n_s16(stbi__f2f( 1.501321110f));\n\n#define dct_long_mul(out, inq, coeff) \\\n int32x4_t out##_l = vmull_s16(vget_low_s16(inq), coeff); \\\n int32x4_t out##_h = vmull_s16(vget_high_s16(inq), coeff)\n\n#define dct_long_mac(out, acc, inq, coeff) \\\n int32x4_t out##_l = vmlal_s16(acc##_l, vget_low_s16(inq), coeff); \\\n int32x4_t out##_h = vmlal_s16(acc##_h, vget_high_s16(inq), coeff)\n\n#define dct_widen(out, inq) \\\n int32x4_t out##_l = vshll_n_s16(vget_low_s16(inq), 12); \\\n int32x4_t out##_h = vshll_n_s16(vget_high_s16(inq), 12)\n\n// wide add\n#define dct_wadd(out, a, b) \\\n int32x4_t out##_l = vaddq_s32(a##_l, b##_l); \\\n int32x4_t out##_h = vaddq_s32(a##_h, b##_h)\n\n// wide sub\n#define dct_wsub(out, a, b) \\\n int32x4_t out##_l = vsubq_s32(a##_l, b##_l); \\\n int32x4_t out##_h = vsubq_s32(a##_h, b##_h)\n\n// butterfly a/b, then shift using \"shiftop\" by \"s\" and pack\n#define dct_bfly32o(out0,out1, a,b,shiftop,s) \\\n { \\\n dct_wadd(sum, a, b); \\\n dct_wsub(dif, a, b); \\\n out0 = vcombine_s16(shiftop(sum_l, s), shiftop(sum_h, s)); \\\n out1 = vcombine_s16(shiftop(dif_l, s), shiftop(dif_h, s)); \\\n }\n\n#define dct_pass(shiftop, shift) \\\n { \\\n /* even part */ \\\n int16x8_t sum26 = vaddq_s16(row2, row6); \\\n dct_long_mul(p1e, sum26, rot0_0); \\\n dct_long_mac(t2e, p1e, row6, rot0_1); \\\n dct_long_mac(t3e, p1e, row2, rot0_2); \\\n int16x8_t sum04 = vaddq_s16(row0, row4); \\\n int16x8_t dif04 = vsubq_s16(row0, row4); \\\n dct_widen(t0e, sum04); \\\n dct_widen(t1e, dif04); \\\n dct_wadd(x0, t0e, t3e); \\\n dct_wsub(x3, t0e, t3e); \\\n dct_wadd(x1, t1e, t2e); \\\n dct_wsub(x2, t1e, t2e); \\\n /* odd part */ \\\n int16x8_t sum15 = vaddq_s16(row1, row5); \\\n int16x8_t sum17 = vaddq_s16(row1, row7); \\\n int16x8_t sum35 = vaddq_s16(row3, row5); \\\n int16x8_t sum37 = vaddq_s16(row3, row7); \\\n int16x8_t sumodd = vaddq_s16(sum17, sum35); \\\n dct_long_mul(p5o, sumodd, rot1_0); \\\n dct_long_mac(p1o, p5o, sum17, rot1_1); \\\n dct_long_mac(p2o, p5o, sum35, rot1_2); \\\n dct_long_mul(p3o, sum37, rot2_0); \\\n dct_long_mul(p4o, sum15, rot2_1); \\\n dct_wadd(sump13o, p1o, p3o); \\\n dct_wadd(sump24o, p2o, p4o); \\\n dct_wadd(sump23o, p2o, p3o); \\\n dct_wadd(sump14o, p1o, p4o); \\\n dct_long_mac(x4, sump13o, row7, rot3_0); \\\n dct_long_mac(x5, sump24o, row5, rot3_1); \\\n dct_long_mac(x6, sump23o, row3, rot3_2); \\\n dct_long_mac(x7, sump14o, row1, rot3_3); \\\n dct_bfly32o(row0,row7, x0,x7,shiftop,shift); \\\n dct_bfly32o(row1,row6, x1,x6,shiftop,shift); \\\n dct_bfly32o(row2,row5, x2,x5,shiftop,shift); \\\n dct_bfly32o(row3,row4, x3,x4,shiftop,shift); \\\n }\n\n // load\n row0 = vld1q_s16(data + 0*8);\n row1 = vld1q_s16(data + 1*8);\n row2 = vld1q_s16(data + 2*8);\n row3 = vld1q_s16(data + 3*8);\n row4 = vld1q_s16(data + 4*8);\n row5 = vld1q_s16(data + 5*8);\n row6 = vld1q_s16(data + 6*8);\n row7 = vld1q_s16(data + 7*8);\n\n // add DC bias\n row0 = vaddq_s16(row0, vsetq_lane_s16(1024, vdupq_n_s16(0), 0));\n\n // column pass\n dct_pass(vrshrn_n_s32, 10);\n\n // 16bit 8x8 transpose\n {\n// these three map to a single VTRN.16, VTRN.32, and VSWP, respectively.\n// whether compilers actually get this is another story, sadly.\n#define dct_trn16(x, y) { int16x8x2_t t = vtrnq_s16(x, y); x = t.val[0]; y = t.val[1]; }\n#define dct_trn32(x, y) { int32x4x2_t t = vtrnq_s32(vreinterpretq_s32_s16(x), vreinterpretq_s32_s16(y)); x = vreinterpretq_s16_s32(t.val[0]); y = vreinterpretq_s16_s32(t.val[1]); }\n#define dct_trn64(x, y) { int16x8_t x0 = x; int16x8_t y0 = y; x = vcombine_s16(vget_low_s16(x0), vget_low_s16(y0)); y = vcombine_s16(vget_high_s16(x0), vget_high_s16(y0)); }\n\n // pass 1\n dct_trn16(row0, row1); // a0b0a2b2a4b4a6b6\n dct_trn16(row2, row3);\n dct_trn16(row4, row5);\n dct_trn16(row6, row7);\n\n // pass 2\n dct_trn32(row0, row2); // a0b0c0d0a4b4c4d4\n dct_trn32(row1, row3);\n dct_trn32(row4, row6);\n dct_trn32(row5, row7);\n\n // pass 3\n dct_trn64(row0, row4); // a0b0c0d0e0f0g0h0\n dct_trn64(row1, row5);\n dct_trn64(row2, row6);\n dct_trn64(row3, row7);\n\n#undef dct_trn16\n#undef dct_trn32\n#undef dct_trn64\n }\n\n // row pass\n // vrshrn_n_s32 only supports shifts up to 16, we need\n // 17. so do a non-rounding shift of 16 first then follow\n // up with a rounding shift by 1.\n dct_pass(vshrn_n_s32, 16);\n\n {\n // pack and round\n uint8x8_t p0 = vqrshrun_n_s16(row0, 1);\n uint8x8_t p1 = vqrshrun_n_s16(row1, 1);\n uint8x8_t p2 = vqrshrun_n_s16(row2, 1);\n uint8x8_t p3 = vqrshrun_n_s16(row3, 1);\n uint8x8_t p4 = vqrshrun_n_s16(row4, 1);\n uint8x8_t p5 = vqrshrun_n_s16(row5, 1);\n uint8x8_t p6 = vqrshrun_n_s16(row6, 1);\n uint8x8_t p7 = vqrshrun_n_s16(row7, 1);\n\n // again, these can translate into one instruction, but often don't.\n#define dct_trn8_8(x, y) { uint8x8x2_t t = vtrn_u8(x, y); x = t.val[0]; y = t.val[1]; }\n#define dct_trn8_16(x, y) { uint16x4x2_t t = vtrn_u16(vreinterpret_u16_u8(x), vreinterpret_u16_u8(y)); x = vreinterpret_u8_u16(t.val[0]); y = vreinterpret_u8_u16(t.val[1]); }\n#define dct_trn8_32(x, y) { uint32x2x2_t t = vtrn_u32(vreinterpret_u32_u8(x), vreinterpret_u32_u8(y)); x = vreinterpret_u8_u32(t.val[0]); y = vreinterpret_u8_u32(t.val[1]); }\n\n // sadly can't use interleaved stores here since we only write\n // 8 bytes to each scan line!\n\n // 8x8 8-bit transpose pass 1\n dct_trn8_8(p0, p1);\n dct_trn8_8(p2, p3);\n dct_trn8_8(p4, p5);\n dct_trn8_8(p6, p7);\n\n // pass 2\n dct_trn8_16(p0, p2);\n dct_trn8_16(p1, p3);\n dct_trn8_16(p4, p6);\n dct_trn8_16(p5, p7);\n\n // pass 3\n dct_trn8_32(p0, p4);\n dct_trn8_32(p1, p5);\n dct_trn8_32(p2, p6);\n dct_trn8_32(p3, p7);\n\n // store\n vst1_u8(out, p0); out += out_stride;\n vst1_u8(out, p1); out += out_stride;\n vst1_u8(out, p2); out += out_stride;\n vst1_u8(out, p3); out += out_stride;\n vst1_u8(out, p4); out += out_stride;\n vst1_u8(out, p5); out += out_stride;\n vst1_u8(out, p6); out += out_stride;\n vst1_u8(out, p7);\n\n#undef dct_trn8_8\n#undef dct_trn8_16\n#undef dct_trn8_32\n }\n\n#undef dct_long_mul\n#undef dct_long_mac\n#undef dct_widen\n#undef dct_wadd\n#undef dct_wsub\n#undef dct_bfly32o\n#undef dct_pass\n}\n\n#endif // STBI_NEON\n\n#define STBI__MARKER_none 0xff\n// if there's a pending marker from the entropy stream, return that\n// otherwise, fetch from the stream and get a marker. if there's no\n// marker, return 0xff, which is never a valid marker value\nstatic stbi_uc stbi__get_marker(stbi__jpeg *j)\n{\n stbi_uc x;\n if (j->marker != STBI__MARKER_none) { x = j->marker; j->marker = STBI__MARKER_none; return x; }\n x = stbi__get8(j->s);\n if (x != 0xff) return STBI__MARKER_none;\n while (x == 0xff)\n x = stbi__get8(j->s); // consume repeated 0xff fill bytes\n return x;\n}\n\n// in each scan, we'll have scan_n components, and the order\n// of the components is specified by order[]\n#define STBI__RESTART(x) ((x) >= 0xd0 && (x) <= 0xd7)\n\n// after a restart interval, stbi__jpeg_reset the entropy decoder and\n// the dc prediction\nstatic void stbi__jpeg_reset(stbi__jpeg *j)\n{\n j->code_bits = 0;\n j->code_buffer = 0;\n j->nomore = 0;\n j->img_comp[0].dc_pred = j->img_comp[1].dc_pred = j->img_comp[2].dc_pred = j->img_comp[3].dc_pred = 0;\n j->marker = STBI__MARKER_none;\n j->todo = j->restart_interval ? j->restart_interval : 0x7fffffff;\n j->eob_run = 0;\n // no more than 1<<31 MCUs if no restart_interal? that's plenty safe,\n // since we don't even allow 1<<30 pixels\n}\n\nstatic int stbi__parse_entropy_coded_data(stbi__jpeg *z)\n{\n stbi__jpeg_reset(z);\n if (!z->progressive) {\n if (z->scan_n == 1) {\n int i,j;\n STBI_SIMD_ALIGN(short, data[64]);\n int n = z->order[0];\n // non-interleaved data, we just need to process one block at a time,\n // in trivial scanline order\n // number of blocks to do just depends on how many actual \"pixels\" this\n // component has, independent of interleaved MCU blocking and such\n int w = (z->img_comp[n].x+7) >> 3;\n int h = (z->img_comp[n].y+7) >> 3;\n for (j=0; j < h; ++j) {\n for (i=0; i < w; ++i) {\n int ha = z->img_comp[n].ha;\n if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0;\n z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data);\n // every data block is an MCU, so countdown the restart interval\n if (--z->todo <= 0) {\n if (z->code_bits < 24) stbi__grow_buffer_unsafe(z);\n // if it's NOT a restart, then just bail, so we get corrupt data\n // rather than no data\n if (!STBI__RESTART(z->marker)) return 1;\n stbi__jpeg_reset(z);\n }\n }\n }\n return 1;\n } else { // interleaved\n int i,j,k,x,y;\n STBI_SIMD_ALIGN(short, data[64]);\n for (j=0; j < z->img_mcu_y; ++j) {\n for (i=0; i < z->img_mcu_x; ++i) {\n // scan an interleaved mcu... process scan_n components in order\n for (k=0; k < z->scan_n; ++k) {\n int n = z->order[k];\n // scan out an mcu's worth of this component; that's just determined\n // by the basic H and V specified for the component\n for (y=0; y < z->img_comp[n].v; ++y) {\n for (x=0; x < z->img_comp[n].h; ++x) {\n int x2 = (i*z->img_comp[n].h + x)*8;\n int y2 = (j*z->img_comp[n].v + y)*8;\n int ha = z->img_comp[n].ha;\n if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0;\n z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*y2+x2, z->img_comp[n].w2, data);\n }\n }\n }\n // after all interleaved components, that's an interleaved MCU,\n // so now count down the restart interval\n if (--z->todo <= 0) {\n if (z->code_bits < 24) stbi__grow_buffer_unsafe(z);\n if (!STBI__RESTART(z->marker)) return 1;\n stbi__jpeg_reset(z);\n }\n }\n }\n return 1;\n }\n } else {\n if (z->scan_n == 1) {\n int i,j;\n int n = z->order[0];\n // non-interleaved data, we just need to process one block at a time,\n // in trivial scanline order\n // number of blocks to do just depends on how many actual \"pixels\" this\n // component has, independent of interleaved MCU blocking and such\n int w = (z->img_comp[n].x+7) >> 3;\n int h = (z->img_comp[n].y+7) >> 3;\n for (j=0; j < h; ++j) {\n for (i=0; i < w; ++i) {\n short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w);\n if (z->spec_start == 0) {\n if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n))\n return 0;\n } else {\n int ha = z->img_comp[n].ha;\n if (!stbi__jpeg_decode_block_prog_ac(z, data, &z->huff_ac[ha], z->fast_ac[ha]))\n return 0;\n }\n // every data block is an MCU, so countdown the restart interval\n if (--z->todo <= 0) {\n if (z->code_bits < 24) stbi__grow_buffer_unsafe(z);\n if (!STBI__RESTART(z->marker)) return 1;\n stbi__jpeg_reset(z);\n }\n }\n }\n return 1;\n } else { // interleaved\n int i,j,k,x,y;\n for (j=0; j < z->img_mcu_y; ++j) {\n for (i=0; i < z->img_mcu_x; ++i) {\n // scan an interleaved mcu... process scan_n components in order\n for (k=0; k < z->scan_n; ++k) {\n int n = z->order[k];\n // scan out an mcu's worth of this component; that's just determined\n // by the basic H and V specified for the component\n for (y=0; y < z->img_comp[n].v; ++y) {\n for (x=0; x < z->img_comp[n].h; ++x) {\n int x2 = (i*z->img_comp[n].h + x);\n int y2 = (j*z->img_comp[n].v + y);\n short *data = z->img_comp[n].coeff + 64 * (x2 + y2 * z->img_comp[n].coeff_w);\n if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n))\n return 0;\n }\n }\n }\n // after all interleaved components, that's an interleaved MCU,\n // so now count down the restart interval\n if (--z->todo <= 0) {\n if (z->code_bits < 24) stbi__grow_buffer_unsafe(z);\n if (!STBI__RESTART(z->marker)) return 1;\n stbi__jpeg_reset(z);\n }\n }\n }\n return 1;\n }\n }\n}\n\nstatic void stbi__jpeg_dequantize(short *data, stbi__uint16 *dequant)\n{\n int i;\n for (i=0; i < 64; ++i)\n data[i] *= dequant[i];\n}\n\nstatic void stbi__jpeg_finish(stbi__jpeg *z)\n{\n if (z->progressive) {\n // dequantize and idct the data\n int i,j,n;\n for (n=0; n < z->s->img_n; ++n) {\n int w = (z->img_comp[n].x+7) >> 3;\n int h = (z->img_comp[n].y+7) >> 3;\n for (j=0; j < h; ++j) {\n for (i=0; i < w; ++i) {\n short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w);\n stbi__jpeg_dequantize(data, z->dequant[z->img_comp[n].tq]);\n z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data);\n }\n }\n }\n }\n}\n\nstatic int stbi__process_marker(stbi__jpeg *z, int m)\n{\n int L;\n switch (m) {\n case STBI__MARKER_none: // no marker found\n return stbi__err(\"expected marker\",\"Corrupt JPEG\");\n\n case 0xDD: // DRI - specify restart interval\n if (stbi__get16be(z->s) != 4) return stbi__err(\"bad DRI len\",\"Corrupt JPEG\");\n z->restart_interval = stbi__get16be(z->s);\n return 1;\n\n case 0xDB: // DQT - define quantization table\n L = stbi__get16be(z->s)-2;\n while (L > 0) {\n int q = stbi__get8(z->s);\n int p = q >> 4, sixteen = (p != 0);\n int t = q & 15,i;\n if (p != 0 && p != 1) return stbi__err(\"bad DQT type\",\"Corrupt JPEG\");\n if (t > 3) return stbi__err(\"bad DQT table\",\"Corrupt JPEG\");\n\n for (i=0; i < 64; ++i)\n z->dequant[t][stbi__jpeg_dezigzag[i]] = (stbi__uint16)(sixteen ? stbi__get16be(z->s) : stbi__get8(z->s));\n L -= (sixteen ? 129 : 65);\n }\n return L==0;\n\n case 0xC4: // DHT - define huffman table\n L = stbi__get16be(z->s)-2;\n while (L > 0) {\n stbi_uc *v;\n int sizes[16],i,n=0;\n int q = stbi__get8(z->s);\n int tc = q >> 4;\n int th = q & 15;\n if (tc > 1 || th > 3) return stbi__err(\"bad DHT header\",\"Corrupt JPEG\");\n for (i=0; i < 16; ++i) {\n sizes[i] = stbi__get8(z->s);\n n += sizes[i];\n }\n if(n > 256) return stbi__err(\"bad DHT header\",\"Corrupt JPEG\"); // Loop over i < n would write past end of values!\n L -= 17;\n if (tc == 0) {\n if (!stbi__build_huffman(z->huff_dc+th, sizes)) return 0;\n v = z->huff_dc[th].values;\n } else {\n if (!stbi__build_huffman(z->huff_ac+th, sizes)) return 0;\n v = z->huff_ac[th].values;\n }\n for (i=0; i < n; ++i)\n v[i] = stbi__get8(z->s);\n if (tc != 0)\n stbi__build_fast_ac(z->fast_ac[th], z->huff_ac + th);\n L -= n;\n }\n return L==0;\n }\n\n // check for comment block or APP blocks\n if ((m >= 0xE0 && m <= 0xEF) || m == 0xFE) {\n L = stbi__get16be(z->s);\n if (L < 2) {\n if (m == 0xFE)\n return stbi__err(\"bad COM len\",\"Corrupt JPEG\");\n else\n return stbi__err(\"bad APP len\",\"Corrupt JPEG\");\n }\n L -= 2;\n\n if (m == 0xE0 && L >= 5) { // JFIF APP0 segment\n static const unsigned char tag[5] = {'J','F','I','F','\\0'};\n int ok = 1;\n int i;\n for (i=0; i < 5; ++i)\n if (stbi__get8(z->s) != tag[i])\n ok = 0;\n L -= 5;\n if (ok)\n z->jfif = 1;\n } else if (m == 0xEE && L >= 12) { // Adobe APP14 segment\n static const unsigned char tag[6] = {'A','d','o','b','e','\\0'};\n int ok = 1;\n int i;\n for (i=0; i < 6; ++i)\n if (stbi__get8(z->s) != tag[i])\n ok = 0;\n L -= 6;\n if (ok) {\n stbi__get8(z->s); // version\n stbi__get16be(z->s); // flags0\n stbi__get16be(z->s); // flags1\n z->app14_color_transform = stbi__get8(z->s); // color transform\n L -= 6;\n }\n }\n\n stbi__skip(z->s, L);\n return 1;\n }\n\n return stbi__err(\"unknown marker\",\"Corrupt JPEG\");\n}\n\n// after we see SOS\nstatic int stbi__process_scan_header(stbi__jpeg *z)\n{\n int i;\n int Ls = stbi__get16be(z->s);\n z->scan_n = stbi__get8(z->s);\n if (z->scan_n < 1 || z->scan_n > 4 || z->scan_n > (int) z->s->img_n) return stbi__err(\"bad SOS component count\",\"Corrupt JPEG\");\n if (Ls != 6+2*z->scan_n) return stbi__err(\"bad SOS len\",\"Corrupt JPEG\");\n for (i=0; i < z->scan_n; ++i) {\n int id = stbi__get8(z->s), which;\n int q = stbi__get8(z->s);\n for (which = 0; which < z->s->img_n; ++which)\n if (z->img_comp[which].id == id)\n break;\n if (which == z->s->img_n) return 0; // no match\n z->img_comp[which].hd = q >> 4; if (z->img_comp[which].hd > 3) return stbi__err(\"bad DC huff\",\"Corrupt JPEG\");\n z->img_comp[which].ha = q & 15; if (z->img_comp[which].ha > 3) return stbi__err(\"bad AC huff\",\"Corrupt JPEG\");\n z->order[i] = which;\n }\n\n {\n int aa;\n z->spec_start = stbi__get8(z->s);\n z->spec_end = stbi__get8(z->s); // should be 63, but might be 0\n aa = stbi__get8(z->s);\n z->succ_high = (aa >> 4);\n z->succ_low = (aa & 15);\n if (z->progressive) {\n if (z->spec_start > 63 || z->spec_end > 63 || z->spec_start > z->spec_end || z->succ_high > 13 || z->succ_low > 13)\n return stbi__err(\"bad SOS\", \"Corrupt JPEG\");\n } else {\n if (z->spec_start != 0) return stbi__err(\"bad SOS\",\"Corrupt JPEG\");\n if (z->succ_high != 0 || z->succ_low != 0) return stbi__err(\"bad SOS\",\"Corrupt JPEG\");\n z->spec_end = 63;\n }\n }\n\n return 1;\n}\n\nstatic int stbi__free_jpeg_components(stbi__jpeg *z, int ncomp, int why)\n{\n int i;\n for (i=0; i < ncomp; ++i) {\n if (z->img_comp[i].raw_data) {\n STBI_FREE(z->img_comp[i].raw_data);\n z->img_comp[i].raw_data = NULL;\n z->img_comp[i].data = NULL;\n }\n if (z->img_comp[i].raw_coeff) {\n STBI_FREE(z->img_comp[i].raw_coeff);\n z->img_comp[i].raw_coeff = 0;\n z->img_comp[i].coeff = 0;\n }\n if (z->img_comp[i].linebuf) {\n STBI_FREE(z->img_comp[i].linebuf);\n z->img_comp[i].linebuf = NULL;\n }\n }\n return why;\n}\n\nstatic int stbi__process_frame_header(stbi__jpeg *z, int scan)\n{\n stbi__context *s = z->s;\n int Lf,p,i,q, h_max=1,v_max=1,c;\n Lf = stbi__get16be(s); if (Lf < 11) return stbi__err(\"bad SOF len\",\"Corrupt JPEG\"); // JPEG\n p = stbi__get8(s); if (p != 8) return stbi__err(\"only 8-bit\",\"JPEG format not supported: 8-bit only\"); // JPEG baseline\n s->img_y = stbi__get16be(s); if (s->img_y == 0) return stbi__err(\"no header height\", \"JPEG format not supported: delayed height\"); // Legal, but we don't handle it--but neither does IJG\n s->img_x = stbi__get16be(s); if (s->img_x == 0) return stbi__err(\"0 width\",\"Corrupt JPEG\"); // JPEG requires\n if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err(\"too large\",\"Very large image (corrupt?)\");\n if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err(\"too large\",\"Very large image (corrupt?)\");\n c = stbi__get8(s);\n if (c != 3 && c != 1 && c != 4) return stbi__err(\"bad component count\",\"Corrupt JPEG\");\n s->img_n = c;\n for (i=0; i < c; ++i) {\n z->img_comp[i].data = NULL;\n z->img_comp[i].linebuf = NULL;\n }\n\n if (Lf != 8+3*s->img_n) return stbi__err(\"bad SOF len\",\"Corrupt JPEG\");\n\n z->rgb = 0;\n for (i=0; i < s->img_n; ++i) {\n static const unsigned char rgb[3] = { 'R', 'G', 'B' };\n z->img_comp[i].id = stbi__get8(s);\n if (s->img_n == 3 && z->img_comp[i].id == rgb[i])\n ++z->rgb;\n q = stbi__get8(s);\n z->img_comp[i].h = (q >> 4); if (!z->img_comp[i].h || z->img_comp[i].h > 4) return stbi__err(\"bad H\",\"Corrupt JPEG\");\n z->img_comp[i].v = q & 15; if (!z->img_comp[i].v || z->img_comp[i].v > 4) return stbi__err(\"bad V\",\"Corrupt JPEG\");\n z->img_comp[i].tq = stbi__get8(s); if (z->img_comp[i].tq > 3) return stbi__err(\"bad TQ\",\"Corrupt JPEG\");\n }\n\n if (scan != STBI__SCAN_load) return 1;\n\n if (!stbi__mad3sizes_valid(s->img_x, s->img_y, s->img_n, 0)) return stbi__err(\"too large\", \"Image too large to decode\");\n\n for (i=0; i < s->img_n; ++i) {\n if (z->img_comp[i].h > h_max) h_max = z->img_comp[i].h;\n if (z->img_comp[i].v > v_max) v_max = z->img_comp[i].v;\n }\n\n // check that plane subsampling factors are integer ratios; our resamplers can't deal with fractional ratios\n // and I've never seen a non-corrupted JPEG file actually use them\n for (i=0; i < s->img_n; ++i) {\n if (h_max % z->img_comp[i].h != 0) return stbi__err(\"bad H\",\"Corrupt JPEG\");\n if (v_max % z->img_comp[i].v != 0) return stbi__err(\"bad V\",\"Corrupt JPEG\");\n }\n\n // compute interleaved mcu info\n z->img_h_max = h_max;\n z->img_v_max = v_max;\n z->img_mcu_w = h_max * 8;\n z->img_mcu_h = v_max * 8;\n // these sizes can't be more than 17 bits\n z->img_mcu_x = (s->img_x + z->img_mcu_w-1) / z->img_mcu_w;\n z->img_mcu_y = (s->img_y + z->img_mcu_h-1) / z->img_mcu_h;\n\n for (i=0; i < s->img_n; ++i) {\n // number of effective pixels (e.g. for non-interleaved MCU)\n z->img_comp[i].x = (s->img_x * z->img_comp[i].h + h_max-1) / h_max;\n z->img_comp[i].y = (s->img_y * z->img_comp[i].v + v_max-1) / v_max;\n // to simplify generation, we'll allocate enough memory to decode\n // the bogus oversized data from using interleaved MCUs and their\n // big blocks (e.g. a 16x16 iMCU on an image of width 33); we won't\n // discard the extra data until colorspace conversion\n //\n // img_mcu_x, img_mcu_y: <=17 bits; comp[i].h and .v are <=4 (checked earlier)\n // so these muls can't overflow with 32-bit ints (which we require)\n z->img_comp[i].w2 = z->img_mcu_x * z->img_comp[i].h * 8;\n z->img_comp[i].h2 = z->img_mcu_y * z->img_comp[i].v * 8;\n z->img_comp[i].coeff = 0;\n z->img_comp[i].raw_coeff = 0;\n z->img_comp[i].linebuf = NULL;\n z->img_comp[i].raw_data = stbi__malloc_mad2(z->img_comp[i].w2, z->img_comp[i].h2, 15);\n if (z->img_comp[i].raw_data == NULL)\n return stbi__free_jpeg_components(z, i+1, stbi__err(\"outofmem\", \"Out of memory\"));\n // align blocks for idct using mmx/sse\n z->img_comp[i].data = (stbi_uc*) (((size_t) z->img_comp[i].raw_data + 15) & ~15);\n if (z->progressive) {\n // w2, h2 are multiples of 8 (see above)\n z->img_comp[i].coeff_w = z->img_comp[i].w2 / 8;\n z->img_comp[i].coeff_h = z->img_comp[i].h2 / 8;\n z->img_comp[i].raw_coeff = stbi__malloc_mad3(z->img_comp[i].w2, z->img_comp[i].h2, sizeof(short), 15);\n if (z->img_comp[i].raw_coeff == NULL)\n return stbi__free_jpeg_components(z, i+1, stbi__err(\"outofmem\", \"Out of memory\"));\n z->img_comp[i].coeff = (short*) (((size_t) z->img_comp[i].raw_coeff + 15) & ~15);\n }\n }\n\n return 1;\n}\n\n// use comparisons since in some cases we handle more than one case (e.g. SOF)\n#define stbi__DNL(x) ((x) == 0xdc)\n#define stbi__SOI(x) ((x) == 0xd8)\n#define stbi__EOI(x) ((x) == 0xd9)\n#define stbi__SOF(x) ((x) == 0xc0 || (x) == 0xc1 || (x) == 0xc2)\n#define stbi__SOS(x) ((x) == 0xda)\n\n#define stbi__SOF_progressive(x) ((x) == 0xc2)\n\nstatic int stbi__decode_jpeg_header(stbi__jpeg *z, int scan)\n{\n int m;\n z->jfif = 0;\n z->app14_color_transform = -1; // valid values are 0,1,2\n z->marker = STBI__MARKER_none; // initialize cached marker to empty\n m = stbi__get_marker(z);\n if (!stbi__SOI(m)) return stbi__err(\"no SOI\",\"Corrupt JPEG\");\n if (scan == STBI__SCAN_type) return 1;\n m = stbi__get_marker(z);\n while (!stbi__SOF(m)) {\n if (!stbi__process_marker(z,m)) return 0;\n m = stbi__get_marker(z);\n while (m == STBI__MARKER_none) {\n // some files have extra padding after their blocks, so ok, we'll scan\n if (stbi__at_eof(z->s)) return stbi__err(\"no SOF\", \"Corrupt JPEG\");\n m = stbi__get_marker(z);\n }\n }\n z->progressive = stbi__SOF_progressive(m);\n if (!stbi__process_frame_header(z, scan)) return 0;\n return 1;\n}\n\nstatic int stbi__skip_jpeg_junk_at_end(stbi__jpeg *j)\n{\n // some JPEGs have junk at end, skip over it but if we find what looks\n // like a valid marker, resume there\n while (!stbi__at_eof(j->s)) {\n int x = stbi__get8(j->s);\n while (x == 255) { // might be a marker\n if (stbi__at_eof(j->s)) return STBI__MARKER_none;\n x = stbi__get8(j->s);\n if (x != 0x00 && x != 0xff) {\n // not a stuffed zero or lead-in to another marker, looks\n // like an actual marker, return it\n return x;\n }\n // stuffed zero has x=0 now which ends the loop, meaning we go\n // back to regular scan loop.\n // repeated 0xff keeps trying to read the next byte of the marker.\n }\n }\n return STBI__MARKER_none;\n}\n\n// decode image to YCbCr format\nstatic int stbi__decode_jpeg_image(stbi__jpeg *j)\n{\n int m;\n for (m = 0; m < 4; m++) {\n j->img_comp[m].raw_data = NULL;\n j->img_comp[m].raw_coeff = NULL;\n }\n j->restart_interval = 0;\n if (!stbi__decode_jpeg_header(j, STBI__SCAN_load)) return 0;\n m = stbi__get_marker(j);\n while (!stbi__EOI(m)) {\n if (stbi__SOS(m)) {\n if (!stbi__process_scan_header(j)) return 0;\n if (!stbi__parse_entropy_coded_data(j)) return 0;\n if (j->marker == STBI__MARKER_none ) {\n j->marker = stbi__skip_jpeg_junk_at_end(j);\n // if we reach eof without hitting a marker, stbi__get_marker() below will fail and we'll eventually return 0\n }\n m = stbi__get_marker(j);\n if (STBI__RESTART(m))\n m = stbi__get_marker(j);\n } else if (stbi__DNL(m)) {\n int Ld = stbi__get16be(j->s);\n stbi__uint32 NL = stbi__get16be(j->s);\n if (Ld != 4) return stbi__err(\"bad DNL len\", \"Corrupt JPEG\");\n if (NL != j->s->img_y) return stbi__err(\"bad DNL height\", \"Corrupt JPEG\");\n m = stbi__get_marker(j);\n } else {\n if (!stbi__process_marker(j, m)) return 1;\n m = stbi__get_marker(j);\n }\n }\n if (j->progressive)\n stbi__jpeg_finish(j);\n return 1;\n}\n\n// static jfif-centered resampling (across block boundaries)\n\ntypedef stbi_uc *(*resample_row_func)(stbi_uc *out, stbi_uc *in0, stbi_uc *in1,\n int w, int hs);\n\n#define stbi__div4(x) ((stbi_uc) ((x) >> 2))\n\nstatic stbi_uc *resample_row_1(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)\n{\n STBI_NOTUSED(out);\n STBI_NOTUSED(in_far);\n STBI_NOTUSED(w);\n STBI_NOTUSED(hs);\n return in_near;\n}\n\nstatic stbi_uc* stbi__resample_row_v_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)\n{\n // need to generate two samples vertically for every one in input\n int i;\n STBI_NOTUSED(hs);\n for (i=0; i < w; ++i)\n out[i] = stbi__div4(3*in_near[i] + in_far[i] + 2);\n return out;\n}\n\nstatic stbi_uc* stbi__resample_row_h_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)\n{\n // need to generate two samples horizontally for every one in input\n int i;\n stbi_uc *input = in_near;\n\n if (w == 1) {\n // if only one sample, can't do any interpolation\n out[0] = out[1] = input[0];\n return out;\n }\n\n out[0] = input[0];\n out[1] = stbi__div4(input[0]*3 + input[1] + 2);\n for (i=1; i < w-1; ++i) {\n int n = 3*input[i]+2;\n out[i*2+0] = stbi__div4(n+input[i-1]);\n out[i*2+1] = stbi__div4(n+input[i+1]);\n }\n out[i*2+0] = stbi__div4(input[w-2]*3 + input[w-1] + 2);\n out[i*2+1] = input[w-1];\n\n STBI_NOTUSED(in_far);\n STBI_NOTUSED(hs);\n\n return out;\n}\n\n#define stbi__div16(x) ((stbi_uc) ((x) >> 4))\n\nstatic stbi_uc *stbi__resample_row_hv_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)\n{\n // need to generate 2x2 samples for every one in input\n int i,t0,t1;\n if (w == 1) {\n out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2);\n return out;\n }\n\n t1 = 3*in_near[0] + in_far[0];\n out[0] = stbi__div4(t1+2);\n for (i=1; i < w; ++i) {\n t0 = t1;\n t1 = 3*in_near[i]+in_far[i];\n out[i*2-1] = stbi__div16(3*t0 + t1 + 8);\n out[i*2 ] = stbi__div16(3*t1 + t0 + 8);\n }\n out[w*2-1] = stbi__div4(t1+2);\n\n STBI_NOTUSED(hs);\n\n return out;\n}\n\n#if defined(STBI_SSE2) || defined(STBI_NEON)\nstatic stbi_uc *stbi__resample_row_hv_2_simd(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)\n{\n // need to generate 2x2 samples for every one in input\n int i=0,t0,t1;\n\n if (w == 1) {\n out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2);\n return out;\n }\n\n t1 = 3*in_near[0] + in_far[0];\n // process groups of 8 pixels for as long as we can.\n // note we can't handle the last pixel in a row in this loop\n // because we need to handle the filter boundary conditions.\n for (; i < ((w-1) & ~7); i += 8) {\n#if defined(STBI_SSE2)\n // load and perform the vertical filtering pass\n // this uses 3*x + y = 4*x + (y - x)\n __m128i zero = _mm_setzero_si128();\n __m128i farb = _mm_loadl_epi64((__m128i *) (in_far + i));\n __m128i nearb = _mm_loadl_epi64((__m128i *) (in_near + i));\n __m128i farw = _mm_unpacklo_epi8(farb, zero);\n __m128i nearw = _mm_unpacklo_epi8(nearb, zero);\n __m128i diff = _mm_sub_epi16(farw, nearw);\n __m128i nears = _mm_slli_epi16(nearw, 2);\n __m128i curr = _mm_add_epi16(nears, diff); // current row\n\n // horizontal filter works the same based on shifted vers of current\n // row. \"prev\" is current row shifted right by 1 pixel; we need to\n // insert the previous pixel value (from t1).\n // \"next\" is current row shifted left by 1 pixel, with first pixel\n // of next block of 8 pixels added in.\n __m128i prv0 = _mm_slli_si128(curr, 2);\n __m128i nxt0 = _mm_srli_si128(curr, 2);\n __m128i prev = _mm_insert_epi16(prv0, t1, 0);\n __m128i next = _mm_insert_epi16(nxt0, 3*in_near[i+8] + in_far[i+8], 7);\n\n // horizontal filter, polyphase implementation since it's convenient:\n // even pixels = 3*cur + prev = cur*4 + (prev - cur)\n // odd pixels = 3*cur + next = cur*4 + (next - cur)\n // note the shared term.\n __m128i bias = _mm_set1_epi16(8);\n __m128i curs = _mm_slli_epi16(curr, 2);\n __m128i prvd = _mm_sub_epi16(prev, curr);\n __m128i nxtd = _mm_sub_epi16(next, curr);\n __m128i curb = _mm_add_epi16(curs, bias);\n __m128i even = _mm_add_epi16(prvd, curb);\n __m128i odd = _mm_add_epi16(nxtd, curb);\n\n // interleave even and odd pixels, then undo scaling.\n __m128i int0 = _mm_unpacklo_epi16(even, odd);\n __m128i int1 = _mm_unpackhi_epi16(even, odd);\n __m128i de0 = _mm_srli_epi16(int0, 4);\n __m128i de1 = _mm_srli_epi16(int1, 4);\n\n // pack and write output\n __m128i outv = _mm_packus_epi16(de0, de1);\n _mm_storeu_si128((__m128i *) (out + i*2), outv);\n#elif defined(STBI_NEON)\n // load and perform the vertical filtering pass\n // this uses 3*x + y = 4*x + (y - x)\n uint8x8_t farb = vld1_u8(in_far + i);\n uint8x8_t nearb = vld1_u8(in_near + i);\n int16x8_t diff = vreinterpretq_s16_u16(vsubl_u8(farb, nearb));\n int16x8_t nears = vreinterpretq_s16_u16(vshll_n_u8(nearb, 2));\n int16x8_t curr = vaddq_s16(nears, diff); // current row\n\n // horizontal filter works the same based on shifted vers of current\n // row. \"prev\" is current row shifted right by 1 pixel; we need to\n // insert the previous pixel value (from t1).\n // \"next\" is current row shifted left by 1 pixel, with first pixel\n // of next block of 8 pixels added in.\n int16x8_t prv0 = vextq_s16(curr, curr, 7);\n int16x8_t nxt0 = vextq_s16(curr, curr, 1);\n int16x8_t prev = vsetq_lane_s16(t1, prv0, 0);\n int16x8_t next = vsetq_lane_s16(3*in_near[i+8] + in_far[i+8], nxt0, 7);\n\n // horizontal filter, polyphase implementation since it's convenient:\n // even pixels = 3*cur + prev = cur*4 + (prev - cur)\n // odd pixels = 3*cur + next = cur*4 + (next - cur)\n // note the shared term.\n int16x8_t curs = vshlq_n_s16(curr, 2);\n int16x8_t prvd = vsubq_s16(prev, curr);\n int16x8_t nxtd = vsubq_s16(next, curr);\n int16x8_t even = vaddq_s16(curs, prvd);\n int16x8_t odd = vaddq_s16(curs, nxtd);\n\n // undo scaling and round, then store with even/odd phases interleaved\n uint8x8x2_t o;\n o.val[0] = vqrshrun_n_s16(even, 4);\n o.val[1] = vqrshrun_n_s16(odd, 4);\n vst2_u8(out + i*2, o);\n#endif\n\n // \"previous\" value for next iter\n t1 = 3*in_near[i+7] + in_far[i+7];\n }\n\n t0 = t1;\n t1 = 3*in_near[i] + in_far[i];\n out[i*2] = stbi__div16(3*t1 + t0 + 8);\n\n for (++i; i < w; ++i) {\n t0 = t1;\n t1 = 3*in_near[i]+in_far[i];\n out[i*2-1] = stbi__div16(3*t0 + t1 + 8);\n out[i*2 ] = stbi__div16(3*t1 + t0 + 8);\n }\n out[w*2-1] = stbi__div4(t1+2);\n\n STBI_NOTUSED(hs);\n\n return out;\n}\n#endif\n\nstatic stbi_uc *stbi__resample_row_generic(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs)\n{\n // resample with nearest-neighbor\n int i,j;\n STBI_NOTUSED(in_far);\n for (i=0; i < w; ++i)\n for (j=0; j < hs; ++j)\n out[i*hs+j] = in_near[i];\n return out;\n}\n\n// this is a reduced-precision calculation of YCbCr-to-RGB introduced\n// to make sure the code produces the same results in both SIMD and scalar\n#define stbi__float2fixed(x) (((int) ((x) * 4096.0f + 0.5f)) << 8)\nstatic void stbi__YCbCr_to_RGB_row(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step)\n{\n int i;\n for (i=0; i < count; ++i) {\n int y_fixed = (y[i] << 20) + (1<<19); // rounding\n int r,g,b;\n int cr = pcr[i] - 128;\n int cb = pcb[i] - 128;\n r = y_fixed + cr* stbi__float2fixed(1.40200f);\n g = y_fixed + (cr*-stbi__float2fixed(0.71414f)) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000);\n b = y_fixed + cb* stbi__float2fixed(1.77200f);\n r >>= 20;\n g >>= 20;\n b >>= 20;\n if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; }\n if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; }\n if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; }\n out[0] = (stbi_uc)r;\n out[1] = (stbi_uc)g;\n out[2] = (stbi_uc)b;\n out[3] = 255;\n out += step;\n }\n}\n\n#if defined(STBI_SSE2) || defined(STBI_NEON)\nstatic void stbi__YCbCr_to_RGB_simd(stbi_uc *out, stbi_uc const *y, stbi_uc const *pcb, stbi_uc const *pcr, int count, int step)\n{\n int i = 0;\n\n#ifdef STBI_SSE2\n // step == 3 is pretty ugly on the final interleave, and i'm not convinced\n // it's useful in practice (you wouldn't use it for textures, for example).\n // so just accelerate step == 4 case.\n if (step == 4) {\n // this is a fairly straightforward implementation and not super-optimized.\n __m128i signflip = _mm_set1_epi8(-0x80);\n __m128i cr_const0 = _mm_set1_epi16( (short) ( 1.40200f*4096.0f+0.5f));\n __m128i cr_const1 = _mm_set1_epi16( - (short) ( 0.71414f*4096.0f+0.5f));\n __m128i cb_const0 = _mm_set1_epi16( - (short) ( 0.34414f*4096.0f+0.5f));\n __m128i cb_const1 = _mm_set1_epi16( (short) ( 1.77200f*4096.0f+0.5f));\n __m128i y_bias = _mm_set1_epi8((char) (unsigned char) 128);\n __m128i xw = _mm_set1_epi16(255); // alpha channel\n\n for (; i+7 < count; i += 8) {\n // load\n __m128i y_bytes = _mm_loadl_epi64((__m128i *) (y+i));\n __m128i cr_bytes = _mm_loadl_epi64((__m128i *) (pcr+i));\n __m128i cb_bytes = _mm_loadl_epi64((__m128i *) (pcb+i));\n __m128i cr_biased = _mm_xor_si128(cr_bytes, signflip); // -128\n __m128i cb_biased = _mm_xor_si128(cb_bytes, signflip); // -128\n\n // unpack to short (and left-shift cr, cb by 8)\n __m128i yw = _mm_unpacklo_epi8(y_bias, y_bytes);\n __m128i crw = _mm_unpacklo_epi8(_mm_setzero_si128(), cr_biased);\n __m128i cbw = _mm_unpacklo_epi8(_mm_setzero_si128(), cb_biased);\n\n // color transform\n __m128i yws = _mm_srli_epi16(yw, 4);\n __m128i cr0 = _mm_mulhi_epi16(cr_const0, crw);\n __m128i cb0 = _mm_mulhi_epi16(cb_const0, cbw);\n __m128i cb1 = _mm_mulhi_epi16(cbw, cb_const1);\n __m128i cr1 = _mm_mulhi_epi16(crw, cr_const1);\n __m128i rws = _mm_add_epi16(cr0, yws);\n __m128i gwt = _mm_add_epi16(cb0, yws);\n __m128i bws = _mm_add_epi16(yws, cb1);\n __m128i gws = _mm_add_epi16(gwt, cr1);\n\n // descale\n __m128i rw = _mm_srai_epi16(rws, 4);\n __m128i bw = _mm_srai_epi16(bws, 4);\n __m128i gw = _mm_srai_epi16(gws, 4);\n\n // back to byte, set up for transpose\n __m128i brb = _mm_packus_epi16(rw, bw);\n __m128i gxb = _mm_packus_epi16(gw, xw);\n\n // transpose to interleave channels\n __m128i t0 = _mm_unpacklo_epi8(brb, gxb);\n __m128i t1 = _mm_unpackhi_epi8(brb, gxb);\n __m128i o0 = _mm_unpacklo_epi16(t0, t1);\n __m128i o1 = _mm_unpackhi_epi16(t0, t1);\n\n // store\n _mm_storeu_si128((__m128i *) (out + 0), o0);\n _mm_storeu_si128((__m128i *) (out + 16), o1);\n out += 32;\n }\n }\n#endif\n\n#ifdef STBI_NEON\n // in this version, step=3 support would be easy to add. but is there demand?\n if (step == 4) {\n // this is a fairly straightforward implementation and not super-optimized.\n uint8x8_t signflip = vdup_n_u8(0x80);\n int16x8_t cr_const0 = vdupq_n_s16( (short) ( 1.40200f*4096.0f+0.5f));\n int16x8_t cr_const1 = vdupq_n_s16( - (short) ( 0.71414f*4096.0f+0.5f));\n int16x8_t cb_const0 = vdupq_n_s16( - (short) ( 0.34414f*4096.0f+0.5f));\n int16x8_t cb_const1 = vdupq_n_s16( (short) ( 1.77200f*4096.0f+0.5f));\n\n for (; i+7 < count; i += 8) {\n // load\n uint8x8_t y_bytes = vld1_u8(y + i);\n uint8x8_t cr_bytes = vld1_u8(pcr + i);\n uint8x8_t cb_bytes = vld1_u8(pcb + i);\n int8x8_t cr_biased = vreinterpret_s8_u8(vsub_u8(cr_bytes, signflip));\n int8x8_t cb_biased = vreinterpret_s8_u8(vsub_u8(cb_bytes, signflip));\n\n // expand to s16\n int16x8_t yws = vreinterpretq_s16_u16(vshll_n_u8(y_bytes, 4));\n int16x8_t crw = vshll_n_s8(cr_biased, 7);\n int16x8_t cbw = vshll_n_s8(cb_biased, 7);\n\n // color transform\n int16x8_t cr0 = vqdmulhq_s16(crw, cr_const0);\n int16x8_t cb0 = vqdmulhq_s16(cbw, cb_const0);\n int16x8_t cr1 = vqdmulhq_s16(crw, cr_const1);\n int16x8_t cb1 = vqdmulhq_s16(cbw, cb_const1);\n int16x8_t rws = vaddq_s16(yws, cr0);\n int16x8_t gws = vaddq_s16(vaddq_s16(yws, cb0), cr1);\n int16x8_t bws = vaddq_s16(yws, cb1);\n\n // undo scaling, round, convert to byte\n uint8x8x4_t o;\n o.val[0] = vqrshrun_n_s16(rws, 4);\n o.val[1] = vqrshrun_n_s16(gws, 4);\n o.val[2] = vqrshrun_n_s16(bws, 4);\n o.val[3] = vdup_n_u8(255);\n\n // store, interleaving r/g/b/a\n vst4_u8(out, o);\n out += 8*4;\n }\n }\n#endif\n\n for (; i < count; ++i) {\n int y_fixed = (y[i] << 20) + (1<<19); // rounding\n int r,g,b;\n int cr = pcr[i] - 128;\n int cb = pcb[i] - 128;\n r = y_fixed + cr* stbi__float2fixed(1.40200f);\n g = y_fixed + cr*-stbi__float2fixed(0.71414f) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000);\n b = y_fixed + cb* stbi__float2fixed(1.77200f);\n r >>= 20;\n g >>= 20;\n b >>= 20;\n if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; }\n if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; }\n if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; }\n out[0] = (stbi_uc)r;\n out[1] = (stbi_uc)g;\n out[2] = (stbi_uc)b;\n out[3] = 255;\n out += step;\n }\n}\n#endif\n\n// set up the kernels\nstatic void stbi__setup_jpeg(stbi__jpeg *j)\n{\n j->idct_block_kernel = stbi__idct_block;\n j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_row;\n j->resample_row_hv_2_kernel = stbi__resample_row_hv_2;\n\n#ifdef STBI_SSE2\n if (stbi__sse2_available()) {\n j->idct_block_kernel = stbi__idct_simd;\n j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd;\n j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd;\n }\n#endif\n\n#ifdef STBI_NEON\n j->idct_block_kernel = stbi__idct_simd;\n j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd;\n j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd;\n#endif\n}\n\n// clean up the temporary component buffers\nstatic void stbi__cleanup_jpeg(stbi__jpeg *j)\n{\n stbi__free_jpeg_components(j, j->s->img_n, 0);\n}\n\ntypedef struct\n{\n resample_row_func resample;\n stbi_uc *line0,*line1;\n int hs,vs; // expansion factor in each axis\n int w_lores; // horizontal pixels pre-expansion\n int ystep; // how far through vertical expansion we are\n int ypos; // which pre-expansion row we're on\n} stbi__resample;\n\n// fast 0..255 * 0..255 => 0..255 rounded multiplication\nstatic stbi_uc stbi__blinn_8x8(stbi_uc x, stbi_uc y)\n{\n unsigned int t = x*y + 128;\n return (stbi_uc) ((t + (t >>8)) >> 8);\n}\n\nstatic stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp)\n{\n int n, decode_n, is_rgb;\n z->s->img_n = 0; // make stbi__cleanup_jpeg safe\n\n // validate req_comp\n if (req_comp < 0 || req_comp > 4) return stbi__errpuc(\"bad req_comp\", \"Internal error\");\n\n // load a jpeg image from whichever source, but leave in YCbCr format\n if (!stbi__decode_jpeg_image(z)) { stbi__cleanup_jpeg(z); return NULL; }\n\n // determine actual number of components to generate\n n = req_comp ? req_comp : z->s->img_n >= 3 ? 3 : 1;\n\n is_rgb = z->s->img_n == 3 && (z->rgb == 3 || (z->app14_color_transform == 0 && !z->jfif));\n\n if (z->s->img_n == 3 && n < 3 && !is_rgb)\n decode_n = 1;\n else\n decode_n = z->s->img_n;\n\n // nothing to do if no components requested; check this now to avoid\n // accessing uninitialized coutput[0] later\n if (decode_n <= 0) { stbi__cleanup_jpeg(z); return NULL; }\n\n // resample and color-convert\n {\n int k;\n unsigned int i,j;\n stbi_uc *output;\n stbi_uc *coutput[4] = { NULL, NULL, NULL, NULL };\n\n stbi__resample res_comp[4];\n\n for (k=0; k < decode_n; ++k) {\n stbi__resample *r = &res_comp[k];\n\n // allocate line buffer big enough for upsampling off the edges\n // with upsample factor of 4\n z->img_comp[k].linebuf = (stbi_uc *) stbi__malloc(z->s->img_x + 3);\n if (!z->img_comp[k].linebuf) { stbi__cleanup_jpeg(z); return stbi__errpuc(\"outofmem\", \"Out of memory\"); }\n\n r->hs = z->img_h_max / z->img_comp[k].h;\n r->vs = z->img_v_max / z->img_comp[k].v;\n r->ystep = r->vs >> 1;\n r->w_lores = (z->s->img_x + r->hs-1) / r->hs;\n r->ypos = 0;\n r->line0 = r->line1 = z->img_comp[k].data;\n\n if (r->hs == 1 && r->vs == 1) r->resample = resample_row_1;\n else if (r->hs == 1 && r->vs == 2) r->resample = stbi__resample_row_v_2;\n else if (r->hs == 2 && r->vs == 1) r->resample = stbi__resample_row_h_2;\n else if (r->hs == 2 && r->vs == 2) r->resample = z->resample_row_hv_2_kernel;\n else r->resample = stbi__resample_row_generic;\n }\n\n // can't error after this so, this is safe\n output = (stbi_uc *) stbi__malloc_mad3(n, z->s->img_x, z->s->img_y, 1);\n if (!output) { stbi__cleanup_jpeg(z); return stbi__errpuc(\"outofmem\", \"Out of memory\"); }\n\n // now go ahead and resample\n for (j=0; j < z->s->img_y; ++j) {\n stbi_uc *out = output + n * z->s->img_x * j;\n for (k=0; k < decode_n; ++k) {\n stbi__resample *r = &res_comp[k];\n int y_bot = r->ystep >= (r->vs >> 1);\n coutput[k] = r->resample(z->img_comp[k].linebuf,\n y_bot ? r->line1 : r->line0,\n y_bot ? r->line0 : r->line1,\n r->w_lores, r->hs);\n if (++r->ystep >= r->vs) {\n r->ystep = 0;\n r->line0 = r->line1;\n if (++r->ypos < z->img_comp[k].y)\n r->line1 += z->img_comp[k].w2;\n }\n }\n if (n >= 3) {\n stbi_uc *y = coutput[0];\n if (z->s->img_n == 3) {\n if (is_rgb) {\n for (i=0; i < z->s->img_x; ++i) {\n out[0] = y[i];\n out[1] = coutput[1][i];\n out[2] = coutput[2][i];\n out[3] = 255;\n out += n;\n }\n } else {\n z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n);\n }\n } else if (z->s->img_n == 4) {\n if (z->app14_color_transform == 0) { // CMYK\n for (i=0; i < z->s->img_x; ++i) {\n stbi_uc m = coutput[3][i];\n out[0] = stbi__blinn_8x8(coutput[0][i], m);\n out[1] = stbi__blinn_8x8(coutput[1][i], m);\n out[2] = stbi__blinn_8x8(coutput[2][i], m);\n out[3] = 255;\n out += n;\n }\n } else if (z->app14_color_transform == 2) { // YCCK\n z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n);\n for (i=0; i < z->s->img_x; ++i) {\n stbi_uc m = coutput[3][i];\n out[0] = stbi__blinn_8x8(255 - out[0], m);\n out[1] = stbi__blinn_8x8(255 - out[1], m);\n out[2] = stbi__blinn_8x8(255 - out[2], m);\n out += n;\n }\n } else { // YCbCr + alpha? Ignore the fourth channel for now\n z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n);\n }\n } else\n for (i=0; i < z->s->img_x; ++i) {\n out[0] = out[1] = out[2] = y[i];\n out[3] = 255; // not used if n==3\n out += n;\n }\n } else {\n if (is_rgb) {\n if (n == 1)\n for (i=0; i < z->s->img_x; ++i)\n *out++ = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]);\n else {\n for (i=0; i < z->s->img_x; ++i, out += 2) {\n out[0] = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]);\n out[1] = 255;\n }\n }\n } else if (z->s->img_n == 4 && z->app14_color_transform == 0) {\n for (i=0; i < z->s->img_x; ++i) {\n stbi_uc m = coutput[3][i];\n stbi_uc r = stbi__blinn_8x8(coutput[0][i], m);\n stbi_uc g = stbi__blinn_8x8(coutput[1][i], m);\n stbi_uc b = stbi__blinn_8x8(coutput[2][i], m);\n out[0] = stbi__compute_y(r, g, b);\n out[1] = 255;\n out += n;\n }\n } else if (z->s->img_n == 4 && z->app14_color_transform == 2) {\n for (i=0; i < z->s->img_x; ++i) {\n out[0] = stbi__blinn_8x8(255 - coutput[0][i], coutput[3][i]);\n out[1] = 255;\n out += n;\n }\n } else {\n stbi_uc *y = coutput[0];\n if (n == 1)\n for (i=0; i < z->s->img_x; ++i) out[i] = y[i];\n else\n for (i=0; i < z->s->img_x; ++i) { *out++ = y[i]; *out++ = 255; }\n }\n }\n }\n stbi__cleanup_jpeg(z);\n *out_x = z->s->img_x;\n *out_y = z->s->img_y;\n if (comp) *comp = z->s->img_n >= 3 ? 3 : 1; // report original components, not output\n return output;\n }\n}\n\nstatic void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)\n{\n unsigned char* result;\n stbi__jpeg* j = (stbi__jpeg*) stbi__malloc(sizeof(stbi__jpeg));\n if (!j) return stbi__errpuc(\"outofmem\", \"Out of memory\");\n memset(j, 0, sizeof(stbi__jpeg));\n STBI_NOTUSED(ri);\n j->s = s;\n stbi__setup_jpeg(j);\n result = load_jpeg_image(j, x,y,comp,req_comp);\n STBI_FREE(j);\n return result;\n}\n\nstatic int stbi__jpeg_test(stbi__context *s)\n{\n int r;\n stbi__jpeg* j = (stbi__jpeg*)stbi__malloc(sizeof(stbi__jpeg));\n if (!j) return stbi__err(\"outofmem\", \"Out of memory\");\n memset(j, 0, sizeof(stbi__jpeg));\n j->s = s;\n stbi__setup_jpeg(j);\n r = stbi__decode_jpeg_header(j, STBI__SCAN_type);\n stbi__rewind(s);\n STBI_FREE(j);\n return r;\n}\n\nstatic int stbi__jpeg_info_raw(stbi__jpeg *j, int *x, int *y, int *comp)\n{\n if (!stbi__decode_jpeg_header(j, STBI__SCAN_header)) {\n stbi__rewind( j->s );\n return 0;\n }\n if (x) *x = j->s->img_x;\n if (y) *y = j->s->img_y;\n if (comp) *comp = j->s->img_n >= 3 ? 3 : 1;\n return 1;\n}\n\nstatic int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp)\n{\n int result;\n stbi__jpeg* j = (stbi__jpeg*) (stbi__malloc(sizeof(stbi__jpeg)));\n if (!j) return stbi__err(\"outofmem\", \"Out of memory\");\n memset(j, 0, sizeof(stbi__jpeg));\n j->s = s;\n result = stbi__jpeg_info_raw(j, x, y, comp);\n STBI_FREE(j);\n return result;\n}\n#endif\n\n// public domain zlib decode v0.2 Sean Barrett 2006-11-18\n// simple implementation\n// - all input must be provided in an upfront buffer\n// - all output is written to a single output buffer (can malloc/realloc)\n// performance\n// - fast huffman\n\n#ifndef STBI_NO_ZLIB\n\n// fast-way is faster to check than jpeg huffman, but slow way is slower\n#define STBI__ZFAST_BITS 9 // accelerate all cases in default tables\n#define STBI__ZFAST_MASK ((1 << STBI__ZFAST_BITS) - 1)\n#define STBI__ZNSYMS 288 // number of symbols in literal/length alphabet\n\n// zlib-style huffman encoding\n// (jpegs packs from left, zlib from right, so can't share code)\ntypedef struct\n{\n stbi__uint16 fast[1 << STBI__ZFAST_BITS];\n stbi__uint16 firstcode[16];\n int maxcode[17];\n stbi__uint16 firstsymbol[16];\n stbi_uc size[STBI__ZNSYMS];\n stbi__uint16 value[STBI__ZNSYMS];\n} stbi__zhuffman;\n\nstbi_inline static int stbi__bitreverse16(int n)\n{\n n = ((n & 0xAAAA) >> 1) | ((n & 0x5555) << 1);\n n = ((n & 0xCCCC) >> 2) | ((n & 0x3333) << 2);\n n = ((n & 0xF0F0) >> 4) | ((n & 0x0F0F) << 4);\n n = ((n & 0xFF00) >> 8) | ((n & 0x00FF) << 8);\n return n;\n}\n\nstbi_inline static int stbi__bit_reverse(int v, int bits)\n{\n STBI_ASSERT(bits <= 16);\n // to bit reverse n bits, reverse 16 and shift\n // e.g. 11 bits, bit reverse and shift away 5\n return stbi__bitreverse16(v) >> (16-bits);\n}\n\nstatic int stbi__zbuild_huffman(stbi__zhuffman *z, const stbi_uc *sizelist, int num)\n{\n int i,k=0;\n int code, next_code[16], sizes[17];\n\n // DEFLATE spec for generating codes\n memset(sizes, 0, sizeof(sizes));\n memset(z->fast, 0, sizeof(z->fast));\n for (i=0; i < num; ++i)\n ++sizes[sizelist[i]];\n sizes[0] = 0;\n for (i=1; i < 16; ++i)\n if (sizes[i] > (1 << i))\n return stbi__err(\"bad sizes\", \"Corrupt PNG\");\n code = 0;\n for (i=1; i < 16; ++i) {\n next_code[i] = code;\n z->firstcode[i] = (stbi__uint16) code;\n z->firstsymbol[i] = (stbi__uint16) k;\n code = (code + sizes[i]);\n if (sizes[i])\n if (code-1 >= (1 << i)) return stbi__err(\"bad codelengths\",\"Corrupt PNG\");\n z->maxcode[i] = code << (16-i); // preshift for inner loop\n code <<= 1;\n k += sizes[i];\n }\n z->maxcode[16] = 0x10000; // sentinel\n for (i=0; i < num; ++i) {\n int s = sizelist[i];\n if (s) {\n int c = next_code[s] - z->firstcode[s] + z->firstsymbol[s];\n stbi__uint16 fastv = (stbi__uint16) ((s << 9) | i);\n z->size [c] = (stbi_uc ) s;\n z->value[c] = (stbi__uint16) i;\n if (s <= STBI__ZFAST_BITS) {\n int j = stbi__bit_reverse(next_code[s],s);\n while (j < (1 << STBI__ZFAST_BITS)) {\n z->fast[j] = fastv;\n j += (1 << s);\n }\n }\n ++next_code[s];\n }\n }\n return 1;\n}\n\n// zlib-from-memory implementation for PNG reading\n// because PNG allows splitting the zlib stream arbitrarily,\n// and it's annoying structurally to have PNG call ZLIB call PNG,\n// we require PNG read all the IDATs and combine them into a single\n// memory buffer\n\ntypedef struct\n{\n stbi_uc *zbuffer, *zbuffer_end;\n int num_bits;\n stbi__uint32 code_buffer;\n\n char *zout;\n char *zout_start;\n char *zout_end;\n int z_expandable;\n\n stbi__zhuffman z_length, z_distance;\n} stbi__zbuf;\n\nstbi_inline static int stbi__zeof(stbi__zbuf *z)\n{\n return (z->zbuffer >= z->zbuffer_end);\n}\n\nstbi_inline static stbi_uc stbi__zget8(stbi__zbuf *z)\n{\n return stbi__zeof(z) ? 0 : *z->zbuffer++;\n}\n\nstatic void stbi__fill_bits(stbi__zbuf *z)\n{\n do {\n if (z->code_buffer >= (1U << z->num_bits)) {\n z->zbuffer = z->zbuffer_end; /* treat this as EOF so we fail. */\n return;\n }\n z->code_buffer |= (unsigned int) stbi__zget8(z) << z->num_bits;\n z->num_bits += 8;\n } while (z->num_bits <= 24);\n}\n\nstbi_inline static unsigned int stbi__zreceive(stbi__zbuf *z, int n)\n{\n unsigned int k;\n if (z->num_bits < n) stbi__fill_bits(z);\n k = z->code_buffer & ((1 << n) - 1);\n z->code_buffer >>= n;\n z->num_bits -= n;\n return k;\n}\n\nstatic int stbi__zhuffman_decode_slowpath(stbi__zbuf *a, stbi__zhuffman *z)\n{\n int b,s,k;\n // not resolved by fast table, so compute it the slow way\n // use jpeg approach, which requires MSbits at top\n k = stbi__bit_reverse(a->code_buffer, 16);\n for (s=STBI__ZFAST_BITS+1; ; ++s)\n if (k < z->maxcode[s])\n break;\n if (s >= 16) return -1; // invalid code!\n // code size is s, so:\n b = (k >> (16-s)) - z->firstcode[s] + z->firstsymbol[s];\n if (b >= STBI__ZNSYMS) return -1; // some data was corrupt somewhere!\n if (z->size[b] != s) return -1; // was originally an assert, but report failure instead.\n a->code_buffer >>= s;\n a->num_bits -= s;\n return z->value[b];\n}\n\nstbi_inline static int stbi__zhuffman_decode(stbi__zbuf *a, stbi__zhuffman *z)\n{\n int b,s;\n if (a->num_bits < 16) {\n if (stbi__zeof(a)) {\n return -1; /* report error for unexpected end of data. */\n }\n stbi__fill_bits(a);\n }\n b = z->fast[a->code_buffer & STBI__ZFAST_MASK];\n if (b) {\n s = b >> 9;\n a->code_buffer >>= s;\n a->num_bits -= s;\n return b & 511;\n }\n return stbi__zhuffman_decode_slowpath(a, z);\n}\n\nstatic int stbi__zexpand(stbi__zbuf *z, char *zout, int n) // need to make room for n bytes\n{\n char *q;\n unsigned int cur, limit, old_limit;\n z->zout = zout;\n if (!z->z_expandable) return stbi__err(\"output buffer limit\",\"Corrupt PNG\");\n cur = (unsigned int) (z->zout - z->zout_start);\n limit = old_limit = (unsigned) (z->zout_end - z->zout_start);\n if (UINT_MAX - cur < (unsigned) n) return stbi__err(\"outofmem\", \"Out of memory\");\n while (cur + n > limit) {\n if(limit > UINT_MAX / 2) return stbi__err(\"outofmem\", \"Out of memory\");\n limit *= 2;\n }\n q = (char *) STBI_REALLOC_SIZED(z->zout_start, old_limit, limit);\n STBI_NOTUSED(old_limit);\n if (q == NULL) return stbi__err(\"outofmem\", \"Out of memory\");\n z->zout_start = q;\n z->zout = q + cur;\n z->zout_end = q + limit;\n return 1;\n}\n\nstatic const int stbi__zlength_base[31] = {\n 3,4,5,6,7,8,9,10,11,13,\n 15,17,19,23,27,31,35,43,51,59,\n 67,83,99,115,131,163,195,227,258,0,0 };\n\nstatic const int stbi__zlength_extra[31]=\n{ 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 };\n\nstatic const int stbi__zdist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,\n257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0};\n\nstatic const int stbi__zdist_extra[32] =\n{ 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13};\n\nstatic int stbi__parse_huffman_block(stbi__zbuf *a)\n{\n char *zout = a->zout;\n for(;;) {\n int z = stbi__zhuffman_decode(a, &a->z_length);\n if (z < 256) {\n if (z < 0) return stbi__err(\"bad huffman code\",\"Corrupt PNG\"); // error in huffman codes\n if (zout >= a->zout_end) {\n if (!stbi__zexpand(a, zout, 1)) return 0;\n zout = a->zout;\n }\n *zout++ = (char) z;\n } else {\n stbi_uc *p;\n int len,dist;\n if (z == 256) {\n a->zout = zout;\n return 1;\n }\n if (z >= 286) return stbi__err(\"bad huffman code\",\"Corrupt PNG\"); // per DEFLATE, length codes 286 and 287 must not appear in compressed data\n z -= 257;\n len = stbi__zlength_base[z];\n if (stbi__zlength_extra[z]) len += stbi__zreceive(a, stbi__zlength_extra[z]);\n z = stbi__zhuffman_decode(a, &a->z_distance);\n if (z < 0 || z >= 30) return stbi__err(\"bad huffman code\",\"Corrupt PNG\"); // per DEFLATE, distance codes 30 and 31 must not appear in compressed data\n dist = stbi__zdist_base[z];\n if (stbi__zdist_extra[z]) dist += stbi__zreceive(a, stbi__zdist_extra[z]);\n if (zout - a->zout_start < dist) return stbi__err(\"bad dist\",\"Corrupt PNG\");\n if (zout + len > a->zout_end) {\n if (!stbi__zexpand(a, zout, len)) return 0;\n zout = a->zout;\n }\n p = (stbi_uc *) (zout - dist);\n if (dist == 1) { // run of one byte; common in images.\n stbi_uc v = *p;\n if (len) { do *zout++ = v; while (--len); }\n } else {\n if (len) { do *zout++ = *p++; while (--len); }\n }\n }\n }\n}\n\nstatic int stbi__compute_huffman_codes(stbi__zbuf *a)\n{\n static const stbi_uc length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 };\n stbi__zhuffman z_codelength;\n stbi_uc lencodes[286+32+137];//padding for maximum single op\n stbi_uc codelength_sizes[19];\n int i,n;\n\n int hlit = stbi__zreceive(a,5) + 257;\n int hdist = stbi__zreceive(a,5) + 1;\n int hclen = stbi__zreceive(a,4) + 4;\n int ntot = hlit + hdist;\n\n memset(codelength_sizes, 0, sizeof(codelength_sizes));\n for (i=0; i < hclen; ++i) {\n int s = stbi__zreceive(a,3);\n codelength_sizes[length_dezigzag[i]] = (stbi_uc) s;\n }\n if (!stbi__zbuild_huffman(&z_codelength, codelength_sizes, 19)) return 0;\n\n n = 0;\n while (n < ntot) {\n int c = stbi__zhuffman_decode(a, &z_codelength);\n if (c < 0 || c >= 19) return stbi__err(\"bad codelengths\", \"Corrupt PNG\");\n if (c < 16)\n lencodes[n++] = (stbi_uc) c;\n else {\n stbi_uc fill = 0;\n if (c == 16) {\n c = stbi__zreceive(a,2)+3;\n if (n == 0) return stbi__err(\"bad codelengths\", \"Corrupt PNG\");\n fill = lencodes[n-1];\n } else if (c == 17) {\n c = stbi__zreceive(a,3)+3;\n } else if (c == 18) {\n c = stbi__zreceive(a,7)+11;\n } else {\n return stbi__err(\"bad codelengths\", \"Corrupt PNG\");\n }\n if (ntot - n < c) return stbi__err(\"bad codelengths\", \"Corrupt PNG\");\n memset(lencodes+n, fill, c);\n n += c;\n }\n }\n if (n != ntot) return stbi__err(\"bad codelengths\",\"Corrupt PNG\");\n if (!stbi__zbuild_huffman(&a->z_length, lencodes, hlit)) return 0;\n if (!stbi__zbuild_huffman(&a->z_distance, lencodes+hlit, hdist)) return 0;\n return 1;\n}\n\nstatic int stbi__parse_uncompressed_block(stbi__zbuf *a)\n{\n stbi_uc header[4];\n int len,nlen,k;\n if (a->num_bits & 7)\n stbi__zreceive(a, a->num_bits & 7); // discard\n // drain the bit-packed data into header\n k = 0;\n while (a->num_bits > 0) {\n header[k++] = (stbi_uc) (a->code_buffer & 255); // suppress MSVC run-time check\n a->code_buffer >>= 8;\n a->num_bits -= 8;\n }\n if (a->num_bits < 0) return stbi__err(\"zlib corrupt\",\"Corrupt PNG\");\n // now fill header the normal way\n while (k < 4)\n header[k++] = stbi__zget8(a);\n len = header[1] * 256 + header[0];\n nlen = header[3] * 256 + header[2];\n if (nlen != (len ^ 0xffff)) return stbi__err(\"zlib corrupt\",\"Corrupt PNG\");\n if (a->zbuffer + len > a->zbuffer_end) return stbi__err(\"read past buffer\",\"Corrupt PNG\");\n if (a->zout + len > a->zout_end)\n if (!stbi__zexpand(a, a->zout, len)) return 0;\n memcpy(a->zout, a->zbuffer, len);\n a->zbuffer += len;\n a->zout += len;\n return 1;\n}\n\nstatic int stbi__parse_zlib_header(stbi__zbuf *a)\n{\n int cmf = stbi__zget8(a);\n int cm = cmf & 15;\n /* int cinfo = cmf >> 4; */\n int flg = stbi__zget8(a);\n if (stbi__zeof(a)) return stbi__err(\"bad zlib header\",\"Corrupt PNG\"); // zlib spec\n if ((cmf*256+flg) % 31 != 0) return stbi__err(\"bad zlib header\",\"Corrupt PNG\"); // zlib spec\n if (flg & 32) return stbi__err(\"no preset dict\",\"Corrupt PNG\"); // preset dictionary not allowed in png\n if (cm != 8) return stbi__err(\"bad compression\",\"Corrupt PNG\"); // DEFLATE required for png\n // window = 1 << (8 + cinfo)... but who cares, we fully buffer output\n return 1;\n}\n\nstatic const stbi_uc stbi__zdefault_length[STBI__ZNSYMS] =\n{\n 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,\n 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,\n 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,\n 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,\n 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,\n 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,\n 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,\n 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,\n 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8\n};\nstatic const stbi_uc stbi__zdefault_distance[32] =\n{\n 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5\n};\n/*\nInit algorithm:\n{\n int i; // use <= to match clearly with spec\n for (i=0; i <= 143; ++i) stbi__zdefault_length[i] = 8;\n for ( ; i <= 255; ++i) stbi__zdefault_length[i] = 9;\n for ( ; i <= 279; ++i) stbi__zdefault_length[i] = 7;\n for ( ; i <= 287; ++i) stbi__zdefault_length[i] = 8;\n\n for (i=0; i <= 31; ++i) stbi__zdefault_distance[i] = 5;\n}\n*/\n\nstatic int stbi__parse_zlib(stbi__zbuf *a, int parse_header)\n{\n int final, type;\n if (parse_header)\n if (!stbi__parse_zlib_header(a)) return 0;\n a->num_bits = 0;\n a->code_buffer = 0;\n do {\n final = stbi__zreceive(a,1);\n type = stbi__zreceive(a,2);\n if (type == 0) {\n if (!stbi__parse_uncompressed_block(a)) return 0;\n } else if (type == 3) {\n return 0;\n } else {\n if (type == 1) {\n // use fixed code lengths\n if (!stbi__zbuild_huffman(&a->z_length , stbi__zdefault_length , STBI__ZNSYMS)) return 0;\n if (!stbi__zbuild_huffman(&a->z_distance, stbi__zdefault_distance, 32)) return 0;\n } else {\n if (!stbi__compute_huffman_codes(a)) return 0;\n }\n if (!stbi__parse_huffman_block(a)) return 0;\n }\n } while (!final);\n return 1;\n}\n\nstatic int stbi__do_zlib(stbi__zbuf *a, char *obuf, int olen, int exp, int parse_header)\n{\n a->zout_start = obuf;\n a->zout = obuf;\n a->zout_end = obuf + olen;\n a->z_expandable = exp;\n\n return stbi__parse_zlib(a, parse_header);\n}\n\nSTBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen)\n{\n stbi__zbuf a;\n char *p = (char *) stbi__malloc(initial_size);\n if (p == NULL) return NULL;\n a.zbuffer = (stbi_uc *) buffer;\n a.zbuffer_end = (stbi_uc *) buffer + len;\n if (stbi__do_zlib(&a, p, initial_size, 1, 1)) {\n if (outlen) *outlen = (int) (a.zout - a.zout_start);\n return a.zout_start;\n } else {\n STBI_FREE(a.zout_start);\n return NULL;\n }\n}\n\nSTBIDEF char *stbi_zlib_decode_malloc(char const *buffer, int len, int *outlen)\n{\n return stbi_zlib_decode_malloc_guesssize(buffer, len, 16384, outlen);\n}\n\nSTBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header)\n{\n stbi__zbuf a;\n char *p = (char *) stbi__malloc(initial_size);\n if (p == NULL) return NULL;\n a.zbuffer = (stbi_uc *) buffer;\n a.zbuffer_end = (stbi_uc *) buffer + len;\n if (stbi__do_zlib(&a, p, initial_size, 1, parse_header)) {\n if (outlen) *outlen = (int) (a.zout - a.zout_start);\n return a.zout_start;\n } else {\n STBI_FREE(a.zout_start);\n return NULL;\n }\n}\n\nSTBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, char const *ibuffer, int ilen)\n{\n stbi__zbuf a;\n a.zbuffer = (stbi_uc *) ibuffer;\n a.zbuffer_end = (stbi_uc *) ibuffer + ilen;\n if (stbi__do_zlib(&a, obuffer, olen, 0, 1))\n return (int) (a.zout - a.zout_start);\n else\n return -1;\n}\n\nSTBIDEF char *stbi_zlib_decode_noheader_malloc(char const *buffer, int len, int *outlen)\n{\n stbi__zbuf a;\n char *p = (char *) stbi__malloc(16384);\n if (p == NULL) return NULL;\n a.zbuffer = (stbi_uc *) buffer;\n a.zbuffer_end = (stbi_uc *) buffer+len;\n if (stbi__do_zlib(&a, p, 16384, 1, 0)) {\n if (outlen) *outlen = (int) (a.zout - a.zout_start);\n return a.zout_start;\n } else {\n STBI_FREE(a.zout_start);\n return NULL;\n }\n}\n\nSTBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen)\n{\n stbi__zbuf a;\n a.zbuffer = (stbi_uc *) ibuffer;\n a.zbuffer_end = (stbi_uc *) ibuffer + ilen;\n if (stbi__do_zlib(&a, obuffer, olen, 0, 0))\n return (int) (a.zout - a.zout_start);\n else\n return -1;\n}\n#endif\n\n// public domain \"baseline\" PNG decoder v0.10 Sean Barrett 2006-11-18\n// simple implementation\n// - only 8-bit samples\n// - no CRC checking\n// - allocates lots of intermediate memory\n// - avoids problem of streaming data between subsystems\n// - avoids explicit window management\n// performance\n// - uses stb_zlib, a PD zlib implementation with fast huffman decoding\n\n#ifndef STBI_NO_PNG\ntypedef struct\n{\n stbi__uint32 length;\n stbi__uint32 type;\n} stbi__pngchunk;\n\nstatic stbi__pngchunk stbi__get_chunk_header(stbi__context *s)\n{\n stbi__pngchunk c;\n c.length = stbi__get32be(s);\n c.type = stbi__get32be(s);\n return c;\n}\n\nstatic int stbi__check_png_header(stbi__context *s)\n{\n static const stbi_uc png_sig[8] = { 137,80,78,71,13,10,26,10 };\n int i;\n for (i=0; i < 8; ++i)\n if (stbi__get8(s) != png_sig[i]) return stbi__err(\"bad png sig\",\"Not a PNG\");\n return 1;\n}\n\ntypedef struct\n{\n stbi__context *s;\n stbi_uc *idata, *expanded, *out;\n int depth;\n} stbi__png;\n\n\nenum {\n STBI__F_none=0,\n STBI__F_sub=1,\n STBI__F_up=2,\n STBI__F_avg=3,\n STBI__F_paeth=4,\n // synthetic filters used for first scanline to avoid needing a dummy row of 0s\n STBI__F_avg_first,\n STBI__F_paeth_first\n};\n\nstatic stbi_uc first_row_filter[5] =\n{\n STBI__F_none,\n STBI__F_sub,\n STBI__F_none,\n STBI__F_avg_first,\n STBI__F_paeth_first\n};\n\nstatic int stbi__paeth(int a, int b, int c)\n{\n int p = a + b - c;\n int pa = abs(p-a);\n int pb = abs(p-b);\n int pc = abs(p-c);\n if (pa <= pb && pa <= pc) return a;\n if (pb <= pc) return b;\n return c;\n}\n\nstatic const stbi_uc stbi__depth_scale_table[9] = { 0, 0xff, 0x55, 0, 0x11, 0,0,0, 0x01 };\n\n// create the png data from post-deflated data\nstatic int stbi__create_png_image_raw(stbi__png *a, stbi_uc *raw, stbi__uint32 raw_len, int out_n, stbi__uint32 x, stbi__uint32 y, int depth, int color)\n{\n int bytes = (depth == 16? 2 : 1);\n stbi__context *s = a->s;\n stbi__uint32 i,j,stride = x*out_n*bytes;\n stbi__uint32 img_len, img_width_bytes;\n int k;\n int img_n = s->img_n; // copy it into a local for later\n\n int output_bytes = out_n*bytes;\n int filter_bytes = img_n*bytes;\n int width = x;\n\n STBI_ASSERT(out_n == s->img_n || out_n == s->img_n+1);\n a->out = (stbi_uc *) stbi__malloc_mad3(x, y, output_bytes, 0); // extra bytes to write off the end into\n if (!a->out) return stbi__err(\"outofmem\", \"Out of memory\");\n\n if (!stbi__mad3sizes_valid(img_n, x, depth, 7)) return stbi__err(\"too large\", \"Corrupt PNG\");\n img_width_bytes = (((img_n * x * depth) + 7) >> 3);\n img_len = (img_width_bytes + 1) * y;\n\n // we used to check for exact match between raw_len and img_len on non-interlaced PNGs,\n // but issue #276 reported a PNG in the wild that had extra data at the end (all zeros),\n // so just check for raw_len < img_len always.\n if (raw_len < img_len) return stbi__err(\"not enough pixels\",\"Corrupt PNG\");\n\n for (j=0; j < y; ++j) {\n stbi_uc *cur = a->out + stride*j;\n stbi_uc *prior;\n int filter = *raw++;\n\n if (filter > 4)\n return stbi__err(\"invalid filter\",\"Corrupt PNG\");\n\n if (depth < 8) {\n if (img_width_bytes > x) return stbi__err(\"invalid width\",\"Corrupt PNG\");\n cur += x*out_n - img_width_bytes; // store output to the rightmost img_len bytes, so we can decode in place\n filter_bytes = 1;\n width = img_width_bytes;\n }\n prior = cur - stride; // bugfix: need to compute this after 'cur +=' computation above\n\n // if first row, use special filter that doesn't sample previous row\n if (j == 0) filter = first_row_filter[filter];\n\n // handle first byte explicitly\n for (k=0; k < filter_bytes; ++k) {\n switch (filter) {\n case STBI__F_none : cur[k] = raw[k]; break;\n case STBI__F_sub : cur[k] = raw[k]; break;\n case STBI__F_up : cur[k] = STBI__BYTECAST(raw[k] + prior[k]); break;\n case STBI__F_avg : cur[k] = STBI__BYTECAST(raw[k] + (prior[k]>>1)); break;\n case STBI__F_paeth : cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(0,prior[k],0)); break;\n case STBI__F_avg_first : cur[k] = raw[k]; break;\n case STBI__F_paeth_first: cur[k] = raw[k]; break;\n }\n }\n\n if (depth == 8) {\n if (img_n != out_n)\n cur[img_n] = 255; // first pixel\n raw += img_n;\n cur += out_n;\n prior += out_n;\n } else if (depth == 16) {\n if (img_n != out_n) {\n cur[filter_bytes] = 255; // first pixel top byte\n cur[filter_bytes+1] = 255; // first pixel bottom byte\n }\n raw += filter_bytes;\n cur += output_bytes;\n prior += output_bytes;\n } else {\n raw += 1;\n cur += 1;\n prior += 1;\n }\n\n // this is a little gross, so that we don't switch per-pixel or per-component\n if (depth < 8 || img_n == out_n) {\n int nk = (width - 1)*filter_bytes;\n #define STBI__CASE(f) \\\n case f: \\\n for (k=0; k < nk; ++k)\n switch (filter) {\n // \"none\" filter turns into a memcpy here; make that explicit.\n case STBI__F_none: memcpy(cur, raw, nk); break;\n STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k-filter_bytes]); } break;\n STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break;\n STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k-filter_bytes])>>1)); } break;\n STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],prior[k],prior[k-filter_bytes])); } break;\n STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k-filter_bytes] >> 1)); } break;\n STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],0,0)); } break;\n }\n #undef STBI__CASE\n raw += nk;\n } else {\n STBI_ASSERT(img_n+1 == out_n);\n #define STBI__CASE(f) \\\n case f: \\\n for (i=x-1; i >= 1; --i, cur[filter_bytes]=255,raw+=filter_bytes,cur+=output_bytes,prior+=output_bytes) \\\n for (k=0; k < filter_bytes; ++k)\n switch (filter) {\n STBI__CASE(STBI__F_none) { cur[k] = raw[k]; } break;\n STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k- output_bytes]); } break;\n STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break;\n STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k- output_bytes])>>1)); } break;\n STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],prior[k],prior[k- output_bytes])); } break;\n STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k- output_bytes] >> 1)); } break;\n STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],0,0)); } break;\n }\n #undef STBI__CASE\n\n // the loop above sets the high byte of the pixels' alpha, but for\n // 16 bit png files we also need the low byte set. we'll do that here.\n if (depth == 16) {\n cur = a->out + stride*j; // start at the beginning of the row again\n for (i=0; i < x; ++i,cur+=output_bytes) {\n cur[filter_bytes+1] = 255;\n }\n }\n }\n }\n\n // we make a separate pass to expand bits to pixels; for performance,\n // this could run two scanlines behind the above code, so it won't\n // intefere with filtering but will still be in the cache.\n if (depth < 8) {\n for (j=0; j < y; ++j) {\n stbi_uc *cur = a->out + stride*j;\n stbi_uc *in = a->out + stride*j + x*out_n - img_width_bytes;\n // unpack 1/2/4-bit into a 8-bit buffer. allows us to keep the common 8-bit path optimal at minimal cost for 1/2/4-bit\n // png guarante byte alignment, if width is not multiple of 8/4/2 we'll decode dummy trailing data that will be skipped in the later loop\n stbi_uc scale = (color == 0) ? stbi__depth_scale_table[depth] : 1; // scale grayscale values to 0..255 range\n\n // note that the final byte might overshoot and write more data than desired.\n // we can allocate enough data that this never writes out of memory, but it\n // could also overwrite the next scanline. can it overwrite non-empty data\n // on the next scanline? yes, consider 1-pixel-wide scanlines with 1-bit-per-pixel.\n // so we need to explicitly clamp the final ones\n\n if (depth == 4) {\n for (k=x*img_n; k >= 2; k-=2, ++in) {\n *cur++ = scale * ((*in >> 4) );\n *cur++ = scale * ((*in ) & 0x0f);\n }\n if (k > 0) *cur++ = scale * ((*in >> 4) );\n } else if (depth == 2) {\n for (k=x*img_n; k >= 4; k-=4, ++in) {\n *cur++ = scale * ((*in >> 6) );\n *cur++ = scale * ((*in >> 4) & 0x03);\n *cur++ = scale * ((*in >> 2) & 0x03);\n *cur++ = scale * ((*in ) & 0x03);\n }\n if (k > 0) *cur++ = scale * ((*in >> 6) );\n if (k > 1) *cur++ = scale * ((*in >> 4) & 0x03);\n if (k > 2) *cur++ = scale * ((*in >> 2) & 0x03);\n } else if (depth == 1) {\n for (k=x*img_n; k >= 8; k-=8, ++in) {\n *cur++ = scale * ((*in >> 7) );\n *cur++ = scale * ((*in >> 6) & 0x01);\n *cur++ = scale * ((*in >> 5) & 0x01);\n *cur++ = scale * ((*in >> 4) & 0x01);\n *cur++ = scale * ((*in >> 3) & 0x01);\n *cur++ = scale * ((*in >> 2) & 0x01);\n *cur++ = scale * ((*in >> 1) & 0x01);\n *cur++ = scale * ((*in ) & 0x01);\n }\n if (k > 0) *cur++ = scale * ((*in >> 7) );\n if (k > 1) *cur++ = scale * ((*in >> 6) & 0x01);\n if (k > 2) *cur++ = scale * ((*in >> 5) & 0x01);\n if (k > 3) *cur++ = scale * ((*in >> 4) & 0x01);\n if (k > 4) *cur++ = scale * ((*in >> 3) & 0x01);\n if (k > 5) *cur++ = scale * ((*in >> 2) & 0x01);\n if (k > 6) *cur++ = scale * ((*in >> 1) & 0x01);\n }\n if (img_n != out_n) {\n int q;\n // insert alpha = 255\n cur = a->out + stride*j;\n if (img_n == 1) {\n for (q=x-1; q >= 0; --q) {\n cur[q*2+1] = 255;\n cur[q*2+0] = cur[q];\n }\n } else {\n STBI_ASSERT(img_n == 3);\n for (q=x-1; q >= 0; --q) {\n cur[q*4+3] = 255;\n cur[q*4+2] = cur[q*3+2];\n cur[q*4+1] = cur[q*3+1];\n cur[q*4+0] = cur[q*3+0];\n }\n }\n }\n }\n } else if (depth == 16) {\n // force the image data from big-endian to platform-native.\n // this is done in a separate pass due to the decoding relying\n // on the data being untouched, but could probably be done\n // per-line during decode if care is taken.\n stbi_uc *cur = a->out;\n stbi__uint16 *cur16 = (stbi__uint16*)cur;\n\n for(i=0; i < x*y*out_n; ++i,cur16++,cur+=2) {\n *cur16 = (cur[0] << 8) | cur[1];\n }\n }\n\n return 1;\n}\n\nstatic int stbi__create_png_image(stbi__png *a, stbi_uc *image_data, stbi__uint32 image_data_len, int out_n, int depth, int color, int interlaced)\n{\n int bytes = (depth == 16 ? 2 : 1);\n int out_bytes = out_n * bytes;\n stbi_uc *final;\n int p;\n if (!interlaced)\n return stbi__create_png_image_raw(a, image_data, image_data_len, out_n, a->s->img_x, a->s->img_y, depth, color);\n\n // de-interlacing\n final = (stbi_uc *) stbi__malloc_mad3(a->s->img_x, a->s->img_y, out_bytes, 0);\n if (!final) return stbi__err(\"outofmem\", \"Out of memory\");\n for (p=0; p < 7; ++p) {\n int xorig[] = { 0,4,0,2,0,1,0 };\n int yorig[] = { 0,0,4,0,2,0,1 };\n int xspc[] = { 8,8,4,4,2,2,1 };\n int yspc[] = { 8,8,8,4,4,2,2 };\n int i,j,x,y;\n // pass1_x[4] = 0, pass1_x[5] = 1, pass1_x[12] = 1\n x = (a->s->img_x - xorig[p] + xspc[p]-1) / xspc[p];\n y = (a->s->img_y - yorig[p] + yspc[p]-1) / yspc[p];\n if (x && y) {\n stbi__uint32 img_len = ((((a->s->img_n * x * depth) + 7) >> 3) + 1) * y;\n if (!stbi__create_png_image_raw(a, image_data, image_data_len, out_n, x, y, depth, color)) {\n STBI_FREE(final);\n return 0;\n }\n for (j=0; j < y; ++j) {\n for (i=0; i < x; ++i) {\n int out_y = j*yspc[p]+yorig[p];\n int out_x = i*xspc[p]+xorig[p];\n memcpy(final + out_y*a->s->img_x*out_bytes + out_x*out_bytes,\n a->out + (j*x+i)*out_bytes, out_bytes);\n }\n }\n STBI_FREE(a->out);\n image_data += img_len;\n image_data_len -= img_len;\n }\n }\n a->out = final;\n\n return 1;\n}\n\nstatic int stbi__compute_transparency(stbi__png *z, stbi_uc tc[3], int out_n)\n{\n stbi__context *s = z->s;\n stbi__uint32 i, pixel_count = s->img_x * s->img_y;\n stbi_uc *p = z->out;\n\n // compute color-based transparency, assuming we've\n // already got 255 as the alpha value in the output\n STBI_ASSERT(out_n == 2 || out_n == 4);\n\n if (out_n == 2) {\n for (i=0; i < pixel_count; ++i) {\n p[1] = (p[0] == tc[0] ? 0 : 255);\n p += 2;\n }\n } else {\n for (i=0; i < pixel_count; ++i) {\n if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2])\n p[3] = 0;\n p += 4;\n }\n }\n return 1;\n}\n\nstatic int stbi__compute_transparency16(stbi__png *z, stbi__uint16 tc[3], int out_n)\n{\n stbi__context *s = z->s;\n stbi__uint32 i, pixel_count = s->img_x * s->img_y;\n stbi__uint16 *p = (stbi__uint16*) z->out;\n\n // compute color-based transparency, assuming we've\n // already got 65535 as the alpha value in the output\n STBI_ASSERT(out_n == 2 || out_n == 4);\n\n if (out_n == 2) {\n for (i = 0; i < pixel_count; ++i) {\n p[1] = (p[0] == tc[0] ? 0 : 65535);\n p += 2;\n }\n } else {\n for (i = 0; i < pixel_count; ++i) {\n if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2])\n p[3] = 0;\n p += 4;\n }\n }\n return 1;\n}\n\nstatic int stbi__expand_png_palette(stbi__png *a, stbi_uc *palette, int len, int pal_img_n)\n{\n stbi__uint32 i, pixel_count = a->s->img_x * a->s->img_y;\n stbi_uc *p, *temp_out, *orig = a->out;\n\n p = (stbi_uc *) stbi__malloc_mad2(pixel_count, pal_img_n, 0);\n if (p == NULL) return stbi__err(\"outofmem\", \"Out of memory\");\n\n // between here and free(out) below, exitting would leak\n temp_out = p;\n\n if (pal_img_n == 3) {\n for (i=0; i < pixel_count; ++i) {\n int n = orig[i]*4;\n p[0] = palette[n ];\n p[1] = palette[n+1];\n p[2] = palette[n+2];\n p += 3;\n }\n } else {\n for (i=0; i < pixel_count; ++i) {\n int n = orig[i]*4;\n p[0] = palette[n ];\n p[1] = palette[n+1];\n p[2] = palette[n+2];\n p[3] = palette[n+3];\n p += 4;\n }\n }\n STBI_FREE(a->out);\n a->out = temp_out;\n\n STBI_NOTUSED(len);\n\n return 1;\n}\n\nstatic int stbi__unpremultiply_on_load_global = 0;\nstatic int stbi__de_iphone_flag_global = 0;\n\nSTBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply)\n{\n stbi__unpremultiply_on_load_global = flag_true_if_should_unpremultiply;\n}\n\nSTBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert)\n{\n stbi__de_iphone_flag_global = flag_true_if_should_convert;\n}\n\n#ifndef STBI_THREAD_LOCAL\n#define stbi__unpremultiply_on_load stbi__unpremultiply_on_load_global\n#define stbi__de_iphone_flag stbi__de_iphone_flag_global\n#else\nstatic STBI_THREAD_LOCAL int stbi__unpremultiply_on_load_local, stbi__unpremultiply_on_load_set;\nstatic STBI_THREAD_LOCAL int stbi__de_iphone_flag_local, stbi__de_iphone_flag_set;\n\nSTBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply)\n{\n stbi__unpremultiply_on_load_local = flag_true_if_should_unpremultiply;\n stbi__unpremultiply_on_load_set = 1;\n}\n\nSTBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert)\n{\n stbi__de_iphone_flag_local = flag_true_if_should_convert;\n stbi__de_iphone_flag_set = 1;\n}\n\n#define stbi__unpremultiply_on_load (stbi__unpremultiply_on_load_set \\\n ? stbi__unpremultiply_on_load_local \\\n : stbi__unpremultiply_on_load_global)\n#define stbi__de_iphone_flag (stbi__de_iphone_flag_set \\\n ? stbi__de_iphone_flag_local \\\n : stbi__de_iphone_flag_global)\n#endif // STBI_THREAD_LOCAL\n\nstatic void stbi__de_iphone(stbi__png *z)\n{\n stbi__context *s = z->s;\n stbi__uint32 i, pixel_count = s->img_x * s->img_y;\n stbi_uc *p = z->out;\n\n if (s->img_out_n == 3) { // convert bgr to rgb\n for (i=0; i < pixel_count; ++i) {\n stbi_uc t = p[0];\n p[0] = p[2];\n p[2] = t;\n p += 3;\n }\n } else {\n STBI_ASSERT(s->img_out_n == 4);\n if (stbi__unpremultiply_on_load) {\n // convert bgr to rgb and unpremultiply\n for (i=0; i < pixel_count; ++i) {\n stbi_uc a = p[3];\n stbi_uc t = p[0];\n if (a) {\n stbi_uc half = a / 2;\n p[0] = (p[2] * 255 + half) / a;\n p[1] = (p[1] * 255 + half) / a;\n p[2] = ( t * 255 + half) / a;\n } else {\n p[0] = p[2];\n p[2] = t;\n }\n p += 4;\n }\n } else {\n // convert bgr to rgb\n for (i=0; i < pixel_count; ++i) {\n stbi_uc t = p[0];\n p[0] = p[2];\n p[2] = t;\n p += 4;\n }\n }\n }\n}\n\n#define STBI__PNG_TYPE(a,b,c,d) (((unsigned) (a) << 24) + ((unsigned) (b) << 16) + ((unsigned) (c) << 8) + (unsigned) (d))\n\nstatic int stbi__parse_png_file(stbi__png *z, int scan, int req_comp)\n{\n stbi_uc palette[1024], pal_img_n=0;\n stbi_uc has_trans=0, tc[3]={0};\n stbi__uint16 tc16[3];\n stbi__uint32 ioff=0, idata_limit=0, i, pal_len=0;\n int first=1,k,interlace=0, color=0, is_iphone=0;\n stbi__context *s = z->s;\n\n z->expanded = NULL;\n z->idata = NULL;\n z->out = NULL;\n\n if (!stbi__check_png_header(s)) return 0;\n\n if (scan == STBI__SCAN_type) return 1;\n\n for (;;) {\n stbi__pngchunk c = stbi__get_chunk_header(s);\n switch (c.type) {\n case STBI__PNG_TYPE('C','g','B','I'):\n is_iphone = 1;\n stbi__skip(s, c.length);\n break;\n case STBI__PNG_TYPE('I','H','D','R'): {\n int comp,filter;\n if (!first) return stbi__err(\"multiple IHDR\",\"Corrupt PNG\");\n first = 0;\n if (c.length != 13) return stbi__err(\"bad IHDR len\",\"Corrupt PNG\");\n s->img_x = stbi__get32be(s);\n s->img_y = stbi__get32be(s);\n if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err(\"too large\",\"Very large image (corrupt?)\");\n if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err(\"too large\",\"Very large image (corrupt?)\");\n z->depth = stbi__get8(s); if (z->depth != 1 && z->depth != 2 && z->depth != 4 && z->depth != 8 && z->depth != 16) return stbi__err(\"1/2/4/8/16-bit only\",\"PNG not supported: 1/2/4/8/16-bit only\");\n color = stbi__get8(s); if (color > 6) return stbi__err(\"bad ctype\",\"Corrupt PNG\");\n if (color == 3 && z->depth == 16) return stbi__err(\"bad ctype\",\"Corrupt PNG\");\n if (color == 3) pal_img_n = 3; else if (color & 1) return stbi__err(\"bad ctype\",\"Corrupt PNG\");\n comp = stbi__get8(s); if (comp) return stbi__err(\"bad comp method\",\"Corrupt PNG\");\n filter= stbi__get8(s); if (filter) return stbi__err(\"bad filter method\",\"Corrupt PNG\");\n interlace = stbi__get8(s); if (interlace>1) return stbi__err(\"bad interlace method\",\"Corrupt PNG\");\n if (!s->img_x || !s->img_y) return stbi__err(\"0-pixel image\",\"Corrupt PNG\");\n if (!pal_img_n) {\n s->img_n = (color & 2 ? 3 : 1) + (color & 4 ? 1 : 0);\n if ((1 << 30) / s->img_x / s->img_n < s->img_y) return stbi__err(\"too large\", \"Image too large to decode\");\n } else {\n // if paletted, then pal_n is our final components, and\n // img_n is # components to decompress/filter.\n s->img_n = 1;\n if ((1 << 30) / s->img_x / 4 < s->img_y) return stbi__err(\"too large\",\"Corrupt PNG\");\n }\n // even with SCAN_header, have to scan to see if we have a tRNS\n break;\n }\n\n case STBI__PNG_TYPE('P','L','T','E'): {\n if (first) return stbi__err(\"first not IHDR\", \"Corrupt PNG\");\n if (c.length > 256*3) return stbi__err(\"invalid PLTE\",\"Corrupt PNG\");\n pal_len = c.length / 3;\n if (pal_len * 3 != c.length) return stbi__err(\"invalid PLTE\",\"Corrupt PNG\");\n for (i=0; i < pal_len; ++i) {\n palette[i*4+0] = stbi__get8(s);\n palette[i*4+1] = stbi__get8(s);\n palette[i*4+2] = stbi__get8(s);\n palette[i*4+3] = 255;\n }\n break;\n }\n\n case STBI__PNG_TYPE('t','R','N','S'): {\n if (first) return stbi__err(\"first not IHDR\", \"Corrupt PNG\");\n if (z->idata) return stbi__err(\"tRNS after IDAT\",\"Corrupt PNG\");\n if (pal_img_n) {\n if (scan == STBI__SCAN_header) { s->img_n = 4; return 1; }\n if (pal_len == 0) return stbi__err(\"tRNS before PLTE\",\"Corrupt PNG\");\n if (c.length > pal_len) return stbi__err(\"bad tRNS len\",\"Corrupt PNG\");\n pal_img_n = 4;\n for (i=0; i < c.length; ++i)\n palette[i*4+3] = stbi__get8(s);\n } else {\n if (!(s->img_n & 1)) return stbi__err(\"tRNS with alpha\",\"Corrupt PNG\");\n if (c.length != (stbi__uint32) s->img_n*2) return stbi__err(\"bad tRNS len\",\"Corrupt PNG\");\n has_trans = 1;\n // non-paletted with tRNS = constant alpha. if header-scanning, we can stop now.\n if (scan == STBI__SCAN_header) { ++s->img_n; return 1; }\n if (z->depth == 16) {\n for (k = 0; k < s->img_n; ++k) tc16[k] = (stbi__uint16)stbi__get16be(s); // copy the values as-is\n } else {\n for (k = 0; k < s->img_n; ++k) tc[k] = (stbi_uc)(stbi__get16be(s) & 255) * stbi__depth_scale_table[z->depth]; // non 8-bit images will be larger\n }\n }\n break;\n }\n\n case STBI__PNG_TYPE('I','D','A','T'): {\n if (first) return stbi__err(\"first not IHDR\", \"Corrupt PNG\");\n if (pal_img_n && !pal_len) return stbi__err(\"no PLTE\",\"Corrupt PNG\");\n if (scan == STBI__SCAN_header) {\n // header scan definitely stops at first IDAT\n if (pal_img_n)\n s->img_n = pal_img_n;\n return 1;\n }\n if (c.length > (1u << 30)) return stbi__err(\"IDAT size limit\", \"IDAT section larger than 2^30 bytes\");\n if ((int)(ioff + c.length) < (int)ioff) return 0;\n if (ioff + c.length > idata_limit) {\n stbi__uint32 idata_limit_old = idata_limit;\n stbi_uc *p;\n if (idata_limit == 0) idata_limit = c.length > 4096 ? c.length : 4096;\n while (ioff + c.length > idata_limit)\n idata_limit *= 2;\n STBI_NOTUSED(idata_limit_old);\n p = (stbi_uc *) STBI_REALLOC_SIZED(z->idata, idata_limit_old, idata_limit); if (p == NULL) return stbi__err(\"outofmem\", \"Out of memory\");\n z->idata = p;\n }\n if (!stbi__getn(s, z->idata+ioff,c.length)) return stbi__err(\"outofdata\",\"Corrupt PNG\");\n ioff += c.length;\n break;\n }\n\n case STBI__PNG_TYPE('I','E','N','D'): {\n stbi__uint32 raw_len, bpl;\n if (first) return stbi__err(\"first not IHDR\", \"Corrupt PNG\");\n if (scan != STBI__SCAN_load) return 1;\n if (z->idata == NULL) return stbi__err(\"no IDAT\",\"Corrupt PNG\");\n // initial guess for decoded data size to avoid unnecessary reallocs\n bpl = (s->img_x * z->depth + 7) / 8; // bytes per line, per component\n raw_len = bpl * s->img_y * s->img_n /* pixels */ + s->img_y /* filter mode per row */;\n z->expanded = (stbi_uc *) stbi_zlib_decode_malloc_guesssize_headerflag((char *) z->idata, ioff, raw_len, (int *) &raw_len, !is_iphone);\n if (z->expanded == NULL) return 0; // zlib should set error\n STBI_FREE(z->idata); z->idata = NULL;\n if ((req_comp == s->img_n+1 && req_comp != 3 && !pal_img_n) || has_trans)\n s->img_out_n = s->img_n+1;\n else\n s->img_out_n = s->img_n;\n if (!stbi__create_png_image(z, z->expanded, raw_len, s->img_out_n, z->depth, color, interlace)) return 0;\n if (has_trans) {\n if (z->depth == 16) {\n if (!stbi__compute_transparency16(z, tc16, s->img_out_n)) return 0;\n } else {\n if (!stbi__compute_transparency(z, tc, s->img_out_n)) return 0;\n }\n }\n if (is_iphone && stbi__de_iphone_flag && s->img_out_n > 2)\n stbi__de_iphone(z);\n if (pal_img_n) {\n // pal_img_n == 3 or 4\n s->img_n = pal_img_n; // record the actual colors we had\n s->img_out_n = pal_img_n;\n if (req_comp >= 3) s->img_out_n = req_comp;\n if (!stbi__expand_png_palette(z, palette, pal_len, s->img_out_n))\n return 0;\n } else if (has_trans) {\n // non-paletted image with tRNS -> source image has (constant) alpha\n ++s->img_n;\n }\n STBI_FREE(z->expanded); z->expanded = NULL;\n // end of PNG chunk, read and skip CRC\n stbi__get32be(s);\n return 1;\n }\n\n default:\n // if critical, fail\n if (first) return stbi__err(\"first not IHDR\", \"Corrupt PNG\");\n if ((c.type & (1 << 29)) == 0) {\n #ifndef STBI_NO_FAILURE_STRINGS\n // not threadsafe\n static char invalid_chunk[] = \"XXXX PNG chunk not known\";\n invalid_chunk[0] = STBI__BYTECAST(c.type >> 24);\n invalid_chunk[1] = STBI__BYTECAST(c.type >> 16);\n invalid_chunk[2] = STBI__BYTECAST(c.type >> 8);\n invalid_chunk[3] = STBI__BYTECAST(c.type >> 0);\n #endif\n return stbi__err(invalid_chunk, \"PNG not supported: unknown PNG chunk type\");\n }\n stbi__skip(s, c.length);\n break;\n }\n // end of PNG chunk, read and skip CRC\n stbi__get32be(s);\n }\n}\n\nstatic void *stbi__do_png(stbi__png *p, int *x, int *y, int *n, int req_comp, stbi__result_info *ri)\n{\n void *result=NULL;\n if (req_comp < 0 || req_comp > 4) return stbi__errpuc(\"bad req_comp\", \"Internal error\");\n if (stbi__parse_png_file(p, STBI__SCAN_load, req_comp)) {\n if (p->depth <= 8)\n ri->bits_per_channel = 8;\n else if (p->depth == 16)\n ri->bits_per_channel = 16;\n else\n return stbi__errpuc(\"bad bits_per_channel\", \"PNG not supported: unsupported color depth\");\n result = p->out;\n p->out = NULL;\n if (req_comp && req_comp != p->s->img_out_n) {\n if (ri->bits_per_channel == 8)\n result = stbi__convert_format((unsigned char *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y);\n else\n result = stbi__convert_format16((stbi__uint16 *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y);\n p->s->img_out_n = req_comp;\n if (result == NULL) return result;\n }\n *x = p->s->img_x;\n *y = p->s->img_y;\n if (n) *n = p->s->img_n;\n }\n STBI_FREE(p->out); p->out = NULL;\n STBI_FREE(p->expanded); p->expanded = NULL;\n STBI_FREE(p->idata); p->idata = NULL;\n\n return result;\n}\n\nstatic void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)\n{\n stbi__png p;\n p.s = s;\n return stbi__do_png(&p, x,y,comp,req_comp, ri);\n}\n\nstatic int stbi__png_test(stbi__context *s)\n{\n int r;\n r = stbi__check_png_header(s);\n stbi__rewind(s);\n return r;\n}\n\nstatic int stbi__png_info_raw(stbi__png *p, int *x, int *y, int *comp)\n{\n if (!stbi__parse_png_file(p, STBI__SCAN_header, 0)) {\n stbi__rewind( p->s );\n return 0;\n }\n if (x) *x = p->s->img_x;\n if (y) *y = p->s->img_y;\n if (comp) *comp = p->s->img_n;\n return 1;\n}\n\nstatic int stbi__png_info(stbi__context *s, int *x, int *y, int *comp)\n{\n stbi__png p;\n p.s = s;\n return stbi__png_info_raw(&p, x, y, comp);\n}\n\nstatic int stbi__png_is16(stbi__context *s)\n{\n stbi__png p;\n p.s = s;\n if (!stbi__png_info_raw(&p, NULL, NULL, NULL))\n\t return 0;\n if (p.depth != 16) {\n stbi__rewind(p.s);\n return 0;\n }\n return 1;\n}\n#endif\n\n// Microsoft/Windows BMP image\n\n#ifndef STBI_NO_BMP\nstatic int stbi__bmp_test_raw(stbi__context *s)\n{\n int r;\n int sz;\n if (stbi__get8(s) != 'B') return 0;\n if (stbi__get8(s) != 'M') return 0;\n stbi__get32le(s); // discard filesize\n stbi__get16le(s); // discard reserved\n stbi__get16le(s); // discard reserved\n stbi__get32le(s); // discard data offset\n sz = stbi__get32le(s);\n r = (sz == 12 || sz == 40 || sz == 56 || sz == 108 || sz == 124);\n return r;\n}\n\nstatic int stbi__bmp_test(stbi__context *s)\n{\n int r = stbi__bmp_test_raw(s);\n stbi__rewind(s);\n return r;\n}\n\n\n// returns 0..31 for the highest set bit\nstatic int stbi__high_bit(unsigned int z)\n{\n int n=0;\n if (z == 0) return -1;\n if (z >= 0x10000) { n += 16; z >>= 16; }\n if (z >= 0x00100) { n += 8; z >>= 8; }\n if (z >= 0x00010) { n += 4; z >>= 4; }\n if (z >= 0x00004) { n += 2; z >>= 2; }\n if (z >= 0x00002) { n += 1;/* >>= 1;*/ }\n return n;\n}\n\nstatic int stbi__bitcount(unsigned int a)\n{\n a = (a & 0x55555555) + ((a >> 1) & 0x55555555); // max 2\n a = (a & 0x33333333) + ((a >> 2) & 0x33333333); // max 4\n a = (a + (a >> 4)) & 0x0f0f0f0f; // max 8 per 4, now 8 bits\n a = (a + (a >> 8)); // max 16 per 8 bits\n a = (a + (a >> 16)); // max 32 per 8 bits\n return a & 0xff;\n}\n\n// extract an arbitrarily-aligned N-bit value (N=bits)\n// from v, and then make it 8-bits long and fractionally\n// extend it to full full range.\nstatic int stbi__shiftsigned(unsigned int v, int shift, int bits)\n{\n static unsigned int mul_table[9] = {\n 0,\n 0xff/*0b11111111*/, 0x55/*0b01010101*/, 0x49/*0b01001001*/, 0x11/*0b00010001*/,\n 0x21/*0b00100001*/, 0x41/*0b01000001*/, 0x81/*0b10000001*/, 0x01/*0b00000001*/,\n };\n static unsigned int shift_table[9] = {\n 0, 0,0,1,0,2,4,6,0,\n };\n if (shift < 0)\n v <<= -shift;\n else\n v >>= shift;\n STBI_ASSERT(v < 256);\n v >>= (8-bits);\n STBI_ASSERT(bits >= 0 && bits <= 8);\n return (int) ((unsigned) v * mul_table[bits]) >> shift_table[bits];\n}\n\ntypedef struct\n{\n int bpp, offset, hsz;\n unsigned int mr,mg,mb,ma, all_a;\n int extra_read;\n} stbi__bmp_data;\n\nstatic int stbi__bmp_set_mask_defaults(stbi__bmp_data *info, int compress)\n{\n // BI_BITFIELDS specifies masks explicitly, don't override\n if (compress == 3)\n return 1;\n\n if (compress == 0) {\n if (info->bpp == 16) {\n info->mr = 31u << 10;\n info->mg = 31u << 5;\n info->mb = 31u << 0;\n } else if (info->bpp == 32) {\n info->mr = 0xffu << 16;\n info->mg = 0xffu << 8;\n info->mb = 0xffu << 0;\n info->ma = 0xffu << 24;\n info->all_a = 0; // if all_a is 0 at end, then we loaded alpha channel but it was all 0\n } else {\n // otherwise, use defaults, which is all-0\n info->mr = info->mg = info->mb = info->ma = 0;\n }\n return 1;\n }\n return 0; // error\n}\n\nstatic void *stbi__bmp_parse_header(stbi__context *s, stbi__bmp_data *info)\n{\n int hsz;\n if (stbi__get8(s) != 'B' || stbi__get8(s) != 'M') return stbi__errpuc(\"not BMP\", \"Corrupt BMP\");\n stbi__get32le(s); // discard filesize\n stbi__get16le(s); // discard reserved\n stbi__get16le(s); // discard reserved\n info->offset = stbi__get32le(s);\n info->hsz = hsz = stbi__get32le(s);\n info->mr = info->mg = info->mb = info->ma = 0;\n info->extra_read = 14;\n\n if (info->offset < 0) return stbi__errpuc(\"bad BMP\", \"bad BMP\");\n\n if (hsz != 12 && hsz != 40 && hsz != 56 && hsz != 108 && hsz != 124) return stbi__errpuc(\"unknown BMP\", \"BMP type not supported: unknown\");\n if (hsz == 12) {\n s->img_x = stbi__get16le(s);\n s->img_y = stbi__get16le(s);\n } else {\n s->img_x = stbi__get32le(s);\n s->img_y = stbi__get32le(s);\n }\n if (stbi__get16le(s) != 1) return stbi__errpuc(\"bad BMP\", \"bad BMP\");\n info->bpp = stbi__get16le(s);\n if (hsz != 12) {\n int compress = stbi__get32le(s);\n if (compress == 1 || compress == 2) return stbi__errpuc(\"BMP RLE\", \"BMP type not supported: RLE\");\n if (compress >= 4) return stbi__errpuc(\"BMP JPEG/PNG\", \"BMP type not supported: unsupported compression\"); // this includes PNG/JPEG modes\n if (compress == 3 && info->bpp != 16 && info->bpp != 32) return stbi__errpuc(\"bad BMP\", \"bad BMP\"); // bitfields requires 16 or 32 bits/pixel\n stbi__get32le(s); // discard sizeof\n stbi__get32le(s); // discard hres\n stbi__get32le(s); // discard vres\n stbi__get32le(s); // discard colorsused\n stbi__get32le(s); // discard max important\n if (hsz == 40 || hsz == 56) {\n if (hsz == 56) {\n stbi__get32le(s);\n stbi__get32le(s);\n stbi__get32le(s);\n stbi__get32le(s);\n }\n if (info->bpp == 16 || info->bpp == 32) {\n if (compress == 0) {\n stbi__bmp_set_mask_defaults(info, compress);\n } else if (compress == 3) {\n info->mr = stbi__get32le(s);\n info->mg = stbi__get32le(s);\n info->mb = stbi__get32le(s);\n info->extra_read += 12;\n // not documented, but generated by photoshop and handled by mspaint\n if (info->mr == info->mg && info->mg == info->mb) {\n // ?!?!?\n return stbi__errpuc(\"bad BMP\", \"bad BMP\");\n }\n } else\n return stbi__errpuc(\"bad BMP\", \"bad BMP\");\n }\n } else {\n // V4/V5 header\n int i;\n if (hsz != 108 && hsz != 124)\n return stbi__errpuc(\"bad BMP\", \"bad BMP\");\n info->mr = stbi__get32le(s);\n info->mg = stbi__get32le(s);\n info->mb = stbi__get32le(s);\n info->ma = stbi__get32le(s);\n if (compress != 3) // override mr/mg/mb unless in BI_BITFIELDS mode, as per docs\n stbi__bmp_set_mask_defaults(info, compress);\n stbi__get32le(s); // discard color space\n for (i=0; i < 12; ++i)\n stbi__get32le(s); // discard color space parameters\n if (hsz == 124) {\n stbi__get32le(s); // discard rendering intent\n stbi__get32le(s); // discard offset of profile data\n stbi__get32le(s); // discard size of profile data\n stbi__get32le(s); // discard reserved\n }\n }\n }\n return (void *) 1;\n}\n\n\nstatic void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)\n{\n stbi_uc *out;\n unsigned int mr=0,mg=0,mb=0,ma=0, all_a;\n stbi_uc pal[256][4];\n int psize=0,i,j,width;\n int flip_vertically, pad, target;\n stbi__bmp_data info;\n STBI_NOTUSED(ri);\n\n info.all_a = 255;\n if (stbi__bmp_parse_header(s, &info) == NULL)\n return NULL; // error code already set\n\n flip_vertically = ((int) s->img_y) > 0;\n s->img_y = abs((int) s->img_y);\n\n if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n\n mr = info.mr;\n mg = info.mg;\n mb = info.mb;\n ma = info.ma;\n all_a = info.all_a;\n\n if (info.hsz == 12) {\n if (info.bpp < 24)\n psize = (info.offset - info.extra_read - 24) / 3;\n } else {\n if (info.bpp < 16)\n psize = (info.offset - info.extra_read - info.hsz) >> 2;\n }\n if (psize == 0) {\n // accept some number of extra bytes after the header, but if the offset points either to before\n // the header ends or implies a large amount of extra data, reject the file as malformed\n int bytes_read_so_far = s->callback_already_read + (int)(s->img_buffer - s->img_buffer_original);\n int header_limit = 1024; // max we actually read is below 256 bytes currently.\n int extra_data_limit = 256*4; // what ordinarily goes here is a palette; 256 entries*4 bytes is its max size.\n if (bytes_read_so_far <= 0 || bytes_read_so_far > header_limit) {\n return stbi__errpuc(\"bad header\", \"Corrupt BMP\");\n }\n // we established that bytes_read_so_far is positive and sensible.\n // the first half of this test rejects offsets that are either too small positives, or\n // negative, and guarantees that info.offset >= bytes_read_so_far > 0. this in turn\n // ensures the number computed in the second half of the test can't overflow.\n if (info.offset < bytes_read_so_far || info.offset - bytes_read_so_far > extra_data_limit) {\n return stbi__errpuc(\"bad offset\", \"Corrupt BMP\");\n } else {\n stbi__skip(s, info.offset - bytes_read_so_far);\n }\n }\n\n if (info.bpp == 24 && ma == 0xff000000)\n s->img_n = 3;\n else\n s->img_n = ma ? 4 : 3;\n if (req_comp && req_comp >= 3) // we can directly decode 3 or 4\n target = req_comp;\n else\n target = s->img_n; // if they want monochrome, we'll post-convert\n\n // sanity-check size\n if (!stbi__mad3sizes_valid(target, s->img_x, s->img_y, 0))\n return stbi__errpuc(\"too large\", \"Corrupt BMP\");\n\n out = (stbi_uc *) stbi__malloc_mad3(target, s->img_x, s->img_y, 0);\n if (!out) return stbi__errpuc(\"outofmem\", \"Out of memory\");\n if (info.bpp < 16) {\n int z=0;\n if (psize == 0 || psize > 256) { STBI_FREE(out); return stbi__errpuc(\"invalid\", \"Corrupt BMP\"); }\n for (i=0; i < psize; ++i) {\n pal[i][2] = stbi__get8(s);\n pal[i][1] = stbi__get8(s);\n pal[i][0] = stbi__get8(s);\n if (info.hsz != 12) stbi__get8(s);\n pal[i][3] = 255;\n }\n stbi__skip(s, info.offset - info.extra_read - info.hsz - psize * (info.hsz == 12 ? 3 : 4));\n if (info.bpp == 1) width = (s->img_x + 7) >> 3;\n else if (info.bpp == 4) width = (s->img_x + 1) >> 1;\n else if (info.bpp == 8) width = s->img_x;\n else { STBI_FREE(out); return stbi__errpuc(\"bad bpp\", \"Corrupt BMP\"); }\n pad = (-width)&3;\n if (info.bpp == 1) {\n for (j=0; j < (int) s->img_y; ++j) {\n int bit_offset = 7, v = stbi__get8(s);\n for (i=0; i < (int) s->img_x; ++i) {\n int color = (v>>bit_offset)&0x1;\n out[z++] = pal[color][0];\n out[z++] = pal[color][1];\n out[z++] = pal[color][2];\n if (target == 4) out[z++] = 255;\n if (i+1 == (int) s->img_x) break;\n if((--bit_offset) < 0) {\n bit_offset = 7;\n v = stbi__get8(s);\n }\n }\n stbi__skip(s, pad);\n }\n } else {\n for (j=0; j < (int) s->img_y; ++j) {\n for (i=0; i < (int) s->img_x; i += 2) {\n int v=stbi__get8(s),v2=0;\n if (info.bpp == 4) {\n v2 = v & 15;\n v >>= 4;\n }\n out[z++] = pal[v][0];\n out[z++] = pal[v][1];\n out[z++] = pal[v][2];\n if (target == 4) out[z++] = 255;\n if (i+1 == (int) s->img_x) break;\n v = (info.bpp == 8) ? stbi__get8(s) : v2;\n out[z++] = pal[v][0];\n out[z++] = pal[v][1];\n out[z++] = pal[v][2];\n if (target == 4) out[z++] = 255;\n }\n stbi__skip(s, pad);\n }\n }\n } else {\n int rshift=0,gshift=0,bshift=0,ashift=0,rcount=0,gcount=0,bcount=0,acount=0;\n int z = 0;\n int easy=0;\n stbi__skip(s, info.offset - info.extra_read - info.hsz);\n if (info.bpp == 24) width = 3 * s->img_x;\n else if (info.bpp == 16) width = 2*s->img_x;\n else /* bpp = 32 and pad = 0 */ width=0;\n pad = (-width) & 3;\n if (info.bpp == 24) {\n easy = 1;\n } else if (info.bpp == 32) {\n if (mb == 0xff && mg == 0xff00 && mr == 0x00ff0000 && ma == 0xff000000)\n easy = 2;\n }\n if (!easy) {\n if (!mr || !mg || !mb) { STBI_FREE(out); return stbi__errpuc(\"bad masks\", \"Corrupt BMP\"); }\n // right shift amt to put high bit in position #7\n rshift = stbi__high_bit(mr)-7; rcount = stbi__bitcount(mr);\n gshift = stbi__high_bit(mg)-7; gcount = stbi__bitcount(mg);\n bshift = stbi__high_bit(mb)-7; bcount = stbi__bitcount(mb);\n ashift = stbi__high_bit(ma)-7; acount = stbi__bitcount(ma);\n if (rcount > 8 || gcount > 8 || bcount > 8 || acount > 8) { STBI_FREE(out); return stbi__errpuc(\"bad masks\", \"Corrupt BMP\"); }\n }\n for (j=0; j < (int) s->img_y; ++j) {\n if (easy) {\n for (i=0; i < (int) s->img_x; ++i) {\n unsigned char a;\n out[z+2] = stbi__get8(s);\n out[z+1] = stbi__get8(s);\n out[z+0] = stbi__get8(s);\n z += 3;\n a = (easy == 2 ? stbi__get8(s) : 255);\n all_a |= a;\n if (target == 4) out[z++] = a;\n }\n } else {\n int bpp = info.bpp;\n for (i=0; i < (int) s->img_x; ++i) {\n stbi__uint32 v = (bpp == 16 ? (stbi__uint32) stbi__get16le(s) : stbi__get32le(s));\n unsigned int a;\n out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mr, rshift, rcount));\n out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mg, gshift, gcount));\n out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mb, bshift, bcount));\n a = (ma ? stbi__shiftsigned(v & ma, ashift, acount) : 255);\n all_a |= a;\n if (target == 4) out[z++] = STBI__BYTECAST(a);\n }\n }\n stbi__skip(s, pad);\n }\n }\n\n // if alpha channel is all 0s, replace with all 255s\n if (target == 4 && all_a == 0)\n for (i=4*s->img_x*s->img_y-1; i >= 0; i -= 4)\n out[i] = 255;\n\n if (flip_vertically) {\n stbi_uc t;\n for (j=0; j < (int) s->img_y>>1; ++j) {\n stbi_uc *p1 = out + j *s->img_x*target;\n stbi_uc *p2 = out + (s->img_y-1-j)*s->img_x*target;\n for (i=0; i < (int) s->img_x*target; ++i) {\n t = p1[i]; p1[i] = p2[i]; p2[i] = t;\n }\n }\n }\n\n if (req_comp && req_comp != target) {\n out = stbi__convert_format(out, target, req_comp, s->img_x, s->img_y);\n if (out == NULL) return out; // stbi__convert_format frees input on failure\n }\n\n *x = s->img_x;\n *y = s->img_y;\n if (comp) *comp = s->img_n;\n return out;\n}\n#endif\n\n// Targa Truevision - TGA\n// by Jonathan Dummer\n#ifndef STBI_NO_TGA\n// returns STBI_rgb or whatever, 0 on error\nstatic int stbi__tga_get_comp(int bits_per_pixel, int is_grey, int* is_rgb16)\n{\n // only RGB or RGBA (incl. 16bit) or grey allowed\n if (is_rgb16) *is_rgb16 = 0;\n switch(bits_per_pixel) {\n case 8: return STBI_grey;\n case 16: if(is_grey) return STBI_grey_alpha;\n // fallthrough\n case 15: if(is_rgb16) *is_rgb16 = 1;\n return STBI_rgb;\n case 24: // fallthrough\n case 32: return bits_per_pixel/8;\n default: return 0;\n }\n}\n\nstatic int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp)\n{\n int tga_w, tga_h, tga_comp, tga_image_type, tga_bits_per_pixel, tga_colormap_bpp;\n int sz, tga_colormap_type;\n stbi__get8(s); // discard Offset\n tga_colormap_type = stbi__get8(s); // colormap type\n if( tga_colormap_type > 1 ) {\n stbi__rewind(s);\n return 0; // only RGB or indexed allowed\n }\n tga_image_type = stbi__get8(s); // image type\n if ( tga_colormap_type == 1 ) { // colormapped (paletted) image\n if (tga_image_type != 1 && tga_image_type != 9) {\n stbi__rewind(s);\n return 0;\n }\n stbi__skip(s,4); // skip index of first colormap entry and number of entries\n sz = stbi__get8(s); // check bits per palette color entry\n if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) {\n stbi__rewind(s);\n return 0;\n }\n stbi__skip(s,4); // skip image x and y origin\n tga_colormap_bpp = sz;\n } else { // \"normal\" image w/o colormap - only RGB or grey allowed, +/- RLE\n if ( (tga_image_type != 2) && (tga_image_type != 3) && (tga_image_type != 10) && (tga_image_type != 11) ) {\n stbi__rewind(s);\n return 0; // only RGB or grey allowed, +/- RLE\n }\n stbi__skip(s,9); // skip colormap specification and image x/y origin\n tga_colormap_bpp = 0;\n }\n tga_w = stbi__get16le(s);\n if( tga_w < 1 ) {\n stbi__rewind(s);\n return 0; // test width\n }\n tga_h = stbi__get16le(s);\n if( tga_h < 1 ) {\n stbi__rewind(s);\n return 0; // test height\n }\n tga_bits_per_pixel = stbi__get8(s); // bits per pixel\n stbi__get8(s); // ignore alpha bits\n if (tga_colormap_bpp != 0) {\n if((tga_bits_per_pixel != 8) && (tga_bits_per_pixel != 16)) {\n // when using a colormap, tga_bits_per_pixel is the size of the indexes\n // I don't think anything but 8 or 16bit indexes makes sense\n stbi__rewind(s);\n return 0;\n }\n tga_comp = stbi__tga_get_comp(tga_colormap_bpp, 0, NULL);\n } else {\n tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3) || (tga_image_type == 11), NULL);\n }\n if(!tga_comp) {\n stbi__rewind(s);\n return 0;\n }\n if (x) *x = tga_w;\n if (y) *y = tga_h;\n if (comp) *comp = tga_comp;\n return 1; // seems to have passed everything\n}\n\nstatic int stbi__tga_test(stbi__context *s)\n{\n int res = 0;\n int sz, tga_color_type;\n stbi__get8(s); // discard Offset\n tga_color_type = stbi__get8(s); // color type\n if ( tga_color_type > 1 ) goto errorEnd; // only RGB or indexed allowed\n sz = stbi__get8(s); // image type\n if ( tga_color_type == 1 ) { // colormapped (paletted) image\n if (sz != 1 && sz != 9) goto errorEnd; // colortype 1 demands image type 1 or 9\n stbi__skip(s,4); // skip index of first colormap entry and number of entries\n sz = stbi__get8(s); // check bits per palette color entry\n if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd;\n stbi__skip(s,4); // skip image x and y origin\n } else { // \"normal\" image w/o colormap\n if ( (sz != 2) && (sz != 3) && (sz != 10) && (sz != 11) ) goto errorEnd; // only RGB or grey allowed, +/- RLE\n stbi__skip(s,9); // skip colormap specification and image x/y origin\n }\n if ( stbi__get16le(s) < 1 ) goto errorEnd; // test width\n if ( stbi__get16le(s) < 1 ) goto errorEnd; // test height\n sz = stbi__get8(s); // bits per pixel\n if ( (tga_color_type == 1) && (sz != 8) && (sz != 16) ) goto errorEnd; // for colormapped images, bpp is size of an index\n if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd;\n\n res = 1; // if we got this far, everything's good and we can return 1 instead of 0\n\nerrorEnd:\n stbi__rewind(s);\n return res;\n}\n\n// read 16bit value and convert to 24bit RGB\nstatic void stbi__tga_read_rgb16(stbi__context *s, stbi_uc* out)\n{\n stbi__uint16 px = (stbi__uint16)stbi__get16le(s);\n stbi__uint16 fiveBitMask = 31;\n // we have 3 channels with 5bits each\n int r = (px >> 10) & fiveBitMask;\n int g = (px >> 5) & fiveBitMask;\n int b = px & fiveBitMask;\n // Note that this saves the data in RGB(A) order, so it doesn't need to be swapped later\n out[0] = (stbi_uc)((r * 255)/31);\n out[1] = (stbi_uc)((g * 255)/31);\n out[2] = (stbi_uc)((b * 255)/31);\n\n // some people claim that the most significant bit might be used for alpha\n // (possibly if an alpha-bit is set in the \"image descriptor byte\")\n // but that only made 16bit test images completely translucent..\n // so let's treat all 15 and 16bit TGAs as RGB with no alpha.\n}\n\nstatic void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)\n{\n // read in the TGA header stuff\n int tga_offset = stbi__get8(s);\n int tga_indexed = stbi__get8(s);\n int tga_image_type = stbi__get8(s);\n int tga_is_RLE = 0;\n int tga_palette_start = stbi__get16le(s);\n int tga_palette_len = stbi__get16le(s);\n int tga_palette_bits = stbi__get8(s);\n int tga_x_origin = stbi__get16le(s);\n int tga_y_origin = stbi__get16le(s);\n int tga_width = stbi__get16le(s);\n int tga_height = stbi__get16le(s);\n int tga_bits_per_pixel = stbi__get8(s);\n int tga_comp, tga_rgb16=0;\n int tga_inverted = stbi__get8(s);\n // int tga_alpha_bits = tga_inverted & 15; // the 4 lowest bits - unused (useless?)\n // image data\n unsigned char *tga_data;\n unsigned char *tga_palette = NULL;\n int i, j;\n unsigned char raw_data[4] = {0};\n int RLE_count = 0;\n int RLE_repeating = 0;\n int read_next_pixel = 1;\n STBI_NOTUSED(ri);\n STBI_NOTUSED(tga_x_origin); // @TODO\n STBI_NOTUSED(tga_y_origin); // @TODO\n\n if (tga_height > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n if (tga_width > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n\n // do a tiny bit of precessing\n if ( tga_image_type >= 8 )\n {\n tga_image_type -= 8;\n tga_is_RLE = 1;\n }\n tga_inverted = 1 - ((tga_inverted >> 5) & 1);\n\n // If I'm paletted, then I'll use the number of bits from the palette\n if ( tga_indexed ) tga_comp = stbi__tga_get_comp(tga_palette_bits, 0, &tga_rgb16);\n else tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3), &tga_rgb16);\n\n if(!tga_comp) // shouldn't really happen, stbi__tga_test() should have ensured basic consistency\n return stbi__errpuc(\"bad format\", \"Can't find out TGA pixelformat\");\n\n // tga info\n *x = tga_width;\n *y = tga_height;\n if (comp) *comp = tga_comp;\n\n if (!stbi__mad3sizes_valid(tga_width, tga_height, tga_comp, 0))\n return stbi__errpuc(\"too large\", \"Corrupt TGA\");\n\n tga_data = (unsigned char*)stbi__malloc_mad3(tga_width, tga_height, tga_comp, 0);\n if (!tga_data) return stbi__errpuc(\"outofmem\", \"Out of memory\");\n\n // skip to the data's starting position (offset usually = 0)\n stbi__skip(s, tga_offset );\n\n if ( !tga_indexed && !tga_is_RLE && !tga_rgb16 ) {\n for (i=0; i < tga_height; ++i) {\n int row = tga_inverted ? tga_height -i - 1 : i;\n stbi_uc *tga_row = tga_data + row*tga_width*tga_comp;\n stbi__getn(s, tga_row, tga_width * tga_comp);\n }\n } else {\n // do I need to load a palette?\n if ( tga_indexed)\n {\n if (tga_palette_len == 0) { /* you have to have at least one entry! */\n STBI_FREE(tga_data);\n return stbi__errpuc(\"bad palette\", \"Corrupt TGA\");\n }\n\n // any data to skip? (offset usually = 0)\n stbi__skip(s, tga_palette_start );\n // load the palette\n tga_palette = (unsigned char*)stbi__malloc_mad2(tga_palette_len, tga_comp, 0);\n if (!tga_palette) {\n STBI_FREE(tga_data);\n return stbi__errpuc(\"outofmem\", \"Out of memory\");\n }\n if (tga_rgb16) {\n stbi_uc *pal_entry = tga_palette;\n STBI_ASSERT(tga_comp == STBI_rgb);\n for (i=0; i < tga_palette_len; ++i) {\n stbi__tga_read_rgb16(s, pal_entry);\n pal_entry += tga_comp;\n }\n } else if (!stbi__getn(s, tga_palette, tga_palette_len * tga_comp)) {\n STBI_FREE(tga_data);\n STBI_FREE(tga_palette);\n return stbi__errpuc(\"bad palette\", \"Corrupt TGA\");\n }\n }\n // load the data\n for (i=0; i < tga_width * tga_height; ++i)\n {\n // if I'm in RLE mode, do I need to get a RLE stbi__pngchunk?\n if ( tga_is_RLE )\n {\n if ( RLE_count == 0 )\n {\n // yep, get the next byte as a RLE command\n int RLE_cmd = stbi__get8(s);\n RLE_count = 1 + (RLE_cmd & 127);\n RLE_repeating = RLE_cmd >> 7;\n read_next_pixel = 1;\n } else if ( !RLE_repeating )\n {\n read_next_pixel = 1;\n }\n } else\n {\n read_next_pixel = 1;\n }\n // OK, if I need to read a pixel, do it now\n if ( read_next_pixel )\n {\n // load however much data we did have\n if ( tga_indexed )\n {\n // read in index, then perform the lookup\n int pal_idx = (tga_bits_per_pixel == 8) ? stbi__get8(s) : stbi__get16le(s);\n if ( pal_idx >= tga_palette_len ) {\n // invalid index\n pal_idx = 0;\n }\n pal_idx *= tga_comp;\n for (j = 0; j < tga_comp; ++j) {\n raw_data[j] = tga_palette[pal_idx+j];\n }\n } else if(tga_rgb16) {\n STBI_ASSERT(tga_comp == STBI_rgb);\n stbi__tga_read_rgb16(s, raw_data);\n } else {\n // read in the data raw\n for (j = 0; j < tga_comp; ++j) {\n raw_data[j] = stbi__get8(s);\n }\n }\n // clear the reading flag for the next pixel\n read_next_pixel = 0;\n } // end of reading a pixel\n\n // copy data\n for (j = 0; j < tga_comp; ++j)\n tga_data[i*tga_comp+j] = raw_data[j];\n\n // in case we're in RLE mode, keep counting down\n --RLE_count;\n }\n // do I need to invert the image?\n if ( tga_inverted )\n {\n for (j = 0; j*2 < tga_height; ++j)\n {\n int index1 = j * tga_width * tga_comp;\n int index2 = (tga_height - 1 - j) * tga_width * tga_comp;\n for (i = tga_width * tga_comp; i > 0; --i)\n {\n unsigned char temp = tga_data[index1];\n tga_data[index1] = tga_data[index2];\n tga_data[index2] = temp;\n ++index1;\n ++index2;\n }\n }\n }\n // clear my palette, if I had one\n if ( tga_palette != NULL )\n {\n STBI_FREE( tga_palette );\n }\n }\n\n // swap RGB - if the source data was RGB16, it already is in the right order\n if (tga_comp >= 3 && !tga_rgb16)\n {\n unsigned char* tga_pixel = tga_data;\n for (i=0; i < tga_width * tga_height; ++i)\n {\n unsigned char temp = tga_pixel[0];\n tga_pixel[0] = tga_pixel[2];\n tga_pixel[2] = temp;\n tga_pixel += tga_comp;\n }\n }\n\n // convert to target component count\n if (req_comp && req_comp != tga_comp)\n tga_data = stbi__convert_format(tga_data, tga_comp, req_comp, tga_width, tga_height);\n\n // the things I do to get rid of an error message, and yet keep\n // Microsoft's C compilers happy... [8^(\n tga_palette_start = tga_palette_len = tga_palette_bits =\n tga_x_origin = tga_y_origin = 0;\n STBI_NOTUSED(tga_palette_start);\n // OK, done\n return tga_data;\n}\n#endif\n\n// *************************************************************************************************\n// Photoshop PSD loader -- PD by Thatcher Ulrich, integration by Nicolas Schulz, tweaked by STB\n\n#ifndef STBI_NO_PSD\nstatic int stbi__psd_test(stbi__context *s)\n{\n int r = (stbi__get32be(s) == 0x38425053);\n stbi__rewind(s);\n return r;\n}\n\nstatic int stbi__psd_decode_rle(stbi__context *s, stbi_uc *p, int pixelCount)\n{\n int count, nleft, len;\n\n count = 0;\n while ((nleft = pixelCount - count) > 0) {\n len = stbi__get8(s);\n if (len == 128) {\n // No-op.\n } else if (len < 128) {\n // Copy next len+1 bytes literally.\n len++;\n if (len > nleft) return 0; // corrupt data\n count += len;\n while (len) {\n *p = stbi__get8(s);\n p += 4;\n len--;\n }\n } else if (len > 128) {\n stbi_uc val;\n // Next -len+1 bytes in the dest are replicated from next source byte.\n // (Interpret len as a negative 8-bit int.)\n len = 257 - len;\n if (len > nleft) return 0; // corrupt data\n val = stbi__get8(s);\n count += len;\n while (len) {\n *p = val;\n p += 4;\n len--;\n }\n }\n }\n\n return 1;\n}\n\nstatic void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc)\n{\n int pixelCount;\n int channelCount, compression;\n int channel, i;\n int bitdepth;\n int w,h;\n stbi_uc *out;\n STBI_NOTUSED(ri);\n\n // Check identifier\n if (stbi__get32be(s) != 0x38425053) // \"8BPS\"\n return stbi__errpuc(\"not PSD\", \"Corrupt PSD image\");\n\n // Check file type version.\n if (stbi__get16be(s) != 1)\n return stbi__errpuc(\"wrong version\", \"Unsupported version of PSD image\");\n\n // Skip 6 reserved bytes.\n stbi__skip(s, 6 );\n\n // Read the number of channels (R, G, B, A, etc).\n channelCount = stbi__get16be(s);\n if (channelCount < 0 || channelCount > 16)\n return stbi__errpuc(\"wrong channel count\", \"Unsupported number of channels in PSD image\");\n\n // Read the rows and columns of the image.\n h = stbi__get32be(s);\n w = stbi__get32be(s);\n\n if (h > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n if (w > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n\n // Make sure the depth is 8 bits.\n bitdepth = stbi__get16be(s);\n if (bitdepth != 8 && bitdepth != 16)\n return stbi__errpuc(\"unsupported bit depth\", \"PSD bit depth is not 8 or 16 bit\");\n\n // Make sure the color mode is RGB.\n // Valid options are:\n // 0: Bitmap\n // 1: Grayscale\n // 2: Indexed color\n // 3: RGB color\n // 4: CMYK color\n // 7: Multichannel\n // 8: Duotone\n // 9: Lab color\n if (stbi__get16be(s) != 3)\n return stbi__errpuc(\"wrong color format\", \"PSD is not in RGB color format\");\n\n // Skip the Mode Data. (It's the palette for indexed color; other info for other modes.)\n stbi__skip(s,stbi__get32be(s) );\n\n // Skip the image resources. (resolution, pen tool paths, etc)\n stbi__skip(s, stbi__get32be(s) );\n\n // Skip the reserved data.\n stbi__skip(s, stbi__get32be(s) );\n\n // Find out if the data is compressed.\n // Known values:\n // 0: no compression\n // 1: RLE compressed\n compression = stbi__get16be(s);\n if (compression > 1)\n return stbi__errpuc(\"bad compression\", \"PSD has an unknown compression format\");\n\n // Check size\n if (!stbi__mad3sizes_valid(4, w, h, 0))\n return stbi__errpuc(\"too large\", \"Corrupt PSD\");\n\n // Create the destination image.\n\n if (!compression && bitdepth == 16 && bpc == 16) {\n out = (stbi_uc *) stbi__malloc_mad3(8, w, h, 0);\n ri->bits_per_channel = 16;\n } else\n out = (stbi_uc *) stbi__malloc(4 * w*h);\n\n if (!out) return stbi__errpuc(\"outofmem\", \"Out of memory\");\n pixelCount = w*h;\n\n // Initialize the data to zero.\n //memset( out, 0, pixelCount * 4 );\n\n // Finally, the image data.\n if (compression) {\n // RLE as used by .PSD and .TIFF\n // Loop until you get the number of unpacked bytes you are expecting:\n // Read the next source byte into n.\n // If n is between 0 and 127 inclusive, copy the next n+1 bytes literally.\n // Else if n is between -127 and -1 inclusive, copy the next byte -n+1 times.\n // Else if n is 128, noop.\n // Endloop\n\n // The RLE-compressed data is preceded by a 2-byte data count for each row in the data,\n // which we're going to just skip.\n stbi__skip(s, h * channelCount * 2 );\n\n // Read the RLE data by channel.\n for (channel = 0; channel < 4; channel++) {\n stbi_uc *p;\n\n p = out+channel;\n if (channel >= channelCount) {\n // Fill this channel with default data.\n for (i = 0; i < pixelCount; i++, p += 4)\n *p = (channel == 3 ? 255 : 0);\n } else {\n // Read the RLE data.\n if (!stbi__psd_decode_rle(s, p, pixelCount)) {\n STBI_FREE(out);\n return stbi__errpuc(\"corrupt\", \"bad RLE data\");\n }\n }\n }\n\n } else {\n // We're at the raw image data. It's each channel in order (Red, Green, Blue, Alpha, ...)\n // where each channel consists of an 8-bit (or 16-bit) value for each pixel in the image.\n\n // Read the data by channel.\n for (channel = 0; channel < 4; channel++) {\n if (channel >= channelCount) {\n // Fill this channel with default data.\n if (bitdepth == 16 && bpc == 16) {\n stbi__uint16 *q = ((stbi__uint16 *) out) + channel;\n stbi__uint16 val = channel == 3 ? 65535 : 0;\n for (i = 0; i < pixelCount; i++, q += 4)\n *q = val;\n } else {\n stbi_uc *p = out+channel;\n stbi_uc val = channel == 3 ? 255 : 0;\n for (i = 0; i < pixelCount; i++, p += 4)\n *p = val;\n }\n } else {\n if (ri->bits_per_channel == 16) { // output bpc\n stbi__uint16 *q = ((stbi__uint16 *) out) + channel;\n for (i = 0; i < pixelCount; i++, q += 4)\n *q = (stbi__uint16) stbi__get16be(s);\n } else {\n stbi_uc *p = out+channel;\n if (bitdepth == 16) { // input bpc\n for (i = 0; i < pixelCount; i++, p += 4)\n *p = (stbi_uc) (stbi__get16be(s) >> 8);\n } else {\n for (i = 0; i < pixelCount; i++, p += 4)\n *p = stbi__get8(s);\n }\n }\n }\n }\n }\n\n // remove weird white matte from PSD\n if (channelCount >= 4) {\n if (ri->bits_per_channel == 16) {\n for (i=0; i < w*h; ++i) {\n stbi__uint16 *pixel = (stbi__uint16 *) out + 4*i;\n if (pixel[3] != 0 && pixel[3] != 65535) {\n float a = pixel[3] / 65535.0f;\n float ra = 1.0f / a;\n float inv_a = 65535.0f * (1 - ra);\n pixel[0] = (stbi__uint16) (pixel[0]*ra + inv_a);\n pixel[1] = (stbi__uint16) (pixel[1]*ra + inv_a);\n pixel[2] = (stbi__uint16) (pixel[2]*ra + inv_a);\n }\n }\n } else {\n for (i=0; i < w*h; ++i) {\n unsigned char *pixel = out + 4*i;\n if (pixel[3] != 0 && pixel[3] != 255) {\n float a = pixel[3] / 255.0f;\n float ra = 1.0f / a;\n float inv_a = 255.0f * (1 - ra);\n pixel[0] = (unsigned char) (pixel[0]*ra + inv_a);\n pixel[1] = (unsigned char) (pixel[1]*ra + inv_a);\n pixel[2] = (unsigned char) (pixel[2]*ra + inv_a);\n }\n }\n }\n }\n\n // convert to desired output format\n if (req_comp && req_comp != 4) {\n if (ri->bits_per_channel == 16)\n out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, 4, req_comp, w, h);\n else\n out = stbi__convert_format(out, 4, req_comp, w, h);\n if (out == NULL) return out; // stbi__convert_format frees input on failure\n }\n\n if (comp) *comp = 4;\n *y = h;\n *x = w;\n\n return out;\n}\n#endif\n\n// *************************************************************************************************\n// Softimage PIC loader\n// by Tom Seddon\n//\n// See http://softimage.wiki.softimage.com/index.php/INFO:_PIC_file_format\n// See http://ozviz.wasp.uwa.edu.au/~pbourke/dataformats/softimagepic/\n\n#ifndef STBI_NO_PIC\nstatic int stbi__pic_is4(stbi__context *s,const char *str)\n{\n int i;\n for (i=0; i<4; ++i)\n if (stbi__get8(s) != (stbi_uc)str[i])\n return 0;\n\n return 1;\n}\n\nstatic int stbi__pic_test_core(stbi__context *s)\n{\n int i;\n\n if (!stbi__pic_is4(s,\"\\x53\\x80\\xF6\\x34\"))\n return 0;\n\n for(i=0;i<84;++i)\n stbi__get8(s);\n\n if (!stbi__pic_is4(s,\"PICT\"))\n return 0;\n\n return 1;\n}\n\ntypedef struct\n{\n stbi_uc size,type,channel;\n} stbi__pic_packet;\n\nstatic stbi_uc *stbi__readval(stbi__context *s, int channel, stbi_uc *dest)\n{\n int mask=0x80, i;\n\n for (i=0; i<4; ++i, mask>>=1) {\n if (channel & mask) {\n if (stbi__at_eof(s)) return stbi__errpuc(\"bad file\",\"PIC file too short\");\n dest[i]=stbi__get8(s);\n }\n }\n\n return dest;\n}\n\nstatic void stbi__copyval(int channel,stbi_uc *dest,const stbi_uc *src)\n{\n int mask=0x80,i;\n\n for (i=0;i<4; ++i, mask>>=1)\n if (channel&mask)\n dest[i]=src[i];\n}\n\nstatic stbi_uc *stbi__pic_load_core(stbi__context *s,int width,int height,int *comp, stbi_uc *result)\n{\n int act_comp=0,num_packets=0,y,chained;\n stbi__pic_packet packets[10];\n\n // this will (should...) cater for even some bizarre stuff like having data\n // for the same channel in multiple packets.\n do {\n stbi__pic_packet *packet;\n\n if (num_packets==sizeof(packets)/sizeof(packets[0]))\n return stbi__errpuc(\"bad format\",\"too many packets\");\n\n packet = &packets[num_packets++];\n\n chained = stbi__get8(s);\n packet->size = stbi__get8(s);\n packet->type = stbi__get8(s);\n packet->channel = stbi__get8(s);\n\n act_comp |= packet->channel;\n\n if (stbi__at_eof(s)) return stbi__errpuc(\"bad file\",\"file too short (reading packets)\");\n if (packet->size != 8) return stbi__errpuc(\"bad format\",\"packet isn't 8bpp\");\n } while (chained);\n\n *comp = (act_comp & 0x10 ? 4 : 3); // has alpha channel?\n\n for(y=0; ytype) {\n default:\n return stbi__errpuc(\"bad format\",\"packet has bad compression type\");\n\n case 0: {//uncompressed\n int x;\n\n for(x=0;xchannel,dest))\n return 0;\n break;\n }\n\n case 1://Pure RLE\n {\n int left=width, i;\n\n while (left>0) {\n stbi_uc count,value[4];\n\n count=stbi__get8(s);\n if (stbi__at_eof(s)) return stbi__errpuc(\"bad file\",\"file too short (pure read count)\");\n\n if (count > left)\n count = (stbi_uc) left;\n\n if (!stbi__readval(s,packet->channel,value)) return 0;\n\n for(i=0; ichannel,dest,value);\n left -= count;\n }\n }\n break;\n\n case 2: {//Mixed RLE\n int left=width;\n while (left>0) {\n int count = stbi__get8(s), i;\n if (stbi__at_eof(s)) return stbi__errpuc(\"bad file\",\"file too short (mixed read count)\");\n\n if (count >= 128) { // Repeated\n stbi_uc value[4];\n\n if (count==128)\n count = stbi__get16be(s);\n else\n count -= 127;\n if (count > left)\n return stbi__errpuc(\"bad file\",\"scanline overrun\");\n\n if (!stbi__readval(s,packet->channel,value))\n return 0;\n\n for(i=0;ichannel,dest,value);\n } else { // Raw\n ++count;\n if (count>left) return stbi__errpuc(\"bad file\",\"scanline overrun\");\n\n for(i=0;ichannel,dest))\n return 0;\n }\n left-=count;\n }\n break;\n }\n }\n }\n }\n\n return result;\n}\n\nstatic void *stbi__pic_load(stbi__context *s,int *px,int *py,int *comp,int req_comp, stbi__result_info *ri)\n{\n stbi_uc *result;\n int i, x,y, internal_comp;\n STBI_NOTUSED(ri);\n\n if (!comp) comp = &internal_comp;\n\n for (i=0; i<92; ++i)\n stbi__get8(s);\n\n x = stbi__get16be(s);\n y = stbi__get16be(s);\n\n if (y > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n if (x > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n\n if (stbi__at_eof(s)) return stbi__errpuc(\"bad file\",\"file too short (pic header)\");\n if (!stbi__mad3sizes_valid(x, y, 4, 0)) return stbi__errpuc(\"too large\", \"PIC image too large to decode\");\n\n stbi__get32be(s); //skip `ratio'\n stbi__get16be(s); //skip `fields'\n stbi__get16be(s); //skip `pad'\n\n // intermediate buffer is RGBA\n result = (stbi_uc *) stbi__malloc_mad3(x, y, 4, 0);\n if (!result) return stbi__errpuc(\"outofmem\", \"Out of memory\");\n memset(result, 0xff, x*y*4);\n\n if (!stbi__pic_load_core(s,x,y,comp, result)) {\n STBI_FREE(result);\n result=0;\n }\n *px = x;\n *py = y;\n if (req_comp == 0) req_comp = *comp;\n result=stbi__convert_format(result,4,req_comp,x,y);\n\n return result;\n}\n\nstatic int stbi__pic_test(stbi__context *s)\n{\n int r = stbi__pic_test_core(s);\n stbi__rewind(s);\n return r;\n}\n#endif\n\n// *************************************************************************************************\n// GIF loader -- public domain by Jean-Marc Lienher -- simplified/shrunk by stb\n\n#ifndef STBI_NO_GIF\ntypedef struct\n{\n stbi__int16 prefix;\n stbi_uc first;\n stbi_uc suffix;\n} stbi__gif_lzw;\n\ntypedef struct\n{\n int w,h;\n stbi_uc *out; // output buffer (always 4 components)\n stbi_uc *background; // The current \"background\" as far as a gif is concerned\n stbi_uc *history;\n int flags, bgindex, ratio, transparent, eflags;\n stbi_uc pal[256][4];\n stbi_uc lpal[256][4];\n stbi__gif_lzw codes[8192];\n stbi_uc *color_table;\n int parse, step;\n int lflags;\n int start_x, start_y;\n int max_x, max_y;\n int cur_x, cur_y;\n int line_size;\n int delay;\n} stbi__gif;\n\nstatic int stbi__gif_test_raw(stbi__context *s)\n{\n int sz;\n if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') return 0;\n sz = stbi__get8(s);\n if (sz != '9' && sz != '7') return 0;\n if (stbi__get8(s) != 'a') return 0;\n return 1;\n}\n\nstatic int stbi__gif_test(stbi__context *s)\n{\n int r = stbi__gif_test_raw(s);\n stbi__rewind(s);\n return r;\n}\n\nstatic void stbi__gif_parse_colortable(stbi__context *s, stbi_uc pal[256][4], int num_entries, int transp)\n{\n int i;\n for (i=0; i < num_entries; ++i) {\n pal[i][2] = stbi__get8(s);\n pal[i][1] = stbi__get8(s);\n pal[i][0] = stbi__get8(s);\n pal[i][3] = transp == i ? 0 : 255;\n }\n}\n\nstatic int stbi__gif_header(stbi__context *s, stbi__gif *g, int *comp, int is_info)\n{\n stbi_uc version;\n if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8')\n return stbi__err(\"not GIF\", \"Corrupt GIF\");\n\n version = stbi__get8(s);\n if (version != '7' && version != '9') return stbi__err(\"not GIF\", \"Corrupt GIF\");\n if (stbi__get8(s) != 'a') return stbi__err(\"not GIF\", \"Corrupt GIF\");\n\n stbi__g_failure_reason = \"\";\n g->w = stbi__get16le(s);\n g->h = stbi__get16le(s);\n g->flags = stbi__get8(s);\n g->bgindex = stbi__get8(s);\n g->ratio = stbi__get8(s);\n g->transparent = -1;\n\n if (g->w > STBI_MAX_DIMENSIONS) return stbi__err(\"too large\",\"Very large image (corrupt?)\");\n if (g->h > STBI_MAX_DIMENSIONS) return stbi__err(\"too large\",\"Very large image (corrupt?)\");\n\n if (comp != 0) *comp = 4; // can't actually tell whether it's 3 or 4 until we parse the comments\n\n if (is_info) return 1;\n\n if (g->flags & 0x80)\n stbi__gif_parse_colortable(s,g->pal, 2 << (g->flags & 7), -1);\n\n return 1;\n}\n\nstatic int stbi__gif_info_raw(stbi__context *s, int *x, int *y, int *comp)\n{\n stbi__gif* g = (stbi__gif*) stbi__malloc(sizeof(stbi__gif));\n if (!g) return stbi__err(\"outofmem\", \"Out of memory\");\n if (!stbi__gif_header(s, g, comp, 1)) {\n STBI_FREE(g);\n stbi__rewind( s );\n return 0;\n }\n if (x) *x = g->w;\n if (y) *y = g->h;\n STBI_FREE(g);\n return 1;\n}\n\nstatic void stbi__out_gif_code(stbi__gif *g, stbi__uint16 code)\n{\n stbi_uc *p, *c;\n int idx;\n\n // recurse to decode the prefixes, since the linked-list is backwards,\n // and working backwards through an interleaved image would be nasty\n if (g->codes[code].prefix >= 0)\n stbi__out_gif_code(g, g->codes[code].prefix);\n\n if (g->cur_y >= g->max_y) return;\n\n idx = g->cur_x + g->cur_y;\n p = &g->out[idx];\n g->history[idx / 4] = 1;\n\n c = &g->color_table[g->codes[code].suffix * 4];\n if (c[3] > 128) { // don't render transparent pixels;\n p[0] = c[2];\n p[1] = c[1];\n p[2] = c[0];\n p[3] = c[3];\n }\n g->cur_x += 4;\n\n if (g->cur_x >= g->max_x) {\n g->cur_x = g->start_x;\n g->cur_y += g->step;\n\n while (g->cur_y >= g->max_y && g->parse > 0) {\n g->step = (1 << g->parse) * g->line_size;\n g->cur_y = g->start_y + (g->step >> 1);\n --g->parse;\n }\n }\n}\n\nstatic stbi_uc *stbi__process_gif_raster(stbi__context *s, stbi__gif *g)\n{\n stbi_uc lzw_cs;\n stbi__int32 len, init_code;\n stbi__uint32 first;\n stbi__int32 codesize, codemask, avail, oldcode, bits, valid_bits, clear;\n stbi__gif_lzw *p;\n\n lzw_cs = stbi__get8(s);\n if (lzw_cs > 12) return NULL;\n clear = 1 << lzw_cs;\n first = 1;\n codesize = lzw_cs + 1;\n codemask = (1 << codesize) - 1;\n bits = 0;\n valid_bits = 0;\n for (init_code = 0; init_code < clear; init_code++) {\n g->codes[init_code].prefix = -1;\n g->codes[init_code].first = (stbi_uc) init_code;\n g->codes[init_code].suffix = (stbi_uc) init_code;\n }\n\n // support no starting clear code\n avail = clear+2;\n oldcode = -1;\n\n len = 0;\n for(;;) {\n if (valid_bits < codesize) {\n if (len == 0) {\n len = stbi__get8(s); // start new block\n if (len == 0)\n return g->out;\n }\n --len;\n bits |= (stbi__int32) stbi__get8(s) << valid_bits;\n valid_bits += 8;\n } else {\n stbi__int32 code = bits & codemask;\n bits >>= codesize;\n valid_bits -= codesize;\n // @OPTIMIZE: is there some way we can accelerate the non-clear path?\n if (code == clear) { // clear code\n codesize = lzw_cs + 1;\n codemask = (1 << codesize) - 1;\n avail = clear + 2;\n oldcode = -1;\n first = 0;\n } else if (code == clear + 1) { // end of stream code\n stbi__skip(s, len);\n while ((len = stbi__get8(s)) > 0)\n stbi__skip(s,len);\n return g->out;\n } else if (code <= avail) {\n if (first) {\n return stbi__errpuc(\"no clear code\", \"Corrupt GIF\");\n }\n\n if (oldcode >= 0) {\n p = &g->codes[avail++];\n if (avail > 8192) {\n return stbi__errpuc(\"too many codes\", \"Corrupt GIF\");\n }\n\n p->prefix = (stbi__int16) oldcode;\n p->first = g->codes[oldcode].first;\n p->suffix = (code == avail) ? p->first : g->codes[code].first;\n } else if (code == avail)\n return stbi__errpuc(\"illegal code in raster\", \"Corrupt GIF\");\n\n stbi__out_gif_code(g, (stbi__uint16) code);\n\n if ((avail & codemask) == 0 && avail <= 0x0FFF) {\n codesize++;\n codemask = (1 << codesize) - 1;\n }\n\n oldcode = code;\n } else {\n return stbi__errpuc(\"illegal code in raster\", \"Corrupt GIF\");\n }\n }\n }\n}\n\n// this function is designed to support animated gifs, although stb_image doesn't support it\n// two back is the image from two frames ago, used for a very specific disposal format\nstatic stbi_uc *stbi__gif_load_next(stbi__context *s, stbi__gif *g, int *comp, int req_comp, stbi_uc *two_back)\n{\n int dispose;\n int first_frame;\n int pi;\n int pcount;\n STBI_NOTUSED(req_comp);\n\n // on first frame, any non-written pixels get the background colour (non-transparent)\n first_frame = 0;\n if (g->out == 0) {\n if (!stbi__gif_header(s, g, comp,0)) return 0; // stbi__g_failure_reason set by stbi__gif_header\n if (!stbi__mad3sizes_valid(4, g->w, g->h, 0))\n return stbi__errpuc(\"too large\", \"GIF image is too large\");\n pcount = g->w * g->h;\n g->out = (stbi_uc *) stbi__malloc(4 * pcount);\n g->background = (stbi_uc *) stbi__malloc(4 * pcount);\n g->history = (stbi_uc *) stbi__malloc(pcount);\n if (!g->out || !g->background || !g->history)\n return stbi__errpuc(\"outofmem\", \"Out of memory\");\n\n // image is treated as \"transparent\" at the start - ie, nothing overwrites the current background;\n // background colour is only used for pixels that are not rendered first frame, after that \"background\"\n // color refers to the color that was there the previous frame.\n memset(g->out, 0x00, 4 * pcount);\n memset(g->background, 0x00, 4 * pcount); // state of the background (starts transparent)\n memset(g->history, 0x00, pcount); // pixels that were affected previous frame\n first_frame = 1;\n } else {\n // second frame - how do we dispose of the previous one?\n dispose = (g->eflags & 0x1C) >> 2;\n pcount = g->w * g->h;\n\n if ((dispose == 3) && (two_back == 0)) {\n dispose = 2; // if I don't have an image to revert back to, default to the old background\n }\n\n if (dispose == 3) { // use previous graphic\n for (pi = 0; pi < pcount; ++pi) {\n if (g->history[pi]) {\n memcpy( &g->out[pi * 4], &two_back[pi * 4], 4 );\n }\n }\n } else if (dispose == 2) {\n // restore what was changed last frame to background before that frame;\n for (pi = 0; pi < pcount; ++pi) {\n if (g->history[pi]) {\n memcpy( &g->out[pi * 4], &g->background[pi * 4], 4 );\n }\n }\n } else {\n // This is a non-disposal case eithe way, so just\n // leave the pixels as is, and they will become the new background\n // 1: do not dispose\n // 0: not specified.\n }\n\n // background is what out is after the undoing of the previou frame;\n memcpy( g->background, g->out, 4 * g->w * g->h );\n }\n\n // clear my history;\n memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame\n\n for (;;) {\n int tag = stbi__get8(s);\n switch (tag) {\n case 0x2C: /* Image Descriptor */\n {\n stbi__int32 x, y, w, h;\n stbi_uc *o;\n\n x = stbi__get16le(s);\n y = stbi__get16le(s);\n w = stbi__get16le(s);\n h = stbi__get16le(s);\n if (((x + w) > (g->w)) || ((y + h) > (g->h)))\n return stbi__errpuc(\"bad Image Descriptor\", \"Corrupt GIF\");\n\n g->line_size = g->w * 4;\n g->start_x = x * 4;\n g->start_y = y * g->line_size;\n g->max_x = g->start_x + w * 4;\n g->max_y = g->start_y + h * g->line_size;\n g->cur_x = g->start_x;\n g->cur_y = g->start_y;\n\n // if the width of the specified rectangle is 0, that means\n // we may not see *any* pixels or the image is malformed;\n // to make sure this is caught, move the current y down to\n // max_y (which is what out_gif_code checks).\n if (w == 0)\n g->cur_y = g->max_y;\n\n g->lflags = stbi__get8(s);\n\n if (g->lflags & 0x40) {\n g->step = 8 * g->line_size; // first interlaced spacing\n g->parse = 3;\n } else {\n g->step = g->line_size;\n g->parse = 0;\n }\n\n if (g->lflags & 0x80) {\n stbi__gif_parse_colortable(s,g->lpal, 2 << (g->lflags & 7), g->eflags & 0x01 ? g->transparent : -1);\n g->color_table = (stbi_uc *) g->lpal;\n } else if (g->flags & 0x80) {\n g->color_table = (stbi_uc *) g->pal;\n } else\n return stbi__errpuc(\"missing color table\", \"Corrupt GIF\");\n\n o = stbi__process_gif_raster(s, g);\n if (!o) return NULL;\n\n // if this was the first frame,\n pcount = g->w * g->h;\n if (first_frame && (g->bgindex > 0)) {\n // if first frame, any pixel not drawn to gets the background color\n for (pi = 0; pi < pcount; ++pi) {\n if (g->history[pi] == 0) {\n g->pal[g->bgindex][3] = 255; // just in case it was made transparent, undo that; It will be reset next frame if need be;\n memcpy( &g->out[pi * 4], &g->pal[g->bgindex], 4 );\n }\n }\n }\n\n return o;\n }\n\n case 0x21: // Comment Extension.\n {\n int len;\n int ext = stbi__get8(s);\n if (ext == 0xF9) { // Graphic Control Extension.\n len = stbi__get8(s);\n if (len == 4) {\n g->eflags = stbi__get8(s);\n g->delay = 10 * stbi__get16le(s); // delay - 1/100th of a second, saving as 1/1000ths.\n\n // unset old transparent\n if (g->transparent >= 0) {\n g->pal[g->transparent][3] = 255;\n }\n if (g->eflags & 0x01) {\n g->transparent = stbi__get8(s);\n if (g->transparent >= 0) {\n g->pal[g->transparent][3] = 0;\n }\n } else {\n // don't need transparent\n stbi__skip(s, 1);\n g->transparent = -1;\n }\n } else {\n stbi__skip(s, len);\n break;\n }\n }\n while ((len = stbi__get8(s)) != 0) {\n stbi__skip(s, len);\n }\n break;\n }\n\n case 0x3B: // gif stream termination code\n return (stbi_uc *) s; // using '1' causes warning on some compilers\n\n default:\n return stbi__errpuc(\"unknown code\", \"Corrupt GIF\");\n }\n }\n}\n\nstatic void *stbi__load_gif_main_outofmem(stbi__gif *g, stbi_uc *out, int **delays)\n{\n STBI_FREE(g->out);\n STBI_FREE(g->history);\n STBI_FREE(g->background);\n\n if (out) STBI_FREE(out);\n if (delays && *delays) STBI_FREE(*delays);\n return stbi__errpuc(\"outofmem\", \"Out of memory\");\n}\n\nstatic void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp)\n{\n if (stbi__gif_test(s)) {\n int layers = 0;\n stbi_uc *u = 0;\n stbi_uc *out = 0;\n stbi_uc *two_back = 0;\n stbi__gif g;\n int stride;\n int out_size = 0;\n int delays_size = 0;\n\n STBI_NOTUSED(out_size);\n STBI_NOTUSED(delays_size);\n\n memset(&g, 0, sizeof(g));\n if (delays) {\n *delays = 0;\n }\n\n do {\n u = stbi__gif_load_next(s, &g, comp, req_comp, two_back);\n if (u == (stbi_uc *) s) u = 0; // end of animated gif marker\n\n if (u) {\n *x = g.w;\n *y = g.h;\n ++layers;\n stride = g.w * g.h * 4;\n\n if (out) {\n void *tmp = (stbi_uc*) STBI_REALLOC_SIZED( out, out_size, layers * stride );\n if (!tmp)\n return stbi__load_gif_main_outofmem(&g, out, delays);\n else {\n out = (stbi_uc*) tmp;\n out_size = layers * stride;\n }\n\n if (delays) {\n int *new_delays = (int*) STBI_REALLOC_SIZED( *delays, delays_size, sizeof(int) * layers );\n if (!new_delays)\n return stbi__load_gif_main_outofmem(&g, out, delays);\n *delays = new_delays;\n delays_size = layers * sizeof(int);\n }\n } else {\n out = (stbi_uc*)stbi__malloc( layers * stride );\n if (!out)\n return stbi__load_gif_main_outofmem(&g, out, delays);\n out_size = layers * stride;\n if (delays) {\n *delays = (int*) stbi__malloc( layers * sizeof(int) );\n if (!*delays)\n return stbi__load_gif_main_outofmem(&g, out, delays);\n delays_size = layers * sizeof(int);\n }\n }\n memcpy( out + ((layers - 1) * stride), u, stride );\n if (layers >= 2) {\n two_back = out - 2 * stride;\n }\n\n if (delays) {\n (*delays)[layers - 1U] = g.delay;\n }\n }\n } while (u != 0);\n\n // free temp buffer;\n STBI_FREE(g.out);\n STBI_FREE(g.history);\n STBI_FREE(g.background);\n\n // do the final conversion after loading everything;\n if (req_comp && req_comp != 4)\n out = stbi__convert_format(out, 4, req_comp, layers * g.w, g.h);\n\n *z = layers;\n return out;\n } else {\n return stbi__errpuc(\"not GIF\", \"Image was not as a gif type.\");\n }\n}\n\nstatic void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)\n{\n stbi_uc *u = 0;\n stbi__gif g;\n memset(&g, 0, sizeof(g));\n STBI_NOTUSED(ri);\n\n u = stbi__gif_load_next(s, &g, comp, req_comp, 0);\n if (u == (stbi_uc *) s) u = 0; // end of animated gif marker\n if (u) {\n *x = g.w;\n *y = g.h;\n\n // moved conversion to after successful load so that the same\n // can be done for multiple frames.\n if (req_comp && req_comp != 4)\n u = stbi__convert_format(u, 4, req_comp, g.w, g.h);\n } else if (g.out) {\n // if there was an error and we allocated an image buffer, free it!\n STBI_FREE(g.out);\n }\n\n // free buffers needed for multiple frame loading;\n STBI_FREE(g.history);\n STBI_FREE(g.background);\n\n return u;\n}\n\nstatic int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp)\n{\n return stbi__gif_info_raw(s,x,y,comp);\n}\n#endif\n\n// *************************************************************************************************\n// Radiance RGBE HDR loader\n// originally by Nicolas Schulz\n#ifndef STBI_NO_HDR\nstatic int stbi__hdr_test_core(stbi__context *s, const char *signature)\n{\n int i;\n for (i=0; signature[i]; ++i)\n if (stbi__get8(s) != signature[i])\n return 0;\n stbi__rewind(s);\n return 1;\n}\n\nstatic int stbi__hdr_test(stbi__context* s)\n{\n int r = stbi__hdr_test_core(s, \"#?RADIANCE\\n\");\n stbi__rewind(s);\n if(!r) {\n r = stbi__hdr_test_core(s, \"#?RGBE\\n\");\n stbi__rewind(s);\n }\n return r;\n}\n\n#define STBI__HDR_BUFLEN 1024\nstatic char *stbi__hdr_gettoken(stbi__context *z, char *buffer)\n{\n int len=0;\n char c = '\\0';\n\n c = (char) stbi__get8(z);\n\n while (!stbi__at_eof(z) && c != '\\n') {\n buffer[len++] = c;\n if (len == STBI__HDR_BUFLEN-1) {\n // flush to end of line\n while (!stbi__at_eof(z) && stbi__get8(z) != '\\n')\n ;\n break;\n }\n c = (char) stbi__get8(z);\n }\n\n buffer[len] = 0;\n return buffer;\n}\n\nstatic void stbi__hdr_convert(float *output, stbi_uc *input, int req_comp)\n{\n if ( input[3] != 0 ) {\n float f1;\n // Exponent\n f1 = (float) ldexp(1.0f, input[3] - (int)(128 + 8));\n if (req_comp <= 2)\n output[0] = (input[0] + input[1] + input[2]) * f1 / 3;\n else {\n output[0] = input[0] * f1;\n output[1] = input[1] * f1;\n output[2] = input[2] * f1;\n }\n if (req_comp == 2) output[1] = 1;\n if (req_comp == 4) output[3] = 1;\n } else {\n switch (req_comp) {\n case 4: output[3] = 1; /* fallthrough */\n case 3: output[0] = output[1] = output[2] = 0;\n break;\n case 2: output[1] = 1; /* fallthrough */\n case 1: output[0] = 0;\n break;\n }\n }\n}\n\nstatic float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)\n{\n char buffer[STBI__HDR_BUFLEN];\n char *token;\n int valid = 0;\n int width, height;\n stbi_uc *scanline;\n float *hdr_data;\n int len;\n unsigned char count, value;\n int i, j, k, c1,c2, z;\n const char *headerToken;\n STBI_NOTUSED(ri);\n\n // Check identifier\n headerToken = stbi__hdr_gettoken(s,buffer);\n if (strcmp(headerToken, \"#?RADIANCE\") != 0 && strcmp(headerToken, \"#?RGBE\") != 0)\n return stbi__errpf(\"not HDR\", \"Corrupt HDR image\");\n\n // Parse header\n for(;;) {\n token = stbi__hdr_gettoken(s,buffer);\n if (token[0] == 0) break;\n if (strcmp(token, \"FORMAT=32-bit_rle_rgbe\") == 0) valid = 1;\n }\n\n if (!valid) return stbi__errpf(\"unsupported format\", \"Unsupported HDR format\");\n\n // Parse width and height\n // can't use sscanf() if we're not using stdio!\n token = stbi__hdr_gettoken(s,buffer);\n if (strncmp(token, \"-Y \", 3)) return stbi__errpf(\"unsupported data layout\", \"Unsupported HDR format\");\n token += 3;\n height = (int) strtol(token, &token, 10);\n while (*token == ' ') ++token;\n if (strncmp(token, \"+X \", 3)) return stbi__errpf(\"unsupported data layout\", \"Unsupported HDR format\");\n token += 3;\n width = (int) strtol(token, NULL, 10);\n\n if (height > STBI_MAX_DIMENSIONS) return stbi__errpf(\"too large\",\"Very large image (corrupt?)\");\n if (width > STBI_MAX_DIMENSIONS) return stbi__errpf(\"too large\",\"Very large image (corrupt?)\");\n\n *x = width;\n *y = height;\n\n if (comp) *comp = 3;\n if (req_comp == 0) req_comp = 3;\n\n if (!stbi__mad4sizes_valid(width, height, req_comp, sizeof(float), 0))\n return stbi__errpf(\"too large\", \"HDR image is too large\");\n\n // Read data\n hdr_data = (float *) stbi__malloc_mad4(width, height, req_comp, sizeof(float), 0);\n if (!hdr_data)\n return stbi__errpf(\"outofmem\", \"Out of memory\");\n\n // Load image data\n // image data is stored as some number of sca\n if ( width < 8 || width >= 32768) {\n // Read flat data\n for (j=0; j < height; ++j) {\n for (i=0; i < width; ++i) {\n stbi_uc rgbe[4];\n main_decode_loop:\n stbi__getn(s, rgbe, 4);\n stbi__hdr_convert(hdr_data + j * width * req_comp + i * req_comp, rgbe, req_comp);\n }\n }\n } else {\n // Read RLE-encoded data\n scanline = NULL;\n\n for (j = 0; j < height; ++j) {\n c1 = stbi__get8(s);\n c2 = stbi__get8(s);\n len = stbi__get8(s);\n if (c1 != 2 || c2 != 2 || (len & 0x80)) {\n // not run-length encoded, so we have to actually use THIS data as a decoded\n // pixel (note this can't be a valid pixel--one of RGB must be >= 128)\n stbi_uc rgbe[4];\n rgbe[0] = (stbi_uc) c1;\n rgbe[1] = (stbi_uc) c2;\n rgbe[2] = (stbi_uc) len;\n rgbe[3] = (stbi_uc) stbi__get8(s);\n stbi__hdr_convert(hdr_data, rgbe, req_comp);\n i = 1;\n j = 0;\n STBI_FREE(scanline);\n goto main_decode_loop; // yes, this makes no sense\n }\n len <<= 8;\n len |= stbi__get8(s);\n if (len != width) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf(\"invalid decoded scanline length\", \"corrupt HDR\"); }\n if (scanline == NULL) {\n scanline = (stbi_uc *) stbi__malloc_mad2(width, 4, 0);\n if (!scanline) {\n STBI_FREE(hdr_data);\n return stbi__errpf(\"outofmem\", \"Out of memory\");\n }\n }\n\n for (k = 0; k < 4; ++k) {\n int nleft;\n i = 0;\n while ((nleft = width - i) > 0) {\n count = stbi__get8(s);\n if (count > 128) {\n // Run\n value = stbi__get8(s);\n count -= 128;\n if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf(\"corrupt\", \"bad RLE data in HDR\"); }\n for (z = 0; z < count; ++z)\n scanline[i++ * 4 + k] = value;\n } else {\n // Dump\n if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf(\"corrupt\", \"bad RLE data in HDR\"); }\n for (z = 0; z < count; ++z)\n scanline[i++ * 4 + k] = stbi__get8(s);\n }\n }\n }\n for (i=0; i < width; ++i)\n stbi__hdr_convert(hdr_data+(j*width + i)*req_comp, scanline + i*4, req_comp);\n }\n if (scanline)\n STBI_FREE(scanline);\n }\n\n return hdr_data;\n}\n\nstatic int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp)\n{\n char buffer[STBI__HDR_BUFLEN];\n char *token;\n int valid = 0;\n int dummy;\n\n if (!x) x = &dummy;\n if (!y) y = &dummy;\n if (!comp) comp = &dummy;\n\n if (stbi__hdr_test(s) == 0) {\n stbi__rewind( s );\n return 0;\n }\n\n for(;;) {\n token = stbi__hdr_gettoken(s,buffer);\n if (token[0] == 0) break;\n if (strcmp(token, \"FORMAT=32-bit_rle_rgbe\") == 0) valid = 1;\n }\n\n if (!valid) {\n stbi__rewind( s );\n return 0;\n }\n token = stbi__hdr_gettoken(s,buffer);\n if (strncmp(token, \"-Y \", 3)) {\n stbi__rewind( s );\n return 0;\n }\n token += 3;\n *y = (int) strtol(token, &token, 10);\n while (*token == ' ') ++token;\n if (strncmp(token, \"+X \", 3)) {\n stbi__rewind( s );\n return 0;\n }\n token += 3;\n *x = (int) strtol(token, NULL, 10);\n *comp = 3;\n return 1;\n}\n#endif // STBI_NO_HDR\n\n#ifndef STBI_NO_BMP\nstatic int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp)\n{\n void *p;\n stbi__bmp_data info;\n\n info.all_a = 255;\n p = stbi__bmp_parse_header(s, &info);\n if (p == NULL) {\n stbi__rewind( s );\n return 0;\n }\n if (x) *x = s->img_x;\n if (y) *y = s->img_y;\n if (comp) {\n if (info.bpp == 24 && info.ma == 0xff000000)\n *comp = 3;\n else\n *comp = info.ma ? 4 : 3;\n }\n return 1;\n}\n#endif\n\n#ifndef STBI_NO_PSD\nstatic int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp)\n{\n int channelCount, dummy, depth;\n if (!x) x = &dummy;\n if (!y) y = &dummy;\n if (!comp) comp = &dummy;\n if (stbi__get32be(s) != 0x38425053) {\n stbi__rewind( s );\n return 0;\n }\n if (stbi__get16be(s) != 1) {\n stbi__rewind( s );\n return 0;\n }\n stbi__skip(s, 6);\n channelCount = stbi__get16be(s);\n if (channelCount < 0 || channelCount > 16) {\n stbi__rewind( s );\n return 0;\n }\n *y = stbi__get32be(s);\n *x = stbi__get32be(s);\n depth = stbi__get16be(s);\n if (depth != 8 && depth != 16) {\n stbi__rewind( s );\n return 0;\n }\n if (stbi__get16be(s) != 3) {\n stbi__rewind( s );\n return 0;\n }\n *comp = 4;\n return 1;\n}\n\nstatic int stbi__psd_is16(stbi__context *s)\n{\n int channelCount, depth;\n if (stbi__get32be(s) != 0x38425053) {\n stbi__rewind( s );\n return 0;\n }\n if (stbi__get16be(s) != 1) {\n stbi__rewind( s );\n return 0;\n }\n stbi__skip(s, 6);\n channelCount = stbi__get16be(s);\n if (channelCount < 0 || channelCount > 16) {\n stbi__rewind( s );\n return 0;\n }\n STBI_NOTUSED(stbi__get32be(s));\n STBI_NOTUSED(stbi__get32be(s));\n depth = stbi__get16be(s);\n if (depth != 16) {\n stbi__rewind( s );\n return 0;\n }\n return 1;\n}\n#endif\n\n#ifndef STBI_NO_PIC\nstatic int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp)\n{\n int act_comp=0,num_packets=0,chained,dummy;\n stbi__pic_packet packets[10];\n\n if (!x) x = &dummy;\n if (!y) y = &dummy;\n if (!comp) comp = &dummy;\n\n if (!stbi__pic_is4(s,\"\\x53\\x80\\xF6\\x34\")) {\n stbi__rewind(s);\n return 0;\n }\n\n stbi__skip(s, 88);\n\n *x = stbi__get16be(s);\n *y = stbi__get16be(s);\n if (stbi__at_eof(s)) {\n stbi__rewind( s);\n return 0;\n }\n if ( (*x) != 0 && (1 << 28) / (*x) < (*y)) {\n stbi__rewind( s );\n return 0;\n }\n\n stbi__skip(s, 8);\n\n do {\n stbi__pic_packet *packet;\n\n if (num_packets==sizeof(packets)/sizeof(packets[0]))\n return 0;\n\n packet = &packets[num_packets++];\n chained = stbi__get8(s);\n packet->size = stbi__get8(s);\n packet->type = stbi__get8(s);\n packet->channel = stbi__get8(s);\n act_comp |= packet->channel;\n\n if (stbi__at_eof(s)) {\n stbi__rewind( s );\n return 0;\n }\n if (packet->size != 8) {\n stbi__rewind( s );\n return 0;\n }\n } while (chained);\n\n *comp = (act_comp & 0x10 ? 4 : 3);\n\n return 1;\n}\n#endif\n\n// *************************************************************************************************\n// Portable Gray Map and Portable Pixel Map loader\n// by Ken Miller\n//\n// PGM: http://netpbm.sourceforge.net/doc/pgm.html\n// PPM: http://netpbm.sourceforge.net/doc/ppm.html\n//\n// Known limitations:\n// Does not support comments in the header section\n// Does not support ASCII image data (formats P2 and P3)\n\n#ifndef STBI_NO_PNM\n\nstatic int stbi__pnm_test(stbi__context *s)\n{\n char p, t;\n p = (char) stbi__get8(s);\n t = (char) stbi__get8(s);\n if (p != 'P' || (t != '5' && t != '6')) {\n stbi__rewind( s );\n return 0;\n }\n return 1;\n}\n\nstatic void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)\n{\n stbi_uc *out;\n STBI_NOTUSED(ri);\n\n ri->bits_per_channel = stbi__pnm_info(s, (int *)&s->img_x, (int *)&s->img_y, (int *)&s->img_n);\n if (ri->bits_per_channel == 0)\n return 0;\n\n if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc(\"too large\",\"Very large image (corrupt?)\");\n\n *x = s->img_x;\n *y = s->img_y;\n if (comp) *comp = s->img_n;\n\n if (!stbi__mad4sizes_valid(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0))\n return stbi__errpuc(\"too large\", \"PNM too large\");\n\n out = (stbi_uc *) stbi__malloc_mad4(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0);\n if (!out) return stbi__errpuc(\"outofmem\", \"Out of memory\");\n if (!stbi__getn(s, out, s->img_n * s->img_x * s->img_y * (ri->bits_per_channel / 8))) {\n STBI_FREE(out);\n return stbi__errpuc(\"bad PNM\", \"PNM file truncated\");\n }\n\n if (req_comp && req_comp != s->img_n) {\n if (ri->bits_per_channel == 16) {\n out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, s->img_n, req_comp, s->img_x, s->img_y);\n } else {\n out = stbi__convert_format(out, s->img_n, req_comp, s->img_x, s->img_y);\n }\n if (out == NULL) return out; // stbi__convert_format frees input on failure\n }\n return out;\n}\n\nstatic int stbi__pnm_isspace(char c)\n{\n return c == ' ' || c == '\\t' || c == '\\n' || c == '\\v' || c == '\\f' || c == '\\r';\n}\n\nstatic void stbi__pnm_skip_whitespace(stbi__context *s, char *c)\n{\n for (;;) {\n while (!stbi__at_eof(s) && stbi__pnm_isspace(*c))\n *c = (char) stbi__get8(s);\n\n if (stbi__at_eof(s) || *c != '#')\n break;\n\n while (!stbi__at_eof(s) && *c != '\\n' && *c != '\\r' )\n *c = (char) stbi__get8(s);\n }\n}\n\nstatic int stbi__pnm_isdigit(char c)\n{\n return c >= '0' && c <= '9';\n}\n\nstatic int stbi__pnm_getinteger(stbi__context *s, char *c)\n{\n int value = 0;\n\n while (!stbi__at_eof(s) && stbi__pnm_isdigit(*c)) {\n value = value*10 + (*c - '0');\n *c = (char) stbi__get8(s);\n if((value > 214748364) || (value == 214748364 && *c > '7'))\n return stbi__err(\"integer parse overflow\", \"Parsing an integer in the PPM header overflowed a 32-bit int\");\n }\n\n return value;\n}\n\nstatic int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp)\n{\n int maxv, dummy;\n char c, p, t;\n\n if (!x) x = &dummy;\n if (!y) y = &dummy;\n if (!comp) comp = &dummy;\n\n stbi__rewind(s);\n\n // Get identifier\n p = (char) stbi__get8(s);\n t = (char) stbi__get8(s);\n if (p != 'P' || (t != '5' && t != '6')) {\n stbi__rewind(s);\n return 0;\n }\n\n *comp = (t == '6') ? 3 : 1; // '5' is 1-component .pgm; '6' is 3-component .ppm\n\n c = (char) stbi__get8(s);\n stbi__pnm_skip_whitespace(s, &c);\n\n *x = stbi__pnm_getinteger(s, &c); // read width\n if(*x == 0)\n return stbi__err(\"invalid width\", \"PPM image header had zero or overflowing width\");\n stbi__pnm_skip_whitespace(s, &c);\n\n *y = stbi__pnm_getinteger(s, &c); // read height\n if (*y == 0)\n return stbi__err(\"invalid width\", \"PPM image header had zero or overflowing width\");\n stbi__pnm_skip_whitespace(s, &c);\n\n maxv = stbi__pnm_getinteger(s, &c); // read max value\n if (maxv > 65535)\n return stbi__err(\"max value > 65535\", \"PPM image supports only 8-bit and 16-bit images\");\n else if (maxv > 255)\n return 16;\n else\n return 8;\n}\n\nstatic int stbi__pnm_is16(stbi__context *s)\n{\n if (stbi__pnm_info(s, NULL, NULL, NULL) == 16)\n\t return 1;\n return 0;\n}\n#endif\n\nstatic int stbi__info_main(stbi__context *s, int *x, int *y, int *comp)\n{\n #ifndef STBI_NO_JPEG\n if (stbi__jpeg_info(s, x, y, comp)) return 1;\n #endif\n\n #ifndef STBI_NO_PNG\n if (stbi__png_info(s, x, y, comp)) return 1;\n #endif\n\n #ifndef STBI_NO_GIF\n if (stbi__gif_info(s, x, y, comp)) return 1;\n #endif\n\n #ifndef STBI_NO_BMP\n if (stbi__bmp_info(s, x, y, comp)) return 1;\n #endif\n\n #ifndef STBI_NO_PSD\n if (stbi__psd_info(s, x, y, comp)) return 1;\n #endif\n\n #ifndef STBI_NO_PIC\n if (stbi__pic_info(s, x, y, comp)) return 1;\n #endif\n\n #ifndef STBI_NO_PNM\n if (stbi__pnm_info(s, x, y, comp)) return 1;\n #endif\n\n #ifndef STBI_NO_HDR\n if (stbi__hdr_info(s, x, y, comp)) return 1;\n #endif\n\n // test tga last because it's a crappy test!\n #ifndef STBI_NO_TGA\n if (stbi__tga_info(s, x, y, comp))\n return 1;\n #endif\n return stbi__err(\"unknown image type\", \"Image not of any known type, or corrupt\");\n}\n\nstatic int stbi__is_16_main(stbi__context *s)\n{\n #ifndef STBI_NO_PNG\n if (stbi__png_is16(s)) return 1;\n #endif\n\n #ifndef STBI_NO_PSD\n if (stbi__psd_is16(s)) return 1;\n #endif\n\n #ifndef STBI_NO_PNM\n if (stbi__pnm_is16(s)) return 1;\n #endif\n return 0;\n}\n\n#ifndef STBI_NO_STDIO\nSTBIDEF int stbi_info(char const *filename, int *x, int *y, int *comp)\n{\n FILE *f = stbi__fopen(filename, \"rb\");\n int result;\n if (!f) return stbi__err(\"can't fopen\", \"Unable to open file\");\n result = stbi_info_from_file(f, x, y, comp);\n fclose(f);\n return result;\n}\n\nSTBIDEF int stbi_info_from_file(FILE *f, int *x, int *y, int *comp)\n{\n int r;\n stbi__context s;\n long pos = ftell(f);\n stbi__start_file(&s, f);\n r = stbi__info_main(&s,x,y,comp);\n fseek(f,pos,SEEK_SET);\n return r;\n}\n\nSTBIDEF int stbi_is_16_bit(char const *filename)\n{\n FILE *f = stbi__fopen(filename, \"rb\");\n int result;\n if (!f) return stbi__err(\"can't fopen\", \"Unable to open file\");\n result = stbi_is_16_bit_from_file(f);\n fclose(f);\n return result;\n}\n\nSTBIDEF int stbi_is_16_bit_from_file(FILE *f)\n{\n int r;\n stbi__context s;\n long pos = ftell(f);\n stbi__start_file(&s, f);\n r = stbi__is_16_main(&s);\n fseek(f,pos,SEEK_SET);\n return r;\n}\n#endif // !STBI_NO_STDIO\n\nSTBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp)\n{\n stbi__context s;\n stbi__start_mem(&s,buffer,len);\n return stbi__info_main(&s,x,y,comp);\n}\n\nSTBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *c, void *user, int *x, int *y, int *comp)\n{\n stbi__context s;\n stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user);\n return stbi__info_main(&s,x,y,comp);\n}\n\nSTBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len)\n{\n stbi__context s;\n stbi__start_mem(&s,buffer,len);\n return stbi__is_16_main(&s);\n}\n\nSTBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *c, void *user)\n{\n stbi__context s;\n stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user);\n return stbi__is_16_main(&s);\n}\n\n#endif // STB_IMAGE_IMPLEMENTATION\n\n/*\n revision history:\n 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs\n 2.19 (2018-02-11) fix warning\n 2.18 (2018-01-30) fix warnings\n 2.17 (2018-01-29) change sbti__shiftsigned to avoid clang -O2 bug\n 1-bit BMP\n *_is_16_bit api\n avoid warnings\n 2.16 (2017-07-23) all functions have 16-bit variants;\n STBI_NO_STDIO works again;\n compilation fixes;\n fix rounding in unpremultiply;\n optimize vertical flip;\n disable raw_len validation;\n documentation fixes\n 2.15 (2017-03-18) fix png-1,2,4 bug; now all Imagenet JPGs decode;\n warning fixes; disable run-time SSE detection on gcc;\n uniform handling of optional \"return\" values;\n thread-safe initialization of zlib tables\n 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs\n 2.13 (2016-11-29) add 16-bit API, only supported for PNG right now\n 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes\n 2.11 (2016-04-02) allocate large structures on the stack\n remove white matting for transparent PSD\n fix reported channel count for PNG & BMP\n re-enable SSE2 in non-gcc 64-bit\n support RGB-formatted JPEG\n read 16-bit PNGs (only as 8-bit)\n 2.10 (2016-01-22) avoid warning introduced in 2.09 by STBI_REALLOC_SIZED\n 2.09 (2016-01-16) allow comments in PNM files\n 16-bit-per-pixel TGA (not bit-per-component)\n info() for TGA could break due to .hdr handling\n info() for BMP to shares code instead of sloppy parse\n can use STBI_REALLOC_SIZED if allocator doesn't support realloc\n code cleanup\n 2.08 (2015-09-13) fix to 2.07 cleanup, reading RGB PSD as RGBA\n 2.07 (2015-09-13) fix compiler warnings\n partial animated GIF support\n limited 16-bpc PSD support\n #ifdef unused functions\n bug with < 92 byte PIC,PNM,HDR,TGA\n 2.06 (2015-04-19) fix bug where PSD returns wrong '*comp' value\n 2.05 (2015-04-19) fix bug in progressive JPEG handling, fix warning\n 2.04 (2015-04-15) try to re-enable SIMD on MinGW 64-bit\n 2.03 (2015-04-12) extra corruption checking (mmozeiko)\n stbi_set_flip_vertically_on_load (nguillemot)\n fix NEON support; fix mingw support\n 2.02 (2015-01-19) fix incorrect assert, fix warning\n 2.01 (2015-01-17) fix various warnings; suppress SIMD on gcc 32-bit without -msse2\n 2.00b (2014-12-25) fix STBI_MALLOC in progressive JPEG\n 2.00 (2014-12-25) optimize JPG, including x86 SSE2 & NEON SIMD (ryg)\n progressive JPEG (stb)\n PGM/PPM support (Ken Miller)\n STBI_MALLOC,STBI_REALLOC,STBI_FREE\n GIF bugfix -- seemingly never worked\n STBI_NO_*, STBI_ONLY_*\n 1.48 (2014-12-14) fix incorrectly-named assert()\n 1.47 (2014-12-14) 1/2/4-bit PNG support, both direct and paletted (Omar Cornut & stb)\n optimize PNG (ryg)\n fix bug in interlaced PNG with user-specified channel count (stb)\n 1.46 (2014-08-26)\n fix broken tRNS chunk (colorkey-style transparency) in non-paletted PNG\n 1.45 (2014-08-16)\n fix MSVC-ARM internal compiler error by wrapping malloc\n 1.44 (2014-08-07)\n various warning fixes from Ronny Chevalier\n 1.43 (2014-07-15)\n fix MSVC-only compiler problem in code changed in 1.42\n 1.42 (2014-07-09)\n don't define _CRT_SECURE_NO_WARNINGS (affects user code)\n fixes to stbi__cleanup_jpeg path\n added STBI_ASSERT to avoid requiring assert.h\n 1.41 (2014-06-25)\n fix search&replace from 1.36 that messed up comments/error messages\n 1.40 (2014-06-22)\n fix gcc struct-initialization warning\n 1.39 (2014-06-15)\n fix to TGA optimization when req_comp != number of components in TGA;\n fix to GIF loading because BMP wasn't rewinding (whoops, no GIFs in my test suite)\n add support for BMP version 5 (more ignored fields)\n 1.38 (2014-06-06)\n suppress MSVC warnings on integer casts truncating values\n fix accidental rename of 'skip' field of I/O\n 1.37 (2014-06-04)\n remove duplicate typedef\n 1.36 (2014-06-03)\n convert to header file single-file library\n if de-iphone isn't set, load iphone images color-swapped instead of returning NULL\n 1.35 (2014-05-27)\n various warnings\n fix broken STBI_SIMD path\n fix bug where stbi_load_from_file no longer left file pointer in correct place\n fix broken non-easy path for 32-bit BMP (possibly never used)\n TGA optimization by Arseny Kapoulkine\n 1.34 (unknown)\n use STBI_NOTUSED in stbi__resample_row_generic(), fix one more leak in tga failure case\n 1.33 (2011-07-14)\n make stbi_is_hdr work in STBI_NO_HDR (as specified), minor compiler-friendly improvements\n 1.32 (2011-07-13)\n support for \"info\" function for all supported filetypes (SpartanJ)\n 1.31 (2011-06-20)\n a few more leak fixes, bug in PNG handling (SpartanJ)\n 1.30 (2011-06-11)\n added ability to load files via callbacks to accomidate custom input streams (Ben Wenger)\n removed deprecated format-specific test/load functions\n removed support for installable file formats (stbi_loader) -- would have been broken for IO callbacks anyway\n error cases in bmp and tga give messages and don't leak (Raymond Barbiero, grisha)\n fix inefficiency in decoding 32-bit BMP (David Woo)\n 1.29 (2010-08-16)\n various warning fixes from Aurelien Pocheville\n 1.28 (2010-08-01)\n fix bug in GIF palette transparency (SpartanJ)\n 1.27 (2010-08-01)\n cast-to-stbi_uc to fix warnings\n 1.26 (2010-07-24)\n fix bug in file buffering for PNG reported by SpartanJ\n 1.25 (2010-07-17)\n refix trans_data warning (Won Chun)\n 1.24 (2010-07-12)\n perf improvements reading from files on platforms with lock-heavy fgetc()\n minor perf improvements for jpeg\n deprecated type-specific functions so we'll get feedback if they're needed\n attempt to fix trans_data warning (Won Chun)\n 1.23 fixed bug in iPhone support\n 1.22 (2010-07-10)\n removed image *writing* support\n stbi_info support from Jetro Lauha\n GIF support from Jean-Marc Lienher\n iPhone PNG-extensions from James Brown\n warning-fixes from Nicolas Schulz and Janez Zemva (i.stbi__err. Janez (U+017D)emva)\n 1.21 fix use of 'stbi_uc' in header (reported by jon blow)\n 1.20 added support for Softimage PIC, by Tom Seddon\n 1.19 bug in interlaced PNG corruption check (found by ryg)\n 1.18 (2008-08-02)\n fix a threading bug (local mutable static)\n 1.17 support interlaced PNG\n 1.16 major bugfix - stbi__convert_format converted one too many pixels\n 1.15 initialize some fields for thread safety\n 1.14 fix threadsafe conversion bug\n header-file-only version (#define STBI_HEADER_FILE_ONLY before including)\n 1.13 threadsafe\n 1.12 const qualifiers in the API\n 1.11 Support installable IDCT, colorspace conversion routines\n 1.10 Fixes for 64-bit (don't use \"unsigned long\")\n optimized upsampling by Fabian \"ryg\" Giesen\n 1.09 Fix format-conversion for PSD code (bad global variables!)\n 1.08 Thatcher Ulrich's PSD code integrated by Nicolas Schulz\n 1.07 attempt to fix C++ warning/errors again\n 1.06 attempt to fix C++ warning/errors again\n 1.05 fix TGA loading to return correct *comp and use good luminance calc\n 1.04 default float alpha is 1, not 255; use 'void *' for stbi_image_free\n 1.03 bugfixes to STBI_NO_STDIO, STBI_NO_HDR\n 1.02 support for (subset of) HDR files, float interface for preferred access to them\n 1.01 fix bug: possible bug in handling right-side up bmps... not sure\n fix bug: the stbi__bmp_load() and stbi__tga_load() functions didn't work at all\n 1.00 interface to zlib that skips zlib header\n 0.99 correct handling of alpha in palette\n 0.98 TGA loader by lonesock; dynamically add loaders (untested)\n 0.97 jpeg errors on too large a file; also catch another malloc failure\n 0.96 fix detection of invalid v value - particleman@mollyrocket forum\n 0.95 during header scan, seek to markers in case of padding\n 0.94 STBI_NO_STDIO to disable stdio usage; rename all #defines the same\n 0.93 handle jpegtran output; verbose errors\n 0.92 read 4,8,16,24,32-bit BMP files of several formats\n 0.91 output 24-bit Windows 3.0 BMP files\n 0.90 fix a few more warnings; bump version number to approach 1.0\n 0.61 bugfixes due to Marc LeBlanc, Christopher Lloyd\n 0.60 fix compiling as c++\n 0.59 fix warnings: merge Dave Moore's -Wall fixes\n 0.58 fix bug: zlib uncompressed mode len/nlen was wrong endian\n 0.57 fix bug: jpg last huffman symbol before marker was >9 bits but less than 16 available\n 0.56 fix bug: zlib uncompressed mode len vs. nlen\n 0.55 fix bug: restart_interval not initialized to 0\n 0.54 allow NULL for 'int *comp'\n 0.53 fix bug in png 3->4; speedup png decoding\n 0.52 png handles req_comp=3,4 directly; minor cleanup; jpeg comments\n 0.51 obey req_comp requests, 1-component jpegs return as 1-component,\n on 'test' only check type, not whether we support this variant\n 0.50 (2006-11-19)\n first released version\n*/\n\n\n/*\n------------------------------------------------------------------------------\nThis software is available under 2 licenses -- choose whichever you prefer.\n------------------------------------------------------------------------------\nALTERNATIVE A - MIT License\nCopyright (c) 2017 Sean Barrett\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\nof the Software, and to permit persons to whom the Software is furnished to do\nso, subject to the following conditions:\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n------------------------------------------------------------------------------\nALTERNATIVE B - Public Domain (www.unlicense.org)\nThis is free and unencumbered software released into the public domain.\nAnyone is free to copy, modify, publish, use, compile, sell, or distribute this\nsoftware, either in source code form or as a compiled binary, for any purpose,\ncommercial or non-commercial, and by any means.\nIn jurisdictions that recognize copyright laws, the author or authors of this\nsoftware dedicate any and all copyright interest in the software to the public\ndomain. We make this dedication for the benefit of the public at large and to\nthe detriment of our heirs and successors. We intend this dedication to be an\novert act of relinquishment in perpetuity of all present and future rights to\nthis software under copyright law.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\nACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n------------------------------------------------------------------------------\n*/\n" }, { "path": "examples/stb_image_write.h", "content": "/* stb_image_write - v1.16 - public domain - http://nothings.org/stb\n writes out PNG/BMP/TGA/JPEG/HDR images to C stdio - Sean Barrett 2010-2015\n no warranty implied; use at your own risk\n\n Before #including,\n\n #define STB_IMAGE_WRITE_IMPLEMENTATION\n\n in the file that you want to have the implementation.\n\n Will probably not work correctly with strict-aliasing optimizations.\n\nABOUT:\n\n This header file is a library for writing images to C stdio or a callback.\n\n The PNG output is not optimal; it is 20-50% larger than the file\n written by a decent optimizing implementation; though providing a custom\n zlib compress function (see STBIW_ZLIB_COMPRESS) can mitigate that.\n This library is designed for source code compactness and simplicity,\n not optimal image file size or run-time performance.\n\nBUILDING:\n\n You can #define STBIW_ASSERT(x) before the #include to avoid using assert.h.\n You can #define STBIW_MALLOC(), STBIW_REALLOC(), and STBIW_FREE() to replace\n malloc,realloc,free.\n You can #define STBIW_MEMMOVE() to replace memmove()\n You can #define STBIW_ZLIB_COMPRESS to use a custom zlib-style compress function\n for PNG compression (instead of the builtin one), it must have the following signature:\n unsigned char * my_compress(unsigned char *data, int data_len, int *out_len, int quality);\n The returned data will be freed with STBIW_FREE() (free() by default),\n so it must be heap allocated with STBIW_MALLOC() (malloc() by default),\n\nUNICODE:\n\n If compiling for Windows and you wish to use Unicode filenames, compile\n with\n #define STBIW_WINDOWS_UTF8\n and pass utf8-encoded filenames. Call stbiw_convert_wchar_to_utf8 to convert\n Windows wchar_t filenames to utf8.\n\nUSAGE:\n\n There are five functions, one for each image file format:\n\n int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes);\n int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data);\n int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data);\n int stbi_write_jpg(char const *filename, int w, int h, int comp, const void *data, int quality);\n int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data);\n\n void stbi_flip_vertically_on_write(int flag); // flag is non-zero to flip data vertically\n\n There are also five equivalent functions that use an arbitrary write function. You are\n expected to open/close your file-equivalent before and after calling these:\n\n int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes);\n int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data);\n int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data);\n int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data);\n int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality);\n\n where the callback is:\n void stbi_write_func(void *context, void *data, int size);\n\n You can configure it with these global variables:\n int stbi_write_tga_with_rle; // defaults to true; set to 0 to disable RLE\n int stbi_write_png_compression_level; // defaults to 8; set to higher for more compression\n int stbi_write_force_png_filter; // defaults to -1; set to 0..5 to force a filter mode\n\n\n You can define STBI_WRITE_NO_STDIO to disable the file variant of these\n functions, so the library will not use stdio.h at all. However, this will\n also disable HDR writing, because it requires stdio for formatted output.\n\n Each function returns 0 on failure and non-0 on success.\n\n The functions create an image file defined by the parameters. The image\n is a rectangle of pixels stored from left-to-right, top-to-bottom.\n Each pixel contains 'comp' channels of data stored interleaved with 8-bits\n per channel, in the following order: 1=Y, 2=YA, 3=RGB, 4=RGBA. (Y is\n monochrome color.) The rectangle is 'w' pixels wide and 'h' pixels tall.\n The *data pointer points to the first byte of the top-left-most pixel.\n For PNG, \"stride_in_bytes\" is the distance in bytes from the first byte of\n a row of pixels to the first byte of the next row of pixels.\n\n PNG creates output files with the same number of components as the input.\n The BMP format expands Y to RGB in the file format and does not\n output alpha.\n\n PNG supports writing rectangles of data even when the bytes storing rows of\n data are not consecutive in memory (e.g. sub-rectangles of a larger image),\n by supplying the stride between the beginning of adjacent rows. The other\n formats do not. (Thus you cannot write a native-format BMP through the BMP\n writer, both because it is in BGR order and because it may have padding\n at the end of the line.)\n\n PNG allows you to set the deflate compression level by setting the global\n variable 'stbi_write_png_compression_level' (it defaults to 8).\n\n HDR expects linear float data. Since the format is always 32-bit rgb(e)\n data, alpha (if provided) is discarded, and for monochrome data it is\n replicated across all three channels.\n\n TGA supports RLE or non-RLE compressed data. To use non-RLE-compressed\n data, set the global variable 'stbi_write_tga_with_rle' to 0.\n\n JPEG does ignore alpha channels in input data; quality is between 1 and 100.\n Higher quality looks better but results in a bigger image.\n JPEG baseline (no JPEG progressive).\n\nCREDITS:\n\n\n Sean Barrett - PNG/BMP/TGA\n Baldur Karlsson - HDR\n Jean-Sebastien Guay - TGA monochrome\n Tim Kelsey - misc enhancements\n Alan Hickman - TGA RLE\n Emmanuel Julien - initial file IO callback implementation\n Jon Olick - original jo_jpeg.cpp code\n Daniel Gibson - integrate JPEG, allow external zlib\n Aarni Koskela - allow choosing PNG filter\n\n bugfixes:\n github:Chribba\n Guillaume Chereau\n github:jry2\n github:romigrou\n Sergio Gonzalez\n Jonas Karlsson\n Filip Wasil\n Thatcher Ulrich\n github:poppolopoppo\n Patrick Boettcher\n github:xeekworx\n Cap Petschulat\n Simon Rodriguez\n Ivan Tikhonov\n github:ignotion\n Adam Schackart\n Andrew Kensler\n\nLICENSE\n\n See end of file for license information.\n\n*/\n\n#ifndef INCLUDE_STB_IMAGE_WRITE_H\n#define INCLUDE_STB_IMAGE_WRITE_H\n\n#include \n\n// if STB_IMAGE_WRITE_STATIC causes problems, try defining STBIWDEF to 'inline' or 'static inline'\n#ifndef STBIWDEF\n#ifdef STB_IMAGE_WRITE_STATIC\n#define STBIWDEF static\n#else\n#ifdef __cplusplus\n#define STBIWDEF extern \"C\"\n#else\n#define STBIWDEF extern\n#endif\n#endif\n#endif\n\n#ifndef STB_IMAGE_WRITE_STATIC // C++ forbids static forward declarations\nSTBIWDEF int stbi_write_tga_with_rle;\nSTBIWDEF int stbi_write_png_compression_level;\nSTBIWDEF int stbi_write_force_png_filter;\n#endif\n\n#ifndef STBI_WRITE_NO_STDIO\nSTBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes);\nSTBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data);\nSTBIWDEF int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data);\nSTBIWDEF int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data);\nSTBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality);\n\n#ifdef STBIW_WINDOWS_UTF8\nSTBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input);\n#endif\n#endif\n\ntypedef void stbi_write_func(void *context, void *data, int size);\n\nSTBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes);\nSTBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data);\nSTBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data);\nSTBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data);\nSTBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality);\n\nSTBIWDEF void stbi_flip_vertically_on_write(int flip_boolean);\n\n#endif//INCLUDE_STB_IMAGE_WRITE_H\n\n#ifdef STB_IMAGE_WRITE_IMPLEMENTATION\n\n#ifdef _WIN32\n #ifndef _CRT_SECURE_NO_WARNINGS\n #define _CRT_SECURE_NO_WARNINGS\n #endif\n #ifndef _CRT_NONSTDC_NO_DEPRECATE\n #define _CRT_NONSTDC_NO_DEPRECATE\n #endif\n#endif\n\n#ifndef STBI_WRITE_NO_STDIO\n#include \n#endif // STBI_WRITE_NO_STDIO\n\n#include \n#include \n#include \n#include \n\n#if defined(STBIW_MALLOC) && defined(STBIW_FREE) && (defined(STBIW_REALLOC) || defined(STBIW_REALLOC_SIZED))\n// ok\n#elif !defined(STBIW_MALLOC) && !defined(STBIW_FREE) && !defined(STBIW_REALLOC) && !defined(STBIW_REALLOC_SIZED)\n// ok\n#else\n#error \"Must define all or none of STBIW_MALLOC, STBIW_FREE, and STBIW_REALLOC (or STBIW_REALLOC_SIZED).\"\n#endif\n\n#ifndef STBIW_MALLOC\n#define STBIW_MALLOC(sz) malloc(sz)\n#define STBIW_REALLOC(p,newsz) realloc(p,newsz)\n#define STBIW_FREE(p) free(p)\n#endif\n\n#ifndef STBIW_REALLOC_SIZED\n#define STBIW_REALLOC_SIZED(p,oldsz,newsz) STBIW_REALLOC(p,newsz)\n#endif\n\n\n#ifndef STBIW_MEMMOVE\n#define STBIW_MEMMOVE(a,b,sz) memmove(a,b,sz)\n#endif\n\n\n#ifndef STBIW_ASSERT\n#include \n#define STBIW_ASSERT(x) assert(x)\n#endif\n\n#define STBIW_UCHAR(x) (unsigned char) ((x) & 0xff)\n\n#ifdef STB_IMAGE_WRITE_STATIC\nstatic int stbi_write_png_compression_level = 8;\nstatic int stbi_write_tga_with_rle = 1;\nstatic int stbi_write_force_png_filter = -1;\n#else\nint stbi_write_png_compression_level = 8;\nint stbi_write_tga_with_rle = 1;\nint stbi_write_force_png_filter = -1;\n#endif\n\nstatic int stbi__flip_vertically_on_write = 0;\n\nSTBIWDEF void stbi_flip_vertically_on_write(int flag)\n{\n stbi__flip_vertically_on_write = flag;\n}\n\ntypedef struct\n{\n stbi_write_func *func;\n void *context;\n unsigned char buffer[64];\n int buf_used;\n} stbi__write_context;\n\n// initialize a callback-based context\nstatic void stbi__start_write_callbacks(stbi__write_context *s, stbi_write_func *c, void *context)\n{\n s->func = c;\n s->context = context;\n}\n\n#ifndef STBI_WRITE_NO_STDIO\n\nstatic void stbi__stdio_write(void *context, void *data, int size)\n{\n fwrite(data,1,size,(FILE*) context);\n}\n\n#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8)\n#ifdef __cplusplus\n#define STBIW_EXTERN extern \"C\"\n#else\n#define STBIW_EXTERN extern\n#endif\nSTBIW_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide);\nSTBIW_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default);\n\nSTBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input)\n{\n return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL);\n}\n#endif\n\nstatic FILE *stbiw__fopen(char const *filename, char const *mode)\n{\n FILE *f;\n#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8)\n wchar_t wMode[64];\n wchar_t wFilename[1024];\n if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename)))\n return 0;\n\n if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode)))\n return 0;\n\n#if defined(_MSC_VER) && _MSC_VER >= 1400\n if (0 != _wfopen_s(&f, wFilename, wMode))\n f = 0;\n#else\n f = _wfopen(wFilename, wMode);\n#endif\n\n#elif defined(_MSC_VER) && _MSC_VER >= 1400\n if (0 != fopen_s(&f, filename, mode))\n f=0;\n#else\n f = fopen(filename, mode);\n#endif\n return f;\n}\n\nstatic int stbi__start_write_file(stbi__write_context *s, const char *filename)\n{\n FILE *f = stbiw__fopen(filename, \"wb\");\n stbi__start_write_callbacks(s, stbi__stdio_write, (void *) f);\n return f != NULL;\n}\n\nstatic void stbi__end_write_file(stbi__write_context *s)\n{\n fclose((FILE *)s->context);\n}\n\n#endif // !STBI_WRITE_NO_STDIO\n\ntypedef unsigned int stbiw_uint32;\ntypedef int stb_image_write_test[sizeof(stbiw_uint32)==4 ? 1 : -1];\n\nstatic void stbiw__writefv(stbi__write_context *s, const char *fmt, va_list v)\n{\n while (*fmt) {\n switch (*fmt++) {\n case ' ': break;\n case '1': { unsigned char x = STBIW_UCHAR(va_arg(v, int));\n s->func(s->context,&x,1);\n break; }\n case '2': { int x = va_arg(v,int);\n unsigned char b[2];\n b[0] = STBIW_UCHAR(x);\n b[1] = STBIW_UCHAR(x>>8);\n s->func(s->context,b,2);\n break; }\n case '4': { stbiw_uint32 x = va_arg(v,int);\n unsigned char b[4];\n b[0]=STBIW_UCHAR(x);\n b[1]=STBIW_UCHAR(x>>8);\n b[2]=STBIW_UCHAR(x>>16);\n b[3]=STBIW_UCHAR(x>>24);\n s->func(s->context,b,4);\n break; }\n default:\n STBIW_ASSERT(0);\n return;\n }\n }\n}\n\nstatic void stbiw__writef(stbi__write_context *s, const char *fmt, ...)\n{\n va_list v;\n va_start(v, fmt);\n stbiw__writefv(s, fmt, v);\n va_end(v);\n}\n\nstatic void stbiw__write_flush(stbi__write_context *s)\n{\n if (s->buf_used) {\n s->func(s->context, &s->buffer, s->buf_used);\n s->buf_used = 0;\n }\n}\n\nstatic void stbiw__putc(stbi__write_context *s, unsigned char c)\n{\n s->func(s->context, &c, 1);\n}\n\nstatic void stbiw__write1(stbi__write_context *s, unsigned char a)\n{\n if ((size_t)s->buf_used + 1 > sizeof(s->buffer))\n stbiw__write_flush(s);\n s->buffer[s->buf_used++] = a;\n}\n\nstatic void stbiw__write3(stbi__write_context *s, unsigned char a, unsigned char b, unsigned char c)\n{\n int n;\n if ((size_t)s->buf_used + 3 > sizeof(s->buffer))\n stbiw__write_flush(s);\n n = s->buf_used;\n s->buf_used = n+3;\n s->buffer[n+0] = a;\n s->buffer[n+1] = b;\n s->buffer[n+2] = c;\n}\n\nstatic void stbiw__write_pixel(stbi__write_context *s, int rgb_dir, int comp, int write_alpha, int expand_mono, unsigned char *d)\n{\n unsigned char bg[3] = { 255, 0, 255}, px[3];\n int k;\n\n if (write_alpha < 0)\n stbiw__write1(s, d[comp - 1]);\n\n switch (comp) {\n case 2: // 2 pixels = mono + alpha, alpha is written separately, so same as 1-channel case\n case 1:\n if (expand_mono)\n stbiw__write3(s, d[0], d[0], d[0]); // monochrome bmp\n else\n stbiw__write1(s, d[0]); // monochrome TGA\n break;\n case 4:\n if (!write_alpha) {\n // composite against pink background\n for (k = 0; k < 3; ++k)\n px[k] = bg[k] + ((d[k] - bg[k]) * d[3]) / 255;\n stbiw__write3(s, px[1 - rgb_dir], px[1], px[1 + rgb_dir]);\n break;\n }\n /* FALLTHROUGH */\n case 3:\n stbiw__write3(s, d[1 - rgb_dir], d[1], d[1 + rgb_dir]);\n break;\n }\n if (write_alpha > 0)\n stbiw__write1(s, d[comp - 1]);\n}\n\nstatic void stbiw__write_pixels(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, void *data, int write_alpha, int scanline_pad, int expand_mono)\n{\n stbiw_uint32 zero = 0;\n int i,j, j_end;\n\n if (y <= 0)\n return;\n\n if (stbi__flip_vertically_on_write)\n vdir *= -1;\n\n if (vdir < 0) {\n j_end = -1; j = y-1;\n } else {\n j_end = y; j = 0;\n }\n\n for (; j != j_end; j += vdir) {\n for (i=0; i < x; ++i) {\n unsigned char *d = (unsigned char *) data + (j*x+i)*comp;\n stbiw__write_pixel(s, rgb_dir, comp, write_alpha, expand_mono, d);\n }\n stbiw__write_flush(s);\n s->func(s->context, &zero, scanline_pad);\n }\n}\n\nstatic int stbiw__outfile(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, int expand_mono, void *data, int alpha, int pad, const char *fmt, ...)\n{\n if (y < 0 || x < 0) {\n return 0;\n } else {\n va_list v;\n va_start(v, fmt);\n stbiw__writefv(s, fmt, v);\n va_end(v);\n stbiw__write_pixels(s,rgb_dir,vdir,x,y,comp,data,alpha,pad, expand_mono);\n return 1;\n }\n}\n\nstatic int stbi_write_bmp_core(stbi__write_context *s, int x, int y, int comp, const void *data)\n{\n if (comp != 4) {\n // write RGB bitmap\n int pad = (-x*3) & 3;\n return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *) data,0,pad,\n \"11 4 22 4\" \"4 44 22 444444\",\n 'B', 'M', 14+40+(x*3+pad)*y, 0,0, 14+40, // file header\n 40, x,y, 1,24, 0,0,0,0,0,0); // bitmap header\n } else {\n // RGBA bitmaps need a v4 header\n // use BI_BITFIELDS mode with 32bpp and alpha mask\n // (straight BI_RGB with alpha mask doesn't work in most readers)\n return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *)data,1,0,\n \"11 4 22 4\" \"4 44 22 444444 4444 4 444 444 444 444\",\n 'B', 'M', 14+108+x*y*4, 0, 0, 14+108, // file header\n 108, x,y, 1,32, 3,0,0,0,0,0, 0xff0000,0xff00,0xff,0xff000000u, 0, 0,0,0, 0,0,0, 0,0,0, 0,0,0); // bitmap V4 header\n }\n}\n\nSTBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data)\n{\n stbi__write_context s = { 0 };\n stbi__start_write_callbacks(&s, func, context);\n return stbi_write_bmp_core(&s, x, y, comp, data);\n}\n\n#ifndef STBI_WRITE_NO_STDIO\nSTBIWDEF int stbi_write_bmp(char const *filename, int x, int y, int comp, const void *data)\n{\n stbi__write_context s = { 0 };\n if (stbi__start_write_file(&s,filename)) {\n int r = stbi_write_bmp_core(&s, x, y, comp, data);\n stbi__end_write_file(&s);\n return r;\n } else\n return 0;\n}\n#endif //!STBI_WRITE_NO_STDIO\n\nstatic int stbi_write_tga_core(stbi__write_context *s, int x, int y, int comp, void *data)\n{\n int has_alpha = (comp == 2 || comp == 4);\n int colorbytes = has_alpha ? comp-1 : comp;\n int format = colorbytes < 2 ? 3 : 2; // 3 color channels (RGB/RGBA) = 2, 1 color channel (Y/YA) = 3\n\n if (y < 0 || x < 0)\n return 0;\n\n if (!stbi_write_tga_with_rle) {\n return stbiw__outfile(s, -1, -1, x, y, comp, 0, (void *) data, has_alpha, 0,\n \"111 221 2222 11\", 0, 0, format, 0, 0, 0, 0, 0, x, y, (colorbytes + has_alpha) * 8, has_alpha * 8);\n } else {\n int i,j,k;\n int jend, jdir;\n\n stbiw__writef(s, \"111 221 2222 11\", 0,0,format+8, 0,0,0, 0,0,x,y, (colorbytes + has_alpha) * 8, has_alpha * 8);\n\n if (stbi__flip_vertically_on_write) {\n j = 0;\n jend = y;\n jdir = 1;\n } else {\n j = y-1;\n jend = -1;\n jdir = -1;\n }\n for (; j != jend; j += jdir) {\n unsigned char *row = (unsigned char *) data + j * x * comp;\n int len;\n\n for (i = 0; i < x; i += len) {\n unsigned char *begin = row + i * comp;\n int diff = 1;\n len = 1;\n\n if (i < x - 1) {\n ++len;\n diff = memcmp(begin, row + (i + 1) * comp, comp);\n if (diff) {\n const unsigned char *prev = begin;\n for (k = i + 2; k < x && len < 128; ++k) {\n if (memcmp(prev, row + k * comp, comp)) {\n prev += comp;\n ++len;\n } else {\n --len;\n break;\n }\n }\n } else {\n for (k = i + 2; k < x && len < 128; ++k) {\n if (!memcmp(begin, row + k * comp, comp)) {\n ++len;\n } else {\n break;\n }\n }\n }\n }\n\n if (diff) {\n unsigned char header = STBIW_UCHAR(len - 1);\n stbiw__write1(s, header);\n for (k = 0; k < len; ++k) {\n stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin + k * comp);\n }\n } else {\n unsigned char header = STBIW_UCHAR(len - 129);\n stbiw__write1(s, header);\n stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin);\n }\n }\n }\n stbiw__write_flush(s);\n }\n return 1;\n}\n\nSTBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data)\n{\n stbi__write_context s = { 0 };\n stbi__start_write_callbacks(&s, func, context);\n return stbi_write_tga_core(&s, x, y, comp, (void *) data);\n}\n\n#ifndef STBI_WRITE_NO_STDIO\nSTBIWDEF int stbi_write_tga(char const *filename, int x, int y, int comp, const void *data)\n{\n stbi__write_context s = { 0 };\n if (stbi__start_write_file(&s,filename)) {\n int r = stbi_write_tga_core(&s, x, y, comp, (void *) data);\n stbi__end_write_file(&s);\n return r;\n } else\n return 0;\n}\n#endif\n\n// *************************************************************************************************\n// Radiance RGBE HDR writer\n// by Baldur Karlsson\n\n#define stbiw__max(a, b) ((a) > (b) ? (a) : (b))\n\n#ifndef STBI_WRITE_NO_STDIO\n\nstatic void stbiw__linear_to_rgbe(unsigned char *rgbe, float *linear)\n{\n int exponent;\n float maxcomp = stbiw__max(linear[0], stbiw__max(linear[1], linear[2]));\n\n if (maxcomp < 1e-32f) {\n rgbe[0] = rgbe[1] = rgbe[2] = rgbe[3] = 0;\n } else {\n float normalize = (float) frexp(maxcomp, &exponent) * 256.0f/maxcomp;\n\n rgbe[0] = (unsigned char)(linear[0] * normalize);\n rgbe[1] = (unsigned char)(linear[1] * normalize);\n rgbe[2] = (unsigned char)(linear[2] * normalize);\n rgbe[3] = (unsigned char)(exponent + 128);\n }\n}\n\nstatic void stbiw__write_run_data(stbi__write_context *s, int length, unsigned char databyte)\n{\n unsigned char lengthbyte = STBIW_UCHAR(length+128);\n STBIW_ASSERT(length+128 <= 255);\n s->func(s->context, &lengthbyte, 1);\n s->func(s->context, &databyte, 1);\n}\n\nstatic void stbiw__write_dump_data(stbi__write_context *s, int length, unsigned char *data)\n{\n unsigned char lengthbyte = STBIW_UCHAR(length);\n STBIW_ASSERT(length <= 128); // inconsistent with spec but consistent with official code\n s->func(s->context, &lengthbyte, 1);\n s->func(s->context, data, length);\n}\n\nstatic void stbiw__write_hdr_scanline(stbi__write_context *s, int width, int ncomp, unsigned char *scratch, float *scanline)\n{\n unsigned char scanlineheader[4] = { 2, 2, 0, 0 };\n unsigned char rgbe[4];\n float linear[3];\n int x;\n\n scanlineheader[2] = (width&0xff00)>>8;\n scanlineheader[3] = (width&0x00ff);\n\n /* skip RLE for images too small or large */\n if (width < 8 || width >= 32768) {\n for (x=0; x < width; x++) {\n switch (ncomp) {\n case 4: /* fallthrough */\n case 3: linear[2] = scanline[x*ncomp + 2];\n linear[1] = scanline[x*ncomp + 1];\n linear[0] = scanline[x*ncomp + 0];\n break;\n default:\n linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0];\n break;\n }\n stbiw__linear_to_rgbe(rgbe, linear);\n s->func(s->context, rgbe, 4);\n }\n } else {\n int c,r;\n /* encode into scratch buffer */\n for (x=0; x < width; x++) {\n switch(ncomp) {\n case 4: /* fallthrough */\n case 3: linear[2] = scanline[x*ncomp + 2];\n linear[1] = scanline[x*ncomp + 1];\n linear[0] = scanline[x*ncomp + 0];\n break;\n default:\n linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0];\n break;\n }\n stbiw__linear_to_rgbe(rgbe, linear);\n scratch[x + width*0] = rgbe[0];\n scratch[x + width*1] = rgbe[1];\n scratch[x + width*2] = rgbe[2];\n scratch[x + width*3] = rgbe[3];\n }\n\n s->func(s->context, scanlineheader, 4);\n\n /* RLE each component separately */\n for (c=0; c < 4; c++) {\n unsigned char *comp = &scratch[width*c];\n\n x = 0;\n while (x < width) {\n // find first run\n r = x;\n while (r+2 < width) {\n if (comp[r] == comp[r+1] && comp[r] == comp[r+2])\n break;\n ++r;\n }\n if (r+2 >= width)\n r = width;\n // dump up to first run\n while (x < r) {\n int len = r-x;\n if (len > 128) len = 128;\n stbiw__write_dump_data(s, len, &comp[x]);\n x += len;\n }\n // if there's a run, output it\n if (r+2 < width) { // same test as what we break out of in search loop, so only true if we break'd\n // find next byte after run\n while (r < width && comp[r] == comp[x])\n ++r;\n // output run up to r\n while (x < r) {\n int len = r-x;\n if (len > 127) len = 127;\n stbiw__write_run_data(s, len, comp[x]);\n x += len;\n }\n }\n }\n }\n }\n}\n\nstatic int stbi_write_hdr_core(stbi__write_context *s, int x, int y, int comp, float *data)\n{\n if (y <= 0 || x <= 0 || data == NULL)\n return 0;\n else {\n // Each component is stored separately. Allocate scratch space for full output scanline.\n unsigned char *scratch = (unsigned char *) STBIW_MALLOC(x*4);\n int i, len;\n char buffer[128];\n char header[] = \"#?RADIANCE\\n# Written by stb_image_write.h\\nFORMAT=32-bit_rle_rgbe\\n\";\n s->func(s->context, header, sizeof(header)-1);\n\n#ifdef __STDC_LIB_EXT1__\n len = sprintf_s(buffer, sizeof(buffer), \"EXPOSURE= 1.0000000000000\\n\\n-Y %d +X %d\\n\", y, x);\n#else\n len = sprintf(buffer, \"EXPOSURE= 1.0000000000000\\n\\n-Y %d +X %d\\n\", y, x);\n#endif\n s->func(s->context, buffer, len);\n\n for(i=0; i < y; i++)\n stbiw__write_hdr_scanline(s, x, comp, scratch, data + comp*x*(stbi__flip_vertically_on_write ? y-1-i : i));\n STBIW_FREE(scratch);\n return 1;\n }\n}\n\nSTBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const float *data)\n{\n stbi__write_context s = { 0 };\n stbi__start_write_callbacks(&s, func, context);\n return stbi_write_hdr_core(&s, x, y, comp, (float *) data);\n}\n\nSTBIWDEF int stbi_write_hdr(char const *filename, int x, int y, int comp, const float *data)\n{\n stbi__write_context s = { 0 };\n if (stbi__start_write_file(&s,filename)) {\n int r = stbi_write_hdr_core(&s, x, y, comp, (float *) data);\n stbi__end_write_file(&s);\n return r;\n } else\n return 0;\n}\n#endif // STBI_WRITE_NO_STDIO\n\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// PNG writer\n//\n\n#ifndef STBIW_ZLIB_COMPRESS\n// stretchy buffer; stbiw__sbpush() == vector<>::push_back() -- stbiw__sbcount() == vector<>::size()\n#define stbiw__sbraw(a) ((int *) (void *) (a) - 2)\n#define stbiw__sbm(a) stbiw__sbraw(a)[0]\n#define stbiw__sbn(a) stbiw__sbraw(a)[1]\n\n#define stbiw__sbneedgrow(a,n) ((a)==0 || stbiw__sbn(a)+n >= stbiw__sbm(a))\n#define stbiw__sbmaybegrow(a,n) (stbiw__sbneedgrow(a,(n)) ? stbiw__sbgrow(a,n) : 0)\n#define stbiw__sbgrow(a,n) stbiw__sbgrowf((void **) &(a), (n), sizeof(*(a)))\n\n#define stbiw__sbpush(a, v) (stbiw__sbmaybegrow(a,1), (a)[stbiw__sbn(a)++] = (v))\n#define stbiw__sbcount(a) ((a) ? stbiw__sbn(a) : 0)\n#define stbiw__sbfree(a) ((a) ? STBIW_FREE(stbiw__sbraw(a)),0 : 0)\n\nstatic void *stbiw__sbgrowf(void **arr, int increment, int itemsize)\n{\n int m = *arr ? 2*stbiw__sbm(*arr)+increment : increment+1;\n void *p = STBIW_REALLOC_SIZED(*arr ? stbiw__sbraw(*arr) : 0, *arr ? (stbiw__sbm(*arr)*itemsize + sizeof(int)*2) : 0, itemsize * m + sizeof(int)*2);\n STBIW_ASSERT(p);\n if (p) {\n if (!*arr) ((int *) p)[1] = 0;\n *arr = (void *) ((int *) p + 2);\n stbiw__sbm(*arr) = m;\n }\n return *arr;\n}\n\nstatic unsigned char *stbiw__zlib_flushf(unsigned char *data, unsigned int *bitbuffer, int *bitcount)\n{\n while (*bitcount >= 8) {\n stbiw__sbpush(data, STBIW_UCHAR(*bitbuffer));\n *bitbuffer >>= 8;\n *bitcount -= 8;\n }\n return data;\n}\n\nstatic int stbiw__zlib_bitrev(int code, int codebits)\n{\n int res=0;\n while (codebits--) {\n res = (res << 1) | (code & 1);\n code >>= 1;\n }\n return res;\n}\n\nstatic unsigned int stbiw__zlib_countm(unsigned char *a, unsigned char *b, int limit)\n{\n int i;\n for (i=0; i < limit && i < 258; ++i)\n if (a[i] != b[i]) break;\n return i;\n}\n\nstatic unsigned int stbiw__zhash(unsigned char *data)\n{\n stbiw_uint32 hash = data[0] + (data[1] << 8) + (data[2] << 16);\n hash ^= hash << 3;\n hash += hash >> 5;\n hash ^= hash << 4;\n hash += hash >> 17;\n hash ^= hash << 25;\n hash += hash >> 6;\n return hash;\n}\n\n#define stbiw__zlib_flush() (out = stbiw__zlib_flushf(out, &bitbuf, &bitcount))\n#define stbiw__zlib_add(code,codebits) \\\n (bitbuf |= (code) << bitcount, bitcount += (codebits), stbiw__zlib_flush())\n#define stbiw__zlib_huffa(b,c) stbiw__zlib_add(stbiw__zlib_bitrev(b,c),c)\n// default huffman tables\n#define stbiw__zlib_huff1(n) stbiw__zlib_huffa(0x30 + (n), 8)\n#define stbiw__zlib_huff2(n) stbiw__zlib_huffa(0x190 + (n)-144, 9)\n#define stbiw__zlib_huff3(n) stbiw__zlib_huffa(0 + (n)-256,7)\n#define stbiw__zlib_huff4(n) stbiw__zlib_huffa(0xc0 + (n)-280,8)\n#define stbiw__zlib_huff(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : (n) <= 255 ? stbiw__zlib_huff2(n) : (n) <= 279 ? stbiw__zlib_huff3(n) : stbiw__zlib_huff4(n))\n#define stbiw__zlib_huffb(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : stbiw__zlib_huff2(n))\n\n#define stbiw__ZHASH 16384\n\n#endif // STBIW_ZLIB_COMPRESS\n\nSTBIWDEF unsigned char * stbi_zlib_compress(unsigned char *data, int data_len, int *out_len, int quality)\n{\n#ifdef STBIW_ZLIB_COMPRESS\n // user provided a zlib compress implementation, use that\n return STBIW_ZLIB_COMPRESS(data, data_len, out_len, quality);\n#else // use builtin\n static unsigned short lengthc[] = { 3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258, 259 };\n static unsigned char lengtheb[]= { 0,0,0,0,0,0,0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 };\n static unsigned short distc[] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577, 32768 };\n static unsigned char disteb[] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13 };\n unsigned int bitbuf=0;\n int i,j, bitcount=0;\n unsigned char *out = NULL;\n unsigned char ***hash_table = (unsigned char***) STBIW_MALLOC(stbiw__ZHASH * sizeof(unsigned char**));\n if (hash_table == NULL)\n return NULL;\n if (quality < 5) quality = 5;\n\n stbiw__sbpush(out, 0x78); // DEFLATE 32K window\n stbiw__sbpush(out, 0x5e); // FLEVEL = 1\n stbiw__zlib_add(1,1); // BFINAL = 1\n stbiw__zlib_add(1,2); // BTYPE = 1 -- fixed huffman\n\n for (i=0; i < stbiw__ZHASH; ++i)\n hash_table[i] = NULL;\n\n i=0;\n while (i < data_len-3) {\n // hash next 3 bytes of data to be compressed\n int h = stbiw__zhash(data+i)&(stbiw__ZHASH-1), best=3;\n unsigned char *bestloc = 0;\n unsigned char **hlist = hash_table[h];\n int n = stbiw__sbcount(hlist);\n for (j=0; j < n; ++j) {\n if (hlist[j]-data > i-32768) { // if entry lies within window\n int d = stbiw__zlib_countm(hlist[j], data+i, data_len-i);\n if (d >= best) { best=d; bestloc=hlist[j]; }\n }\n }\n // when hash table entry is too long, delete half the entries\n if (hash_table[h] && stbiw__sbn(hash_table[h]) == 2*quality) {\n STBIW_MEMMOVE(hash_table[h], hash_table[h]+quality, sizeof(hash_table[h][0])*quality);\n stbiw__sbn(hash_table[h]) = quality;\n }\n stbiw__sbpush(hash_table[h],data+i);\n\n if (bestloc) {\n // \"lazy matching\" - check match at *next* byte, and if it's better, do cur byte as literal\n h = stbiw__zhash(data+i+1)&(stbiw__ZHASH-1);\n hlist = hash_table[h];\n n = stbiw__sbcount(hlist);\n for (j=0; j < n; ++j) {\n if (hlist[j]-data > i-32767) {\n int e = stbiw__zlib_countm(hlist[j], data+i+1, data_len-i-1);\n if (e > best) { // if next match is better, bail on current match\n bestloc = NULL;\n break;\n }\n }\n }\n }\n\n if (bestloc) {\n int d = (int) (data+i - bestloc); // distance back\n STBIW_ASSERT(d <= 32767 && best <= 258);\n for (j=0; best > lengthc[j+1]-1; ++j);\n stbiw__zlib_huff(j+257);\n if (lengtheb[j]) stbiw__zlib_add(best - lengthc[j], lengtheb[j]);\n for (j=0; d > distc[j+1]-1; ++j);\n stbiw__zlib_add(stbiw__zlib_bitrev(j,5),5);\n if (disteb[j]) stbiw__zlib_add(d - distc[j], disteb[j]);\n i += best;\n } else {\n stbiw__zlib_huffb(data[i]);\n ++i;\n }\n }\n // write out final bytes\n for (;i < data_len; ++i)\n stbiw__zlib_huffb(data[i]);\n stbiw__zlib_huff(256); // end of block\n // pad with 0 bits to byte boundary\n while (bitcount)\n stbiw__zlib_add(0,1);\n\n for (i=0; i < stbiw__ZHASH; ++i)\n (void) stbiw__sbfree(hash_table[i]);\n STBIW_FREE(hash_table);\n\n // store uncompressed instead if compression was worse\n if (stbiw__sbn(out) > data_len + 2 + ((data_len+32766)/32767)*5) {\n stbiw__sbn(out) = 2; // truncate to DEFLATE 32K window and FLEVEL = 1\n for (j = 0; j < data_len;) {\n int blocklen = data_len - j;\n if (blocklen > 32767) blocklen = 32767;\n stbiw__sbpush(out, data_len - j == blocklen); // BFINAL = ?, BTYPE = 0 -- no compression\n stbiw__sbpush(out, STBIW_UCHAR(blocklen)); // LEN\n stbiw__sbpush(out, STBIW_UCHAR(blocklen >> 8));\n stbiw__sbpush(out, STBIW_UCHAR(~blocklen)); // NLEN\n stbiw__sbpush(out, STBIW_UCHAR(~blocklen >> 8));\n memcpy(out+stbiw__sbn(out), data+j, blocklen);\n stbiw__sbn(out) += blocklen;\n j += blocklen;\n }\n }\n\n {\n // compute adler32 on input\n unsigned int s1=1, s2=0;\n int blocklen = (int) (data_len % 5552);\n j=0;\n while (j < data_len) {\n for (i=0; i < blocklen; ++i) { s1 += data[j+i]; s2 += s1; }\n s1 %= 65521; s2 %= 65521;\n j += blocklen;\n blocklen = 5552;\n }\n stbiw__sbpush(out, STBIW_UCHAR(s2 >> 8));\n stbiw__sbpush(out, STBIW_UCHAR(s2));\n stbiw__sbpush(out, STBIW_UCHAR(s1 >> 8));\n stbiw__sbpush(out, STBIW_UCHAR(s1));\n }\n *out_len = stbiw__sbn(out);\n // make returned pointer freeable\n STBIW_MEMMOVE(stbiw__sbraw(out), out, *out_len);\n return (unsigned char *) stbiw__sbraw(out);\n#endif // STBIW_ZLIB_COMPRESS\n}\n\nstatic unsigned int stbiw__crc32(unsigned char *buffer, int len)\n{\n#ifdef STBIW_CRC32\n return STBIW_CRC32(buffer, len);\n#else\n static unsigned int crc_table[256] =\n {\n 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,\n 0x0eDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,\n 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,\n 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,\n 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,\n 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,\n 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,\n 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,\n 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,\n 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,\n 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,\n 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,\n 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,\n 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,\n 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,\n 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,\n 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,\n 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,\n 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,\n 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,\n 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,\n 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,\n 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,\n 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,\n 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,\n 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,\n 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,\n 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,\n 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,\n 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,\n 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,\n 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D\n };\n\n unsigned int crc = ~0u;\n int i;\n for (i=0; i < len; ++i)\n crc = (crc >> 8) ^ crc_table[buffer[i] ^ (crc & 0xff)];\n return ~crc;\n#endif\n}\n\n#define stbiw__wpng4(o,a,b,c,d) ((o)[0]=STBIW_UCHAR(a),(o)[1]=STBIW_UCHAR(b),(o)[2]=STBIW_UCHAR(c),(o)[3]=STBIW_UCHAR(d),(o)+=4)\n#define stbiw__wp32(data,v) stbiw__wpng4(data, (v)>>24,(v)>>16,(v)>>8,(v));\n#define stbiw__wptag(data,s) stbiw__wpng4(data, s[0],s[1],s[2],s[3])\n\nstatic void stbiw__wpcrc(unsigned char **data, int len)\n{\n unsigned int crc = stbiw__crc32(*data - len - 4, len+4);\n stbiw__wp32(*data, crc);\n}\n\nstatic unsigned char stbiw__paeth(int a, int b, int c)\n{\n int p = a + b - c, pa = abs(p-a), pb = abs(p-b), pc = abs(p-c);\n if (pa <= pb && pa <= pc) return STBIW_UCHAR(a);\n if (pb <= pc) return STBIW_UCHAR(b);\n return STBIW_UCHAR(c);\n}\n\n// @OPTIMIZE: provide an option that always forces left-predict or paeth predict\nstatic void stbiw__encode_png_line(unsigned char *pixels, int stride_bytes, int width, int height, int y, int n, int filter_type, signed char *line_buffer)\n{\n static int mapping[] = { 0,1,2,3,4 };\n static int firstmap[] = { 0,1,0,5,6 };\n int *mymap = (y != 0) ? mapping : firstmap;\n int i;\n int type = mymap[filter_type];\n unsigned char *z = pixels + stride_bytes * (stbi__flip_vertically_on_write ? height-1-y : y);\n int signed_stride = stbi__flip_vertically_on_write ? -stride_bytes : stride_bytes;\n\n if (type==0) {\n memcpy(line_buffer, z, width*n);\n return;\n }\n\n // first loop isn't optimized since it's just one pixel\n for (i = 0; i < n; ++i) {\n switch (type) {\n case 1: line_buffer[i] = z[i]; break;\n case 2: line_buffer[i] = z[i] - z[i-signed_stride]; break;\n case 3: line_buffer[i] = z[i] - (z[i-signed_stride]>>1); break;\n case 4: line_buffer[i] = (signed char) (z[i] - stbiw__paeth(0,z[i-signed_stride],0)); break;\n case 5: line_buffer[i] = z[i]; break;\n case 6: line_buffer[i] = z[i]; break;\n }\n }\n switch (type) {\n case 1: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-n]; break;\n case 2: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-signed_stride]; break;\n case 3: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - ((z[i-n] + z[i-signed_stride])>>1); break;\n case 4: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], z[i-signed_stride], z[i-signed_stride-n]); break;\n case 5: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - (z[i-n]>>1); break;\n case 6: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], 0,0); break;\n }\n}\n\nSTBIWDEF unsigned char *stbi_write_png_to_mem(const unsigned char *pixels, int stride_bytes, int x, int y, int n, int *out_len)\n{\n int force_filter = stbi_write_force_png_filter;\n int ctype[5] = { -1, 0, 4, 2, 6 };\n unsigned char sig[8] = { 137,80,78,71,13,10,26,10 };\n unsigned char *out,*o, *filt, *zlib;\n signed char *line_buffer;\n int j,zlen;\n\n if (stride_bytes == 0)\n stride_bytes = x * n;\n\n if (force_filter >= 5) {\n force_filter = -1;\n }\n\n filt = (unsigned char *) STBIW_MALLOC((x*n+1) * y); if (!filt) return 0;\n line_buffer = (signed char *) STBIW_MALLOC(x * n); if (!line_buffer) { STBIW_FREE(filt); return 0; }\n for (j=0; j < y; ++j) {\n int filter_type;\n if (force_filter > -1) {\n filter_type = force_filter;\n stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, force_filter, line_buffer);\n } else { // Estimate the best filter by running through all of them:\n int best_filter = 0, best_filter_val = 0x7fffffff, est, i;\n for (filter_type = 0; filter_type < 5; filter_type++) {\n stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, filter_type, line_buffer);\n\n // Estimate the entropy of the line using this filter; the less, the better.\n est = 0;\n for (i = 0; i < x*n; ++i) {\n est += abs((signed char) line_buffer[i]);\n }\n if (est < best_filter_val) {\n best_filter_val = est;\n best_filter = filter_type;\n }\n }\n if (filter_type != best_filter) { // If the last iteration already got us the best filter, don't redo it\n stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, best_filter, line_buffer);\n filter_type = best_filter;\n }\n }\n // when we get here, filter_type contains the filter type, and line_buffer contains the data\n filt[j*(x*n+1)] = (unsigned char) filter_type;\n STBIW_MEMMOVE(filt+j*(x*n+1)+1, line_buffer, x*n);\n }\n STBIW_FREE(line_buffer);\n zlib = stbi_zlib_compress(filt, y*( x*n+1), &zlen, stbi_write_png_compression_level);\n STBIW_FREE(filt);\n if (!zlib) return 0;\n\n // each tag requires 12 bytes of overhead\n out = (unsigned char *) STBIW_MALLOC(8 + 12+13 + 12+zlen + 12);\n if (!out) return 0;\n *out_len = 8 + 12+13 + 12+zlen + 12;\n\n o=out;\n STBIW_MEMMOVE(o,sig,8); o+= 8;\n stbiw__wp32(o, 13); // header length\n stbiw__wptag(o, \"IHDR\");\n stbiw__wp32(o, x);\n stbiw__wp32(o, y);\n *o++ = 8;\n *o++ = STBIW_UCHAR(ctype[n]);\n *o++ = 0;\n *o++ = 0;\n *o++ = 0;\n stbiw__wpcrc(&o,13);\n\n stbiw__wp32(o, zlen);\n stbiw__wptag(o, \"IDAT\");\n STBIW_MEMMOVE(o, zlib, zlen);\n o += zlen;\n STBIW_FREE(zlib);\n stbiw__wpcrc(&o, zlen);\n\n stbiw__wp32(o,0);\n stbiw__wptag(o, \"IEND\");\n stbiw__wpcrc(&o,0);\n\n STBIW_ASSERT(o == out + *out_len);\n\n return out;\n}\n\n#ifndef STBI_WRITE_NO_STDIO\nSTBIWDEF int stbi_write_png(char const *filename, int x, int y, int comp, const void *data, int stride_bytes)\n{\n FILE *f;\n int len;\n unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len);\n if (png == NULL) return 0;\n\n f = stbiw__fopen(filename, \"wb\");\n if (!f) { STBIW_FREE(png); return 0; }\n fwrite(png, 1, len, f);\n fclose(f);\n STBIW_FREE(png);\n return 1;\n}\n#endif\n\nSTBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int stride_bytes)\n{\n int len;\n unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len);\n if (png == NULL) return 0;\n func(context, png, len);\n STBIW_FREE(png);\n return 1;\n}\n\n\n/* ***************************************************************************\n *\n * JPEG writer\n *\n * This is based on Jon Olick's jo_jpeg.cpp:\n * public domain Simple, Minimalistic JPEG writer - http://www.jonolick.com/code.html\n */\n\nstatic const unsigned char stbiw__jpg_ZigZag[] = { 0,1,5,6,14,15,27,28,2,4,7,13,16,26,29,42,3,8,12,17,25,30,41,43,9,11,18,\n 24,31,40,44,53,10,19,23,32,39,45,52,54,20,22,33,38,46,51,55,60,21,34,37,47,50,56,59,61,35,36,48,49,57,58,62,63 };\n\nstatic void stbiw__jpg_writeBits(stbi__write_context *s, int *bitBufP, int *bitCntP, const unsigned short *bs) {\n int bitBuf = *bitBufP, bitCnt = *bitCntP;\n bitCnt += bs[1];\n bitBuf |= bs[0] << (24 - bitCnt);\n while(bitCnt >= 8) {\n unsigned char c = (bitBuf >> 16) & 255;\n stbiw__putc(s, c);\n if(c == 255) {\n stbiw__putc(s, 0);\n }\n bitBuf <<= 8;\n bitCnt -= 8;\n }\n *bitBufP = bitBuf;\n *bitCntP = bitCnt;\n}\n\nstatic void stbiw__jpg_DCT(float *d0p, float *d1p, float *d2p, float *d3p, float *d4p, float *d5p, float *d6p, float *d7p) {\n float d0 = *d0p, d1 = *d1p, d2 = *d2p, d3 = *d3p, d4 = *d4p, d5 = *d5p, d6 = *d6p, d7 = *d7p;\n float z1, z2, z3, z4, z5, z11, z13;\n\n float tmp0 = d0 + d7;\n float tmp7 = d0 - d7;\n float tmp1 = d1 + d6;\n float tmp6 = d1 - d6;\n float tmp2 = d2 + d5;\n float tmp5 = d2 - d5;\n float tmp3 = d3 + d4;\n float tmp4 = d3 - d4;\n\n // Even part\n float tmp10 = tmp0 + tmp3; // phase 2\n float tmp13 = tmp0 - tmp3;\n float tmp11 = tmp1 + tmp2;\n float tmp12 = tmp1 - tmp2;\n\n d0 = tmp10 + tmp11; // phase 3\n d4 = tmp10 - tmp11;\n\n z1 = (tmp12 + tmp13) * 0.707106781f; // c4\n d2 = tmp13 + z1; // phase 5\n d6 = tmp13 - z1;\n\n // Odd part\n tmp10 = tmp4 + tmp5; // phase 2\n tmp11 = tmp5 + tmp6;\n tmp12 = tmp6 + tmp7;\n\n // The rotator is modified from fig 4-8 to avoid extra negations.\n z5 = (tmp10 - tmp12) * 0.382683433f; // c6\n z2 = tmp10 * 0.541196100f + z5; // c2-c6\n z4 = tmp12 * 1.306562965f + z5; // c2+c6\n z3 = tmp11 * 0.707106781f; // c4\n\n z11 = tmp7 + z3; // phase 5\n z13 = tmp7 - z3;\n\n *d5p = z13 + z2; // phase 6\n *d3p = z13 - z2;\n *d1p = z11 + z4;\n *d7p = z11 - z4;\n\n *d0p = d0; *d2p = d2; *d4p = d4; *d6p = d6;\n}\n\nstatic void stbiw__jpg_calcBits(int val, unsigned short bits[2]) {\n int tmp1 = val < 0 ? -val : val;\n val = val < 0 ? val-1 : val;\n bits[1] = 1;\n while(tmp1 >>= 1) {\n ++bits[1];\n }\n bits[0] = val & ((1<0)&&(DU[end0pos]==0); --end0pos) {\n }\n // end0pos = first element in reverse order !=0\n if(end0pos == 0) {\n stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB);\n return DU[0];\n }\n for(i = 1; i <= end0pos; ++i) {\n int startpos = i;\n int nrzeroes;\n unsigned short bits[2];\n for (; DU[i]==0 && i<=end0pos; ++i) {\n }\n nrzeroes = i-startpos;\n if ( nrzeroes >= 16 ) {\n int lng = nrzeroes>>4;\n int nrmarker;\n for (nrmarker=1; nrmarker <= lng; ++nrmarker)\n stbiw__jpg_writeBits(s, bitBuf, bitCnt, M16zeroes);\n nrzeroes &= 15;\n }\n stbiw__jpg_calcBits(DU[i], bits);\n stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTAC[(nrzeroes<<4)+bits[1]]);\n stbiw__jpg_writeBits(s, bitBuf, bitCnt, bits);\n }\n if(end0pos != 63) {\n stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB);\n }\n return DU[0];\n}\n\nstatic int stbi_write_jpg_core(stbi__write_context *s, int width, int height, int comp, const void* data, int quality) {\n // Constants that don't pollute global namespace\n static const unsigned char std_dc_luminance_nrcodes[] = {0,0,1,5,1,1,1,1,1,1,0,0,0,0,0,0,0};\n static const unsigned char std_dc_luminance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11};\n static const unsigned char std_ac_luminance_nrcodes[] = {0,0,2,1,3,3,2,4,3,5,5,4,4,0,0,1,0x7d};\n static const unsigned char std_ac_luminance_values[] = {\n 0x01,0x02,0x03,0x00,0x04,0x11,0x05,0x12,0x21,0x31,0x41,0x06,0x13,0x51,0x61,0x07,0x22,0x71,0x14,0x32,0x81,0x91,0xa1,0x08,\n 0x23,0x42,0xb1,0xc1,0x15,0x52,0xd1,0xf0,0x24,0x33,0x62,0x72,0x82,0x09,0x0a,0x16,0x17,0x18,0x19,0x1a,0x25,0x26,0x27,0x28,\n 0x29,0x2a,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58,0x59,\n 0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x83,0x84,0x85,0x86,0x87,0x88,0x89,\n 0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4,0xb5,0xb6,\n 0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xe1,0xe2,\n 0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa\n };\n static const unsigned char std_dc_chrominance_nrcodes[] = {0,0,3,1,1,1,1,1,1,1,1,1,0,0,0,0,0};\n static const unsigned char std_dc_chrominance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11};\n static const unsigned char std_ac_chrominance_nrcodes[] = {0,0,2,1,2,4,4,3,4,7,5,4,4,0,1,2,0x77};\n static const unsigned char std_ac_chrominance_values[] = {\n 0x00,0x01,0x02,0x03,0x11,0x04,0x05,0x21,0x31,0x06,0x12,0x41,0x51,0x07,0x61,0x71,0x13,0x22,0x32,0x81,0x08,0x14,0x42,0x91,\n 0xa1,0xb1,0xc1,0x09,0x23,0x33,0x52,0xf0,0x15,0x62,0x72,0xd1,0x0a,0x16,0x24,0x34,0xe1,0x25,0xf1,0x17,0x18,0x19,0x1a,0x26,\n 0x27,0x28,0x29,0x2a,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58,\n 0x59,0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x82,0x83,0x84,0x85,0x86,0x87,\n 0x88,0x89,0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4,\n 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,\n 0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa\n };\n // Huffman tables\n static const unsigned short YDC_HT[256][2] = { {0,2},{2,3},{3,3},{4,3},{5,3},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9}};\n static const unsigned short UVDC_HT[256][2] = { {0,2},{1,2},{2,2},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9},{1022,10},{2046,11}};\n static const unsigned short YAC_HT[256][2] = {\n {10,4},{0,2},{1,2},{4,3},{11,4},{26,5},{120,7},{248,8},{1014,10},{65410,16},{65411,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {12,4},{27,5},{121,7},{502,9},{2038,11},{65412,16},{65413,16},{65414,16},{65415,16},{65416,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {28,5},{249,8},{1015,10},{4084,12},{65417,16},{65418,16},{65419,16},{65420,16},{65421,16},{65422,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {58,6},{503,9},{4085,12},{65423,16},{65424,16},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {59,6},{1016,10},{65430,16},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {122,7},{2039,11},{65438,16},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {123,7},{4086,12},{65446,16},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {250,8},{4087,12},{65454,16},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {504,9},{32704,15},{65462,16},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {505,9},{65470,16},{65471,16},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {506,9},{65479,16},{65480,16},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {1017,10},{65488,16},{65489,16},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {1018,10},{65497,16},{65498,16},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {2040,11},{65506,16},{65507,16},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {65515,16},{65516,16},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{0,0},{0,0},{0,0},{0,0},{0,0},\n {2041,11},{65525,16},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0}\n };\n static const unsigned short UVAC_HT[256][2] = {\n {0,2},{1,2},{4,3},{10,4},{24,5},{25,5},{56,6},{120,7},{500,9},{1014,10},{4084,12},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {11,4},{57,6},{246,8},{501,9},{2038,11},{4085,12},{65416,16},{65417,16},{65418,16},{65419,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {26,5},{247,8},{1015,10},{4086,12},{32706,15},{65420,16},{65421,16},{65422,16},{65423,16},{65424,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {27,5},{248,8},{1016,10},{4087,12},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{65430,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {58,6},{502,9},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{65438,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {59,6},{1017,10},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{65446,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {121,7},{2039,11},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{65454,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {122,7},{2040,11},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{65462,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {249,8},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{65470,16},{65471,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {503,9},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{65479,16},{65480,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {504,9},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{65488,16},{65489,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {505,9},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{65497,16},{65498,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {506,9},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{65506,16},{65507,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {2041,11},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{65515,16},{65516,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0},\n {16352,14},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{65525,16},{0,0},{0,0},{0,0},{0,0},{0,0},\n {1018,10},{32707,15},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0}\n };\n static const int YQT[] = {16,11,10,16,24,40,51,61,12,12,14,19,26,58,60,55,14,13,16,24,40,57,69,56,14,17,22,29,51,87,80,62,18,22,\n 37,56,68,109,103,77,24,35,55,64,81,104,113,92,49,64,78,87,103,121,120,101,72,92,95,98,112,100,103,99};\n static const int UVQT[] = {17,18,24,47,99,99,99,99,18,21,26,66,99,99,99,99,24,26,56,99,99,99,99,99,47,66,99,99,99,99,99,99,\n 99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99};\n static const float aasf[] = { 1.0f * 2.828427125f, 1.387039845f * 2.828427125f, 1.306562965f * 2.828427125f, 1.175875602f * 2.828427125f,\n 1.0f * 2.828427125f, 0.785694958f * 2.828427125f, 0.541196100f * 2.828427125f, 0.275899379f * 2.828427125f };\n\n int row, col, i, k, subsample;\n float fdtbl_Y[64], fdtbl_UV[64];\n unsigned char YTable[64], UVTable[64];\n\n if(!data || !width || !height || comp > 4 || comp < 1) {\n return 0;\n }\n\n quality = quality ? quality : 90;\n subsample = quality <= 90 ? 1 : 0;\n quality = quality < 1 ? 1 : quality > 100 ? 100 : quality;\n quality = quality < 50 ? 5000 / quality : 200 - quality * 2;\n\n for(i = 0; i < 64; ++i) {\n int uvti, yti = (YQT[i]*quality+50)/100;\n YTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (yti < 1 ? 1 : yti > 255 ? 255 : yti);\n uvti = (UVQT[i]*quality+50)/100;\n UVTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (uvti < 1 ? 1 : uvti > 255 ? 255 : uvti);\n }\n\n for(row = 0, k = 0; row < 8; ++row) {\n for(col = 0; col < 8; ++col, ++k) {\n fdtbl_Y[k] = 1 / (YTable [stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]);\n fdtbl_UV[k] = 1 / (UVTable[stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]);\n }\n }\n\n // Write Headers\n {\n static const unsigned char head0[] = { 0xFF,0xD8,0xFF,0xE0,0,0x10,'J','F','I','F',0,1,1,0,0,1,0,1,0,0,0xFF,0xDB,0,0x84,0 };\n static const unsigned char head2[] = { 0xFF,0xDA,0,0xC,3,1,0,2,0x11,3,0x11,0,0x3F,0 };\n const unsigned char head1[] = { 0xFF,0xC0,0,0x11,8,(unsigned char)(height>>8),STBIW_UCHAR(height),(unsigned char)(width>>8),STBIW_UCHAR(width),\n 3,1,(unsigned char)(subsample?0x22:0x11),0,2,0x11,1,3,0x11,1,0xFF,0xC4,0x01,0xA2,0 };\n s->func(s->context, (void*)head0, sizeof(head0));\n s->func(s->context, (void*)YTable, sizeof(YTable));\n stbiw__putc(s, 1);\n s->func(s->context, UVTable, sizeof(UVTable));\n s->func(s->context, (void*)head1, sizeof(head1));\n s->func(s->context, (void*)(std_dc_luminance_nrcodes+1), sizeof(std_dc_luminance_nrcodes)-1);\n s->func(s->context, (void*)std_dc_luminance_values, sizeof(std_dc_luminance_values));\n stbiw__putc(s, 0x10); // HTYACinfo\n s->func(s->context, (void*)(std_ac_luminance_nrcodes+1), sizeof(std_ac_luminance_nrcodes)-1);\n s->func(s->context, (void*)std_ac_luminance_values, sizeof(std_ac_luminance_values));\n stbiw__putc(s, 1); // HTUDCinfo\n s->func(s->context, (void*)(std_dc_chrominance_nrcodes+1), sizeof(std_dc_chrominance_nrcodes)-1);\n s->func(s->context, (void*)std_dc_chrominance_values, sizeof(std_dc_chrominance_values));\n stbiw__putc(s, 0x11); // HTUACinfo\n s->func(s->context, (void*)(std_ac_chrominance_nrcodes+1), sizeof(std_ac_chrominance_nrcodes)-1);\n s->func(s->context, (void*)std_ac_chrominance_values, sizeof(std_ac_chrominance_values));\n s->func(s->context, (void*)head2, sizeof(head2));\n }\n\n // Encode 8x8 macroblocks\n {\n static const unsigned short fillBits[] = {0x7F, 7};\n int DCY=0, DCU=0, DCV=0;\n int bitBuf=0, bitCnt=0;\n // comp == 2 is grey+alpha (alpha is ignored)\n int ofsG = comp > 2 ? 1 : 0, ofsB = comp > 2 ? 2 : 0;\n const unsigned char *dataR = (const unsigned char *)data;\n const unsigned char *dataG = dataR + ofsG;\n const unsigned char *dataB = dataR + ofsB;\n int x, y, pos;\n if(subsample) {\n for(y = 0; y < height; y += 16) {\n for(x = 0; x < width; x += 16) {\n float Y[256], U[256], V[256];\n for(row = y, pos = 0; row < y+16; ++row) {\n // row >= height => use last input row\n int clamped_row = (row < height) ? row : height - 1;\n int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp;\n for(col = x; col < x+16; ++col, ++pos) {\n // if col >= width => use pixel from last input column\n int p = base_p + ((col < width) ? col : (width-1))*comp;\n float r = dataR[p], g = dataG[p], b = dataB[p];\n Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128;\n U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b;\n V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b;\n }\n }\n DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+0, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT);\n DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+8, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT);\n DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+128, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT);\n DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+136, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT);\n\n // subsample U,V\n {\n float subU[64], subV[64];\n int yy, xx;\n for(yy = 0, pos = 0; yy < 8; ++yy) {\n for(xx = 0; xx < 8; ++xx, ++pos) {\n int j = yy*32+xx*2;\n subU[pos] = (U[j+0] + U[j+1] + U[j+16] + U[j+17]) * 0.25f;\n subV[pos] = (V[j+0] + V[j+1] + V[j+16] + V[j+17]) * 0.25f;\n }\n }\n DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subU, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT);\n DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subV, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT);\n }\n }\n }\n } else {\n for(y = 0; y < height; y += 8) {\n for(x = 0; x < width; x += 8) {\n float Y[64], U[64], V[64];\n for(row = y, pos = 0; row < y+8; ++row) {\n // row >= height => use last input row\n int clamped_row = (row < height) ? row : height - 1;\n int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp;\n for(col = x; col < x+8; ++col, ++pos) {\n // if col >= width => use pixel from last input column\n int p = base_p + ((col < width) ? col : (width-1))*comp;\n float r = dataR[p], g = dataG[p], b = dataB[p];\n Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128;\n U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b;\n V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b;\n }\n }\n\n DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y, 8, fdtbl_Y, DCY, YDC_HT, YAC_HT);\n DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, U, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT);\n DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, V, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT);\n }\n }\n }\n\n // Do the bit alignment of the EOI marker\n stbiw__jpg_writeBits(s, &bitBuf, &bitCnt, fillBits);\n }\n\n // EOI\n stbiw__putc(s, 0xFF);\n stbiw__putc(s, 0xD9);\n\n return 1;\n}\n\nSTBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality)\n{\n stbi__write_context s = { 0 };\n stbi__start_write_callbacks(&s, func, context);\n return stbi_write_jpg_core(&s, x, y, comp, (void *) data, quality);\n}\n\n\n#ifndef STBI_WRITE_NO_STDIO\nSTBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality)\n{\n stbi__write_context s = { 0 };\n if (stbi__start_write_file(&s,filename)) {\n int r = stbi_write_jpg_core(&s, x, y, comp, data, quality);\n stbi__end_write_file(&s);\n return r;\n } else\n return 0;\n}\n#endif\n\n#endif // STB_IMAGE_WRITE_IMPLEMENTATION\n\n/* Revision history\n 1.16 (2021-07-11)\n make Deflate code emit uncompressed blocks when it would otherwise expand\n support writing BMPs with alpha channel\n 1.15 (2020-07-13) unknown\n 1.14 (2020-02-02) updated JPEG writer to downsample chroma channels\n 1.13\n 1.12\n 1.11 (2019-08-11)\n\n 1.10 (2019-02-07)\n support utf8 filenames in Windows; fix warnings and platform ifdefs\n 1.09 (2018-02-11)\n fix typo in zlib quality API, improve STB_I_W_STATIC in C++\n 1.08 (2018-01-29)\n add stbi__flip_vertically_on_write, external zlib, zlib quality, choose PNG filter\n 1.07 (2017-07-24)\n doc fix\n 1.06 (2017-07-23)\n writing JPEG (using Jon Olick's code)\n 1.05 ???\n 1.04 (2017-03-03)\n monochrome BMP expansion\n 1.03 ???\n 1.02 (2016-04-02)\n avoid allocating large structures on the stack\n 1.01 (2016-01-16)\n STBIW_REALLOC_SIZED: support allocators with no realloc support\n avoid race-condition in crc initialization\n minor compile issues\n 1.00 (2015-09-14)\n installable file IO function\n 0.99 (2015-09-13)\n warning fixes; TGA rle support\n 0.98 (2015-04-08)\n added STBIW_MALLOC, STBIW_ASSERT etc\n 0.97 (2015-01-18)\n fixed HDR asserts, rewrote HDR rle logic\n 0.96 (2015-01-17)\n add HDR output\n fix monochrome BMP\n 0.95 (2014-08-17)\n add monochrome TGA output\n 0.94 (2014-05-31)\n rename private functions to avoid conflicts with stb_image.h\n 0.93 (2014-05-27)\n warning fixes\n 0.92 (2010-08-01)\n casts to unsigned char to fix warnings\n 0.91 (2010-07-17)\n first public release\n 0.90 first internal release\n*/\n\n/*\n------------------------------------------------------------------------------\nThis software is available under 2 licenses -- choose whichever you prefer.\n------------------------------------------------------------------------------\nALTERNATIVE A - MIT License\nCopyright (c) 2017 Sean Barrett\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\nof the Software, and to permit persons to whom the Software is furnished to do\nso, subject to the following conditions:\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n------------------------------------------------------------------------------\nALTERNATIVE B - Public Domain (www.unlicense.org)\nThis is free and unencumbered software released into the public domain.\nAnyone is free to copy, modify, publish, use, compile, sell, or distribute this\nsoftware, either in source code form or as a compiled binary, for any purpose,\ncommercial or non-commercial, and by any means.\nIn jurisdictions that recognize copyright laws, the author or authors of this\nsoftware dedicate any and all copyright interest in the software to the public\ndomain. We make this dedication for the benefit of the public at large and to\nthe detriment of our heirs and successors. We intend this dedication to be an\novert act of relinquishment in perpetuity of all present and future rights to\nthis software under copyright law.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\nACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n------------------------------------------------------------------------------\n*/\n" }, { "path": "examples/test-cmake/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.14)\nproject(ggml-simple)\n\nset(CMAKE_CXX_STANDARD 17)\n\nfind_package(ggml CONFIG REQUIRED)\n\nset(TEST_TARGET test-cmake)\nadd_executable(test-cmake test-cmake.cpp)\ntarget_link_libraries(test-cmake PRIVATE ggml::ggml)\n" }, { "path": "examples/test-cmake/README.md", "content": "## cmake-test\n\nThis directory can be built as a separate project with an installed ggml.\n" }, { "path": "examples/test-cmake/test-cmake.cpp", "content": "#include \"ggml-backend.h\"\n\nint main(void) {\n ggml_backend_load_all();\n return 0;\n}\n" }, { "path": "examples/yolo/CMakeLists.txt", "content": "#\n# yolov3-tiny\n\nset(TEST_TARGET yolov3-tiny)\nadd_executable(${TEST_TARGET} yolov3-tiny.cpp yolo-image.cpp)\ntarget_link_libraries(${TEST_TARGET} PRIVATE ggml common)\n" }, { "path": "examples/yolo/README.md", "content": "This example shows how to implement YOLO object detection with ggml using pretrained model.\n\n# YOLOv3-tiny\n\nDownload the model weights:\n\n```bash\n$ wget https://pjreddie.com/media/files/yolov3-tiny.weights\n$ sha1sum yolov3-tiny.weights \n40f3c11883bef62fd850213bc14266632ed4414f yolov3-tiny.weights\n```\n\nConvert the weights to GGUF format:\n\n```bash\n$ ./convert-yolov3-tiny.py yolov3-tiny.weights\nyolov3-tiny.weights converted to yolov3-tiny.gguf\n```\n\nAlternatively, you can download the converted model from [HuggingFace](https://huggingface.co/rgerganov/yolo-gguf/resolve/main/yolov3-tiny.gguf)\n\nObject detection:\n\n```bash\n$ wget https://raw.githubusercontent.com/pjreddie/darknet/master/data/dog.jpg\n$ ./yolov3-tiny -m yolov3-tiny.gguf -i dog.jpg\nload_model: using CUDA backend\nggml_cuda_init: GGML_CUDA_FORCE_MMQ: no\nggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no\nggml_cuda_init: found 1 CUDA devices:\n Device 0: NVIDIA T1200 Laptop GPU, compute capability 7.5, VMM: yes\nLayer 0 output shape: 416 x 416 x 16 x 1\nLayer 1 output shape: 208 x 208 x 16 x 1\nLayer 2 output shape: 208 x 208 x 32 x 1\nLayer 3 output shape: 104 x 104 x 32 x 1\nLayer 4 output shape: 104 x 104 x 64 x 1\nLayer 5 output shape: 52 x 52 x 64 x 1\nLayer 6 output shape: 52 x 52 x 128 x 1\nLayer 7 output shape: 26 x 26 x 128 x 1\nLayer 8 output shape: 26 x 26 x 256 x 1\nLayer 9 output shape: 13 x 13 x 256 x 1\nLayer 10 output shape: 13 x 13 x 512 x 1\nLayer 11 output shape: 13 x 13 x 512 x 1\nLayer 12 output shape: 13 x 13 x 1024 x 1\nLayer 13 output shape: 13 x 13 x 256 x 1\nLayer 14 output shape: 13 x 13 x 512 x 1\nLayer 15 output shape: 13 x 13 x 255 x 1\nLayer 18 output shape: 13 x 13 x 128 x 1\nLayer 19 output shape: 26 x 26 x 128 x 1\nLayer 20 output shape: 26 x 26 x 384 x 1\nLayer 21 output shape: 26 x 26 x 256 x 1\nLayer 22 output shape: 26 x 26 x 255 x 1\ndog: 57%\ncar: 52%\ntruck: 56%\ncar: 62%\nbicycle: 59%\nDetected objects saved in 'predictions.jpg' (time: 0.057000 sec.)\n```" }, { "path": "examples/yolo/convert-yolov3-tiny.py", "content": "#!/usr/bin/env python3\nimport sys\nimport gguf\nimport numpy as np\n\ndef save_conv2d_layer(f, gguf_writer, prefix, inp_c, filters, size, batch_normalize=True):\n biases = np.fromfile(f, dtype=np.float32, count=filters)\n gguf_writer.add_tensor(prefix + \"_biases\", biases, raw_shape=(1, filters, 1, 1))\n\n if batch_normalize:\n scales = np.fromfile(f, dtype=np.float32, count=filters)\n gguf_writer.add_tensor(prefix + \"_scales\", scales, raw_shape=(1, filters, 1, 1))\n rolling_mean = np.fromfile(f, dtype=np.float32, count=filters)\n gguf_writer.add_tensor(prefix + \"_rolling_mean\", rolling_mean, raw_shape=(1, filters, 1, 1))\n rolling_variance = np.fromfile(f, dtype=np.float32, count=filters)\n gguf_writer.add_tensor(prefix + \"_rolling_variance\", rolling_variance, raw_shape=(1, filters, 1, 1))\n\n weights_count = filters * inp_c * size * size\n l0_weights = np.fromfile(f, dtype=np.float32, count=weights_count)\n ## ggml doesn't support f32 convolution yet, use f16 instead\n l0_weights = l0_weights.astype(np.float16)\n gguf_writer.add_tensor(prefix + \"_weights\", l0_weights, raw_shape=(filters, inp_c, size, size))\n\n\nif __name__ == '__main__':\n if len(sys.argv) != 2:\n print(\"Usage: %s \" % sys.argv[0])\n sys.exit(1)\n outfile = 'yolov3-tiny.gguf'\n gguf_writer = gguf.GGUFWriter(outfile, 'yolov3-tiny')\n\n f = open(sys.argv[1], 'rb')\n f.read(20) # skip header\n save_conv2d_layer(f, gguf_writer, \"l0\", 3, 16, 3)\n save_conv2d_layer(f, gguf_writer, \"l1\", 16, 32, 3)\n save_conv2d_layer(f, gguf_writer, \"l2\", 32, 64, 3)\n save_conv2d_layer(f, gguf_writer, \"l3\", 64, 128, 3)\n save_conv2d_layer(f, gguf_writer, \"l4\", 128, 256, 3)\n save_conv2d_layer(f, gguf_writer, \"l5\", 256, 512, 3)\n save_conv2d_layer(f, gguf_writer, \"l6\", 512, 1024, 3)\n save_conv2d_layer(f, gguf_writer, \"l7\", 1024, 256, 1)\n save_conv2d_layer(f, gguf_writer, \"l8\", 256, 512, 3)\n save_conv2d_layer(f, gguf_writer, \"l9\", 512, 255, 1, batch_normalize=False)\n save_conv2d_layer(f, gguf_writer, \"l10\", 256, 128, 1)\n save_conv2d_layer(f, gguf_writer, \"l11\", 384, 256, 3)\n save_conv2d_layer(f, gguf_writer, \"l12\", 256, 255, 1, batch_normalize=False)\n f.close()\n \n gguf_writer.write_header_to_file()\n gguf_writer.write_kv_data_to_file()\n gguf_writer.write_tensors_to_file()\n gguf_writer.close()\n print(\"{} converted to {}\".format(sys.argv[1], outfile))\n" }, { "path": "examples/yolo/data/coco.names", "content": "person\nbicycle\ncar\nmotorbike\naeroplane\nbus\ntrain\ntruck\nboat\ntraffic light\nfire hydrant\nstop sign\nparking meter\nbench\nbird\ncat\ndog\nhorse\nsheep\ncow\nelephant\nbear\nzebra\ngiraffe\nbackpack\numbrella\nhandbag\ntie\nsuitcase\nfrisbee\nskis\nsnowboard\nsports ball\nkite\nbaseball bat\nbaseball glove\nskateboard\nsurfboard\ntennis racket\nbottle\nwine glass\ncup\nfork\nknife\nspoon\nbowl\nbanana\napple\nsandwich\norange\nbroccoli\ncarrot\nhot dog\npizza\ndonut\ncake\nchair\nsofa\npottedplant\nbed\ndiningtable\ntoilet\ntvmonitor\nlaptop\nmouse\nremote\nkeyboard\ncell phone\nmicrowave\noven\ntoaster\nsink\nrefrigerator\nbook\nclock\nvase\nscissors\nteddy bear\nhair drier\ntoothbrush\n" }, { "path": "examples/yolo/yolo-image.cpp", "content": "#define STB_IMAGE_IMPLEMENTATION\n#include \"stb_image.h\"\n#define STB_IMAGE_WRITE_IMPLEMENTATION\n#include \"stb_image_write.h\"\n\n#include \"yolo-image.h\"\n\nstatic void draw_box(yolo_image & a, int x1, int y1, int x2, int y2, float r, float g, float b)\n{\n if (x1 < 0) x1 = 0;\n if (x1 >= a.w) x1 = a.w-1;\n if (x2 < 0) x2 = 0;\n if (x2 >= a.w) x2 = a.w-1;\n\n if (y1 < 0) y1 = 0;\n if (y1 >= a.h) y1 = a.h-1;\n if (y2 < 0) y2 = 0;\n if (y2 >= a.h) y2 = a.h-1;\n\n for (int i = x1; i <= x2; ++i){\n a.data[i + y1*a.w + 0*a.w*a.h] = r;\n a.data[i + y2*a.w + 0*a.w*a.h] = r;\n\n a.data[i + y1*a.w + 1*a.w*a.h] = g;\n a.data[i + y2*a.w + 1*a.w*a.h] = g;\n\n a.data[i + y1*a.w + 2*a.w*a.h] = b;\n a.data[i + y2*a.w + 2*a.w*a.h] = b;\n }\n for (int i = y1; i <= y2; ++i){\n a.data[x1 + i*a.w + 0*a.w*a.h] = r;\n a.data[x2 + i*a.w + 0*a.w*a.h] = r;\n\n a.data[x1 + i*a.w + 1*a.w*a.h] = g;\n a.data[x2 + i*a.w + 1*a.w*a.h] = g;\n\n a.data[x1 + i*a.w + 2*a.w*a.h] = b;\n a.data[x2 + i*a.w + 2*a.w*a.h] = b;\n }\n}\n\nvoid draw_box_width(yolo_image & a, int x1, int y1, int x2, int y2, int w, float r, float g, float b)\n{\n for (int i = 0; i < w; ++i) {\n draw_box(a, x1+i, y1+i, x2-i, y2-i, r, g, b);\n }\n}\n\nbool save_image(const yolo_image & im, const char *name, int quality)\n{\n uint8_t *data = (uint8_t*)calloc(im.w*im.h*im.c, sizeof(uint8_t));\n for (int k = 0; k < im.c; ++k) {\n for (int i = 0; i < im.w*im.h; ++i) {\n data[i*im.c+k] = (uint8_t) (255*im.data[i + k*im.w*im.h]);\n }\n }\n int success = stbi_write_jpg(name, im.w, im.h, im.c, data, quality);\n free(data);\n if (!success) {\n fprintf(stderr, \"Failed to write image %s\\n\", name);\n return false;\n }\n return true;\n}\n\nbool load_image(const char *fname, yolo_image & img)\n{\n int w, h, c;\n uint8_t * data = stbi_load(fname, &w, &h, &c, 3);\n if (!data) {\n return false;\n }\n c = 3;\n img.w = w;\n img.h = h;\n img.c = c;\n img.data.resize(w*h*c);\n for (int k = 0; k < c; ++k){\n for (int j = 0; j < h; ++j){\n for (int i = 0; i < w; ++i){\n int dst_index = i + w*j + w*h*k;\n int src_index = k + c*i + c*w*j;\n img.data[dst_index] = (float)data[src_index]/255.;\n }\n }\n }\n stbi_image_free(data);\n return true;\n}\n\nstatic yolo_image resize_image(const yolo_image & im, int w, int h)\n{\n yolo_image resized(w, h, im.c);\n yolo_image part(w, im.h, im.c);\n float w_scale = (float)(im.w - 1) / (w - 1);\n float h_scale = (float)(im.h - 1) / (h - 1);\n for (int k = 0; k < im.c; ++k){\n for (int r = 0; r < im.h; ++r) {\n for (int c = 0; c < w; ++c) {\n float val = 0;\n if (c == w-1 || im.w == 1){\n val = im.get_pixel(im.w-1, r, k);\n } else {\n float sx = c*w_scale;\n int ix = (int) sx;\n float dx = sx - ix;\n val = (1 - dx) * im.get_pixel(ix, r, k) + dx * im.get_pixel(ix+1, r, k);\n }\n part.set_pixel(c, r, k, val);\n }\n }\n }\n for (int k = 0; k < im.c; ++k){\n for (int r = 0; r < h; ++r){\n float sy = r*h_scale;\n int iy = (int) sy;\n float dy = sy - iy;\n for (int c = 0; c < w; ++c){\n float val = (1-dy) * part.get_pixel(c, iy, k);\n resized.set_pixel(c, r, k, val);\n }\n if (r == h-1 || im.h == 1) continue;\n for (int c = 0; c < w; ++c){\n float val = dy * part.get_pixel(c, iy+1, k);\n resized.add_pixel(c, r, k, val);\n }\n }\n }\n return resized;\n}\n\nstatic void embed_image(const yolo_image & source, yolo_image & dest, int dx, int dy)\n{\n for (int k = 0; k < source.c; ++k) {\n for (int y = 0; y < source.h; ++y) {\n for (int x = 0; x < source.w; ++x) {\n float val = source.get_pixel(x, y, k);\n dest.set_pixel(dx+x, dy+y, k, val);\n }\n }\n }\n}\n\nyolo_image letterbox_image(const yolo_image & im, int w, int h)\n{\n int new_w = im.w;\n int new_h = im.h;\n if (((float)w/im.w) < ((float)h/im.h)) {\n new_w = w;\n new_h = (im.h * w)/im.w;\n } else {\n new_h = h;\n new_w = (im.w * h)/im.h;\n }\n yolo_image resized = resize_image(im, new_w, new_h);\n yolo_image boxed(w, h, im.c);\n boxed.fill(0.5);\n embed_image(resized, boxed, (w-new_w)/2, (h-new_h)/2);\n return boxed;\n}\n\nstatic yolo_image tile_images(const yolo_image & a, const yolo_image & b, int dx)\n{\n if (a.w == 0) {\n return b;\n }\n yolo_image c(a.w + b.w + dx, (a.h > b.h) ? a.h : b.h, a.c);\n c.fill(1.0f);\n embed_image(a, c, 0, 0);\n embed_image(b, c, a.w + dx, 0);\n return c;\n}\n\nstatic yolo_image border_image(const yolo_image & a, int border)\n{\n yolo_image b(a.w + 2*border, a.h + 2*border, a.c);\n b.fill(1.0f);\n embed_image(a, b, border, border);\n return b;\n}\n\nyolo_image get_label(const std::vector & alphabet, const std::string & label, int size)\n{\n size = size/10;\n size = std::min(size, 7);\n yolo_image result(0,0,0);\n for (int i = 0; i < (int)label.size(); ++i) {\n int ch = label[i];\n yolo_image img = alphabet[size*128 + ch];\n result = tile_images(result, img, -size - 1 + (size+1)/2);\n }\n return border_image(result, (int)(result.h*.25));\n}\n\nvoid draw_label(yolo_image & im, int row, int col, const yolo_image & label, const float * rgb)\n{\n int w = label.w;\n int h = label.h;\n if (row - h >= 0) {\n row = row - h;\n }\n for (int j = 0; j < h && j + row < im.h; j++) {\n for (int i = 0; i < w && i + col < im.w; i++) {\n for (int k = 0; k < label.c; k++) {\n float val = label.get_pixel(i, j, k);\n im.set_pixel(i + col, j + row, k, rgb[k] * val);\n }\n }\n }\n}" }, { "path": "examples/yolo/yolo-image.h", "content": "#pragma once\n\n#include \n#include \n#include \n\nstruct yolo_image {\n int w, h, c;\n std::vector data;\n\n yolo_image() : w(0), h(0), c(0) {}\n yolo_image(int w, int h, int c) : w(w), h(h), c(c), data(w*h*c) {}\n\n float get_pixel(int x, int y, int c) const {\n assert(x >= 0 && x < w && y >= 0 && y < h && c >= 0 && c < this->c);\n return data[c*w*h + y*w + x];\n }\n\n void set_pixel(int x, int y, int c, float val) {\n assert(x >= 0 && x < w && y >= 0 && y < h && c >= 0 && c < this->c);\n data[c*w*h + y*w + x] = val;\n }\n\n void add_pixel(int x, int y, int c, float val) {\n assert(x >= 0 && x < w && y >= 0 && y < h && c >= 0 && c < this->c);\n data[c*w*h + y*w + x] += val;\n }\n\n void fill(float val) {\n std::fill(data.begin(), data.end(), val);\n }\n};\n\nbool load_image(const char *fname, yolo_image & img);\nvoid draw_box_width(yolo_image & a, int x1, int y1, int x2, int y2, int w, float r, float g, float b);\nyolo_image letterbox_image(const yolo_image & im, int w, int h);\nbool save_image(const yolo_image & im, const char *name, int quality);\nyolo_image get_label(const std::vector & alphabet, const std::string & label, int size);\nvoid draw_label(yolo_image & im, int row, int col, const yolo_image & label, const float * rgb);\n" }, { "path": "examples/yolo/yolov3-tiny.cpp", "content": "#include \"ggml.h\"\n#include \"gguf.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n\n#include \"yolo-image.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\nstruct conv2d_layer {\n struct ggml_tensor * weights;\n struct ggml_tensor * biases;\n struct ggml_tensor * scales;\n struct ggml_tensor * rolling_mean;\n struct ggml_tensor * rolling_variance;\n int padding = 1;\n bool batch_normalize = true;\n bool activate = true; // true for leaky relu, false for linear\n};\n\nstruct yolo_model {\n int width = 416;\n int height = 416;\n std::vector conv2d_layers;\n ggml_backend_t backend;\n ggml_backend_buffer_t buffer;\n struct ggml_context * ctx;\n};\n\nstruct yolo_layer {\n int classes = 80;\n std::vector mask;\n std::vector anchors;\n std::vector predictions;\n int w;\n int h;\n\n yolo_layer(int classes, const std::vector & mask, const std::vector & anchors, struct ggml_tensor * prev_layer)\n : classes(classes), mask(mask), anchors(anchors)\n {\n w = prev_layer->ne[0];\n h = prev_layer->ne[1];\n predictions.resize(ggml_nbytes(prev_layer)/sizeof(float));\n ggml_backend_tensor_get(prev_layer, predictions.data(), 0, ggml_nbytes(prev_layer));\n }\n\n int entry_index(int location, int entry) const {\n int n = location / (w*h);\n int loc = location % (w*h);\n return n*w*h*(4+classes+1) + entry*w*h + loc;\n }\n};\n\nstruct box {\n float x, y, w, h;\n};\n\nstruct detection {\n box bbox;\n std::vector prob;\n float objectness;\n};\n\nstatic bool load_model(const std::string & fname, yolo_model & model) {\n struct ggml_context * tmp_ctx = nullptr;\n struct gguf_init_params gguf_params = {\n /*.no_alloc =*/ false,\n /*.ctx =*/ &tmp_ctx,\n };\n gguf_context * gguf_ctx = gguf_init_from_file(fname.c_str(), gguf_params);\n if (!gguf_ctx) {\n fprintf(stderr, \"%s: gguf_init_from_file() failed\\n\", __func__);\n return false;\n }\n\n int num_tensors = gguf_get_n_tensors(gguf_ctx);\n struct ggml_init_params params {\n /*.mem_size =*/ ggml_tensor_overhead() * num_tensors,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n model.ctx = ggml_init(params);\n for (int i = 0; i < num_tensors; i++) {\n const char * name = gguf_get_tensor_name(gguf_ctx, i);\n struct ggml_tensor * src = ggml_get_tensor(tmp_ctx, name);\n struct ggml_tensor * dst = ggml_dup_tensor(model.ctx, src);\n ggml_set_name(dst, name);\n }\n model.buffer = ggml_backend_alloc_ctx_tensors(model.ctx, model.backend);\n // copy tensors from main memory to backend\n for (struct ggml_tensor * cur = ggml_get_first_tensor(model.ctx); cur != NULL; cur = ggml_get_next_tensor(model.ctx, cur)) {\n struct ggml_tensor * src = ggml_get_tensor(tmp_ctx, ggml_get_name(cur));\n size_t n_size = ggml_nbytes(src);\n ggml_backend_tensor_set(cur, ggml_get_data(src), 0, n_size);\n }\n gguf_free(gguf_ctx);\n ggml_free(tmp_ctx);\n \n model.width = 416;\n model.height = 416;\n model.conv2d_layers.resize(13);\n model.conv2d_layers[7].padding = 0;\n model.conv2d_layers[9].padding = 0;\n model.conv2d_layers[9].batch_normalize = false;\n model.conv2d_layers[9].activate = false;\n model.conv2d_layers[10].padding = 0;\n model.conv2d_layers[12].padding = 0;\n model.conv2d_layers[12].batch_normalize = false;\n model.conv2d_layers[12].activate = false;\n for (int i = 0; i < (int)model.conv2d_layers.size(); i++) {\n char name[256];\n snprintf(name, sizeof(name), \"l%d_weights\", i);\n model.conv2d_layers[i].weights = ggml_get_tensor(model.ctx, name);\n snprintf(name, sizeof(name), \"l%d_biases\", i);\n model.conv2d_layers[i].biases = ggml_get_tensor(model.ctx, name);\n if (model.conv2d_layers[i].batch_normalize) {\n snprintf(name, sizeof(name), \"l%d_scales\", i);\n model.conv2d_layers[i].scales = ggml_get_tensor(model.ctx, name);\n snprintf(name, sizeof(name), \"l%d_rolling_mean\", i);\n model.conv2d_layers[i].rolling_mean = ggml_get_tensor(model.ctx, name);\n snprintf(name, sizeof(name), \"l%d_rolling_variance\", i);\n model.conv2d_layers[i].rolling_variance = ggml_get_tensor(model.ctx, name);\n }\n }\n return true;\n}\n\nstatic bool load_labels(const char * filename, std::vector & labels)\n{\n std::ifstream file_in(filename);\n if (!file_in) {\n return false;\n }\n std::string line;\n while (std::getline(file_in, line)) {\n labels.push_back(line);\n }\n GGML_ASSERT(labels.size() == 80);\n return true;\n}\n\nstatic bool load_alphabet(std::vector & alphabet)\n{\n alphabet.resize(8 * 128);\n for (int j = 0; j < 8; j++) {\n for (int i = 32; i < 127; i++) {\n char fname[256];\n snprintf(fname, sizeof(fname), \"data/labels/%d_%d.png\", i, j);\n if (!load_image(fname, alphabet[j*128 + i])) {\n fprintf(stderr, \"Cannot load '%s'\\n\", fname);\n return false;\n }\n }\n }\n return true;\n}\n\nstatic ggml_tensor * apply_conv2d(ggml_context * ctx, ggml_tensor * input, const conv2d_layer & layer)\n{\n struct ggml_tensor * result = ggml_conv_2d(ctx, layer.weights, input, 1, 1, layer.padding, layer.padding, 1, 1);\n if (layer.batch_normalize) {\n result = ggml_sub(ctx, result, ggml_repeat(ctx, layer.rolling_mean, result));\n result = ggml_div(ctx, result, ggml_sqrt(ctx, ggml_repeat(ctx, layer.rolling_variance, result)));\n result = ggml_mul(ctx, result, ggml_repeat(ctx, layer.scales, result));\n }\n result = ggml_add(ctx, result, ggml_repeat(ctx, layer.biases, result));\n if (layer.activate) {\n result = ggml_leaky_relu(ctx, result, 0.1f, true);\n }\n return result;\n}\n\nstatic void activate_array(float * x, const int n)\n{\n // logistic activation\n for (int i = 0; i < n; i++) {\n x[i] = 1./(1. + exp(-x[i]));\n }\n}\n\nstatic void apply_yolo(yolo_layer & layer)\n{\n int w = layer.w;\n int h = layer.h;\n int N = layer.mask.size();\n float * data = layer.predictions.data();\n for (int n = 0; n < N; n++) {\n int index = layer.entry_index(n*w*h, 0);\n activate_array(data + index, 2*w*h);\n index = layer.entry_index(n*w*h, 4);\n activate_array(data + index, (1+layer.classes)*w*h);\n }\n}\n\nstatic box get_yolo_box(const yolo_layer & layer, int n, int index, int i, int j, int lw, int lh, int w, int h, int stride)\n{\n const float * predictions = layer.predictions.data();\n box b;\n b.x = (i + predictions[index + 0*stride]) / lw;\n b.y = (j + predictions[index + 1*stride]) / lh;\n b.w = exp(predictions[index + 2*stride]) * layer.anchors[2*n] / w;\n b.h = exp(predictions[index + 3*stride]) * layer.anchors[2*n+1] / h;\n return b;\n}\n\nstatic void correct_yolo_box(box & b, int im_w, int im_h, int net_w, int net_h)\n{\n int new_w = 0;\n int new_h = 0;\n if (((float)net_w/im_w) < ((float)net_h/im_h)) {\n new_w = net_w;\n new_h = (im_h * net_w)/im_w;\n } else {\n new_h = net_h;\n new_w = (im_w * net_h)/im_h;\n }\n b.x = (b.x - (net_w - new_w)/2./net_w) / ((float)new_w/net_w);\n b.y = (b.y - (net_h - new_h)/2./net_h) / ((float)new_h/net_h);\n b.w *= (float)net_w/new_w;\n b.h *= (float)net_h/new_h;\n}\n\nstatic void get_yolo_detections(const yolo_layer & layer, std::vector & detections, int im_w, int im_h, int netw, int neth, float thresh)\n{\n int w = layer.w;\n int h = layer.h;\n int N = layer.mask.size();\n const float * predictions = layer.predictions.data();\n std::vector result;\n for (int i = 0; i < w*h; i++) {\n for (int n = 0; n < N; n++) {\n int obj_index = layer.entry_index(n*w*h + i, 4);\n float objectness = predictions[obj_index];\n if (objectness <= thresh) {\n continue;\n }\n detection det;\n int box_index = layer.entry_index(n*w*h + i, 0);\n int row = i / w;\n int col = i % w;\n det.bbox = get_yolo_box(layer, layer.mask[n], box_index, col, row, w, h, netw, neth, w*h);\n correct_yolo_box(det.bbox, im_w, im_h, netw, neth);\n det.objectness = objectness;\n det.prob.resize(layer.classes);\n for (int j = 0; j < layer.classes; j++) {\n int class_index = layer.entry_index(n*w*h + i, 4 + 1 + j);\n float prob = objectness*predictions[class_index];\n det.prob[j] = (prob > thresh) ? prob : 0;\n }\n detections.push_back(det);\n }\n }\n}\n\nstatic float overlap(float x1, float w1, float x2, float w2)\n{\n float l1 = x1 - w1/2;\n float l2 = x2 - w2/2;\n float left = l1 > l2 ? l1 : l2;\n float r1 = x1 + w1/2;\n float r2 = x2 + w2/2;\n float right = r1 < r2 ? r1 : r2;\n return right - left;\n}\n\nstatic float box_intersection(const box & a, const box & b)\n{\n float w = overlap(a.x, a.w, b.x, b.w);\n float h = overlap(a.y, a.h, b.y, b.h);\n if (w < 0 || h < 0) return 0;\n float area = w*h;\n return area;\n}\n\nstatic float box_union(const box & a, const box & b)\n{\n float i = box_intersection(a, b);\n float u = a.w*a.h + b.w*b.h - i;\n return u;\n}\n\nstatic float box_iou(const box & a, const box & b)\n{\n return box_intersection(a, b)/box_union(a, b);\n}\n\nstatic void do_nms_sort(std::vector & dets, int classes, float thresh)\n{\n int k = (int)dets.size()-1;\n for (int i = 0; i <= k; ++i) {\n if (dets[i].objectness == 0) {\n std::swap(dets[i], dets[k]);\n --k;\n --i;\n }\n }\n int total = k+1;\n for (int k = 0; k < classes; ++k) {\n std::sort(dets.begin(), dets.begin()+total, [=](const detection & a, const detection & b) {\n return a.prob[k] > b.prob[k];\n });\n for (int i = 0; i < total; ++i) {\n if (dets[i].prob[k] == 0) {\n continue;\n }\n box a = dets[i].bbox;\n for (int j = i+1; j < total; ++j){\n box b = dets[j].bbox;\n if (box_iou(a, b) > thresh) {\n dets[j].prob[k] = 0;\n }\n }\n }\n }\n}\n\nstatic float get_color(int c, int x, int max)\n{\n float colors[6][3] = { {1,0,1}, {0,0,1}, {0,1,1}, {0,1,0}, {1,1,0}, {1,0,0} };\n float ratio = ((float)x/max)*5;\n int i = floor(ratio);\n int j = ceil(ratio);\n ratio -= i;\n float r = (1-ratio) * colors[i][c] + ratio*colors[j][c];\n return r;\n}\n\nstatic void draw_detections(yolo_image & im, const std::vector & dets, float thresh, const std::vector & labels, const std::vector & alphabet)\n{\n int classes = (int)labels.size();\n for (int i = 0; i < (int)dets.size(); i++) {\n std::string labelstr;\n int cl = -1;\n for (int j = 0; j < (int)dets[i].prob.size(); j++) {\n if (dets[i].prob[j] > thresh) {\n if (cl < 0) {\n labelstr = labels[j];\n cl = j;\n } else {\n labelstr += \", \";\n labelstr += labels[j];\n }\n printf(\"%s: %.0f%%\\n\", labels[j].c_str(), dets[i].prob[j]*100);\n }\n }\n if (cl >= 0) {\n int width = im.h * .006;\n int offset = cl*123457 % classes;\n float red = get_color(2,offset,classes);\n float green = get_color(1,offset,classes);\n float blue = get_color(0,offset,classes);\n float rgb[3];\n\n rgb[0] = red;\n rgb[1] = green;\n rgb[2] = blue;\n box b = dets[i].bbox;\n\n int left = (b.x-b.w/2.)*im.w;\n int right = (b.x+b.w/2.)*im.w;\n int top = (b.y-b.h/2.)*im.h;\n int bot = (b.y+b.h/2.)*im.h;\n\n if (left < 0) left = 0;\n if (right > im.w-1) right = im.w-1;\n if (top < 0) top = 0;\n if (bot > im.h-1) bot = im.h-1;\n\n draw_box_width(im, left, top, right, bot, width, red, green, blue);\n yolo_image label = get_label(alphabet, labelstr, (im.h*.03));\n draw_label(im, top + width, left, label, rgb);\n }\n }\n}\n\nstatic void print_shape(int layer, const ggml_tensor * t)\n{\n printf(\"Layer %2d output shape: %3d x %3d x %4d x %3d\\n\", layer, (int)t->ne[0], (int)t->ne[1], (int)t->ne[2], (int)t->ne[3]);\n}\n\nstatic struct ggml_cgraph * build_graph(struct ggml_context * ctx_cgraph, const yolo_model & model) {\n struct ggml_cgraph * gf = ggml_new_graph(ctx_cgraph);\n\n struct ggml_tensor * input = ggml_new_tensor_4d(ctx_cgraph, GGML_TYPE_F32, model.width, model.height, 3, 1);\n ggml_set_name(input, \"input\");\n struct ggml_tensor * result = apply_conv2d(ctx_cgraph, input, model.conv2d_layers[0]);\n print_shape(0, result);\n result = ggml_pool_2d(ctx_cgraph, result, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);\n print_shape(1, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[1]);\n print_shape(2, result);\n result = ggml_pool_2d(ctx_cgraph, result, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);\n print_shape(3, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[2]);\n print_shape(4, result);\n result = ggml_pool_2d(ctx_cgraph, result, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);\n print_shape(5, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[3]);\n print_shape(6, result);\n result = ggml_pool_2d(ctx_cgraph, result, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);\n print_shape(7, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[4]);\n struct ggml_tensor * layer_8 = result;\n print_shape(8, result);\n result = ggml_pool_2d(ctx_cgraph, result, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);\n print_shape(9, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[5]);\n print_shape(10, result);\n result = ggml_pool_2d(ctx_cgraph, result, GGML_OP_POOL_MAX, 2, 2, 1, 1, 0.5, 0.5);\n print_shape(11, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[6]);\n print_shape(12, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[7]);\n struct ggml_tensor * layer_13 = result;\n print_shape(13, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[8]);\n print_shape(14, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[9]);\n struct ggml_tensor * layer_15 = result;\n ggml_set_output(layer_15);\n ggml_set_name(layer_15, \"layer_15\");\n\n print_shape(15, result);\n result = apply_conv2d(ctx_cgraph, layer_13, model.conv2d_layers[10]);\n print_shape(18, result);\n result = ggml_upscale(ctx_cgraph, result, 2, GGML_SCALE_MODE_NEAREST);\n print_shape(19, result);\n result = ggml_concat(ctx_cgraph, result, layer_8, 2);\n print_shape(20, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[11]);\n print_shape(21, result);\n result = apply_conv2d(ctx_cgraph, result, model.conv2d_layers[12]);\n struct ggml_tensor * layer_22 = result;\n ggml_set_output(layer_22);\n ggml_set_name(layer_22, \"layer_22\");\n print_shape(22, result);\n\n ggml_build_forward_expand(gf, layer_15);\n ggml_build_forward_expand(gf, layer_22);\n return gf;\n}\n\nvoid detect(yolo_image & img, struct ggml_cgraph * gf, const yolo_model & model, float thresh, const std::vector & labels, const std::vector & alphabet)\n{\n std::vector detections;\n yolo_image sized = letterbox_image(img, model.width, model.height);\n struct ggml_tensor * input = ggml_graph_get_tensor(gf, \"input\");\n ggml_backend_tensor_set(input, sized.data.data(), 0, ggml_nbytes(input));\n\n if (ggml_backend_graph_compute(model.backend, gf) != GGML_STATUS_SUCCESS) {\n fprintf(stderr, \"%s: ggml_backend_graph_compute() failed\\n\", __func__);\n return;\n }\n\n struct ggml_tensor * layer_15 = ggml_graph_get_tensor(gf, \"layer_15\");\n yolo_layer yolo16{ 80, {3, 4, 5}, {10, 14, 23, 27, 37,58, 81, 82, 135, 169, 344, 319}, layer_15};\n apply_yolo(yolo16);\n get_yolo_detections(yolo16, detections, img.w, img.h, model.width, model.height, thresh);\n\n struct ggml_tensor * layer_22 = ggml_graph_get_tensor(gf, \"layer_22\");\n yolo_layer yolo23{ 80, {0, 1, 2}, {10, 14, 23, 27, 37,58, 81, 82, 135, 169, 344, 319}, layer_22};\n apply_yolo(yolo23);\n get_yolo_detections(yolo23, detections, img.w, img.h, model.width, model.height, thresh);\n\n do_nms_sort(detections, yolo23.classes, .45);\n draw_detections(img, detections, thresh, labels, alphabet);\n}\n\nstruct yolo_params {\n float thresh = 0.5;\n std::string model = \"yolov3-tiny.gguf\";\n std::string fname_inp = \"input.jpg\";\n std::string fname_out = \"predictions.jpg\";\n int n_threads = std::max(1U, std::thread::hardware_concurrency()/2);\n std::string device;\n};\n\nvoid yolo_print_usage(int argc, char ** argv, const yolo_params & params) {\n fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n fprintf(stderr, \"\\n\");\n fprintf(stderr, \"options:\\n\");\n fprintf(stderr, \" -h, --help show this help message and exit\\n\");\n fprintf(stderr, \" -d, --device DEV device to use\\n\");\n fprintf(stderr, \" -t, --threads N number of threads for the CPU backend (default: %d)\\n\", params.n_threads);\n fprintf(stderr, \" -th, --thresh T detection threshold (default: %.2f)\\n\", params.thresh);\n fprintf(stderr, \" -m, --model FNAME model path (default: %s)\\n\", params.model.c_str());\n fprintf(stderr, \" -i, --inp FNAME input file (default: %s)\\n\", params.fname_inp.c_str());\n fprintf(stderr, \" -o, --out FNAME output file (default: %s)\\n\", params.fname_out.c_str());\n fprintf(stderr, \"\\n\");\n}\n\nbool yolo_params_parse(int argc, char ** argv, yolo_params & params) {\n for (int i = 1; i < argc; i++) {\n std::string arg = argv[i];\n if (arg == \"-th\" || arg == \"--thresh\") {\n params.thresh = std::stof(argv[++i]);\n if (params.thresh < 0 || params.thresh > 1) {\n fprintf(stderr, \"error: invalid threshold: %.2f\\n\", params.thresh);\n return false;\n }\n } else if (arg == \"-m\" || arg == \"--model\") {\n params.model = argv[++i];\n } else if (arg == \"-i\" || arg == \"--inp\") {\n params.fname_inp = argv[++i];\n } else if (arg == \"-o\" || arg == \"--out\") {\n params.fname_out = argv[++i];\n } else if (arg == \"-t\" || arg == \"--threads\") {\n if (++i >= argc) {\n return false;\n }\n params.n_threads = std::stoi(argv[i]);\n if (params.n_threads <= 0) {\n fprintf(stderr, \"error: invalid number of threads: %d\\n\", params.n_threads);\n return false;\n }\n } else if (arg == \"-d\" || arg == \"--device\") {\n if (++i >= argc) {\n return false;\n }\n params.device = argv[i];\n if (ggml_backend_dev_by_name(params.device.c_str()) == nullptr) {\n fprintf(stderr, \"error: unknown device: %s\\n\", params.device.c_str());\n fprintf(stderr, \"available devices:\\n\");\n for (size_t i = 0; i < ggml_backend_dev_count(); i++) {\n auto * dev = ggml_backend_dev_get(i);\n size_t free, total;\n ggml_backend_dev_memory(dev, &free, &total);\n printf(\" %s: %s (%zu MiB, %zu MiB free)\\n\", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);\n }\n return false;\n }\n } else if (arg == \"-h\" || arg == \"--help\") {\n yolo_print_usage(argc, argv, params);\n exit(0);\n } else {\n fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n yolo_print_usage(argc, argv, params);\n exit(0);\n }\n }\n\n return true;\n}\n\nstatic ggml_backend_t create_backend(const yolo_params & params) {\n ggml_backend_t backend = nullptr;\n\n if (!params.device.empty()) {\n ggml_backend_dev_t dev = ggml_backend_dev_by_name(params.device.c_str());\n if (dev) {\n backend = ggml_backend_dev_init(dev, nullptr);\n if (!backend) {\n fprintf(stderr, \"Failed to create backend for device %s\\n\", params.device.c_str());\n return nullptr;\n }\n }\n }\n\n // try to initialize a GPU backend first\n if (!backend) {\n backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr);\n }\n\n // if there aren't GPU backends fallback to CPU backend\n if (!backend) {\n backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);\n }\n\n if (backend) {\n fprintf(stderr, \"%s: using %s backend\\n\", __func__, ggml_backend_name(backend));\n\n // set the number of threads\n ggml_backend_dev_t dev = ggml_backend_get_device(backend);\n ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;\n if (reg) {\n auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, \"ggml_backend_set_n_threads\");\n if (ggml_backend_set_n_threads_fn) {\n ggml_backend_set_n_threads_fn(backend, params.n_threads);\n }\n }\n }\n\n return backend;\n}\n\nint main(int argc, char *argv[])\n{\n ggml_backend_load_all();\n ggml_time_init();\n yolo_model model;\n\n yolo_params params;\n if (!yolo_params_parse(argc, argv, params)) {\n return 1;\n }\n model.backend = create_backend(params);\n if (!model.backend) {\n fprintf(stderr, \"Failed to create backend\\n\");\n return 1;\n }\n\n if (!load_model(params.model, model)) {\n fprintf(stderr, \"%s: failed to load model from '%s'\\n\", __func__, params.model.c_str());\n return 1;\n }\n yolo_image img(0,0,0);\n if (!load_image(params.fname_inp.c_str(), img)) {\n fprintf(stderr, \"%s: failed to load image from '%s'\\n\", __func__, params.fname_inp.c_str());\n return 1;\n }\n std::vector labels;\n if (!load_labels(\"data/coco.names\", labels)) {\n fprintf(stderr, \"%s: failed to load labels from 'data/coco.names'\\n\", __func__);\n return 1;\n }\n std::vector alphabet;\n if (!load_alphabet(alphabet)) {\n fprintf(stderr, \"%s: failed to load alphabet\\n\", __func__);\n return 1;\n }\n\n struct ggml_init_params params0 = {\n /*.mem_size =*/ ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead(),\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()\n };\n struct ggml_context * ctx_cgraph = ggml_init(params0);\n struct ggml_cgraph * gf = build_graph(ctx_cgraph, model);\n\n ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));\n ggml_gallocr_alloc_graph(allocr, gf);\n\n const int64_t t_start_ms = ggml_time_ms();\n detect(img, gf, model, params.thresh, labels, alphabet);\n const int64_t t_detect_ms = ggml_time_ms() - t_start_ms;\n if (!save_image(img, params.fname_out.c_str(), 80)) {\n fprintf(stderr, \"%s: failed to save image to '%s'\\n\", __func__, params.fname_out.c_str());\n return 1;\n }\n printf(\"Detected objects saved in '%s' (time: %f sec.)\\n\", params.fname_out.c_str(), t_detect_ms / 1000.0f);\n\n ggml_free(ctx_cgraph);\n ggml_gallocr_free(allocr);\n ggml_free(model.ctx);\n ggml_backend_buffer_free(model.buffer);\n ggml_backend_free(model.backend);\n return 0;\n}\n" }, { "path": "ggml.pc.in", "content": "prefix=@CMAKE_INSTALL_PREFIX@\nexec_prefix=${prefix}\nincludedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@\nlibdir=${prefix}/@CMAKE_INSTALL_LIBDIR@\n\nName: ggml\nDescription: The GGML Tensor Library for Machine Learning\nVersion: @GGML_VERSION@\nCflags: -I${includedir}\nLibs: -L${libdir} -lggml\n" }, { "path": "include/ggml-alloc.h", "content": "#pragma once\n\n#include \"ggml.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\ntypedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;\ntypedef struct ggml_backend_buffer * ggml_backend_buffer_t;\ntypedef struct ggml_backend * ggml_backend_t;\n\n// Tensor allocator\nstruct ggml_tallocr {\n ggml_backend_buffer_t buffer;\n void * base;\n size_t alignment;\n size_t offset;\n};\n\nGGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer);\nGGML_API enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor);\n\n// Graph allocator\n/*\n Example usage:\n ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type());\n\n // optional: create a worst-case graph and reserve the buffers to avoid reallocations\n ggml_gallocr_reserve(galloc, build_graph(max_batch));\n\n // allocate the graph\n struct ggml_cgraph * graph = build_graph(batch);\n ggml_gallocr_alloc_graph(galloc, graph);\n\n printf(\"compute buffer size: %zu bytes\\n\", ggml_gallocr_get_buffer_size(galloc, 0));\n\n // evaluate the graph\n ggml_backend_graph_compute(backend, graph);\n*/\n\n// special tensor flags for use with the graph allocator:\n// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses\n// ggml_set_output(): output tensors are never freed and never overwritten\n\ntypedef struct ggml_gallocr * ggml_gallocr_t;\n\nGGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft);\nGGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs);\nGGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);\n\n// pre-allocate buffers from a measure graph - does not allocate or modify the graph\n// call with a worst-case graph to avoid buffer reallocations\n// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed\n// returns false if the buffer allocation failed\n// ggml_gallocr_resrve_n_size writes the buffer sizes per galloc buffer that would be allocated by ggml_gallocr_reserve_n to sizes\nGGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph);\nGGML_API void ggml_gallocr_reserve_n_size(\n ggml_gallocr_t galloc,\n struct ggml_cgraph * graph,\n const int * node_buffer_ids,\n const int * leaf_buffer_ids,\n size_t * sizes);\nGGML_API bool ggml_gallocr_reserve_n(\n ggml_gallocr_t galloc,\n struct ggml_cgraph * graph,\n const int * node_buffer_ids,\n const int * leaf_buffer_ids);\n\n// automatic reallocation if the topology changes when using a single buffer\n// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers)\nGGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph);\n\nGGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id);\n\n// Utils\n// Create a buffer and allocate all the tensors in a ggml_context\n// ggml_backend_alloc_ctx_tensors_from_buft_size returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft\nGGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);\nGGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);\nGGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-backend.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-alloc.h\"\n\n#ifdef GGML_BACKEND_SHARED\n# if defined(_WIN32) && !defined(__MINGW32__)\n# ifdef GGML_BACKEND_BUILD\n# define GGML_BACKEND_API __declspec(dllexport) extern\n# else\n# define GGML_BACKEND_API __declspec(dllimport) extern\n# endif\n# else\n# define GGML_BACKEND_API __attribute__ ((visibility (\"default\"))) extern\n# endif\n#else\n# define GGML_BACKEND_API extern\n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;\n typedef struct ggml_backend_buffer * ggml_backend_buffer_t;\n typedef struct ggml_backend_event * ggml_backend_event_t;\n typedef struct ggml_backend * ggml_backend_t;\n typedef void * ggml_backend_graph_plan_t;\n typedef struct ggml_backend_reg * ggml_backend_reg_t;\n typedef struct ggml_backend_device * ggml_backend_dev_t;\n\n\n //\n // Backend buffer type\n //\n\n GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);\n GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);\n GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);\n GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);\n GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);\n GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);\n GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft);\n\n //\n // Backend buffer\n //\n\n enum ggml_backend_buffer_usage {\n GGML_BACKEND_BUFFER_USAGE_ANY = 0,\n GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,\n GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,\n };\n\n GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);\n GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);\n GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);\n GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);\n GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);\n GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);\n GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);\n GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor);\n GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);\n GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);\n GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);\n GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer);\n GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);\n GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);\n\n // tensor copy between different backends\n GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);\n\n //\n // Backend (stream)\n //\n\n GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);\n GGML_API const char * ggml_backend_name(ggml_backend_t backend);\n GGML_API void ggml_backend_free(ggml_backend_t backend);\n\n GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);\n GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);\n GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);\n GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);\n\n GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);\n\n // \"offset\" refers to the offset in tensor->data for setting/getting data\n GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);\n GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);\n\n GGML_API void ggml_backend_synchronize(ggml_backend_t backend);\n\n GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);\n GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);\n\n GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);\n GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);\n GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);\n\n // NOTE: will be removed, use device version instead\n GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);\n GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);\n GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);\n\n // asynchronous copy\n // the copy is performed after all the currently queued operations in backend_src\n // backend_dst will wait for the copy to complete before performing other operations\n // automatic fallback to sync copy if async is not supported\n GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);\n\n GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend);\n\n //\n // Events\n //\n\n GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device);\n GGML_API void ggml_backend_event_free(ggml_backend_event_t event);\n GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend);\n GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);\n GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event);\n\n //\n // Backend device\n //\n\n enum ggml_backend_dev_type {\n // CPU device using system memory\n GGML_BACKEND_DEVICE_TYPE_CPU,\n // GPU device using dedicated memory\n GGML_BACKEND_DEVICE_TYPE_GPU,\n // integrated GPU device using host memory\n GGML_BACKEND_DEVICE_TYPE_IGPU,\n // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)\n GGML_BACKEND_DEVICE_TYPE_ACCEL\n };\n\n // functionality supported by the device\n struct ggml_backend_dev_caps {\n // asynchronous operations\n bool async;\n // pinned host buffer\n bool host_buffer;\n // creating buffers from host ptr\n bool buffer_from_host_ptr;\n // event synchronization\n bool events;\n };\n\n // all the device properties\n struct ggml_backend_dev_props {\n // device name\n const char * name;\n // device description\n const char * description;\n // device free memory in bytes\n size_t memory_free;\n // device total memory in bytes\n size_t memory_total;\n // device type\n enum ggml_backend_dev_type type;\n // device id\n // for PCI devices, this should be the PCI bus id formatted as \"domain:bus:device.function\" (e.g. \"0000:01:00.0\")\n // if the id is unknown, this should be NULL\n const char * device_id;\n // device capabilities\n struct ggml_backend_dev_caps caps;\n };\n\n GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device);\n GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device);\n GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total);\n GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device);\n GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props);\n GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device);\n GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params);\n GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device);\n GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device);\n GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size);\n\n GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);\n GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);\n GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);\n\n //\n // Backend (reg)\n //\n\n GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg);\n GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg);\n GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index);\n GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name);\n\n // Common functions that may be obtained using ggml_backend_reg_get_proc_address\n\n // Split buffer type for tensor parallelism\n typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);\n // Set the number of threads for the backend\n typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);\n // Get additional buffer types provided by the device (returns a NULL-terminated array)\n typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);\n // Set the abort callback for the backend\n typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);\n // Get a list of feature flags supported by the backend (returns a NULL-terminated array)\n struct ggml_backend_feature {\n const char * name;\n const char * value;\n };\n typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);\n\n //\n // Backend registry\n //\n\n GGML_API void ggml_backend_register(ggml_backend_reg_t reg);\n\n GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);\n\n // Backend (reg) enumeration\n GGML_API size_t ggml_backend_reg_count(void);\n GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index);\n GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name);\n\n // Device enumeration\n GGML_API size_t ggml_backend_dev_count(void);\n GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index);\n GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name);\n GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type);\n\n // Direct backend (stream) initialization\n // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params)\n GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params);\n // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params)\n GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params);\n // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)\n GGML_API ggml_backend_t ggml_backend_init_best(void);\n\n // Load a backend from a dynamic library and register it\n GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);\n // Unload a backend if loaded dynamically and unregister it\n GGML_API void ggml_backend_unload(ggml_backend_reg_t reg);\n // Load all known backends from dynamic libraries\n GGML_API void ggml_backend_load_all(void);\n GGML_API void ggml_backend_load_all_from_path(const char * dir_path);\n\n //\n // Backend scheduler\n //\n\n // The backend scheduler allows for multiple backend devices to be used together\n // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends\n // The backends are selected based on:\n // - the backend that supports the operation\n // - the location of the pre-allocated tensors (e.g. the weights)\n /*\n Example usage:\n\n // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned\n // preferably to run on the same backend as the buffer\n ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);\n\n sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true);\n\n // initialize buffers from a max size graph (optional)\n reserve_graph = build_graph(sched, max_batch_size);\n\n // manually assign nodes to a backend (optional, should not be needed in most cases)\n struct ggml_tensor * node = ggml_mul_mat(ctx, ...);\n ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);\n\n ggml_backend_sched_reserve(sched, reserve_graph);\n\n // compute\n graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation\n for (int i = 0; i < 10; ++i) {\n ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically\n }\n\n // if there are graph inputs:\n graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called)\n ggml_backend_sched_reset(sched); // clear the allocation of the previous graph\n ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it\n ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors\n ggml_backend_sched_graph_compute(sched, graph); // execute the graph\n\n // as an alternative to the above it is also possible to assign the inputs to a dedicated context and\n // allocate them statically via ggml_backend_alloc_ctx_tensors\n }\n */\n\n typedef struct ggml_backend_sched * ggml_backend_sched_t;\n\n // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback)\n // when ask == true, the scheduler wants to know if the user wants to observe this node\n // this allows the scheduler to batch nodes together in order to evaluate them in a single call\n //\n // when ask == false, the scheduler is passing the node tensor to the user for observation\n // if the user returns false, the scheduler will cancel the graph compute\n //\n typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);\n\n // Initialize a backend scheduler, backends with low index are given priority over backends with high index\n GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload);\n GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);\n\n // Initialize backend buffers from a measure graph\n GGML_API void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes);\n GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success\n\n GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched);\n GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i);\n\n // Get the number of splits of the last graph\n GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);\n GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);\n\n GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend);\n GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);\n\n GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);\n GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);\n\n // Split graph without allocating it\n GGML_API void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);\n\n // Allocate and compute graph on the backend scheduler\n GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success\n GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);\n GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);\n GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);\n\n // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.\n // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.\n // The correct way to use this API is to discard the deallocated tensors and create new ones.\n GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);\n\n // Set a callback to be called for each resulting node during graph compute\n GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);\n\n //\n // Utils\n //\n\n struct ggml_backend_graph_copy {\n ggml_backend_buffer_t buffer;\n struct ggml_context * ctx_allocated;\n struct ggml_context * ctx_unallocated;\n struct ggml_cgraph * graph;\n };\n\n // Copy a graph to a different backend\n GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);\n GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);\n\n typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);\n\n // Compare the output of two backends\n GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes);\n\n // Tensor initialization\n GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);\n GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor);\n\n // CPU buffer types are always available\n GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);\n GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-blas.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_blas_init(void);\n\nGGML_BACKEND_API bool ggml_backend_is_blas(ggml_backend_t backend);\n\n// number of threads used for conversion to float\n// for openblas and blis, this will also set the number of threads used for blas operations\nGGML_BACKEND_API void ggml_backend_blas_set_n_threads(ggml_backend_t backend_blas, int n_threads);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_blas_reg(void);\n\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-cann.h", "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#pragma once\n\n#include \"ggml-backend.h\"\n#include \"ggml.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n/**\n * @brief Maximum number of CANN devices supported.\n */\n#define GGML_CANN_MAX_DEVICES 16\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_cann_reg(void);\n\n/**\n * @brief Initializes the CANN backend for a specified device.\n *\n * This function initializes the CANN backend for the given device.\n * It verifies the device index, allocates a context, and creates a backend\n * instance.\n *\n * @param device The index of the device to initialize.\n * @return A pointer to the initialized backend instance, or nullptr on failure.\n */\nGGML_BACKEND_API ggml_backend_t ggml_backend_cann_init(int32_t device);\n\n/**\n * @brief Checks if a given backend is a CANN backend.\n *\n * This function verifies if the provided backend is a CANN backend by comparing\n * its GUID with the CANN backend's GUID.\n *\n * @param backend The backend instance to check.\n * @return True if the backend is a CANN backend, false otherwise.\n */\nGGML_BACKEND_API bool ggml_backend_is_cann(ggml_backend_t backend);\n\n/**\n * @brief Retrieves the CANN buffer type for a specified device.\n *\n * This function initializes and returns the buffer type interface associated\n * with the given device. It ensures thread-safe access using a mutex.\n *\n * @param device The device index for which to retrieve the buffer type.\n * @return A pointer to the buffer type interface for the specified device, or\n * nullptr if the device index is out of range.\n */\nGGML_BACKEND_API ggml_backend_buffer_type_t\nggml_backend_cann_buffer_type(int32_t device);\n\n/**\n * @brief Retrieves the number of CANN devices available.\n *\n * This function returns the number of CANN devices available based on\n * information obtained from `ggml_cann_info()`.\n *\n * @return The number of CANN devices available.\n */\nGGML_BACKEND_API int32_t ggml_backend_cann_get_device_count(void);\n\n/**\n * @brief pinned host buffer for use with the CPU backend for faster copies between CPU and NPU.\n *\n * @return A pointer to the host buffer type interface.\n */\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cann_host_buffer_type(void);\n\n/**\n * @brief Retrieves the description of a specific CANN device.\n *\n * This function sets the specified device, retrieves the SoC name,\n * and writes it into the provided description buffer.\n *\n * @param device The device index to retrieve the description for.\n * @param description Pointer to a buffer where the description will be written.\n * @param description_size Size of the description buffer.\n */\nGGML_BACKEND_API void ggml_backend_cann_get_device_description(\n int32_t device, char* description, size_t description_size);\n\n/**\n * @brief Retrieves the memory information of a specific CANN device.\n *\n * This function sets the specified device, retrieves the free and total\n * memory information of the specified type (ACL_HBM_MEM), and stores them\n * in the provided pointers.\n *\n * @param device The device index to retrieve memory information for.\n * @param free Pointer to a variable where the free memory size will be stored.\n * @param total Pointer to a variable where the total memory size will be\n * stored.\n */\nGGML_BACKEND_API void ggml_backend_cann_get_device_memory(int32_t device,\n size_t* free,\n size_t* total);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-cpp.h", "content": "#pragma once\n\n#ifndef __cplusplus\n#error \"This header is for C++ only\"\n#endif\n\n#include \"ggml.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n#include \"gguf.h\"\n#include \n\n// Smart pointers for ggml types\n\n// ggml\n\nstruct ggml_context_deleter { void operator()(ggml_context * ctx) { ggml_free(ctx); } };\nstruct gguf_context_deleter { void operator()(gguf_context * ctx) { gguf_free(ctx); } };\n\ntypedef std::unique_ptr ggml_context_ptr;\ntypedef std::unique_ptr gguf_context_ptr;\n\n// ggml-alloc\n\nstruct ggml_gallocr_deleter { void operator()(ggml_gallocr_t galloc) { ggml_gallocr_free(galloc); } };\n\ntypedef std::unique_ptr ggml_gallocr_ptr;\n\n// ggml-backend\n\nstruct ggml_backend_deleter { void operator()(ggml_backend_t backend) { ggml_backend_free(backend); } };\nstruct ggml_backend_buffer_deleter { void operator()(ggml_backend_buffer_t buffer) { ggml_backend_buffer_free(buffer); } };\nstruct ggml_backend_event_deleter { void operator()(ggml_backend_event_t event) { ggml_backend_event_free(event); } };\nstruct ggml_backend_sched_deleter { void operator()(ggml_backend_sched_t sched) { ggml_backend_sched_free(sched); } };\n\ntypedef std::unique_ptr ggml_backend_ptr;\ntypedef std::unique_ptr ggml_backend_buffer_ptr;\ntypedef std::unique_ptr ggml_backend_event_ptr;\ntypedef std::unique_ptr ggml_backend_sched_ptr;\n" }, { "path": "include/ggml-cpu.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n // the compute plan that needs to be prepared for ggml_graph_compute()\n // since https://github.com/ggml-org/ggml/issues/287\n struct ggml_cplan {\n size_t work_size; // size of work buffer, calculated by `ggml_graph_plan()`\n uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()`\n\n int n_threads;\n struct ggml_threadpool * threadpool;\n\n // abort ggml_graph_compute when true\n ggml_abort_callback abort_callback;\n void * abort_callback_data;\n\n // use only reference implementations\n bool use_ref;\n };\n\n // numa strategies\n enum ggml_numa_strategy {\n GGML_NUMA_STRATEGY_DISABLED = 0,\n GGML_NUMA_STRATEGY_DISTRIBUTE = 1,\n GGML_NUMA_STRATEGY_ISOLATE = 2,\n GGML_NUMA_STRATEGY_NUMACTL = 3,\n GGML_NUMA_STRATEGY_MIRROR = 4,\n GGML_NUMA_STRATEGY_COUNT\n };\n\n GGML_BACKEND_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems\n GGML_BACKEND_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node\n\n GGML_BACKEND_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value);\n GGML_BACKEND_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value);\n\n GGML_BACKEND_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);\n GGML_BACKEND_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);\n\n GGML_BACKEND_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i);\n GGML_BACKEND_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value);\n\n GGML_BACKEND_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);\n GGML_BACKEND_API void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);\n\n GGML_BACKEND_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i);\n GGML_BACKEND_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value);\n\n GGML_BACKEND_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);\n GGML_BACKEND_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);\n\n GGML_BACKEND_API struct ggml_threadpool * ggml_threadpool_new (struct ggml_threadpool_params * params);\n GGML_BACKEND_API void ggml_threadpool_free (struct ggml_threadpool * threadpool);\n GGML_BACKEND_API int ggml_threadpool_get_n_threads (struct ggml_threadpool * threadpool);\n GGML_BACKEND_API void ggml_threadpool_pause (struct ggml_threadpool * threadpool);\n GGML_BACKEND_API void ggml_threadpool_resume (struct ggml_threadpool * threadpool);\n\n // ggml_graph_plan() has to be called before ggml_graph_compute()\n // when plan.work_size > 0, caller must allocate memory for plan.work_data\n GGML_BACKEND_API struct ggml_cplan ggml_graph_plan(\n const struct ggml_cgraph * cgraph,\n int n_threads, /* = GGML_DEFAULT_N_THREADS */\n struct ggml_threadpool * threadpool /* = NULL */ );\n GGML_BACKEND_API enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);\n\n // same as ggml_graph_compute() but the work data is allocated as a part of the context\n // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data\n GGML_BACKEND_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);\n\n //\n // system info\n //\n\n // x86\n GGML_BACKEND_API int ggml_cpu_has_sse3 (void);\n GGML_BACKEND_API int ggml_cpu_has_ssse3 (void);\n GGML_BACKEND_API int ggml_cpu_has_avx (void);\n GGML_BACKEND_API int ggml_cpu_has_avx_vnni (void);\n GGML_BACKEND_API int ggml_cpu_has_avx2 (void);\n GGML_BACKEND_API int ggml_cpu_has_bmi2 (void);\n GGML_BACKEND_API int ggml_cpu_has_f16c (void);\n GGML_BACKEND_API int ggml_cpu_has_fma (void);\n GGML_BACKEND_API int ggml_cpu_has_avx512 (void);\n GGML_BACKEND_API int ggml_cpu_has_avx512_vbmi(void);\n GGML_BACKEND_API int ggml_cpu_has_avx512_vnni(void);\n GGML_BACKEND_API int ggml_cpu_has_avx512_bf16(void);\n GGML_BACKEND_API int ggml_cpu_has_amx_int8 (void);\n // ARM\n GGML_BACKEND_API int ggml_cpu_has_neon (void);\n GGML_BACKEND_API int ggml_cpu_has_arm_fma (void);\n GGML_BACKEND_API int ggml_cpu_has_fp16_va (void);\n GGML_BACKEND_API int ggml_cpu_has_dotprod (void);\n GGML_BACKEND_API int ggml_cpu_has_matmul_int8(void);\n GGML_BACKEND_API int ggml_cpu_has_sve (void);\n GGML_BACKEND_API int ggml_cpu_get_sve_cnt (void); // sve vector length in bytes\n GGML_BACKEND_API int ggml_cpu_has_sme (void);\n // other\n GGML_BACKEND_API int ggml_cpu_has_riscv_v (void);\n GGML_BACKEND_API int ggml_cpu_get_rvv_vlen (void); // risc-v vector length in bytes\n GGML_BACKEND_API int ggml_cpu_has_vsx (void);\n GGML_BACKEND_API int ggml_cpu_has_vxe (void);\n GGML_BACKEND_API int ggml_cpu_has_wasm_simd (void);\n GGML_BACKEND_API int ggml_cpu_has_llamafile (void);\n\n // Internal types and functions exposed for tests and benchmarks\n\n typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx,\n const void * GGML_RESTRICT y, size_t by, int nrc);\n\n struct ggml_type_traits_cpu {\n ggml_from_float_t from_float;\n ggml_vec_dot_t vec_dot;\n enum ggml_type vec_dot_type;\n int64_t nrows; // number of rows to process simultaneously\n };\n\n GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type);\n\n GGML_BACKEND_API void ggml_cpu_init(void);\n\n //\n // CPU backend\n //\n\n GGML_BACKEND_API ggml_backend_t ggml_backend_cpu_init(void);\n\n GGML_BACKEND_API bool ggml_backend_is_cpu (ggml_backend_t backend);\n GGML_BACKEND_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads);\n GGML_BACKEND_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool);\n GGML_BACKEND_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);\n\n GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref);\n\n GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cpu_reg(void);\n\n GGML_BACKEND_API void ggml_cpu_fp32_to_fp32(const float *, float *, int64_t);\n GGML_BACKEND_API void ggml_cpu_fp32_to_i32 (const float *, int32_t *, int64_t);\n GGML_BACKEND_API void ggml_cpu_fp32_to_fp16(const float *, ggml_fp16_t *, int64_t);\n GGML_BACKEND_API void ggml_cpu_fp16_to_fp32(const ggml_fp16_t *, float *, int64_t);\n GGML_BACKEND_API void ggml_cpu_fp32_to_bf16(const float *, ggml_bf16_t *, int64_t);\n GGML_BACKEND_API void ggml_cpu_bf16_to_fp32(const ggml_bf16_t *, float *, int64_t);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-cuda.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#ifdef GGML_USE_HIP\n#define GGML_CUDA_NAME \"ROCm\"\n#define GGML_CUBLAS_NAME \"hipBLAS\"\n#elif defined(GGML_USE_MUSA)\n#define GGML_CUDA_NAME \"MUSA\"\n#define GGML_CUBLAS_NAME \"muBLAS\"\n#else\n#define GGML_CUDA_NAME \"CUDA\"\n#define GGML_CUBLAS_NAME \"cuBLAS\"\n#endif\n#define GGML_CUDA_MAX_DEVICES 16\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_cuda_init(int device);\n\nGGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend);\n\n// device buffer\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);\n\n// split tensor buffer that splits matrices by rows across multiple devices\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split);\n\n// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);\n\nGGML_BACKEND_API int ggml_backend_cuda_get_device_count(void);\nGGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);\nGGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);\n\nGGML_BACKEND_API bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);\nGGML_BACKEND_API void ggml_backend_cuda_unregister_host_buffer(void * buffer);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_cuda_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-hexagon.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_hexagon_init(void);\n\nGGML_BACKEND_API bool ggml_backend_is_hexagon(ggml_backend_t backend);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_hexagon_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-metal.h", "content": "// Note: this description is outdated\n//\n// An interface allowing to compute ggml_cgraph with Metal\n//\n// This is a fully functional interface that extends ggml with GPU support for Apple devices.\n// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.)\n//\n// How it works?\n//\n// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this\n// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you\n// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.)\n//\n// You only need to make sure that all memory buffers that you used during the graph creation\n// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is\n// used during the graph evaluation to determine the arguments of the compute kernels.\n//\n// Synchronization between device and host memory (for example for input and output tensors)\n// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions.\n//\n\n#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#include \n#include \n\nstruct ggml_tensor;\nstruct ggml_cgraph;\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n//\n// backend API\n// user-code should use only these functions\n//\n\n// TODO: remove in the future\nGGML_BACKEND_API ggml_backend_t ggml_backend_metal_init(void);\n\nGGML_BACKEND_API bool ggml_backend_is_metal(ggml_backend_t backend);\n\nGGML_BACKEND_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data);\n\n// helper to check if the device supports a specific family\n// ideally, the user code should be doing these checks\n// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf\nGGML_BACKEND_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family);\n\n// capture all command buffers committed the next time `ggml_backend_graph_compute` is called\nGGML_BACKEND_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_metal_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-opencl.h", "content": "#ifndef GGML_OPENCL_H\n#define GGML_OPENCL_H\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n//\n// backend API\n//\nGGML_BACKEND_API ggml_backend_t ggml_backend_opencl_init(void);\nGGML_BACKEND_API bool ggml_backend_is_opencl(ggml_backend_t backend);\n\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_buffer_type(void);\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_host_buffer_type(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_opencl_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif // GGML_OPENCL_H\n" }, { "path": "include/ggml-openvino.h", "content": "#pragma once\n\n#include \"ggml-backend.h\"\n\n#include \n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#define GGML_OPENVINO_NAME \"OPENVINO\"\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_openvino_init(int device);\n\nGGML_BACKEND_API bool ggml_backend_is_openvino(ggml_backend_t backend);\n\nGGML_BACKEND_API bool ggml_backend_buffer_is_openvino(ggml_backend_buffer_t buffer);\n\nGGML_BACKEND_API bool ggml_backend_buft_is_openvino(ggml_backend_buffer_type_t buft);\n\nGGML_BACKEND_API bool ggml_backend_buft_is_openvino_host(ggml_backend_buffer_type_t buft);\n\nGGML_BACKEND_API size_t ggml_backend_openvino_buffer_get_ctx_id(ggml_backend_buffer_t buffer);\n\n// device buffer\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_buffer_type(int device);\n\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_host_buffer_type(int device);\n\nGGML_BACKEND_API int ggml_backend_openvino_get_device_count(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_openvino_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-opt.h", "content": "// This file contains functionality for training models using GGML.\n// It is not strictly needed vs. just vanilla GGML but it provides a more high-level interface for common needs such as datasets.\n// At the bottom of this file especially there are relatively high-level functions that are suitable use or adaptation in user code.\n//\n// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de)\n\n#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#include \n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n struct ggml_opt_dataset;\n struct ggml_opt_context;\n struct ggml_opt_result;\n\n typedef struct ggml_opt_dataset * ggml_opt_dataset_t;\n typedef struct ggml_opt_context * ggml_opt_context_t;\n typedef struct ggml_opt_result * ggml_opt_result_t;\n\n // ====== Loss ======\n\n // built-in loss types, i.e. the built-in quantities minimized by the optimizer\n // custom loss types can be defined via mean or sum which simply reduce the outputs for all datapoints to a single value\n enum ggml_opt_loss_type {\n GGML_OPT_LOSS_TYPE_MEAN,\n GGML_OPT_LOSS_TYPE_SUM,\n GGML_OPT_LOSS_TYPE_CROSS_ENTROPY,\n GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR,\n };\n\n // ====== Dataset ======\n\n GGML_API ggml_opt_dataset_t ggml_opt_dataset_init(\n enum ggml_type type_data, // the type for the internal data tensor\n enum ggml_type type_label, // the type for the internal labels tensor\n int64_t ne_datapoint, // number of elements per datapoint\n int64_t ne_label, // number of elements per label\n int64_t ndata, // total number of datapoints/labels\n int64_t ndata_shard); // number of datapoints/labels per shard (unit at which the dataset is shuffled/copied)\n GGML_API void ggml_opt_dataset_free(ggml_opt_dataset_t dataset);\n\n // get underlying tensors that store the data\n GGML_API int64_t ggml_opt_dataset_ndata (ggml_opt_dataset_t dataset);\n GGML_API struct ggml_tensor * ggml_opt_dataset_data (ggml_opt_dataset_t dataset); // shape = [ne_datapoint, ndata]\n GGML_API struct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset); // shape = [nd_label, ndata]\n\n // shuffle idata first datapoints from dataset with RNG from opt_ctx, shuffle all datapoints if idata is negative\n GGML_API void ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata);\n\n // get batch at position ibatch from dataset and copy the data to data_batch and labels_batch\n GGML_API void ggml_opt_dataset_get_batch(\n ggml_opt_dataset_t dataset,\n struct ggml_tensor * data_batch, // shape = [ne_datapoint, ndata_batch]\n struct ggml_tensor * labels_batch, // shape = [ne_label, ndata_batch]\n int64_t ibatch);\n GGML_API void ggml_opt_dataset_get_batch_host(\n ggml_opt_dataset_t dataset,\n void * data_batch,\n size_t nb_data_batch,\n void * labels_batch,\n int64_t ibatch);\n\n // ====== Model / Context ======\n\n enum ggml_opt_build_type {\n GGML_OPT_BUILD_TYPE_FORWARD = 10,\n GGML_OPT_BUILD_TYPE_GRAD = 20,\n GGML_OPT_BUILD_TYPE_OPT = 30,\n };\n\n enum ggml_opt_optimizer_type {\n GGML_OPT_OPTIMIZER_TYPE_ADAMW,\n GGML_OPT_OPTIMIZER_TYPE_SGD,\n\n GGML_OPT_OPTIMIZER_TYPE_COUNT\n };\n\n // parameters that control which optimizer is used and how said optimizer tries to find the minimal loss\n struct ggml_opt_optimizer_params {\n struct {\n float alpha; // learning rate\n float beta1; // first AdamW momentum\n float beta2; // second AdamW momentum\n float eps; // epsilon for numerical stability\n float wd; // weight decay - 0.0f to disable\n } adamw;\n struct {\n float alpha; // learning rate\n float wd; // weight decay\n } sgd;\n };\n\n // callback to calculate optimizer parameters prior to a backward pass\n // userdata can be used to pass arbitrary data\n typedef struct ggml_opt_optimizer_params (*ggml_opt_get_optimizer_params)(void * userdata);\n\n // returns the default optimizer params (constant, hard-coded values)\n // userdata is not used\n GGML_API struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata);\n\n // casts userdata to ggml_opt_optimizer_params and returns it\n GGML_API struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata);\n\n // parameters for initializing a new optimization context\n struct ggml_opt_params {\n ggml_backend_sched_t backend_sched; // defines which backends are used to construct the compute graphs\n\n // by default the forward graph needs to be reconstructed for each eval\n // if ctx_compute, inputs, and outputs are set the graphs are instead allocated statically\n struct ggml_context * ctx_compute;\n struct ggml_tensor * inputs;\n struct ggml_tensor * outputs;\n\n enum ggml_opt_loss_type loss_type;\n enum ggml_opt_build_type build_type;\n\n int32_t opt_period; // after how many gradient accumulation steps an optimizer step should be done\n\n ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters\n void * get_opt_pars_ud; // userdata for calculating optimizer parameters\n\n // only GGML_OPT_OPTIMIZER_TYPE_ADAMW needs m, v momenta per parameter tensor\n enum ggml_opt_optimizer_type optimizer;\n };\n\n // get parameters for an optimization context with defaults set where possible\n // parameters for which no sensible defaults exist are supplied as arguments to this function\n GGML_API struct ggml_opt_params ggml_opt_default_params(\n ggml_backend_sched_t backend_sched,\n enum ggml_opt_loss_type loss_type);\n\n GGML_API ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params);\n GGML_API void ggml_opt_free(ggml_opt_context_t opt_ctx);\n\n // set gradients to zero, initialize loss, and optionally reset the optimizer\n GGML_API void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer);\n\n GGML_API bool ggml_opt_static_graphs(ggml_opt_context_t opt_ctx); // whether the graphs are allocated_statically\n\n // get underlying tensors that store data\n // if not using static graphs these pointers become invalid with the next call to ggml_opt_alloc\n GGML_API struct ggml_tensor * ggml_opt_inputs( ggml_opt_context_t opt_ctx); // forward graph input tensor\n GGML_API struct ggml_tensor * ggml_opt_outputs( ggml_opt_context_t opt_ctx); // forward graph output tensor\n GGML_API struct ggml_tensor * ggml_opt_labels( ggml_opt_context_t opt_ctx); // labels to compare outputs against\n GGML_API struct ggml_tensor * ggml_opt_loss( ggml_opt_context_t opt_ctx); // scalar tensor that contains the loss\n GGML_API struct ggml_tensor * ggml_opt_pred( ggml_opt_context_t opt_ctx); // predictions made by outputs\n GGML_API struct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx); // number of matching predictions between outputs and labels\n\n // get the gradient accumulator for a node from the forward graph\n GGML_API struct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node);\n\n GGML_API enum ggml_opt_optimizer_type ggml_opt_context_optimizer_type(ggml_opt_context_t); //TODO consistent naming scheme\n\n GGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer_type);\n\n // ====== Optimization Result ======\n\n GGML_API ggml_opt_result_t ggml_opt_result_init(void);\n GGML_API void ggml_opt_result_free(ggml_opt_result_t result);\n GGML_API void ggml_opt_result_reset(ggml_opt_result_t result);\n\n // get data from result, uncertainties are optional and can be ignored by passing NULL\n GGML_API void ggml_opt_result_ndata( ggml_opt_result_t result, int64_t * ndata); // writes 1 value, number of datapoints\n GGML_API void ggml_opt_result_loss( ggml_opt_result_t result, double * loss, double * unc); // writes 1 value\n GGML_API void ggml_opt_result_pred( ggml_opt_result_t result, int32_t * pred); // writes ndata values\n GGML_API void ggml_opt_result_accuracy(ggml_opt_result_t result, double * accuracy, double * unc); // writes 1 value\n\n // ====== Computation ======\n\n // if not using static graphs, this function must be called prior to ggml_opt_alloc\n GGML_API void ggml_opt_prepare_alloc(\n ggml_opt_context_t opt_ctx,\n struct ggml_context * ctx_compute,\n struct ggml_cgraph * gf,\n struct ggml_tensor * inputs,\n struct ggml_tensor * outputs);\n\n // allocate the next graph for evaluation, either forward or forward + backward\n // must be called exactly once prior to calling ggml_opt_eval\n GGML_API void ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward);\n\n // do forward pass, increment result if not NULL, do backward pass if allocated\n GGML_API void ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result);\n\n // ############################################################################\n // ## The high-level functions start here. They do not depend on any private ##\n // ## functions or structs and can be copied to and adapted for user code. ##\n // ############################################################################\n\n // ====== Intended Usage ======\n //\n // 1. Select the appropriate loss for your problem.\n // 2. Create a dataset and set the data for the \"data\" tensor. Also set the \"labels\" tensor if your loss needs them.\n // Setting the shard size to 1 will be fine, it's the granularity with which data is shuffled/loaded (bigger values are faster).\n // 3. Create a GGML graph for your model with no_alloc == true. Use two separate contexts for the tensors.\n // The first context should contain the model parameters and inputs and be allocated statically in user code.\n // The second context should contain all other tensors and will be (re)allocated automatically.\n // Due to this automated allocation the data of the second context is not defined when accessed in user code.\n // Note that the second dimension of the inputs/outputs are interpreted as the number of datapoints in those tensors.\n // 4. Call ggml_opt_fit. If you need more control you can use ggml_opt_epoch instead.\n\n // signature for a callback while evaluating opt_ctx on dataset, called after an evaluation\n typedef void (*ggml_opt_epoch_callback)(\n bool train, // true after training evaluation, false after validation evaluation\n ggml_opt_context_t opt_ctx,\n ggml_opt_dataset_t dataset,\n ggml_opt_result_t result, // result associated with the dataset subsection\n int64_t ibatch, // number of batches that have been evaluated so far\n int64_t ibatch_max, // total number of batches in this dataset subsection\n int64_t t_start_us); // time at which the evaluation on the dataset subsection was started\n\n // do training on front of dataset, do evaluation only on back of dataset\n GGML_API void ggml_opt_epoch(\n ggml_opt_context_t opt_ctx,\n ggml_opt_dataset_t dataset,\n ggml_opt_result_t result_train, // result to increment during training, ignored if NULL\n ggml_opt_result_t result_eval, // result to increment during evaluation, ignored if NULL\n int64_t idata_split, // data index at which to split training and evaluation\n ggml_opt_epoch_callback callback_train,\n ggml_opt_epoch_callback callback_eval);\n\n // callback that prints a progress bar on stderr\n GGML_API void ggml_opt_epoch_callback_progress_bar(\n bool train,\n ggml_opt_context_t opt_ctx,\n ggml_opt_dataset_t dataset,\n ggml_opt_result_t result,\n int64_t ibatch,\n int64_t ibatch_max,\n int64_t t_start_us);\n\n // fit model defined by inputs and outputs to dataset\n GGML_API void ggml_opt_fit(\n ggml_backend_sched_t backend_sched, // backend scheduler for constructing the compute graphs\n struct ggml_context * ctx_compute, // context with temporarily allocated tensors to calculate the outputs\n struct ggml_tensor * inputs, // input tensor with shape [ne_datapoint, ndata_batch]\n struct ggml_tensor * outputs, // output tensor, must have shape [ne_label, ndata_batch] if labels are used\n ggml_opt_dataset_t dataset, // dataset with data and optionally also labels\n enum ggml_opt_loss_type loss_type, // loss to minimize\n enum ggml_opt_optimizer_type optimizer, // sgd or adamw\n ggml_opt_get_optimizer_params get_opt_pars, // callback to get optimizer params, userdata is pointer to epoch (of type int64_t)\n int64_t nepoch, // how many times the dataset should be iterated over\n int64_t nbatch_logical, // datapoints optimizer step, must be a multiple of ndata_batch in inputs/outputs\n float val_split, // fraction of the dataset to use for validation, must be in [0.0f, 1.0f)\n bool silent); // whether or not info prints to stderr should be suppressed\n\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-rpc.h", "content": "#pragma once\n\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#define RPC_PROTO_MAJOR_VERSION 3\n#define RPC_PROTO_MINOR_VERSION 6\n#define RPC_PROTO_PATCH_VERSION 1\n\n#ifdef __cplusplus\nstatic_assert(GGML_OP_COUNT == 96, \"GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION\");\n#endif\n\n#define GGML_RPC_MAX_SERVERS 16\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device);\nGGML_BACKEND_API bool ggml_backend_is_rpc(ggml_backend_t backend);\n\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, uint32_t device);\n\nGGML_BACKEND_API void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total);\n\nGGML_BACKEND_API void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir,\n size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_reg(void);\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-sycl.h", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#define GGML_SYCL_NAME \"SYCL\"\n#define GGML_SYCL_MAX_DEVICES 48\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_sycl_init(int device);\n\nGGML_BACKEND_API bool ggml_backend_is_sycl(ggml_backend_t backend);\n\n// devide buffer\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device);\n\n// split tensor buffer that splits matrices by rows across multiple devices\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split);\n\n// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type(void);\n\nGGML_BACKEND_API void ggml_backend_sycl_print_sycl_devices(void);\nGGML_BACKEND_API void ggml_backend_sycl_get_gpu_list(int *id_list, int max_len);\nGGML_BACKEND_API void ggml_backend_sycl_get_device_description(int device,\n char *description,\n size_t description_size);\nGGML_BACKEND_API int ggml_backend_sycl_get_device_count();\nGGML_BACKEND_API void ggml_backend_sycl_get_device_memory(int device, size_t *free, size_t *total);\n\n// SYCL doesn't support registering host memory, keep here for reference\n// GGML_BACKEND_API bool ggml_backend_sycl_register_host_buffer(void * buffer, size_t size);\n// GGML_BACKEND_API void ggml_backend_sycl_unregister_host_buffer(void * buffer);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_sycl_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-virtgpu.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_virtgpu_reg();\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-vulkan.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#define GGML_VK_NAME \"Vulkan\"\n#define GGML_VK_MAX_DEVICES 16\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_vk_init(size_t dev_num);\n\nGGML_BACKEND_API bool ggml_backend_is_vk(ggml_backend_t backend);\nGGML_BACKEND_API int ggml_backend_vk_get_device_count(void);\nGGML_BACKEND_API void ggml_backend_vk_get_device_description(int device, char * description, size_t description_size);\nGGML_BACKEND_API void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total);\n\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num);\n// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_vk_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-webgpu.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#define GGML_WEBGPU_NAME \"WebGPU\"\n\n// Needed for examples in ggml\nGGML_BACKEND_API ggml_backend_t ggml_backend_webgpu_init(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_webgpu_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-zdnn.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// device buffer\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_zdnn_buffer_type(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_zdnn_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml-zendnn.h", "content": "#pragma once\n\n#include \"ggml-backend.h\"\n#include \"ggml.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_zendnn_init(void);\n\nGGML_BACKEND_API bool ggml_backend_is_zendnn(ggml_backend_t backend);\n\n// number of threads used for zendnn operations\nGGML_BACKEND_API void ggml_backend_zendnn_set_n_threads(ggml_backend_t backend_zendnn, int n_threads);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_zendnn_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/ggml.h", "content": "#pragma once\n\n//\n// GGML Tensor Library\n//\n// This documentation is still a work in progress.\n// If you wish some specific topics to be covered, feel free to drop a comment:\n//\n// https://github.com/ggml-org/whisper.cpp/issues/40\n//\n// ## Overview\n//\n// This library implements:\n//\n// - a set of tensor operations\n// - automatic differentiation\n// - basic optimization algorithms\n//\n// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,\n// but is not limited to, the following:\n//\n// - linear regression\n// - support vector machines\n// - neural networks\n//\n// The library allows the user to define a certain function using the available tensor operations. This function\n// definition is represented internally via a computation graph. Each tensor operation in the function definition\n// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the\n// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized\n// using one of the available optimization algorithms.\n//\n// For example, here we define the function: f(x) = a*x^2 + b\n//\n// {\n// struct ggml_init_params params = {\n// .mem_size = 16*1024*1024,\n// .mem_buffer = NULL,\n// };\n//\n// // memory allocation happens here\n// struct ggml_context * ctx = ggml_init(params);\n//\n// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);\n//\n// ggml_set_param(ctx, x); // x is an input variable\n//\n// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);\n// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);\n// struct ggml_tensor * x2 = ggml_mul(ctx, x, x);\n// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b);\n//\n// ...\n// }\n//\n// Notice that the function definition above does not involve any actual computation. The computation is performed only\n// when the user explicitly requests it. For example, to compute the function's value at x = 2.0:\n//\n// {\n// ...\n//\n// struct ggml_cgraph * gf = ggml_new_graph(ctx);\n// ggml_build_forward_expand(gf, f);\n//\n// // set the input variable and parameter values\n// ggml_set_f32(x, 2.0f);\n// ggml_set_f32(a, 3.0f);\n// ggml_set_f32(b, 4.0f);\n//\n// ggml_graph_compute_with_ctx(ctx, &gf, n_threads);\n//\n// printf(\"f = %f\\n\", ggml_get_f32_1d(f, 0));\n//\n// ...\n// }\n//\n// The actual computation is performed in the ggml_graph_compute() function.\n//\n// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the\n// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know\n// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory\n// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was\n// actually needed.\n//\n// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic\n// differentiation and optimization algorithms.\n//\n// The described approach allows to define the function graph once and then compute its forward or backward graphs\n// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way\n// the user can avoid the memory allocation overhead at runtime.\n//\n// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class\n// citizens, but in theory the library can be extended to support FP8 and integer data types.\n//\n// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary\n// and binary operations. Most of the available operations fall into one of these two categories. With time, it became\n// clear that the library needs to support more complex operations. The way to support these operations is not clear\n// yet, but a few examples are demonstrated in the following operations:\n//\n// - ggml_permute()\n// - ggml_conv_1d_1s()\n// - ggml_conv_1d_2s()\n//\n// For each tensor operator, the library implements a forward and backward computation function. The forward function\n// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the\n// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a\n// calculus class, or watch the following video:\n//\n// What is Automatic Differentiation?\n// https://www.youtube.com/watch?v=wG_nF1awSSY\n//\n//\n// ## Tensor data (struct ggml_tensor)\n//\n// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of\n// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains\n// pointers to the \"source\" tensors - i.e. the tensors that were used to compute the current tensor. For example:\n//\n// {\n// struct ggml_tensor * c = ggml_add(ctx, a, b);\n//\n// assert(c->src[0] == a);\n// assert(c->src[1] == b);\n// }\n//\n// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the\n// number of elements in each dimension (\"ne\") as well as the number of bytes (\"nb\", a.k.a. stride). This allows\n// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and\n// permutation. All tensor operations have to take the stride into account and not assume that the tensor is\n// contiguous in memory.\n//\n// The data of the tensor is accessed via the \"data\" pointer. For example:\n//\n// {\n// const int nx = 2;\n// const int ny = 3;\n//\n// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);\n//\n// for (int y = 0; y < ny; y++) {\n// for (int x = 0; x < nx; x++) {\n// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;\n// }\n// }\n//\n// ...\n// }\n//\n// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.\n//\n// ## The matrix multiplication operator (ggml_mul_mat)\n//\n// TODO\n//\n//\n// ## Multi-threading\n//\n// TODO\n//\n//\n// ## Overview of ggml.c\n//\n// TODO\n//\n//\n// ## SIMD optimizations\n//\n// TODO\n//\n//\n// ## Debugging ggml\n//\n// TODO\n//\n//\n\n#ifdef GGML_SHARED\n# if defined(_WIN32) && !defined(__MINGW32__)\n# ifdef GGML_BUILD\n# define GGML_API __declspec(dllexport) extern\n# else\n# define GGML_API __declspec(dllimport) extern\n# endif\n# else\n# define GGML_API __attribute__ ((visibility (\"default\"))) extern\n# endif\n#else\n# define GGML_API extern\n#endif\n\n// TODO: support for clang\n#ifdef __GNUC__\n# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))\n#elif defined(_MSC_VER)\n# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func\n#else\n# define GGML_DEPRECATED(func, hint) func\n#endif\n\n#ifndef __GNUC__\n# define GGML_ATTRIBUTE_FORMAT(...)\n#elif defined(__MINGW32__) && !defined(__clang__)\n# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))\n#else\n# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))\n#endif\n\n#if defined(_WIN32) && !defined(_WIN32_WINNT)\n# define _WIN32_WINNT 0x0A00\n#endif\n\n#include \n#include \n#include \n#include \n\n#define GGML_FILE_MAGIC 0x67676d6c // \"ggml\"\n#define GGML_FILE_VERSION 2\n\n#define GGML_QNT_VERSION 2 // bump this on quantization format changes\n#define GGML_QNT_VERSION_FACTOR 1000 // do not change this\n\n#define GGML_MAX_DIMS 4\n#define GGML_MAX_PARAMS 2048\n#define GGML_MAX_SRC 10\n#define GGML_MAX_N_THREADS 512\n#define GGML_MAX_OP_PARAMS 64\n\n#ifndef GGML_MAX_NAME\n# define GGML_MAX_NAME 64\n#endif\n\n#define GGML_DEFAULT_N_THREADS 4\n#define GGML_DEFAULT_GRAPH_SIZE 2048\n\n#if UINTPTR_MAX == 0xFFFFFFFF\n #define GGML_MEM_ALIGN 4\n#elif defined(__EMSCRIPTEN__)\n// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm.\n// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.)\n// ref: https://github.com/ggml-org/llama.cpp/pull/18628\n #define GGML_MEM_ALIGN 8\n#else\n #define GGML_MEM_ALIGN 16\n#endif\n\n#define GGML_EXIT_SUCCESS 0\n#define GGML_EXIT_ABORTED 1\n\n// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726\n#define GGML_ROPE_TYPE_NORMAL 0\n#define GGML_ROPE_TYPE_NEOX 2\n#define GGML_ROPE_TYPE_MROPE 8\n#define GGML_ROPE_TYPE_VISION 24\n#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000\n\n#define GGML_MROPE_SECTIONS 4\n\n#define GGML_UNUSED(x) (void)(x)\n#ifdef __CUDACC__\ntemplate\n__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {}\n#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__)\n#else\n#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0)\n#endif // __CUDACC__\n\n#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))\n\n#ifndef NDEBUG\n# define GGML_UNREACHABLE() do { fprintf(stderr, \"statement should be unreachable\\n\"); abort(); } while(0)\n#elif defined(__GNUC__)\n# define GGML_UNREACHABLE() __builtin_unreachable()\n#elif defined(_MSC_VER)\n# define GGML_UNREACHABLE() __assume(0)\n#else\n# define GGML_UNREACHABLE() ((void) 0)\n#endif\n\n#ifdef __cplusplus\n# define GGML_NORETURN [[noreturn]]\n#elif defined(_MSC_VER)\n# define GGML_NORETURN __declspec(noreturn)\n#else\n# define GGML_NORETURN _Noreturn\n#endif\n\n#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__)\n#define GGML_ASSERT(x) if (!(x)) GGML_ABORT(\"GGML_ASSERT(%s) failed\", #x)\n\n// used to copy the number of elements and stride in bytes of tensors into local variables.\n// main purpose is to reduce code duplication and improve readability.\n//\n// example:\n//\n// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);\n// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);\n//\n#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \\\n const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \\\n GGML_UNUSED(prefix##0);\n#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \\\n GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \\\n const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \\\n GGML_UNUSED(prefix##1);\n#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \\\n GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \\\n const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \\\n GGML_UNUSED(prefix##2);\n#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \\\n GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \\\n const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \\\n GGML_UNUSED(prefix##3);\n\n#define GGML_TENSOR_UNARY_OP_LOCALS \\\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \\\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n#define GGML_TENSOR_BINARY_OP_LOCALS \\\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \\\n GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \\\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n#define GGML_TENSOR_TERNARY_OP_LOCALS \\\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \\\n GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \\\n GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb2, src2, nb) \\\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n#define GGML_TENSOR_BINARY_OP_LOCALS01 \\\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \\\n GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \\\n GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n // Function type used in fatal error callbacks\n typedef void (*ggml_abort_callback_t)(const char * error_message);\n\n // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout)\n // Returns the old callback for chaining\n GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback);\n\n GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4)\n GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...);\n\n enum ggml_status {\n GGML_STATUS_ALLOC_FAILED = -2,\n GGML_STATUS_FAILED = -1,\n GGML_STATUS_SUCCESS = 0,\n GGML_STATUS_ABORTED = 1,\n };\n\n // get ggml_status name string\n GGML_API const char * ggml_status_to_string(enum ggml_status status);\n\n // ieee 754-2008 half-precision float16\n // todo: make this not an integral type\n typedef uint16_t ggml_fp16_t;\n GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);\n GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);\n GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);\n GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);\n\n // google brain half-precision bfloat16\n typedef struct { uint16_t bits; } ggml_bf16_t;\n GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);\n GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16\n GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);\n GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);\n GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);\n\n struct ggml_object;\n struct ggml_context;\n struct ggml_cgraph;\n\n // NOTE: always add types at the end of the enum to keep backward compatibility\n enum ggml_type {\n GGML_TYPE_F32 = 0,\n GGML_TYPE_F16 = 1,\n GGML_TYPE_Q4_0 = 2,\n GGML_TYPE_Q4_1 = 3,\n // GGML_TYPE_Q4_2 = 4, support has been removed\n // GGML_TYPE_Q4_3 = 5, support has been removed\n GGML_TYPE_Q5_0 = 6,\n GGML_TYPE_Q5_1 = 7,\n GGML_TYPE_Q8_0 = 8,\n GGML_TYPE_Q8_1 = 9,\n GGML_TYPE_Q2_K = 10,\n GGML_TYPE_Q3_K = 11,\n GGML_TYPE_Q4_K = 12,\n GGML_TYPE_Q5_K = 13,\n GGML_TYPE_Q6_K = 14,\n GGML_TYPE_Q8_K = 15,\n GGML_TYPE_IQ2_XXS = 16,\n GGML_TYPE_IQ2_XS = 17,\n GGML_TYPE_IQ3_XXS = 18,\n GGML_TYPE_IQ1_S = 19,\n GGML_TYPE_IQ4_NL = 20,\n GGML_TYPE_IQ3_S = 21,\n GGML_TYPE_IQ2_S = 22,\n GGML_TYPE_IQ4_XS = 23,\n GGML_TYPE_I8 = 24,\n GGML_TYPE_I16 = 25,\n GGML_TYPE_I32 = 26,\n GGML_TYPE_I64 = 27,\n GGML_TYPE_F64 = 28,\n GGML_TYPE_IQ1_M = 29,\n GGML_TYPE_BF16 = 30,\n // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files\n // GGML_TYPE_Q4_0_4_8 = 32,\n // GGML_TYPE_Q4_0_8_8 = 33,\n GGML_TYPE_TQ1_0 = 34,\n GGML_TYPE_TQ2_0 = 35,\n // GGML_TYPE_IQ4_NL_4_4 = 36,\n // GGML_TYPE_IQ4_NL_4_8 = 37,\n // GGML_TYPE_IQ4_NL_8_8 = 38,\n GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)\n GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)\n GGML_TYPE_COUNT = 41,\n };\n\n // precision\n enum ggml_prec {\n GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default\n GGML_PREC_F32 = 10,\n };\n\n // model file types\n enum ggml_ftype {\n GGML_FTYPE_UNKNOWN = -1,\n GGML_FTYPE_ALL_F32 = 0,\n GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16\n GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors\n GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors\n GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors\n GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors\n GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors\n GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors\n GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors\n GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors\n GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors\n GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors\n GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors\n GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors\n GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors\n GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors\n };\n\n // available tensor operations:\n enum ggml_op {\n GGML_OP_NONE = 0,\n\n GGML_OP_DUP,\n GGML_OP_ADD,\n GGML_OP_ADD_ID,\n GGML_OP_ADD1,\n GGML_OP_ACC,\n GGML_OP_SUB,\n GGML_OP_MUL,\n GGML_OP_DIV,\n GGML_OP_SQR,\n GGML_OP_SQRT,\n GGML_OP_LOG,\n GGML_OP_SIN,\n GGML_OP_COS,\n GGML_OP_SUM,\n GGML_OP_SUM_ROWS,\n GGML_OP_CUMSUM,\n GGML_OP_MEAN,\n GGML_OP_ARGMAX,\n GGML_OP_COUNT_EQUAL,\n GGML_OP_REPEAT,\n GGML_OP_REPEAT_BACK,\n GGML_OP_CONCAT,\n GGML_OP_SILU_BACK,\n GGML_OP_NORM, // normalize\n GGML_OP_RMS_NORM,\n GGML_OP_RMS_NORM_BACK,\n GGML_OP_GROUP_NORM,\n GGML_OP_L2_NORM,\n\n GGML_OP_MUL_MAT,\n GGML_OP_MUL_MAT_ID,\n GGML_OP_OUT_PROD,\n\n GGML_OP_SCALE,\n GGML_OP_SET,\n GGML_OP_CPY,\n GGML_OP_CONT,\n GGML_OP_RESHAPE,\n GGML_OP_VIEW,\n GGML_OP_PERMUTE,\n GGML_OP_TRANSPOSE,\n GGML_OP_GET_ROWS,\n GGML_OP_GET_ROWS_BACK,\n GGML_OP_SET_ROWS,\n GGML_OP_DIAG,\n GGML_OP_DIAG_MASK_INF,\n GGML_OP_DIAG_MASK_ZERO,\n GGML_OP_SOFT_MAX,\n GGML_OP_SOFT_MAX_BACK,\n GGML_OP_ROPE,\n GGML_OP_ROPE_BACK,\n GGML_OP_CLAMP,\n GGML_OP_CONV_TRANSPOSE_1D,\n GGML_OP_IM2COL,\n GGML_OP_IM2COL_BACK,\n GGML_OP_IM2COL_3D,\n GGML_OP_CONV_2D,\n GGML_OP_CONV_3D,\n GGML_OP_CONV_2D_DW,\n GGML_OP_CONV_TRANSPOSE_2D,\n GGML_OP_POOL_1D,\n GGML_OP_POOL_2D,\n GGML_OP_POOL_2D_BACK,\n GGML_OP_UPSCALE,\n GGML_OP_PAD,\n GGML_OP_PAD_REFLECT_1D,\n GGML_OP_ROLL,\n GGML_OP_ARANGE,\n GGML_OP_TIMESTEP_EMBEDDING,\n GGML_OP_ARGSORT,\n GGML_OP_TOP_K,\n GGML_OP_LEAKY_RELU,\n GGML_OP_TRI,\n GGML_OP_FILL,\n\n GGML_OP_FLASH_ATTN_EXT,\n GGML_OP_FLASH_ATTN_BACK,\n GGML_OP_SSM_CONV,\n GGML_OP_SSM_SCAN,\n GGML_OP_WIN_PART,\n GGML_OP_WIN_UNPART,\n GGML_OP_GET_REL_POS,\n GGML_OP_ADD_REL_POS,\n GGML_OP_RWKV_WKV6,\n GGML_OP_GATED_LINEAR_ATTN,\n GGML_OP_RWKV_WKV7,\n GGML_OP_SOLVE_TRI,\n GGML_OP_GATED_DELTA_NET,\n\n GGML_OP_UNARY,\n\n GGML_OP_MAP_CUSTOM1,\n GGML_OP_MAP_CUSTOM2,\n GGML_OP_MAP_CUSTOM3,\n\n GGML_OP_CUSTOM,\n\n GGML_OP_CROSS_ENTROPY_LOSS,\n GGML_OP_CROSS_ENTROPY_LOSS_BACK,\n GGML_OP_OPT_STEP_ADAMW,\n GGML_OP_OPT_STEP_SGD,\n\n GGML_OP_GLU,\n\n GGML_OP_COUNT,\n };\n\n enum ggml_unary_op {\n GGML_UNARY_OP_ABS,\n GGML_UNARY_OP_SGN,\n GGML_UNARY_OP_NEG,\n GGML_UNARY_OP_STEP,\n GGML_UNARY_OP_TANH,\n GGML_UNARY_OP_ELU,\n GGML_UNARY_OP_RELU,\n GGML_UNARY_OP_SIGMOID,\n GGML_UNARY_OP_GELU,\n GGML_UNARY_OP_GELU_QUICK,\n GGML_UNARY_OP_SILU,\n GGML_UNARY_OP_HARDSWISH,\n GGML_UNARY_OP_HARDSIGMOID,\n GGML_UNARY_OP_EXP,\n GGML_UNARY_OP_EXPM1,\n GGML_UNARY_OP_SOFTPLUS,\n GGML_UNARY_OP_GELU_ERF,\n GGML_UNARY_OP_XIELU,\n GGML_UNARY_OP_FLOOR,\n GGML_UNARY_OP_CEIL,\n GGML_UNARY_OP_ROUND,\n GGML_UNARY_OP_TRUNC,\n\n GGML_UNARY_OP_COUNT,\n };\n\n enum ggml_glu_op {\n GGML_GLU_OP_REGLU,\n GGML_GLU_OP_GEGLU,\n GGML_GLU_OP_SWIGLU,\n GGML_GLU_OP_SWIGLU_OAI,\n GGML_GLU_OP_GEGLU_ERF,\n GGML_GLU_OP_GEGLU_QUICK,\n\n GGML_GLU_OP_COUNT,\n };\n\n enum ggml_object_type {\n GGML_OBJECT_TYPE_TENSOR,\n GGML_OBJECT_TYPE_GRAPH,\n GGML_OBJECT_TYPE_WORK_BUFFER\n };\n\n enum ggml_log_level {\n GGML_LOG_LEVEL_NONE = 0,\n GGML_LOG_LEVEL_DEBUG = 1,\n GGML_LOG_LEVEL_INFO = 2,\n GGML_LOG_LEVEL_WARN = 3,\n GGML_LOG_LEVEL_ERROR = 4,\n GGML_LOG_LEVEL_CONT = 5, // continue previous log\n };\n\n // this tensor...\n enum ggml_tensor_flag {\n GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph\n GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph\n GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters\n GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up)\n GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed\n };\n\n enum ggml_tri_type {\n GGML_TRI_TYPE_UPPER_DIAG = 0,\n GGML_TRI_TYPE_UPPER = 1,\n GGML_TRI_TYPE_LOWER_DIAG = 2,\n GGML_TRI_TYPE_LOWER = 3\n };\n\n struct ggml_init_params {\n // memory pool\n size_t mem_size; // bytes\n void * mem_buffer; // if NULL, memory will be allocated internally\n bool no_alloc; // don't allocate memory for the tensor data\n };\n\n // n-dimensional tensor\n struct ggml_tensor {\n enum ggml_type type;\n\n struct ggml_backend_buffer * buffer;\n\n int64_t ne[GGML_MAX_DIMS]; // number of elements\n size_t nb[GGML_MAX_DIMS]; // stride in bytes:\n // nb[0] = ggml_type_size(type)\n // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding\n // nb[i] = nb[i-1] * ne[i-1]\n\n // compute data\n enum ggml_op op;\n\n // op params - allocated as int32_t for alignment\n int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];\n\n int32_t flags;\n\n struct ggml_tensor * src[GGML_MAX_SRC];\n\n // source tensor and offset for views\n struct ggml_tensor * view_src;\n size_t view_offs;\n\n void * data;\n\n char name[GGML_MAX_NAME];\n\n void * extra; // extra things e.g. for ggml-cuda.cu\n\n char padding[8];\n };\n\n static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);\n\n // Abort callback\n // If not NULL, called before ggml computation\n // If it returns true, the computation is aborted\n typedef bool (*ggml_abort_callback)(void * data);\n\n\n //\n // GUID\n //\n\n // GUID types\n typedef uint8_t ggml_guid[16];\n typedef ggml_guid * ggml_guid_t;\n\n GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);\n\n // misc\n\n GGML_API const char * ggml_version(void);\n GGML_API const char * ggml_commit(void);\n\n GGML_API void ggml_time_init(void); // call this once at the beginning of the program\n GGML_API int64_t ggml_time_ms(void);\n GGML_API int64_t ggml_time_us(void);\n GGML_API int64_t ggml_cycles(void);\n GGML_API int64_t ggml_cycles_per_ms(void);\n\n // accepts a UTF-8 path, even on Windows\n GGML_API FILE * ggml_fopen(const char * fname, const char * mode);\n\n GGML_API void ggml_print_object (const struct ggml_object * obj);\n GGML_API void ggml_print_objects(const struct ggml_context * ctx);\n\n GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor);\n GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor);\n GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor);\n GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN\n\n GGML_API int64_t ggml_blck_size(enum ggml_type type);\n GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block\n GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row\n\n GGML_DEPRECATED(\n GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float\n \"use ggml_row_size() instead\");\n\n GGML_API const char * ggml_type_name(enum ggml_type type);\n GGML_API const char * ggml_op_name (enum ggml_op op);\n GGML_API const char * ggml_op_symbol(enum ggml_op op);\n\n GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);\n GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op);\n GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name\n\n GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);\n\n GGML_API bool ggml_is_quantized(enum ggml_type type);\n\n // TODO: temporary until model loading of ggml examples is refactored\n GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);\n\n GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);\n GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor);\n GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor);\n GGML_API bool ggml_is_view (const struct ggml_tensor * tensor);\n GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);\n GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);\n GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);\n GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);\n GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars\n\n // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation)\n GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor);\n GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()\n GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1\n GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2\n\n // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok)\n GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor);\n\n // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN\n GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor);\n\n // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements\n GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor);\n\n GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);\n GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);\n\n GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1);\n\n // use this to compute the memory overhead of a tensor\n GGML_API size_t ggml_tensor_overhead(void);\n\n GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);\n\n // main\n\n GGML_API struct ggml_context * ggml_init (struct ggml_init_params params);\n GGML_API void ggml_reset(struct ggml_context * ctx);\n GGML_API void ggml_free (struct ggml_context * ctx);\n\n GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);\n\n GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);\n GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);\n\n GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);\n GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);\n GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);\n\n GGML_API struct ggml_tensor * ggml_new_tensor(\n struct ggml_context * ctx,\n enum ggml_type type,\n int n_dims,\n const int64_t *ne);\n\n GGML_API struct ggml_tensor * ggml_new_tensor_1d(\n struct ggml_context * ctx,\n enum ggml_type type,\n int64_t ne0);\n\n GGML_API struct ggml_tensor * ggml_new_tensor_2d(\n struct ggml_context * ctx,\n enum ggml_type type,\n int64_t ne0,\n int64_t ne1);\n\n GGML_API struct ggml_tensor * ggml_new_tensor_3d(\n struct ggml_context * ctx,\n enum ggml_type type,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2);\n\n GGML_API struct ggml_tensor * ggml_new_tensor_4d(\n struct ggml_context * ctx,\n enum ggml_type type,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3);\n\n GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes);\n\n GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);\n GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);\n\n // Context tensor enumeration and lookup\n GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);\n GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);\n GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);\n\n // Converts a flat index into coordinates\n GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);\n\n GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);\n GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor);\n\n GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);\n GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);\n\n GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);\n GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);\n GGML_ATTRIBUTE_FORMAT(2, 3)\n GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);\n\n // Tensor flags\n GGML_API void ggml_set_input(struct ggml_tensor * tensor);\n GGML_API void ggml_set_output(struct ggml_tensor * tensor);\n GGML_API void ggml_set_param(struct ggml_tensor * tensor);\n GGML_API void ggml_set_loss(struct ggml_tensor * tensor);\n\n //\n // operations on tensors with backpropagation\n //\n\n GGML_API struct ggml_tensor * ggml_dup(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // in-place, returns view(a)\n GGML_API struct ggml_tensor * ggml_dup_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_add(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_add_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_add_cast(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n enum ggml_type type);\n\n // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]]\n GGML_API struct ggml_tensor * ggml_add_id(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * ids);\n\n GGML_API struct ggml_tensor * ggml_add1(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_add1_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n // dst = a\n // view(dst, nb1, nb2, nb3, offset) += b\n // return dst\n GGML_API struct ggml_tensor * ggml_acc(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t nb2,\n size_t nb3,\n size_t offset);\n\n GGML_API struct ggml_tensor * ggml_acc_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t nb2,\n size_t nb3,\n size_t offset);\n\n GGML_API struct ggml_tensor * ggml_sub(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_sub_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_mul(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_mul_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_div(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_div_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_sqr(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_sqr_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_sqrt(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_sqrt_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_log(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_log_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_expm1(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_expm1_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_softplus(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_softplus_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_sin(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_sin_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_cos(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_cos_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // return scalar\n GGML_API struct ggml_tensor * ggml_sum(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]\n GGML_API struct ggml_tensor * ggml_sum_rows(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_cumsum(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // mean along rows\n GGML_API struct ggml_tensor * ggml_mean(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // argmax along rows\n GGML_API struct ggml_tensor * ggml_argmax(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // count number of equal elements in a and b\n GGML_API struct ggml_tensor * ggml_count_equal(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n // if a is the same shape as b, and a is not parameter, return a\n // otherwise, return a new tensor: repeat(a) to fit in b\n GGML_API struct ggml_tensor * ggml_repeat(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n // repeat a to the specified shape\n GGML_API struct ggml_tensor * ggml_repeat_4d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3);\n\n // sums repetitions in a into shape of b\n GGML_API struct ggml_tensor * ggml_repeat_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride\n\n // concat a and b along dim\n // used in stable-diffusion\n GGML_API struct ggml_tensor * ggml_concat(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int dim);\n\n GGML_API struct ggml_tensor * ggml_abs(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_abs_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_sgn(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_sgn_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_neg(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_neg_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_step(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_step_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_tanh(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_tanh_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_elu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_elu_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_relu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_leaky_relu(\n struct ggml_context * ctx,\n struct ggml_tensor * a, float negative_slope, bool inplace);\n\n GGML_API struct ggml_tensor * ggml_relu_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_sigmoid(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_sigmoid_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_gelu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_gelu_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // GELU using erf (error function) when possible\n // some backends may fallback to approximation based on Abramowitz and Stegun formula\n GGML_API struct ggml_tensor * ggml_gelu_erf(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_gelu_erf_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_gelu_quick(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_silu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_silu_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // a - x\n // b - dy\n GGML_API struct ggml_tensor * ggml_silu_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n // hardswish(x) = x * relu6(x + 3) / 6\n GGML_API struct ggml_tensor * ggml_hardswish(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // hardsigmoid(x) = relu6(x + 3) / 6\n GGML_API struct ggml_tensor * ggml_hardsigmoid(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_exp(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_exp_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_floor(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_floor_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_ceil(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_ceil_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_round(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_round_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n /**\n * Truncates the fractional part of each element in the tensor (towards zero).\n * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0\n * Similar to std::trunc in C/C++.\n */\n\n GGML_API struct ggml_tensor * ggml_trunc(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_trunc_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n\n\n // xIELU activation function\n // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0)\n // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions\n // that constrain the positive and negative source alpha values respectively\n GGML_API struct ggml_tensor * ggml_xielu(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float alpha_n,\n float alpha_p,\n float beta,\n float eps);\n\n // gated linear unit ops\n // A: n columns, r rows,\n // result is n / 2 columns, r rows,\n // expects gate in second half of row, unless swapped is true\n GGML_API struct ggml_tensor * ggml_glu(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_glu_op op,\n bool swapped);\n\n GGML_API struct ggml_tensor * ggml_reglu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_reglu_swapped(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_geglu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_geglu_swapped(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_swiglu(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_swiglu_swapped(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_geglu_erf(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_geglu_erf_swapped(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_geglu_quick(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n GGML_API struct ggml_tensor * ggml_geglu_quick_swapped(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // A: n columns, r rows,\n // B: n columns, r rows,\n GGML_API struct ggml_tensor * ggml_glu_split(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n enum ggml_glu_op op);\n\n GGML_API struct ggml_tensor * ggml_reglu_split(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_geglu_split(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_swiglu_split(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_geglu_erf_split(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_geglu_quick_split(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n GGML_API struct ggml_tensor * ggml_swiglu_oai(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n float alpha,\n float limit);\n\n // normalize along rows\n GGML_API struct ggml_tensor * ggml_norm(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float eps);\n\n GGML_API struct ggml_tensor * ggml_norm_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float eps);\n\n GGML_API struct ggml_tensor * ggml_rms_norm(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float eps);\n\n GGML_API struct ggml_tensor * ggml_rms_norm_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float eps);\n\n // group normalize along ne0*ne1*n_groups\n // used in stable-diffusion\n GGML_API struct ggml_tensor * ggml_group_norm(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_groups,\n float eps);\n\n GGML_API struct ggml_tensor * ggml_group_norm_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_groups,\n float eps);\n\n // l2 normalize along rows\n // used in rwkv v7\n GGML_API struct ggml_tensor * ggml_l2_norm(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float eps);\n\n GGML_API struct ggml_tensor * ggml_l2_norm_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float eps);\n\n // a - x\n // b - dy\n GGML_API struct ggml_tensor * ggml_rms_norm_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n float eps);\n\n // A: k columns, n rows => [ne03, ne02, n, k]\n // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]\n // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]\n GGML_API struct ggml_tensor * ggml_mul_mat(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n // change the precision of a matrix multiplication\n // set to GGML_PREC_F32 for higher precision (useful for phi-2)\n GGML_API void ggml_mul_mat_set_prec(\n struct ggml_tensor * a,\n enum ggml_prec prec);\n\n // indirect matrix multiplication\n GGML_API struct ggml_tensor * ggml_mul_mat_id(\n struct ggml_context * ctx,\n struct ggml_tensor * as,\n struct ggml_tensor * b,\n struct ggml_tensor * ids);\n\n // A: m columns, n rows,\n // B: p columns, n rows,\n // result is m columns, p rows\n GGML_API struct ggml_tensor * ggml_out_prod(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n //\n // operations on tensors without backpropagation\n //\n\n GGML_API struct ggml_tensor * ggml_scale(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float s);\n\n // in-place, returns view(a)\n GGML_API struct ggml_tensor * ggml_scale_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float s);\n\n // x = s * a + b\n GGML_API struct ggml_tensor * ggml_scale_bias(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float s,\n float b);\n\n GGML_API struct ggml_tensor * ggml_scale_bias_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float s,\n float b);\n\n // b -> view(a,offset,nb1,nb2,3), return modified a\n GGML_API struct ggml_tensor * ggml_set(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t nb2,\n size_t nb3,\n size_t offset); // in bytes\n\n // b -> view(a,offset,nb1,nb2,3), return view(a)\n GGML_API struct ggml_tensor * ggml_set_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t nb2,\n size_t nb3,\n size_t offset); // in bytes\n\n GGML_API struct ggml_tensor * ggml_set_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t offset); // in bytes\n\n GGML_API struct ggml_tensor * ggml_set_1d_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t offset); // in bytes\n\n // b -> view(a,offset,nb1,nb2,3), return modified a\n GGML_API struct ggml_tensor * ggml_set_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t offset); // in bytes\n\n // b -> view(a,offset,nb1,nb2,3), return view(a)\n GGML_API struct ggml_tensor * ggml_set_2d_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n size_t nb1,\n size_t offset); // in bytes\n\n // a -> b, return view(b)\n GGML_API struct ggml_tensor * ggml_cpy(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n // note: casting from f32 to i32 will discard the fractional part\n GGML_API struct ggml_tensor * ggml_cast(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_type type);\n\n // make contiguous\n GGML_API struct ggml_tensor * ggml_cont(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // make contiguous, with new shape\n GGML_API struct ggml_tensor * ggml_cont_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0);\n\n GGML_API struct ggml_tensor * ggml_cont_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1);\n\n GGML_API struct ggml_tensor * ggml_cont_3d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2);\n\n GGML_API struct ggml_tensor * ggml_cont_4d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3);\n\n // return view(a), b specifies the new shape\n // TODO: when we start computing gradient, make a copy instead of view\n GGML_API struct ggml_tensor * ggml_reshape(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n // return view(a)\n // TODO: when we start computing gradient, make a copy instead of view\n GGML_API struct ggml_tensor * ggml_reshape_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0);\n\n GGML_API struct ggml_tensor * ggml_reshape_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1);\n\n // return view(a)\n // TODO: when we start computing gradient, make a copy instead of view\n GGML_API struct ggml_tensor * ggml_reshape_3d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2);\n\n GGML_API struct ggml_tensor * ggml_reshape_4d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3);\n\n // offset in bytes\n GGML_API struct ggml_tensor * ggml_view_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n size_t offset);\n\n GGML_API struct ggml_tensor * ggml_view_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n size_t nb1, // row stride in bytes\n size_t offset);\n\n GGML_API struct ggml_tensor * ggml_view_3d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n size_t nb1, // row stride in bytes\n size_t nb2, // slice stride in bytes\n size_t offset);\n\n GGML_API struct ggml_tensor * ggml_view_4d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3,\n size_t nb1, // row stride in bytes\n size_t nb2, // slice stride in bytes\n size_t nb3,\n size_t offset);\n\n GGML_API struct ggml_tensor * ggml_permute(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int axis0,\n int axis1,\n int axis2,\n int axis3);\n\n // alias for ggml_permute(ctx, a, 1, 0, 2, 3)\n GGML_API struct ggml_tensor * ggml_transpose(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // supports 4D a:\n // a [n_embd, ne1, ne2, ne3]\n // b I32 [n_rows, ne2, ne3, 1]\n //\n // return [n_embd, n_rows, ne2, ne3]\n GGML_API struct ggml_tensor * ggml_get_rows(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // data\n struct ggml_tensor * b); // row indices\n\n GGML_API struct ggml_tensor * ggml_get_rows_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // gradients of ggml_get_rows result\n struct ggml_tensor * b, // row indices\n struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape\n\n // a TD [n_embd, ne1, ne2, ne3]\n // b TS [n_embd, n_rows, ne02, ne03] | ne02 == ne2, ne03 == ne3\n // c I64 [n_rows, ne11, ne12, 1] | c[i] in [0, ne1)\n //\n // undefined behavior if destination rows overlap\n //\n // broadcast:\n // ne2 % ne11 == 0\n // ne3 % ne12 == 0\n //\n // return view(a)\n GGML_API struct ggml_tensor * ggml_set_rows(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // destination\n struct ggml_tensor * b, // source\n struct ggml_tensor * c); // row indices\n\n GGML_API struct ggml_tensor * ggml_diag(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // set elements above the diagonal to -INF\n GGML_API struct ggml_tensor * ggml_diag_mask_inf(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past);\n\n // in-place, returns view(a)\n GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past);\n\n // set elements above the diagonal to 0\n GGML_API struct ggml_tensor * ggml_diag_mask_zero(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past);\n\n // in-place, returns view(a)\n GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int n_past);\n\n GGML_API struct ggml_tensor * ggml_soft_max(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // in-place, returns view(a)\n GGML_API struct ggml_tensor * ggml_soft_max_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a);\n\n // a [ne0, ne01, ne02, ne03]\n // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional\n //\n // broadcast:\n // ne02 % ne12 == 0\n // ne03 % ne13 == 0\n //\n // fused soft_max(a*scale + mask*(ALiBi slope))\n // max_bias = 0.0f for no ALiBi\n GGML_API struct ggml_tensor * ggml_soft_max_ext(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * mask,\n float scale,\n float max_bias);\n\n GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * mask,\n float scale,\n float max_bias);\n\n GGML_API void ggml_soft_max_add_sinks(\n struct ggml_tensor * a,\n struct ggml_tensor * sinks);\n\n GGML_API struct ggml_tensor * ggml_soft_max_ext_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n float scale,\n float max_bias);\n\n // in-place, returns view(a)\n GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n float scale,\n float max_bias);\n\n // rotary position embedding\n // if (mode & 1) - skip n_past elements (NOT SUPPORTED)\n // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style\n //\n // b is an int32 vector with size a->ne[2], it contains the positions\n GGML_API struct ggml_tensor * ggml_rope(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int n_dims,\n int mode);\n\n // in-place, returns view(a)\n GGML_API struct ggml_tensor * ggml_rope_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int n_dims,\n int mode);\n\n // custom RoPE\n // c is freq factors (e.g. phi3-128k), (optional)\n GGML_API struct ggml_tensor * ggml_rope_ext(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n int n_dims,\n int mode,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow);\n\n GGML_API struct ggml_tensor * ggml_rope_multi(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n int n_dims,\n int sections[GGML_MROPE_SECTIONS],\n int mode,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow);\n\n // in-place, returns view(a)\n GGML_API struct ggml_tensor * ggml_rope_ext_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n int n_dims,\n int mode,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow);\n\n GGML_API struct ggml_tensor * ggml_rope_multi_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n int n_dims,\n int sections[GGML_MROPE_SECTIONS],\n int mode,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow);\n\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int n_dims,\n int mode,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow),\n \"use ggml_rope_ext instead\");\n\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int n_dims,\n int mode,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow),\n \"use ggml_rope_ext_inplace instead\");\n\n // compute correction dims for YaRN RoPE scaling\n GGML_API void ggml_rope_yarn_corr_dims(\n int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);\n\n // rotary position embedding backward, i.e compute dx from dy\n // a - dy\n GGML_API struct ggml_tensor * ggml_rope_ext_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // gradients of ggml_rope result\n struct ggml_tensor * b, // positions\n struct ggml_tensor * c, // freq factors\n int n_dims,\n int mode,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow);\n\n GGML_API struct ggml_tensor * ggml_rope_multi_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n int n_dims,\n int sections[4],\n int mode,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow);\n\n\n // clamp\n // in-place, returns view(a)\n GGML_API struct ggml_tensor * ggml_clamp(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float min,\n float max);\n\n // im2col\n // converts data into a format that effectively results in a convolution when combined with matrix multiplication\n GGML_API struct ggml_tensor * ggml_im2col(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel\n struct ggml_tensor * b, // data\n int s0, // stride dimension 0\n int s1, // stride dimension 1\n int p0, // padding dimension 0\n int p1, // padding dimension 1\n int d0, // dilation dimension 0\n int d1, // dilation dimension 1\n bool is_2D,\n enum ggml_type dst_type);\n\n GGML_API struct ggml_tensor * ggml_im2col_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel\n struct ggml_tensor * b, // gradient of im2col output\n int64_t * ne, // shape of im2col input\n int s0, // stride dimension 0\n int s1, // stride dimension 1\n int p0, // padding dimension 0\n int p1, // padding dimension 1\n int d0, // dilation dimension 0\n int d1, // dilation dimension 1\n bool is_2D);\n\n GGML_API struct ggml_tensor * ggml_conv_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel\n struct ggml_tensor * b, // data\n int s0, // stride\n int p0, // padding\n int d0); // dilation\n\n // conv_1d with padding = half\n // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)\n GGML_API struct ggml_tensor* ggml_conv_1d_ph(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel\n struct ggml_tensor * b, // data\n int s, // stride\n int d); // dilation\n\n // depthwise\n // TODO: this is very likely wrong for some cases! - needs more testing\n GGML_API struct ggml_tensor * ggml_conv_1d_dw(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel\n struct ggml_tensor * b, // data\n int s0, // stride\n int p0, // padding\n int d0); // dilation\n\n GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel\n struct ggml_tensor * b, // data\n int s0, // stride\n int d0); // dilation\n\n GGML_API struct ggml_tensor * ggml_conv_transpose_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel\n struct ggml_tensor * b, // data\n int s0, // stride\n int p0, // padding\n int d0); // dilation\n\n GGML_API struct ggml_tensor * ggml_conv_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel\n struct ggml_tensor * b, // data\n int s0, // stride dimension 0\n int s1, // stride dimension 1\n int p0, // padding dimension 0\n int p1, // padding dimension 1\n int d0, // dilation dimension 0\n int d1); // dilation dimension 1\n\n GGML_API struct ggml_tensor * ggml_im2col_3d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int64_t IC,\n int s0, // stride width\n int s1, // stride height\n int s2, // stride depth\n int p0, // padding width\n int p1, // padding height\n int p2, // padding depth\n int d0, // dilation width\n int d1, // dilation height\n int d2, // dilation depth\n enum ggml_type dst_type);\n\n // a: [OC*IC, KD, KH, KW]\n // b: [N*IC, ID, IH, IW]\n // result: [N*OC, OD, OH, OW]\n GGML_API struct ggml_tensor * ggml_conv_3d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int64_t IC,\n int s0, // stride width\n int s1, // stride height\n int s2, // stride depth\n int p0, // padding width\n int p1, // padding height\n int p2, // padding depth\n int d0, // dilation width\n int d1, // dilation height\n int d2 // dilation depth\n );\n\n // kernel size is a->ne[0] x a->ne[1]\n // stride is equal to kernel size\n // padding is zero\n // example:\n // a: 16 16 3 768\n // b: 1024 1024 3 1\n // res: 64 64 768 1\n // used in sam\n GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n // kernel size is a->ne[0] x a->ne[1]\n // stride is 1\n // padding is half\n // example:\n // a: 3 3 256 256\n // b: 64 64 256 1\n // res: 64 64 256 1\n // used in sam\n GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b);\n\n // depthwise (via im2col and mul_mat)\n GGML_API struct ggml_tensor * ggml_conv_2d_dw(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel\n struct ggml_tensor * b, // data\n int s0, // stride dimension 0\n int s1, // stride dimension 1\n int p0, // padding dimension 0\n int p1, // padding dimension 1\n int d0, // dilation dimension 0\n int d1); // dilation dimension 1\n\n // Depthwise 2D convolution\n // may be faster than ggml_conv_2d_dw, but not available in all backends\n // a: KW KH 1 C convolution kernel\n // b: W H C N input data\n // res: W_out H_out C N\n GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int stride0,\n int stride1,\n int pad0,\n int pad1,\n int dilation0,\n int dilation1);\n\n GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n int stride);\n\n GGML_API struct ggml_tensor * ggml_conv_2d_direct(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC]\n struct ggml_tensor * b, // input data [W, H, C, N]\n int s0, // stride dimension 0\n int s1, // stride dimension 1\n int p0, // padding dimension 0\n int p1, // padding dimension 1\n int d0, // dilation dimension 0\n int d1); // dilation dimension 1\n\n GGML_API struct ggml_tensor * ggml_conv_3d_direct(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC]\n struct ggml_tensor * b, // input [W, H, D, C * N]\n int s0, // stride\n int s1,\n int s2,\n int p0, // padding\n int p1,\n int p2,\n int d0, // dilation\n int d1,\n int d2,\n int n_channels,\n int n_batch,\n int n_channels_out);\n\n enum ggml_op_pool {\n GGML_OP_POOL_MAX,\n GGML_OP_POOL_AVG,\n GGML_OP_POOL_COUNT,\n };\n\n GGML_API struct ggml_tensor * ggml_pool_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_op_pool op,\n int k0, // kernel size\n int s0, // stride\n int p0); // padding\n\n // the result will have 2*p0 padding for the first dimension\n // and 2*p1 padding for the second dimension\n GGML_API struct ggml_tensor * ggml_pool_2d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_op_pool op,\n int k0,\n int k1,\n int s0,\n int s1,\n float p0,\n float p1);\n\n GGML_API struct ggml_tensor * ggml_pool_2d_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * af, // \"a\"/input used in forward pass\n enum ggml_op_pool op,\n int k0,\n int k1,\n int s0,\n int s1,\n float p0,\n float p1);\n\n enum ggml_scale_mode {\n GGML_SCALE_MODE_NEAREST = 0,\n GGML_SCALE_MODE_BILINEAR = 1,\n GGML_SCALE_MODE_BICUBIC = 2,\n\n GGML_SCALE_MODE_COUNT\n };\n\n enum ggml_scale_flag {\n GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8),\n GGML_SCALE_FLAG_ANTIALIAS = (1 << 9),\n };\n\n // interpolate\n // multiplies ne0 and ne1 by scale factor\n GGML_API struct ggml_tensor * ggml_upscale(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int scale_factor,\n enum ggml_scale_mode mode);\n\n // interpolate\n // interpolate scale to specified dimensions\n GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n enum ggml_scale_mode mode),\n \"use ggml_interpolate instead\");\n\n // Up- or downsamples the input to the specified size.\n // 2D scale modes (eg. bilinear) are applied to the first two dimensions.\n GGML_API struct ggml_tensor * ggml_interpolate(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3,\n uint32_t mode); // ggml_scale_mode [ | ggml_scale_flag...]\n\n // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]\n GGML_API struct ggml_tensor * ggml_pad(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int p0,\n int p1,\n int p2,\n int p3);\n\n // pad each dimension with values on the other side of the torus (looping around)\n GGML_API struct ggml_tensor * ggml_pad_circular(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int p0,\n int p1,\n int p2,\n int p3);\n\n GGML_API struct ggml_tensor * ggml_pad_ext(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int lp0,\n int rp0,\n int lp1,\n int rp1,\n int lp2,\n int rp2,\n int lp3,\n int rp3\n );\n\n // pad each dimension with values on the other side of the torus (looping around)\n GGML_API struct ggml_tensor * ggml_pad_ext_circular(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int lp0,\n int rp0,\n int lp1,\n int rp1,\n int lp2,\n int rp2,\n int lp3,\n int rp3);\n\n // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c]\n GGML_API struct ggml_tensor * ggml_pad_reflect_1d(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int p0,\n int p1);\n\n // Move tensor elements by an offset given for each dimension. Elements that\n // are shifted beyond the last position are wrapped around to the beginning.\n GGML_API struct ggml_tensor * ggml_roll(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int shift0,\n int shift1,\n int shift2,\n int shift3);\n\n // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing\n // zeroes everywhere outside the masked area\n GGML_API struct ggml_tensor * ggml_tri(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_tri_type type);\n\n // Fill tensor a with constant c\n GGML_API struct ggml_tensor * ggml_fill(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float c);\n\n GGML_API struct ggml_tensor * ggml_fill_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n float c);\n\n // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151\n // timesteps: [N,]\n // return: [N, dim]\n GGML_API struct ggml_tensor * ggml_timestep_embedding(\n struct ggml_context * ctx,\n struct ggml_tensor * timesteps,\n int dim,\n int max_period);\n\n // sort rows\n enum ggml_sort_order {\n GGML_SORT_ORDER_ASC,\n GGML_SORT_ORDER_DESC,\n };\n\n GGML_API struct ggml_tensor * ggml_argsort(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_sort_order order);\n\n // similar to ggml_top_k but implemented as `argsort` + `view`\n GGML_API struct ggml_tensor * ggml_argsort_top_k(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int k);\n\n // top k elements per row\n // note: the resulting top k indices are in no particular order\n GGML_API struct ggml_tensor * ggml_top_k(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int k);\n\n GGML_API struct ggml_tensor * ggml_arange(\n struct ggml_context * ctx,\n float start,\n float stop,\n float step);\n\n // q: [n_embd_k, n_batch, n_head, ne3 ]\n // k: [n_embd_k, n_kv, n_head_kv, ne3 ]\n // v: [n_embd_v, n_kv, n_head_kv, ne3 ] !! not transposed !!\n // mask: [n_kv, n_batch, ne32, ne33]\n // res: [n_embd_v, n_head, n_batch, ne3 ] !! permuted !!\n //\n // broadcast:\n // n_head % n_head_kv == 0\n // n_head % ne32 == 0\n // ne3 % ne33 == 0\n //\n GGML_API struct ggml_tensor * ggml_flash_attn_ext(\n struct ggml_context * ctx,\n struct ggml_tensor * q,\n struct ggml_tensor * k,\n struct ggml_tensor * v,\n struct ggml_tensor * mask,\n float scale,\n float max_bias,\n float logit_softcap);\n\n GGML_API void ggml_flash_attn_ext_set_prec(\n struct ggml_tensor * a,\n enum ggml_prec prec);\n\n GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(\n const struct ggml_tensor * a);\n\n GGML_API void ggml_flash_attn_ext_add_sinks(\n struct ggml_tensor * a,\n struct ggml_tensor * sinks);\n\n // TODO: needs to be adapted to ggml_flash_attn_ext\n GGML_API struct ggml_tensor * ggml_flash_attn_back(\n struct ggml_context * ctx,\n struct ggml_tensor * q,\n struct ggml_tensor * k,\n struct ggml_tensor * v,\n struct ggml_tensor * d,\n bool masked);\n\n GGML_API struct ggml_tensor * ggml_ssm_conv(\n struct ggml_context * ctx,\n struct ggml_tensor * sx,\n struct ggml_tensor * c);\n\n GGML_API struct ggml_tensor * ggml_ssm_scan(\n struct ggml_context * ctx,\n struct ggml_tensor * s,\n struct ggml_tensor * x,\n struct ggml_tensor * dt,\n struct ggml_tensor * A,\n struct ggml_tensor * B,\n struct ggml_tensor * C,\n struct ggml_tensor * ids);\n\n // partition into non-overlapping windows with padding if needed\n // example:\n // a: 768 64 64 1\n // w: 14\n // res: 768 14 14 25\n // used in sam\n GGML_API struct ggml_tensor * ggml_win_part(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int w);\n\n // reverse of ggml_win_part\n // used in sam\n GGML_API struct ggml_tensor * ggml_win_unpart(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int w0,\n int h0,\n int w);\n\n GGML_API struct ggml_tensor * ggml_unary(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_unary_op op);\n\n GGML_API struct ggml_tensor * ggml_unary_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n enum ggml_unary_op op);\n\n // used in sam\n GGML_API struct ggml_tensor * ggml_get_rel_pos(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n int qh,\n int kh);\n\n // used in sam\n GGML_API struct ggml_tensor * ggml_add_rel_pos(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * pw,\n struct ggml_tensor * ph);\n\n GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * pw,\n struct ggml_tensor * ph);\n\n GGML_API struct ggml_tensor * ggml_rwkv_wkv6(\n struct ggml_context * ctx,\n struct ggml_tensor * k,\n struct ggml_tensor * v,\n struct ggml_tensor * r,\n struct ggml_tensor * tf,\n struct ggml_tensor * td,\n struct ggml_tensor * state);\n\n GGML_API struct ggml_tensor * ggml_gated_linear_attn(\n struct ggml_context * ctx,\n struct ggml_tensor * k,\n struct ggml_tensor * v,\n struct ggml_tensor * q,\n struct ggml_tensor * g,\n struct ggml_tensor * state,\n float scale);\n\n GGML_API struct ggml_tensor * ggml_rwkv_wkv7(\n struct ggml_context * ctx,\n struct ggml_tensor * r,\n struct ggml_tensor * w,\n struct ggml_tensor * k,\n struct ggml_tensor * v,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * state);\n\n /* Solves a specific equation of the form Ax=B, where A is a triangular matrix\n * without zeroes on the diagonal (i.e. invertible).\n * B can have any number of columns, but must have the same number of rows as A\n * If A is [n, n] and B is [n, m], then the result will be [n, m] as well\n * Has O(n^3) complexity (unlike most matrix ops out there), so use on cases\n * where n > 100 sparingly, pre-chunk if necessary.\n *\n * If left = false, solves xA=B instead\n * If lower = false, assumes upper triangular instead\n * If uni = true, assumes diagonal of A to be all ones (will override actual values)\n *\n * TODO: currently only lower, right, non-unitriangular variant is implemented\n */\n GGML_API struct ggml_tensor * ggml_solve_tri(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n bool left,\n bool lower,\n bool uni);\n\n // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST]\n // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306\n GGML_API struct ggml_tensor * ggml_gated_delta_net(\n struct ggml_context * ctx,\n struct ggml_tensor * q,\n struct ggml_tensor * k,\n struct ggml_tensor * v,\n struct ggml_tensor * g,\n struct ggml_tensor * beta,\n struct ggml_tensor * state);\n\n // custom operators\n\n typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);\n typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);\n typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);\n\n#define GGML_N_TASKS_MAX (-1)\n // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks\n\n GGML_API struct ggml_tensor * ggml_map_custom1(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n ggml_custom1_op_t fun,\n int n_tasks,\n void * userdata);\n\n GGML_API struct ggml_tensor * ggml_map_custom1_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n ggml_custom1_op_t fun,\n int n_tasks,\n void * userdata);\n\n GGML_API struct ggml_tensor * ggml_map_custom2(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n ggml_custom2_op_t fun,\n int n_tasks,\n void * userdata);\n\n GGML_API struct ggml_tensor * ggml_map_custom2_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n ggml_custom2_op_t fun,\n int n_tasks,\n void * userdata);\n\n GGML_API struct ggml_tensor * ggml_map_custom3(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n ggml_custom3_op_t fun,\n int n_tasks,\n void * userdata);\n\n GGML_API struct ggml_tensor * ggml_map_custom3_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * b,\n struct ggml_tensor * c,\n ggml_custom3_op_t fun,\n int n_tasks,\n void * userdata);\n\n typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata);\n\n GGML_API struct ggml_tensor * ggml_custom_4d(\n struct ggml_context * ctx,\n enum ggml_type type,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3,\n struct ggml_tensor ** args,\n int n_args,\n ggml_custom_op_t fun,\n int n_tasks,\n void * userdata);\n\n GGML_API struct ggml_tensor * ggml_custom_inplace(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor ** args,\n int n_args,\n ggml_custom_op_t fun,\n int n_tasks,\n void * userdata);\n\n // loss function\n\n GGML_API struct ggml_tensor * ggml_cross_entropy_loss(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // logits\n struct ggml_tensor * b); // labels\n\n GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(\n struct ggml_context * ctx,\n struct ggml_tensor * a, // logits\n struct ggml_tensor * b, // labels\n struct ggml_tensor * c); // gradients of cross_entropy_loss result\n\n // AdamW optimizer step\n // Paper: https://arxiv.org/pdf/1711.05101v3.pdf\n // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html\n GGML_API struct ggml_tensor * ggml_opt_step_adamw(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * grad,\n struct ggml_tensor * m,\n struct ggml_tensor * v,\n struct ggml_tensor * adamw_params); // parameters such as the learning rate\n\n // stochastic gradient descent step (with weight decay)\n GGML_API struct ggml_tensor * ggml_opt_step_sgd(\n struct ggml_context * ctx,\n struct ggml_tensor * a,\n struct ggml_tensor * grad,\n struct ggml_tensor * sgd_params); // alpha, weight decay\n\n // build forward multiple tensors and select one of them for computing\n // this is useful for creating graphs that have constant topology but compute different things based on the input\n // ref: https://github.com/ggml-org/llama.cpp/pull/18550\n //\n // nodes:\n // | - build forward into the graph but do not compute\n // c - build forward into the graph and compute\n //\n // | | ... c ... |\n // | | ... c ... |\n // | | ... c ... |\n // [0 1 ... idx ... n-1] <-- ggml_build_forward_select(..., n, idx)\n // c\n // c\n //\n // example:\n // struct ggml_tensor * curs[3];\n //\n // curs[0] = compute0(...);\n // curs[1] = compute1(...);\n // curs[2] = compute2(...);\n //\n // int idx = select_branch(some_input);\n //\n // struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx);\n //\n GGML_API struct ggml_tensor * ggml_build_forward_select(\n struct ggml_cgraph * cgraph,\n struct ggml_tensor ** tensors,\n int n_tensors,\n int idx);\n\n GGML_API void ggml_build_forward_expand(\n struct ggml_cgraph * cgraph,\n struct ggml_tensor * tensor);\n\n GGML_API void ggml_build_backward_expand(\n struct ggml_context * ctx, // context for gradient computation\n struct ggml_cgraph * cgraph,\n struct ggml_tensor ** grad_accs);\n\n // graph allocation in a context\n GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false\n GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads);\n GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads);\n GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst);\n GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1\n GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph);\n\n GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph);\n GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i]\n GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph);\n GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph);\n\n GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);\n\n GGML_API size_t ggml_graph_overhead(void);\n GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);\n\n GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name);\n GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);\n GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);\n\n // print info and performance information for the graph\n GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);\n\n // dump the graph into a file using the dot format\n GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename);\n\n // TODO these functions were sandwiched in the old optimization interface, is there a better place for them?\n typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);\n\n // Set callback for all future logging events.\n // If this is not called, or NULL is supplied, everything is output on stderr.\n GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data);\n GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data);\n\n GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);\n\n //\n // quantization\n //\n\n // - ggml_quantize_init can be called multiple times with the same type\n // it will only initialize the quantization tables for the first call or after ggml_quantize_free\n // automatically called by ggml_quantize_chunk for convenience\n //\n // - ggml_quantize_free will free any memory allocated by ggml_quantize_init\n // call this at the end of the program to avoid memory leaks\n //\n // note: these are thread-safe\n //\n GGML_API void ggml_quantize_init(enum ggml_type type);\n GGML_API void ggml_quantize_free(void);\n\n // some quantization type cannot be used without an importance matrix\n GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);\n\n // calls ggml_quantize_init internally (i.e. can allocate memory)\n GGML_API size_t ggml_quantize_chunk(\n enum ggml_type type,\n const float * src,\n void * dst,\n int64_t start,\n int64_t nrows,\n int64_t n_per_row,\n const float * imatrix);\n\n#ifdef __cplusplus\n // restrict not standard in C++\n# if defined(__GNUC__)\n# define GGML_RESTRICT __restrict__\n# elif defined(__clang__)\n# define GGML_RESTRICT __restrict\n# elif defined(_MSC_VER)\n# define GGML_RESTRICT __restrict\n# else\n# define GGML_RESTRICT\n# endif\n#else\n# if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L)\n# define GGML_RESTRICT __restrict\n# else\n# define GGML_RESTRICT restrict\n# endif\n#endif\n typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\n typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\n struct ggml_type_traits {\n const char * type_name;\n int64_t blck_size;\n int64_t blck_size_interleave; // interleave elements in blocks\n size_t type_size;\n bool is_quantized;\n ggml_to_float_t to_float;\n ggml_from_float_t from_float_ref;\n };\n\n GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type);\n\n // ggml threadpool\n // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend\n // the goal should be to create an API that other backends can use move everything to the ggml base\n\n // scheduling priorities\n enum ggml_sched_priority {\n GGML_SCHED_PRIO_LOW = -1,\n GGML_SCHED_PRIO_NORMAL,\n GGML_SCHED_PRIO_MEDIUM,\n GGML_SCHED_PRIO_HIGH,\n GGML_SCHED_PRIO_REALTIME\n };\n\n // threadpool params\n // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults\n struct ggml_threadpool_params {\n bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)\n int n_threads; // number of threads\n enum ggml_sched_priority prio; // thread priority\n uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling)\n bool strict_cpu; // strict cpu placement\n bool paused; // start in paused state\n };\n\n struct ggml_threadpool; // forward declaration, see ggml.c\n\n typedef struct ggml_threadpool * ggml_threadpool_t;\n\n GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);\n GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads);\n GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "include/gguf.h", "content": "// This file contains functionality related to \"GGUF\" files, the binary file format used by ggml.\n// GGUF files have the following structure:\n//\n// 1. File magic \"GGUF\" (4 bytes).\n// 2. File version (uint32_t).\n// 3. Number of ggml tensors in file (int64_t).\n// 4. Number of key-value-pairs in file (int64_t).\n// 5. For each KV pair:\n// 1. The key (string).\n// 2. The value type (gguf_type).\n// 3a. If the value type is GGUF_TYPE_ARRAY:\n// 1. The type of the array (gguf_type).\n// 2. The number of elements in the array (uint64_t).\n// 3. The binary representation of each element in the array.\n// 3b. Otherwise:\n// 1. The binary representation of the value.\n// 6. For each ggml tensor:\n// 1. The tensor name (string).\n// 2. The number of dimensions of the tensor (uint32_t).\n// 3. For each dimension:\n// 1. The size of the tensor in the dimension (int64_t).\n// 4. The tensor data type (ggml_type).\n// 5. The tensor data offset in the tensor data binary blob (uint64_t).\n// 7. The tensor data binary blob (optional, aligned).\n//\n// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator.\n// All enums are stored as int32_t.\n// All bool values are stored as int8_t.\n// If the special key \"general.alignment\" (uint32_t) is defined it is used for alignment,\n// otherwise GGUF_DEFAULT_ALIGNMENT is used.\n//\n// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de)\n\n#pragma once\n\n#include \"ggml.h\"\n\n#include \n#include \n\n#define GGUF_MAGIC \"GGUF\"\n#define GGUF_VERSION 3\n\n#define GGUF_KEY_GENERAL_ALIGNMENT \"general.alignment\"\n\n#define GGUF_DEFAULT_ALIGNMENT 32\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n // types that can be stored as GGUF KV data\n enum gguf_type {\n GGUF_TYPE_UINT8 = 0,\n GGUF_TYPE_INT8 = 1,\n GGUF_TYPE_UINT16 = 2,\n GGUF_TYPE_INT16 = 3,\n GGUF_TYPE_UINT32 = 4,\n GGUF_TYPE_INT32 = 5,\n GGUF_TYPE_FLOAT32 = 6,\n GGUF_TYPE_BOOL = 7,\n GGUF_TYPE_STRING = 8,\n GGUF_TYPE_ARRAY = 9,\n GGUF_TYPE_UINT64 = 10,\n GGUF_TYPE_INT64 = 11,\n GGUF_TYPE_FLOAT64 = 12,\n GGUF_TYPE_COUNT, // marks the end of the enum\n };\n\n struct gguf_context;\n\n struct gguf_init_params {\n bool no_alloc;\n\n // if not NULL, create a ggml_context and allocate the tensor data in it\n struct ggml_context ** ctx;\n };\n\n GGML_API struct gguf_context * gguf_init_empty(void);\n GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);\n //GGML_API struct gguf_context * gguf_init_from_buffer(..);\n\n GGML_API void gguf_free(struct gguf_context * ctx);\n\n GGML_API const char * gguf_type_name(enum gguf_type type);\n\n GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx);\n GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx);\n GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx);\n\n GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx);\n GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found\n GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id);\n\n GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id);\n GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id);\n\n // will abort if the wrong type is used for the key\n GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id);\n GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id);\n GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id);\n GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id);\n GGML_API size_t gguf_get_arr_n (const struct gguf_context * ctx, int64_t key_id);\n\n // get raw pointer to the first element of the array with the given key_id\n // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference)\n GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id);\n\n // get ith C string from array with given key_id\n GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i);\n\n GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx);\n GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found\n GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id);\n GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id);\n GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id);\n GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id);\n\n // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist)\n GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key);\n\n // overrides an existing KV pair or adds a new one, the new KV pair is always at the back\n GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val);\n GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val);\n GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);\n GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val);\n GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);\n GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val);\n GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val);\n GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val);\n GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val);\n GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val);\n GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val);\n GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);\n\n // creates a new array with n elements of the given type and copies the corresponding number of bytes from data\n GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n);\n\n // creates a new array with n strings and copies the corresponding strings from data\n GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n);\n\n // set or add KV pairs from another context\n GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src);\n\n // add tensor to GGUF context, tensor name must be unique\n GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);\n\n // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated\n // in such a way that the tensor data remains as one contiguous block (except for padding)\n GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);\n\n // assumes that at least gguf_get_tensor_size bytes can be read from data\n GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data);\n\n // writing gguf files can be done in 3 ways:\n //\n // - write the entire gguf_context to a binary file in a single pass:\n //\n // gguf_write_to_file(ctx, fname, /*only_meta =*/ false);\n //\n // - write only the meta data to a file, then re-open the file and append the tensor data:\n //\n // gguf_write_to_file(ctx, fname, /*only_meta =*/ true);\n // FILE * f = fopen(fname, \"ab\");\n // fwrite(f, ...); // write tensor data\n // fclose(f);\n //\n // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:\n //\n // FILE * f = fopen(fname, \"wb\");\n // const size_t size_meta = gguf_get_meta_size(ctx);\n // fseek(f, size_meta, SEEK_SET);\n // fwrite(f, ...); // write tensor data\n // void * data = malloc(size_meta);\n // gguf_get_meta_data(ctx, data);\n // rewind(f);\n // fwrite(data, 1, data, f);\n // free(data);\n // fclose(f);\n //\n\n // write the entire context to a binary file\n GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);\n\n // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding\n GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);\n\n // writes the meta data to pointer \"data\"\n GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "requirements.txt", "content": "accelerate==0.19.0\nnumpy>=2.0.2\nsentencepiece~=0.1.98\ntorchvision>=0.15.2\ntransformers>=4.35.2,<5.0.0\ngguf>=0.1.0\nkeras==3.5.0\ntensorflow==2.18.0\n\n--extra-index-url https://download.pytorch.org/whl/cpu\ntorch~=2.5.1\n" }, { "path": "scripts/gen-authors.sh", "content": "#!/usr/bin/env bash\n\nprintf \"# date: $(date)\\n\" > AUTHORS\nprintf \"# this file is auto-generated by scripts/gen-authors.sh\\n\\n\" >> AUTHORS\n\ngit log --format='%an <%ae>' --reverse --date=short master | awk '!seen[$0]++' | sort >> AUTHORS\n\n# if necessary, update your name here. for example: jdoe -> John Doe\nsed -i '' 's/^jdoe/John Doe/g' AUTHORS\n" }, { "path": "scripts/release.sh", "content": "#!/bin/bash\n#\n# Automated release script for ggml.\n#\n# Note: Sync from llama.cpp should be done separately via PR process\n# prior to running this script.\n#\n# Usage:\n# ./scripts/release.sh prepare [major|minor|patch] [--dry-run]\n# ./scripts/release.sh finalize [--dry-run]\n#\n# Two-stage release process:\n#\n# Stage 1 - Prepare:\n# $ ./scripts/release.sh prepare minor\n# This creates a release candidate branch with version bump and removes -dev suffix.\n# The branch should then be manually pushed and a PR created, reviewed, and merged.\n#\n# Stage 2 - Finalize:\n# $ ./scripts/release.sh finalize\n# After the RC PR is merged, this reads the current version from CMakeLists.txt,\n# creates the release tag, and prepares the next development cycle.\n#\n# Prepare stage:\n# 1. Creates release candidate branch\n# 2. Updates version and removes -dev suffix\n# 3. Commits the version bump\n#\n# Finalize stage:\n# 1. Reads current release version from CMakeLists.txt\n# 2. Creates signed git tag on master\n# 3. Adds -dev suffix back for next development cycle\n# 4. Creates branch and commit for development version\n#\n\nset -e\n\nif [ ! -f \"CMakeLists.txt\" ] || [ ! -d \"scripts\" ]; then\n echo \"Error: Must be run from ggml root directory\"\n exit 1\nfi\n\n# Parse command line arguments\nCOMMAND=\"\"\nVERSION_TYPE=\"\"\nDRY_RUN=false\n\n# First argument should be the command\nif [ $# -eq 0 ]; then\n echo \"Error: Missing command\"\n echo \"Usage: $0 prepare [major|minor|patch] [--dry-run]\"\n echo \" $0 finalize [--dry-run]\"\n exit 1\nfi\n\nCOMMAND=\"$1\"\nshift\n\n# Parse remaining arguments\nfor arg in \"$@\"; do\n case $arg in\n --dry-run)\n DRY_RUN=true\n ;;\n major|minor|patch)\n if [ \"$COMMAND\" = \"prepare\" ]; then\n VERSION_TYPE=\"$arg\"\n else\n echo \"Error: Version type only valid for 'prepare' command\"\n exit 1\n fi\n ;;\n *)\n echo \"Error: Unknown argument '$arg'\"\n echo \"Usage: $0 prepare [major|minor|patch] [--dry-run]\"\n echo \" $0 finalize [--dry-run]\"\n exit 1\n ;;\n esac\ndone\n\n# Validate command\nif [[ ! \"$COMMAND\" =~ ^(prepare|finalize)$ ]]; then\n echo \"Error: Command must be 'prepare' or 'finalize'\"\n echo \"Usage: $0 prepare [major|minor|patch] [--dry-run]\"\n echo \" $0 finalize [--dry-run]\"\n exit 1\nfi\n\n# For prepare command, default to patch if no version type specified\nif [ \"$COMMAND\" = \"prepare\" ]; then\n VERSION_TYPE=\"${VERSION_TYPE:-patch}\"\n if [[ ! \"$VERSION_TYPE\" =~ ^(major|minor|patch)$ ]]; then\n echo \"Error: Version type must be 'major', 'minor', or 'patch'\"\n echo \"Usage: $0 prepare [major|minor|patch] [--dry-run]\"\n exit 1\n fi\nfi\n\n# Common validation functions\ncheck_git_status() {\n # Check for uncommitted changes (skip in dry-run)\n if [ \"$DRY_RUN\" = false ] && ! git diff-index --quiet HEAD --; then\n echo \"Error: You have uncommitted changes. Please commit or stash them first.\"\n exit 1\n fi\n}\n\ncheck_master_branch() {\n # Ensure we're on master branch\n CURRENT_BRANCH=$(git branch --show-current)\n if [ \"$CURRENT_BRANCH\" != \"master\" ]; then\n if [ \"$DRY_RUN\" = true ]; then\n echo \"[dry run] Warning: Not on master branch (currently on: $CURRENT_BRANCH). Continuing with dry-run...\"\n echo \"\"\n else\n echo \"Error: Must be on master branch. Currently on: $CURRENT_BRANCH\"\n exit 1\n fi\n fi\n}\n\ncheck_master_up_to_date() {\n # Check if we have the latest from master (skip in dry-run)\n if [ \"$DRY_RUN\" = false ]; then\n echo \"Checking if local master is up-to-date with remote...\"\n git fetch origin master\n LOCAL=$(git rev-parse HEAD)\n REMOTE=$(git rev-parse origin/master)\n\n if [ \"$LOCAL\" != \"$REMOTE\" ]; then\n echo \"Error: Your local master branch is not up-to-date with origin/master.\"\n echo \"Please run 'git pull origin master' first.\"\n exit 1\n fi\n echo \"✓ Local master is up-to-date with remote\"\n echo \"\"\n elif [ \"$(git branch --show-current)\" = \"master\" ]; then\n echo \"[dry run] Warning: Dry-run mode - not checking if master is up-to-date with remote\"\n echo \"\"\n fi\n}\n\nprepare_release() {\n if [ \"$DRY_RUN\" = true ]; then\n echo \"[dry-run] Preparing release (no changes will be made)\"\n else\n echo \"Starting release preparation...\"\n fi\n echo \"\"\n\n check_git_status\n check_master_branch\n check_master_up_to_date\n\n # Extract current version from CMakeLists.txt\n echo \"Step 1: Reading current version...\"\n MAJOR=$(grep \"set(GGML_VERSION_MAJOR\" CMakeLists.txt | sed 's/.*MAJOR \\([0-9]*\\).*/\\1/')\n MINOR=$(grep \"set(GGML_VERSION_MINOR\" CMakeLists.txt | sed 's/.*MINOR \\([0-9]*\\).*/\\1/')\n PATCH=$(grep \"set(GGML_VERSION_PATCH\" CMakeLists.txt | sed 's/.*PATCH \\([0-9]*\\).*/\\1/')\n\n echo \"Current version: $MAJOR.$MINOR.$PATCH\"\n\n # Calculate new version\n case $VERSION_TYPE in\n major)\n NEW_MAJOR=$((MAJOR + 1))\n NEW_MINOR=0\n NEW_PATCH=0\n ;;\n minor)\n NEW_MAJOR=$MAJOR\n NEW_MINOR=$((MINOR + 1))\n NEW_PATCH=0\n ;;\n patch)\n NEW_MAJOR=$MAJOR\n NEW_MINOR=$MINOR\n NEW_PATCH=$((PATCH + 1))\n ;;\n esac\n\n NEW_VERSION=\"$NEW_MAJOR.$NEW_MINOR.$NEW_PATCH\"\n RC_BRANCH=\"ggml-rc-v$NEW_VERSION\"\n echo \"New release version: $NEW_VERSION\"\n echo \"Release candidate branch: $RC_BRANCH\"\n echo \"\"\n\n # Create release candidate branch\n echo \"Step 2: Creating release candidate branch...\"\n if [ \"$DRY_RUN\" = true ]; then\n echo \" [dry-run] Would create branch: $RC_BRANCH\"\n else\n git checkout -b \"$RC_BRANCH\"\n echo \"✓ Created and switched to branch: $RC_BRANCH\"\n fi\n echo \"\"\n\n # Update CMakeLists.txt for release\n echo \"Step 3: Updating version in CMakeLists.txt...\"\n if [ \"$DRY_RUN\" = true ]; then\n echo \" [dry-run] Would update GGML_VERSION_MAJOR to $NEW_MAJOR\"\n echo \" [dry-run] Would update GGML_VERSION_MINOR to $NEW_MINOR\"\n echo \" [dry-run] Would update GGML_VERSION_PATCH to $NEW_PATCH\"\n else\n sed -i'' -e \"s/set(GGML_VERSION_MAJOR [0-9]*)/set(GGML_VERSION_MAJOR $NEW_MAJOR)/\" CMakeLists.txt\n sed -i'' -e \"s/set(GGML_VERSION_MINOR [0-9]*)/set(GGML_VERSION_MINOR $NEW_MINOR)/\" CMakeLists.txt\n sed -i'' -e \"s/set(GGML_VERSION_PATCH [0-9]*)/set(GGML_VERSION_PATCH $NEW_PATCH)/\" CMakeLists.txt\n fi\n echo \"\"\n\n # Commit version bump\n echo \"Step 4: Committing version bump...\"\n if [ \"$DRY_RUN\" = true ]; then\n echo \" [dry-run] Would commit: 'ggml : bump version to $NEW_VERSION'\"\n else\n git add CMakeLists.txt\n git commit -m \"ggml : bump version to $NEW_VERSION\"\n fi\n echo \"\"\n\n echo \"\"\n if [ \"$DRY_RUN\" = true ]; then\n echo \"[dry-run] Summary (no changes were made):\"\n echo \" • Would have created branch: $RC_BRANCH\"\n echo \" • Would have updated version to: $NEW_VERSION\"\n else\n echo \"Release preparation completed!\"\n echo \"Summary:\"\n echo \" • Created branch: $RC_BRANCH\"\n echo \" • Updated version to: $NEW_VERSION\"\n echo \"\"\n echo \"Next steps:\"\n echo \" • Push branch to remote: git push origin $RC_BRANCH\"\n echo \" • Create a Pull Request from $RC_BRANCH to master\"\n echo \" • After PR is merged, run: ./scripts/release.sh finalize\"\n fi\n}\n\nfinalize_release() {\n if [ \"$DRY_RUN\" = true ]; then\n echo \"[dry-run] Finalizing release (no changes will be made)\"\n else\n echo \"Starting release finalization...\"\n fi\n echo \"\"\n\n check_git_status\n check_master_branch\n check_master_up_to_date\n\n # Read current version from CMakeLists.txt\n echo \"Step 1: Reading current release version...\"\n MAJOR=$(grep \"set(GGML_VERSION_MAJOR\" CMakeLists.txt | sed 's/.*MAJOR \\([0-9]*\\).*/\\1/')\n MINOR=$(grep \"set(GGML_VERSION_MINOR\" CMakeLists.txt | sed 's/.*MINOR \\([0-9]*\\).*/\\1/')\n PATCH=$(grep \"set(GGML_VERSION_PATCH\" CMakeLists.txt | sed 's/.*PATCH \\([0-9]*\\).*/\\1/')\n\n RELEASE_VERSION=\"$MAJOR.$MINOR.$PATCH\"\n echo \"Release version: $RELEASE_VERSION\"\n echo \"\"\n\n # Create git tag\n echo \"Step 2: Creating signed git tag...\"\n if [ \"$DRY_RUN\" = true ]; then\n echo \" [dry-run] Would create signed tag: v$RELEASE_VERSION with message 'Release version $RELEASE_VERSION'\"\n else\n git tag -s \"v$RELEASE_VERSION\" -m \"Release version $RELEASE_VERSION\"\n echo \"✓ Created signed tag: v$RELEASE_VERSION\"\n fi\n echo \"\"\n\n\n echo \"\"\n if [ \"$DRY_RUN\" = true ]; then\n echo \"[dry-run] Summary (no changes were made):\"\n echo \" • Would have created tag: v$RELEASE_VERSION\"\n else\n echo \"Release finalization completed!\"\n echo \"Summary:\"\n echo \" • Created signed tag: v$RELEASE_VERSION\"\n echo \"\"\n echo \"Next steps:\"\n echo \" • Push tag to remote: git push origin v$RELEASE_VERSION\"\n echo \" • The release is now complete!\"\n fi\n}\n\n# Execute the appropriate command\ncase $COMMAND in\n prepare)\n prepare_release\n ;;\n finalize)\n finalize_release\n ;;\nesac\n" }, { "path": "scripts/sync-llama-am.sh", "content": "#!/bin/bash\n#\n# Synchronize llama.cpp changes to ggml\n#\n# Usage:\n#\n# $ cd /path/to/ggml\n# $ ./scripts/sync-llama-am.sh -skip hash0,hash1,hash2... -C 3\n#\n\nset -e\n\nsd=$(dirname $0)\ncd $sd/../\n\nSRC_GGML=$(pwd)\nSRC_LLAMA=$(cd ../llama.cpp; pwd)\n\nif [ ! -d $SRC_LLAMA ]; then\n echo \"llama.cpp not found at $SRC_LLAMA\"\n exit 1\nfi\n\nlc=$(cat $SRC_GGML/scripts/sync-llama.last)\necho \"Syncing llama.cpp changes since commit $lc\"\n\nto_skip=\"\"\n\n# context for git patches in number of lines\nctx=\"8\"\n\nwhile [ \"$1\" != \"\" ]; do\n case $1 in\n -skip )\n shift\n to_skip=$1\n ;;\n -C )\n shift\n ctx=$1\n ;;\n esac\n shift\ndone\n\ncd $SRC_LLAMA\n\ngit log --oneline $lc..HEAD\ngit log --oneline $lc..HEAD --reverse | grep -v \"(ggml/[0-9]*)\" | grep -v \"(whisper/[0-9]*)\" | cut -d' ' -f1 > $SRC_GGML/llama-commits\n\nif [ ! -s $SRC_GGML/llama-commits ]; then\n rm -v $SRC_GGML/llama-commits\n echo \"No new commits\"\n exit 0\nfi\n\nif [ -f $SRC_GGML/llama-src.patch ]; then\n rm -v $SRC_GGML/llama-src.patch\nfi\n\nwhile read c; do\n if [ -n \"$to_skip\" ]; then\n if [[ $to_skip == *\"$c\"* ]]; then\n echo \"Skipping $c\"\n continue\n fi\n fi\n\n git format-patch -U${ctx} -k $c~1..$c --stdout -- \\\n ggml/CMakeLists.txt \\\n ggml/src/CMakeLists.txt \\\n ggml/cmake/BuildTypes.cmake \\\n ggml/cmake/GitVars.cmake \\\n ggml/cmake/common.cmake \\\n ggml/cmake/ggml-config.cmake.in \\\n ggml/src/ggml-cpu/cmake/FindSIMD.cmake \\\n ggml/src/ggml* \\\n ggml/src/gguf* \\\n ggml/include/ggml*.h \\\n ggml/include/gguf*.h \\\n tests/test-opt.cpp \\\n tests/test-quantize-fns.cpp \\\n tests/test-quantize-perf.cpp \\\n tests/test-backend-ops.cpp \\\n LICENSE \\\n scripts/gen-authors.sh \\\n >> $SRC_GGML/llama-src.patch\ndone < $SRC_GGML/llama-commits\n\nrm -v $SRC_GGML/llama-commits\n\n# delete files if empty\nif [ ! -s $SRC_GGML/llama-src.patch ]; then\n rm -v $SRC_GGML/llama-src.patch\nfi\n\ncd $SRC_GGML\n\nif [ -f $SRC_GGML/llama-src.patch ]; then\n # replace PR numbers\n #\n # Subject: some text (#1234)\n # Subject: some text (llama/1234)\n cat llama-src.patch | sed -e 's/^Subject: \\(.*\\) (#\\([0-9]*\\))/Subject: \\1 (llama\\/\\2)/' > llama-src.patch.tmp\n mv llama-src.patch.tmp llama-src.patch\n\n cat llama-src.patch | sed -e 's/^\\(.*\\) (#\\([0-9]*\\))$/\\1 (llama\\/\\2)/' > llama-src.patch.tmp\n mv llama-src.patch.tmp llama-src.patch\n\n # replace filenames:\n #\n # ggml/CMakelists.txt -> CMakeLists.txt\n # ggml/src/CMakelists.txt -> src/CMakeLists.txt\n #\n # ggml/cmake/BuildTypes.cmake -> cmake/BuildTypes.cmake\n # ggml/cmake/GitVars.cmake -> cmake/GitVars.cmake\n # ggml/cmake/common.cmake -> cmake/common.cmake\n # ggml/cmake/ggml-config.cmake.in -> cmake/ggml-config.cmake.in\n # ggml/src/ggml-cpu/cmake/FindSIMD.cmake -> src/ggml-cpu/cmake/FindSIMD.cmake\n #\n # ggml/src/ggml* -> src/ggml*\n # ggml/src/gguf* -> src/gguf*\n #\n # ggml/include/ggml*.h -> include/ggml*.h\n # ggml/include/gguf*.h -> include/gguf*.h\n #\n # tests/test-opt.cpp -> tests/test-opt.cpp\n # tests/test-quantize-fns.cpp -> tests/test-quantize-fns.cpp\n # tests/test-quantize-perf.cpp -> tests/test-quantize-perf.cpp\n # tests/test-backend-ops.cpp -> tests/test-backend-ops.cpp\n #\n # LICENSE -> LICENSE\n # scripts/gen-authors.sh -> scripts/gen-authors.sh\n\n cat llama-src.patch | sed -E \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/CMakeLists\\.txt/\\1CMakeLists.txt/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/src\\/CMakeLists\\.txt/\\1src\\/CMakeLists.txt/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/cmake\\/BuildTypes\\.cmake/\\1cmake\\/BuildTypes\\.cmake/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/cmake\\/GitVars\\.cmake/\\1cmake\\/GitVars\\.cmake/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/cmake\\/common\\.cmake/\\1cmake\\/common\\.cmake/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/cmake\\/ggml-config\\.cmake\\.in/\\1cmake\\/ggml-config\\.cmake\\.in/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/src\\/ggml-cpu\\/cmake\\/FindSIMD\\.cmake/\\1src\\/ggml-cpu\\/cmake\\/FindSIMD\\.cmake/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/src\\/ggml(.*)/\\1src\\/ggml\\2/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/src\\/gguf(.*)/\\1src\\/gguf\\2/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/include\\/ggml(.*)\\.h/\\1include\\/ggml\\2.h/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)ggml\\/include\\/gguf(.*)\\.h/\\1include\\/gguf\\2.h/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)tests\\/test-opt\\.cpp/\\1tests\\/test-opt.cpp/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)tests\\/test-quantize-fns\\.cpp/\\1tests\\/test-quantize-fns.cpp/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)tests\\/test-quantize-perf\\.cpp/\\1tests\\/test-quantize-perf.cpp/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)tests\\/test-backend-ops\\.cpp/\\1tests\\/test-backend-ops.cpp/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)LICENSE/\\1LICENSE/g' \\\n -e 's/(^[[:space:]]| [ab]\\/)scripts\\/gen-authors\\.sh/\\1scripts\\/gen-authors.sh/g' \\\n > llama-src.patch.tmp\n mv llama-src.patch.tmp llama-src.patch\n\n git am -C${ctx} llama-src.patch\n\n rm -v $SRC_GGML/llama-src.patch\nfi\n\n# update last commit\ncd $SRC_LLAMA\ngit log -1 --format=%H > $SRC_GGML/scripts/sync-llama.last\n\necho \"Done\"\n\nexit 0\n" }, { "path": "scripts/sync-llama.last", "content": "ae40cd27c85aa30b9cd56033da1d6a954290f7ea\n" }, { "path": "scripts/sync-llama.sh", "content": "#!/bin/bash\n\ncp -rpv ../llama.cpp/ggml/CMakeLists.txt CMakeLists.txt\ncp -rpv ../llama.cpp/ggml/src/CMakeLists.txt src/CMakeLists.txt\n\ncp -rpv ../llama.cpp/ggml/cmake/* cmake/\ncp -rpv ../llama.cpp/ggml/src/ggml-cpu/cmake/* src/ggml-cpu/cmake/\n\ncp -rpv ../llama.cpp/ggml/src/ggml* src/\ncp -rpv ../llama.cpp/ggml/src/gguf* src/\n\ncp -rpv ../llama.cpp/ggml/include/ggml*.h include/\ncp -rpv ../llama.cpp/ggml/include/gguf*.h include/\n\ncp -rpv ../llama.cpp/tests/test-opt.cpp tests/test-opt.cpp\ncp -rpv ../llama.cpp/tests/test-quantize-fns.cpp tests/test-quantize-fns.cpp\ncp -rpv ../llama.cpp/tests/test-quantize-perf.cpp tests/test-quantize-perf.cpp\ncp -rpv ../llama.cpp/tests/test-backend-ops.cpp tests/test-backend-ops.cpp\n\ncp -rpv ../llama.cpp/LICENSE ./LICENSE\ncp -rpv ../llama.cpp/scripts/gen-authors.sh ./scripts/gen-authors.sh\n" }, { "path": "scripts/sync-whisper-am.sh", "content": "#!/bin/bash\n#\n# Synchronize whisper.cpp changes to ggml\n#\n# Usage:\n#\n# $ cd /path/to/ggml\n# $ ./scripts/sync-whisper-am.sh -skip hash0,hash1,hash2...\n#\n\nset -e\n\nsd=$(dirname $0)\ncd $sd/../\n\nSRC_GGML=$(pwd)\nSRC_WHISPER=$(cd ../whisper.cpp; pwd)\n\nif [ ! -d $SRC_WHISPER ]; then\n echo \"whisper.cpp not found at $SRC_WHISPER\"\n exit 1\nfi\n\nlc=$(cat $SRC_GGML/scripts/sync-whisper.last)\necho \"Syncing whisper.cpp changes since commit $lc\"\n\nto_skip=\"\"\nif [ \"$1\" == \"-skip\" ]; then\n to_skip=$2\nfi\n\ncd $SRC_WHISPER\n\ngit log --oneline $lc..HEAD\ngit log --oneline $lc..HEAD --reverse | grep -v \"(ggml/[0-9]*)\" | grep -v \"(llama/[0-9]*)\" | cut -d' ' -f1 > $SRC_GGML/whisper-commits\n\nif [ ! -s $SRC_GGML/whisper-commits ]; then\n rm -v $SRC_GGML/whisper-commits\n echo \"No new commits\"\n exit 0\nfi\n\nif [ -f $SRC_GGML/whisper-src.patch ]; then\n rm -v $SRC_GGML/whisper-src.patch\nfi\n\nwhile read c; do\n if [ -n \"$to_skip\" ]; then\n if [[ $to_skip == *\"$c\"* ]]; then\n echo \"Skipping $c\"\n continue\n fi\n fi\n\n git format-patch -k $c~1..$c --stdout -- \\\n ggml/CMakeLists.txt \\\n ggml/src/CMakeLists.txt \\\n ggml/cmake/FindSIMD.cmake \\\n ggml/src/ggml* \\\n ggml/src/gguf* \\\n ggml/include/ggml*.h \\\n ggml/include/gguf*.h \\\n examples/common-ggml.h \\\n examples/common-ggml.cpp \\\n LICENSE \\\n scripts/gen-authors.sh \\\n >> $SRC_GGML/whisper-src.patch\ndone < $SRC_GGML/whisper-commits\n\nrm -v $SRC_GGML/whisper-commits\n\n# delete files if empty\nif [ ! -s $SRC_GGML/whisper-src.patch ]; then\n rm -v $SRC_GGML/whisper-src.patch\nfi\n\ncd $SRC_GGML\n\nif [ -f $SRC_GGML/whisper-src.patch ]; then\n # replace PR numbers\n #\n # Subject: some text (#1234)\n # Subject: some text (whisper/1234)\n cat whisper-src.patch | sed -e 's/^Subject: \\(.*\\) (#\\([0-9]*\\))/Subject: \\1 (whisper\\/\\2)/' > whisper-src.patch.tmp\n mv whisper-src.patch.tmp whisper-src.patch\n\n cat whisper-src.patch | sed -e 's/^\\(.*\\) (#\\([0-9]*\\))$/\\1 (whisper\\/\\2)/' > whisper-src.patch.tmp\n mv whisper-src.patch.tmp whisper-src.patch\n\n # replace filenames:\n #\n # ggml/CMakelists.txt -> CMakeLists.txt\n # ggml/src/CMakelists.txt -> src/CMakeLists.txt\n # ggml/cmake/FindSIMD.cmake -> cmake/FindSIMD.cmake\n #\n # ggml/src/ggml* -> src/ggml*\n # ggml/src/gguf* -> src/gguf*\n #\n # ggml/include/ggml*.h -> include/ggml*.h\n # ggml/include/gguf*.h -> include/gguf*.h\n #\n # examples/common.h -> examples/common.h\n # examples/common.cpp -> examples/common.cpp\n # examples/common-ggml.h -> examples/common-ggml.h\n # examples/common-ggml.cpp -> examples/common-ggml.cpp\n #\n # LICENSE -> LICENSE\n # scripts/gen-authors.sh -> scripts/gen-authors.sh\n\n cat whisper-src.patch | sed -E \\\n -e 's/\\/ggml\\/CMakeLists\\.txt/\\/CMakeLists.txt/g' \\\n -e 's/\\/ggml\\/src\\/CMakeLists\\.txt/\\/src\\/CMakeLists.txt/g' \\\n -e 's/\\/ggml\\/cmake\\/FindSIMD\\.cmake/\\/cmake\\/FindSIMD.cmake/g' \\\n -e 's/\\/ggml\\/src\\/ggml(.*)/\\/src\\/ggml\\1/g' \\\n -e 's/\\/ggml\\/src\\/gguf(.*)/\\/src\\/gguf\\1/g' \\\n -e 's/\\/ggml\\/include\\/ggml(.*)\\.h/\\/include\\/ggml\\1.h/g' \\\n -e 's/\\/ggml\\/include\\/gguf(.*)\\.h/\\/include\\/gguf\\1.h/g' \\\n -e 's/\\/examples\\/common\\.h/\\/examples\\/common.h/g' \\\n -e 's/\\/examples\\/common\\.cpp/\\/examples\\/common.cpp/g' \\\n -e 's/\\/examples\\/common-ggml\\.h/\\/examples\\/common-ggml.h/g' \\\n -e 's/\\/examples\\/common-ggml\\.cpp/\\/examples\\/common-ggml.cpp/g' \\\n -e 's/\\/LICENSE/\\/LICENSE/g' \\\n -e 's/\\/scripts\\/gen-authors\\.sh/\\/scripts\\/gen-authors.sh/g' \\\n > whisper-src.patch.tmp\n mv whisper-src.patch.tmp whisper-src.patch\n\n git am whisper-src.patch\n\n rm -v $SRC_GGML/whisper-src.patch\nfi\n\n# update last commit\ncd $SRC_WHISPER\ngit log -1 --format=%H > $SRC_GGML/scripts/sync-whisper.last\n\necho \"Done\"\n\nexit 0\n" }, { "path": "scripts/sync-whisper.last", "content": "79218f51d02ffe70575ef7fba3496dfc7adda027\n" }, { "path": "scripts/sync-whisper.sh", "content": "#!/bin/bash\n\ncp -rpv ../whisper.cpp/ggml/CMakeLists.txt CMakeLists.txt\ncp -rpv ../whisper.cpp/ggml/src/CMakeLists.txt src/CMakeLists.txt\ncp -rpv ../whisper.cpp/ggml/cmake/FindSIMD.cmake cmake/FindSIMD.cmake\n\ncp -rpv ../whisper.cpp/ggml/src/ggml* src/\ncp -rpv ../whisper.cpp/ggml/src/gguf* src/\n\ncp -rpv ../whisper.cpp/ggml/include/ggml*.h include/\ncp -rpv ../whisper.cpp/ggml/include/gguf*.h include/\n\ncp -rpv ../whisper.cpp/examples/common-ggml.h examples/common-ggml.h\ncp -rpv ../whisper.cpp/examples/common-ggml.cpp examples/common-ggml.cpp\n\ncp -rpv ../whisper.cpp/LICENSE ./LICENSE\ncp -rpv ../whisper.cpp/scripts/gen-authors.sh ./scripts/gen-authors.sh\n" }, { "path": "src/CMakeLists.txt", "content": "include(CheckCXXCompilerFlag)\ninclude(\"../cmake/common.cmake\")\n\nadd_compile_definitions(GGML_SCHED_MAX_COPIES=${GGML_SCHED_MAX_COPIES})\n\n# enable libstdc++ assertions for debug builds\nif (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n add_compile_definitions($<$:_GLIBCXX_ASSERTIONS>)\nendif()\n\nif (NOT MSVC)\n if (GGML_SANITIZE_THREAD)\n add_compile_options(-fsanitize=thread)\n link_libraries (-fsanitize=thread)\n endif()\n\n if (GGML_SANITIZE_ADDRESS)\n add_compile_options(-fsanitize=address -fno-omit-frame-pointer)\n link_libraries (-fsanitize=address)\n endif()\n\n if (GGML_SANITIZE_UNDEFINED)\n add_compile_options(-fsanitize=undefined)\n link_libraries (-fsanitize=undefined)\n endif()\nendif()\n\nif (GGML_FATAL_WARNINGS)\n if (CMAKE_CXX_COMPILER_ID MATCHES \"GNU\" OR CMAKE_CXX_COMPILER_ID MATCHES \"Clang\")\n list(APPEND C_FLAGS -Werror)\n list(APPEND CXX_FLAGS -Werror)\n elseif (CMAKE_CXX_COMPILER_ID STREQUAL \"MSVC\")\n add_compile_options(/WX)\n endif()\nendif()\n\nif (GGML_ALL_WARNINGS)\n if (NOT MSVC)\n list(APPEND WARNING_FLAGS -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function)\n list(APPEND C_FLAGS -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes\n -Werror=implicit-int -Werror=implicit-function-declaration)\n list(APPEND CXX_FLAGS -Wmissing-declarations -Wmissing-noreturn)\n\n list(APPEND C_FLAGS ${WARNING_FLAGS})\n list(APPEND CXX_FLAGS ${WARNING_FLAGS})\n\n ggml_get_flags(${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})\n\n add_compile_options(\"$<$:${C_FLAGS};${GF_C_FLAGS}>\"\n \"$<$:${CXX_FLAGS};${GF_CXX_FLAGS}>\")\n else()\n # todo : msvc\n set(C_FLAGS \"\")\n set(CXX_FLAGS \"\")\n endif()\nendif()\n\nif (GGML_LTO)\n include(CheckIPOSupported)\n check_ipo_supported(RESULT result OUTPUT output)\n if (result)\n set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)\n else()\n message(WARNING \"IPO is not supported: ${output}\")\n endif()\nendif()\n\nif (GGML_CCACHE AND NOT CMAKE_C_COMPILER_LAUNCHER AND NOT CMAKE_CXX_COMPILER_LAUNCHER)\n find_program(GGML_CCACHE_FOUND ccache)\n find_program(GGML_SCCACHE_FOUND sccache)\n\n if (GGML_CCACHE_FOUND OR GGML_SCCACHE_FOUND)\n if(GGML_CCACHE_FOUND)\n set(GGML_CCACHE_VARIANT ccache)\n else()\n set(GGML_CCACHE_VARIANT sccache)\n endif()\n # TODO: should not be set globally\n if (GGML_SYCL AND GGML_CCACHE_FOUND AND WIN32)\n set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE \"ccache compiler_type=icl\")\n else ()\n set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE \"${GGML_CCACHE_VARIANT}\")\n endif ()\n set(ENV{CCACHE_SLOPPINESS} time_macros)\n message(STATUS \"${GGML_CCACHE_VARIANT} found, compilation results will be cached. Disable with GGML_CCACHE=OFF.\")\n else()\n message(STATUS \"Warning: ccache not found - consider installing it for faster compilation or disable this warning with GGML_CCACHE=OFF\")\n endif ()\nendif()\n\n# this version of Apple ld64 is buggy\nexecute_process(\n COMMAND ${CMAKE_C_COMPILER} ${CMAKE_EXE_LINKER_FLAGS} -Wl,-v\n ERROR_VARIABLE output\n OUTPUT_QUIET\n)\n\nif (output MATCHES \"dyld-1015\\.7\")\n add_compile_definitions(HAVE_BUGGY_APPLE_LINKER)\nendif()\n\n# architecture specific\n# TODO: probably these flags need to be tweaked on some architectures\n# feel free to update the Makefile for your architecture and send a pull request or issue\nmessage(STATUS \"CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}\")\nif (MSVC)\n string(TOLOWER \"${CMAKE_GENERATOR_PLATFORM}\" CMAKE_GENERATOR_PLATFORM_LWR)\n message(STATUS \"CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}\")\nelse ()\n set(CMAKE_GENERATOR_PLATFORM_LWR \"\")\nendif ()\nggml_get_system_arch()\nmessage(STATUS \"GGML_SYSTEM_ARCH: ${GGML_SYSTEM_ARCH}\")\n\nif (NOT MSVC)\n if (GGML_STATIC)\n if (UNIX AND NOT APPLE)\n set(CMAKE_FIND_LIBRARY_SUFFIXES \".a;.so\")\n endif()\n add_link_options(-static)\n if (MINGW)\n add_link_options(-static-libgcc -static-libstdc++)\n endif()\n endif()\n if (GGML_GPROF)\n add_compile_options(-pg)\n endif()\nendif()\n\n#\n# POSIX conformance\n#\n\n# clock_gettime came in POSIX.1b (1993)\n# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional\n# posix_memalign came in POSIX.1-2001 / SUSv3\n# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985)\n\n# Somehow in OpenBSD whenever POSIX conformance is specified\n# some string functions rely on locale_t availability,\n# which was introduced in POSIX.1-2008, forcing us to go higher\nif (CMAKE_SYSTEM_NAME MATCHES \"OpenBSD\")\n add_compile_definitions(_XOPEN_SOURCE=700)\nelseif (CMAKE_SYSTEM_NAME MATCHES \"AIX\")\n # Don't define _XOPEN_SOURCE. We need _ALL_SOURCE, which is the default,\n # in order to define _SC_PHYS_PAGES.\nelse()\n add_compile_definitions(_XOPEN_SOURCE=600)\nendif()\n\n# Data types, macros and functions related to controlling CPU affinity and\n# some memory allocation are available on Linux through GNU extensions in libc\nif (CMAKE_SYSTEM_NAME MATCHES \"Linux\" OR CMAKE_SYSTEM_NAME MATCHES \"Android\")\n add_compile_definitions(_GNU_SOURCE)\nendif()\n\n# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1,\n# and on macOS its availability depends on enabling Darwin extensions\n# similarly on DragonFly, enabling BSD extensions is necessary\nif (\n CMAKE_SYSTEM_NAME MATCHES \"Darwin\" OR\n CMAKE_SYSTEM_NAME MATCHES \"iOS\" OR\n CMAKE_SYSTEM_NAME MATCHES \"tvOS\" OR\n CMAKE_SYSTEM_NAME MATCHES \"DragonFly\"\n)\n add_compile_definitions(_DARWIN_C_SOURCE)\nendif()\n\n# alloca is a non-standard interface that is not visible on BSDs when\n# POSIX conformance is specified, but not all of them provide a clean way\n# to enable it in such cases\nif (CMAKE_SYSTEM_NAME MATCHES \"FreeBSD\")\n add_compile_definitions(__BSD_VISIBLE)\nendif()\nif (CMAKE_SYSTEM_NAME MATCHES \"NetBSD\")\n add_compile_definitions(_NETBSD_SOURCE)\nendif()\nif (CMAKE_SYSTEM_NAME MATCHES \"OpenBSD\")\n add_compile_definitions(_BSD_SOURCE)\nendif()\n\nif (WIN32)\n add_compile_definitions(_CRT_SECURE_NO_WARNINGS)\nendif()\n\n# ggml\n\nif (GGML_BACKEND_DL AND NOT BUILD_SHARED_LIBS)\n message(FATAL_ERROR \"GGML_BACKEND_DL requires BUILD_SHARED_LIBS\")\nendif()\n\nadd_library(ggml-base\n ../include/ggml.h\n ../include/ggml-alloc.h\n ../include/ggml-backend.h\n ../include/ggml-cpp.h\n ../include/ggml-opt.h\n ../include/gguf.h\n ggml.c\n ggml.cpp\n ggml-alloc.c\n ggml-backend.cpp\n ggml-opt.cpp\n ggml-threading.cpp\n ggml-threading.h\n ggml-quants.c\n ggml-quants.h\n gguf.cpp)\n\nset_target_properties(ggml-base PROPERTIES\n VERSION ${GGML_VERSION}\n SOVERSION ${GGML_VERSION_MAJOR}\n)\n\ntarget_include_directories(ggml-base PRIVATE .)\nif (GGML_BACKEND_DL)\n target_compile_definitions(ggml-base PUBLIC GGML_BACKEND_DL)\nendif()\n\nif (GGML_SCHED_NO_REALLOC)\n target_compile_definitions(ggml-base PUBLIC GGML_SCHED_NO_REALLOC)\nendif()\n\nadd_library(ggml\n ggml-backend-dl.cpp\n ggml-backend-reg.cpp)\nadd_library(ggml::ggml ALIAS ggml)\n\nset_target_properties(ggml PROPERTIES\n VERSION ${GGML_VERSION}\n SOVERSION ${GGML_VERSION_MAJOR}\n)\n\nif (GGML_BACKEND_DIR)\n if (NOT GGML_BACKEND_DL)\n message(FATAL_ERROR \"GGML_BACKEND_DIR requires GGML_BACKEND_DL\")\n endif()\n target_compile_definitions(ggml PUBLIC GGML_BACKEND_DIR=\"${GGML_BACKEND_DIR}\")\nendif()\n\ntarget_link_libraries(ggml PUBLIC ggml-base)\n\nif (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n target_link_libraries(ggml PRIVATE dl)\nendif()\n\nfunction(ggml_add_backend_library backend)\n if (GGML_BACKEND_DL)\n add_library(${backend} MODULE ${ARGN})\n # write the shared library to the output directory\n set_target_properties(${backend} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})\n target_compile_definitions(${backend} PRIVATE GGML_BACKEND_DL)\n add_dependencies(ggml ${backend})\n if (GGML_BACKEND_DIR)\n install(TARGETS ${backend} LIBRARY DESTINATION ${GGML_BACKEND_DIR})\n else()\n install(TARGETS ${backend} LIBRARY DESTINATION ${CMAKE_INSTALL_BINDIR})\n endif()\n else()\n add_library(${backend} ${ARGN})\n target_link_libraries(ggml PUBLIC ${backend})\n install(TARGETS ${backend} LIBRARY)\n endif()\n\n target_link_libraries(${backend} PRIVATE ggml-base)\n target_include_directories(${backend} PRIVATE ..)\n\n if (${BUILD_SHARED_LIBS})\n target_compile_definitions(${backend} PRIVATE GGML_BACKEND_BUILD)\n target_compile_definitions(${backend} PUBLIC GGML_BACKEND_SHARED)\n endif()\n\n # Set versioning properties for all backend libraries\n # Building a MODULE library with a version is not supported on macOS (https://gitlab.kitware.com/cmake/cmake/-/issues/20782)\n if (NOT (APPLE AND GGML_BACKEND_DL))\n set_target_properties(${backend} PROPERTIES\n VERSION ${GGML_VERSION}\n SOVERSION ${GGML_VERSION_MAJOR}\n )\n endif()\n\n if(NOT GGML_AVAILABLE_BACKENDS)\n set(GGML_AVAILABLE_BACKENDS \"${backend}\"\n CACHE INTERNAL \"List of backends for cmake package\")\n else()\n list(FIND GGML_AVAILABLE_BACKENDS \"${backend}\" has_backend)\n if(has_backend EQUAL -1)\n set(GGML_AVAILABLE_BACKENDS \"${GGML_AVAILABLE_BACKENDS};${backend}\"\n CACHE INTERNAL \"List of backends for cmake package\")\n endif()\n endif()\nendfunction()\n\nfunction(ggml_add_backend backend)\n string(TOUPPER \"GGML_${backend}\" backend_id)\n if (${backend_id})\n string(TOLOWER \"ggml-${backend}\" backend_target)\n add_subdirectory(${backend_target})\n message(STATUS \"Including ${backend} backend\")\n if (NOT GGML_BACKEND_DL)\n string(TOUPPER \"GGML_USE_${backend}\" backend_use)\n target_compile_definitions(ggml PUBLIC ${backend_use})\n endif()\n endif()\nendfunction()\n\nfunction(ggml_add_cpu_backend_variant tag_name)\n set(GGML_CPU_TAG_NAME ${tag_name})\n # other: OPENMP LLAMAFILE CPU_HBM\n if (GGML_SYSTEM_ARCH STREQUAL \"x86\")\n foreach (feat NATIVE\n SSE42\n AVX AVX2 BMI2 AVX_VNNI FMA F16C\n AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16\n AMX_TILE AMX_INT8 AMX_BF16)\n set(GGML_${feat} OFF)\n endforeach()\n\n foreach (feat ${ARGN})\n set(GGML_${feat} ON)\n endforeach()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"ARM\")\n foreach (feat ${ARGN})\n set(GGML_INTERNAL_${feat} ON)\n endforeach()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"PowerPC\")\n foreach (feat ${ARGN})\n set(GGML_INTERNAL_${feat} ON)\n endforeach()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"s390x\")\n foreach (feat VXE2 NNPA)\n set(GGML_INTERNAL_${feat} OFF)\n endforeach()\n\n foreach (feat ${ARGN})\n set(GGML_INTERNAL_${feat} ON)\n endforeach()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"riscv64\")\n foreach (feat RVV)\n set(GGML_INTERNAL_${feat} OFF)\n endforeach()\n\n foreach (feat ${ARGN})\n set(GGML_INTERNAL_${feat} ON)\n endforeach()\n endif()\n\n ggml_add_cpu_backend_variant_impl(${tag_name})\nendfunction()\n\nggml_add_backend(CPU)\n\nif (GGML_CPU_ALL_VARIANTS)\n if (NOT GGML_BACKEND_DL)\n message(FATAL_ERROR \"GGML_CPU_ALL_VARIANTS requires GGML_BACKEND_DL\")\n elseif (GGML_CPU_ARM_ARCH)\n message(FATAL_ERROR \"Cannot use both GGML_CPU_ARM_ARCH and GGML_CPU_ALL_VARIANTS\")\n endif()\n if (GGML_SYSTEM_ARCH STREQUAL \"x86\")\n ggml_add_cpu_backend_variant(x64)\n ggml_add_cpu_backend_variant(sse42 SSE42)\n ggml_add_cpu_backend_variant(sandybridge SSE42 AVX)\n if (NOT MSVC)\n # __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512\n ggml_add_cpu_backend_variant(ivybridge SSE42 AVX F16C)\n ggml_add_cpu_backend_variant(piledriver SSE42 AVX F16C FMA)\n endif()\n ggml_add_cpu_backend_variant(haswell SSE42 AVX F16C FMA AVX2 BMI2)\n ggml_add_cpu_backend_variant(skylakex SSE42 AVX F16C FMA AVX2 BMI2 AVX512)\n ggml_add_cpu_backend_variant(cannonlake SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VBMI)\n ggml_add_cpu_backend_variant(cascadelake SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VNNI)\n ggml_add_cpu_backend_variant(icelake SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VBMI AVX512_VNNI)\n if (NOT MSVC)\n # MSVC 2022 doesn't support BF16 intrinsics without `/arch:AVX10.1` ?!\n # https://learn.microsoft.com/en-us/cpp/intrinsics/x64-amd64-intrinsics-list?view=msvc-170\n # https://learn.microsoft.com/en-us/cpp/build/reference/arch-x64?view=msvc-170\n ggml_add_cpu_backend_variant(cooperlake SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VNNI AVX512_BF16)\n ggml_add_cpu_backend_variant(zen4 SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16)\n endif()\n ggml_add_cpu_backend_variant(alderlake SSE42 AVX F16C FMA AVX2 BMI2 AVX_VNNI)\n if (NOT MSVC)\n # MSVC doesn't support AMX\n ggml_add_cpu_backend_variant(sapphirerapids SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16 AMX_TILE AMX_INT8)\n endif()\n elseif(GGML_SYSTEM_ARCH STREQUAL \"ARM\")\n if (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n # Many of these features are optional so we build versions with popular\n # combinations and name the backends based on the version they were\n # first released with\n ggml_add_cpu_backend_variant(armv8.0_1)\n ggml_add_cpu_backend_variant(armv8.2_1 DOTPROD)\n ggml_add_cpu_backend_variant(armv8.2_2 DOTPROD FP16_VECTOR_ARITHMETIC)\n ggml_add_cpu_backend_variant(armv8.2_3 DOTPROD FP16_VECTOR_ARITHMETIC SVE)\n ggml_add_cpu_backend_variant(armv8.6_1 DOTPROD FP16_VECTOR_ARITHMETIC SVE MATMUL_INT8)\n ggml_add_cpu_backend_variant(armv8.6_2 DOTPROD FP16_VECTOR_ARITHMETIC SVE MATMUL_INT8 SVE2)\n ggml_add_cpu_backend_variant(armv9.2_1 DOTPROD FP16_VECTOR_ARITHMETIC SVE MATMUL_INT8 SME)\n ggml_add_cpu_backend_variant(armv9.2_2 DOTPROD FP16_VECTOR_ARITHMETIC SVE MATMUL_INT8 SVE2 SME)\n elseif (CMAKE_SYSTEM_NAME MATCHES \"Android\")\n # Android-specific backends with SoC-compatible feature sets\n ggml_add_cpu_backend_variant(android_armv8.0_1)\n ggml_add_cpu_backend_variant(android_armv8.2_1 DOTPROD)\n ggml_add_cpu_backend_variant(android_armv8.2_2 DOTPROD FP16_VECTOR_ARITHMETIC)\n ggml_add_cpu_backend_variant(android_armv8.6_1 DOTPROD FP16_VECTOR_ARITHMETIC MATMUL_INT8)\n ggml_add_cpu_backend_variant(android_armv9.0_1 DOTPROD MATMUL_INT8 FP16_VECTOR_ARITHMETIC SVE2)\n ggml_add_cpu_backend_variant(android_armv9.2_1 DOTPROD MATMUL_INT8 FP16_VECTOR_ARITHMETIC SVE SME)\n ggml_add_cpu_backend_variant(android_armv9.2_2 DOTPROD MATMUL_INT8 FP16_VECTOR_ARITHMETIC SVE SVE2 SME)\n elseif (APPLE)\n ggml_add_cpu_backend_variant(apple_m1 DOTPROD)\n ggml_add_cpu_backend_variant(apple_m2_m3 DOTPROD MATMUL_INT8)\n ggml_add_cpu_backend_variant(apple_m4 DOTPROD MATMUL_INT8 NOSVE SME)\n else()\n message(FATAL_ERROR \"Unsupported ARM target OS: ${CMAKE_SYSTEM_NAME}\")\n endif()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"PowerPC\")\n if (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n ggml_add_cpu_backend_variant(power0)\n ggml_add_cpu_backend_variant(power7_1 POWER7)\n ggml_add_cpu_backend_variant(power7_2 POWER7 VSX)\n ggml_add_cpu_backend_variant(power8_1 POWER8)\n ggml_add_cpu_backend_variant(power8_2 POWER8 VSX)\n ggml_add_cpu_backend_variant(power9 POWER9 VSX)\n ggml_add_cpu_backend_variant(power10 POWER10 VSX)\n ggml_add_cpu_backend_variant(power11 POWER11 VSX)\n else()\n message(FATAL_ERROR \"Unsupported PowerPC target OS: ${CMAKE_SYSTEM_NAME}\")\n endif()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"s390x\")\n if (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n ggml_add_cpu_backend_variant(z15 Z15 VXE2)\n ggml_add_cpu_backend_variant(z16 Z16 VXE2 NNPA)\n else()\n message(FATAL_ERROR \"Unsupported s390x target OS: ${CMAKE_SYSTEM_NAME}\")\n endif()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"riscv64\")\n if (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n ggml_add_cpu_backend_variant(riscv64_0)\n ggml_add_cpu_backend_variant(riscv64_v RVV)\n else()\n message(FATAL_ERROR \"Unsupported RISC-V target OS: ${CMAKE_SYSTEM_NAME}\")\n endif()\n else()\n message(FATAL_ERROR \"GGML_CPU_ALL_VARIANTS not yet supported with ${GGML_SYSTEM_ARCH} on ${CMAKE_SYSTEM_NAME}\")\n endif()\nelseif (GGML_CPU)\n ggml_add_cpu_backend_variant_impl(\"\")\nendif()\n\nggml_add_backend(BLAS)\nggml_add_backend(CANN)\nggml_add_backend(CUDA)\nggml_add_backend(HIP)\nggml_add_backend(METAL)\nggml_add_backend(MUSA)\nggml_add_backend(RPC)\nggml_add_backend(VirtGPU)\nggml_add_backend(SYCL)\nggml_add_backend(Vulkan)\nggml_add_backend(WebGPU)\nggml_add_backend(zDNN)\nggml_add_backend(OpenCL)\nggml_add_backend(Hexagon)\nggml_add_backend(ZenDNN)\nggml_add_backend(OPENVINO)\n\nforeach (target ggml-base ggml)\n target_include_directories(${target} PUBLIC $ $)\n target_compile_features (${target} PRIVATE c_std_11 cxx_std_17) # don't bump\nendforeach()\n\ntarget_link_libraries(ggml-base PRIVATE Threads::Threads)\n\nfind_library(MATH_LIBRARY m)\nif (MATH_LIBRARY)\n if (NOT WIN32 OR NOT DEFINED ENV{ONEAPI_ROOT})\n target_link_libraries(ggml-base PRIVATE m)\n endif()\nendif()\n\nif (CMAKE_SYSTEM_NAME MATCHES \"Android\")\n target_link_libraries(ggml-base PRIVATE dl)\nendif()\n\nif(CMAKE_SYSTEM_NAME MATCHES \"visionOS\")\n target_compile_definitions(ggml-base PUBLIC _DARWIN_C_SOURCE)\nendif()\n\nif (BUILD_SHARED_LIBS)\n foreach (target ggml-base ggml)\n set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON)\n target_compile_definitions(${target} PRIVATE GGML_BUILD)\n target_compile_definitions(${target} PUBLIC GGML_SHARED)\n endforeach()\nendif()\n" }, { "path": "src/ggml-alloc.c", "content": "#include \"ggml-alloc.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml.h\"\n#include \"ggml-impl.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n\n#define MAX(a, b) ((a) > (b) ? (a) : (b))\n#define MAX_FREE_BLOCKS 256\n\n//#define GGML_ALLOCATOR_DEBUG\n\n//#define AT_PRINTF(...) GGML_LOG_DEBUG(__VA_ARGS__)\n#define AT_PRINTF(...)\n\n// ops that return true for this function must not use restrict pointers for their backend implementations\nbool ggml_op_can_inplace(enum ggml_op op) {\n switch (op) {\n case GGML_OP_FILL:\n case GGML_OP_SCALE:\n case GGML_OP_DIAG_MASK_ZERO:\n case GGML_OP_DIAG_MASK_INF:\n case GGML_OP_ADD:\n case GGML_OP_ADD_ID:\n case GGML_OP_ADD1:\n case GGML_OP_SUB:\n case GGML_OP_MUL:\n case GGML_OP_DIV:\n case GGML_OP_SQR:\n case GGML_OP_SQRT:\n case GGML_OP_LOG:\n case GGML_OP_UNARY:\n case GGML_OP_ROPE:\n case GGML_OP_ROPE_BACK:\n case GGML_OP_SILU_BACK:\n case GGML_OP_RMS_NORM:\n case GGML_OP_RMS_NORM_BACK:\n case GGML_OP_SOFT_MAX:\n case GGML_OP_SOFT_MAX_BACK:\n return true;\n\n default:\n return false;\n }\n}\n\nstatic size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) {\n assert(alignment && !(alignment & (alignment - 1))); // power of 2\n size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment;\n return offset + align;\n}\n\n// tallocr\n\nstruct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer) {\n void * base = ggml_backend_buffer_get_base(buffer);\n size_t align = ggml_backend_buffer_get_alignment(buffer);\n\n assert(align && !(align & (align - 1))); // power of 2\n\n struct ggml_tallocr talloc = (struct ggml_tallocr) {\n /*.buffer = */ buffer,\n /*.base = */ base,\n /*.alignment = */ align,\n /*.offset = */ aligned_offset(base, 0, align),\n };\n return talloc;\n}\n\nenum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) {\n size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor);\n size = GGML_PAD(size, talloc->alignment);\n\n if (talloc->offset + size > ggml_backend_buffer_get_size(talloc->buffer)) {\n GGML_LOG_ERROR(\"%s: not enough space in the buffer to allocate %s (needed %zu, available %zu)\\n\",\n __func__, tensor->name, size, ggml_backend_buffer_get_size(talloc->buffer) - talloc->offset);\n GGML_ABORT(\"not enough space in the buffer\");\n }\n\n void * addr = (char *)ggml_backend_buffer_get_base(talloc->buffer) + talloc->offset;\n talloc->offset += size;\n\n assert(((uintptr_t)addr % talloc->alignment) == 0);\n\n return ggml_backend_tensor_alloc(talloc->buffer, tensor, addr);\n}\n\n// dynamic tensor allocator\n\n#define GGML_VBUFFER_MAX_CHUNKS 16\n\n// relative memory address within an allocation that can be split into multiple buffers (chunks)\nstruct buffer_address {\n int chunk; // index of a backend buffer\n size_t offset; // local memory offset within the buffer\n};\n\nstatic const struct buffer_address GGML_BUFFER_ADDRESS_INVALID = { -1, SIZE_MAX };\n\nstatic bool ggml_buffer_address_less(struct buffer_address a, struct buffer_address b) {\n return a.chunk != b.chunk ? a.chunk < b.chunk : a.offset < b.offset;\n}\n\nstruct free_block {\n size_t offset;\n size_t size;\n};\n\nstruct tallocr_chunk {\n struct free_block free_blocks[MAX_FREE_BLOCKS];\n int n_free_blocks;\n size_t max_size;\n};\n\nstruct ggml_dyn_tallocr {\n size_t alignment;\n size_t max_chunk_size;\n struct tallocr_chunk * chunks[GGML_VBUFFER_MAX_CHUNKS];\n int n_chunks;\n\n#ifdef GGML_ALLOCATOR_DEBUG\n struct {\n const struct ggml_tensor * tensor;\n struct buffer_address addr;\n } allocated_tensors[1024];\n#endif\n};\n\nstatic void ggml_dyn_tallocr_insert_block(struct tallocr_chunk * chunk, size_t offset, size_t size) {\n GGML_ASSERT(chunk->n_free_blocks < MAX_FREE_BLOCKS && \"out of free blocks\");\n // insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster)\n int insert_pos = 0;\n while (insert_pos < chunk->n_free_blocks && chunk->free_blocks[insert_pos].offset < offset) {\n insert_pos++;\n }\n // shift all blocks from insert_pos onward to make room for the new block\n for (int i = chunk->n_free_blocks; i > insert_pos; i--) {\n chunk->free_blocks[i] = chunk->free_blocks[i-1];\n }\n // insert the new block\n chunk->free_blocks[insert_pos].offset = offset;\n chunk->free_blocks[insert_pos].size = size;\n chunk->n_free_blocks++;\n}\n\nstatic void ggml_dyn_tallocr_remove_block(struct tallocr_chunk * chunk, int idx) {\n // shift all elements after idx by 1 to the left, overwriting the element at idx\n for (int i = idx; i < chunk->n_free_blocks; i++) {\n chunk->free_blocks[i] = chunk->free_blocks[i+1];\n }\n chunk->n_free_blocks--;\n}\n\nstatic int ggml_dyn_tallocr_new_chunk(struct ggml_dyn_tallocr * alloc, size_t min_size) {\n if (alloc->n_chunks >= GGML_VBUFFER_MAX_CHUNKS) {\n return -1;\n }\n struct tallocr_chunk * chunk = calloc(1, sizeof(struct tallocr_chunk));\n chunk->n_free_blocks = 1;\n chunk->free_blocks[0].offset = 0;\n // available space in a chunk is limited to max_chunk_size, but can be higher if:\n // 1. a single tensor exceeds the maximum, and cannot fit any other way\n // 2. we are running out of chunks\n // backends will either manage to allocate the larger size, or report an error.\n chunk->free_blocks[0].size = MAX(min_size, alloc->max_chunk_size);\n if (alloc->n_chunks == GGML_VBUFFER_MAX_CHUNKS - 1) {\n chunk->free_blocks[0].size = SIZE_MAX/2;\n }\n alloc->chunks[alloc->n_chunks] = chunk;\n alloc->n_chunks++;\n return alloc->n_chunks - 1;\n}\n\n#ifdef GGML_ALLOCATOR_DEBUG\nstatic void add_allocated_tensor(struct ggml_dyn_tallocr * alloc, struct buffer_address addr, const struct ggml_tensor * tensor) {\n for (int i = 0; i < 1024; i++) {\n if (alloc->allocated_tensors[i].tensor == NULL) {\n alloc->allocated_tensors[i].tensor = tensor;\n alloc->allocated_tensors[i].addr = addr;\n return;\n }\n }\n GGML_ABORT(\"out of allocated_tensors\");\n}\nstatic void remove_allocated_tensor(struct ggml_dyn_tallocr * alloc, struct buffer_address addr, const struct ggml_tensor * tensor) {\n for (int i = 0; i < 1024; i++) {\n if (alloc->allocated_tensors[i].addr.chunk == addr.chunk && alloc->allocated_tensors[i].addr.offset == addr.offset) {\n alloc->allocated_tensors[i].tensor = NULL;\n return;\n }\n }\n GGML_ABORT(\"tried to free tensor %s not found\\n\", tensor->name);\n}\n#endif\n\nstatic struct buffer_address ggml_dyn_tallocr_alloc(struct ggml_dyn_tallocr * alloc, size_t size, const struct ggml_tensor * tensor) {\n size = aligned_offset(NULL, size, alloc->alignment);\n\n AT_PRINTF(\"%s: allocating %s (%zu bytes) - \", __func__, tensor->name, size);\n\n int best_fit_chunk = -1;\n int best_fit_block = -1;\n size_t max_avail = 0;\n\n // find the best fitting free block besides the last block, within any chunk\n for (int c = 0; c < alloc->n_chunks; ++c) {\n struct tallocr_chunk * chunk = alloc->chunks[c];\n size_t best_fit_size = SIZE_MAX;\n for (int i = 0; i < chunk->n_free_blocks - 1; i++) {\n struct free_block * block = &chunk->free_blocks[i];\n max_avail = MAX(max_avail, block->size);\n if (block->size >= size && block->size <= best_fit_size) {\n best_fit_chunk = c;\n best_fit_block = i;\n best_fit_size = block->size;\n }\n }\n }\n\n if (best_fit_block == -1) {\n // no suitable block found, try the last block (this may grow a chunks size)\n int64_t best_reuse = INT64_MIN;\n for (int c = 0; c < alloc->n_chunks; ++c) {\n struct tallocr_chunk * chunk = alloc->chunks[c];\n if (chunk->n_free_blocks > 0) {\n struct free_block * block = &chunk->free_blocks[chunk->n_free_blocks - 1];\n max_avail = MAX(max_avail, block->size);\n int64_t reuse_factor = chunk->max_size - block->offset - size;\n // reuse_factor < 0 : amount of extra memory that needs to be allocated\n // reuse_factor = 0 : allocated free space exactly matches tensor size\n // reuse_factor > 0 : superfluous memory that will remain unused\n bool better_reuse = best_reuse < 0 && reuse_factor > best_reuse;\n bool better_fit = reuse_factor >= 0 && reuse_factor < best_reuse;\n if (block->size >= size && (better_reuse || better_fit)) {\n best_fit_chunk = c;\n best_fit_block = chunk->n_free_blocks - 1;\n best_reuse = reuse_factor;\n }\n }\n }\n }\n\n if (best_fit_block == -1) {\n // none of the existing chunks have enough space left\n best_fit_chunk = ggml_dyn_tallocr_new_chunk(alloc, size);\n best_fit_block = 0;\n }\n if (best_fit_chunk == -1) {\n // since the last chunk always has virtually endless memory, this should never happen\n GGML_LOG_ERROR(\"%s: not enough space in the buffer to allocate %zu bytes, largest block available %zu bytes\\n\",\n __func__, size, max_avail);\n GGML_ABORT(\"graph allocation: failed to reserve memory\");\n }\n\n struct tallocr_chunk * chunk = alloc->chunks[best_fit_chunk];\n struct free_block * block = &chunk->free_blocks[best_fit_block];\n struct buffer_address addr = {.chunk = best_fit_chunk, .offset = block->offset };\n block->offset += size;\n block->size -= size;\n if (block->size == 0) {\n // remove block if empty\n ggml_dyn_tallocr_remove_block(chunk, best_fit_block);\n }\n\n AT_PRINTF(\"block %d, offset %zu, chunk %d\\n\", best_fit_block, addr.offset, addr.chunk);\n\n#ifdef GGML_ALLOCATOR_DEBUG\n add_allocated_tensor(alloc, addr, tensor);\n size_t cur_max = addr.offset + size;\n if (cur_max > chunk->max_size) {\n // sort allocated_tensors by chunk/offset\n for (int i = 0; i < 1024; i++) {\n for (int j = i + 1; j < 1024; j++) {\n if (ggml_buffer_address_less(alloc->allocated_tensors[j].addr, alloc->allocated_tensors[i].addr)) {\n const struct ggml_tensor * tmp_tensor = alloc->allocated_tensors[i].tensor;\n struct buffer_address tmp_addr = alloc->allocated_tensors[i].addr;\n alloc->allocated_tensors[i].tensor = alloc->allocated_tensors[j].tensor;\n alloc->allocated_tensors[i].addr = alloc->allocated_tensors[j].addr;\n alloc->allocated_tensors[j].tensor = tmp_tensor;\n alloc->allocated_tensors[j].addr = tmp_addr;\n }\n }\n }\n GGML_LOG_DEBUG(\"max_size[%d] = %.2f MB: tensors: \", addr.chunk, cur_max / 1024.0 / 1024.0);\n for (int i = 0; i < 1024; i++) {\n if (alloc->allocated_tensors[i].tensor) {\n GGML_LOG_DEBUG(\"%s [%d: %zx-%zx] (%.2f MB) \", alloc->allocated_tensors[i].tensor->name,\n alloc->allocated_tensors[i].addr.chunk,\n alloc->allocated_tensors[i].addr.offset,\n alloc->allocated_tensors[i].addr.offset + ggml_nbytes(alloc->allocated_tensors[i].tensor),\n ggml_nbytes(alloc->allocated_tensors[i].tensor) / 1024.0 / 1024.0);\n }\n }\n GGML_LOG_DEBUG(\"\\n\");\n }\n#endif\n\n chunk->max_size = MAX(chunk->max_size, addr.offset + size);\n\n return addr;\n\n GGML_UNUSED(tensor);\n}\n\n// this is a very naive implementation, but for our case the number of free blocks should be very small\nstatic void ggml_dyn_tallocr_free_bytes(struct ggml_dyn_tallocr * alloc, struct buffer_address addr, size_t size) {\n size = aligned_offset(NULL, size, alloc->alignment);\n\n struct tallocr_chunk * chunk = alloc->chunks[addr.chunk];\n\n // see if we can merge with an existing block\n for (int i = 0; i < chunk->n_free_blocks; i++) {\n struct free_block * block = &chunk->free_blocks[i];\n // check if ptr is at the end of the block\n if (block->offset + block->size == addr.offset) {\n block->size += size;\n // check if we can merge with the next block\n if (i < chunk->n_free_blocks - 1) {\n struct free_block * next = &chunk->free_blocks[i+1];\n if (block->offset + block->size == next->offset) {\n block->size += next->size;\n ggml_dyn_tallocr_remove_block(chunk, i+1);\n }\n }\n return;\n }\n // check if ptr is at the beginning of the block\n if (addr.offset + size == block->offset) {\n block->offset = addr.offset;\n block->size += size;\n // check if we can merge with the previous block\n if (i > 0) {\n struct free_block * prev = &chunk->free_blocks[i-1];\n if (prev->offset + prev->size == block->offset) {\n prev->size += block->size;\n ggml_dyn_tallocr_remove_block(chunk, i);\n }\n }\n return;\n }\n }\n // otherwise, add a new block\n ggml_dyn_tallocr_insert_block(chunk, addr.offset, size);\n}\n\nstatic void ggml_dyn_tallocr_reset(struct ggml_dyn_tallocr * alloc) {\n for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS; i++) {\n free(alloc->chunks[i]);\n alloc->chunks[i] = NULL;\n }\n alloc->n_chunks = 0;\n\n#ifdef GGML_ALLOCATOR_DEBUG\n for (int i = 0; i < 1024; i++) {\n alloc->allocated_tensors[i].tensor = NULL;\n }\n#endif\n}\n\nstatic struct ggml_dyn_tallocr * ggml_dyn_tallocr_new(size_t alignment, size_t max_buffer_size) {\n struct ggml_dyn_tallocr * alloc = (struct ggml_dyn_tallocr *)malloc(sizeof(struct ggml_dyn_tallocr));\n\n *alloc = (struct ggml_dyn_tallocr) {\n /*.alignment = */ alignment,\n /*.max_chunk_size = */ MIN(max_buffer_size, SIZE_MAX/2), // clamp to avoid overflows\n /*.chunks = */ {NULL},\n /*.n_chunks = */ 0,\n#ifdef GGML_ALLOCATOR_DEBUG\n /*.allocated_tensors = */ {{0}},\n#endif\n };\n\n ggml_dyn_tallocr_reset(alloc);\n\n return alloc;\n}\n\nstatic void ggml_dyn_tallocr_free(struct ggml_dyn_tallocr * alloc) {\n for (int i = 0; i < alloc->n_chunks; ++i) {\n free(alloc->chunks[i]);\n }\n free(alloc);\n}\n\nstatic size_t ggml_dyn_tallocr_max_size(struct ggml_dyn_tallocr * alloc, int chunk) {\n return chunk < alloc->n_chunks ? alloc->chunks[chunk]->max_size : 0;\n}\n\n\n// virtual buffer with contiguous memory range, split into multiple backend buffers (chunks)\n\nstruct vbuffer {\n ggml_backend_buffer_t chunks[GGML_VBUFFER_MAX_CHUNKS];\n};\n\nstatic void ggml_vbuffer_free(struct vbuffer * buf) {\n if (buf == NULL) {\n return;\n }\n for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS; ++i) {\n ggml_backend_buffer_free(buf->chunks[i]);\n }\n free(buf);\n}\n\nstatic size_t ggml_vbuffer_chunk_size(struct vbuffer * buf, int chunk) {\n return buf->chunks[chunk] ? ggml_backend_buffer_get_size(buf->chunks[chunk]) : 0;\n}\n\nstatic size_t ggml_vbuffer_size(struct vbuffer * buf) {\n size_t size = 0;\n for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS && buf->chunks[i]; ++i) {\n size += ggml_backend_buffer_get_size(buf->chunks[i]);\n }\n return size;\n}\n\nstatic struct vbuffer * ggml_vbuffer_alloc(ggml_backend_buffer_type_t buft, const struct ggml_dyn_tallocr * talloc, enum ggml_backend_buffer_usage usage) {\n struct vbuffer * buf = (struct vbuffer *)calloc(1, sizeof(struct vbuffer));\n if (buf == NULL) {\n return NULL;\n }\n\n for (int n = 0; n < talloc->n_chunks; n++) {\n size_t chunk_size = talloc->chunks[n]->max_size;\n buf->chunks[n] = ggml_backend_buft_alloc_buffer(buft, chunk_size);\n if (buf->chunks[n] == NULL) {\n ggml_vbuffer_free(buf);\n return NULL;\n }\n ggml_backend_buffer_set_usage(buf->chunks[n], usage);\n }\n return buf;\n}\n\nstatic void ggml_vbuffer_tensor_alloc(struct vbuffer * buf, struct ggml_tensor * tensor, struct buffer_address buf_addr) {\n void * base = ggml_backend_buffer_get_base(buf->chunks[buf_addr.chunk]);\n void * addr = (char *)base + buf_addr.offset;\n ggml_backend_tensor_alloc(buf->chunks[buf_addr.chunk], tensor, addr);\n}\n\nstatic void ggml_vbuffer_reset(struct vbuffer * buf) {\n for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS && buf->chunks[i]; ++i) {\n ggml_backend_buffer_reset(buf->chunks[i]);\n }\n}\n\n\n/////////////////////////////////////\n\n// graph allocator\n\nstruct hash_node {\n int n_children;\n int n_views;\n int buffer_id;\n struct buffer_address addr;\n bool allocated;\n};\n\nstruct tensor_alloc {\n int buffer_id;\n struct buffer_address addr;\n size_t size_max; // 0 = pre-allocated, unused, or view\n};\n\nstruct leaf_alloc {\n struct tensor_alloc leaf;\n};\n\nstruct node_alloc {\n struct tensor_alloc dst;\n struct tensor_alloc src[GGML_MAX_SRC];\n};\n\nstruct ggml_gallocr {\n ggml_backend_buffer_type_t * bufts; // [n_buffers]\n struct vbuffer ** buffers; // [n_buffers]\n struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers]\n int n_buffers;\n\n struct ggml_hash_set hash_set;\n struct hash_node * hash_values; // [hash_set.size]\n\n struct node_alloc * node_allocs; // [n_nodes]\n int n_nodes;\n\n struct leaf_alloc * leaf_allocs; // [n_leafs]\n int n_leafs;\n};\n\nggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs) {\n ggml_gallocr_t galloc = (ggml_gallocr_t)calloc(1, sizeof(struct ggml_gallocr));\n GGML_ASSERT(galloc != NULL);\n\n galloc->bufts = calloc(n_bufs, sizeof(ggml_backend_buffer_type_t));\n GGML_ASSERT(galloc->bufts != NULL);\n\n galloc->buffers = calloc(n_bufs, sizeof(struct vbuffer *));\n GGML_ASSERT(galloc->buffers != NULL);\n\n galloc->buf_tallocs = calloc(n_bufs, sizeof(struct ggml_dyn_tallocr *));\n GGML_ASSERT(galloc->buf_tallocs != NULL);\n\n for (int i = 0; i < n_bufs; i++) {\n galloc->bufts[i] = bufts[i];\n galloc->buffers[i] = NULL;\n\n // check if the same buffer type is used multiple times and reuse the same allocator\n for (int j = 0; j < i; j++) {\n if (bufts[i] == bufts[j]) {\n galloc->buf_tallocs[i] = galloc->buf_tallocs[j];\n break;\n }\n }\n\n if (galloc->buf_tallocs[i] == NULL) {\n size_t alignment = ggml_backend_buft_get_alignment(bufts[i]);\n size_t max_size = ggml_backend_buft_get_max_size(bufts[i]);\n galloc->buf_tallocs[i] = ggml_dyn_tallocr_new(alignment, max_size);\n }\n }\n galloc->n_buffers = n_bufs;\n\n return galloc;\n}\n\nggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft) {\n return ggml_gallocr_new_n(&buft, 1);\n}\n\nvoid ggml_gallocr_free(ggml_gallocr_t galloc) {\n if (galloc == NULL) {\n return;\n }\n\n for (int i = 0; i < galloc->n_buffers; i++) {\n if (galloc->buffers != NULL) {\n // skip if already freed\n bool freed = false;\n for (int j = 0; j < i; j++) {\n if (galloc->buffers[j] == galloc->buffers[i]) {\n freed = true;\n break;\n }\n }\n if (!freed) {\n ggml_vbuffer_free(galloc->buffers[i]);\n }\n }\n if (galloc->buf_tallocs != NULL) {\n // skip if already freed\n bool freed = false;\n for (int j = 0; j < i; j++) {\n if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {\n freed = true;\n break;\n }\n }\n if (!freed) {\n ggml_dyn_tallocr_free(galloc->buf_tallocs[i]);\n }\n }\n }\n\n ggml_hash_set_free(&galloc->hash_set);\n free(galloc->hash_values);\n free(galloc->bufts);\n free(galloc->buffers);\n free(galloc->buf_tallocs);\n free(galloc->node_allocs);\n free(galloc->leaf_allocs);\n free(galloc);\n}\n\ntypedef struct ggml_gallocr * ggml_gallocr_t;\n\nstatic struct hash_node * ggml_gallocr_hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) {\n size_t i = ggml_hash_find_or_insert(&galloc->hash_set, t);\n return &galloc->hash_values[i];\n}\n\nstatic bool ggml_gallocr_is_own(ggml_gallocr_t galloc, struct ggml_tensor * t) {\n return ggml_gallocr_hash_get(galloc, t)->allocated;\n}\n\nstatic bool ggml_gallocr_is_allocated(ggml_gallocr_t galloc, struct ggml_tensor * t) {\n return t->data != NULL // tensor data already set externally\n || t->buffer // tensor on external buffer (but not yet allocated)\n || ggml_gallocr_is_own(galloc, t); // tensor will be allocated by galloc\n}\n\n// free the extra space at the end if the new tensor is smaller\nstatic void ggml_gallocr_free_extra_space(ggml_gallocr_t galloc, struct ggml_tensor * node, struct ggml_tensor * parent) {\n struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);\n struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);\n\n size_t parent_size = ggml_backend_buft_get_alloc_size(galloc->bufts[p_hn->buffer_id], parent);\n size_t node_size = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node);\n\n GGML_ASSERT(parent_size >= node_size);\n\n // note: we want after the freeing the chunks to continue to be aligned\n struct ggml_dyn_tallocr * p_alloc = galloc->buf_tallocs[p_hn->buffer_id];\n parent_size = aligned_offset(NULL, parent_size, p_alloc->alignment);\n node_size = aligned_offset(NULL, node_size, p_alloc->alignment);\n\n if (parent_size > node_size) {\n struct buffer_address p_addr = p_hn->addr;\n p_addr.offset += node_size;\n size_t extra_size = parent_size - node_size;\n AT_PRINTF(\"freeing extra %zu bytes from parent %s for %s\\n\", extra_size, parent->name, node->name);\n ggml_dyn_tallocr_free_bytes(p_alloc, p_addr, extra_size);\n }\n}\n\nstatic void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) {\n GGML_ASSERT(buffer_id >= 0);\n struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);\n\n if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_impl_is_view(node)) {\n hn->allocated = true;\n assert(hn->addr.offset == 0);\n\n // try to reuse a parent's buffer (inplace)\n if (ggml_op_can_inplace(node->op)) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n struct ggml_tensor * parent = node->src[i];\n if (parent == NULL) {\n continue;\n }\n\n // if the node's data is external, then we cannot re-use it\n if (!ggml_gallocr_is_own(galloc, parent)) {\n AT_PRINTF(\"not reusing parent %s for %s as %p is external\\n\", parent->name, node->name, parent->data);\n continue;\n }\n\n // outputs cannot be reused\n if (parent->flags & GGML_TENSOR_FLAG_OUTPUT || (parent->view_src != NULL && parent->view_src->flags & GGML_TENSOR_FLAG_OUTPUT)) {\n AT_PRINTF(\"not reusing parent %s for %s as it is an output\\n\", parent->name, node->name);\n continue;\n }\n\n if (!ggml_are_same_layout(node, parent)) {\n AT_PRINTF(\"not reusing parent %s for %s as layouts are different\\n\", parent->name, node->name);\n continue;\n }\n\n struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);\n if (p_hn->n_children == 1 && p_hn->n_views == 0) {\n if (ggml_impl_is_view(parent)) {\n struct ggml_tensor * view_src = parent->view_src;\n struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);\n if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {\n AT_PRINTF(\"reusing view parent %s (%s) for %s\\n\", parent->name, view_src->name, node->name);\n assert(view_src_hn->addr.chunk == p_hn->addr.chunk && view_src_hn->addr.offset == p_hn->addr.offset);\n hn->buffer_id = p_hn->buffer_id;\n hn->addr = p_hn->addr;\n p_hn->allocated = false; // avoid freeing the parent\n view_src_hn->allocated = false;\n ggml_gallocr_free_extra_space(galloc, node, view_src);\n return;\n }\n } else {\n AT_PRINTF(\"reusing parent %s for %s\\n\", parent->name, node->name);\n hn->buffer_id = p_hn->buffer_id;\n hn->addr = p_hn->addr;\n p_hn->allocated = false; // avoid freeing the parent\n ggml_gallocr_free_extra_space(galloc, node, parent);\n return;\n }\n }\n }\n }\n // allocate tensor from the buffer\n struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];\n ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];\n size_t size = ggml_backend_buft_get_alloc_size(buft, node);\n hn->buffer_id = buffer_id;\n hn->addr = ggml_dyn_tallocr_alloc(alloc, size, node);\n }\n}\n\nstatic void ggml_gallocr_free_node(ggml_gallocr_t galloc, struct ggml_tensor * node) {\n // graph outputs are never freed\n if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {\n AT_PRINTF(\"not freeing output %s\\n\", node->name);\n return;\n }\n\n struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);\n int buffer_id = hn->buffer_id;\n struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];\n ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];\n size_t size = ggml_backend_buft_get_alloc_size(buft, node);\n\n AT_PRINTF(\"%s: freeing %s at {chunk=%d, offset=%zu} (%zu bytes) - n_free_blocks = %d\\n\",\n __func__, node->name, hn->addr.chunk, hn->addr.offset, size, alloc->chunks[hn->addr.chunk]->n_free_blocks);\n#ifdef GGML_ALLOCATOR_DEBUG\n remove_allocated_tensor(alloc, hn->addr, node);\n#endif\n\n ggml_dyn_tallocr_free_bytes(alloc, hn->addr, size);\n hn->allocated = false;\n}\n\nstatic int get_node_buffer_id(const int * node_buffer_ids, int i) {\n return node_buffer_ids ? node_buffer_ids[i] : 0;\n}\n\nstatic void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {\n // clear hash tables\n ggml_hash_set_reset(&galloc->hash_set);\n memset(galloc->hash_values, 0, sizeof(struct hash_node) * galloc->hash_set.size);\n\n // allocate leafs\n // these may be tensors that the application is not using in the graph, but may still want to allocate for other purposes\n for (int i = 0; i < graph->n_leafs; i++) {\n struct ggml_tensor * leaf = graph->leafs[i];\n ggml_gallocr_allocate_node(galloc, leaf, get_node_buffer_id(leaf_buffer_ids, i));\n }\n\n // count number of children and views\n // allocate other graph inputs and leafs first to avoid overwriting them\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n\n // TODO: better way to add external dependencies\n // GGML_OP_NONE does not appear normally in the graph nodes, but is used by ggml-backend to add dependencies to\n // control when some tensors are allocated and freed. in this case, the dependencies are in `src`, but the node\n // itself is never used and should not be considered a dependency\n if (ggml_impl_is_view(node) && node->op != GGML_OP_NONE) {\n struct ggml_tensor * view_src = node->view_src;\n ggml_gallocr_hash_get(galloc, view_src)->n_views += 1;\n }\n\n if (node->flags & GGML_TENSOR_FLAG_INPUT) {\n ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i));\n }\n\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n\n ggml_gallocr_hash_get(galloc, src)->n_children += 1;\n\n // allocate explicit inputs\n if (src->flags & GGML_TENSOR_FLAG_INPUT) {\n ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i));\n }\n }\n }\n\n // allocate tensors\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n int buffer_id = get_node_buffer_id(node_buffer_ids, i);\n\n // allocate parents (only leafs need to be allocated at this point)\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * parent = node->src[j];\n if (parent == NULL) {\n continue;\n }\n ggml_gallocr_allocate_node(galloc, parent, buffer_id);\n }\n\n // allocate node\n ggml_gallocr_allocate_node(galloc, node, buffer_id);\n\n AT_PRINTF(\"exec: %s (%s) <= \", ggml_op_desc(node), node->name);\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * parent = node->src[j];\n if (parent == NULL) {\n continue;\n }\n AT_PRINTF(\"%s\", parent->name);\n if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {\n AT_PRINTF(\", \");\n }\n }\n AT_PRINTF(\"\\n\");\n\n // update parents\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * parent = node->src[j];\n if (parent == NULL) {\n continue;\n }\n struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);\n p_hn->n_children -= 1;\n\n AT_PRINTF(\"parent %s: %d children, %d views, allocated: %d\\n\",\n parent->name, p_hn->n_children, p_hn->n_views, p_hn->allocated);\n\n if (p_hn->n_children == 0 && p_hn->n_views == 0) {\n if (ggml_impl_is_view(parent)) {\n struct ggml_tensor * view_src = parent->view_src;\n struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);\n view_src_hn->n_views -= 1;\n AT_PRINTF(\"view_src %s: %d children, %d views\\n\",\n view_src->name, view_src_hn->n_children, view_src_hn->n_views);\n if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src_hn->allocated) {\n ggml_gallocr_free_node(galloc, view_src);\n }\n }\n else if (p_hn->allocated) {\n ggml_gallocr_free_node(galloc, parent);\n }\n }\n AT_PRINTF(\"\\n\");\n }\n }\n}\n\nstatic bool ggml_gallocr_reserve_n_impl(\n ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids, bool no_alloc) {\n size_t min_hash_size = graph->n_nodes + graph->n_leafs;\n // add 25% margin to avoid hash collisions\n min_hash_size += min_hash_size / 4;\n\n // initialize hash table\n if (galloc->hash_set.size < min_hash_size) {\n ggml_hash_set_free(&galloc->hash_set);\n galloc->hash_set = ggml_hash_set_new(min_hash_size);\n GGML_ASSERT(galloc->hash_set.keys != NULL);\n\n free(galloc->hash_values);\n galloc->hash_values = malloc(sizeof(struct hash_node) * galloc->hash_set.size);\n GGML_ASSERT(galloc->hash_values != NULL);\n }\n\n // reset allocators\n for (int i = 0; i < galloc->n_buffers; i++) {\n ggml_dyn_tallocr_reset(galloc->buf_tallocs[i]);\n }\n\n // allocate in hash table\n ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids);\n\n // set the node_allocs from the hash table\n if (galloc->n_nodes < graph->n_nodes) {\n free(galloc->node_allocs);\n galloc->node_allocs = calloc(graph->n_nodes, sizeof(struct node_alloc));\n GGML_ASSERT(galloc->node_allocs != NULL);\n }\n galloc->n_nodes = graph->n_nodes;\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n struct node_alloc * node_alloc = &galloc->node_allocs[i];\n if (node->view_src || node->data) {\n node_alloc->dst.buffer_id = -1;\n node_alloc->dst.addr = GGML_BUFFER_ADDRESS_INVALID;\n node_alloc->dst.size_max = 0;\n } else {\n struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);\n node_alloc->dst.buffer_id = hn->buffer_id;\n node_alloc->dst.addr = hn->addr;\n node_alloc->dst.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node);\n }\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (!src || src->view_src || src->data) {\n node_alloc->src[j].buffer_id = -1;\n node_alloc->src[j].addr = GGML_BUFFER_ADDRESS_INVALID;\n node_alloc->src[j].size_max = 0;\n } else {\n struct hash_node * hn = ggml_gallocr_hash_get(galloc, src);\n node_alloc->src[j].buffer_id = hn->buffer_id;\n node_alloc->src[j].addr = hn->addr;\n node_alloc->src[j].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], src);\n }\n }\n }\n if (galloc->n_leafs < graph->n_leafs) {\n free(galloc->leaf_allocs);\n galloc->leaf_allocs = calloc(graph->n_leafs, sizeof(galloc->leaf_allocs[0]));\n GGML_ASSERT(galloc->leaf_allocs != NULL);\n }\n galloc->n_leafs = graph->n_leafs;\n for (int i = 0; i < graph->n_leafs; i++) {\n struct ggml_tensor * leaf = graph->leafs[i];\n struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf);\n if (leaf->view_src || leaf->data) {\n galloc->leaf_allocs[i].leaf.buffer_id = -1;\n galloc->leaf_allocs[i].leaf.addr = GGML_BUFFER_ADDRESS_INVALID;\n galloc->leaf_allocs[i].leaf.size_max = 0;\n } else {\n galloc->leaf_allocs[i].leaf.buffer_id = hn->buffer_id;\n galloc->leaf_allocs[i].leaf.addr = hn->addr;\n galloc->leaf_allocs[i].leaf.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf);\n }\n }\n\n // reallocate buffers if needed\n for (int i = 0; i < galloc->n_buffers; i++) {\n // if the buffer type is used multiple times, we reuse the same buffer\n for (int j = 0; j < i; j++) {\n if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {\n galloc->buffers[i] = galloc->buffers[j];\n break;\n }\n }\n\n // even if there are no tensors allocated in this buffer, we still need to allocate it to initialize views\n bool realloc = galloc->buffers[i] == NULL;\n size_t new_size = 0;\n for (int c = 0; c < galloc->buf_tallocs[i]->n_chunks; c++) {\n size_t cur_chunk_size = galloc->buffers[i] ? ggml_vbuffer_chunk_size(galloc->buffers[i], c) : 0;\n size_t new_chunk_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i], c);\n new_size += new_chunk_size;\n if (new_chunk_size > cur_chunk_size) {\n realloc = true;\n }\n }\n if (realloc) {\n#ifndef NDEBUG\n {\n size_t cur_size = galloc->buffers[i] ? ggml_vbuffer_size(galloc->buffers[i]) : 0;\n if (cur_size > 0) {\n GGML_LOG_DEBUG(\"%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\\n\",\n __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);\n }\n }\n#endif\n ggml_vbuffer_free(galloc->buffers[i]);\n if (no_alloc) {\n galloc->buffers[i] = NULL;\n } else {\n galloc->buffers[i] = ggml_vbuffer_alloc(galloc->bufts[i], galloc->buf_tallocs[i], GGML_BACKEND_BUFFER_USAGE_COMPUTE);\n if (galloc->buffers[i] == NULL) {\n GGML_LOG_ERROR(\"%s: failed to allocate %s buffer of size %zu\\n\", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size);\n return false;\n }\n }\n }\n }\n\n return true;\n}\n\nvoid ggml_gallocr_reserve_n_size(\n ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids, size_t * sizes) {\n GGML_ASSERT(ggml_gallocr_reserve_n_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids, /*no_alloc =*/ true));\n for (int i = 0; i < galloc->n_buffers; i++) {\n sizes[i] = 0;\n for (int c = 0; c < galloc->buf_tallocs[i]->n_chunks; c++) {\n sizes[i] += galloc->buf_tallocs[i]->chunks[c]->max_size;\n }\n }\n}\n\nbool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {\n return ggml_gallocr_reserve_n_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids, /*no_alloc =*/ false);\n}\n\nbool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) {\n return ggml_gallocr_reserve_n(galloc, graph, NULL, NULL);\n}\n\nstatic void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * tensor, struct tensor_alloc * tensor_alloc) {\n int buffer_id = tensor_alloc->buffer_id;\n assert(tensor->data || tensor->view_src || ggml_backend_buft_get_alloc_size(galloc->bufts[buffer_id], tensor) <= tensor_alloc->size_max);\n\n if (tensor->view_src != NULL) {\n if (tensor->buffer == NULL) {\n assert(tensor_alloc->addr.offset == SIZE_MAX);\n if (tensor->view_src->buffer == NULL) {\n // this tensor was allocated without ggml-backend\n return;\n }\n ggml_backend_view_init(tensor);\n }\n } else {\n if (tensor->data == NULL) {\n assert(tensor_alloc->addr.offset != SIZE_MAX);\n assert(ggml_backend_buft_get_alloc_size(galloc->bufts[buffer_id], tensor) <= tensor_alloc->size_max);\n ggml_vbuffer_tensor_alloc(galloc->buffers[buffer_id], tensor, tensor_alloc->addr);\n } else {\n if (tensor->buffer == NULL) {\n // this tensor was allocated without ggml-backend\n return;\n }\n }\n }\n}\n\nstatic bool ggml_gallocr_node_needs_realloc(ggml_gallocr_t galloc, struct ggml_tensor * node, struct tensor_alloc * talloc) {\n size_t node_size = 0;\n if (!node->data && !node->view_src) {\n // If we previously had data but don't now then reallocate\n if (talloc->buffer_id < 0) {\n return false;\n }\n node_size = ggml_backend_buft_get_alloc_size(galloc->bufts[talloc->buffer_id], node);\n }\n return talloc->size_max >= node_size;\n}\n\nstatic bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {\n if (galloc->n_nodes != graph->n_nodes) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: graph has different number of nodes\\n\", __func__);\n#endif\n return true;\n }\n\n if (galloc->n_leafs != graph->n_leafs) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: graph has different number of leafs\\n\", __func__);\n#endif\n return true;\n }\n\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n struct node_alloc * node_alloc = &galloc->node_allocs[i];\n\n if (!ggml_gallocr_node_needs_realloc(galloc, node, &node_alloc->dst)) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: node %s is not valid\\n\", __func__, node->name);\n#endif\n return true;\n }\n\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n if (!ggml_gallocr_node_needs_realloc(galloc, src, &node_alloc->src[j])) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: src %d (%s) of node %s is not valid\\n\", __func__, j, src->name, node->name);\n#endif\n return true;\n }\n }\n }\n\n return false;\n}\n\nbool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {\n if (ggml_gallocr_needs_realloc(galloc, graph)) {\n if (galloc->n_buffers == 1) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: reallocating buffers automatically\\n\", __func__);\n#endif\n if (!ggml_gallocr_reserve(galloc, graph)) {\n return false;\n }\n } else {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: cannot reallocate multi buffer graph automatically, call reserve\\n\", __func__);\n#endif\n return false;\n }\n }\n\n // reset buffers\n for (int i = 0; i < galloc->n_buffers; i++) {\n if (galloc->buffers[i] != NULL) {\n ggml_vbuffer_reset(galloc->buffers[i]);\n }\n }\n\n // allocate the graph tensors from the previous assignments\n // leafs\n for (int i = 0; i < graph->n_leafs; i++) {\n struct ggml_tensor * leaf = graph->leafs[i];\n struct leaf_alloc * leaf_alloc = &galloc->leaf_allocs[i];\n ggml_gallocr_init_tensor(galloc, leaf, &leaf_alloc->leaf);\n }\n // nodes\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n struct node_alloc * node_alloc = &galloc->node_allocs[i];\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n ggml_gallocr_init_tensor(galloc, src, &node_alloc->src[j]);\n }\n ggml_gallocr_init_tensor(galloc, node, &node_alloc->dst);\n }\n\n return true;\n}\n\nsize_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) {\n GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers);\n\n if (galloc->buffers[buffer_id] == NULL) {\n return 0;\n }\n\n for (int i = 0; i < buffer_id; i++) {\n if (galloc->buffers[i] == galloc->buffers[buffer_id]) {\n // this buffer is the same as a previous one due to the same buffer type being used multiple times\n // only return the buffer size the first time it appears to avoid double counting\n return 0;\n }\n }\n\n return ggml_vbuffer_size(galloc->buffers[buffer_id]);\n}\n\n// utils\n\nstatic void free_buffers(ggml_backend_buffer_t ** buffers, const size_t * n_buffers) {\n for (size_t i = 0; i < *n_buffers; i++) {\n ggml_backend_buffer_free((*buffers)[i]);\n }\n free(*buffers);\n}\n\nstatic bool alloc_tensor_range(struct ggml_context * ctx,\n struct ggml_tensor * first, struct ggml_tensor * last,\n ggml_backend_buffer_type_t buft, size_t size,\n ggml_backend_buffer_t ** buffers, size_t * n_buffers) {\n\n ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size);\n if (buffer == NULL) {\n GGML_LOG_ERROR(\"%s: failed to allocate %s buffer of size %zu\\n\", __func__, ggml_backend_buft_name(buft), size);\n free_buffers(buffers, n_buffers);\n return false;\n }\n\n *buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1));\n (*buffers)[(*n_buffers)++] = buffer;\n\n struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);\n\n for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) {\n enum ggml_status status = GGML_STATUS_SUCCESS;\n if (t->data == NULL) {\n if (t->view_src == NULL) {\n status = ggml_tallocr_alloc(&tallocr, t);\n } else if (t->buffer == NULL) {\n status = ggml_backend_view_init(t);\n }\n } else {\n if (t->view_src != NULL && t->buffer == NULL) {\n // view of a pre-allocated tensor\n status = ggml_backend_view_init(t);\n }\n }\n if (status != GGML_STATUS_SUCCESS) {\n GGML_LOG_ERROR(\"%s: failed to initialize tensor %s\\n\", __func__, t->name);\n free_buffers(buffers, n_buffers);\n return false;\n }\n }\n\n return true;\n}\n\nstatic ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft_impl(\n struct ggml_context * ctx, ggml_backend_buffer_type_t buft, size_t * nbytes_total, bool no_alloc) {\n GGML_ASSERT(ggml_get_no_alloc(ctx) == true);\n\n size_t alignment = ggml_backend_buft_get_alignment(buft);\n size_t max_size = ggml_backend_buft_get_max_size(buft);\n\n ggml_backend_buffer_t * buffers = NULL;\n size_t n_buffers = 0;\n *nbytes_total = 0;\n\n size_t cur_buf_size = 0;\n struct ggml_tensor * first = ggml_get_first_tensor(ctx);\n for (struct ggml_tensor * t = first; t != NULL; t = ggml_get_next_tensor(ctx, t)) {\n size_t this_size = 0;\n if (t->data == NULL && t->view_src == NULL) {\n this_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);\n }\n\n if (cur_buf_size > 0 && (cur_buf_size + this_size) > max_size) {\n // allocate tensors in the current buffer\n if (!no_alloc && !alloc_tensor_range(ctx, first, t, buft, cur_buf_size, &buffers, &n_buffers)) {\n return NULL;\n }\n first = t;\n *nbytes_total += cur_buf_size;\n cur_buf_size = this_size;\n } else {\n cur_buf_size += this_size;\n }\n }\n\n // allocate remaining tensors\n if (cur_buf_size > 0) {\n *nbytes_total += cur_buf_size;\n if (!no_alloc && !alloc_tensor_range(ctx, first, NULL, buft, cur_buf_size, &buffers, &n_buffers)) {\n return NULL;\n }\n }\n\n if (no_alloc) {\n return NULL;\n }\n\n if (n_buffers == 0) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: all tensors in the context are already allocated\\n\", __func__);\n#endif\n GGML_ASSERT(!buffers);\n return NULL;\n }\n\n ggml_backend_buffer_t buffer;\n if (n_buffers == 1) {\n buffer = buffers[0];\n } else {\n buffer = ggml_backend_multi_buffer_alloc_buffer(buffers, n_buffers);\n }\n if (buffers) {\n free(buffers); // can be NULL if context is empty or no_alloc\n }\n return buffer;\n}\n\nsize_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {\n size_t nbytes_total = 0;\n ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft_impl(ctx, buft, &nbytes_total, /*no_alloc=*/ true);\n GGML_ASSERT(!buf);\n return nbytes_total;\n}\n\nggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {\n size_t nbytes_total = 0;\n return ggml_backend_alloc_ctx_tensors_from_buft_impl(ctx, buft, &nbytes_total, /*no_alloc =*/ false);\n}\n\nggml_backend_buffer_t ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend) {\n return ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_get_default_buffer_type(backend));\n}\n" }, { "path": "src/ggml-backend-dl.cpp", "content": "#include \"ggml-backend-dl.h\"\n\n#ifdef _WIN32\n\ndl_handle * dl_load_library(const fs::path & path) {\n // suppress error dialogs for missing DLLs\n DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);\n SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);\n\n HMODULE handle = LoadLibraryW(path.wstring().c_str());\n\n SetErrorMode(old_mode);\n\n return handle;\n}\n\nvoid * dl_get_sym(dl_handle * handle, const char * name) {\n DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);\n SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);\n\n void * p = (void *) GetProcAddress(handle, name);\n\n SetErrorMode(old_mode);\n\n return p;\n}\n\nconst char * dl_error() {\n return \"\";\n}\n\n#else\n\ndl_handle * dl_load_library(const fs::path & path) {\n dl_handle * handle = dlopen(path.string().c_str(), RTLD_NOW | RTLD_LOCAL);\n return handle;\n}\n\nvoid * dl_get_sym(dl_handle * handle, const char * name) {\n return dlsym(handle, name);\n}\n\nconst char * dl_error() {\n const char *rslt = dlerror();\n return rslt != nullptr ? rslt : \"\";\n}\n\n#endif\n" }, { "path": "src/ggml-backend-dl.h", "content": "#pragma once\n\n#ifdef _WIN32\n# define WIN32_LEAN_AND_MEAN\n# ifndef NOMINMAX\n# define NOMINMAX\n# endif\n# include \n# include \n#else\n# include \n# include \n#endif\n#include \n\nnamespace fs = std::filesystem;\n\n#ifdef _WIN32\n\nusing dl_handle = std::remove_pointer_t;\n\nstruct dl_handle_deleter {\n void operator()(HMODULE handle) {\n FreeLibrary(handle);\n }\n};\n\n#else\n\nusing dl_handle = void;\n\nstruct dl_handle_deleter {\n void operator()(void * handle) {\n dlclose(handle);\n }\n};\n\n#endif\n\nusing dl_handle_ptr = std::unique_ptr;\n\ndl_handle * dl_load_library(const fs::path & path);\nvoid * dl_get_sym(dl_handle * handle, const char * name);\nconst char * dl_error();\n\n" }, { "path": "src/ggml-backend-impl.h", "content": "#pragma once\n\n// ggml-backend internal header\n\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n #define GGML_BACKEND_API_VERSION 2\n\n //\n // Backend buffer type\n //\n\n struct ggml_backend_buffer_type_i {\n const char * (*get_name) (ggml_backend_buffer_type_t buft);\n // allocate a buffer of this type\n ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);\n // tensor alignment\n size_t (*get_alignment) (ggml_backend_buffer_type_t buft);\n // (optional) max buffer size that can be allocated (defaults to SIZE_MAX)\n size_t (*get_max_size) (ggml_backend_buffer_type_t buft);\n // (optional) data size needed to allocate the tensor, including padding (defaults to ggml_nbytes)\n size_t (*get_alloc_size)(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);\n // (optional) check if tensor data is in host memory and uses standard ggml tensor layout (defaults to false)\n bool (*is_host) (ggml_backend_buffer_type_t buft);\n };\n\n struct ggml_backend_buffer_type {\n struct ggml_backend_buffer_type_i iface;\n ggml_backend_dev_t device;\n void * context;\n };\n\n //\n // Backend buffer\n //\n\n struct ggml_backend_buffer_i {\n // (optional) free the buffer\n void (*free_buffer) (ggml_backend_buffer_t buffer);\n // base address of the buffer\n void * (*get_base) (ggml_backend_buffer_t buffer);\n // (optional) initialize a tensor in the buffer (eg. add tensor extras)\n enum ggml_status (*init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);\n // tensor data access\n void (*memset_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);\n void (*set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n void (*get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);\n // (optional) tensor copy: dst is in the buffer, src may be in any buffer, including buffers from a different backend (return false if not supported)\n bool (*cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst);\n // clear the entire buffer\n void (*clear) (ggml_backend_buffer_t buffer, uint8_t value);\n // (optional) reset any internal state due to tensor initialization, such as tensor extras\n void (*reset) (ggml_backend_buffer_t buffer);\n };\n\n struct ggml_backend_buffer {\n struct ggml_backend_buffer_i iface;\n ggml_backend_buffer_type_t buft;\n void * context;\n size_t size;\n enum ggml_backend_buffer_usage usage;\n };\n\n GGML_API ggml_backend_buffer_t ggml_backend_buffer_init(\n ggml_backend_buffer_type_t buft,\n struct ggml_backend_buffer_i iface,\n void * context,\n size_t size);\n\n // do not use directly, use ggml_backend_tensor_copy instead\n GGML_API bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);\n\n // multi-buffer\n // buffer that contains a collection of buffers\n GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);\n GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);\n GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);\n\n //\n // Backend (stream)\n //\n\n struct ggml_backend_i {\n const char * (*get_name)(ggml_backend_t backend);\n\n void (*free)(ggml_backend_t backend);\n\n // (optional) asynchronous tensor data access\n void (*set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n void (*get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);\n bool (*cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);\n\n // (optional) complete all pending operations (required if the backend supports async operations)\n void (*synchronize)(ggml_backend_t backend);\n\n // (optional) graph plans (not used currently)\n // compute graph with a plan\n ggml_backend_graph_plan_t (*graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);\n void (*graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);\n // update the plan with a new graph - this should be faster than creating a new plan when the graph has the same topology\n void (*graph_plan_update) (ggml_backend_t backend, ggml_backend_graph_plan_t plan, const struct ggml_cgraph * cgraph);\n // compute the graph with the plan\n enum ggml_status (*graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);\n\n // compute graph (always async if supported by the backend)\n enum ggml_status (*graph_compute) (ggml_backend_t backend, struct ggml_cgraph * cgraph);\n\n // (optional) event synchronization\n // record an event on this stream\n void (*event_record)(ggml_backend_t backend, ggml_backend_event_t event);\n // wait for an event on on a different stream\n void (*event_wait) (ggml_backend_t backend, ggml_backend_event_t event);\n\n // (optional) sort/optimize the nodes in the graph\n void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph);\n };\n\n struct ggml_backend {\n ggml_guid_t guid;\n struct ggml_backend_i iface;\n ggml_backend_dev_t device;\n void * context;\n };\n\n struct ggml_backend_event {\n struct ggml_backend_device * device;\n void * context;\n };\n\n //\n // Backend device\n //\n\n // Note: if additional properties are needed, we should add a struct with all of them\n // the current functions to obtain the properties can remain, since they are more convenient for often used properties\n struct ggml_backend_device_i {\n // device name: short identifier for this device, such as \"CPU\" or \"CUDA0\"\n const char * (*get_name)(ggml_backend_dev_t dev);\n\n // device description: short informative description of the device, could be the model name\n const char * (*get_description)(ggml_backend_dev_t dev);\n\n // device memory in bytes: 0 bytes to indicate no memory to report\n void (*get_memory)(ggml_backend_dev_t dev, size_t * free, size_t * total);\n\n // device type\n enum ggml_backend_dev_type (*get_type)(ggml_backend_dev_t dev);\n\n // device properties\n void (*get_props)(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props);\n\n // backend (stream) initialization\n ggml_backend_t (*init_backend)(ggml_backend_dev_t dev, const char * params);\n\n // preferred buffer type\n ggml_backend_buffer_type_t (*get_buffer_type)(ggml_backend_dev_t dev);\n\n // (optional) host buffer type (in system memory, typically this is a pinned memory buffer for faster transfers between host and device)\n ggml_backend_buffer_type_t (*get_host_buffer_type)(ggml_backend_dev_t dev);\n\n // (optional) buffer from pointer: create a buffer from a host pointer (useful for memory mapped models and importing data from other libraries)\n ggml_backend_buffer_t (*buffer_from_host_ptr)(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size);\n\n // check if the backend can compute an operation\n bool (*supports_op)(ggml_backend_dev_t dev, const struct ggml_tensor * op);\n\n // check if the backend can use tensors allocated in a buffer type\n bool (*supports_buft)(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft);\n\n // (optional) check if the backend wants to run an operation, even if the weights are allocated in an incompatible buffer\n // these should be expensive operations that may benefit from running on this backend instead of the CPU backend\n bool (*offload_op)(ggml_backend_dev_t dev, const struct ggml_tensor * op);\n\n // (optional) event synchronization\n ggml_backend_event_t (*event_new) (ggml_backend_dev_t dev);\n void (*event_free) (ggml_backend_dev_t dev, ggml_backend_event_t event);\n void (*event_synchronize) (ggml_backend_dev_t dev, ggml_backend_event_t event);\n };\n\n struct ggml_backend_device {\n struct ggml_backend_device_i iface;\n ggml_backend_reg_t reg;\n void * context;\n };\n\n //\n // Backend (reg)\n //\n\n struct ggml_backend_reg_i {\n const char * (*get_name)(ggml_backend_reg_t reg);\n\n // enumerate available devices\n size_t (*get_device_count)(ggml_backend_reg_t reg);\n ggml_backend_dev_t (*get_device)(ggml_backend_reg_t reg, size_t index);\n\n // (optional) get a pointer to a function in the backend\n // backends can add custom functions that are not part of the standard ggml-backend interface\n void * (*get_proc_address)(ggml_backend_reg_t reg, const char * name);\n };\n\n struct ggml_backend_reg {\n int api_version; // initialize to GGML_BACKEND_API_VERSION\n struct ggml_backend_reg_i iface;\n void * context;\n };\n\n // Add backend dynamic loading support to the backend\n\n // Initialize the backend\n typedef ggml_backend_reg_t (*ggml_backend_init_t)(void);\n // Optional: obtain a score for the backend based on the system configuration\n // Higher scores are preferred, 0 means the backend is not supported in the current system\n typedef int (*ggml_backend_score_t)(void);\n\n#ifdef GGML_BACKEND_DL\n# ifdef __cplusplus\n# define GGML_BACKEND_DL_IMPL(reg_fn) \\\n extern \"C\" { \\\n GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void); \\\n } \\\n ggml_backend_reg_t ggml_backend_init(void) { \\\n return reg_fn(); \\\n }\n# define GGML_BACKEND_DL_SCORE_IMPL(score_fn) \\\n extern \"C\" { \\\n GGML_BACKEND_API int ggml_backend_score(void); \\\n } \\\n int ggml_backend_score(void) { \\\n return score_fn(); \\\n }\n# else\n# define GGML_BACKEND_DL_IMPL(reg_fn) \\\n GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void); \\\n ggml_backend_reg_t ggml_backend_init(void) { \\\n return reg_fn(); \\\n }\n# define GGML_BACKEND_DL_SCORE_IMPL(score_fn) \\\n GGML_BACKEND_API int ggml_backend_score(void); \\\n int ggml_backend_score(void) { \\\n return score_fn(); \\\n }\n# endif\n#else\n# define GGML_BACKEND_DL_IMPL(reg_fn)\n# define GGML_BACKEND_DL_SCORE_IMPL(score_fn)\n#endif\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-backend-reg.cpp", "content": "#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-backend-dl.h\"\n#include \"ggml-impl.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#ifdef _WIN32\n# define WIN32_LEAN_AND_MEAN\n# ifndef NOMINMAX\n# define NOMINMAX\n# endif\n# include \n#elif defined(__APPLE__)\n# include \n# include \n#else\n# include \n# include \n#endif\n\n// Backend registry\n#ifdef GGML_USE_CPU\n#include \"ggml-cpu.h\"\n#endif\n\n#ifdef GGML_USE_CUDA\n#include \"ggml-cuda.h\"\n#endif\n\n#ifdef GGML_USE_METAL\n#include \"ggml-metal.h\"\n#endif\n\n#ifdef GGML_USE_SYCL\n#include \"ggml-sycl.h\"\n#endif\n\n#ifdef GGML_USE_VULKAN\n#include \"ggml-vulkan.h\"\n#endif\n\n#ifdef GGML_USE_WEBGPU\n#include \"ggml-webgpu.h\"\n#endif\n\n#ifdef GGML_USE_ZDNN\n#include \"ggml-zdnn.h\"\n#endif\n\n#ifdef GGML_USE_OPENCL\n#include \"ggml-opencl.h\"\n#endif\n\n#ifdef GGML_USE_HEXAGON\n#include \"ggml-hexagon.h\"\n#endif\n\n#ifdef GGML_USE_BLAS\n#include \"ggml-blas.h\"\n#endif\n\n#ifdef GGML_USE_RPC\n#include \"ggml-rpc.h\"\n#endif\n\n#ifdef GGML_USE_VIRTGPU_FRONTEND\n#include \"ggml-virtgpu.h\"\n#endif\n\n#ifdef GGML_USE_CANN\n#include \"ggml-cann.h\"\n#endif\n\n#ifdef GGML_USE_ZENDNN\n#include \"ggml-zendnn.h\"\n#endif\n\n#ifdef GGML_USE_OPENVINO\n#include \"ggml-openvino.h\"\n#endif\n\nnamespace fs = std::filesystem;\n\nstatic std::string path_str(const fs::path & path) {\n try {\n#if defined(__cpp_lib_char8_t)\n // C++20 and later: u8string() returns std::u8string\n const std::u8string u8str = path.u8string();\n return std::string(reinterpret_cast(u8str.data()), u8str.size());\n#else\n // C++17: u8string() returns std::string\n return path.u8string();\n#endif\n } catch (...) {\n return std::string();\n }\n}\n\nstruct ggml_backend_reg_entry {\n ggml_backend_reg_t reg;\n dl_handle_ptr handle;\n};\n\nstruct ggml_backend_registry {\n std::vector backends;\n std::vector devices;\n\n ggml_backend_registry() {\n#ifdef GGML_USE_CUDA\n register_backend(ggml_backend_cuda_reg());\n#endif\n#ifdef GGML_USE_METAL\n register_backend(ggml_backend_metal_reg());\n#endif\n#ifdef GGML_USE_SYCL\n register_backend(ggml_backend_sycl_reg());\n#endif\n#ifdef GGML_USE_VULKAN\n // Add runtime disable check\n if (getenv(\"GGML_DISABLE_VULKAN\") == nullptr) {\n register_backend(ggml_backend_vk_reg());\n } else {\n GGML_LOG_DEBUG(\"Vulkan backend disabled by GGML_DISABLE_VULKAN environment variable\\n\");\n }\n#endif\n#ifdef GGML_USE_WEBGPU\n register_backend(ggml_backend_webgpu_reg());\n#endif\n#ifdef GGML_USE_ZDNN\n register_backend(ggml_backend_zdnn_reg());\n#endif\n#ifdef GGML_USE_VIRTGPU_FRONTEND\n register_backend(ggml_backend_virtgpu_reg());\n#endif\n\n#ifdef GGML_USE_OPENCL\n register_backend(ggml_backend_opencl_reg());\n#endif\n#ifdef GGML_USE_ZENDNN\n register_backend(ggml_backend_zendnn_reg());\n#endif\n#ifdef GGML_USE_HEXAGON\n register_backend(ggml_backend_hexagon_reg());\n#endif\n#ifdef GGML_USE_CANN\n register_backend(ggml_backend_cann_reg());\n#endif\n#ifdef GGML_USE_BLAS\n register_backend(ggml_backend_blas_reg());\n#endif\n#ifdef GGML_USE_RPC\n register_backend(ggml_backend_rpc_reg());\n#endif\n#ifdef GGML_USE_OPENVINO\n register_backend(ggml_backend_openvino_reg());\n#endif\n#ifdef GGML_USE_CPU\n register_backend(ggml_backend_cpu_reg());\n#endif\n }\n\n ~ggml_backend_registry() {\n // FIXME: backends cannot be safely unloaded without a function to destroy all the backend resources,\n // since backend threads may still be running and accessing resources from the dynamic library\n for (auto & entry : backends) {\n if (entry.handle) {\n entry.handle.release(); // NOLINT\n }\n }\n }\n\n void register_backend(ggml_backend_reg_t reg, dl_handle_ptr handle = nullptr) {\n if (!reg) {\n return;\n }\n\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: registered backend %s (%zu devices)\\n\",\n __func__, ggml_backend_reg_name(reg), ggml_backend_reg_dev_count(reg));\n#endif\n backends.push_back({ reg, std::move(handle) });\n for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); i++) {\n register_device(ggml_backend_reg_dev_get(reg, i));\n }\n }\n\n void register_device(ggml_backend_dev_t device) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: registered device %s (%s)\\n\", __func__, ggml_backend_dev_name(device), ggml_backend_dev_description(device));\n#endif\n devices.push_back(device);\n }\n\n ggml_backend_reg_t load_backend(const fs::path & path, bool silent) {\n dl_handle_ptr handle { dl_load_library(path) };\n if (!handle) {\n if (!silent) {\n GGML_LOG_ERROR(\"%s: failed to load %s: %s\\n\", __func__, path_str(path).c_str(), dl_error());\n }\n return nullptr;\n }\n\n auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), \"ggml_backend_score\");\n if (score_fn && score_fn() == 0) {\n if (!silent) {\n GGML_LOG_INFO(\"%s: backend %s is not supported on this system\\n\", __func__, path_str(path).c_str());\n }\n return nullptr;\n }\n\n auto backend_init_fn = (ggml_backend_init_t) dl_get_sym(handle.get(), \"ggml_backend_init\");\n if (!backend_init_fn) {\n if (!silent) {\n GGML_LOG_ERROR(\"%s: failed to find ggml_backend_init in %s\\n\", __func__, path_str(path).c_str());\n }\n return nullptr;\n }\n\n ggml_backend_reg_t reg = backend_init_fn();\n if (!reg || reg->api_version != GGML_BACKEND_API_VERSION) {\n if (!silent) {\n if (!reg) {\n GGML_LOG_ERROR(\"%s: failed to initialize backend from %s: ggml_backend_init returned NULL\\n\",\n __func__, path_str(path).c_str());\n } else {\n GGML_LOG_ERROR(\"%s: failed to initialize backend from %s: incompatible API version (backend: %d, current: %d)\\n\",\n __func__, path_str(path).c_str(), reg->api_version, GGML_BACKEND_API_VERSION);\n }\n }\n return nullptr;\n }\n\n GGML_LOG_INFO(\"%s: loaded %s backend from %s\\n\", __func__, ggml_backend_reg_name(reg), path_str(path).c_str());\n\n register_backend(reg, std::move(handle));\n\n return reg;\n }\n\n void unload_backend(ggml_backend_reg_t reg, bool silent) {\n auto it = std::find_if(backends.begin(), backends.end(),\n [reg](const ggml_backend_reg_entry & entry) { return entry.reg == reg; });\n\n if (it == backends.end()) {\n if (!silent) {\n GGML_LOG_ERROR(\"%s: backend not found\\n\", __func__);\n }\n return;\n }\n\n if (!silent) {\n GGML_LOG_DEBUG(\"%s: unloading %s backend\\n\", __func__, ggml_backend_reg_name(reg));\n }\n\n // remove devices\n devices.erase(\n std::remove_if(devices.begin(), devices.end(),\n [reg](ggml_backend_dev_t dev) { return ggml_backend_dev_backend_reg(dev) == reg; }),\n devices.end());\n\n // remove backend\n backends.erase(it);\n }\n};\n\nstatic ggml_backend_registry & get_reg() {\n static ggml_backend_registry reg;\n return reg;\n}\n\n// Internal API\nvoid ggml_backend_register(ggml_backend_reg_t reg) {\n get_reg().register_backend(reg);\n}\n\nvoid ggml_backend_device_register(ggml_backend_dev_t device) {\n get_reg().register_device(device);\n}\n\n// Backend (reg) enumeration\nstatic bool striequals(const char * a, const char * b) {\n for (; *a && *b; a++, b++) {\n if (std::tolower(*a) != std::tolower(*b)) {\n return false;\n }\n }\n return *a == *b;\n}\n\nsize_t ggml_backend_reg_count() {\n return get_reg().backends.size();\n}\n\nggml_backend_reg_t ggml_backend_reg_get(size_t index) {\n GGML_ASSERT(index < ggml_backend_reg_count());\n return get_reg().backends[index].reg;\n}\n\nggml_backend_reg_t ggml_backend_reg_by_name(const char * name) {\n for (size_t i = 0; i < ggml_backend_reg_count(); i++) {\n ggml_backend_reg_t reg = ggml_backend_reg_get(i);\n if (striequals(ggml_backend_reg_name(reg), name)) {\n return reg;\n }\n }\n return nullptr;\n}\n\n// Device enumeration\nsize_t ggml_backend_dev_count() {\n return get_reg().devices.size();\n}\n\nggml_backend_dev_t ggml_backend_dev_get(size_t index) {\n GGML_ASSERT(index < ggml_backend_dev_count());\n return get_reg().devices[index];\n}\n\nggml_backend_dev_t ggml_backend_dev_by_name(const char * name) {\n for (size_t i = 0; i < ggml_backend_dev_count(); i++) {\n ggml_backend_dev_t dev = ggml_backend_dev_get(i);\n if (striequals(ggml_backend_dev_name(dev), name)) {\n return dev;\n }\n }\n return nullptr;\n}\n\nggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type) {\n for (size_t i = 0; i < ggml_backend_dev_count(); i++) {\n ggml_backend_dev_t dev = ggml_backend_dev_get(i);\n if (ggml_backend_dev_type(dev) == type) {\n return dev;\n }\n }\n return nullptr;\n}\n\n// Convenience functions\nggml_backend_t ggml_backend_init_by_name(const char * name, const char * params) {\n ggml_backend_dev_t dev = ggml_backend_dev_by_name(name);\n if (!dev) {\n return nullptr;\n }\n return ggml_backend_dev_init(dev, params);\n}\n\nggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params) {\n ggml_backend_dev_t dev = ggml_backend_dev_by_type(type);\n if (!dev) {\n return nullptr;\n }\n return ggml_backend_dev_init(dev, params);\n}\n\nggml_backend_t ggml_backend_init_best(void) {\n ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);\n dev = dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU);\n dev = dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n if (!dev) {\n return nullptr;\n }\n return ggml_backend_dev_init(dev, nullptr);\n}\n\n// Dynamic loading\nggml_backend_reg_t ggml_backend_load(const char * path) {\n return get_reg().load_backend(path, false);\n}\n\nvoid ggml_backend_unload(ggml_backend_reg_t reg) {\n get_reg().unload_backend(reg, true);\n}\n\nstatic fs::path get_executable_path() {\n#if defined(__APPLE__)\n // get executable path\n std::vector path;\n uint32_t size;\n while (true) {\n size = path.size();\n if (_NSGetExecutablePath(path.data(), &size) == 0) {\n break;\n }\n path.resize(size);\n }\n std::string base_path(path.data(), size);\n // remove executable name\n auto last_slash = base_path.find_last_of('/');\n if (last_slash != std::string::npos) {\n base_path = base_path.substr(0, last_slash);\n }\n return base_path + \"/\";\n#elif defined(__linux__) || defined(__FreeBSD__)\n std::string base_path = \".\";\n std::vector path(1024);\n while (true) {\n // get executable path\n# if defined(__linux__)\n ssize_t len = readlink(\"/proc/self/exe\", path.data(), path.size());\n# elif defined(__FreeBSD__)\n ssize_t len = readlink(\"/proc/curproc/file\", path.data(), path.size());\n# endif\n if (len == -1) {\n break;\n }\n if (len < (ssize_t) path.size()) {\n base_path = std::string(path.data(), len);\n // remove executable name\n auto last_slash = base_path.find_last_of('/');\n if (last_slash != std::string::npos) {\n base_path = base_path.substr(0, last_slash);\n }\n break;\n }\n path.resize(path.size() * 2);\n }\n\n return base_path + \"/\";\n#elif defined(_WIN32)\n std::vector path(MAX_PATH);\n DWORD len = GetModuleFileNameW(NULL, path.data(), path.size());\n if (len == 0) {\n return {};\n }\n std::wstring base_path(path.data(), len);\n // remove executable name\n auto last_slash = base_path.find_last_of('\\\\');\n if (last_slash != std::string::npos) {\n base_path = base_path.substr(0, last_slash);\n }\n return base_path + L\"\\\\\";\n#else\n return {};\n#endif\n}\n\nstatic fs::path backend_filename_prefix() {\n#ifdef _WIN32\n return fs::u8path(\"ggml-\");\n#else\n return fs::u8path(\"libggml-\");\n#endif\n}\n\nstatic fs::path backend_filename_extension() {\n#ifdef _WIN32\n return fs::u8path(\".dll\");\n#else\n return fs::u8path(\".so\");\n#endif\n}\n\nstatic ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, const char * user_search_path) {\n // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths\n const fs::path name_path = fs::u8path(name);\n const fs::path file_prefix = backend_filename_prefix().native() + name_path.native() + fs::u8path(\"-\").native();\n const fs::path file_extension = backend_filename_extension();\n\n std::vector search_paths;\n if (user_search_path == nullptr) {\n#ifdef GGML_BACKEND_DIR\n search_paths.push_back(fs::u8path(GGML_BACKEND_DIR));\n#endif\n // default search paths: executable directory, current directory\n search_paths.push_back(get_executable_path());\n search_paths.push_back(fs::current_path());\n } else {\n search_paths.push_back(fs::u8path(user_search_path));\n }\n\n int best_score = 0;\n fs::path best_path;\n std::error_code ec;\n\n for (const auto & search_path : search_paths) {\n if (!fs::exists(search_path, ec)) {\n if (ec) {\n GGML_LOG_DEBUG(\"%s: posix_stat(%s) failure, error-message: %s\\n\", __func__, path_str(search_path).c_str(), ec.message().c_str());\n } else {\n GGML_LOG_DEBUG(\"%s: search path %s does not exist\\n\", __func__, path_str(search_path).c_str());\n }\n continue;\n }\n fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);\n for (const auto & entry : dir_it) {\n if (entry.is_regular_file(ec)) {\n auto filename = entry.path().filename();\n auto ext = entry.path().extension();\n if (filename.native().find(file_prefix) == 0 && ext == file_extension) {\n dl_handle_ptr handle { dl_load_library(entry) };\n if (!handle && !silent) {\n GGML_LOG_ERROR(\"%s: failed to load %s: %s\\n\", __func__, path_str(entry.path()).c_str(), dl_error());\n }\n if (handle) {\n auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), \"ggml_backend_score\");\n if (score_fn) {\n int s = score_fn();\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: %s score: %d\\n\", __func__, path_str(entry.path()).c_str(), s);\n#endif\n if (s > best_score) {\n best_score = s;\n best_path = entry.path();\n }\n } else {\n if (!silent) {\n GGML_LOG_INFO(\"%s: failed to find ggml_backend_score in %s\\n\", __func__, path_str(entry.path()).c_str());\n }\n }\n }\n }\n }\n }\n }\n\n if (best_score == 0) {\n // try to load the base backend\n for (const auto & search_path : search_paths) {\n fs::path filename = backend_filename_prefix().native() + name_path.native() + backend_filename_extension().native();\n fs::path path = search_path / filename;\n if (std::error_code ec; fs::exists(path, ec)) {\n return get_reg().load_backend(path, silent);\n } else {\n if (ec) {\n GGML_LOG_DEBUG(\"%s: posix_stat(%s) failure, error-message: %s\\n\", __func__, path_str(path).c_str(), ec.message().c_str());\n }\n }\n }\n return nullptr;\n }\n\n return get_reg().load_backend(best_path, silent);\n}\n\nvoid ggml_backend_load_all() {\n ggml_backend_load_all_from_path(nullptr);\n}\n\nvoid ggml_backend_load_all_from_path(const char * dir_path) {\n#ifdef NDEBUG\n bool silent = true;\n#else\n bool silent = false;\n#endif\n\n ggml_backend_load_best(\"blas\", silent, dir_path);\n ggml_backend_load_best(\"zendnn\", silent, dir_path);\n ggml_backend_load_best(\"cann\", silent, dir_path);\n ggml_backend_load_best(\"cuda\", silent, dir_path);\n ggml_backend_load_best(\"hip\", silent, dir_path);\n ggml_backend_load_best(\"metal\", silent, dir_path);\n ggml_backend_load_best(\"rpc\", silent, dir_path);\n ggml_backend_load_best(\"sycl\", silent, dir_path);\n ggml_backend_load_best(\"vulkan\", silent, dir_path);\n ggml_backend_load_best(\"virtgpu\", silent, dir_path);\n ggml_backend_load_best(\"opencl\", silent, dir_path);\n ggml_backend_load_best(\"hexagon\", silent, dir_path);\n ggml_backend_load_best(\"musa\", silent, dir_path);\n ggml_backend_load_best(\"openvino\", silent, dir_path);\n ggml_backend_load_best(\"cpu\", silent, dir_path);\n // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend\n const char * backend_path = std::getenv(\"GGML_BACKEND_PATH\");\n if (backend_path) {\n ggml_backend_load(backend_path);\n }\n}\n" }, { "path": "src/ggml-backend.cpp", "content": "// Note: porting this file to C++ is a work in progress\n\n#ifdef _WIN32\n#define WIN32_LEAN_AND_MEAN\n#ifndef NOMINMAX\n# define NOMINMAX\n#endif\n#include \n#endif\n\n#include \"ggml-backend.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-impl.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#ifdef __APPLE__\n#include \n#include \n#endif\n\n\n// backend buffer type\n\nconst char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) {\n GGML_ASSERT(buft);\n return buft->iface.get_name(buft);\n}\n\nggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n GGML_ASSERT(buft);\n if (size == 0) {\n // return a dummy buffer for zero-sized allocations\n return ggml_backend_buffer_init(buft, {}, NULL, 0);\n }\n return buft->iface.alloc_buffer(buft, size);\n}\n\nsize_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {\n GGML_ASSERT(buft);\n return buft->iface.get_alignment(buft);\n}\n\nsize_t ggml_backend_buft_get_max_size(ggml_backend_buffer_type_t buft) {\n GGML_ASSERT(buft);\n // get_max_size is optional, defaults to SIZE_MAX\n if (buft->iface.get_max_size) {\n return buft->iface.get_max_size(buft);\n }\n return SIZE_MAX;\n}\n\nsize_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {\n GGML_ASSERT(buft);\n // get_alloc_size is optional, defaults to ggml_nbytes\n if (buft->iface.get_alloc_size) {\n size_t size = buft->iface.get_alloc_size(buft, tensor);\n assert(size >= ggml_nbytes(tensor));\n return size;\n }\n return ggml_nbytes(tensor);\n}\n\nbool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) {\n GGML_ASSERT(buft);\n if (buft->iface.is_host) {\n return buft->iface.is_host(buft);\n }\n return false;\n}\n\nggml_backend_dev_t ggml_backend_buft_get_device(ggml_backend_buffer_type_t buft) {\n GGML_ASSERT(buft);\n return buft->device;\n}\n\n// backend buffer\n\nggml_backend_buffer_t ggml_backend_buffer_init(\n ggml_backend_buffer_type_t buft,\n struct ggml_backend_buffer_i iface,\n void * context,\n size_t size) {\n ggml_backend_buffer_t buffer = new ggml_backend_buffer {\n /* .interface = */ iface,\n /* .buft = */ buft,\n /* .context = */ context,\n /* .size = */ size,\n /* .usage = */ GGML_BACKEND_BUFFER_USAGE_ANY\n };\n\n return buffer;\n}\n\nconst char * ggml_backend_buffer_name(ggml_backend_buffer_t buffer) {\n return ggml_backend_buft_name(ggml_backend_buffer_get_type(buffer));\n}\n\nvoid ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {\n if (buffer == NULL) {\n return;\n }\n\n if (buffer->iface.free_buffer != NULL) {\n buffer->iface.free_buffer(buffer);\n }\n delete buffer;\n}\n\nsize_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {\n GGML_ASSERT(buffer);\n return buffer->size;\n}\n\nvoid * ggml_backend_buffer_get_base(ggml_backend_buffer_t buffer) {\n GGML_ASSERT(buffer);\n // get_base is optional if the buffer is zero-sized\n if (buffer->size == 0) {\n return NULL;\n }\n\n // FIXME JG: a multi_buffer has a non-zero size, according to the above comment get_base is not optional,\n // I don't know whether the above comment is correct\n if (!buffer->iface.get_base) {\n return NULL;\n }\n\n void * base = buffer->iface.get_base(buffer);\n\n GGML_ASSERT(base != NULL && \"backend buffer base cannot be NULL\");\n\n return base;\n}\n\nenum ggml_status ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {\n GGML_ASSERT(buffer);\n // init_tensor is optional\n if (buffer->iface.init_tensor) {\n return buffer->iface.init_tensor(buffer, tensor);\n }\n return GGML_STATUS_SUCCESS;\n}\n\nvoid ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n GGML_ASSERT(buffer);\n // clear is optional if the buffer is zero-sized\n if (buffer->size == 0) {\n return;\n }\n\n buffer->iface.clear(buffer, value);\n}\n\nsize_t ggml_backend_buffer_get_alignment(ggml_backend_buffer_t buffer) {\n return ggml_backend_buft_get_alignment(ggml_backend_buffer_get_type(buffer));\n}\n\nsize_t ggml_backend_buffer_get_max_size(ggml_backend_buffer_t buffer) {\n return ggml_backend_buft_get_max_size(ggml_backend_buffer_get_type(buffer));\n}\n\nsize_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor) {\n return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(buffer), tensor);\n}\n\nbool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {\n return ggml_backend_buft_is_host(ggml_backend_buffer_get_type(buffer));\n}\n\nvoid ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {\n GGML_ASSERT(buffer);\n buffer->usage = usage;\n\n // FIXME: add a generic callback to the buffer interface\n if (ggml_backend_buffer_is_multi_buffer(buffer)) {\n ggml_backend_multi_buffer_set_usage(buffer, usage);\n }\n}\n\nenum ggml_backend_buffer_usage ggml_backend_buffer_get_usage(ggml_backend_buffer_t buffer) {\n GGML_ASSERT(buffer);\n return buffer->usage;\n}\n\nggml_backend_buffer_type_t ggml_backend_buffer_get_type(ggml_backend_buffer_t buffer) {\n GGML_ASSERT(buffer);\n return buffer->buft;\n}\n\nvoid ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {\n GGML_ASSERT(buffer);\n if (buffer->iface.reset) {\n buffer->iface.reset(buffer);\n }\n}\n\nbool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {\n ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;\n if (dst_buf->iface.cpy_tensor) {\n return dst_buf->iface.cpy_tensor(dst_buf, src, dst);\n }\n return false;\n}\n\n// backend\n\nggml_guid_t ggml_backend_guid(ggml_backend_t backend) {\n if (backend == NULL) {\n return NULL;\n }\n return backend->guid;\n}\n\nconst char * ggml_backend_name(ggml_backend_t backend) {\n if (backend == NULL) {\n return \"NULL\";\n }\n return backend->iface.get_name(backend);\n}\n\nvoid ggml_backend_free(ggml_backend_t backend) {\n if (backend == NULL) {\n return;\n }\n\n backend->iface.free(backend);\n}\n\nggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend) {\n GGML_ASSERT(backend);\n return ggml_backend_dev_buffer_type(backend->device);\n}\n\nggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size) {\n return ggml_backend_buft_alloc_buffer(ggml_backend_get_default_buffer_type(backend), size);\n}\n\nsize_t ggml_backend_get_alignment(ggml_backend_t backend) {\n return ggml_backend_buft_get_alignment(ggml_backend_get_default_buffer_type(backend));\n}\n\nsize_t ggml_backend_get_max_size(ggml_backend_t backend) {\n return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend));\n}\n\nvoid ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n GGML_ASSERT(backend);\n GGML_ASSERT(tensor);\n GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor write out of bounds\");\n\n if (backend->iface.set_tensor_async == NULL) {\n ggml_backend_synchronize(backend);\n ggml_backend_tensor_set(tensor, data, offset, size);\n } else {\n backend->iface.set_tensor_async(backend, tensor, data, offset, size);\n }\n}\n\nvoid ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n GGML_ASSERT(backend);\n GGML_ASSERT(tensor);\n GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor read out of bounds\");\n\n if (backend->iface.get_tensor_async == NULL) {\n ggml_backend_synchronize(backend);\n ggml_backend_tensor_get(tensor, data, offset, size);\n } else {\n backend->iface.get_tensor_async(backend, tensor, data, offset, size);\n }\n}\n\nvoid ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n GGML_ASSERT(tensor);\n ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n if (size == 0) {\n return;\n }\n\n GGML_ASSERT(buf != NULL && \"tensor buffer not set\");\n GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor write out of bounds\");\n\n buf->iface.set_tensor(buf, tensor, data, offset, size);\n}\n\nvoid ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n GGML_ASSERT(tensor);\n ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n if (size == 0) {\n return;\n }\n\n GGML_ASSERT(buf != NULL && \"tensor buffer not set\");\n GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor read out of bounds\");\n\n buf->iface.get_tensor(buf, tensor, data, offset, size);\n}\n\nvoid ggml_backend_tensor_memset(struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {\n GGML_ASSERT(tensor);\n ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n if (size == 0) {\n return;\n }\n\n GGML_ASSERT(buf != NULL && \"tensor buffer not set\");\n GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor write out of bounds\");\n GGML_ASSERT(buf->iface.memset_tensor != NULL && \"memset not implemented by backend buffer\");\n\n buf->iface.memset_tensor(buf, tensor, value, offset, size);\n}\n\nvoid ggml_backend_synchronize(ggml_backend_t backend) {\n GGML_ASSERT(backend);\n if (backend->iface.synchronize == NULL) {\n return;\n }\n\n backend->iface.synchronize(backend);\n}\n\nggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n GGML_ASSERT(backend);\n GGML_ASSERT(backend->iface.graph_plan_create != NULL);\n\n return backend->iface.graph_plan_create(backend, cgraph);\n}\n\nvoid ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {\n GGML_ASSERT(backend);\n GGML_ASSERT(backend->iface.graph_plan_free != NULL);\n\n backend->iface.graph_plan_free(backend, plan);\n}\n\nenum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {\n GGML_ASSERT(backend);\n GGML_ASSERT(backend->iface.graph_plan_compute != NULL);\n\n return backend->iface.graph_plan_compute(backend, plan);\n}\n\nenum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph);\n ggml_backend_synchronize(backend);\n return err;\n}\n\nenum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n GGML_ASSERT(backend);\n return backend->iface.graph_compute(backend, cgraph);\n}\n\nbool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {\n GGML_ASSERT(backend);\n return ggml_backend_dev_supports_op(backend->device, op);\n}\n\nbool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {\n GGML_ASSERT(backend);\n return ggml_backend_dev_supports_buft(backend->device, buft);\n}\n\nbool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op) {\n GGML_ASSERT(backend);\n return ggml_backend_dev_offload_op(backend->device, op);\n}\n\nggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend) {\n GGML_ASSERT(backend);\n return backend->device;\n}\n\n// backend copy\n\nvoid ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst) {\n GGML_ASSERT(ggml_are_same_layout(src, dst) && \"cannot copy tensors with different layouts\");\n\n if (src == dst) {\n return;\n }\n\n if (ggml_backend_buffer_is_host(src->buffer)) {\n ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));\n } else if (ggml_backend_buffer_is_host(dst->buffer)) {\n ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));\n } else if (!ggml_backend_buffer_copy_tensor(src, dst)) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: warning: slow copy from %s to %s\\n\", __func__, ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer));\n#endif\n size_t nbytes = ggml_nbytes(src);\n void * data = malloc(nbytes);\n ggml_backend_tensor_get(src, data, 0, nbytes);\n ggml_backend_tensor_set(dst, data, 0, nbytes);\n free(data);\n }\n}\n\nvoid ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst) {\n GGML_ASSERT(ggml_are_same_layout(src, dst) && \"cannot copy tensors with different layouts\");\n\n if (src == dst) {\n return;\n }\n\n GGML_ASSERT(backend_dst);\n if (backend_dst->iface.cpy_tensor_async != NULL) {\n if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {\n return;\n }\n }\n\n // an async copy would normally happen after all the queued operations on both backends are completed\n // to simulate the same behavior, we need to synchronize both backends first, and do a blocking copy\n ggml_backend_synchronize(backend_src);\n ggml_backend_synchronize(backend_dst);\n ggml_backend_tensor_copy(src, dst);\n}\n\n// events\n\nggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device) {\n // null device is allowed for the transition period to the device interface\n if (device == NULL || device->iface.event_new == NULL) {\n return NULL;\n }\n return device->iface.event_new(device);\n}\n\nvoid ggml_backend_event_free(ggml_backend_event_t event) {\n if (event == NULL) {\n return;\n }\n event->device->iface.event_free(event->device, event);\n}\n\nvoid ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend) {\n GGML_ASSERT(backend);\n GGML_ASSERT(backend->iface.event_record != NULL);\n\n backend->iface.event_record(backend, event);\n}\n\nvoid ggml_backend_event_synchronize(ggml_backend_event_t event) {\n GGML_ASSERT(event);\n GGML_ASSERT(event->device->iface.event_synchronize);\n\n event->device->iface.event_synchronize(event->device, event);\n}\n\nvoid ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {\n GGML_ASSERT(backend);\n GGML_ASSERT(backend->iface.event_wait != NULL);\n\n backend->iface.event_wait(backend, event);\n}\n\nstatic void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n GGML_ASSERT(backend);\n if (backend->iface.graph_optimize != NULL) {\n backend->iface.graph_optimize(backend, cgraph);\n }\n}\n\n// Backend device\n\nconst char * ggml_backend_dev_name(ggml_backend_dev_t device) {\n GGML_ASSERT(device);\n return device->iface.get_name(device);\n}\n\nconst char * ggml_backend_dev_description(ggml_backend_dev_t device) {\n GGML_ASSERT(device);\n return device->iface.get_description(device);\n}\n\nvoid ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total) {\n GGML_ASSERT(device);\n device->iface.get_memory(device, free, total);\n}\n\nenum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device) {\n GGML_ASSERT(device);\n return device->iface.get_type(device);\n}\n\nvoid ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props) {\n memset(props, 0, sizeof(*props));\n device->iface.get_props(device, props);\n}\n\nggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device) {\n GGML_ASSERT(device);\n return device->reg;\n}\n\nggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params) {\n GGML_ASSERT(device);\n return device->iface.init_backend(device, params);\n}\n\nggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device) {\n GGML_ASSERT(device);\n return device->iface.get_buffer_type(device);\n}\n\nggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device) {\n GGML_ASSERT(device);\n if (device->iface.get_host_buffer_type == NULL) {\n return NULL;\n }\n\n return device->iface.get_host_buffer_type(device);\n}\n\nggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size) {\n GGML_ASSERT(device);\n return device->iface.buffer_from_host_ptr(device, ptr, size, max_tensor_size);\n}\n\nbool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {\n GGML_ASSERT(device);\n return device->iface.supports_op(device, op);\n}\n\nbool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) {\n GGML_ASSERT(device);\n return device->iface.supports_buft(device, buft);\n}\n\nbool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {\n GGML_ASSERT(device);\n if (device->iface.offload_op != NULL) {\n return device->iface.offload_op(device, op);\n }\n\n return false;\n}\n\n// Backend (reg)\n\nconst char * ggml_backend_reg_name(ggml_backend_reg_t reg) {\n GGML_ASSERT(reg);\n return reg->iface.get_name(reg);\n}\n\nsize_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg) {\n GGML_ASSERT(reg);\n return reg->iface.get_device_count(reg);\n}\n\nggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index) {\n GGML_ASSERT(reg);\n return reg->iface.get_device(reg, index);\n}\n\nvoid * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n GGML_ASSERT(reg);\n if (!reg->iface.get_proc_address) {\n return NULL;\n }\n return reg->iface.get_proc_address(reg, name);\n}\n\n// multi-buffer buffer\n\nstruct ggml_backend_multi_buffer_context {\n ggml_backend_buffer_t * buffers;\n size_t n_buffers;\n};\n\nstatic void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n GGML_ASSERT(buffer);\n ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;\n for (size_t i = 0; i < ctx->n_buffers; i++) {\n ggml_backend_buffer_free(ctx->buffers[i]);\n }\n\n free(ctx->buffers);\n free(ctx);\n}\n\nstatic void ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n GGML_ASSERT(buffer);\n ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;\n for (size_t i = 0; i < ctx->n_buffers; i++) {\n ggml_backend_buffer_clear(ctx->buffers[i], value);\n }\n}\n\nstatic const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {\n /* .free_buffer = */ ggml_backend_multi_buffer_free_buffer,\n /* .get_base = */ NULL,\n /* .init_tensor = */ NULL,\n /* .memset_tensor = */ NULL,\n /* .set_tensor = */ NULL,\n /* .get_tensor = */ NULL,\n /* .cpy_tensor = */ NULL,\n /* .clear = */ ggml_backend_multi_buffer_clear,\n /* .reset = */ NULL,\n};\n\nggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {\n ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) malloc(sizeof(struct ggml_backend_multi_buffer_context));\n ctx->n_buffers = n_buffers;\n ctx->buffers = (ggml_backend_buffer_t *) malloc(n_buffers * sizeof(ggml_backend_buffer_t));\n\n GGML_ASSERT(ctx->buffers != NULL);\n\n size_t total_size = 0;\n for (size_t i = 0; i < n_buffers; i++) {\n ctx->buffers[i] = buffers[i];\n total_size += ggml_backend_buffer_get_size(buffers[i]);\n }\n\n return ggml_backend_buffer_init(buffers[0]->buft, ggml_backend_multi_buffer_i, ctx, total_size);\n}\n\nbool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer) {\n GGML_ASSERT(buffer);\n return buffer->iface.free_buffer == ggml_backend_multi_buffer_free_buffer;\n}\n\nvoid ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {\n GGML_ASSERT(buffer);\n GGML_ASSERT(ggml_backend_buffer_is_multi_buffer(buffer));\n ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;\n for (size_t i = 0; i < ctx->n_buffers; i++) {\n ggml_backend_buffer_set_usage(ctx->buffers[i], usage);\n }\n}\n\n// creates a copy of the tensor with the same memory layout\nstatic struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) {\n struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor);\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n dup->nb[i] = tensor->nb[i];\n }\n return dup;\n}\n\nstatic bool ggml_is_view_op(enum ggml_op op) {\n return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;\n}\n\n// scheduler\n\n#ifndef GGML_SCHED_MAX_BACKENDS\n#define GGML_SCHED_MAX_BACKENDS 16\n#endif\n\n#ifndef GGML_SCHED_MAX_SPLIT_INPUTS\n#define GGML_SCHED_MAX_SPLIT_INPUTS 30\n#endif\n\n#ifndef GGML_SCHED_MAX_COPIES\n#define GGML_SCHED_MAX_COPIES 4\n#endif\n\nstruct ggml_backend_sched_split {\n int backend_id;\n int i_start;\n int i_end;\n struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS];\n int n_inputs;\n // graph view of this split\n struct ggml_cgraph graph;\n};\n\nstruct ggml_backend_sched {\n bool is_reset; // true if the scheduler has been reset since the last graph split\n bool is_alloc;\n\n int n_backends;\n\n ggml_backend_t backends[GGML_SCHED_MAX_BACKENDS];\n ggml_backend_buffer_type_t bufts[GGML_SCHED_MAX_BACKENDS];\n ggml_gallocr_t galloc;\n\n // hash map of the nodes in the graph\n struct ggml_hash_set hash_set;\n int * hv_tensor_backend_ids; // [hash_set.size]\n struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies]\n\n int * node_backend_ids; // [graph_size]\n int * leaf_backend_ids; // [graph_size]\n\n int * prev_node_backend_ids; // [graph_size]\n int * prev_leaf_backend_ids; // [graph_size]\n\n // copy of the graph with modified inputs\n struct ggml_cgraph graph;\n\n // graph splits\n struct ggml_backend_sched_split * splits;\n int n_splits;\n int splits_capacity;\n\n // pipeline parallelism support\n int n_copies;\n int cur_copy;\n int next_copy;\n ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];\n struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS];\n int n_graph_inputs;\n\n struct ggml_context * ctx;\n\n ggml_backend_sched_eval_callback callback_eval;\n void * callback_eval_user_data;\n\n char * context_buffer;\n size_t context_buffer_size;\n\n bool op_offload;\n\n int debug;\n\n // used for debugging graph reallocations [GGML_SCHED_DEBUG_REALLOC]\n // ref: https://github.com/ggml-org/llama.cpp/pull/17617\n int debug_realloc;\n int debug_graph_size;\n int debug_prev_graph_size;\n};\n\n#define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor)\n#define tensor_backend_id(tensor) sched->hv_tensor_backend_ids[hash_id(tensor)]\n#define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)]\n#define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id)\n\n// returns the priority of the backend, lower id is higher priority\nstatic int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {\n for (int i = 0; i < sched->n_backends; i++) {\n if (sched->backends[i] == backend) {\n return i;\n }\n }\n return -1;\n}\n\nstatic int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, const struct ggml_tensor * tensor, const struct ggml_tensor * op) {\n ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n if (buffer == NULL) {\n return -1;\n }\n\n // find highest prio backend that supports the buffer type and the op\n for (int i = 0; i < sched->n_backends; i++) {\n if (ggml_backend_supports_buft(sched->backends[i], buffer->buft) &&\n ggml_backend_supports_op(sched->backends[i], op)) {\n return i;\n }\n }\n\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: warning: no backend supports op %s with a weight with buffer type %s used in tensor %s, the weight will need to be copied\\n\",\n __func__, ggml_op_desc(tensor), ggml_backend_buffer_name(buffer), tensor->name);\n#endif\n\n return -1;\n}\n\n#if 0\n#define GGML_SCHED_MAX_SPLITS_DEBUG 4096\nstatic char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS_DEBUG*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only\n#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)\n#define GET_CAUSE(node) causes[hash_id(node)]\n#else\n#define SET_CAUSE(node, ...)\n#define GET_CAUSE(node) \"\"\n#endif\n\n// returns the backend that should be used for the node based on the current locations\nstatic int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * tensor) {\n // assign pre-allocated nodes to their backend\n int cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor, tensor);\n if (cur_backend_id != -1) {\n SET_CAUSE(tensor, \"1.dst\");\n return cur_backend_id;\n }\n\n // view_src\n if (tensor->view_src != NULL) {\n cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src, tensor);\n if (cur_backend_id != -1) {\n SET_CAUSE(tensor, \"1.vsrc\");\n return cur_backend_id;\n }\n }\n\n if (tensor->buffer || (tensor->view_src && tensor->view_src->buffer)) {\n // since the tensor is pre-allocated, it cannot be moved to another backend\n ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n GGML_ABORT(\"pre-allocated tensor (%s) in a buffer (%s) that cannot run the operation (%s)\", tensor->name, ggml_backend_buffer_name(buffer), ggml_op_name(tensor->op));\n }\n\n // graph input\n if (tensor->flags & GGML_TENSOR_FLAG_INPUT) {\n cur_backend_id = sched->n_backends - 1; // last backend (assumed CPU)\n SET_CAUSE(tensor, \"1.inp\");\n return cur_backend_id;\n }\n\n // operations with weights are preferably run on the same backend as the weights\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n const struct ggml_tensor * src = tensor->src[i];\n if (src == NULL) {\n continue;\n }\n // skip ROPE since the rope freqs tensor is too small to choose a backend based on it\n // not an ideal solution\n if (tensor->op != GGML_OP_ROPE && src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {\n int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor);\n // check if a backend with higher prio wants to offload the op\n if (sched->op_offload && src_backend_id == sched->n_backends - 1 && ggml_backend_buffer_is_host(src->buffer)) {\n for (int b = 0; b < src_backend_id; b++) {\n if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) {\n SET_CAUSE(tensor, \"1.off\");\n return b;\n }\n }\n }\n SET_CAUSE(tensor, \"1.wgt%d\", i);\n return src_backend_id;\n }\n }\n\n return -1;\n}\n\nstatic char * fmt_size(size_t size) {\n static char buffer[128];\n if (size >= 1024*1024) {\n snprintf(buffer, sizeof(buffer), \"%zuM\", size/1024/1024);\n } else {\n snprintf(buffer, sizeof(buffer), \"%zuK\", size/1024);\n }\n return buffer;\n}\n\nstatic void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n int cur_split = 0;\n for (int i = 0; i < graph->n_nodes; i++) {\n if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) {\n ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id];\n GGML_LOG_DEBUG(\"\\n## SPLIT #%d: %s # %d inputs\", cur_split, ggml_backend_name(split_backend),\n sched->splits[cur_split].n_inputs);\n for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) {\n if (j == 0) {\n GGML_LOG_DEBUG(\": \");\n }\n GGML_LOG_DEBUG(\"[%s (%5.5s)] \", sched->splits[cur_split].inputs[j]->name,\n fmt_size(ggml_nbytes(sched->splits[cur_split].inputs[j])));\n }\n GGML_LOG_DEBUG(\"\\n\");\n cur_split++;\n }\n struct ggml_tensor * node = graph->nodes[i];\n if (ggml_is_view_op(node->op)) {\n continue;\n }\n if (sched->debug > 1) {\n ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node);\n GGML_LOG_DEBUG(\"node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:\", i, ggml_op_name(node->op), node->name,\n fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : \"NULL\", GET_CAUSE(node),\n graph->use_counts[ggml_hash_find(&graph->visited_hash_set, node)], node->flags & GGML_TENSOR_FLAG_COMPUTE ? 1 : 0);\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src);\n GGML_LOG_DEBUG(\" %20.20s (%5.5s) [%5.5s %8.8s]\", src->name,\n fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : \"NULL\", GET_CAUSE(src));\n }\n GGML_LOG_DEBUG(\"\\n\");\n }\n }\n}\n\nstatic bool ggml_backend_sched_buffer_supported(ggml_backend_sched_t sched, struct ggml_tensor * t, int backend_id) {\n ggml_backend_buffer_t buf = t->view_src ? t->view_src->buffer : t->buffer;\n ggml_backend_buffer_type_t buft = NULL;\n\n if (buf) {\n // the tensor is already allocated\n buft = buf->buft;\n } else {\n // see if the tensor already has a backend assigned, and use the buffer type of that backend\n int tensor_backend_id = tensor_backend_id(t);\n if (tensor_backend_id == -1 && t->view_src) {\n tensor_backend_id = tensor_backend_id(t->view_src);\n }\n if (tensor_backend_id != -1) {\n buft = sched->bufts[tensor_backend_id];\n }\n }\n\n return buft != NULL && ggml_backend_supports_buft(sched->backends[backend_id], buft);\n}\n\nstatic void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, struct ggml_tensor * node, int cur_backend_id, int * node_backend_id) {\n if (ggml_backend_supports_op(sched->backends[cur_backend_id], node)) {\n *node_backend_id = cur_backend_id;\n SET_CAUSE(node, \"2.sup\");\n }\n}\n\n// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend\nvoid ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n // reset splits\n sched->n_splits = 0;\n sched->n_graph_inputs = 0;\n sched->is_reset = false;\n\n struct ggml_init_params params = {\n /* .mem_size = */ sched->context_buffer_size,\n /* .mem_buffer = */ sched->context_buffer,\n /* .no_alloc = */ true\n };\n\n ggml_free(sched->ctx);\n\n sched->ctx = ggml_init(params);\n if (sched->ctx == NULL) {\n GGML_ABORT(\"%s: failed to initialize context\\n\", __func__);\n }\n\n // pass 1: assign backends to ops with pre-allocated inputs\n for (int i = 0; i < graph->n_leafs; i++) {\n struct ggml_tensor * leaf = graph->leafs[i];\n int * leaf_backend_id = &tensor_backend_id(leaf);\n // do not overwrite user assignments\n if (*leaf_backend_id == -1) {\n *leaf_backend_id = ggml_backend_sched_backend_id_from_cur(sched, leaf);\n }\n }\n\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n int * node_backend_id = &tensor_backend_id(node);\n // do not overwrite user assignments\n if (*node_backend_id == -1) {\n *node_backend_id = ggml_backend_sched_backend_id_from_cur(sched, node);\n\n#if 0\n // src\n if (node->op == GGML_OP_NONE) {\n continue;\n }\n\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n int * src_backend_id = &tensor_backend_id(src);\n if (*src_backend_id == -1) {\n *src_backend_id = ggml_backend_sched_backend_id_from_cur(sched, src);\n }\n }\n#endif\n }\n }\n\n // pass 2: expand current backend assignments\n // assign the same backend to adjacent nodes\n // expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend)\n // thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops\n // ops unsupported by the backend being expanded will be left unassigned so that they can be assigned later when the locations of its inputs are known\n // expand gpu down\n {\n int cur_backend_id = -1;\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n if (ggml_is_view_op(node->op)) {\n continue;\n }\n int * node_backend_id = &tensor_backend_id(node);\n if (*node_backend_id != -1) {\n if (*node_backend_id == sched->n_backends - 1) {\n // skip cpu (lowest prio backend)\n cur_backend_id = -1;\n } else {\n cur_backend_id = *node_backend_id;\n }\n } else if (cur_backend_id != -1) {\n ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);\n }\n }\n }\n // expand gpu up\n {\n int cur_backend_id = -1;\n for (int i = graph->n_nodes - 1; i >= 0; i--) {\n struct ggml_tensor * node = graph->nodes[i];\n if (ggml_is_view_op(node->op)) {\n continue;\n }\n int * node_backend_id = &tensor_backend_id(node);\n if (*node_backend_id != -1) {\n if (*node_backend_id == sched->n_backends - 1) {\n // skip cpu (lowest prio backend)\n cur_backend_id = -1;\n } else {\n cur_backend_id = *node_backend_id;\n }\n } else if (cur_backend_id != -1) {\n ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);\n }\n }\n }\n // expand rest down\n {\n int cur_backend_id = -1;\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n if (ggml_is_view_op(node->op)) {\n continue;\n }\n int * node_backend_id = &tensor_backend_id(node);\n if (*node_backend_id != -1) {\n cur_backend_id = *node_backend_id;\n } else if (cur_backend_id != -1) {\n ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);\n }\n }\n }\n // expand rest up\n {\n int cur_backend_id = -1;\n for (int i = graph->n_nodes - 1; i >= 0; i--) {\n struct ggml_tensor * node = graph->nodes[i];\n if (ggml_is_view_op(node->op)) {\n continue;\n }\n int * node_backend_id = &tensor_backend_id(node);\n if (*node_backend_id != -1) {\n cur_backend_id = *node_backend_id;\n } else if (cur_backend_id != -1) {\n ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);\n }\n }\n }\n\n // pass 3: upgrade nodes to higher prio backends with compatible buffer types\n // if the tensor is already in the same buffer type (*) as another higher priority backend, we should move it there\n // however, we also need to verify that the sources are in compatible buffer types\n // (*) the actual requirement is more relaxed, the buffer type of the backend should be supported by all the users of this tensor further down the graph\n // however, this is slow to verify, so we have a more strict requirement that the buffer type is the same\n // this is not uncommon since multiple backends can use host memory, with the same buffer type (eg. BLAS and CPU)\n // additionally, set remaining unassigned nodes to the backend with the most supported inputs\n // only nodes that could not be assigned during expansion due to the backend not supporting the op should be unassigned at this point\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n if (ggml_is_view_op(node->op)) {\n continue;\n }\n int * node_backend_id = &tensor_backend_id(node);\n if (*node_backend_id == -1) {\n // unassigned node: find the backend with the most supported inputs\n int n_supported_best = -1;\n for (int b = 0; b < sched->n_backends; b++) {\n if (ggml_backend_supports_op(sched->backends[b], node)) {\n int n_supported = 0;\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n if ((tensor_backend_id(src) != -1 || tensor_backend_id(src->view_src) != -1) && ggml_backend_sched_buffer_supported(sched, src, b)) {\n n_supported++;\n }\n }\n if (n_supported > n_supported_best) {\n n_supported_best = n_supported;\n *node_backend_id = b;\n SET_CAUSE(node, \"3.best\");\n }\n }\n }\n } else {\n // assigned node: upgrade to higher prio backend if possible\n for (int b = 0; b < *node_backend_id; b++) {\n if (sched->bufts[b] == sched->bufts[*node_backend_id] && ggml_backend_supports_op(sched->backends[b], node)) {\n bool supported = true;\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n if (!ggml_backend_sched_buffer_supported(sched, src, b)) {\n supported = false;\n break;\n }\n }\n if (supported) {\n *node_backend_id = b;\n SET_CAUSE(node, \"3.upg\");\n break;\n }\n }\n }\n }\n }\n\n // pass 4: assign backends to remaining src from dst and view_src\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n int * cur_backend_id = &tensor_backend_id(node);\n if (node->view_src != NULL && *cur_backend_id == -1) {\n *cur_backend_id = tensor_backend_id(node->view_src);\n SET_CAUSE(node, \"4.vsrc\");\n }\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n int * src_backend_id = &tensor_backend_id(src);\n if (*src_backend_id == -1) {\n if (src->view_src != NULL) {\n // views are always on the same backend as the source\n *src_backend_id = tensor_backend_id(src->view_src);\n SET_CAUSE(src, \"4.vsrc\");\n } else {\n *src_backend_id = *cur_backend_id;\n SET_CAUSE(src, \"4.cur\");\n }\n }\n }\n // if the node is still unassigned, assign it to the first backend that supports it\n for (int b = 0; b < sched->n_backends && *cur_backend_id == -1; b++) {\n ggml_backend_sched_set_if_supported(sched, node, b, cur_backend_id);\n }\n GGML_ASSERT(*cur_backend_id != -1);\n }\n\n // pass 5: split graph, find tensors that need to be copied\n {\n int i_split = 0;\n struct ggml_backend_sched_split * split = &sched->splits[0];\n // find the backend of the first split, skipping view ops\n int i = 0;\n for (; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n if (!ggml_is_view_op(node->op)) {\n split->backend_id = tensor_backend_id(node);\n break;\n }\n }\n split->i_start = 0;\n split->n_inputs = 0;\n int cur_backend_id = split->backend_id;\n for (; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n\n if (ggml_is_view_op(node->op)) {\n continue;\n }\n\n const int node_backend_id = tensor_backend_id(node);\n\n GGML_ASSERT(node_backend_id != -1); // all nodes should be assigned by now, this can happen if there is no CPU fallback\n\n // check if we should start a new split based on the sources of the current node\n bool need_new_split = false;\n if (node_backend_id == cur_backend_id && split->n_inputs > 0) {\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n // check if a weight is on a different and incompatible backend\n // by starting a new split, the memory of the previously offloaded weights can be reused\n if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {\n int src_backend_id = tensor_backend_id(src);\n if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {\n need_new_split = true;\n break;\n }\n }\n // check if the split has too many inputs\n // FIXME: count the number of inputs instead of only checking when full\n if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) {\n const size_t id = hash_id(src);\n int src_backend_id = sched->hv_tensor_backend_ids[id];\n bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id);\n if (src_backend_id != cur_backend_id && tensor_id_copy(id, cur_backend_id, 0) == NULL && !supported) {\n need_new_split = true;\n break;\n }\n }\n }\n }\n\n if (node_backend_id != cur_backend_id || need_new_split) {\n split->i_end = i;\n i_split++;\n if (i_split >= sched->splits_capacity) {\n sched->splits_capacity *= 2;\n sched->splits = (ggml_backend_sched_split *)\n realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));\n GGML_ASSERT(sched->splits != NULL);\n }\n split = &sched->splits[i_split];\n split->backend_id = node_backend_id;\n split->i_start = i;\n split->n_inputs = 0;\n cur_backend_id = node_backend_id;\n }\n\n // find inputs that are not on the same backend\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n struct ggml_tensor * src = node->src[j];\n if (src == NULL) {\n continue;\n }\n\n size_t src_id = hash_id(src);\n const int src_backend_id = sched->hv_tensor_backend_ids[src_id];\n GGML_ASSERT(src_backend_id != -1); // all inputs should be assigned by now\n\n if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) {\n if (tensor_id_copy(src_id, src_backend_id, 0) == NULL) {\n ggml_backend_t backend = sched->backends[src_backend_id];\n for (int c = 0; c < sched->n_copies; c++) {\n struct ggml_tensor * tensor_copy;\n if (c == sched->cur_copy) {\n tensor_copy = src; // use the original tensor as the current copy\n } else {\n tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);\n ggml_format_name(tensor_copy, \"%s#%s#%d\", ggml_backend_name(backend), src->name, c);\n }\n ggml_set_input(tensor_copy);\n ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor\n tensor_id_copy(src_id, src_backend_id, c) = tensor_copy;\n SET_CAUSE(tensor_copy, \"4.cpy\");\n }\n int n_graph_inputs = sched->n_graph_inputs++;\n GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);\n sched->graph_inputs[n_graph_inputs] = src;\n }\n }\n\n if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {\n // create a copy of the input in the split's backend\n if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) {\n ggml_backend_t backend = sched->backends[cur_backend_id];\n for (int c = 0; c < sched->n_copies; c++) {\n struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);\n ggml_format_name(tensor_copy, \"%s#%s#%d\", ggml_backend_name(backend), src->name, c);\n if (sched->n_copies > 1) {\n ggml_set_input(tensor_copy);\n ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor\n }\n tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy;\n SET_CAUSE(tensor_copy, \"4.cpy\");\n }\n int n_inputs = split->n_inputs++;\n GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);\n split->inputs[n_inputs] = src;\n }\n node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy);\n }\n }\n }\n split->i_end = graph->n_nodes;\n sched->n_splits = i_split + 1;\n }\n\n if (sched->debug) {\n ggml_backend_sched_print_assignments(sched, graph);\n }\n\n // swap node_backend_ids and leaf _backend_ids with prevs\n {\n int * tmp = sched->node_backend_ids;\n sched->node_backend_ids = sched->prev_node_backend_ids;\n sched->prev_node_backend_ids = tmp;\n\n tmp = sched->leaf_backend_ids;\n sched->leaf_backend_ids = sched->prev_leaf_backend_ids;\n sched->prev_leaf_backend_ids = tmp;\n }\n\n int graph_size = std::max(graph->n_nodes, graph->n_leafs) + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sched->n_copies;\n\n // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC]\n sched->debug_prev_graph_size = sched->debug_graph_size;\n sched->debug_graph_size = graph_size;\n\n if (sched->graph.size < graph_size) {\n sched->graph.size = graph_size;\n sched->graph.nodes = (ggml_tensor **) realloc(sched->graph.nodes, graph_size * sizeof(struct ggml_tensor *));\n sched->graph.leafs = (ggml_tensor **) realloc(sched->graph.leafs, graph_size * sizeof(struct ggml_tensor *));\n GGML_ASSERT(sched->graph.nodes != NULL);\n GGML_ASSERT(sched->graph.leafs != NULL);\n }\n sched->graph.n_nodes = 0;\n sched->graph.n_leafs = 0;\n\n struct ggml_cgraph * graph_copy = &sched->graph;\n\n for (int i = 0; i < sched->n_splits; i++) {\n struct ggml_backend_sched_split * split = &sched->splits[i];\n split->graph = ggml_graph_view(graph, split->i_start, split->i_end);\n\n // Optimize this split of the graph. This needs to happen before we make graph_copy,\n // so they are in sync.\n ggml_backend_graph_optimize(sched->backends[split->backend_id], &split->graph);\n\n // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split\n for (int j = 0; j < split->n_inputs; j++) {\n assert(graph_copy->size > (graph_copy->n_nodes + 1));\n\n struct ggml_tensor * input = split->inputs[j];\n const size_t input_id = hash_id(input);\n struct ggml_tensor * input_cpy = tensor_id_copy(input_id, split->backend_id, sched->cur_copy);\n\n // add a dependency to the input source so that it is not freed before the copy is done\n struct ggml_tensor * input_dep = ggml_view_tensor(sched->ctx, input);\n input_dep->src[0] = input;\n sched->node_backend_ids[graph_copy->n_nodes] = sched->hv_tensor_backend_ids[input_id];\n graph_copy->nodes[graph_copy->n_nodes++] = input_dep;\n\n // add a dependency to the input copy so that it is allocated at the start of the split\n sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id;\n graph_copy->nodes[graph_copy->n_nodes++] = input_cpy;\n }\n\n for (int j = split->i_start; j < split->i_end; j++) {\n assert(graph_copy->size > graph_copy->n_nodes);\n sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]);\n graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j];\n }\n }\n\n if (sched->n_copies > 1) {\n // add input copies as leafs so that they are allocated first\n for (int i = 0; i < sched->n_graph_inputs; i++) {\n struct ggml_tensor * input = sched->graph_inputs[i];\n size_t id = hash_id(input);\n int backend_id = tensor_backend_id(input);\n for (int c = 0; c < sched->n_copies; c++) {\n struct ggml_tensor * input_cpy = tensor_id_copy(id, backend_id, c);\n sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;\n assert(graph_copy->size > graph_copy->n_leafs);\n graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;\n }\n }\n\n for (int i = 0; i < sched->n_splits; i++) {\n struct ggml_backend_sched_split * split = &sched->splits[i];\n int backend_id = split->backend_id;\n for (int j = 0; j < split->n_inputs; j++) {\n struct ggml_tensor * input = split->inputs[j];\n size_t id = hash_id(input);\n for (int c = 0; c < sched->n_copies; c++) {\n struct ggml_tensor * input_cpy = tensor_id_copy(id, backend_id, c);\n sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;\n assert(graph_copy->size > graph_copy->n_leafs);\n graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;\n }\n }\n }\n }\n\n // add leafs from the original graph\n for (int i = 0; i < graph->n_leafs; i++) {\n struct ggml_tensor * leaf = graph->leafs[i];\n sched->leaf_backend_ids[graph_copy->n_leafs] = tensor_backend_id(leaf);\n assert(graph_copy->size > graph_copy->n_leafs);\n graph_copy->leafs[graph_copy->n_leafs++] = leaf;\n }\n}\n\nstatic bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {\n bool backend_ids_changed = false;\n for (int i = 0; i < sched->graph.n_nodes; i++) {\n if (sched->node_backend_ids[i] != sched->prev_node_backend_ids[i] &&\n sched->bufts[sched->node_backend_ids[i]] != sched->bufts[sched->prev_node_backend_ids[i]]) {\n backend_ids_changed = true;\n break;\n }\n }\n if (!backend_ids_changed) {\n for (int i = 0; i < sched->graph.n_leafs; i++) {\n if (sched->leaf_backend_ids[i] != sched->prev_leaf_backend_ids[i] &&\n sched->bufts[sched->leaf_backend_ids[i]] != sched->bufts[sched->prev_leaf_backend_ids[i]]) {\n backend_ids_changed = true;\n break;\n }\n }\n }\n\n // allocate graph\n if (backend_ids_changed || !ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: failed to allocate graph, reserving (backend_ids_changed = %d)\\n\", __func__, backend_ids_changed);\n#endif\n\n if (sched->debug_realloc > 0) {\n // we are interested only in situations where the graph was reallocated even though its size remained the same [GGML_SCHED_DEBUG_REALLOC]\n // example: https://github.com/ggml-org/llama.cpp/pull/17143\n const bool unexpected = !backend_ids_changed && sched->debug_prev_graph_size == sched->debug_graph_size;\n\n if (unexpected || sched->debug_realloc > 1) {\n GGML_ABORT(\"%s: unexpected graph reallocation (graph size = %d, nodes = %d, leafs = %d), debug_realloc = %d\\n\", __func__,\n sched->debug_graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc);\n }\n }\n\n // the re-allocation may cause the split inputs to be moved to a different address\n // synchronize without ggml_backend_sched_synchronize to avoid changing cur_copy\n for (int i = 0; i < sched->n_backends; i++) {\n ggml_backend_synchronize(sched->backends[i]);\n }\n\n ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids);\n if (!ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) {\n GGML_LOG_ERROR(\"%s: failed to allocate graph\\n\", __func__);\n return false;\n }\n }\n\n return true;\n}\n\nstatic enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {\n GGML_ASSERT(sched);\n struct ggml_backend_sched_split * splits = sched->splits;\n\n ggml_tensor * prev_ids_tensor = nullptr;\n std::vector ids;\n std::vector used_ids;\n\n for (int split_id = 0; split_id < sched->n_splits; split_id++) {\n struct ggml_backend_sched_split * split = &splits[split_id];\n int split_backend_id = split->backend_id;\n ggml_backend_t split_backend = sched->backends[split_backend_id];\n\n // copy the input tensors to the split backend\n for (int input_id = 0; input_id < split->n_inputs; input_id++) {\n ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]);\n struct ggml_tensor * input = split->inputs[input_id];\n struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy);\n\n if (input->flags & GGML_TENSOR_FLAG_INPUT) {\n // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done\n if (sched->events[split_backend_id][sched->cur_copy] != NULL) {\n ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);\n } else {\n ggml_backend_synchronize(split_backend);\n }\n ggml_backend_tensor_copy(input, input_cpy);\n } else {\n // wait for the split backend to finish using the input before overwriting it\n if (sched->events[split_backend_id][sched->cur_copy] != NULL) {\n ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);\n } else {\n ggml_backend_synchronize(split_backend);\n }\n\n // when offloading MoE weights, we can reduce the amount of data copied by copying only the experts that are used\n ggml_tensor * node = split->graph.nodes[0];\n if (split->graph.n_nodes > 0 &&\n ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS &&\n ggml_backend_buffer_is_host(input->buffer) && (\n (node->src[0] == input_cpy && node->op == GGML_OP_MUL_MAT_ID)\n //|| (node->src[1] == input_cpy && node->op == GGML_OP_ADD_ID) /* GGML_OP_ADD_ID weights are small and not worth splitting */\n )) {\n\n const int64_t n_expert = node->op == GGML_OP_MUL_MAT_ID ? input->ne[2] : input->ne[1];\n const size_t expert_size = node->op == GGML_OP_MUL_MAT_ID ? input->nb[2] : input->nb[1];\n\n ggml_backend_synchronize(input_backend);\n\n // get the ids\n ggml_tensor * ids_tensor = node->src[2];\n ggml_backend_t ids_backend = split_backend;\n\n // if the ids tensor is also an input of the split, it may not have been copied yet to the split backend\n // in that case, we use the original ids tensor\n for (int i = input_id + 1; i < split->n_inputs; i++) {\n if (ids_tensor == tensor_copy(split->inputs[i], split_backend_id, sched->cur_copy)) {\n ids_tensor = split->inputs[i];\n ids_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[i]);\n break;\n }\n }\n\n if (ids_tensor != prev_ids_tensor) {\n ids.resize(ggml_nbytes(ids_tensor) / sizeof(int32_t));\n ggml_backend_tensor_get_async(ids_backend, ids_tensor, ids.data(), 0, ggml_nbytes(ids_tensor));\n ggml_backend_synchronize(ids_backend);\n\n // find the used experts\n used_ids.clear();\n used_ids.resize(ggml_bitset_size(n_expert));\n for (int64_t i1 = 0; i1 < ids_tensor->ne[1]; i1++) {\n for (int64_t i0 = 0; i0 < ids_tensor->ne[0]; i0++) {\n int32_t id = ids[i1 * ids_tensor->nb[1]/sizeof(int32_t) + i0 * ids_tensor->nb[0]/sizeof(int32_t)];\n GGML_ASSERT(id >= 0 && id < n_expert);\n ggml_bitset_set(used_ids.data(), id);\n }\n }\n\n prev_ids_tensor = ids_tensor;\n }\n\n // group consecutive experts and copy them together\n auto copy_experts = [&](int32_t first_id, int32_t last_id) {\n const size_t expert_offset = first_id * expert_size;\n const size_t expert_size_copy = (last_id - first_id + 1) * expert_size;\n const size_t padding = std::min(expert_size, 512);\n const size_t padding_end = last_id < n_expert - 1 ? padding : 0;\n\n ggml_backend_tensor_set_async(split_backend,\n input_cpy,\n (const uint8_t *)input->data + expert_offset, expert_offset,\n // copy a bit extra at the to ensure there are no NaNs in the padding of the last expert\n // this is necessary for MMQ in the CUDA backend\n expert_size_copy + padding_end);\n };\n\n int id = 0;\n while (!ggml_bitset_get(used_ids.data(), id)) {\n id++;\n }\n int32_t first_id = id;\n int32_t last_id = first_id;\n\n for (++id; id < n_expert; ++id) {\n if (!ggml_bitset_get(used_ids.data(), id)) {\n continue;\n }\n\n if (id == last_id + 1) {\n last_id = id;\n continue;\n }\n\n copy_experts(first_id, last_id);\n\n first_id = id;\n last_id = id;\n }\n copy_experts(first_id, last_id);\n } else {\n // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events\n // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface\n if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {\n ggml_backend_synchronize(input_backend);\n if (sched->events[split_backend_id][sched->cur_copy] != NULL) {\n ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);\n } else {\n ggml_backend_synchronize(split_backend);\n }\n ggml_backend_tensor_copy(input, input_cpy);\n }\n }\n }\n }\n\n if (!sched->callback_eval) {\n enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph);\n if (ec != GGML_STATUS_SUCCESS) {\n return ec;\n }\n } else {\n // similar to ggml_backend_compare_graph_backend\n for (int j0 = 0; j0 < split->graph.n_nodes; j0++) {\n struct ggml_tensor * t = split->graph.nodes[j0];\n\n // check if the user needs data from this node\n bool need = sched->callback_eval(t, true, sched->callback_eval_user_data);\n\n int j1 = j0;\n\n // determine the range [j0, j1] of nodes that can be computed together\n while (!need && j1 < split->graph.n_nodes - 1) {\n t = split->graph.nodes[++j1];\n need = sched->callback_eval(t, true, sched->callback_eval_user_data);\n }\n\n struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1);\n\n enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &gv);\n if (ec != GGML_STATUS_SUCCESS) {\n return ec;\n }\n\n // TODO: pass backend to the callback, then the user can decide if they want to synchronize\n ggml_backend_synchronize(split_backend);\n\n if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) {\n break;\n }\n\n j0 = j1;\n }\n }\n\n // record the event of this copy\n if (split->n_inputs > 0) {\n if (sched->events[split_backend_id][sched->cur_copy] != NULL) {\n ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend);\n }\n }\n }\n\n return GGML_STATUS_SUCCESS;\n}\n\nggml_backend_sched_t ggml_backend_sched_new(\n ggml_backend_t * backends,\n ggml_backend_buffer_type_t * bufts,\n int n_backends,\n size_t graph_size,\n bool parallel,\n bool op_offload) {\n GGML_ASSERT(n_backends > 0);\n GGML_ASSERT(n_backends <= GGML_SCHED_MAX_BACKENDS);\n GGML_ASSERT(ggml_backend_dev_type(ggml_backend_get_device(backends[n_backends - 1])) == GGML_BACKEND_DEVICE_TYPE_CPU);\n\n struct ggml_backend_sched * sched = (ggml_backend_sched *) calloc(1, sizeof(struct ggml_backend_sched));\n\n const char * GGML_SCHED_DEBUG = getenv(\"GGML_SCHED_DEBUG\");\n sched->debug = GGML_SCHED_DEBUG ? atoi(GGML_SCHED_DEBUG) : 0;\n\n sched->debug_realloc = 0;\n#ifdef GGML_SCHED_NO_REALLOC\n sched->debug_realloc = 1;\n#endif\n const char * GGML_SCHED_DEBUG_REALLOC = getenv(\"GGML_SCHED_DEBUG_REALLOC\");\n sched->debug_realloc = GGML_SCHED_DEBUG_REALLOC ? atoi(GGML_SCHED_DEBUG_REALLOC) : sched->debug_realloc;\n\n sched->n_backends = n_backends;\n sched->n_copies = parallel ? GGML_SCHED_MAX_COPIES : 1;\n\n // initialize hash table\n // FIXME: needs to be size*2 to account for leafs (do it in graph_split instead)\n sched->hash_set = ggml_hash_set_new(graph_size);\n sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));\n sched->hv_tensor_copies = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));\n\n const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph\n const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2;\n sched->node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->node_backend_ids[0]));\n sched->leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->leaf_backend_ids[0]));\n sched->prev_node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0]));\n sched->prev_leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0]));\n\n sched->debug_graph_size = 0;\n sched->debug_prev_graph_size = 0;\n\n sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);\n sched->context_buffer = (char *) malloc(sched->context_buffer_size);\n\n const int initial_splits_capacity = 16;\n sched->splits = (ggml_backend_sched_split *) calloc(initial_splits_capacity, sizeof(sched->splits[0]));\n sched->splits_capacity = initial_splits_capacity;\n\n for (int b = 0; b < n_backends; b++) {\n sched->backends[b] = backends[b];\n sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]);\n GGML_ASSERT(ggml_backend_supports_buft(backends[b], sched->bufts[b]));\n\n if (sched->n_copies > 1) {\n for (int c = 0; c < sched->n_copies; c++) {\n sched->events[b][c] = ggml_backend_event_new(backends[b]->device);\n }\n }\n }\n\n sched->galloc = ggml_gallocr_new_n(sched->bufts, n_backends);\n sched->op_offload = op_offload;\n\n ggml_backend_sched_reset(sched);\n\n return sched;\n}\n\nvoid ggml_backend_sched_free(ggml_backend_sched_t sched) {\n if (sched == NULL) {\n return;\n }\n for (int b = 0; b < sched->n_backends; b++) {\n for (int c = 0; c < sched->n_copies; c++) {\n ggml_backend_event_free(sched->events[b][c]);\n }\n }\n ggml_gallocr_free(sched->galloc);\n ggml_free(sched->ctx);\n ggml_hash_set_free(&sched->hash_set);\n free(sched->splits);\n free(sched->hv_tensor_backend_ids);\n free(sched->hv_tensor_copies);\n free(sched->node_backend_ids);\n free(sched->leaf_backend_ids);\n free(sched->prev_node_backend_ids);\n free(sched->prev_leaf_backend_ids);\n free(sched->context_buffer);\n free(sched->graph.nodes);\n free(sched->graph.leafs);\n free(sched);\n}\n\nvoid ggml_backend_sched_reset(ggml_backend_sched_t sched) {\n GGML_ASSERT(sched);\n // reset state for the next run\n if (!sched->is_reset) {\n ggml_hash_set_reset(&sched->hash_set);\n memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));\n memset(sched->hv_tensor_copies, 0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));\n sched->is_reset = true;\n }\n sched->is_alloc = false;\n}\n\nvoid ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes) {\n GGML_ASSERT(sched);\n GGML_ASSERT((int)sched->hash_set.size >= measure_graph->n_nodes + measure_graph->n_leafs);\n GGML_ASSERT(sizes);\n\n ggml_backend_sched_reset(sched);\n\n ggml_backend_sched_synchronize(sched);\n\n ggml_backend_sched_split_graph(sched, measure_graph);\n\n ggml_gallocr_reserve_n_size(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids, sizes);\n}\n\nbool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) {\n GGML_ASSERT(sched);\n GGML_ASSERT((int)sched->hash_set.size >= measure_graph->n_nodes + measure_graph->n_leafs);\n\n ggml_backend_sched_synchronize(sched);\n\n ggml_backend_sched_split_graph(sched, measure_graph);\n\n if (!ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) {\n return false;\n }\n\n ggml_backend_sched_reset(sched);\n\n return true;\n}\n\nbool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n GGML_ASSERT(sched);\n GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + graph->n_leafs);\n GGML_ASSERT(!sched->is_alloc);\n\n sched->cur_copy = sched->next_copy;\n sched->next_copy = (sched->next_copy + 1) % sched->n_copies;\n\n ggml_backend_sched_split_graph(sched, graph);\n\n if (!ggml_backend_sched_alloc_splits(sched)) {\n return false;\n }\n\n sched->is_alloc = true;\n\n return true;\n}\n\nenum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n enum ggml_status err = ggml_backend_sched_graph_compute_async(sched, graph);\n ggml_backend_sched_synchronize(sched);\n return err;\n}\n\nenum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n GGML_ASSERT(sched);\n if (!sched->is_reset && !sched->is_alloc) {\n ggml_backend_sched_reset(sched);\n }\n\n if (!sched->is_alloc) {\n if (!ggml_backend_sched_alloc_graph(sched, graph)) {\n return GGML_STATUS_ALLOC_FAILED;\n }\n }\n\n return ggml_backend_sched_compute_splits(sched);\n}\n\nvoid ggml_backend_sched_synchronize(ggml_backend_sched_t sched) {\n GGML_ASSERT(sched);\n for (int i = 0; i < sched->n_backends; i++) {\n ggml_backend_synchronize(sched->backends[i]);\n }\n if (!sched->is_alloc) {\n // if the graph is not already allocated, always use copy 0 after a synchronization\n // this ensures that during generation the same copy is used every time,\n // which avoids changes in the graph that could cause CUDA or other graphs to be disabled\n sched->next_copy = 0;\n }\n}\n\nvoid ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) {\n GGML_ASSERT(sched);\n sched->callback_eval = callback;\n sched->callback_eval_user_data = user_data;\n}\n\nint ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {\n GGML_ASSERT(sched);\n return sched->n_splits;\n}\n\nint ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched) {\n GGML_ASSERT(sched);\n return sched->n_copies;\n}\n\nint ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched) {\n GGML_ASSERT(sched);\n return sched->n_backends;\n}\n\nggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i) {\n GGML_ASSERT(sched);\n GGML_ASSERT(i >= 0 && i < sched->n_backends);\n return sched->backends[i];\n}\n\nggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend) {\n GGML_ASSERT(sched);\n int backend_index = ggml_backend_sched_backend_id(sched, backend);\n GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);\n\n return sched->bufts[backend_index];\n}\n\nsize_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) {\n GGML_ASSERT(sched);\n int backend_index = ggml_backend_sched_backend_id(sched, backend);\n GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);\n\n return ggml_gallocr_get_buffer_size(sched->galloc, backend_index);\n}\n\nvoid ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) {\n GGML_ASSERT(sched);\n int backend_index = ggml_backend_sched_backend_id(sched, backend);\n GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);\n tensor_backend_id(node) = backend_index;\n SET_CAUSE(node, \"usr\");\n sched->is_reset = false;\n}\n\nggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) {\n GGML_ASSERT(sched);\n int backend_index = tensor_backend_id(node);\n if (backend_index == -1) {\n return NULL;\n }\n return sched->backends[backend_index];\n}\n\n// utils\n\nenum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor) {\n GGML_ASSERT(tensor);\n GGML_ASSERT(tensor->buffer == NULL);\n GGML_ASSERT(tensor->view_src != NULL);\n GGML_ASSERT(tensor->view_src->buffer != NULL);\n GGML_ASSERT(tensor->view_src->data != NULL);\n\n tensor->buffer = tensor->view_src->buffer;\n tensor->data = (char *)tensor->view_src->data + tensor->view_offs;\n return ggml_backend_buffer_init_tensor(tensor->buffer, tensor);\n}\n\nenum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr) {\n GGML_ASSERT(tensor);\n GGML_ASSERT(tensor->buffer == NULL);\n GGML_ASSERT(tensor->data == NULL);\n GGML_ASSERT(tensor->view_src == NULL);\n GGML_ASSERT(addr >= ggml_backend_buffer_get_base(buffer));\n GGML_ASSERT((char *)addr + ggml_backend_buffer_get_alloc_size(buffer, tensor) <=\n (char *)ggml_backend_buffer_get_base(buffer) + ggml_backend_buffer_get_size(buffer));\n\n tensor->buffer = buffer;\n tensor->data = addr;\n return ggml_backend_buffer_init_tensor(buffer, tensor);\n}\n\nstatic struct ggml_tensor * graph_copy_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies,\n struct ggml_context * ctx_allocated, struct ggml_context * ctx_unallocated, struct ggml_tensor * src) {\n\n GGML_ASSERT(src != NULL);\n GGML_ASSERT(src->data && \"graph must be allocated\");\n\n size_t id = ggml_hash_insert(&hash_set, src);\n if (id == GGML_HASHSET_ALREADY_EXISTS) {\n return node_copies[ggml_hash_find(&hash_set, src)];\n }\n\n struct ggml_tensor * dst = ggml_dup_tensor_layout(src->data && !src->view_src ? ctx_allocated : ctx_unallocated, src);\n if (src->view_src != NULL) {\n dst->view_src = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, src->view_src);\n dst->view_offs = src->view_offs;\n }\n dst->op = src->op;\n dst->flags = src->flags;\n memcpy(dst->op_params, src->op_params, sizeof(dst->op_params));\n ggml_set_name(dst, src->name);\n\n // copy src\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n struct ggml_tensor * s = src->src[i];\n if (s == NULL) {\n continue;\n }\n dst->src[i] = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s);\n }\n\n node_copies[id] = dst;\n return dst;\n}\n\nstatic void graph_copy_init_tensor(struct ggml_hash_set * hash_set, struct ggml_tensor ** node_copies, bool * node_init, struct ggml_tensor * src) {\n size_t id = ggml_hash_find(hash_set, src);\n if (node_init[id]) {\n return;\n }\n node_init[id] = true;\n\n struct ggml_tensor * dst = node_copies[id];\n if (dst->view_src != NULL) {\n graph_copy_init_tensor(hash_set, node_copies, node_init, src->view_src);\n enum ggml_status status = ggml_backend_view_init(dst);\n GGML_ASSERT(status == GGML_STATUS_SUCCESS);\n }\n else {\n ggml_backend_tensor_copy(src, dst);\n }\n\n // init src\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n struct ggml_tensor * s = src->src[i];\n if (s == NULL) {\n continue;\n }\n graph_copy_init_tensor(hash_set, node_copies, node_init, s);\n }\n}\n\nstruct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph) {\n GGML_ASSERT(graph);\n struct ggml_hash_set hash_set = ggml_hash_set_new(graph->visited_hash_set.size);\n struct ggml_tensor ** node_copies = (ggml_tensor **) calloc(hash_set.size, sizeof(node_copies[0])); // NOLINT\n bool * node_init = (bool *) calloc(hash_set.size, sizeof(node_init[0]));\n\n struct ggml_init_params params = {\n /* .mem_size = */ ggml_tensor_overhead()*hash_set.size + ggml_graph_overhead_custom(graph->size, false),\n /* .mem_buffer = */ NULL,\n /* .no_alloc = */ true\n };\n\n struct ggml_context * ctx_allocated = ggml_init(params);\n struct ggml_context * ctx_unallocated = ggml_init(params);\n\n if (ctx_allocated == NULL || ctx_unallocated == NULL) {\n GGML_LOG_ERROR(\"%s: failed to allocate context for graph copy\\n\", __func__);\n ggml_hash_set_free(&hash_set);\n free(node_copies);\n free(node_init);\n ggml_free(ctx_allocated);\n ggml_free(ctx_unallocated);\n return {\n /* .buffer = */ NULL,\n /* .ctx_allocated = */ NULL,\n /* .ctx_unallocated = */ NULL,\n /* .graph = */ NULL,\n };\n }\n\n // dup nodes\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, node);\n }\n\n // allocate nodes\n ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx_allocated, backend);\n if (buffer == NULL) {\n GGML_LOG_ERROR(\"%s: failed to allocate buffer for graph copy\\n\", __func__);\n ggml_hash_set_free(&hash_set);\n free(node_copies);\n free(node_init);\n ggml_free(ctx_allocated);\n ggml_free(ctx_unallocated);\n return {\n /* .buffer = */ NULL,\n /* .ctx_allocated = */ NULL,\n /* .ctx_unallocated = */ NULL,\n /* .graph = */ NULL,\n };\n }\n\n //printf(\"copy buffer size: %zu MB\\n\", ggml_backend_buffer_get_size(buffer) / 1024 / 1024);\n\n // copy data and init views\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n graph_copy_init_tensor(&hash_set, node_copies, node_init, node);\n }\n\n // build graph copy\n struct ggml_cgraph * graph_copy = ggml_new_graph_custom(ctx_allocated, graph->size, false);\n for (int i = 0; i < graph->n_nodes; i++) {\n struct ggml_tensor * node = graph->nodes[i];\n struct ggml_tensor * node_copy = node_copies[ggml_hash_find(&hash_set, node)];\n graph_copy->nodes[i] = node_copy;\n }\n graph_copy->n_nodes = graph->n_nodes;\n\n ggml_hash_set_free(&hash_set);\n free(node_copies);\n free(node_init);\n\n return {\n /* .buffer = */ buffer,\n /* .ctx_allocated = */ ctx_allocated,\n /* .ctx_unallocated = */ ctx_unallocated,\n /* .graph = */ graph_copy,\n };\n}\n\nvoid ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy) {\n ggml_backend_buffer_free(copy.buffer);\n ggml_free(copy.ctx_allocated);\n ggml_free(copy.ctx_unallocated);\n}\n\nbool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes) {\n struct ggml_backend_graph_copy copy = ggml_backend_graph_copy(backend2, graph);\n if (copy.buffer == NULL) {\n return false;\n }\n\n struct ggml_cgraph * g1 = graph;\n struct ggml_cgraph * g2 = copy.graph;\n\n assert(g1->n_nodes == g2->n_nodes);\n\n if (num_test_nodes != 0) {\n GGML_ASSERT(test_nodes);\n // Compute the whole graph and only test the output for specific tensors\n ggml_backend_graph_compute(backend1, g1);\n ggml_backend_graph_compute(backend2, g2);\n\n bool verified = false;\n for (int i = 0; i < g1->n_nodes; i++) {\n for (size_t j = 0; j < num_test_nodes; ++j) {\n if (g1->nodes[i] == test_nodes[j]) {\n callback(i, g1->nodes[i], g2->nodes[i], user_data);\n verified = true;\n }\n }\n }\n GGML_ASSERT(verified);\n } else {\n for (int i = 0; i < g1->n_nodes; i++) {\n struct ggml_tensor * t1 = g1->nodes[i];\n struct ggml_tensor * t2 = g2->nodes[i];\n\n assert(t1->op == t2->op && ggml_are_same_layout(t1, t2));\n\n struct ggml_cgraph g1v = ggml_graph_view(g1, i, i + 1);\n struct ggml_cgraph g2v = ggml_graph_view(g2, i, i + 1);\n\n ggml_backend_graph_compute(backend1, &g1v);\n ggml_backend_graph_compute(backend2, &g2v);\n\n if (ggml_is_view_op(t1->op)) {\n continue;\n }\n\n // compare results, calculate rms etc\n if (!callback(i, t1, t2, user_data)) {\n break;\n }\n }\n }\n ggml_backend_graph_copy_free(copy);\n\n return true;\n}\n\n// CPU backend - buffer\n\nstatic void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {\n GGML_ASSERT(buffer);\n uintptr_t data = (uintptr_t)buffer->context;\n\n // align the buffer\n if (data % TENSOR_ALIGNMENT != 0) {\n data = GGML_PAD(data, TENSOR_ALIGNMENT);\n }\n\n return (void *)data;\n}\n\nstatic void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n GGML_ASSERT(buffer);\n ggml_aligned_free(buffer->context, buffer->size);\n}\n\nstatic void ggml_backend_cpu_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {\n GGML_ASSERT(tensor);\n memset((char *)tensor->data + offset, value, size);\n\n GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n GGML_ASSERT(tensor);\n memcpy((char *)tensor->data + offset, data, size);\n\n GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n GGML_ASSERT(tensor);\n memcpy(data, (const char *)tensor->data + offset, size);\n\n GGML_UNUSED(buffer);\n}\n\nstatic bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {\n GGML_ASSERT(src);\n if (ggml_backend_buffer_is_host(src->buffer)) {\n memcpy(dst->data, src->data, ggml_nbytes(src));\n return true;\n }\n return false;\n\n GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n GGML_ASSERT(buffer);\n memset(buffer->context, value, buffer->size);\n}\n\nstatic const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {\n /* .free_buffer = */ ggml_backend_cpu_buffer_free_buffer,\n /* .get_base = */ ggml_backend_cpu_buffer_get_base,\n /* .init_tensor = */ NULL, // no initialization required\n /* .memset_tensor = */ ggml_backend_cpu_buffer_memset_tensor,\n /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,\n /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,\n /* .clear = */ ggml_backend_cpu_buffer_clear,\n /* .reset = */ NULL,\n};\n\nstatic const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {\n /* .free_buffer = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed\n /* .get_base = */ ggml_backend_cpu_buffer_get_base,\n /* .init_tensor = */ NULL, // no initialization required\n /* .memset_tensor = */ ggml_backend_cpu_buffer_memset_tensor,\n /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,\n /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,\n /* .clear = */ ggml_backend_cpu_buffer_clear,\n /* .reset = */ NULL,\n};\n\n// CPU backend buffer type\n\n// this buffer type is defined here to make it available to all backends\n\nstatic const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n return \"CPU\";\n\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n void * data = ggml_aligned_malloc(size);\n\n if (data == NULL) {\n GGML_LOG_ERROR(\"%s: failed to allocate buffer of size %zu\\n\", __func__, size);\n return NULL;\n }\n\n return ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);\n}\n\nstatic size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n return TENSOR_ALIGNMENT;\n\n GGML_UNUSED(buft);\n}\n\nstatic bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) {\n return true;\n\n GGML_UNUSED(buft);\n}\n\nggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {\n static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_cpu_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,\n /* .get_max_size = */ NULL, // defaults to SIZE_MAX\n /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes\n /* .is_host = */ ggml_backend_cpu_buffer_type_is_host,\n },\n /* .device = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n /* .context = */ NULL,\n };\n\n return &ggml_backend_cpu_buffer_type;\n}\n\nstatic const char * ggml_backend_cpu_buffer_from_ptr_type_get_name(ggml_backend_buffer_type_t buft) {\n return \"CPU_Mapped\";\n\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cpu_buffer_from_ptr_type(void) {\n static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_cpu_buffer_from_ptr_type_get_name,\n /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,\n /* .get_max_size = */ NULL, // defaults to SIZE_MAX\n /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes\n /* .is_host = */ ggml_backend_cpu_buffer_type_is_host,\n },\n /* .device = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n /* .context = */ NULL,\n };\n\n return &ggml_backend_cpu_buffer_type;\n}\n\nggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) {\n GGML_ASSERT((uintptr_t)ptr % TENSOR_ALIGNMENT == 0 && \"buffer pointer must be aligned\");\n return ggml_backend_buffer_init(ggml_backend_cpu_buffer_from_ptr_type(), ggml_backend_cpu_buffer_from_ptr_i, ptr, size);\n}\n" }, { "path": "src/ggml-blas/CMakeLists.txt", "content": "if (GGML_STATIC)\n set(BLA_STATIC ON)\nendif()\n#if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)\n# set(BLA_SIZEOF_INTEGER 8)\n#endif()\n\nset(BLA_VENDOR ${GGML_BLAS_VENDOR})\nfind_package(BLAS)\n\nif (BLAS_FOUND)\n message(STATUS \"BLAS found, Libraries: ${BLAS_LIBRARIES}\")\n\n ggml_add_backend_library(ggml-blas\n ggml-blas.cpp\n )\n\n if (${GGML_BLAS_VENDOR} MATCHES \"Apple\")\n add_compile_definitions(ACCELERATE_NEW_LAPACK)\n add_compile_definitions(ACCELERATE_LAPACK_ILP64)\n add_compile_definitions(GGML_BLAS_USE_ACCELERATE)\n elseif (\"${BLAS_INCLUDE_DIRS}\" STREQUAL \"\")\n # BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.\n # see https://gitlab.kitware.com/cmake/cmake/-/issues/20268\n find_package(PkgConfig REQUIRED)\n if (${GGML_BLAS_VENDOR} MATCHES \"Generic\")\n pkg_check_modules(DepBLAS blas)\n elseif (${GGML_BLAS_VENDOR} MATCHES \"OpenBLAS\")\n # As of openblas v0.3.22, the 64-bit is named openblas64.pc\n pkg_check_modules(DepBLAS openblas64)\n if (NOT DepBLAS_FOUND)\n pkg_check_modules(DepBLAS openblas)\n endif()\n elseif (${GGML_BLAS_VENDOR} MATCHES \"FLAME\")\n pkg_check_modules(DepBLAS blis)\n elseif (${GGML_BLAS_VENDOR} MATCHES \"ATLAS\")\n pkg_check_modules(DepBLAS blas-atlas)\n elseif (${GGML_BLAS_VENDOR} MATCHES \"FlexiBLAS\")\n pkg_check_modules(DepBLAS flexiblas_api)\n elseif (${GGML_BLAS_VENDOR} MATCHES \"Intel\")\n # all Intel* libraries share the same include path\n pkg_check_modules(DepBLAS mkl-sdl)\n elseif (${GGML_BLAS_VENDOR} MATCHES \"NVHPC\")\n # this doesn't provide pkg-config\n # suggest to assign BLAS_INCLUDE_DIRS on your own\n if (\"${NVHPC_VERSION}\" STREQUAL \"\")\n message(WARNING \"Better to set NVHPC_VERSION\")\n else()\n set(DepBLAS_FOUND ON)\n set(DepBLAS_INCLUDE_DIRS \"/opt/nvidia/hpc_sdk/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}/${NVHPC_VERSION}/math_libs/include\")\n endif()\n endif()\n if (DepBLAS_FOUND)\n set(BLAS_INCLUDE_DIRS ${DepBLAS_INCLUDE_DIRS})\n else()\n message(WARNING \"BLAS_INCLUDE_DIRS neither been provided nor been automatically\"\n \" detected by pkgconfig, trying to find cblas.h from possible paths...\")\n find_path(BLAS_INCLUDE_DIRS\n NAMES cblas.h\n HINTS\n /usr/include\n /usr/local/include\n /usr/include/openblas\n /opt/homebrew/opt/openblas/include\n /usr/local/opt/openblas/include\n /usr/include/x86_64-linux-gnu/openblas/include\n )\n endif()\n endif()\n\n message(STATUS \"BLAS found, Includes: ${BLAS_INCLUDE_DIRS}\")\n\n target_compile_options(ggml-blas PRIVATE ${BLAS_LINKER_FLAGS})\n\n if (\"${GGML_BLAS_VENDOR}\" STREQUAL \"\")\n message(WARNING \"GGML_BLAS_VENDOR is not set; some methods may not link properly.\")\n endif()\n\n if (\"${GGML_BLAS_VENDOR}\" MATCHES \"Intel\" OR (\"${BLAS_INCLUDE_DIRS}\" MATCHES \"mkl\" AND \"${GGML_BLAS_VENDOR}\" MATCHES \"Generic\"))\n add_compile_definitions(GGML_BLAS_USE_MKL)\n endif()\n\n if (\"${GGML_BLAS_VENDOR}\" MATCHES \"OpenBLAS\")\n add_compile_definitions(GGML_BLAS_USE_OPENBLAS)\n endif()\n\n if (\"${GGML_BLAS_VENDOR}\" MATCHES \"FLAME\" OR \"${GGML_BLAS_VENDOR}\" MATCHES \"AOCL\" OR \"${GGML_BLAS_VENDOR}\" MATCHES \"AOCL_mt\")\n add_compile_definitions(GGML_BLAS_USE_BLIS)\n endif()\n\n if (\"${GGML_BLAS_VENDOR}\" MATCHES \"NVPL\")\n add_compile_definitions(GGML_BLAS_USE_NVPL)\n endif()\n\n target_link_libraries (ggml-blas PRIVATE ${BLAS_LIBRARIES})\n target_include_directories(ggml-blas SYSTEM PRIVATE ${BLAS_INCLUDE_DIRS})\nelse()\n message(FATAL_ERROR \"BLAS not found, please refer to \"\n \"https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors\"\n \" to set correct GGML_BLAS_VENDOR\")\nendif()\n" }, { "path": "src/ggml-blas/ggml-blas.cpp", "content": "#include \"ggml-impl.h\"\n#include \"ggml-blas.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \n#include \n#include \n\n#if defined(GGML_BLAS_USE_ACCELERATE)\n# include \n#elif defined(GGML_BLAS_USE_MKL)\n# include \n#elif defined(GGML_BLAS_USE_BLIS)\n# include \n#elif defined(GGML_BLAS_USE_NVPL)\n# include \n#else\n# include \n#endif\n\nstruct ggml_backend_blas_context {\n int n_threads = GGML_DEFAULT_N_THREADS;\n std::unique_ptr work_data;\n size_t work_size = 0;\n#ifndef GGML_USE_OPENMP\n std::vector> tasks;\n#endif\n};\n\nstatic void ggml_backend_blas_mul_mat(ggml_backend_blas_context * ctx, struct ggml_tensor * dst) {\n const struct ggml_tensor * src0 = dst->src[0];\n const struct ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const enum ggml_type type = src0->type;\n\n GGML_ASSERT(ne0 == ne01);\n GGML_ASSERT(ne1 == ne11);\n GGML_ASSERT(ne2 == ne12);\n GGML_ASSERT(ne3 == ne13);\n\n // we don't support permuted src0 or src1\n GGML_ASSERT(nb00 == ggml_type_size(type));\n GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n // broadcast factors\n const int64_t r2 = ne12/ne02;\n const int64_t r3 = ne13/ne03;\n\n const int64_t ne_plane = ne01*ne00;\n const size_t desired_wsize = type == GGML_TYPE_F32 ? 0 : ne03*ne02*ne_plane*sizeof(float);\n\n if (ctx->work_size < desired_wsize) {\n ctx->work_data.reset(new char[desired_wsize]);\n ctx->work_size = desired_wsize;\n }\n void * wdata = ctx->work_data.get();\n\n // convert src0 to float\n if (type != GGML_TYPE_F32) {\n const auto * type_traits = ggml_get_type_traits(type);\n ggml_to_float_t const to_float = type_traits->to_float;\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n const void * x = (char *) src0->data + i02*nb02 + i03*nb03;\n float * const wplane = (float *) wdata + i02*ne_plane + i03*ne02*ne_plane;\n\n const int min_cols_per_thread = 4096;\n const int min_rows_per_thread = std::max((int)(min_cols_per_thread/ne00), 1);\n const int n_threads = std::max(std::min(ctx->n_threads, (int)(ne01/min_rows_per_thread)), 1);\n\n#ifdef GGML_USE_OPENMP\n #pragma omp parallel for num_threads(n_threads)\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00);\n }\n#else\n for (int i = 1; i < n_threads; i++) {\n const int64_t start = i*ne01/n_threads;\n const int64_t end = (i + 1)*ne01/n_threads;\n if (start < end) {\n ctx->tasks.push_back(std::async(std::launch::async, [=]() {\n for (int64_t i01 = start; i01 < end; i01++) {\n to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00);\n }\n }));\n }\n }\n {\n // reuse the current thread for the first task\n const int64_t start = 0;\n const int64_t end = ne01/n_threads;\n for (int64_t i01 = start; i01 < end; i01++) {\n to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00);\n }\n }\n#endif\n }\n }\n\n#ifndef GGML_USE_OPENMP\n // wait for all tasks to finish\n for (auto & task : ctx->tasks) {\n task.get();\n }\n ctx->tasks.clear();\n#endif\n }\n\n#if defined(GGML_BLAS_USE_OPENBLAS)\n openblas_set_num_threads(ctx->n_threads);\n#elif defined(GGML_BLAS_USE_BLIS)\n bli_thread_set_num_threads(ctx->n_threads);\n#elif defined(GGML_BLAS_USE_NVPL)\n nvpl_blas_set_num_threads(ctx->n_threads);\n#endif\n\n for (int64_t i13 = 0; i13 < ne13; i13++) {\n for (int64_t i12 = 0; i12 < ne12; i12++) {\n const int64_t i03 = i13/r3;\n const int64_t i02 = i12/r2;\n\n const float * x = (float *) ((char *) src0->data + i02*nb02 + i03*nb03);\n const float * y = (float *) ((char *) src1->data + i12*nb12 + i13*nb13);\n float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);\n\n if (type != GGML_TYPE_F32) {\n x = (float *) wdata + i02*ne_plane + i03*ne02*ne_plane;\n }\n\n cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,\n ne1, ne01, ne10,\n 1.0f, y, ne10,\n x, ne00,\n 0.0f, d, ne01);\n }\n }\n}\n\nstatic void ggml_backend_blas_out_prod(ggml_backend_blas_context * ctx, struct ggml_tensor * dst) {\n const struct ggml_tensor * src0 = dst->src[0];\n const struct ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n GGML_ASSERT(ne0 == ne00);\n GGML_ASSERT(ne1 == ne10);\n GGML_ASSERT(ne2 == ne02);\n GGML_ASSERT(ne02 == ne12);\n GGML_ASSERT(ne3 == ne13);\n GGML_ASSERT(ne03 == ne13);\n\n // we don't support permuted src0 or src1\n GGML_ASSERT(nb00 == sizeof(float));\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n // GGML_ASSERT(nb0 <= nb1);\n // GGML_ASSERT(nb1 <= nb2);\n // GGML_ASSERT(nb2 <= nb3);\n\n // Arguments to ggml_compute_forward_out_prod (expressed as major,minor)\n // src0: (k,n)\n // src1: (k,m)\n // dst: (m,n)\n //\n // Arguments to sgemm (see https://github.com/Reference-LAPACK/lapack/blob/master/BLAS/SRC/sgemm.f)\n // Also expressed as (major,minor)\n // a: (m,k): so src1 transposed\n // b: (k,n): so src0\n // c: (m,n)\n //\n // However, if ggml_is_transposed(src1) is true, then\n // src1->data already contains a transposed version, so sgemm mustn't\n // transpose it further.\n\n int n = src0->ne[0];\n int k = src0->ne[1];\n int m = src1->ne[0];\n\n CBLAS_TRANSPOSE transposeA;\n int lda;\n\n if (!ggml_is_transposed(src1)) {\n transposeA = CblasTrans;\n lda = m;\n } else {\n transposeA = CblasNoTrans;\n lda = k;\n }\n\n float * a = (float *) ((char *) src1->data);\n float * b = (float *) ((char *) src0->data);\n float * c = (float *) ((char *) dst->data);\n\n cblas_sgemm(CblasRowMajor, transposeA, CblasNoTrans, m, n, k, 1.0, a, lda, b, n, 0.0, c, n);\n\n GGML_UNUSED(ctx);\n}\n\n// backend interface\n\nstatic const char * ggml_backend_blas_get_name(ggml_backend_t backend) {\n return \"BLAS\";\n\n GGML_UNUSED(backend);\n}\n\nstatic void ggml_backend_blas_free(ggml_backend_t backend) {\n ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend->context;\n delete ctx;\n delete backend;\n}\n\nstatic enum ggml_status ggml_backend_blas_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend->context;\n\n for (int i = 0; i < cgraph->n_nodes; i++) {\n struct ggml_tensor * node = cgraph->nodes[i];\n\n if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n continue;\n }\n\n switch (node->op) {\n case GGML_OP_MUL_MAT:\n ggml_backend_blas_mul_mat(ctx, node);\n break;\n\n case GGML_OP_OUT_PROD:\n ggml_backend_blas_out_prod(ctx, node);\n break;\n\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n break;\n\n default:\n GGML_ABORT(\"%s: unsupported op %s\\n\", __func__, ggml_op_desc(node));\n }\n }\n\n return GGML_STATUS_SUCCESS;\n\n GGML_UNUSED(backend);\n}\n\nstatic struct ggml_backend_i blas_backend_i = {\n /* .get_name = */ ggml_backend_blas_get_name,\n /* .free = */ ggml_backend_blas_free,\n /* .set_tensor_async = */ NULL,\n /* .get_tensor_async = */ NULL,\n /* .cpy_tensor_async = */ NULL,\n /* .synchronize = */ NULL,\n /* .graph_plan_create = */ NULL,\n /* .graph_plan_free = */ NULL,\n /* .graph_plan_update = */ NULL,\n /* .graph_plan_compute = */ NULL,\n /* .graph_compute = */ ggml_backend_blas_graph_compute,\n /* .event_record = */ NULL,\n /* .event_wait = */ NULL,\n /* .graph_optimize = */ NULL,\n};\n\nstatic ggml_guid_t ggml_backend_blas_guid(void) {\n static ggml_guid guid = { 0x12, 0xa8, 0xae, 0xf4, 0xc0, 0x1e, 0x61, 0x97, 0x8f, 0xeb, 0x33, 0x04, 0xa1, 0x33, 0x51, 0x2d };\n return &guid;\n}\n\nggml_backend_t ggml_backend_blas_init(void) {\n ggml_backend_blas_context * ctx = new ggml_backend_blas_context;\n\n ggml_backend_t backend = new ggml_backend {\n /* .guid = */ ggml_backend_blas_guid(),\n /* .iface = */ blas_backend_i,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_blas_reg(), 0),\n /* .context = */ ctx,\n };\n\n#if defined(GGML_BLAS_USE_OPENBLAS) && defined(GGML_USE_OPENMP)\n if (openblas_get_parallel() != OPENBLAS_OPENMP) {\n GGML_LOG_DEBUG(\"%s: warning: ggml is using OpenMP, but OpenBLAS was compiled without OpenMP support\\n\", __func__);\n }\n#endif\n\n#if defined(BLIS_ENABLE_CBLAS) && defined(GGML_USE_OPENMP) && !defined(BLIS_ENABLE_OPENMP)\n GGML_LOG_DEBUG(\"%s: warning: ggml is using OpenMP, but BLIS was compiled without OpenMP support\\n\", __func__);\n#endif\n\n return backend;\n}\n\nbool ggml_backend_is_blas(ggml_backend_t backend) {\n return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_blas_guid());\n}\n\nvoid ggml_backend_blas_set_n_threads(ggml_backend_t backend_blas, int n_threads) {\n GGML_ASSERT(ggml_backend_is_blas(backend_blas));\n\n ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend_blas->context;\n ctx->n_threads = n_threads;\n}\n\n// device interface\n\nstatic const char * ggml_backend_blas_device_get_name(ggml_backend_dev_t dev) {\n return \"BLAS\";\n\n GGML_UNUSED(dev);\n}\n\nstatic const char * ggml_backend_blas_device_get_description(ggml_backend_dev_t dev) {\n #if defined(GGML_BLAS_USE_ACCELERATE)\n return \"Accelerate\";\n #elif defined(GGML_BLAS_USE_MKL)\n return \"MKL\";\n #elif defined(GGML_BLAS_USE_BLIS)\n return \"BLIS\";\n #elif defined(GGML_BLAS_USE_NVPL)\n return \"NVPL\";\n #elif defined(GGML_BLAS_USE_OPENBLAS)\n return \"OpenBLAS\";\n #else\n return \"BLAS\";\n #endif\n\n GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_blas_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n // no memory to report\n *free = 0;\n *total = 0;\n\n GGML_UNUSED(dev);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_blas_device_get_type(ggml_backend_dev_t dev) {\n return GGML_BACKEND_DEVICE_TYPE_ACCEL;\n\n GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_blas_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {\n props->name = ggml_backend_blas_device_get_name(dev);\n props->description = ggml_backend_blas_device_get_description(dev);\n props->type = ggml_backend_blas_device_get_type(dev);\n ggml_backend_blas_device_get_memory(dev, &props->memory_free, &props->memory_total);\n props->caps = {\n /* .async = */ false,\n /* .host_buffer = */ false,\n /* .buffer_from_host_ptr = */ true,\n /* .events = */ false,\n };\n}\n\nstatic ggml_backend_t ggml_backend_blas_device_init_backend(ggml_backend_dev_t dev, const char * params) {\n return ggml_backend_blas_init();\n\n GGML_UNUSED(dev);\n GGML_UNUSED(params);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_blas_device_get_buffer_type(ggml_backend_dev_t dev) {\n return ggml_backend_cpu_buffer_type();\n\n GGML_UNUSED(dev);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_blas_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {\n return ggml_backend_cpu_buffer_from_ptr(ptr, size);\n\n GGML_UNUSED(dev);\n GGML_UNUSED(max_tensor_size);\n}\n\nstatic bool ggml_backend_blas_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n\n switch (op->op) {\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n return true;\n\n case GGML_OP_MUL_MAT:\n {\n // BLAS usually is only faster for large matrices\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n\n const int64_t ne10 = src1->ne[0];\n\n const int64_t ne0 = op->ne[0];\n const int64_t ne1 = op->ne[1];\n\n // TODO: find the optimal value\n const int64_t min_batch = 32;\n\n return ggml_is_contiguous(src0) &&\n ggml_is_contiguous(src1) &&\n src1->type == GGML_TYPE_F32 &&\n (ne0 >= min_batch && ne1 >= min_batch && ne10 >= min_batch) &&\n (src0->type == GGML_TYPE_F32 || ggml_get_type_traits(src0->type)->to_float != NULL);\n }\n\n case GGML_OP_OUT_PROD:\n return op->src[0]->type == GGML_TYPE_F32 &&\n op->src[1]->type == GGML_TYPE_F32 &&\n ggml_is_matrix(src0) &&\n ggml_is_matrix(src1) &&\n ggml_is_contiguous(src0) &&\n (ggml_is_contiguous(src1) || ggml_is_transposed(src1)) &&\n (src0->type == GGML_TYPE_F32 || ggml_get_type_traits(src0->type)->to_float != NULL);\n\n default:\n return false;\n\n }\n\n GGML_UNUSED(dev);\n}\n\nstatic bool ggml_backend_blas_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n return ggml_backend_buft_is_host(buft);\n\n GGML_UNUSED(dev);\n}\n\nstatic const struct ggml_backend_device_i ggml_backend_blas_device_i = {\n /* .get_name = */ ggml_backend_blas_device_get_name,\n /* .get_description = */ ggml_backend_blas_device_get_description,\n /* .get_memory = */ ggml_backend_blas_device_get_memory,\n /* .get_type = */ ggml_backend_blas_device_get_type,\n /* .get_props = */ ggml_backend_blas_device_get_props,\n /* .init_backend = */ ggml_backend_blas_device_init_backend,\n /* .get_buffer_type = */ ggml_backend_blas_device_get_buffer_type,\n /* .get_host_buffer_type = */ NULL,\n /* .buffer_from_host_ptr = */ ggml_backend_blas_device_buffer_from_host_ptr,\n /* .supports_op = */ ggml_backend_blas_device_supports_op,\n /* .supports_buft = */ ggml_backend_blas_device_supports_buft,\n /* .offload_op = */ NULL,\n /* .event_new = */ NULL,\n /* .event_free = */ NULL,\n /* .event_synchronize = */ NULL,\n};\n\n// backend reg interface\n\nstatic const char * ggml_backend_blas_reg_get_name(ggml_backend_reg_t reg) {\n return \"BLAS\";\n\n GGML_UNUSED(reg);\n}\n\nstatic size_t ggml_backend_blas_reg_get_device_count(ggml_backend_reg_t reg) {\n return 1;\n\n GGML_UNUSED(reg);\n}\n\nstatic ggml_backend_dev_t ggml_backend_blas_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n GGML_ASSERT(index == 0);\n\n static ggml_backend_device ggml_backend_blas_device = {\n /* .iface = */ ggml_backend_blas_device_i,\n /* .reg = */ reg,\n /* .context = */ nullptr,\n };\n\n return &ggml_backend_blas_device;\n\n GGML_UNUSED(reg);\n GGML_UNUSED(index);\n}\n\nstatic void * ggml_backend_blas_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n if (std::strcmp(name, \"ggml_backend_set_n_threads\") == 0) {\n return (void *)ggml_backend_blas_set_n_threads;\n }\n return NULL;\n\n GGML_UNUSED(reg);\n GGML_UNUSED(name);\n}\n\nstatic const struct ggml_backend_reg_i ggml_backend_blas_reg_i = {\n /* .get_name = */ ggml_backend_blas_reg_get_name,\n /* .get_device_count = */ ggml_backend_blas_reg_get_device_count,\n /* .get_device = */ ggml_backend_blas_reg_get_device,\n /* .get_proc_address = */ ggml_backend_blas_get_proc_address,\n};\n\nggml_backend_reg_t ggml_backend_blas_reg(void) {\n static struct ggml_backend_reg ggml_backend_blas_reg = {\n /* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_blas_reg_i,\n /* .context = */ NULL,\n };\n\n return &ggml_backend_blas_reg;\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_blas_reg)\n" }, { "path": "src/ggml-cann/CMakeLists.txt", "content": "if (\"cann${CANN_INSTALL_DIR}\" STREQUAL \"cann\" AND DEFINED ENV{ASCEND_TOOLKIT_HOME})\n set(CANN_INSTALL_DIR $ENV{ASCEND_TOOLKIT_HOME})\n message(STATUS \"CANN: updated CANN_INSTALL_DIR from ASCEND_TOOLKIT_HOME=$ENV{ASCEND_TOOLKIT_HOME}\")\nendif()\n\n# Auto-detech Soc type and Soc version, if detect failed, will abort build\nset(SOC_VERSION \"\")\nfunction(detect_ascend_soc_type SOC_VERSION)\n execute_process(\n COMMAND bash -c \"npu-smi info|awk -F' ' 'NF > 0 && NR==7 {print $3}'\"\n OUTPUT_VARIABLE npu_info\n RESULT_VARIABLE npu_result\n OUTPUT_STRIP_TRAILING_WHITESPACE\n )\n if(\"${npu_info}\" STREQUAL \"\" OR ${npu_result})\n message(FATAL_ERROR \"Auto-detech ascend soc type failed, please specify manually or check ascend device working normally.\")\n endif()\n set(${SOC_VERSION} \"Ascend${npu_info}\" PARENT_SCOPE)\nendfunction()\n\nif(NOT SOC_TYPE)\n detect_ascend_soc_type(SOC_VERSION)\n set(SOC_TYPE \"${SOC_VERSION}\")\n message(STATUS \"CANN: SOC_VERSION auto-detected is:${SOC_VERSION}\")\nendif()\n\nstring(TOLOWER ${SOC_TYPE} SOC_VERSION) # SOC_VERSION need lower\n\n# Construct Soc specify compile option: ASCEND_#Soc_Major_SN. Such as ASCEND_910B, ASCEND_310P.\nstring(REGEX MATCH \"[0-9]+[a-zA-Z]\" SOC_TYPE_MAJOR_SN \"${SOC_VERSION}\")\nset(SOC_TYPE_COMPILE_OPTION \"ASCEND_${SOC_TYPE_MAJOR_SN}\")\nstring(TOUPPER ${SOC_TYPE_COMPILE_OPTION} SOC_TYPE_COMPILE_OPTION)\nmessage(STATUS \"CANN: SOC_VERSION = ${SOC_VERSION}\")\noption(USE_ACL_GRAPH \"Enable CANN graph execution (ACL graph mode)\" OFF)\n\nif(USE_ACL_GRAPH AND (SOC_TYPE_MAJOR_SN STREQUAL \"310P\" OR SOC_TYPE_COMPILE_OPTION STREQUAL \"ASCEND_310P\"))\n message(FATAL_ERROR\n \"CANN Graph (ACL graph mode) is not supported on 310P devices. \"\n \"Please build with -DUSE_ACL_GRAPH=OFF or use a supported SOC.\")\nendif()\n\nif (CANN_INSTALL_DIR)\n # Only Support Linux.\n if (NOT UNIX)\n message(FATAL_ERROR \"CANN: CANN toolkit supports unix but not ${CMAKE_SYSTEM_NAME}\")\n endif()\n\n # Supported platforms: x86-64, arm64\n if (CMAKE_SYSTEM_PROCESSOR STREQUAL \"aarch64\")\n elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL \"x86_64\" OR CMAKE_SYSTEM_PROCESSOR STREQUAL \"amd64\")\n else()\n message(FATAL_ERROR \"CANN: CANN toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}\")\n endif()\n\n # Set header and libs\n set(CANN_INCLUDE_DIRS\n ${CANN_INSTALL_DIR}/include\n ${CANN_INSTALL_DIR}/include/aclnn\n ${CANN_INSTALL_DIR}/acllib/include\n )\n\n list(APPEND CANN_LIBRARIES\n ascendcl\n nnopbase\n opapi\n acl_op_compiler\n )\n\n file(GLOB GGML_SOURCES_CANN \"*.cpp\")\n\n ggml_add_backend_library(ggml-cann ${GGML_SOURCES_CANN})\n target_link_libraries(ggml-cann PRIVATE ${CANN_LIBRARIES})\n target_include_directories(ggml-cann PRIVATE ${CANN_INCLUDE_DIRS})\n target_link_directories(ggml-cann PRIVATE ${CANN_INSTALL_DIR}/lib64)\n\n target_compile_definitions(ggml-cann PRIVATE \"-D${SOC_TYPE_COMPILE_OPTION}\")\n\n if (USE_ACL_GRAPH)\n target_compile_definitions(ggml-cann PRIVATE USE_ACL_GRAPH)\n message(STATUS \"CANN: USE_ACL_GRAPH is enabled.\")\n else()\n message(STATUS \"CANN: USE_ACL_GRAPH is disabled.\")\n endif()\n\n message(STATUS \"CANN: CANN_INCLUDE_DIRS = ${CANN_INCLUDE_DIRS}\")\n message(STATUS \"CANN: CANN_LIBRARIES = ${CANN_LIBRARIES}\")\nelse()\n message(FATAL_ERROR \"CANN: Can't find CANN_INSTALL_DIR, did you forget to source set_var.sh?\")\nendif()\n" }, { "path": "src/ggml-cann/acl_tensor.cpp", "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#include \"acl_tensor.h\"\n\n#include \n#include \n\naclDataType ggml_cann_type_mapping(ggml_type type) {\n switch (type) {\n case GGML_TYPE_F32:\n return ACL_FLOAT;\n case GGML_TYPE_F16:\n return ACL_FLOAT16;\n case GGML_TYPE_BF16:\n return ACL_BF16;\n case GGML_TYPE_I8:\n return ACL_INT8;\n case GGML_TYPE_I16:\n return ACL_INT16;\n case GGML_TYPE_I32:\n return ACL_INT32;\n case GGML_TYPE_Q4_0:\n return ACL_INT4;\n case GGML_TYPE_Q8_0:\n return ACL_INT8;\n case GGML_TYPE_I64:\n return ACL_INT64;\n default:\n return ACL_DT_UNDEFINED;\n }\n}\n\nacl_tensor_ptr ggml_cann_create_tensor(const ggml_tensor * tensor,\n int64_t * ne,\n size_t * nb,\n int64_t dims,\n aclFormat format,\n size_t offset) {\n // If tensor is bcasted, Up to GGML_MAX_DIMS additional dimensions will be\n // added.\n int64_t acl_ne[GGML_MAX_DIMS * 2], acl_stride[GGML_MAX_DIMS * 2];\n\n if (ne == nullptr) {\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n acl_ne[i] = tensor->ne[i];\n // The step size of acl is in elements.\n acl_stride[i] = tensor->nb[i] / ggml_element_size(tensor);\n }\n } else {\n // With bcast\n for (int i = 0; i < dims; i++) {\n acl_ne[i] = ne[i];\n acl_stride[i] = nb[i] / ggml_element_size(tensor);\n }\n }\n\n int64_t final_dims = (dims == 0 ? GGML_MAX_DIMS : dims);\n int64_t acl_storage_len = 1;\n for (int i = 0; i < final_dims; i++) {\n acl_storage_len += (acl_ne[i] - 1) * acl_stride[i];\n }\n size_t elem_offset = offset / ggml_element_size(tensor);\n acl_storage_len += elem_offset;\n\n // Reverse ne and stride.\n std::reverse(acl_ne, acl_ne + final_dims);\n std::reverse(acl_stride, acl_stride + final_dims);\n\n aclTensor * raw = aclCreateTensor(acl_ne, final_dims, ggml_cann_type_mapping(tensor->type), acl_stride, elem_offset,\n format, &acl_storage_len, 1, tensor->data);\n\n return acl_tensor_ptr(raw);\n}\n\nacl_int_array_ptr ggml_cann_create_int_array(const int64_t * value, uint64_t size) {\n aclIntArray * raw = aclCreateIntArray(value, size);\n return acl_int_array_ptr(raw);\n}\n\nacl_scalar_ptr ggml_cann_create_scalar(void * value, aclDataType dataType) {\n aclScalar * raw = aclCreateScalar(value, dataType);\n return acl_scalar_ptr(raw);\n}\n\nbool ggml_cann_need_bcast(const ggml_tensor * t0, const ggml_tensor * t1) {\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n if (t1->ne[i] != t0->ne[i] && t1->ne[i] != 1) {\n return true;\n }\n }\n return false;\n}\n\nint64_t ggml_cann_get_bcast_shape(const ggml_tensor * src0,\n const ggml_tensor * src1,\n int64_t * bcast_src0_ne,\n int64_t * bcast_src1_ne,\n size_t * bcast_src0_nb,\n size_t * bcast_src1_nb) {\n GGML_ASSERT(ggml_can_repeat(src1, src0));\n int bcast_dim_cnt = 0;\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n int64_t nr = src0->ne[i] / src1->ne[i];\n bcast_src0_ne[bcast_dim_cnt] = src0->ne[i] / nr;\n bcast_src1_ne[bcast_dim_cnt] = src1->ne[i];\n bcast_src0_nb[bcast_dim_cnt] = src0->nb[i];\n bcast_src1_nb[bcast_dim_cnt] = src1->nb[i];\n bcast_dim_cnt++;\n if (nr != 1) {\n // Need to add an extra dim.\n bcast_src0_ne[bcast_dim_cnt] = nr;\n bcast_src1_ne[bcast_dim_cnt] = 1;\n bcast_src0_nb[bcast_dim_cnt] = bcast_src0_nb[bcast_dim_cnt - 1] * bcast_src0_ne[bcast_dim_cnt - 1];\n bcast_src1_nb[bcast_dim_cnt] = bcast_src1_nb[bcast_dim_cnt - 1] * bcast_src1_ne[bcast_dim_cnt - 1];\n bcast_dim_cnt++;\n }\n }\n return bcast_dim_cnt;\n}\n\nint64_t ggml_cann_get_mulmat_bcast_shape(const int64_t * input_ne,\n const int64_t * weight_ne,\n const int64_t * dst_ne,\n const size_t * input_nb,\n const size_t * weight_nb,\n const size_t * dst_nb,\n int64_t * bcast_input_ne,\n int64_t * bcast_weight_ne,\n int64_t * bcast_dst_ne,\n size_t * bcast_input_nb,\n size_t * bcast_weight_nb,\n size_t * bcast_dst_nb) {\n // input and dst shoule in same shape, except first two dims.\n GGML_ASSERT(input_ne[2] == dst_ne[2]);\n GGML_ASSERT(input_ne[3] == dst_ne[3]);\n\n int bcast_dim_cnt = 0;\n\n // For mul_mat, a dimension needs to be added before the dimension that\n // weight needs to be expanded to satisfy the bcast rule of matrix\n // multiplication.\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n int64_t nr = input_ne[i] / weight_ne[i];\n // Do not use bcast in the first two dimensions because we only support\n // the bcast batch dimension. Just copy them.\n if (i < 2 || nr == 1) {\n bcast_input_ne[bcast_dim_cnt] = input_ne[i];\n bcast_weight_ne[bcast_dim_cnt] = weight_ne[i];\n bcast_dst_ne[bcast_dim_cnt] = dst_ne[i];\n\n bcast_input_nb[bcast_dim_cnt] = input_nb[i];\n bcast_weight_nb[bcast_dim_cnt] = weight_nb[i];\n bcast_dst_nb[bcast_dim_cnt] = dst_nb[i];\n bcast_dim_cnt++;\n } else {\n // Need to add an extra dim.\n bcast_input_ne[bcast_dim_cnt] = nr;\n bcast_dst_ne[bcast_dim_cnt] = nr;\n bcast_weight_ne[bcast_dim_cnt] = 1;\n bcast_input_nb[bcast_dim_cnt] = input_nb[i];\n bcast_dst_nb[bcast_dim_cnt] = dst_nb[i];\n bcast_weight_nb[bcast_dim_cnt] = weight_nb[i];\n bcast_dim_cnt++;\n\n bcast_input_ne[bcast_dim_cnt] = input_ne[i] / nr;\n bcast_dst_ne[bcast_dim_cnt] = dst_ne[i] / nr;\n bcast_weight_ne[bcast_dim_cnt] = weight_ne[i];\n bcast_input_nb[bcast_dim_cnt] = bcast_input_nb[bcast_dim_cnt - 1] * bcast_input_ne[bcast_dim_cnt - 1];\n bcast_dst_nb[bcast_dim_cnt] = bcast_dst_nb[bcast_dim_cnt - 1] * bcast_dst_ne[bcast_dim_cnt - 1];\n bcast_weight_nb[bcast_dim_cnt] = bcast_weight_nb[bcast_dim_cnt - 1] * bcast_weight_ne[bcast_dim_cnt - 1];\n bcast_dim_cnt++;\n }\n }\n return bcast_dim_cnt;\n}\n" }, { "path": "src/ggml-cann/acl_tensor.h", "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#ifndef CANN_ACL_TENSOR_H\n#define CANN_ACL_TENSOR_H\n\n#include \"common.h\"\n\n#include \n\n#include \n#include \n\n/**\n * @brief\tMaps a ggml_type to its corresponding aclDataType.\n *\n * @details\tThis function takes a ggml_type as input and returns the corresponding\n *\t\t\taclDataType. It supports mapping for various ggml_types. If the input type\n *\t\t\tdoes not match any of the predefined ggml_types, the function returns\n * ACL_DT_UNDEFINED.\n *\n * @param\ttype The ggml_type to be mapped.\n * @return\tThe corresponding aclDataType. If the input type is not recognized,\n *\t\t\tACL_DT_UNDEFINED is returned.\n */\naclDataType ggml_cann_type_mapping(ggml_type type);\n\n// Deleter for acl objects.\ntemplate struct acl_deleter {\n void operator()(T * ptr) const noexcept {\n if (ptr) {\n ACL_CHECK(DestroyFunc(ptr));\n }\n }\n};\n\nusing acl_tensor_ptr = std::unique_ptr>;\nusing acl_int_array_ptr = std::unique_ptr>;\nusing acl_scalar_ptr = std::unique_ptr>;\nusing acl_tensor_list_ptr = std::unique_ptr>;\n\n/**\n * @brief Creates an ACL tensor from a ggml_tensor with optional shape.\n *\n * @details This function creates an ACL tensor based on the properties of the\n * provided ggml_tensor. It supports customer shape by adjusting dimensions\n * and strides accordingly. If customer shape is applied, additional\n * dimensions and strides are calculated based on the provided parameters.\n *\n * @param tensor Pointer to the ggml_tensor to be converted to ACL tensor.\n * @param ne Pointer to an array containing dimensions. Defaults to nullptr\n * if no customer shape is applied.\n * @param nb Pointer to an array containing strides. Defaults to nullptr\n * if no customer shape is applied.\n * @param dims Number of dimensions in the tensor. Defaults to 0 if no customer\n * shape is applied.\n * @param format ACL tensor format. Defaults to ACL_FORMAT_ND.\n * @param offset Offset in bytes for the ACL tensor data. Defaults to 0.\n * @return Pointer to the created ACL tensor.\n */\nacl_tensor_ptr ggml_cann_create_tensor(const ggml_tensor * tensor,\n int64_t * ne = nullptr,\n size_t * nb = nullptr,\n int64_t dims = 0,\n aclFormat format = ACL_FORMAT_ND,\n size_t offset = 0);\n\n/**\n * @brief Template for creating an ACL tensor from provided parameters. typename TYPE\n * should be size_t or float.\n *\n * @details This function creates an ACL tensor using the provided data pointer,\n * data type, dimensions, strides, format, offset, and additional parameters.\n * It calculates necessary dimensions and strides based on the provided ne and nb\n * arrays, adjusting them for the ACL tensor creation. The ACL storage length\n * is also calculated based on the provided dimensions and strides.\n *\n * @param data_ptr Pointer to the data buffer for the ACL tensor.\n * @param dtype ACL data type of the tensor.\n * @param type_size Size of each element in the tensor data buffer.\n * @param ne Pointer to an array containing tensor dimensions.\n * @param nb Pointer to an array containing tensor strides.\n * @param dims Number of dimensions of the tensor.\n * @param format ACL tensor format. Defaults to ACL_FORMAT_ND.\n * @param offset Offset in bytes for the ACL tensor data. Defaults to 0.\n * @return Pointer to the created ACL tensor.\n */\ntemplate \nacl_tensor_ptr ggml_cann_create_tensor(void * data_ptr,\n aclDataType dtype,\n TYPE type_size,\n int64_t * ne,\n TYPE * nb,\n int64_t dims,\n aclFormat format = ACL_FORMAT_ND,\n size_t offset = 0) {\n int64_t tmp_ne[GGML_MAX_DIMS * 2];\n int64_t tmp_stride[GGML_MAX_DIMS * 2];\n\n memcpy(tmp_ne, ne, dims * sizeof(int64_t));\n for (int i = 0; i < dims; i++) {\n tmp_stride[i] = nb[i] / type_size;\n }\n\n int64_t acl_storage_len = 1;\n for (int i = 0; i < dims; i++) {\n acl_storage_len += (tmp_ne[i] - 1) * tmp_stride[i];\n }\n\n std::reverse(tmp_ne, tmp_ne + dims);\n std::reverse(tmp_stride, tmp_stride + dims);\n\n aclTensor * raw =\n aclCreateTensor(tmp_ne, dims, dtype, tmp_stride, offset / type_size, format, &acl_storage_len, 1, data_ptr);\n\n return acl_tensor_ptr(raw);\n}\n\n/**\n * @brief Create an ACL int array resource wrapped in a smart pointer.\n *\n * This function constructs an aclIntArray from the provided int64_t values\n * and returns it as an acl_int_array_ptr (a std::unique_ptr with a custom\n * deleter). The returned pointer owns the ACL resource and will automatically\n * destroy it via aclDestroyIntArray().\n *\n * @param value Pointer to the int64_t elements.\n * @param size Number of elements in value.\n *\n * @return A smart pointer managing the created ACL int array.\n */\nacl_int_array_ptr ggml_cann_create_int_array(const int64_t * value, uint64_t size);\n\n/**\n * @brief Create an ACL scalar resource wrapped in a smart pointer.\n *\n * This function constructs an aclScalar from the raw value pointer and ACL\n * data type, then returns it as an acl_scalar_ptr (a std::unique_ptr with\n * a custom deleter). The returned pointer owns the ACL scalar and will\n * automatically destroy it via aclDestroyScalar().\n *\n * @param value Pointer to the raw scalar memory.\n * @param dataType ACL data type of the scalar.\n *\n * @return A smart pointer managing the created ACL scalar.\n */\nacl_scalar_ptr ggml_cann_create_scalar(void * value, aclDataType dataType);\n\n/**\n * @brief Create an ACL tensor list from multiple tensor smart pointers.\n *\n * This function accepts a variadic list of acl_tensor_ptr (a unique_ptr with\n * custom deleter) and produces an aclTensorList using aclCreateTensorList().\n *\n * The lifecycle management of the tensor objects changes as follows:\n * - aclCreateTensorList() takes ownership of the tensors\n * - Each input smart pointer releases ownership using release()\n * - As a result, the tensors will NOT be destroyed by unique_ptr\n * - Instead, they will be destroyed when aclDestroyTensorList() is called\n *\n * This ensures correct ownership transfer and prevents double-free situations.\n *\n * @param acl_tensor_ptr Variadic template parameter; each argument must be\n * a unique_ptr-like type supporting get() and release().\n *\n * @param tensors Variadic list of acl_tensor_ptr objects. Ownership of\n * each tensor is transferred away from these smart pointers.\n *\n * @return A smart pointer (acl_tensor_list_ptr) owning the created ACL tensor list.\n *\n * @note This implementation is C++11 compatible. The ownership-release process is\n * executed using a pack expansion inside an initializer list.\n */\ntemplate acl_tensor_list_ptr ggml_cann_create_tensor_list(acl_tensor_ptr &&... tensors) {\n aclTensor * raw_tensors[] = { tensors.get()... };\n aclTensorList * raw = aclCreateTensorList(raw_tensors, sizeof...(tensors));\n // aclTensor will release by aclTensorList, so release ownership without\n // destroying the tensor\n int dummy[] = { (tensors.release(), 0)... };\n GGML_UNUSED(dummy);\n return acl_tensor_list_ptr(raw);\n}\n\n/**\n * @brief Checks if tensors require broadcasting based on their shapes.\n *\n * @details This function determines if two ggml_tensors need to be broadcasted for\n * element-wise operations. Broadcasting is necessary if the shapes of the\n * tensors are not identical and no dimension in either tensor equals 1.\n *\n * @param t0 Pointer to the first ggml_tensor.\n * @param t1 Pointer to the second ggml_tensor.\n * @return True if broadcasting is needed, False otherwise.\n *\n * @remarks This function iterates over the dimensions of t0 and t1. It checks if each\n * dimension in t1 differs from t0's corresponding dimension and is not equal\n * to 1. If such a dimension is found, broadcasting is required to align t1\n * with t0 for element-wise operations.\n */\nbool ggml_cann_need_bcast(const ggml_tensor * t0, const ggml_tensor * t1);\n\n/**\n * @brief Computes broadcast shapes and strides for two ggml_tensors.\n *\n * @details This function calculates the broadcast shapes and strides for two ggml_tensors,\n * following the broadcasting rules similar to numpy. It adjusts dimensions and\n * strides to ensure compatibility for element-wise operations where one tensor\n * can be broadcasted to match the shape of another tensor.\n *\n * @param src0 Pointer to the first ggml_tensor.\n * @param src1 Pointer to the second ggml_tensor.\n * @param bcast_ne_src0 Output array to store broadcasted dimensions for src0.\n * @param bcast_ne_src1 Output array to store broadcasted dimensions for src1.\n * @param bcast_nb_src0 Output array to store broadcasted strides for src0.\n * @param bcast_nb_src1 Output array to store broadcasted strides for src1.\n * @return Number of dimensions in the broadcasted shape.\n *\n * @pre ggml_can_repeat(src1, src0) must return true, indicating src1 can be broadcasted\n * to match src0.\n *\n * @remarks This function iterates over the dimensions of src0 and src1, calculating the\n * necessary broadcast dimensions and strides. If a dimension requires broadcasting\n * (i.e., its size in src1 is smaller than in src0), an additional dimension is\n * added with size calculated to match src0's dimension. This adjustment ensures\n * that src1 can be element-wise broadcasted to src0's shape.\n *\n * How it works:\n *\n * if dim0 has padding.\n * a -> (2, 2) padding = 2\n * a: [[1, 2, *, *]\n * [2, 3, *, *]]\n * nb = (8, 4, 2)\n *\n * if a should bcast with b -> (2, 4)\n * b' -> (2, 2, 2)\n * b : [[1, 2, 3, 4, *, *]\n * [5, 6, 7, 8, *, *]]\n * nb = (12, 6, 1)\n *\n * after bcast:\n * a' -> (2, 1, 2)\n * a': [[[1, 2], *, *]\n * [[2, 3], *, *]]\n * nb = (8, 4, 2, 1)\n *\n * b' : [[[1, 2], [3, 4], *, *]\n * [[5, 6], [7, 8], *, *]]\n * nb = (12, 6, 2, 1)\n * \\endcode\n *\n * dim1 in a inserted dim, should add nb for dim1,\n * and all other nb moves to next in order.\n */\nint64_t ggml_cann_get_bcast_shape(const ggml_tensor * src0,\n const ggml_tensor * src1,\n int64_t * bcast_ne_src0,\n int64_t * bcast_ne_src1,\n size_t * bcast_nb_src0,\n size_t * bcast_nb_src1);\n\n// Bcast macro to avoid duplicate code.\n#define BCAST_SHAPE(src0, src1) \\\n int64_t bcast_##src0##_ne[GGML_MAX_DIMS * 2]; \\\n int64_t bcast_##src1##_ne[GGML_MAX_DIMS * 2]; \\\n size_t bcast_##src0##_nb[GGML_MAX_DIMS * 2]; \\\n size_t bcast_##src1##_nb[GGML_MAX_DIMS * 2]; \\\n int64_t bcast_dims = ggml_cann_get_bcast_shape(src0, src1, bcast_##src0##_ne, bcast_##src1##_ne, \\\n bcast_##src0##_nb, bcast_##src1##_nb);\n\n#define BCAST_PARAM(tensor) bcast_##tensor##_ne, bcast_##tensor##_nb, bcast_dims\n\n/**\n * @brief Calculates broadcast shapes for matrix multiplication.\n *\n * @details This function computes the broadcast shapes required for matrix multiplication\n * based on the input, weight, and destination tensor shapes. It ensures that the\n * dimensions of weight tensors are expanded appropriately to satisfy matrix\n * multiplication broadcast rules.\n *\n * @param input_ne Array containing the dimensions of the input tensor.\n * @param weight_ne Array containing the dimensions of the weight tensor.\n * @param dst_ne Array containing the dimensions of the destination tensor.\n * @param input_nb Array containing the strides of the input tensor.\n * @param weight_nb Array containing the strides of the weight tensor.\n * @param dst_nb Array containing the strides of the destination tensor.\n * @param bcast_input_ne Output array for broadcasted input tensor dimensions.\n * @param bcast_weight_ne Output array for broadcasted weight tensor dimensions.\n * @param bcast_dst_ne Output array for broadcasted destination tensor dimensions.\n * @param bcast_input_nb Output array for broadcasted input tensor strides.\n * @param bcast_weight_nb Output array for broadcasted weight tensor strides.\n * @param bcast_dst_nb Output array for broadcasted destination tensor strides.\n * @return The number of dimensions in the broadcasted tensors.\n *\n * @remarks This function iterates over the tensor dimensions and calculates the broadcast\n * shapes needed for matrix multiplication. It ensures that dimensions where\n * weight tensor requires expansion are appropriately handled to conform with\n * broadcasting rules.\n * @note compare with ggml_cann_get_bcast_shape, mul_mat broadcast need add this new dim\n * before cast dim.\n * @sa ggml_cann_get_bcast_shape\n */\nint64_t ggml_cann_get_mulmat_bcast_shape(const int64_t * input_ne,\n const int64_t * weight_ne,\n const int64_t * dst_ne,\n const size_t * input_nb,\n const size_t * weight_nb,\n const size_t * dst_nb,\n int64_t * bcast_input_ne,\n int64_t * bcast_weight_ne,\n int64_t * bcast_dst_ne,\n size_t * bcast_input_nb,\n size_t * bcast_weight_nb,\n size_t * bcast_dst_nb);\n\n// Bcast macro to avoid duplicate code.\n#define BCAST_MUL_MAT_SHAPE(input, weight, dst) \\\n int64_t bcast_##input##_ne[GGML_MAX_DIMS * 2]; \\\n int64_t bcast_##weight##_ne[GGML_MAX_DIMS * 2]; \\\n int64_t bcast_##dst##_ne[GGML_MAX_DIMS * 2]; \\\n size_t bcast_##input##_nb[GGML_MAX_DIMS * 2]; \\\n size_t bcast_##weight##_nb[GGML_MAX_DIMS * 2]; \\\n size_t bcast_##dst##_nb[GGML_MAX_DIMS * 2]; \\\n int64_t bcast_dims = ggml_cann_get_mulmat_bcast_shape( \\\n input->ne, weight->ne, dst->ne, input->nb, weight->nb, dst->nb, bcast_##input##_ne, bcast_##weight##_ne, \\\n bcast_##dst##_ne, bcast_##input##_nb, bcast_##weight##_nb, bcast_##dst##_nb);\n\n#define BCAST_MUL_MAT_PARAM(tensor) bcast_##tensor##_ne, bcast_##tensor##_nb, bcast_dims\n\n#endif // CANN_ACL_TENSOR_H\n" }, { "path": "src/ggml-cann/aclnn_ops.cpp", "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#include \"aclnn_ops.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#define GGML_COMMON_DECL_C\n\n#include \"../ggml-common.h\"\n\nvoid bcast_shape(ggml_tensor * src0,\n ggml_tensor * src1,\n ggml_tensor * dst,\n acl_tensor_ptr & acl_src0,\n acl_tensor_ptr & acl_src1,\n acl_tensor_ptr & acl_dst) {\n GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_can_repeat(src1, src0));\n // Need bcast\n if (!ggml_are_same_shape(src0, src1) && ggml_cann_need_bcast(src0, src1)) {\n BCAST_SHAPE(src0, src1)\n acl_src0 = ggml_cann_create_tensor(src0, BCAST_PARAM(src0));\n acl_src1 = ggml_cann_create_tensor(src1, BCAST_PARAM(src1));\n acl_dst = ggml_cann_create_tensor(dst, BCAST_PARAM(src0));\n } else {\n acl_src0 = ggml_cann_create_tensor(src0);\n acl_src1 = ggml_cann_create_tensor(src1);\n acl_dst = ggml_cann_create_tensor(dst);\n }\n}\n\nvoid ggml_cann_op_unary(std::function unary_op,\n ggml_backend_cann_context & ctx,\n ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n unary_op(ctx, acl_src.get(), acl_dst.get());\n}\n\nvoid ggml_cann_op_unary_gated(std::function unary_op,\n ggml_backend_cann_context & ctx,\n ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n acl_tensor_ptr acl_src0, acl_src1;\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n\n acl_src0 = ggml_cann_create_tensor(src0);\n acl_src1 = ggml_cann_create_tensor(src1);\n } else {\n int64_t ne[] = { src0->ne[0] / 2, src0->ne[1], src0->ne[2], src0->ne[3] };\n size_t nb[] = { src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3] };\n acl_src0 = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, 0);\n acl_src1 = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, ne[0] * ggml_element_size(src0));\n if (swapped) {\n std::swap(acl_src0, acl_src1);\n }\n }\n\n unary_op(ctx, acl_src0.get(), acl_dst.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, acl_dst.get(), acl_src1.get());\n}\n\n/**\n * @brief Repeats elements of a tensor along each dimension according to the\n * specified repeat array.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor to be repeated.\n * @param acl_dst The destination tensor after repeating.\n * @param repeat_array The array specifying the number of repetitions along each\n * dimension.\n */\nstatic void aclnn_repeat(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n aclTensor * acl_dst,\n int64_t * repeat_array) {\n // repeat tensor along each dim with repeat_array\n acl_int_array_ptr repeats = ggml_cann_create_int_array(repeat_array, GGML_MAX_DIMS);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Repeat, acl_src, repeats.get(), acl_dst);\n}\n\n/**\n * @brief Casts the data type of a source tensor to a destination tensor.\n *\n * This function casts the data type of the source tensor `acl_src` to the\n * specified data type `cast_data_type` and stores the result in the destination\n * tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose data type will be casted.\n * @param acl_dst The destination tensor where the casted result will be stored.\n * @param cast_data_type The target data type to which the source tensor will be\n * casted.\n */\nstatic void aclnn_cast(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n aclTensor * acl_dst,\n aclDataType cast_data_type) {\n GGML_CANN_CALL_ACLNN_OP(ctx, Cast, acl_src, cast_data_type, acl_dst);\n}\n\nvoid ggml_cann_repeat(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n GGML_ASSERT(ggml_can_repeat(src, dst));\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n int64_t repeatsArray[] = { dst->ne[3] / src->ne[3], dst->ne[2] / src->ne[2], dst->ne[1] / src->ne[1],\n dst->ne[0] / src->ne[0] };\n\n aclnn_repeat(ctx, acl_src.get(), acl_dst.get(), repeatsArray);\n}\n\nvoid aclnn_add(ggml_backend_cann_context & ctx, aclTensor * acl_src0, aclTensor * acl_src1, aclTensor * acl_dst) {\n float alphaValue = 1.0f;\n acl_scalar_ptr alpha = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n if (acl_dst != nullptr) {\n GGML_CANN_CALL_ACLNN_OP(ctx, Add, acl_src0, acl_src1, alpha.get(), acl_dst);\n } else {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_src0, acl_src1, alpha.get());\n }\n}\n\nvoid aclnn_sub(ggml_backend_cann_context & ctx, aclTensor * acl_src0, aclTensor * acl_src1, aclTensor * acl_dst) {\n float alphaValue = 1.0f;\n acl_scalar_ptr alpha = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n if (acl_dst != nullptr) {\n GGML_CANN_CALL_ACLNN_OP(ctx, Sub, acl_src0, acl_src1, alpha.get(), acl_dst);\n } else {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSub, acl_src0, acl_src1, alpha.get());\n }\n}\n\nvoid aclnn_mul(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_other, aclTensor * acl_dst) {\n if (acl_dst != nullptr) {\n GGML_CANN_CALL_ACLNN_OP(ctx, Mul, acl_src, acl_other, acl_dst);\n } else {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, acl_src, acl_other);\n }\n}\n\nvoid aclnn_div(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_other, aclTensor * acl_dst) {\n if (acl_dst != nullptr) {\n GGML_CANN_CALL_ACLNN_OP(ctx, Div, acl_src, acl_other, acl_dst);\n } else {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceDiv, acl_src, acl_other);\n }\n}\n\n/**\n * @brief Multiplies elements of a tensor by a scalar value, optionally\n * in-place.\n *\n * This function multiplies each element of the source tensor `acl_src` by the\n * scalar `scale` and stores the result in the destination tensor `acl_dst`. If\n * `inplace` is true, `acl_dst` will not be used and the operation is performed\n * in-place on `acl_src`.\n * The operation is defined as:\n * \\f[\n * \\text {acl_dst }_i=\\text {acl_src }_i \\times \\text {scale}\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose elements will be multiplied.\n * @param scale The scalar value by which each element of `acl_src` will be\n * multiplied.\n * @param acl_dst The destination tensor where the result will be stored if\n * `inplace` is false.\n * @param inplace Flag indicating whether to perform the operation in-place on\n * `acl_src`.\n */\nstatic void aclnn_muls(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n float scale,\n aclTensor * acl_dst,\n bool inplace) {\n acl_scalar_ptr acl_scale = ggml_cann_create_scalar(&scale, aclDataType::ACL_FLOAT);\n if (inplace) {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMuls, acl_src, acl_scale.get());\n } else {\n GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_src, acl_scale.get(), acl_dst);\n }\n}\n\nvoid ggml_cann_leaky_relu(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n float negative_slope;\n memcpy(&negative_slope, dst->op_params, sizeof(float));\n acl_scalar_ptr acl_negative_slope = ggml_cann_create_scalar(&negative_slope, aclDataType::ACL_FLOAT);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, LeakyRelu, acl_src.get(), acl_negative_slope.get(), acl_dst.get());\n}\n\n/**\n * @brief Concatenates a list of tensors along a specified dimension and stores\n * the result in a destination tensor.\n *\n * @param ctx The context for the CANN backend operations.\n * @param tensorList The list of tensors to be concatenated.\n * @param acl_dst The destination tensor where the concatenated result will be\n * stored.\n * @param concat_dim The dimension along which the tensors will be concatenated.\n */\nstatic void aclnn_concat(ggml_backend_cann_context & ctx,\n aclTensorList * tensorList,\n aclTensor * acl_dst,\n int64_t concat_dim) {\n GGML_CANN_CALL_ACLNN_OP(ctx, Cat, tensorList, concat_dim, acl_dst);\n}\n\nvoid ggml_cann_concat(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n ggml_tensor * src1 = dst->src[1];\n acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0);\n acl_tensor_ptr acl_src1 = ggml_cann_create_tensor(src1);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n GGML_ASSERT(dim >= 0 && dim < 4);\n int32_t acl_dim = 3 - dim;\n\n acl_tensor_list_ptr tensor_list = ggml_cann_create_tensor_list(acl_src0, acl_src1);\n aclnn_concat(ctx, tensor_list.get(), acl_dst.get(), acl_dim);\n}\n\n/**\n * @brief Creates a tensor with values starting from `start`, incremented by\n * `step`, and ending before `stop`.\n *\n * This function performs the operation:\n * \\f[\n * \\text {out }_{i+1}=\\text {out }_i+\\text {step}\n * \\f]\n * the range is [start, stop).\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_dst The destination tensor where the values will be stored.\n * @param start The starting value of the range.\n * @param stop The ending value of the range (exclusive).\n * @param step The step size between consecutive values.\n * @param n_elements The number of elements in the destination tensor.\n */\nstatic void aclnn_arange(ggml_backend_cann_context & ctx,\n aclTensor * acl_dst,\n float start,\n float stop,\n float step,\n int64_t n_elements) {\n int64_t steps = (int64_t) std::ceil((stop - start) / step);\n GGML_ASSERT(n_elements == steps);\n\n acl_scalar_ptr acl_start = ggml_cann_create_scalar(&start, aclDataType::ACL_FLOAT);\n acl_scalar_ptr acl_end = ggml_cann_create_scalar(&stop, aclDataType::ACL_FLOAT);\n acl_scalar_ptr acl_step = ggml_cann_create_scalar(&step, aclDataType::ACL_FLOAT);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Arange, acl_start.get(), acl_end.get(), acl_step.get(), acl_dst);\n}\n\nvoid ggml_cann_arange(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n int64_t n_elements = ggml_nelements(dst);\n float start;\n float stop;\n float step;\n memcpy(&start, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&stop, (float *) dst->op_params + 1, sizeof(float));\n memcpy(&step, (float *) dst->op_params + 2, sizeof(float));\n\n aclnn_arange(ctx, acl_dst.get(), start, stop, step, n_elements);\n}\n\nvoid ggml_cann_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n\n float min;\n float max;\n memcpy(&min, dst->op_params, sizeof(float));\n memcpy(&max, (float *) dst->op_params + 1, sizeof(float));\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n acl_scalar_ptr acl_min = ggml_cann_create_scalar(&min, aclDataType::ACL_FLOAT);\n acl_scalar_ptr acl_max = ggml_cann_create_scalar(&max, aclDataType::ACL_FLOAT);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_src.get(), acl_min.get(), acl_max.get(), acl_dst.get());\n}\n\nvoid ggml_cann_scale(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n\n // scale factor\n float v;\n memcpy(&v, dst->op_params, sizeof(float));\n\n acl_scalar_ptr scale = ggml_cann_create_scalar(&v, aclDataType::ACL_FLOAT);\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_src.get(), scale.get(), acl_dst.get());\n}\n\nvoid ggml_cann_argsort(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), ggml_nelements(dst) * sizeof(int64_t));\n void * buffer = temp_buffer_allocator.get();\n acl_tensor_ptr tmp_tensor =\n ggml_cann_create_tensor(buffer, ACL_INT64, ggml_type_size(dst->type), dst->ne, dst->nb, GGML_MAX_DIMS);\n GGML_CANN_CALL_ACLNN_OP(ctx, Argsort, acl_src.get(), -1, (order == GGML_SORT_ORDER_DESC ? true : false),\n tmp_tensor.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, Cast, tmp_tensor.get(), ggml_cann_type_mapping(dst->type), acl_dst.get());\n}\n\nvoid ggml_cann_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n\n std::vector normData = { dst->ne[0] };\n acl_int_array_ptr norm = ggml_cann_create_int_array(normData.data(), normData.size());\n GGML_CANN_CALL_ACLNN_OP(ctx, LayerNorm, acl_src.get(), norm.get(), nullptr, nullptr, eps, acl_dst.get(), nullptr,\n nullptr);\n}\n\nvoid ggml_cann_l2_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n size_t type_size = ggml_type_size(src->type);\n int64_t n_bytes = src->ne[3] * src->ne[2] * src->ne[1] * type_size;\n ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), n_bytes);\n void * buffer = temp_buffer_allocator.get();\n\n int64_t div_ne[] = { 1, src->ne[1], src->ne[2], src->ne[3] };\n size_t div_nb[GGML_MAX_DIMS];\n div_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n div_nb[i] = div_nb[i - 1] * div_ne[i - 1];\n }\n acl_tensor_ptr acl_div = ggml_cann_create_tensor(buffer, ACL_FLOAT, type_size, div_ne, div_nb, GGML_MAX_DIMS);\n\n std::vector norm_dims = { 3 };\n acl_int_array_ptr dims_array = ggml_cann_create_int_array(norm_dims.data(), norm_dims.size());\n\n float p_value = 2.0f;\n acl_scalar_ptr p_scalar = ggml_cann_create_scalar(&p_value, aclDataType::ACL_FLOAT);\n GGML_CANN_CALL_ACLNN_OP(ctx, Norm, acl_src.get(), p_scalar.get(), dims_array.get(), true, acl_div.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, Div, acl_src.get(), acl_div.get(), acl_dst.get());\n}\n\nvoid ggml_cann_cross_entropy_loss(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n ggml_tensor * src1 = dst->src[1];\n\n const int64_t nc = src0->ne[0];\n const int64_t nr = ggml_nrows(src0);\n\n int64_t logits_ne[] = { nc, nr };\n size_t logits_nb[2];\n logits_nb[0] = ggml_type_size(src0->type);\n logits_nb[1] = logits_nb[0] * logits_ne[0];\n acl_tensor_ptr acl_logits = ggml_cann_create_tensor(src0->data, ACL_FLOAT, sizeof(float), logits_ne, logits_nb, 2);\n\n size_t log_softmax_type_size = sizeof(float);\n int64_t log_softmax_n_bytes = nr * nc * log_softmax_type_size;\n ggml_cann_pool_alloc log_softmax_allocator(ctx.pool(), log_softmax_n_bytes);\n void * log_softmax_buffer = log_softmax_allocator.get();\n\n int64_t log_softmax_ne[] = { nc, nr };\n size_t log_softmax_nb[2];\n log_softmax_nb[0] = log_softmax_type_size;\n log_softmax_nb[1] = log_softmax_nb[0] * log_softmax_ne[0];\n acl_tensor_ptr acl_log_softmax = ggml_cann_create_tensor(log_softmax_buffer, ACL_FLOAT, log_softmax_type_size,\n log_softmax_ne, log_softmax_nb, 2);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, LogSoftmax, acl_logits.get(), 1, acl_log_softmax.get());\n\n int64_t labels_ne[] = { nc, nr };\n size_t labels_nb[2];\n labels_nb[0] = ggml_type_size(src1->type);\n labels_nb[1] = labels_nb[0] * labels_ne[0];\n acl_tensor_ptr acl_labels = ggml_cann_create_tensor(src1->data, ACL_FLOAT, sizeof(float), labels_ne, labels_nb, 2);\n\n size_t mul_type_size = sizeof(float);\n int64_t mul_n_bytes = nr * nc * mul_type_size;\n ggml_cann_pool_alloc mul_allocator(ctx.pool(), mul_n_bytes);\n void * mul_buffer = mul_allocator.get();\n\n int64_t mul_ne[] = { nc, nr };\n size_t mul_nb[2];\n mul_nb[0] = mul_type_size;\n mul_nb[1] = mul_nb[0] * mul_ne[0];\n acl_tensor_ptr acl_mul_result = ggml_cann_create_tensor(mul_buffer, ACL_FLOAT, mul_type_size, mul_ne, mul_nb, 2);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Mul, acl_log_softmax.get(), acl_labels.get(), acl_mul_result.get());\n\n size_t sum_per_sample_type_size = sizeof(float);\n int64_t sum_per_sample_n_bytes = nr * sum_per_sample_type_size;\n ggml_cann_pool_alloc sum_per_sample_allocator(ctx.pool(), sum_per_sample_n_bytes);\n void * sum_per_sample_buffer = sum_per_sample_allocator.get();\n\n int64_t sum_per_sample_ne[] = { nr };\n size_t sum_per_sample_nb[1];\n sum_per_sample_nb[0] = sum_per_sample_type_size;\n acl_tensor_ptr acl_sum_per_sample = ggml_cann_create_tensor(\n sum_per_sample_buffer, ACL_FLOAT, sum_per_sample_type_size, sum_per_sample_ne, sum_per_sample_nb, 1);\n\n std::vector sum_dims = { 1 };\n acl_int_array_ptr dims_array = ggml_cann_create_int_array(sum_dims.data(), sum_dims.size());\n bool keep_dims = false;\n\n GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_mul_result.get(), dims_array.get(), keep_dims, ACL_FLOAT,\n acl_sum_per_sample.get());\n\n size_t total_sum_type_size = sizeof(float);\n int64_t total_sum_n_bytes = 1 * total_sum_type_size;\n ggml_cann_pool_alloc total_sum_allocator(ctx.pool(), total_sum_n_bytes);\n void * total_sum_buffer = total_sum_allocator.get();\n\n int64_t total_sum_ne[] = { 1 };\n size_t total_sum_nb[1];\n total_sum_nb[0] = total_sum_type_size;\n\n acl_tensor_ptr acl_total_sum =\n ggml_cann_create_tensor(total_sum_buffer, ACL_FLOAT, total_sum_type_size, total_sum_ne, total_sum_nb, 1);\n\n std::vector total_sum_dims = { 0 };\n acl_int_array_ptr total_sum_dims_array = ggml_cann_create_int_array(total_sum_dims.data(), total_sum_dims.size());\n\n GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_sum_per_sample.get(), total_sum_dims_array.get(), keep_dims, ACL_FLOAT,\n acl_total_sum.get());\n\n float value = -1.0f / static_cast(nr);\n acl_scalar_ptr scale_factor = ggml_cann_create_scalar(&value, aclDataType::ACL_FLOAT);\n acl_tensor_ptr acl_dst =\n ggml_cann_create_tensor(dst->data, ACL_FLOAT, sizeof(float), total_sum_ne, total_sum_nb, 1);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_total_sum.get(), scale_factor.get(), acl_dst.get());\n}\n\nvoid ggml_cann_group_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n int n_groups = dst->op_params[0];\n\n float eps;\n memcpy(&eps, dst->op_params + 1, sizeof(float));\n\n int64_t N = src->ne[3];\n int64_t C = src->ne[2];\n int64_t HxW = src->ne[1] * src->ne[0];\n\n size_t type_size = ggml_type_size(src->type);\n int64_t ne[] = { n_groups, N };\n size_t nb[] = { type_size, type_size * n_groups };\n size_t n_bytes = N * n_groups;\n\n ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), n_bytes * 2);\n void * buffer = temp_buffer_allocator.get();\n acl_tensor_ptr acl_mean_out = ggml_cann_create_tensor(buffer, ACL_FLOAT, type_size, ne, nb, ACL_FORMAT_ND);\n acl_tensor_ptr acl_rstd_out =\n ggml_cann_create_tensor((char *) buffer + n_bytes, ACL_FLOAT, type_size, ne, nb, ACL_FORMAT_ND);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, GroupNorm, acl_src.get(), nullptr, nullptr, N, C, HxW, n_groups, eps, acl_dst.get(),\n acl_mean_out.get(), acl_rstd_out.get());\n}\n\nvoid ggml_cann_acc(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n ggml_tensor * src1 = dst->src[1];\n\n size_t nb1 = ((int32_t *) dst->op_params)[0];\n size_t nb2 = ((int32_t *) dst->op_params)[1];\n size_t nb3 = ((int32_t *) dst->op_params)[2];\n size_t offset = ((int32_t *) dst->op_params)[3];\n bool inplace = (bool) ((int32_t *) dst->op_params)[4];\n\n size_t param_nb[] = { ggml_element_size(src0), nb1, nb2, nb3 };\n\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, src1->ne, param_nb, GGML_MAX_DIMS, ACL_FORMAT_ND, offset);\n acl_tensor_ptr acl_src1 = ggml_cann_create_tensor(src1);\n\n acl_scalar_ptr alpha = nullptr;\n float alphaValue = 1.0f;\n alpha = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n if (!inplace) {\n size_t cpy_size = ggml_nbytes(dst);\n ACL_CHECK(\n aclrtMemcpyAsync(dst->data, cpy_size, src0->data, cpy_size, ACL_MEMCPY_DEVICE_TO_DEVICE, ctx.stream()));\n acl_tensor_ptr acl_src0 =\n ggml_cann_create_tensor(src0, src1->ne, src0->nb, GGML_MAX_DIMS, ACL_FORMAT_ND, offset);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Add, acl_src0.get(), acl_src1.get(), alpha.get(), acl_dst.get());\n } else {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst.get(), acl_src1.get(), alpha.get());\n }\n}\n\n/**\n * @brief Performs sum reduction on a given tensor along specified dimensions.\n *\n * This function reduces the input tensor by summing along the specified dimensions.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the reduced result will be stored.\n * @param dim An array of dimension indices.\n * @param dim_size The number of dimensions.\n */\nstatic void aclnn_reduce_sum(ggml_backend_cann_context & ctx, ggml_tensor * dst, int64_t * dim, size_t dim_size) {\n GGML_ASSERT(dst->ne[0] == 1);\n ggml_tensor * src = dst->src[0];\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n acl_int_array_ptr reduce_dims = ggml_cann_create_int_array(dim, dim_size);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_src.get(), reduce_dims.get(), true, ggml_cann_type_mapping(dst->type),\n acl_dst.get());\n}\n\nvoid ggml_cann_sum_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n int64_t reduce_dims[] = { 3 };\n aclnn_reduce_sum(ctx, dst, reduce_dims, 1);\n}\n\nvoid ggml_cann_sum(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n int64_t reduce_dims[] = { 0, 1, 2, 3 };\n aclnn_reduce_sum(ctx, dst, reduce_dims, 4);\n}\n\nvoid ggml_cann_upsample_nearest2d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n\n std::vector output_size{ dst->ne[1], dst->ne[0] };\n acl_int_array_ptr output_size_array = ggml_cann_create_int_array(output_size.data(), 2);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, UpsampleNearest2d, acl_src.get(), output_size_array.get(), acl_dst.get());\n}\n\n/**\n * @brief Pads a tensor with a specified value along each dimension.\n *\n * This function performs padding of the source tensor `acl_src` and stores the\n * result in the destination tensor `acl_dst`. The padding values for each\n * dimension are specified in the `paddings` array.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor to be padded.\n * @param acl_dst The destination tensor where the padded result will be stored.\n * @param paddings An array specifying the padding values for each dimension.\n * The size of the array should be twice the number of dimensions of the tensor.\n * @param value The value to be used for padding. The default value is 0.0.\n */\nstatic void aclnn_pad(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n aclTensor * acl_dst,\n int64_t * paddings,\n float value = 0.0f) {\n acl_int_array_ptr acl_pad = ggml_cann_create_int_array(paddings, GGML_MAX_DIMS * 2);\n acl_scalar_ptr acl_value = ggml_cann_create_scalar(&value, aclDataType::ACL_FLOAT);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, ConstantPadNd, acl_src, acl_pad.get(), acl_value.get(), acl_dst);\n}\n\nvoid ggml_cann_pad(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n // padding: value in the array means how much distance will be padding.\n // the position of elements in the array means which dirction to padding,\n // each position means: [dim0.front, dim0.behind, dim1.front, dim1.behind,\n // dim2.front, dim2.behind, dim3.front, dim3.behind]\n const int32_t lp0 = ggml_get_op_params_i32(dst, 0);\n const int32_t rp0 = ggml_get_op_params_i32(dst, 1);\n const int32_t lp1 = ggml_get_op_params_i32(dst, 2);\n const int32_t rp1 = ggml_get_op_params_i32(dst, 3);\n const int32_t lp2 = ggml_get_op_params_i32(dst, 4);\n const int32_t rp2 = ggml_get_op_params_i32(dst, 5);\n const int32_t lp3 = ggml_get_op_params_i32(dst, 6);\n const int32_t rp3 = ggml_get_op_params_i32(dst, 7);\n\n int64_t paddings[] = { lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3 };\n aclnn_pad(ctx, acl_src.get(), acl_dst.get(), paddings);\n}\n\n/**\n * @brief Performs 2D average pooling on the input tensor and stores the result\n * in the destination tensor.\n *\n * This function performs average pooling on the source tensor and stores the\n * result in the destination tensor. The pooling parameters (kernel size,\n * strides, padding) are specified in the `op_params` of the destination tensor.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the result will be stored. The source\n * tensor is referenced by `dst->src[0]`.\n */\nstatic void ggml_cann_avg_pool2d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n GGML_ASSERT(src->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n\n const int32_t * opts = (const int32_t *) dst->op_params;\n const int k0 = opts[1];\n const int k1 = opts[2];\n const int s0 = opts[3];\n const int s1 = opts[4];\n const int p0 = opts[5];\n const int p1 = opts[6];\n\n std::vector kernel_dims = { k1, k0 };\n std::vector stride_dims = { s1, s0 };\n std::vector padding_avg_dims = { p1, p0 }; // (padH, padW)\n\n acl_int_array_ptr kernel_size = ggml_cann_create_int_array(kernel_dims.data(), 2);\n acl_int_array_ptr strides = ggml_cann_create_int_array(stride_dims.data(), 2);\n acl_int_array_ptr paddings_avg = ggml_cann_create_int_array(padding_avg_dims.data(), 2);\n\n bool ceil_mode = false;\n bool count_include_pad = true;\n int64_t divisor_override = 0;\n int8_t cube_math_type = 0;\n#ifdef ASCEND_310P\n cube_math_type = 1;\n#endif\n\n GGML_CANN_CALL_ACLNN_OP(ctx, AvgPool2d, acl_src.get(), kernel_size.get(), strides.get(), paddings_avg.get(),\n ceil_mode, count_include_pad, divisor_override, cube_math_type, acl_dst.get());\n}\n\n/**\n * @brief Performs 2D max pooling on the input tensor and stores the result in\n * the destination tensor.\n *\n * This function performs max pooling on the source tensor and stores the result\n * in the destination tensor. The pooling parameters (kernel size, strides,\n * padding) are specified in the `op_params` of the destination tensor.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the result will be stored. The source\n * tensor is referenced by `dst->src[0]`.\n */\nstatic void ggml_cann_max_pool2d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n GGML_ASSERT(src->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n\n const int32_t * opts = (const int32_t *) dst->op_params;\n const int k0 = opts[1];\n const int k1 = opts[2];\n const int s0 = opts[3];\n const int s1 = opts[4];\n const int p0 = opts[5];\n const int p1 = opts[6];\n\n int64_t temp_ne[] = { src->ne[0] + p0 * 2, src->ne[1] + p1 * 2, src->ne[2], src->ne[3] };\n size_t temp_nb[GGML_MAX_DIMS];\n\n temp_nb[0] = ggml_element_size(src);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n temp_nb[i] = temp_nb[i - 1] * temp_ne[i - 1];\n }\n\n ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), ggml_nbytes(src) + p0 * 2 + p1 * 2 * src->nb[1]);\n void * buffer = temp_buffer_allocator.get();\n acl_tensor_ptr tmp_tensor = ggml_cann_create_tensor(buffer, ACL_FLOAT, ggml_element_size(src), temp_ne, temp_nb,\n GGML_MAX_DIMS, ACL_FORMAT_NCHW);\n\n // pad: see padding in ggml_cann_pad()\n int64_t paddings[] = { p0, p0, p1, p1, 0, 0, 0, 0 };\n float value = -FLT_MAX;\n aclnn_pad(ctx, acl_src.get(), tmp_tensor.get(), paddings, value);\n\n // max_pool\n std::vector kernel_dims = { k1, k0 };\n std::vector stride_dims = { s1, s0 };\n // padding_max_dims: [dim0_start, dim0_end, dim1_start, dim1_end]\n std::vector padding_max_dims = { 0, 0, 0, 0 };\n std::vector dilation_size = { 1, 1 };\n acl_int_array_ptr kernel_size = ggml_cann_create_int_array(kernel_dims.data(), 2);\n acl_int_array_ptr strides = ggml_cann_create_int_array(stride_dims.data(), 2);\n acl_int_array_ptr paddings_max = ggml_cann_create_int_array(padding_max_dims.data(), 4);\n acl_int_array_ptr dilations = ggml_cann_create_int_array(dilation_size.data(), 2);\n\n bool ceil_mode = false;\n int64_t auto_pads = 0;\n GGML_CANN_CALL_ACLNN_OP(ctx, MaxPool, tmp_tensor.get(), kernel_size.get(), strides.get(), auto_pads,\n paddings_max.get(), dilations.get(), ceil_mode, acl_dst.get());\n}\n\nvoid ggml_cann_pool2d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n const int32_t * opts = (const int32_t *) dst->op_params;\n enum ggml_op_pool op = static_cast(opts[0]);\n switch (op) {\n case GGML_OP_POOL_AVG:\n ggml_cann_avg_pool2d(ctx, dst);\n break;\n case GGML_OP_POOL_MAX:\n ggml_cann_max_pool2d(ctx, dst);\n break;\n case GGML_OP_POOL_COUNT:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\n/**\n * @brief Copies data from the source tensor to the destination tensor.\n *\n * This function copies data from the source tensor `acl_src` to the destination\n * tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor from which data will be copied.\n * @param acl_dst The destination tensor where the data will be copied to.\n */\nstatic void cann_copy(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceCopy, acl_dst, acl_src);\n}\n\nvoid ggml_cann_dup(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n\n if (ggml_are_same_shape(src0, dst)) {\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n if (dst->type == src0->type) {\n cann_copy(ctx, acl_src.get(), acl_dst.get());\n } else {\n aclnn_cast(ctx, acl_src.get(), acl_dst.get(), ggml_cann_type_mapping(dst->type));\n }\n } else {\n void * src_trans_buffer = src0->data;\n ggml_cann_pool_alloc src_buffer_allocator;\n if (!ggml_is_contiguous(src0)) {\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n src_buffer_allocator.alloc(ctx.pool(), ggml_nelements(src0) * ggml_type_size(src0->type));\n src_trans_buffer = src_buffer_allocator.get();\n size_t src_trans_nb[GGML_MAX_DIMS];\n src_trans_nb[0] = ggml_type_size(src0->type);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];\n }\n acl_tensor_ptr src_trans_tensor =\n ggml_cann_create_tensor(src_trans_buffer, ggml_cann_type_mapping(src0->type),\n ggml_type_size(src0->type), src0->ne, src_trans_nb, GGML_MAX_DIMS);\n cann_copy(ctx, acl_src.get(), src_trans_tensor.get());\n }\n\n size_t src_reshape_nb[GGML_MAX_DIMS];\n src_reshape_nb[0] = ggml_type_size(src0->type);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n src_reshape_nb[i] = src_reshape_nb[i - 1] * dst->ne[i - 1];\n }\n\n acl_tensor_ptr trans_acl_src =\n ggml_cann_create_tensor(src_trans_buffer, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n dst->ne, src_reshape_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n if (dst->type == src0->type) {\n cann_copy(ctx, trans_acl_src.get(), acl_dst.get());\n } else {\n aclnn_cast(ctx, trans_acl_src.get(), acl_dst.get(), ggml_cann_type_mapping(dst->type));\n }\n }\n}\n\n/**\n * @brief Creates an ACL tensor initialized with zeros using a provided buffer.\n *\n * This function initializes a tensor with zeros using the specified buffer and\n * tensor parameters.\n *\n * @param ctx The context for the CANN backend operations.\n * @param buffer The buffer to be used for the tensor data.\n * @param n_bytes The size of the buffer in bytes.\n * @param ne An array specifying the extents (sizes) of each dimension of the\n * tensor.\n * @param dims The number of dimensions of the tensor.\n * @param type The data type of the tensor.\n * @param type_size The size of each element in the tensor data type.\n * @return A tensor smart pointer initialized with zeros.\n */\nstatic acl_tensor_ptr aclnn_zero(ggml_backend_cann_context & ctx,\n void * buffer,\n size_t n_bytes,\n int64_t * ne,\n int64_t dims,\n aclDataType type,\n size_t type_size) {\n size_t nb[GGML_MAX_DIMS];\n nb[0] = type_size;\n for (int i = 1; i < dims; i++) {\n nb[i] = nb[i - 1] * ne[i - 1];\n }\n\n acl_tensor_ptr zero = ggml_cann_create_tensor(buffer, type, type_size, ne, nb, dims);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, zero.get());\n return zero;\n GGML_UNUSED(n_bytes);\n}\n\n/**\n * @brief Creates an ACL tensor initialized with value using a provided buffer.\n *\n * This function initializes a tensor with value using the specified buffer and\n * tensor parameters.\n *\n * @param ctx The context for the CANN backend operations.\n * @param buffer The buffer to be used for the tensor data.\n * @param n_bytes The size of the buffer in bytes.\n * @param ne An array specifying the extents (sizes) of each dimension of the\n * tensor.\n * @param dims The number of dimensions of the tensor.\n * @param type The data type of the tensor.\n * @param type_size The size of each element in the tensor data type.\n * @param value The value to be used for initializing the tensor (default\n * is 1.0).\n * @return A tensor smart pointer initialized with value.\n */\nstatic acl_tensor_ptr aclnn_values(ggml_backend_cann_context & ctx,\n void * buffer,\n size_t n_bytes,\n int64_t * ne,\n int64_t dims,\n aclDataType type,\n size_t type_size,\n float value = 1.0f) {\n acl_tensor_ptr acl_tensor = aclnn_zero(ctx, buffer, n_bytes, ne, dims, type, type_size);\n float alpha_host = 1.0f;\n acl_scalar_ptr alpha = ggml_cann_create_scalar(&alpha_host, aclDataType::ACL_FLOAT);\n acl_scalar_ptr other = ggml_cann_create_scalar(&value, aclDataType::ACL_FLOAT);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdds, acl_tensor.get(), other.get(), alpha.get());\n return acl_tensor;\n}\n\n/**\n * @brief Fills a tensor with a scalar value.\n *\n * This function fills the destination tensor `acl_dst` with the scalar value\n * `scalar`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param scalar The scalar value used to fill the tensor.\n * @param acl_dst The destination tensor to be filled with the scalar value.\n */\nstatic void aclnn_fill_scalar(ggml_backend_cann_context & ctx, float scalar, aclTensor * acl_dst) {\n acl_scalar_ptr acl_scalar = ggml_cann_create_scalar(&scalar, aclDataType::ACL_FLOAT);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceFillScalar, acl_dst, acl_scalar.get());\n}\n\n/**\n * @brief Get or expand a cached tensor filled with a scalar value.\n *\n * This function manages cached device memory for tensors. If the current\n * cache size is insufficient for the requested tensor shape, the old memory will\n * be released and new memory will be allocated. The allocated buffer is\n * initialized with the given scalar value using CANN operations.\n * Finally, an aclTensor object is created from the cached memory and returned.\n *\n * @param ctx The CANN backend context that manages device memory.\n * @param buffer A pointer to the cached device buffer (will be allocated\n * or reallocated if necessary).\n * @param cache_element The current number of cached elements. This will be\n * updated when the cache is expanded.\n * @param ne The tensor shape array (number of elements in each dimension).\n * @param nb The stride size for each dimension.\n * @param dtype Data type of cached tensor.\n * @param dims The number of tensor dimensions.\n * @param value The scalar value used to fill the tensor (supports zero\n * initialization via memset or arbitrary values via fill_scalar).\n * @return A tensor smart pointer created from the cached buffer.\n */\nstatic acl_tensor_ptr get_cache_acl_tensor(ggml_backend_cann_context & ctx,\n void ** buffer,\n int64_t & cache_element,\n int64_t * ne,\n size_t * nb,\n ggml_type dtype,\n int64_t dims,\n float value) {\n // Calculate total number of elements\n int64_t n_element = 1;\n for (int i = 0; i < dims; i++) {\n n_element *= ne[i];\n }\n size_t size = n_element * ggml_type_size(dtype);\n\n // Allocate or expand cache if needed\n if (cache_element < n_element) {\n if (*buffer != nullptr) {\n aclrtFree(*buffer);\n *buffer = nullptr;\n }\n\n ACL_CHECK(aclrtMalloc(buffer, size, ACL_MEM_MALLOC_HUGE_FIRST));\n cache_element = n_element;\n\n // Initialize cache\n int64_t pool_ne[1] = { n_element };\n size_t pool_nb[1] = { ggml_type_size(dtype) };\n acl_tensor_ptr acl_value =\n ggml_cann_create_tensor(*buffer, ggml_cann_type_mapping(dtype), ggml_type_size(dtype), pool_ne, pool_nb, 1);\n aclnn_fill_scalar(ctx, value, acl_value.get());\n }\n\n return ggml_cann_create_tensor(*buffer, ggml_cann_type_mapping(dtype), ggml_type_size(dtype), ne, nb, dims);\n}\n\nvoid ggml_cann_rms_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n\n // build gamma.\n size_t acl_gamma_nb[GGML_MAX_DIMS];\n // gamma's type is the same with dst.\n acl_gamma_nb[0] = ggml_type_size(dst->type);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n acl_gamma_nb[i] = acl_gamma_nb[i - 1] * src->ne[i - 1];\n }\n acl_tensor_ptr acl_gamma = get_cache_acl_tensor(\n ctx, &ctx.rms_norm_one_tensor_cache.cache, ctx.rms_norm_one_tensor_cache.size, src->ne, acl_gamma_nb, dst->type,\n 1, // dims\n 1.0f // value\n );\n\n // build rstd.\n int64_t acl_rstd_ne[] = { src->ne[1], src->ne[2], src->ne[3] };\n size_t acl_rstd_nb[GGML_MAX_DIMS - 1];\n // rstd will always be F32.\n acl_rstd_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n acl_rstd_nb[i] = acl_rstd_nb[i - 1] * acl_rstd_ne[i - 1];\n }\n acl_tensor_ptr acl_rstd =\n get_cache_acl_tensor(ctx, &ctx.rms_norm_zero_tensor_cache.cache, ctx.rms_norm_zero_tensor_cache.size,\n acl_rstd_ne, acl_rstd_nb, GGML_TYPE_F32, GGML_MAX_DIMS - 1,\n 0.0f // value\n );\n\n GGML_CANN_CALL_ACLNN_OP(ctx, RmsNorm, acl_src.get(), acl_gamma.get(), eps, acl_dst.get(), acl_rstd.get());\n}\n\n// TODO: performace is low.\nvoid ggml_cann_diag_mask(ggml_backend_cann_context & ctx, ggml_tensor * dst, float value) {\n ggml_tensor * src = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n const int n_past = ((int32_t *) dst->op_params)[0];\n\n ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), ggml_nbytes(src));\n void * buffer = one_tensor_allocator.get();\n\n acl_tensor_ptr mask_tensor = ggml_cann_create_tensor(buffer, ggml_cann_type_mapping(src->type),\n ggml_type_size(src->type), src->ne, src->nb, GGML_MAX_DIMS);\n\n aclnn_fill_scalar(ctx, value, mask_tensor.get());\n\n float alphaValue = 1.0f;\n acl_scalar_ptr alpha = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceTriu, mask_tensor.get(), n_past + 1);\n GGML_CANN_CALL_ACLNN_OP(ctx, Tril, acl_src.get(), n_past + 1, acl_dst.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst.get(), mask_tensor.get(), alpha.get());\n}\n\n/**\n * @brief Permutes the dimensions of a tensor according to a specified order.\n *\n * This function permutes the dimensions of the source tensor `acl_src`\n * according to the order specified in the `new_dim` array and stores the result\n * in the destination tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose dimensions will be permuted.\n * @param acl_dst The destination tensor where the permuted result will be\n * stored.\n * @param new_dim An array specifying the new order of dimensions for the\n * tensor.\n * @param dims The number of dimensions in the tensor.\n */\nstatic void aclnn_permute(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n aclTensor * acl_dst,\n int64_t * new_dim,\n uint64_t dims) {\n acl_int_array_ptr acl_dims = ggml_cann_create_int_array(new_dim, dims);\n GGML_CANN_CALL_ACLNN_OP(ctx, Permute, acl_src, acl_dims.get(), acl_dst);\n}\n\nstatic void ggml_cann_im2col_2d_post_process(ggml_backend_cann_context & ctx,\n ggml_tensor * dst,\n ggml_tensor * src1,\n aclTensor * tmp_cast_tensor,\n aclTensor * tmp_im2col_tensor) {\n // Permute: [N, IC * KH * KW, OW * OH] -> [N, OW * OH, IC * KH * KW]\n int64_t dst_ne[] = { dst->ne[0], dst->ne[1] * dst->ne[2], dst->ne[3] };\n size_t dst_nb[] = { dst->nb[0], dst->nb[1], dst->nb[3] };\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, dst_ne, dst_nb, GGML_MAX_DIMS - 1);\n\n int64_t permute_dim[] = { 0, 2, 1 };\n if (src1->type != dst->type) {\n aclnn_permute(ctx, tmp_cast_tensor, acl_dst.get(), permute_dim, 3);\n } else {\n aclnn_permute(ctx, tmp_im2col_tensor, acl_dst.get(), permute_dim, 3);\n }\n}\n\nstatic void ggml_cann_im2col_1d_post_process(ggml_backend_cann_context & ctx,\n ggml_tensor * dst,\n ggml_tensor * src1,\n aclTensor * tmp_cast_tensor,\n aclTensor * tmp_im2col_tensor,\n const std::vector & im2col_op_params) {\n // get params\n const int64_t KH = im2col_op_params[0];\n const int64_t KW = im2col_op_params[1];\n const int64_t IW = im2col_op_params[2];\n const int64_t IC = im2col_op_params[3];\n const int64_t N = im2col_op_params[4];\n const int64_t OH = im2col_op_params[5];\n const int64_t OW = im2col_op_params[6];\n const int64_t s0 = im2col_op_params[7];\n const int64_t p0 = im2col_op_params[8];\n const int64_t d0 = im2col_op_params[9];\n const int64_t n_bytes_factor = im2col_op_params[10];\n\n // Permute: [N, IC * KH * KW, OW * OH] ->\n // [N, OW * OH * n_bytes_factor, IC * KH * KW]\n ggml_cann_pool_alloc tmp_permute_allocator(ctx.pool());\n tmp_permute_allocator.alloc(ggml_nbytes(dst) * n_bytes_factor);\n void * tmp_permute_buffer = tmp_permute_allocator.get();\n\n int64_t tmp_permute_ne[] = { IC * KH * KW, OW * OH * n_bytes_factor, N };\n size_t tmp_permute_nb[GGML_MAX_DIMS - 1];\n tmp_permute_nb[0] = ggml_type_size(dst->type);\n for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n tmp_permute_nb[i] = tmp_permute_nb[i - 1] * tmp_permute_ne[i - 1];\n }\n\n acl_tensor_ptr tmp_permute_tensor =\n ggml_cann_create_tensor(tmp_permute_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n tmp_permute_ne, tmp_permute_nb, GGML_MAX_DIMS - 1, ACL_FORMAT_ND);\n\n int64_t permute_dim[] = { 0, 2, 1 };\n if (src1->type != dst->type) {\n aclnn_permute(ctx, tmp_cast_tensor, tmp_permute_tensor.get(), permute_dim, 3);\n } else {\n aclnn_permute(ctx, tmp_im2col_tensor, tmp_permute_tensor.get(), permute_dim, 3);\n }\n\n // number of times the kernel moves in W dimension\n const int n_step_w = (IW + 2 * p0 - d0 * (KW - 1) - 1) / s0 + 1;\n size_t offset;\n void * cur_dst_buffer = dst->data, *cur_permute_buffer = tmp_permute_buffer;\n\n // memory copy with offset to restore 1D im2col from 2d\n if (IC > 1) {\n offset = IC * KH * KW * n_step_w * ggml_type_size(dst->type);\n size_t cpy_size = KH * KW * ggml_type_size(dst->type);\n\n for (int c = 0; c < IC; c++) {\n cur_permute_buffer = (char *) tmp_permute_buffer + offset + KH * KW * c * ggml_type_size(dst->type);\n cur_dst_buffer = (char *) dst->data + c * KH * KW * n_step_w * ggml_type_size(dst->type);\n\n for (int i = 0; i < n_step_w; i++) {\n ACL_CHECK(aclrtMemcpyAsync(cur_dst_buffer, cpy_size, cur_permute_buffer, cpy_size,\n ACL_MEMCPY_DEVICE_TO_DEVICE, ctx.stream()));\n cur_dst_buffer = (char *) cur_dst_buffer + KH * KW * ggml_type_size(dst->type);\n cur_permute_buffer = (char *) cur_permute_buffer + KH * KW * IC * ggml_type_size(dst->type);\n }\n }\n } else {\n offset = KH * KW * n_step_w * ggml_type_size(dst->type); // equal to ggml_nbytes(dst)\n ACL_CHECK(aclrtMemcpyAsync(dst->data, offset, (char *) tmp_permute_buffer + offset, offset,\n ACL_MEMCPY_DEVICE_TO_DEVICE, ctx.stream()));\n }\n}\n\nvoid ggml_cann_im2col(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0]; // kernel\n ggml_tensor * src1 = dst->src[1]; // input\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n // aclnnIm2col only works on 2D. set s1, p1, d1 to 1 to perform 2D\n // im2col and do post-processing to restore it to 1D.\n const bool is_2D = ((const int32_t *) (dst->op_params))[6] == 1;\n const int32_t s0 = ((const int32_t *) (dst->op_params))[0];\n const int32_t s1 = is_2D ? ((const int32_t *) (dst->op_params))[1] : 1;\n const int32_t p0 = ((const int32_t *) (dst->op_params))[2];\n const int32_t p1 = is_2D ? ((const int32_t *) (dst->op_params))[3] : 1;\n const int32_t d0 = ((const int32_t *) (dst->op_params))[4];\n const int32_t d1 = is_2D ? ((const int32_t *) (dst->op_params))[5] : 1;\n\n const int64_t N = ne13;\n const int64_t IC = ne12;\n const int64_t KH = ne01;\n const int64_t KW = ne00;\n const int64_t IW = ne10;\n\n const int64_t OH = is_2D ? ne2 : 1;\n const int64_t OW = ne1;\n\n // memory allocated increased to 3x when is_2D == false\n const int64_t n_bytes_factor = is_2D ? 1 : 3;\n\n // im2col: [N,C,H,W] -> [N, IC * KH * KW, OW * OH * n_bytes_factor]\n acl_tensor_ptr acl_src1 = ggml_cann_create_tensor(src1);\n int64_t tmp_im2col_ne[] = { OW * OH * n_bytes_factor, IC * KH * KW, N };\n size_t tmp_im2col_nb[GGML_MAX_DIMS - 1];\n\n tmp_im2col_nb[0] = ggml_type_size(src1->type);\n for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n tmp_im2col_nb[i] = tmp_im2col_nb[i - 1] * tmp_im2col_ne[i - 1];\n }\n\n // Calculate im2col.\n // If dst is f16, tmp_buffer is f32, we need alloc src.typesize *\n // dst.elemcount.\n ggml_cann_pool_alloc im2col_allocator(ctx.pool(), ggml_nelements(dst) * ggml_element_size(src1) * n_bytes_factor);\n void * tmp_im2col_buffer = im2col_allocator.get();\n\n acl_tensor_ptr tmp_im2col_tensor =\n ggml_cann_create_tensor(tmp_im2col_buffer, ggml_cann_type_mapping(src1->type), ggml_type_size(src1->type),\n tmp_im2col_ne, tmp_im2col_nb, GGML_MAX_DIMS - 1, ACL_FORMAT_ND);\n\n std::vector kernel_dims = { KH, KW };\n std::vector dilation_size = { d1, d0 };\n std::vector padding_dims = { p1, p0 };\n std::vector stride_dims = { s1, s0 };\n acl_int_array_ptr kernel_size = ggml_cann_create_int_array(kernel_dims.data(), 2);\n acl_int_array_ptr dilations = ggml_cann_create_int_array(dilation_size.data(), 2);\n acl_int_array_ptr paddings = ggml_cann_create_int_array(padding_dims.data(), 2);\n acl_int_array_ptr strides = ggml_cann_create_int_array(stride_dims.data(), 2);\n GGML_CANN_CALL_ACLNN_OP(ctx, Im2col, acl_src1.get(), kernel_size.get(), dilations.get(), paddings.get(),\n strides.get(), tmp_im2col_tensor.get());\n\n // Cast if dst is f16.\n acl_tensor_ptr tmp_cast_tensor;\n ggml_cann_pool_alloc tmp_cast_allocator(ctx.pool());\n void * tmp_cast_buffer = nullptr;\n if (src1->type != dst->type) {\n tmp_cast_allocator.alloc(ggml_nbytes(dst) * n_bytes_factor);\n tmp_cast_buffer = tmp_cast_allocator.get();\n size_t temp_cast_nb[GGML_MAX_DIMS - 1];\n temp_cast_nb[0] = ggml_type_size(dst->type);\n for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n temp_cast_nb[i] = temp_cast_nb[i - 1] * tmp_im2col_ne[i - 1];\n }\n\n tmp_cast_tensor =\n ggml_cann_create_tensor(tmp_cast_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n tmp_im2col_ne, temp_cast_nb, GGML_MAX_DIMS - 1, ACL_FORMAT_ND);\n aclnn_cast(ctx, tmp_im2col_tensor.get(), tmp_cast_tensor.get(), ggml_cann_type_mapping(dst->type));\n }\n\n // post-processing\n if (is_2D) {\n ggml_cann_im2col_2d_post_process(ctx, dst, src1, tmp_cast_tensor.get(), tmp_im2col_tensor.get());\n } else {\n std::vector im2col_op_params = { KH, KW, IW, IC, N, OH, OW, s0, p0, d0, n_bytes_factor };\n ggml_cann_im2col_1d_post_process(ctx, dst, src1, tmp_cast_tensor.get(), tmp_im2col_tensor.get(),\n im2col_op_params);\n }\n}\n\n/**\n * @brief Applies element-wise exponential function to the elements of a tensor.\n *\n * This function computes the exponential of each element in the source tensor\n * `acl_src` and stores the result back into the same tensor.\n * The operation is defined as:\n * \\f[\n * \\text {acl_src }_i=e^{acl\\_src_i}\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The tensor on which the exponential function will be applied.\n */\nstatic void aclnn_exp(ggml_backend_cann_context & ctx, aclTensor * acl_src) {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceExp, acl_src);\n}\n\nvoid aclnn_cos(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n if (acl_dst == nullptr) {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceCos, acl_src);\n } else {\n GGML_CANN_CALL_ACLNN_OP(ctx, Cos, acl_src, acl_dst);\n }\n}\n\nvoid aclnn_sin(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n if (acl_dst == nullptr) {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSin, acl_src);\n } else {\n GGML_CANN_CALL_ACLNN_OP(ctx, Sin, acl_src, acl_dst);\n }\n}\n\nvoid ggml_cann_timestep_embedding(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src = dst->src[0];\n\n GGML_ASSERT(src->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n const int dim = dst->op_params[0];\n const int max_period = dst->op_params[1];\n int half = dim / 2;\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n\n // arange: [0, ..., half)\n float start = 0;\n float stop = half;\n float step = 1;\n int64_t n_elements_arange = half;\n int64_t tmp_arange_ne[] = { half };\n size_t tmp_arange_nb[] = { sizeof(dst->type) };\n\n ggml_cann_pool_alloc arange_allocator(ctx.pool(), half * sizeof(dst->type));\n void * tmp_arange_buffer = arange_allocator.get();\n acl_tensor_ptr tmp_arange_tensor =\n ggml_cann_create_tensor(tmp_arange_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n tmp_arange_ne, tmp_arange_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);\n\n aclnn_arange(ctx, tmp_arange_tensor.get(), start, stop, step, n_elements_arange);\n\n // freq\n float freq_param = -logf(max_period) / half;\n bool inplace = true;\n aclnn_muls(ctx, tmp_arange_tensor.get(), freq_param, nullptr, inplace);\n aclnn_exp(ctx, tmp_arange_tensor.get());\n\n // permute: src [0,1,2,3]->[0,1,3,2]\n int64_t tmp_permute_ne[] = { src->ne[1], src->ne[0], src->ne[2], src->ne[3] };\n size_t tmp_permute_nb[GGML_MAX_DIMS];\n tmp_permute_nb[0] = ggml_type_size(src->type);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n tmp_permute_nb[i] = tmp_permute_nb[i - 1] * tmp_permute_ne[i - 1];\n }\n\n ggml_cann_pool_alloc permute_allocator(ctx.pool(), ggml_nbytes(src));\n void * tmp_permute_buffer = permute_allocator.get();\n acl_tensor_ptr tmp_permute_tensor =\n ggml_cann_create_tensor(tmp_permute_buffer, ggml_cann_type_mapping(src->type), ggml_type_size(src->type),\n tmp_permute_ne, tmp_permute_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n int64_t permute_dim[] = { 0, 1, 3, 2 };\n int64_t num_dims = 4;\n aclnn_permute(ctx, acl_src.get(), tmp_permute_tensor.get(), permute_dim, num_dims);\n\n // timestep * freq\n int64_t tmp_mul_ne[] = { src->ne[1] * half, src->ne[0], src->ne[2], src->ne[3] };\n size_t tmp_mul_nb[GGML_MAX_DIMS];\n tmp_mul_nb[0] = ggml_type_size(src->type);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n tmp_mul_nb[i] = tmp_mul_nb[i - 1] * tmp_mul_ne[i - 1];\n }\n\n int mul_nelements = src->ne[1] * half * src->ne[0] * src->ne[2] * src->ne[3];\n\n ggml_cann_pool_alloc mul_allocator(ctx.pool(), mul_nelements * ggml_type_size(src->type));\n void * tmp_mul_buffer = mul_allocator.get();\n acl_tensor_ptr tmp_mul_tensor =\n ggml_cann_create_tensor(tmp_mul_buffer, ggml_cann_type_mapping(src->type), ggml_type_size(src->type),\n tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n aclnn_mul(ctx, tmp_permute_tensor.get(), tmp_arange_tensor.get(), tmp_mul_tensor.get());\n\n // cos\n ggml_cann_pool_alloc cos_allocator(ctx.pool(), mul_nelements * ggml_type_size(src->type));\n void * tmp_cos_buffer = cos_allocator.get();\n acl_tensor_ptr tmp_cos_tensor =\n ggml_cann_create_tensor(tmp_cos_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n\n aclnn_cos(ctx, tmp_mul_tensor.get(), tmp_cos_tensor.get());\n\n // sin\n ggml_cann_pool_alloc sin_allocator(ctx.pool(), mul_nelements * ggml_type_size(src->type));\n void * tmp_sin_buffer = sin_allocator.get();\n acl_tensor_ptr tmp_sin_tensor =\n ggml_cann_create_tensor(tmp_sin_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n\n aclnn_sin(ctx, tmp_mul_tensor.get(), tmp_sin_tensor.get());\n\n // concat\n int64_t concat_dim = 3;\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n acl_tensor_list_ptr tensor_list = ggml_cann_create_tensor_list(tmp_cos_tensor, tmp_sin_tensor);\n aclnn_concat(ctx, tensor_list.get(), acl_dst.get(), concat_dim);\n}\n\n/**\n * @brief Raises each element of a tensor to the power of the corresponding\n * element in another tensor.\n *\n * This function computes the element-wise power of the destination tensor\n * `acl_dst` raised to the power of the exponent tensor `acl_exp`.\n * The operation is defined as:\n * \\f[\n * \\text {acl_dst }_i=acl\\_dst_i^{\\text {acl_exp }_i}\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_dst The destination tensor, which also serves as the base tensor.\n * @param acl_exp The exponent tensor, each element of which is used to raise\n * the corresponding element in the destination tensor.\n */\nstatic void aclnn_pow_tensor_tensor(ggml_backend_cann_context & ctx, aclTensor * acl_dst, aclTensor * acl_exp) {\n GGML_CANN_CALL_ACLNN_OP(ctx, InplacePowTensorTensor, acl_dst, acl_exp);\n}\n\n/**\n * @brief Generate a range of values and apply a scalar base exponentiation.\n *\n * This function creates an evenly spaced sequence from `start` to `stop` (exclusive),\n * with step size `step`, stores it in a temporary buffer, and then computes:\n *\n * @f[\n * slope[i] = m^{\\left( start + i \\cdot step \\right)}, \\quad 0 \\le i < size\n * @f]\n *\n * The results are written to the provided @p slope_buffer.\n *\n * @param ctx CANN backend context for memory allocation and operator execution.\n * @param slope_buffer Pointer to the output buffer (float array) for the computed slope values.\n * @param m Scalar base for the exponentiation.\n * @param size Number of elements in the generated sequence.\n * @param start Starting exponent offset.\n * @param stop Stopping exponent offset (exclusive).\n * @param step Step size for the exponent increment.\n * @param dtype Data type for slope tensor.\n */\nstatic void aclnn_get_slope_inner(ggml_backend_cann_context & ctx,\n void * slope_buffer,\n float m,\n int64_t size,\n float start,\n float stop,\n float step,\n ggml_type dtype) {\n aclDataType acl_type = ggml_cann_type_mapping(dtype);\n size_t type_size = ggml_type_size(dtype);\n\n int64_t ne[] = { size };\n size_t nb[] = { type_size };\n\n ggml_cann_pool_alloc arange_allocator(ctx.pool(), size * type_size);\n void * arange_buffer = arange_allocator.get();\n\n acl_tensor_ptr arange_tensor = ggml_cann_create_tensor(arange_buffer, acl_type, type_size, ne, nb, 1);\n aclnn_arange(ctx, arange_tensor.get(), start, stop, step, size);\n\n acl_tensor_ptr slope_tensor = ggml_cann_create_tensor(slope_buffer, acl_type, type_size, ne, nb, 1);\n\n acl_scalar_ptr sc = ggml_cann_create_scalar(&m, aclDataType::ACL_FLOAT);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, PowScalarTensor, sc.get(), arange_tensor.get(), slope_tensor.get());\n}\n\n/**\n * @brief Compute slope values for multiple attention heads based on ALiBi bias parameters.\n *\n * This function generates slope values for each attention head according to the ALiBi\n * (Attention with Linear Biases) method. It splits the computation into two ranges depending\n * on whether the head index is less than @p n_head_log2 or not, and uses different base values\n * (`m0` and `m1`) for the exponentiation.\n *\n * @f[\n * slope[h] =\n * \\begin{cases}\n * m_0^{(h + 1)}, & h < n\\_head\\_log2 \\\\\n * m_1^{\\left( 2 \\cdot (h - n\\_head\\_log2) + 1 \\right)}, & h \\geq n\\_head\\_log2\n * \\end{cases}\n * \\quad , \\quad \\text{if } max\\_bias > 0\n * @f]\n *\n * If @p max_bias <= 0, all slope values are set to 1.0.\n *\n * @param ctx CANN backend context for memory allocation and operator execution.\n * @param n_head Total number of attention heads.\n * @param slope_buffer Pointer to the output buffer (float array) for storing slopes.\n * @param max_bias Maximum bias value for slope computation.\n * @param dtype Data type for slope tensor.\n *\n*/\nstatic void aclnn_get_slope(ggml_backend_cann_context & ctx,\n int64_t n_head,\n void * slope_buffer,\n float max_bias,\n ggml_type dtype) {\n const int n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n\n float m0 = powf(2.0f, -(max_bias) / n_head_log2);\n float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n // const float slope = (max_bias > 0.0f) ?\n // h < n_head_log2 ?\n // powf(m0, h + 1) :\n // powf(m1, 2*(h - n_head_log2) + 1) :\n // 1.0f;\n // arange1\n float start = 0 + 1;\n float end = (n_head_log2 - 1) + 1;\n float step = 1;\n float count = n_head_log2;\n // end needs to be +1 because aclnn uses a left-closed, right-open interval.\n aclnn_get_slope_inner(ctx, slope_buffer, m0, count, start, end + 1, step, dtype);\n if (n_head_log2 < n_head) {\n // arange2\n start = 2 * (n_head_log2 - n_head_log2) + 1;\n end = 2 * ((n_head - 1) - n_head_log2) + 1;\n step = 2;\n count = n_head - n_head_log2;\n aclnn_get_slope_inner(ctx, (char *) slope_buffer + n_head_log2 * sizeof(float), m1, count, start, end + 1, step,\n dtype);\n }\n}\n\n/**\n * @brief Add ALiBi (Attention with Linear Biases) positional biases to the attention mask.\n *\n * This function computes the ALiBi slopes for each attention head (if max_bias > 0),\n * multiplies them with the attention mask to produce bias tensors, and adds these biases\n * to the destination tensor (@p dst).\n *\n * The function performs necessary broadcasting of the mask and slope tensors to match\n * the shape of the destination tensor, then applies element-wise multiplication and addition\n * using CANN operators.\n *\n * @param ctx CANN backend context for memory management and operator execution.\n * @param mask Input attention mask tensor, assumed to be contiguous.\n * @param dst Destination tensor to which ALiBi biases will be added.\n * @param dst_ptr Pointer to the memory of the destination tensor.\n * @param max_bias Maximum bias value controlling the slope scaling.\n *\n * @note\n * - Write data into dst_ptr using only the shape information of the dst tensor.\n * - `GGML_MAX_DIMS + 2` is used to extend tensor dimensions for broadcasting.\n */\nstatic void aclnn_add_alibi(ggml_backend_cann_context & ctx,\n ggml_tensor * mask,\n ggml_tensor * dst,\n void * dst_ptr,\n float max_bias) {\n void * slope_buffer = nullptr;\n void * bias_buffer = nullptr;\n\n if (max_bias > 0.0f) {\n int64_t n_heads = dst->ne[2];\n ggml_cann_pool_alloc slope_allocator(ctx.pool(), n_heads * sizeof(float));\n slope_buffer = slope_allocator.get();\n ggml_cann_pool_alloc bias_allocator(ctx.pool(), ggml_nelements(dst) * ggml_element_size(dst));\n bias_buffer = bias_allocator.get();\n aclnn_get_slope(ctx, n_heads, slope_buffer, max_bias, GGML_TYPE_F32);\n }\n\n // broadcast for mask, slop and dst;\n int64_t nr2 = dst->ne[2] / mask->ne[2];\n int64_t nr3 = dst->ne[3] / mask->ne[3];\n\n // broadcast the mask across rows\n int64_t mask_ne[] = { mask->ne[0], dst->ne[1], mask->ne[2], 1, mask->ne[3], 1 };\n size_t mask_nb[] = { mask_nb[0] = mask->nb[0], mask_nb[1] = mask->nb[1], mask_nb[2] = mask->nb[2],\n mask_nb[3] = mask->nb[2], mask_nb[4] = mask->nb[3], mask_nb[5] = mask->nb[3] };\n\n int64_t dst_ne[] = { dst->ne[0], dst->ne[1], mask->ne[2], nr2, mask->ne[3], nr3 };\n size_t dst_nb[] = { dst_nb[0] = dst->nb[0], dst_nb[1] = dst->nb[1], dst_nb[2] = dst->nb[2],\n dst_nb[3] = dst->nb[2], dst_nb[4] = dst->nb[3], dst_nb[5] = dst->nb[3] };\n\n // slope is a 1 dim tensor, slope.ne2 == dst.ne2\n int64_t slope_ne[] = { 1, 1, mask->ne[2], nr2, 1, 1 };\n size_t slope_nb[GGML_MAX_DIMS + 2];\n slope_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS + 2; i++) {\n slope_nb[i] = slope_nb[i - 1] * slope_ne[i - 1];\n }\n\n acl_tensor_ptr acl_slope =\n ggml_cann_create_tensor(slope_buffer, ACL_FLOAT, sizeof(float), slope_ne, slope_nb, GGML_MAX_DIMS + 2);\n acl_tensor_ptr acl_mask = ggml_cann_create_tensor(mask, mask_ne, mask_nb, GGML_MAX_DIMS + 2);\n\n // write data into dst_ptr using only the shape information of the dst tensor.\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst_ptr, ggml_cann_type_mapping(dst->type),\n ggml_type_size(dst->type), dst_ne, dst_nb, GGML_MAX_DIMS + 2);\n\n if (max_bias > 0.0f) {\n int64_t bias_ne[] = { mask->ne[0], dst->ne[1], mask->ne[2], nr2, mask->ne[3], 1 };\n size_t bias_nb[GGML_MAX_DIMS + 2];\n bias_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS + 2; i++) {\n bias_nb[i] = bias_nb[i - 1] * bias_ne[i - 1];\n }\n acl_tensor_ptr bias_tensor =\n ggml_cann_create_tensor(bias_buffer, ACL_FLOAT, sizeof(float), bias_ne, bias_nb, GGML_MAX_DIMS + 2);\n\n aclnn_mul(ctx, acl_slope.get(), acl_mask.get(), bias_tensor.get());\n aclnn_add(ctx, acl_dst.get(), bias_tensor.get());\n } else {\n aclnn_add(ctx, acl_dst.get(), acl_mask.get());\n }\n}\n\nvoid ggml_cann_cpy(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_cann_dup(ctx, dst);\n}\n\n/**\n * @brief Applies the softmax function to a tensor along a specified dimension.\n *\n * This function computes the softmax of the source tensor `acl_src` along the\n * specified dimension `dim` and stores the result in the destination tensor\n * `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor on which the softmax function will be\n * applied.\n * @param dim The dimension along which the softmax function will be computed.\n * @param acl_dst The destination tensor where the softmax results will be\n * stored.\n */\nstatic void aclnn_softmax(ggml_backend_cann_context & ctx, aclTensor * acl_src, int64_t dim, aclTensor * acl_dst) {\n GGML_CANN_CALL_ACLNN_OP(ctx, Softmax, acl_src, dim, acl_dst);\n}\n\nvoid ggml_cann_softmax(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n ggml_tensor * src1 = dst->src[1]; // mask\n\n acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n\n memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));\n\n // input mul scale\n acl_scalar_ptr acl_scale = ggml_cann_create_scalar(&scale, aclDataType::ACL_FLOAT);\n ggml_cann_pool_alloc src_tensor_allocator(ctx.pool(), ggml_nbytes(src0));\n void * src_tensor_buffer = src_tensor_allocator.get();\n acl_tensor_ptr softmax_tensor = ggml_cann_create_tensor(src_tensor_buffer, ggml_cann_type_mapping(src0->type),\n ggml_element_size(src0), src0->ne, src0->nb, GGML_MAX_DIMS);\n\n aclnn_muls(ctx, acl_src0.get(), scale, softmax_tensor.get(), false);\n\n // mask\n if (src1) {\n aclnn_add_alibi(ctx, src1, src0, src_tensor_buffer, max_bias);\n }\n // softmax\n aclnn_softmax(ctx, softmax_tensor.get(), 3, acl_dst.get());\n}\n\n/**\n * @brief Performs index select operation on a 4D tensor using the CANN backend.\n *\n * This function applies the `IndexSelect` operation along a specific dimension\n * of the source tensor (`src_buffer`) using the indices from the index tensor (`index`).\n * It iterates over the last two dimensions of the source tensor, creates the corresponding\n * CANN tensors for the source, index, and output slices, and executes the `IndexSelect`\n * operation for each slice.\n *\n * @param ctx The context for CANN backend operations.\n * @param src_buffer The source buffer containing the 4D input tensor data.\n * @param src_ne The dimensions of the source tensor.\n * @param src_nb The strides (byte offsets) of the source tensor.\n * @param dst_buffer The destination buffer where the output tensor data will be written.\n * @param dst_ne The dimensions of the destination tensor.\n * @param dst_nb The strides (byte offsets) of the destination tensor.\n * @param index The index tensor specifying the indices to select from the source tensor.\n * @param type The data type of the source and destination tensors.\n */\nstatic void aclnn_index_select_4d(ggml_backend_cann_context & ctx,\n void * src_buffer,\n int64_t * src_ne,\n size_t * src_nb,\n void * dst_buffer,\n int64_t * dst_ne,\n size_t * dst_nb,\n ggml_tensor * index,\n ggml_type type) {\n for (int64_t i = 0; i < src_ne[3]; i++) {\n for (int64_t j = 0; j < src_ne[2]; j++) {\n // src\n acl_tensor_ptr acl_src_tensor =\n ggml_cann_create_tensor((char *) src_buffer + i * src_nb[3] + j * src_nb[2],\n ggml_cann_type_mapping(type), ggml_type_size(type), src_ne, src_nb, 2);\n\n // index\n acl_tensor_ptr acl_index = ggml_cann_create_tensor(\n (char *) index->data + (i % index->ne[2]) * index->nb[2] + (j % index->ne[1]) * index->nb[1],\n ggml_cann_type_mapping(index->type), ggml_element_size(index), index->ne, index->nb, 1);\n\n // out\n acl_tensor_ptr acl_out =\n ggml_cann_create_tensor((char *) dst_buffer + i * dst_nb[3] + j * dst_nb[2],\n ggml_cann_type_mapping(type), ggml_type_size(type), dst_ne, dst_nb, 2);\n GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, acl_src_tensor.get(), 0, acl_index.get(), acl_out.get());\n }\n }\n}\n\n/**\n * @brief Performs inplace index copy operation on a 4D tensor using the CANN backend.\n *\n * This function applies the `IndexCopy` operation along a specific dimension of the\n * destination tensor (`dst_buffer`) by copying elements from the source tensor (`src_buffer`)\n * to positions specified by the index tensor (`index`).\n * It iterates over the last two dimensions of the tensors, creates the corresponding\n * CANN tensors for source, index, and destination slices, and performs the index copy\n * operation for each slice.\n *\n * @param ctx The context for CANN backend operations.\n * @param src_buffer The source buffer containing the 4D input tensor data to be copied.\n * @param src_ne The dimensions of the source tensor.\n * @param src_nb The strides (byte offsets) of the source tensor.\n * @param dst_buffer The destination buffer where values will be copied to.\n * @param dst_ne The dimensions of the destination tensor.\n * @param dst_nb The strides (byte offsets) of the destination tensor.\n * @param index The index tensor specifying target positions in the destination tensor.\n * @param type The data type of the source and destination tensors.\n */\nstatic void aclnn_index_copy_4d(ggml_backend_cann_context & ctx,\n void * src_buffer,\n int64_t * src_ne,\n size_t * src_nb,\n void * dst_buffer,\n int64_t * dst_ne,\n size_t * dst_nb,\n ggml_tensor * index,\n ggml_type type) {\n for (int64_t i = 0; i < src_ne[3]; i++) {\n for (int64_t j = 0; j < src_ne[2]; j++) {\n // src\n acl_tensor_ptr acl_src_tensor =\n ggml_cann_create_tensor((char *) src_buffer + i * src_nb[3] + j * src_nb[2],\n ggml_cann_type_mapping(type), ggml_type_size(type), src_ne, src_nb, 2);\n\n // index\n acl_tensor_ptr acl_index = ggml_cann_create_tensor(\n (char *) index->data + (i % index->ne[2]) * index->nb[2] + (j % index->ne[1]) * index->nb[1],\n ggml_cann_type_mapping(index->type), ggml_element_size(index), index->ne, index->nb, 1);\n\n // out\n acl_tensor_ptr acl_out =\n ggml_cann_create_tensor((char *) dst_buffer + i * dst_nb[3] + j * dst_nb[2],\n ggml_cann_type_mapping(type), ggml_type_size(type), dst_ne, dst_nb, 2);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceIndexCopy, acl_out.get(), 0, acl_index.get(), acl_src_tensor.get());\n }\n }\n}\n\nvoid ggml_cann_get_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0]; // src\n ggml_tensor * src1 = dst->src[1]; // index\n\n GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n\n switch (src0->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_F32:\n if (src0->type == dst->type) {\n aclnn_index_select_4d(ctx, src0->data, src0->ne, src0->nb, dst->data, dst->ne, dst->nb, src1,\n dst->type);\n } else {\n acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0);\n ggml_cann_pool_alloc src_buffer_allocator(ctx.pool(), ggml_nelements(src0) * ggml_element_size(dst));\n void * src_trans_buffer = src_buffer_allocator.get();\n size_t src_trans_nb[GGML_MAX_DIMS];\n src_trans_nb[0] = dst->nb[0];\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];\n }\n acl_tensor_ptr src_trans_tensor =\n ggml_cann_create_tensor(src_trans_buffer, ggml_cann_type_mapping(dst->type),\n ggml_type_size(dst->type), src0->ne, src_trans_nb, GGML_MAX_DIMS);\n aclnn_cast(ctx, acl_src0.get(), src_trans_tensor.get(), ggml_cann_type_mapping(dst->type));\n aclnn_index_select_4d(ctx, src_trans_buffer, src0->ne, src_trans_nb, dst->data, dst->ne, dst->nb, src1,\n dst->type);\n }\n break;\n case GGML_TYPE_Q8_0:\n {\n // add 1 dim for bcast mul.\n size_t weight_nb[GGML_MAX_DIMS + 1], scale_nb[GGML_MAX_DIMS + 1], dequant_nb[GGML_MAX_DIMS + 1];\n int64_t weight_ne[GGML_MAX_DIMS + 1], scale_ne[GGML_MAX_DIMS + 1], *dequant_ne;\n int64_t scale_offset = 0;\n // [3,4,5,64] -> [3,4,5,2,32]\n weight_ne[0] = QK8_0;\n weight_ne[1] = src0->ne[0] / QK8_0;\n weight_nb[0] = sizeof(int8_t);\n weight_nb[1] = weight_nb[0] * weight_ne[0];\n for (int i = 2; i < GGML_MAX_DIMS + 1; i++) {\n weight_ne[i] = src0->ne[i - 1];\n weight_nb[i] = weight_nb[i - 1] * weight_ne[i - 1];\n }\n // [3,4,5,64] -> [3,4,5,2,1]\n scale_ne[0] = 1;\n scale_ne[1] = src0->ne[0] / QK8_0;\n scale_nb[0] = sizeof(uint16_t);\n scale_nb[1] = scale_nb[0] * scale_ne[0];\n for (int i = 2; i < GGML_MAX_DIMS + 1; i++) {\n scale_ne[i] = src0->ne[i - 1];\n scale_nb[i] = scale_nb[i - 1] * scale_ne[i - 1];\n }\n // [3,4,5,64] -> [3,4,5,2,32]\n dequant_ne = weight_ne;\n dequant_nb[0] = ggml_type_size(dst->type);\n for (int i = 1; i < GGML_MAX_DIMS + 1; i++) {\n dequant_nb[i] = dequant_nb[i - 1] * dequant_ne[i - 1];\n }\n scale_offset = ggml_nelements(src0) * sizeof(int8_t);\n ggml_cann_pool_alloc dequant_buffer_allocator(ctx.pool(),\n ggml_nelements(src0) * ggml_type_size(dst->type));\n acl_tensor_ptr acl_weight_tensor = ggml_cann_create_tensor(src0->data, ACL_INT8, sizeof(int8_t),\n weight_ne, weight_nb, GGML_MAX_DIMS + 1);\n acl_tensor_ptr acl_scale_tensor =\n ggml_cann_create_tensor(src0->data, ACL_FLOAT16, sizeof(uint16_t), scale_ne, scale_nb,\n GGML_MAX_DIMS + 1, ACL_FORMAT_ND, scale_offset);\n acl_tensor_ptr dequant_tensor =\n ggml_cann_create_tensor(dequant_buffer_allocator.get(), ggml_cann_type_mapping(dst->type),\n ggml_type_size(dst->type), dequant_ne, dequant_nb, GGML_MAX_DIMS + 1);\n aclnn_mul(ctx, acl_weight_tensor.get(), acl_scale_tensor.get(), dequant_tensor.get());\n dequant_nb[0] = ggml_type_size(dst->type);\n dequant_ne = src0->ne;\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n dequant_nb[i] = dequant_nb[i - 1] * src0->ne[i - 1];\n }\n aclnn_index_select_4d(ctx, dequant_buffer_allocator.get(), dequant_ne, dequant_nb, dst->data, dst->ne,\n dst->nb, src1, dst->type);\n break;\n }\n default:\n GGML_ABORT(\"Unsupported tensor type for GGML_OP_GET_ROWS\");\n break;\n }\n}\n\nvoid ggml_cann_set_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0]; // src\n ggml_tensor * src1 = dst->src[1]; // index\n\n switch (dst->type) {\n case GGML_TYPE_F32:\n {\n aclnn_index_copy_4d(ctx, src0->data, src0->ne, src0->nb, dst->data, dst->ne, dst->nb, src1, dst->type);\n break;\n }\n case GGML_TYPE_F16:\n {\n acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0);\n ggml_cann_pool_alloc src_buffer_allocator(ctx.pool(), ggml_nelements(src0) * sizeof(uint16_t));\n void * src_trans_buffer = src_buffer_allocator.get();\n size_t src_trans_nb[GGML_MAX_DIMS];\n src_trans_nb[0] = sizeof(uint16_t);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];\n }\n acl_tensor_ptr src_trans_tensor = ggml_cann_create_tensor(\n src_trans_buffer, ACL_FLOAT16, ggml_type_size(dst->type), src0->ne, src_trans_nb, GGML_MAX_DIMS);\n aclnn_cast(ctx, acl_src0.get(), src_trans_tensor.get(), ggml_cann_type_mapping(dst->type));\n aclnn_index_copy_4d(ctx, src_trans_buffer, src0->ne, src_trans_nb, dst->data, dst->ne, dst->nb, src1,\n dst->type);\n break;\n }\n default:\n GGML_ABORT(\"Unsupported tensor type for GGML_OP_SET_ROWS\");\n break;\n }\n}\n\n/**\n * @brief Repeats elements of a tensor along a specified dimension.\n *\n * This function repeats each element of the source tensor `acl_src` a specified\n * number of times (`repeats`) along the specified dimension `dim` and stores\n * the result in the destination tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose elements will be repeated.\n * @param acl_dst The destination tensor where the repeated elements will be\n * stored.\n * @param dim The dimension along which the elements will be repeated.\n * @param repeats The number of times each element will be repeated.\n * @param output_size The size of the output tensor.\n */\nstatic void aclnn_repeat_interleave(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n aclTensor * acl_dst,\n int64_t dim,\n int64_t repeats,\n int64_t output_size) {\n GGML_CANN_CALL_ACLNN_OP(ctx, RepeatInterleaveIntWithDim, acl_src, repeats, dim, output_size, acl_dst);\n}\n\n/**\n * @brief Performs matrix multiplication with floating-point precision on\n * tensors using the CANN backend.\n *\n * This function performs matrix multiplication of the input tensor and the\n * weight tensor, handling broadcasting and transposing as needed, and stores\n * the result in the destination tensor `dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the result of the matrix\n * multiplication will be stored.\n */\nstatic void ggml_cann_mat_mul_fp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * weight = dst->src[0]; // weight\n ggml_tensor * input = dst->src[1]; // input\n\n // when weight ne2 or ne3 is 1, aclnnMatmulGetWorkspaceSize will auto\n // broadcast, when weight ne2 or ne3 is not 1, weight need repeat.\n BCAST_MUL_MAT_SHAPE(input, weight, dst);\n\n int64_t n_dims = bcast_dims;\n if (bcast_input_ne[3] == bcast_weight_ne[3] && bcast_input_ne[3] == 1) {\n if (bcast_input_ne[2] == 1 && bcast_weight_ne[2] == 1) {\n n_dims = 2;\n } else if (bcast_input_ne[2] == 1) {\n n_dims = 3;\n }\n }\n\n acl_tensor_ptr acl_input_tensor = ggml_cann_create_tensor(input, bcast_input_ne, bcast_input_nb, n_dims);\n int64_t transpose_ne[] = { bcast_weight_ne[1], bcast_weight_ne[0], bcast_weight_ne[2],\n bcast_weight_ne[3], bcast_weight_ne[4], bcast_weight_ne[5] };\n size_t transpose_nb[] = { bcast_weight_nb[1], bcast_weight_nb[0], bcast_weight_nb[2],\n bcast_weight_nb[3], bcast_weight_nb[4], bcast_weight_nb[5] };\n acl_tensor_ptr acl_weight_tensor;\n\n // Only check env once.\n static bool weight_to_nz = parse_bool(get_env_as_lowercase(\"GGML_CANN_WEIGHT_NZ\").value_or(\"on\"));\n if (weight_to_nz && is_matmul_weight(weight)) {\n acl_weight_tensor = ggml_cann_create_tensor(weight, transpose_ne, transpose_nb, n_dims, ACL_FORMAT_FRACTAL_NZ);\n } else {\n acl_weight_tensor = ggml_cann_create_tensor(weight, transpose_ne, transpose_nb, n_dims, ACL_FORMAT_ND);\n }\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, bcast_dst_ne, bcast_dst_nb, n_dims);\n\n switch (n_dims) {\n case 2:\n GGML_CANN_CALL_ACLNN_OP(ctx, Mm, acl_input_tensor.get(), acl_weight_tensor.get(), acl_dst.get(), 2);\n break;\n case 3:\n GGML_CANN_CALL_ACLNN_OP(ctx, BatchMatMul, acl_input_tensor.get(), acl_weight_tensor.get(), acl_dst.get(),\n 2);\n break;\n default:\n // ALLOW_FP32_DOWN_PRECISION, when input is\n // fp32, atlas a2 will transpose it to HFLOAT32.\n GGML_CANN_CALL_ACLNN_OP(ctx, Matmul, acl_input_tensor.get(), acl_weight_tensor.get(), acl_dst.get(), 1);\n break;\n }\n}\n\n/**\n * @brief Performs matrix multiplication with quantized weights and\n * floating-point inputs using the CANN backend.\n *\n * This function performs matrix multiplication of the input tensor `src1` and\n * the weight tensor `src0`, handling broadcasting, transposing, and\n * quantization as needed, and stores the result in the destination tensor\n * `dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the result of the matrix\n * multiplication will be stored.\n */\nstatic void ggml_cann_mul_mat_quant(ggml_backend_cann_context & ctx, ggml_tensor * dst, const enum ggml_type type) {\n ggml_tensor * src0 = dst->src[0]; // weight\n ggml_tensor * src1 = dst->src[1]; // input\n\n // The shape of the weight is NCHW.\n // Matrix multiplication uses HW dims.\n // HC is regarded as batch.\n // weight need transpose.\n float weight_elem_size;\n if (type == GGML_TYPE_Q4_0) {\n weight_elem_size = float(sizeof(uint8_t)) / 2;\n } else if (type == GGML_TYPE_Q8_0) {\n weight_elem_size = float(sizeof(uint8_t));\n } else {\n GGML_ABORT(\"Only support Q4_0 and Q8_0 MUL_MAT\");\n }\n float weight_nb[] = { src0->ne[0] * weight_elem_size, weight_elem_size };\n size_t weight_stride = src0->ne[1] * src0->ne[0] * weight_elem_size;\n size_t weight_size = weight_stride * src0->ne[2] * src0->ne[3];\n\n // scale stored at the end of weight. Also need transpose.\n size_t scale_elem_size = sizeof(uint16_t);\n size_t scale_nb[] = { src0->ne[0] / QK8_0 * scale_elem_size, scale_elem_size };\n size_t scale_stride = src0->ne[1] * src0->ne[0] / QK8_0 * scale_elem_size;\n char * scale_offset = (char *) src0->data + weight_size;\n\n // input\n size_t input_elem_size = sizeof(uint16_t);\n int64_t input_ne[] = { src1->ne[0], src1->ne[1] };\n size_t input_nb[] = { input_elem_size, input_ne[0] * input_elem_size };\n size_t input_stride = input_ne[0] * input_ne[1] * input_elem_size;\n ggml_cann_pool_alloc input_alloctor(ctx.pool());\n void * input_buffer = src1->data;\n\n // case in\n if (src1->type != GGML_TYPE_F16) {\n acl_tensor_ptr acl_src1_tensor = ggml_cann_create_tensor(src1);\n input_buffer = input_alloctor.alloc(ggml_nelements(src1) * input_elem_size);\n\n int64_t * input_cast_ne = src1->ne;\n size_t input_cast_nb[GGML_MAX_DIMS];\n input_cast_nb[0] = sizeof(uint16_t);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n input_cast_nb[i] = input_cast_nb[i - 1] * input_cast_ne[i - 1];\n }\n\n acl_tensor_ptr acl_input_tensor = ggml_cann_create_tensor(input_buffer, ACL_FLOAT16, input_elem_size,\n input_cast_ne, input_cast_nb, GGML_MAX_DIMS);\n aclnn_cast(ctx, acl_src1_tensor.get(), acl_input_tensor.get(), ACL_FLOAT16);\n }\n\n // output\n size_t output_elem_size = sizeof(uint16_t);\n size_t output_nb[] = { output_elem_size, dst->ne[0] * output_elem_size };\n ggml_cann_pool_alloc output_allocator(ctx.pool());\n void * output_buffer = output_allocator.alloc(ggml_nelements(dst) * output_elem_size);\n size_t output_stride = dst->ne[0] * dst->ne[1] * output_elem_size;\n\n // aclnn\n int64_t max_elem_size = 65535;\n int64_t split_size = (src0->ne[1] / max_elem_size) + 1;\n ggml_cann_pool_alloc workspace_allocator(ctx.pool());\n for (int64_t n1 = 0; n1 < src1->ne[3]; n1++) {\n for (int64_t c1 = 0; c1 < src1->ne[2]; c1++) {\n int64_t n0 = n1 / (src1->ne[3] / src0->ne[3]);\n int64_t c0 = c1 / (src1->ne[2] / src0->ne[2]);\n\n int64_t batch1 = (n1 * src1->ne[2]) + c1;\n int64_t batch0 = (n0 * src0->ne[2]) + c0;\n\n acl_tensor_ptr acl_input_tensor = ggml_cann_create_tensor(\n (char *) input_buffer + batch1 * input_stride, ACL_FLOAT16, input_elem_size, input_ne, input_nb, 2);\n\n // first split\n int64_t weight_ne_offset = 0;\n int64_t weight_ne[2] = { max_elem_size > src0->ne[1] ? src0->ne[1] : max_elem_size, src0->ne[0] };\n int64_t scale_ne_offset = 0;\n int64_t scale_ne[2] = { weight_ne[0], weight_ne[1] / QK8_0 };\n int64_t output_ne_offset = 0;\n int64_t output_ne[2] = { weight_ne[0], dst->ne[1] };\n\n acl_tensor_ptr acl_weight_tensor =\n ggml_cann_create_tensor((char *) src0->data + batch0 * weight_stride, ggml_cann_type_mapping(type),\n weight_elem_size, weight_ne, weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);\n acl_tensor_ptr acl_scale_tensor =\n ggml_cann_create_tensor(scale_offset + batch0 * scale_stride, ACL_FLOAT16, scale_elem_size, scale_ne,\n scale_nb, 2, ACL_FORMAT_ND, scale_ne_offset);\n acl_tensor_ptr acl_output_tensor =\n ggml_cann_create_tensor((char *) output_buffer + batch1 * output_stride, ACL_FLOAT16, output_elem_size,\n output_ne, output_nb, 2, ACL_FORMAT_ND, output_ne_offset);\n int64_t antiquantGroupSize = 0;\n if (src0->ne[0] > QK8_0) {\n antiquantGroupSize = QK8_0;\n }\n GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, acl_input_tensor.get(), acl_weight_tensor.get(),\n acl_scale_tensor.get(), nullptr, nullptr, nullptr, nullptr, antiquantGroupSize,\n acl_output_tensor.get());\n\n // other splits\n for (int64_t split = 1; split < split_size; split++) {\n weight_ne_offset += weight_elem_size * weight_ne[0] * weight_ne[1];\n weight_ne[0] =\n max_elem_size * (split + 1) > src0->ne[1] ? src0->ne[1] - (max_elem_size * split) : max_elem_size;\n scale_ne_offset += scale_elem_size * scale_ne[0] * scale_ne[1];\n scale_ne[0] = weight_ne[0];\n output_ne_offset += output_elem_size * output_ne[0] * output_ne[1];\n output_ne[0] = weight_ne[0];\n\n acl_weight_tensor =\n ggml_cann_create_tensor((char *) src0->data + batch0 * weight_stride, ggml_cann_type_mapping(type),\n weight_elem_size, weight_ne, weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);\n acl_scale_tensor =\n ggml_cann_create_tensor(scale_offset + batch0 * scale_stride, ACL_FLOAT16, scale_elem_size,\n scale_ne, scale_nb, 2, ACL_FORMAT_ND, scale_ne_offset);\n acl_output_tensor =\n ggml_cann_create_tensor((char *) output_buffer + batch1 * output_stride, ACL_FLOAT16,\n output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND, output_ne_offset);\n GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, acl_input_tensor.get(), acl_weight_tensor.get(),\n acl_scale_tensor.get(), nullptr, nullptr, nullptr, nullptr, antiquantGroupSize,\n acl_output_tensor.get());\n }\n }\n }\n\n // cast out\n if (dst->type != GGML_TYPE_F16) {\n int64_t * output_cast_ne = dst->ne;\n size_t output_cast_nb[GGML_MAX_DIMS];\n output_cast_nb[0] = sizeof(uint16_t);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n output_cast_nb[i] = output_cast_nb[i - 1] * output_cast_ne[i - 1];\n }\n\n acl_tensor_ptr acl_output_tensor = ggml_cann_create_tensor(output_buffer, ACL_FLOAT16, output_elem_size,\n output_cast_ne, output_cast_nb, GGML_MAX_DIMS);\n acl_tensor_ptr acl_dst_tensor = ggml_cann_create_tensor(dst);\n aclnn_cast(ctx, acl_output_tensor.get(), acl_dst_tensor.get(), ggml_cann_type_mapping(dst->type));\n }\n}\n\nvoid ggml_cann_mul_mat(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n const enum ggml_type type = dst->src[0]->type;\n switch (type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n ggml_cann_mat_mul_fp(ctx, dst);\n break;\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q8_0:\n ggml_cann_mul_mat_quant(ctx, dst, type);\n break;\n default:\n GGML_ABORT(\"Unsupported type for mul_mat\");\n break;\n }\n}\n\n/**\n * @brief Rolls the elements of a tensor along a specified dimension.\n *\n * This function rolls the elements of the source tensor `acl_src` by the\n * specified shifts `shifts` along the specified dimensions `dims`, and stores\n * the result in the destination tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose elements will be rolled.\n * @param acl_dst The destination tensor where the rolled elements will be\n * stored.\n * @param shifts An array specifying the number of positions by which elements\n * are shifted.\n * @param dims An array specifying the dimensions along which elements are\n * shifted.\n */\nstatic void aclnn_roll(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n aclTensor * acl_dst,\n int64_t * shifts,\n int64_t * dims) {\n acl_int_array_ptr acl_shifts = ggml_cann_create_int_array(shifts, 1);\n acl_int_array_ptr acl_dims = ggml_cann_create_int_array(dims, 1);\n GGML_CANN_CALL_ACLNN_OP(ctx, Roll, acl_src, acl_shifts.get(), acl_dims.get(), acl_dst);\n}\n\n/**\n * @brief Fills specified positions of a tensor with a scalar value.\n *\n * This function fills the positions in the source tensor `acl_src` specified by\n * `index` along the dimension `dim` with the scalar value `value`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor where the positions will be filled.\n * @param dim The dimension along which the positions are specified.\n * @param index An array specifying the positions to be filled.\n * @param index_num The number of positions specified in the index array.\n * @param value The scalar value used to fill the specified positions.\n */\nstatic void aclnn_index_fill_tensor(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n int64_t dim,\n int64_t * index,\n int64_t index_num,\n float value) {\n acl_int_array_ptr acl_index = ggml_cann_create_int_array(index, index_num);\n acl_scalar_ptr acl_value = ggml_cann_create_scalar(&value, aclDataType::ACL_FLOAT);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceIndexFillTensor, acl_src, dim, acl_index.get(), acl_value.get());\n}\n\n/**\n * @brief Initializes and caches all intermediate tensors required for RoPE\n * (Rotary Position Embedding), including support for Yarn, mRoPE,\n * i-mRoPE, Neox repeat strategy, independent sectors, frequency factors，\n * and multi-section rotary groups.\n *\n * This function computes and caches the per-dimension θ coefficients used for\n * Q/K rotary embedding. The cache is shared across layers, and recomputed only\n * when any dependent parameter changes.\n *\n * The function now supports:\n * - Yarn RoPE extrapolation (via @param corr_dims and @param ext_factor)\n * - Per-dimension independent sector exponent rules (indep_sects + sections[])\n * - Multi-section RoPE (mRoPE) index mapping (mrope_used + is_imrope)\n * - Frequency factor division (src2)\n * - Neox / normal repeat expansion modes\n *\n * @param ctx CANN backend context, containing memory pool,\n * cached buffers, and runtime stream.\n * @param dst Destination ggml_tensor whose computation\n * depends on RoPE (typically Qcur or Kcur).\n * @param corr_dims [low, high] Yarn correction range.\n * @param ext_factor Yarn extrapolation strength. 0 = disabled.\n * @param theta_scale Base multiplier for per-dimension θ exponent.\n * @param freq_scale Global frequency scaling factor.\n * @param attn_factor Optional scaling applied to sin/cos (if needed).\n * @param is_neox Whether to use Neox-style dimension interleave.\n * @param sections 4-way sector sizes for independent-section RoPE\n * and multi-section mRoPE (t/h/w/e).\n * @param mrope_used Whether to enable multi-section rotary embedding.\n * @param is_imrope Whether to apply interleaved mRoPE rules.\n * @param indep_sects Whether each dimension runs independent exponent\n * resets based on @p sections.\n */\nstatic void aclnn_rope_cache_init(ggml_backend_cann_context & ctx,\n ggml_tensor * dst,\n float * corr_dims,\n float ext_factor,\n float theta_scale,\n float freq_scale,\n float attn_factor,\n bool is_neox,\n int sections[4],\n bool mrope_used,\n bool is_imrope,\n bool indep_sects,\n int64_t rope_dims) {\n ggml_tensor * src1 = dst->src[1]; // position\n ggml_tensor * src2 = dst->src[2]; // freq_factors\n\n int64_t theta_scale_length = rope_dims / 2;\n int64_t position_length = dst->ne[2];\n\n // TODO: check theta_scale_length and position_length.\n if (src2 == nullptr && ctx.rope_cache.cached &&\n ctx.rope_cache.equal(theta_scale_length, position_length, ext_factor, theta_scale, freq_scale, attn_factor,\n is_neox, indep_sects, mrope_used, is_imrope, sections)) {\n // use cache.\n return;\n }\n\n // Step0: calculate tensor shape.\n int64_t theta_scale_ne[] = { theta_scale_length, 1, 1, 1 };\n size_t theta_scale_nb[] = { sizeof(float), theta_scale_length * sizeof(float), theta_scale_length * sizeof(float),\n theta_scale_length * sizeof(float) };\n\n GGML_ASSERT(src1->type == GGML_TYPE_I32);\n int64_t position_ne[] = { 1, 1, position_length, 1 };\n size_t position_nb[] = { sizeof(int32_t), sizeof(int32_t), sizeof(int32_t), sizeof(int32_t) * position_length };\n\n int64_t cache_ne[] = { theta_scale_length, 1, position_length, 1 };\n size_t cache_nb[GGML_MAX_DIMS];\n cache_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n cache_nb[i] = cache_nb[i - 1] * cache_ne[i - 1];\n }\n\n // Step1: Compute the coefficient of theta. During the cache_init process, aside from\n // (1) multiplying by the position,\n // (2) dividing by freq_factors,\n // (3) computing the sine and cosine,\n // the other parameters used in the computation generally do not change in most scenarios.\n // Therefore, we can first compute this part of the result and then cache it.\n\n // Step1.1: prepare theta_scale exponent. if this exponent updated, should update theta_scale_tensor.\n acl_tensor_ptr acl_theta_scale_tensor;\n bool theta_scale_updated = false;\n if (ctx.rope_cache.theta_scale_length != theta_scale_length || ctx.rope_cache.theta_scale != theta_scale ||\n ctx.rope_cache.indep_sects != indep_sects) {\n theta_scale_updated = true;\n if (ctx.rope_cache.theta_scale_exp_host != nullptr) {\n free(ctx.rope_cache.theta_scale_exp_host);\n }\n ctx.rope_cache.theta_scale_exp_host = (float *) malloc(theta_scale_length * sizeof(float));\n GGML_ASSERT(ctx.rope_cache.theta_scale_exp_host != nullptr);\n if (!indep_sects) {\n ctx.rope_cache.theta_scale_exp_host[0] = 1;\n for (int i = 1; i < theta_scale_length; i++) {\n ctx.rope_cache.theta_scale_exp_host[i] = ctx.rope_cache.theta_scale_exp_host[i - 1] * theta_scale;\n }\n } else {\n int sect_dims = sections[0] + sections[1] + sections[2] + sections[3];\n int sec_w = sections[1] + sections[0];\n int sec_e = sections[2] + sec_w;\n\n ctx.rope_cache.theta_scale_exp_host[0] = 1;\n for (int i = 1; i < theta_scale_length; i++) {\n int sector = i % sect_dims;\n if (sector == 0 || sector == sections[0] || sector == sec_w || sector == sec_e) {\n ctx.rope_cache.theta_scale_exp_host[i] = 1;\n continue;\n }\n ctx.rope_cache.theta_scale_exp_host[i] = ctx.rope_cache.theta_scale_exp_host[i - 1] * theta_scale;\n }\n }\n\n if (ctx.rope_cache.theta_scale_cache != nullptr) {\n ACL_CHECK(aclrtFree(ctx.rope_cache.theta_scale_cache));\n }\n ACL_CHECK(aclrtMalloc(&ctx.rope_cache.theta_scale_cache, theta_scale_length * sizeof(float),\n ACL_MEM_MALLOC_HUGE_FIRST));\n\n ACL_CHECK(aclrtMemcpyAsync(ctx.rope_cache.theta_scale_cache, theta_scale_length * sizeof(float),\n ctx.rope_cache.theta_scale_exp_host, theta_scale_length * sizeof(float),\n ACL_MEMCPY_HOST_TO_DEVICE, ctx.stream()));\n }\n acl_theta_scale_tensor = ggml_cann_create_tensor(ctx.rope_cache.theta_scale_cache, ACL_FLOAT, sizeof(float),\n theta_scale_ne, theta_scale_nb, 1);\n\n // Step1.2: prepare rope_yarn_ramp, if this part updated, should update theta_scale_tensor.\n // TODO: acl_yarn_ramp_tensor use rope cache.\n bool yarn_ramp_tensor_updated = false;\n acl_tensor_ptr acl_yarn_ramp_tensor;\n if (ext_factor != 0 && (theta_scale_updated || ctx.rope_cache.theta_scale_length != theta_scale_length ||\n ctx.rope_cache.freq_scale != freq_scale)) {\n yarn_ramp_tensor_updated = true;\n if (ctx.rope_cache.yarn_ramp_cache != nullptr) {\n ACL_CHECK(aclrtFree(ctx.rope_cache.yarn_ramp_cache));\n }\n ACL_CHECK(aclrtMalloc(&ctx.rope_cache.yarn_ramp_cache, theta_scale_length * sizeof(float),\n ACL_MEM_MALLOC_HUGE_FIRST));\n // -rope_yarn_ramp\n // const float y = (i0 / 2 - low) / MAX(0.001f, high - low);\n // return MIN(1, MAX(0, y)) - 1;\n acl_yarn_ramp_tensor = ggml_cann_create_tensor(ctx.rope_cache.yarn_ramp_cache, ACL_FLOAT, sizeof(float),\n theta_scale_ne, theta_scale_nb, 1);\n float zero_value = 0, one_value = 1;\n float denom_safe_value = MAX(0.001f, corr_dims[1] - corr_dims[0]);\n acl_scalar_ptr low = ggml_cann_create_scalar(&corr_dims[0], aclDataType::ACL_FLOAT);\n acl_scalar_ptr zero = ggml_cann_create_scalar(&zero_value, aclDataType::ACL_FLOAT);\n acl_scalar_ptr one = ggml_cann_create_scalar(&one_value, aclDataType::ACL_FLOAT);\n acl_scalar_ptr denom_safe = ggml_cann_create_scalar(&denom_safe_value, aclDataType::ACL_FLOAT);\n acl_scalar_ptr ext_factor_sc = ggml_cann_create_scalar(&ext_factor, aclDataType::ACL_FLOAT);\n\n aclnn_arange(ctx, acl_yarn_ramp_tensor.get(), 0, theta_scale_length, 1, theta_scale_length);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSubs, acl_yarn_ramp_tensor.get(), low.get(), one.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceDivs, acl_yarn_ramp_tensor.get(), denom_safe.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceThreshold, acl_yarn_ramp_tensor.get(), zero.get(), zero.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceClampMax, acl_yarn_ramp_tensor.get(), one.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSubs, acl_yarn_ramp_tensor.get(), one.get(), one.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMuls, acl_yarn_ramp_tensor.get(), ext_factor_sc.get());\n\n // theta_interp = freq_scale * theta_extrap;\n // theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;\n // theta = freq_scale * theta_extrap * (1 - ramp_mix) + theta_extrap * ramp_mix;\n // theta = freq_scale * theta_extrap - freq_scale * theta_extrap * ramp_mix + theta_extrap * ramp_mix;\n // theta = theta_extrap * (freq_scale - freq_scale * ramp_mix + ramp_mix);\n //\n // we cache (freq_scale - freq_scale * ramp_mix + ramp_mix), Considering that the rope_yarn_ramp here is the inverse\n // cache freq_scale + (freq_scale - 1) * ramp_mix\n float freq_scale_1 = freq_scale - 1;\n acl_scalar_ptr freq_scale_sc = ggml_cann_create_scalar(&freq_scale, aclDataType::ACL_FLOAT);\n acl_scalar_ptr freq_scale_1_sc = ggml_cann_create_scalar(&freq_scale_1, aclDataType::ACL_FLOAT);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMuls, acl_yarn_ramp_tensor.get(), freq_scale_1_sc.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdds, acl_yarn_ramp_tensor.get(), freq_scale_sc.get(), one.get());\n } else {\n acl_yarn_ramp_tensor = ggml_cann_create_tensor(ctx.rope_cache.yarn_ramp_cache, ACL_FLOAT, sizeof(float),\n theta_scale_ne, theta_scale_nb, 1);\n }\n // Step 1.3: update theta_scale_tensor according to ext_factor or freq_scale.\n if (ext_factor != 0) {\n if (theta_scale_updated || yarn_ramp_tensor_updated) {\n theta_scale_updated = true;\n aclnn_mul(ctx, acl_theta_scale_tensor.get(), acl_yarn_ramp_tensor.get());\n }\n } else {\n if (freq_scale != 1 && (ctx.rope_cache.freq_scale != freq_scale || theta_scale_updated)) {\n theta_scale_updated = true;\n aclnn_muls(ctx, acl_theta_scale_tensor.get(), freq_scale, nullptr, true);\n }\n }\n\n // Nothing changed, use cache.\n if (!theta_scale_updated) {\n acl_theta_scale_tensor = ggml_cann_create_tensor(ctx.rope_cache.theta_scale_cache, ACL_FLOAT, sizeof(float),\n theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);\n }\n\n // Step 1.4: prepare select index if mrope\n acl_tensor_ptr position_select_index_tensor;\n if (mrope_used) {\n if (ctx.rope_cache.sections[0] != sections[0] || ctx.rope_cache.sections[1] != sections[1] ||\n ctx.rope_cache.sections[2] != sections[2] || ctx.rope_cache.sections[3] != sections[3] ||\n ctx.rope_cache.theta_scale_length != theta_scale_length || ctx.rope_cache.is_imrope != is_imrope) {\n if (ctx.rope_cache.position_select_index_host != nullptr) {\n free(ctx.rope_cache.position_select_index_host);\n }\n ctx.rope_cache.position_select_index_host = (int *) malloc(theta_scale_length * sizeof(int));\n GGML_ASSERT(ctx.rope_cache.position_select_index_host != nullptr);\n int sect_dims = sections[0] + sections[1] + sections[2] + sections[3];\n int sec_w = sections[1] + sections[0];\n int sec_e = sections[2] + sec_w;\n // t,h,w,e\n for (int i = 0; i < theta_scale_length; i++) {\n int sector = i % sect_dims;\n\n if (is_imrope) { // qwen3vl apply interleaved mrope\n if (sector % 3 == 1 && sector < 3 * sections[1]) {\n ctx.rope_cache.position_select_index_host[i] = 1;\n } else if (sector % 3 == 2 && sector < 3 * sections[2]) {\n ctx.rope_cache.position_select_index_host[i] = 2;\n } else if (sector % 3 == 0 && sector < 3 * sections[0]) {\n ctx.rope_cache.position_select_index_host[i] = 0;\n } else {\n ctx.rope_cache.position_select_index_host[i] = 3;\n }\n } else {\n if (sector >= sections[0] && sector < sec_w) {\n ctx.rope_cache.position_select_index_host[i] = 1;\n } else if (sector >= sec_w && sector < sec_e) {\n ctx.rope_cache.position_select_index_host[i] = 2;\n } else if (sector >= sec_e) {\n ctx.rope_cache.position_select_index_host[i] = 3;\n } else {\n ctx.rope_cache.position_select_index_host[i] = 0;\n }\n }\n }\n\n if (ctx.rope_cache.position_select_index != nullptr) {\n ACL_CHECK(aclrtFree(ctx.rope_cache.position_select_index));\n }\n ACL_CHECK(aclrtMalloc(&ctx.rope_cache.position_select_index, theta_scale_length * sizeof(int),\n ACL_MEM_MALLOC_HUGE_FIRST));\n\n ACL_CHECK(aclrtMemcpyAsync(ctx.rope_cache.position_select_index, theta_scale_length * sizeof(int),\n ctx.rope_cache.position_select_index_host, theta_scale_length * sizeof(int),\n ACL_MEMCPY_HOST_TO_DEVICE, ctx.stream()));\n }\n\n position_select_index_tensor = ggml_cann_create_tensor(ctx.rope_cache.position_select_index, ACL_INT32,\n sizeof(int), theta_scale_ne, theta_scale_nb, 1);\n }\n\n // Step2: divide by freq_factors\n ggml_cann_pool_alloc freq_fac_res_allocator(ctx.pool());\n if (src2) {\n freq_fac_res_allocator.alloc(theta_scale_length * sizeof(float));\n void * freq_fac_res_ptr = freq_fac_res_allocator.get();\n acl_tensor_ptr acl_freq_factors_tensor =\n ggml_cann_create_tensor(src2->data, ggml_cann_type_mapping(src2->type), ggml_type_size(src2->type),\n theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);\n acl_tensor_ptr acl_freq_fac_res_tensor = ggml_cann_create_tensor(freq_fac_res_ptr, ACL_FLOAT, sizeof(float),\n theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);\n aclnn_div(ctx, acl_theta_scale_tensor.get(), acl_freq_factors_tensor.get(), acl_freq_fac_res_tensor.get());\n std::swap(acl_theta_scale_tensor, acl_freq_fac_res_tensor);\n }\n\n // Step3: prepare position_tensor\n acl_tensor_ptr acl_position_tensor;\n ggml_cann_pool_alloc mrope_position_acllocator(ctx.pool());\n if (mrope_used) {\n // Step3.1: select current position;\n // position :\n // pos1: [[0, 1 ,2 ,3 ],\n // pos2: [4, 5 ,6 ,7 ],\n // pos3: [8, 9 ,10,11],\n // pos4: [12,13,14,15] ]\n //\n // select index = [0, 1, 2, 2, 1, 0]\n //\n // selected_tensor:\n // [[0, 1 ,2 ,3 ],\n // [4, 5 ,6 ,7 ],\n // [8, 9 ,10,11],\n // [8, 9 ,10,11],\n // [4, 5 ,6 ,7 ],\n // [0, 1 ,2 ,3 ]]\n //\n // transpose, from [seq_len:dims] to [dims:seq_len]\n // [0, 4, 8 ,8 ,4, 0],\n // [1, 5, 9, 9, 5, 1],\n // [2, 6, 10,10,6 ,2],\n // [3, 7, 11,11,7 3 ]]\n //\n // multipy by theta_scale_tensor\n // [theta_scale^0, theta_scale^1, ..., theta_scale ^ n]\n\n int64_t mrope_position_ne[] = { position_length, 4 };\n size_t mrope_position_nb[] = { sizeof(int), position_length * sizeof(int) };\n acl_tensor_ptr mrope_position =\n ggml_cann_create_tensor(src1->data, ggml_cann_type_mapping(src1->type), ggml_type_size(src1->type),\n mrope_position_ne, mrope_position_nb, 2);\n\n // selected position tensor's shape is a transpose of cache tensor.\n int64_t selected_position_ne[] = { position_length, theta_scale_length };\n size_t selected_position_nb[] = { sizeof(float), position_length * sizeof(float) };\n mrope_position_acllocator.alloc(theta_scale_length * position_length * sizeof(float));\n void * mrope_position_buffer = mrope_position_acllocator.get();\n acl_position_tensor =\n ggml_cann_create_tensor(mrope_position_buffer, ggml_cann_type_mapping(src1->type),\n ggml_type_size(src1->type), selected_position_ne, selected_position_nb, 2);\n GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, mrope_position.get(), 0, position_select_index_tensor.get(),\n acl_position_tensor.get());\n\n // transpose\n int64_t transposed_ne[] = { position_length, 1, theta_scale_length, 1 };\n size_t transposed_nb[GGML_MAX_DIMS];\n transposed_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n transposed_nb[i] = transposed_nb[i - 1] * transposed_ne[i - 1];\n }\n\n std::swap(transposed_ne[0], transposed_ne[2]);\n std::swap(transposed_nb[0], transposed_nb[2]);\n\n acl_position_tensor =\n ggml_cann_create_tensor(mrope_position_buffer, ggml_cann_type_mapping(src1->type),\n ggml_type_size(src1->type), transposed_ne, transposed_nb, GGML_MAX_DIMS);\n\n } else {\n // auto bcast.\n acl_position_tensor =\n ggml_cann_create_tensor(src1->data, ggml_cann_type_mapping(src1->type), ggml_type_size(src1->type),\n position_ne, position_nb, GGML_MAX_DIMS);\n }\n\n // Step4: multiply by the position\n int64_t theta_length = theta_scale_length * position_length;\n ggml_cann_pool_alloc theta_allocator(ctx.pool(), theta_length * sizeof(float));\n void * theta_buffer = theta_allocator.get();\n\n acl_tensor_ptr acl_theta_tensor =\n ggml_cann_create_tensor(theta_buffer, ACL_FLOAT, sizeof(float), cache_ne, cache_nb, GGML_MAX_DIMS);\n aclnn_mul(ctx, acl_position_tensor.get(), acl_theta_scale_tensor.get(), acl_theta_tensor.get());\n\n // Step5: calculate sin cos.\n // init sin_repeat && cos_repeat, only to accelerate first layer on each device\n if (position_length > ctx.rope_cache.position_length) {\n ctx.rope_cache.position_length = position_length;\n if (ctx.rope_cache.sin_cache != nullptr) {\n ACL_CHECK(aclrtFree(ctx.rope_cache.sin_cache));\n }\n if (ctx.rope_cache.cos_cache != nullptr) {\n ACL_CHECK(aclrtFree(ctx.rope_cache.cos_cache));\n }\n int64_t repeat_theta_length = theta_scale_length * position_length * 2;\n ACL_CHECK(\n aclrtMalloc(&ctx.rope_cache.sin_cache, repeat_theta_length * sizeof(float), ACL_MEM_MALLOC_HUGE_FIRST));\n ACL_CHECK(\n aclrtMalloc(&ctx.rope_cache.cos_cache, repeat_theta_length * sizeof(float), ACL_MEM_MALLOC_HUGE_FIRST));\n }\n\n // sin/cos\n ggml_cann_pool_alloc sin_allocator(ctx.pool(), theta_length * sizeof(float));\n void * sin_buffer = sin_allocator.get();\n acl_tensor_ptr acl_sin_tensor =\n ggml_cann_create_tensor(sin_buffer, ACL_FLOAT, sizeof(float), cache_ne, cache_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n aclnn_sin(ctx, acl_theta_tensor.get(), acl_sin_tensor.get());\n\n ggml_cann_pool_alloc cos_allocator(ctx.pool(), theta_length * sizeof(float));\n void * cos_buffer = cos_allocator.get();\n acl_tensor_ptr acl_cos_tensor =\n ggml_cann_create_tensor(cos_buffer, ACL_FLOAT, sizeof(float), cache_ne, cache_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n aclnn_cos(ctx, acl_theta_tensor.get(), acl_cos_tensor.get());\n\n if (ext_factor != 0) {\n attn_factor *= 1.0f + 0.1f * logf(1.0f / freq_scale);\n }\n\n // Step 5: multiply by attn_factor\n if (attn_factor != 1) {\n aclnn_muls(ctx, acl_sin_tensor.get(), attn_factor, nullptr, true);\n aclnn_muls(ctx, acl_cos_tensor.get(), attn_factor, nullptr, true);\n }\n\n int64_t sin_reshape_ne[4] = { rope_dims, 1, dst->ne[2], 1 };\n size_t sin_reshape_nb[GGML_MAX_DIMS];\n sin_reshape_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n sin_reshape_nb[i] = sin_reshape_nb[i - 1] * sin_reshape_ne[i - 1];\n }\n acl_tensor_ptr acl_sin_repeat_tensor = ggml_cann_create_tensor(ctx.rope_cache.sin_cache, ACL_FLOAT, sizeof(float),\n sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);\n acl_tensor_ptr acl_cos_repeat_tensor = ggml_cann_create_tensor(ctx.rope_cache.cos_cache, ACL_FLOAT, sizeof(float),\n sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);\n\n // Step 6: repeat\n if (is_neox) {\n // [sinθ1, sinθ1, sinθ2, sinθ2, ..., sinθn, sinθn]\n int64_t repeatsArray[] = { 1, 1, 1, 2 };\n aclnn_repeat(ctx, acl_sin_tensor.get(), acl_sin_repeat_tensor.get(), repeatsArray);\n aclnn_repeat(ctx, acl_cos_tensor.get(), acl_cos_repeat_tensor.get(), repeatsArray);\n } else {\n int64_t num_repeats = 2;\n int64_t dim = 3;\n int64_t output_size = theta_scale_length * num_repeats;\n // [sinθ1, sinθ2, ..., sinθn, sinθ1, sinθ2, ..., sinθn]\n aclnn_repeat_interleave(ctx, acl_sin_tensor.get(), acl_sin_repeat_tensor.get(), dim, num_repeats, output_size);\n aclnn_repeat_interleave(ctx, acl_cos_tensor.get(), acl_cos_repeat_tensor.get(), dim, num_repeats, output_size);\n }\n\n // Update cached value.\n ctx.rope_cache.cached = true;\n ctx.rope_cache.set(theta_scale_length, position_length, ext_factor, theta_scale, freq_scale, attn_factor, is_neox,\n indep_sects, mrope_used, is_imrope, sections);\n}\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\naclnnStatus aclnnRotaryPositionEmbeddingGetWorkspaceSize(const aclTensor * x,\n const aclTensor * cos,\n const aclTensor * sin,\n int64_t mode,\n const aclTensor * yOut,\n uint64_t * workspaceSize,\n aclOpExecutor ** executor);\naclnnStatus aclnnRotaryPositionEmbedding(void * workspace,\n uint64_t workspaceSize,\n aclOpExecutor * executor,\n aclrtStream stream);\n#ifdef __cplusplus\n}\n#endif\n\nvoid ggml_cann_rope(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0]; // input\n\n // param\n float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;\n int sections[4];\n // const int n_past = ((int32_t *) dst->op_params)[0];\n const int n_dims = ((int32_t *) dst->op_params)[1];\n const int mode = ((int32_t *) dst->op_params)[2];\n // const int n_ctx = ((int32_t *) dst->op_params)[3];\n const int n_ctx_orig = ((int32_t *) dst->op_params)[4];\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));\n memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));\n memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));\n memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));\n memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));\n memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));\n memcpy(§ions, (int32_t *) dst->op_params + 11, sizeof(int) * 4);\n\n GGML_ASSERT(n_dims % 2 == 0);\n GGML_ASSERT(n_dims <= ne00);\n\n const float theta_scale = powf(freq_base, -2.0f / n_dims);\n\n float corr_dims[2];\n ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);\n\n bool is_neox = mode & GGML_ROPE_TYPE_NEOX;\n const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; // qwen3vl apply interleaved mrope\n // mrope_used means the GGML_ROPE_TYPE_MROPE bit is set.\n // Note: this bit is also set for imrope and some vision modes,\n // so mrope_used does NOT exclusively indicate pure mrope.\n const bool mrope_used = mode & GGML_ROPE_TYPE_MROPE;\n const bool is_vision = mode == GGML_ROPE_TYPE_VISION;\n\n if (mrope_used) {\n GGML_ASSERT(sections[0] > 0 || sections[1] > 0 || sections[2] > 0);\n }\n\n if (is_vision) {\n GGML_ASSERT(n_dims == ne0 / 2);\n }\n\n if (is_imrope || mrope_used) {\n is_neox = true;\n }\n\n int64_t rope_dims = n_dims;\n\n //Our current RotaryPositionEmbedding does not support the VISION mode,\n //but essentially it only modifies theta_base in mrope,\n //then repeats it at the end in the same way as is_neox.\n //In fact, RoPE is still applied across all dimensions.\n if (is_vision) {\n rope_dims = src0->ne[0];\n }\n int64_t tail_dims = ne00 - rope_dims;\n bool has_tail = tail_dims > 0;\n\n // init ctx.rope_cos/rope_sin cache\n aclnn_rope_cache_init(ctx, dst, corr_dims, ext_factor, theta_scale, freq_scale, attn_factor, is_neox, sections,\n mrope_used, is_imrope, is_vision, rope_dims);\n\n // Cache is generated with ne00 dimensions, so we use ne00 for reshape\n int64_t sin_reshape_ne[4] = { rope_dims, 1, ne02, 1 };\n size_t sin_reshape_nb[GGML_MAX_DIMS];\n sin_reshape_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n sin_reshape_nb[i] = sin_reshape_nb[i - 1] * sin_reshape_ne[i - 1];\n }\n acl_tensor_ptr acl_sin_reshape_tensor = ggml_cann_create_tensor(ctx.rope_cache.sin_cache, ACL_FLOAT, sizeof(float),\n sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);\n acl_tensor_ptr acl_cos_reshape_tensor = ggml_cann_create_tensor(ctx.rope_cache.cos_cache, ACL_FLOAT, sizeof(float),\n sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n#ifdef ASCEND_310P\n // Special ROPE operation for 310P\n\n // roll input\n void * input_roll_buffer;\n acl_tensor_ptr acl_minus_one_tensor;\n void * minus_one_scale_buffer = nullptr;\n ggml_cann_pool_alloc roll_allocator(ctx.pool(), ggml_nbytes(src0));\n ggml_cann_pool_alloc minus_one_scale_allocator(ctx.pool(), sizeof(float) * src0->ne[0]);\n if (!is_neox) {\n // roll input: [q0,q1,q2,q3,...] -> [q1,q0,q3,q2,...]\n input_roll_buffer = roll_allocator.get();\n int64_t input_roll_ne[4] = { 2, src0->ne[1] * (src0->ne[0] / 2), src0->ne[2], src0->ne[3] };\n size_t input_roll_nb[GGML_MAX_DIMS];\n input_roll_nb[0] = ggml_type_size(src0->type);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n input_roll_nb[i] = input_roll_nb[i - 1] * input_roll_ne[i - 1];\n }\n acl_tensor_ptr acl_input_roll_tensor =\n ggml_cann_create_tensor(input_roll_buffer, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n input_roll_ne, input_roll_nb, GGML_MAX_DIMS);\n acl_tensor_ptr acl_input_tensor =\n ggml_cann_create_tensor(src0->data, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n input_roll_ne, input_roll_nb, GGML_MAX_DIMS);\n\n int64_t shifts[] = { 1 };\n int64_t dims[] = { 3 };\n aclnn_roll(ctx, acl_input_tensor.get(), acl_input_roll_tensor.get(), shifts, dims);\n\n // init [-1, 1, -1, 1, ...]\n minus_one_scale_buffer = minus_one_scale_allocator.get();\n\n int64_t minus_one_ne[4] = { src0->ne[0], 1, 1, 1 };\n size_t minus_one_nb[GGML_MAX_DIMS];\n minus_one_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];\n }\n acl_minus_one_tensor = aclnn_values(ctx, minus_one_scale_buffer, sizeof(float) * src0->ne[0], minus_one_ne,\n GGML_MAX_DIMS, ACL_FLOAT, sizeof(float), 1);\n int64_t dim = 3;\n int64_t * index = new int64_t[src0->ne[0]];\n for (int i = 0; i < src0->ne[0]; i++) {\n index[i] = i / 2 * 2;\n }\n int64_t index_num = src0->ne[0];\n float value = -1;\n aclnn_index_fill_tensor(ctx, acl_minus_one_tensor.get(), dim, index, index_num, value);\n } else {\n // roll input: [q0,q1,q2,...] ->\n // [q_half,q_half+1,...,q_end,q0,q1,...q_half-1]\n input_roll_buffer = roll_allocator.get();\n acl_tensor_ptr acl_input_roll_tensor =\n ggml_cann_create_tensor(input_roll_buffer, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n src0->ne, src0->nb, GGML_MAX_DIMS);\n acl_tensor_ptr acl_input_tensor = ggml_cann_create_tensor(src0);\n\n int64_t shifts[] = { src0->ne[0] / 2 };\n int64_t dims[] = { 3 };\n aclnn_roll(ctx, acl_input_tensor.get(), acl_input_roll_tensor.get(), shifts, dims);\n\n // init [-1, -1, -1, 1, 1，1，...]\n minus_one_scale_buffer = minus_one_scale_allocator.get();\n int64_t minus_one_ne[4] = { src0->ne[0], 1, 1, 1 };\n size_t minus_one_nb[GGML_MAX_DIMS];\n minus_one_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];\n }\n acl_minus_one_tensor = aclnn_values(ctx, minus_one_scale_buffer, sizeof(float) * src0->ne[0], minus_one_ne,\n GGML_MAX_DIMS, ACL_FLOAT, sizeof(float), 1);\n // -1 * first half\n int64_t first_half_ne[4] = { src0->ne[0] / 2, 1, 1, 1 };\n size_t first_half_nb[GGML_MAX_DIMS];\n first_half_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n first_half_nb[i] = first_half_nb[i - 1] * first_half_ne[i - 1];\n }\n acl_tensor_ptr acl_first_half_tensor = ggml_cann_create_tensor(minus_one_scale_buffer, ACL_FLOAT, sizeof(float),\n first_half_ne, first_half_nb, GGML_MAX_DIMS);\n bool inplace = true;\n float scale = -1;\n aclnn_muls(ctx, acl_first_half_tensor.get(), scale, nullptr, inplace);\n }\n\n // TODO: n_dims < ne0\n GGML_ASSERT(n_dims == src0->ne[0]);\n\n // input * scale\n ggml_cann_pool_alloc roll_mul_scale_allocator(ctx.pool(), ggml_nbytes(src0));\n void * input_roll_mul_scale_buffer = roll_mul_scale_allocator.get();\n size_t input_nb[GGML_MAX_DIMS];\n input_nb[0] = ggml_type_size(src0->type);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n input_nb[i] = input_nb[i - 1] * src0->ne[i - 1];\n }\n acl_tensor_ptr acl_input_roll_mul_scale_tensor =\n ggml_cann_create_tensor(input_roll_mul_scale_buffer, ggml_cann_type_mapping(src0->type),\n ggml_type_size(src0->type), src0->ne, input_nb, GGML_MAX_DIMS);\n acl_tensor_ptr acl_input_roll_reshape_tensor =\n ggml_cann_create_tensor(input_roll_buffer, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n src0->ne, input_nb, GGML_MAX_DIMS);\n\n aclnn_mul(ctx, acl_input_roll_reshape_tensor.get(), acl_minus_one_tensor.get(),\n acl_input_roll_mul_scale_tensor.get());\n\n // output\n void * output_fp32_buffer;\n if (src0->type == GGML_TYPE_F32) {\n aclnn_mul(ctx, acl_src.get(), acl_cos_reshape_tensor.get());\n aclnn_mul(ctx, acl_input_roll_mul_scale_tensor.get(), acl_sin_reshape_tensor.get());\n aclnn_add(ctx, acl_src.get(), acl_input_roll_mul_scale_tensor.get(), acl_dst.get());\n // TODO: ne0 != n_dims in mode2\n } else if (src0->type == GGML_TYPE_F16) {\n size_t input_fp32_nb[GGML_MAX_DIMS];\n input_fp32_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n input_fp32_nb[i] = input_fp32_nb[i - 1] * dst->ne[i - 1];\n }\n ggml_cann_pool_alloc fp32_allocator1(ctx.pool(), ggml_nelements(dst) * sizeof(float));\n void * input_fp32_buffer1 = fp32_allocator1.get();\n acl_tensor_ptr input_fp32_tensor1 = ggml_cann_create_tensor(input_fp32_buffer1, ACL_FLOAT, sizeof(float),\n dst->ne, input_fp32_nb, GGML_MAX_DIMS);\n ggml_cann_pool_alloc fp32_allocator2(ctx.pool(), ggml_nelements(dst) * sizeof(float));\n void * input_fp32_buffer2 = fp32_allocator2.get();\n acl_tensor_ptr input_fp32_tensor2 = ggml_cann_create_tensor(input_fp32_buffer2, ACL_FLOAT, sizeof(float),\n dst->ne, input_fp32_nb, GGML_MAX_DIMS);\n\n ggml_cann_pool_alloc fp32_allocator(ctx.pool(), ggml_nelements(dst) * sizeof(float));\n output_fp32_buffer = fp32_allocator.get();\n acl_tensor_ptr output_fp32_tensor = ggml_cann_create_tensor(output_fp32_buffer, ACL_FLOAT, sizeof(float),\n dst->ne, input_fp32_nb, GGML_MAX_DIMS);\n aclnn_mul(ctx, acl_src.get(), acl_cos_reshape_tensor.get(), input_fp32_tensor1.get());\n aclnn_mul(ctx, acl_input_roll_mul_scale_tensor.get(), acl_sin_reshape_tensor.get(), input_fp32_tensor2.get());\n aclnn_add(ctx, input_fp32_tensor1.get(), input_fp32_tensor2.get(), output_fp32_tensor.get());\n aclnn_cast(ctx, output_fp32_tensor.get(), acl_dst.get(), ACL_FLOAT16);\n }\n return;\n#endif\n int64_t acl_mode = is_neox ? 0 : 1;\n\n // Pre-define head and tail dimensions for reuse\n int64_t head_ne[GGML_MAX_DIMS] = { rope_dims, ne01, ne02, ne03 };\n int64_t tail_ne[GGML_MAX_DIMS] = { tail_dims, ne01, ne02, ne03 };\n\n // Step 1: Prepare trans tensors for F16 type conversion to F32 if needed\n bool src_dst_need_trans = false;\n ggml_cann_pool_alloc src_trans_allocator(ctx.pool());\n ggml_cann_pool_alloc dst_trans_allocator(ctx.pool());\n acl_tensor_ptr acl_src_trans_tensor;\n acl_tensor_ptr acl_dst_trans_tensor;\n void * src_trans_buffer = nullptr;\n void * dst_trans_buffer = nullptr;\n size_t src_dst_trans_nb[GGML_MAX_DIMS];\n if (src0->type == GGML_TYPE_F16) {\n src_dst_need_trans = true;\n src_trans_buffer = src_trans_allocator.alloc(ggml_nelements(src0) * sizeof(float));\n dst_trans_buffer = dst_trans_allocator.alloc(ggml_nelements(dst) * sizeof(float));\n\n src_dst_trans_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n src_dst_trans_nb[i] = src_dst_trans_nb[i - 1] * src0->ne[i - 1];\n }\n acl_src_trans_tensor = ggml_cann_create_tensor(src_trans_buffer, ACL_FLOAT, sizeof(float), src0->ne,\n src_dst_trans_nb, GGML_MAX_DIMS);\n acl_dst_trans_tensor = ggml_cann_create_tensor(dst_trans_buffer, ACL_FLOAT, sizeof(float), dst->ne,\n src_dst_trans_nb, GGML_MAX_DIMS);\n aclnn_cast(ctx, acl_src.get(), acl_src_trans_tensor.get(), ACL_FLOAT);\n }\n\n // Step 2: Prepare head tensors for tail splitting if needed\n acl_tensor_ptr acl_src_head;\n acl_tensor_ptr acl_dst_head;\n if (has_tail) {\n // Create head views for RotaryPositionEmbedding (only first rope_dims dimensions)\n // RotaryPositionEmbedding requires contiguous dst tensor, so we use a temporary buffer\n if (src_dst_need_trans) {\n // Use F32 trans tensor strides\n acl_src_head = ggml_cann_create_tensor((char *) src_trans_buffer, ACL_FLOAT, sizeof(float), head_ne,\n src_dst_trans_nb, GGML_MAX_DIMS);\n } else {\n // Use original F32 tensor strides\n acl_src_head = ggml_cann_create_tensor((char *) src0->data, ACL_FLOAT, sizeof(float), head_ne, src0->nb,\n GGML_MAX_DIMS);\n }\n\n int64_t head_elements = rope_dims * ne01 * ne02 * ne03;\n ggml_cann_pool_alloc dst_head_contiguous_allocator(ctx.pool(), head_elements * sizeof(float));\n void * dst_head_contiguous_buffer = dst_head_contiguous_allocator.get();\n\n size_t head_contiguous_nb[GGML_MAX_DIMS];\n head_contiguous_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n head_contiguous_nb[i] = head_contiguous_nb[i - 1] * head_ne[i - 1];\n }\n acl_dst_head = ggml_cann_create_tensor(dst_head_contiguous_buffer, ACL_FLOAT, sizeof(float), head_ne,\n head_contiguous_nb, GGML_MAX_DIMS);\n }\n\n // Step 3: Execute RotaryPositionEmbedding\n if (has_tail) {\n // Rotate only the head portion (first rope_dims dimensions)\n GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src_head.get(), acl_cos_reshape_tensor.get(),\n acl_sin_reshape_tensor.get(), acl_mode, acl_dst_head.get());\n\n // Copy head result from contiguous buffer back to destination tensor\n if (src_dst_need_trans) {\n acl_tensor_ptr acl_dst_head_target = ggml_cann_create_tensor(\n (char *) dst_trans_buffer, ACL_FLOAT, sizeof(float), head_ne, src_dst_trans_nb, GGML_MAX_DIMS);\n cann_copy(ctx, acl_dst_head.get(), acl_dst_head_target.get());\n } else {\n acl_tensor_ptr acl_dst_head_target =\n ggml_cann_create_tensor((char *) dst->data, ACL_FLOAT, sizeof(float), head_ne, dst->nb, GGML_MAX_DIMS);\n cann_copy(ctx, acl_dst_head.get(), acl_dst_head_target.get());\n }\n } else if (src_dst_need_trans) {\n // Rotate full tensor (no tail), using trans tensors\n GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src_trans_tensor.get(), acl_cos_reshape_tensor.get(),\n acl_sin_reshape_tensor.get(), acl_mode, acl_dst_trans_tensor.get());\n } else {\n // Rotate full tensor (no tail), using original tensors\n GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src.get(), acl_cos_reshape_tensor.get(),\n acl_sin_reshape_tensor.get(), acl_mode, acl_dst.get());\n }\n\n // Step 4: Copy unrotated tail portion from source to destination\n if (has_tail) {\n size_t src_tail_offset;\n size_t dst_tail_offset;\n\n auto copy_tail_device = [&](void * src_ptr, void * dst_ptr, aclDataType dtype, size_t elem_size,\n size_t * nb_src_arr, size_t * nb_dst_arr) {\n acl_tensor_ptr acl_src_tail =\n ggml_cann_create_tensor(src_ptr, dtype, elem_size, tail_ne, nb_src_arr, GGML_MAX_DIMS);\n acl_tensor_ptr acl_dst_tail =\n ggml_cann_create_tensor(dst_ptr, dtype, elem_size, tail_ne, nb_dst_arr, GGML_MAX_DIMS);\n cann_copy(ctx, acl_src_tail.get(), acl_dst_tail.get());\n };\n\n if (src_dst_need_trans) {\n // Use F32 trans tensor strides and offsets\n src_tail_offset = rope_dims * src_dst_trans_nb[0];\n dst_tail_offset = rope_dims * src_dst_trans_nb[0];\n copy_tail_device((char *) src_trans_buffer + src_tail_offset, (char *) dst_trans_buffer + dst_tail_offset,\n ACL_FLOAT, sizeof(float), src_dst_trans_nb, src_dst_trans_nb);\n } else {\n // Use original tensor strides and offsets\n src_tail_offset = rope_dims * nb00;\n dst_tail_offset = rope_dims * nb0;\n copy_tail_device((char *) src0->data + src_tail_offset, (char *) dst->data + dst_tail_offset,\n ggml_cann_type_mapping(dst->type), ggml_element_size(dst), src0->nb, dst->nb);\n }\n }\n\n // Step 5: Cast back to F16 if needed\n if (src_dst_need_trans) {\n aclnn_cast(ctx, acl_dst_trans_tensor.get(), acl_dst.get(), ACL_FLOAT16);\n }\n}\n\nvoid ggml_cann_argmax(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, dst->ne, dst->nb, 3);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, ArgMax, acl_src.get(), 3, false, acl_dst.get());\n}\n\nvoid ggml_cann_conv_transpose_1d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n ggml_tensor * src1 = dst->src[1];\n\n // stride\n int64_t s0 = ((const int32_t *) (dst->op_params))[0];\n\n acl_tensor_ptr acl_input = ggml_cann_create_tensor(src1, src1->ne, src1->nb, 3, ACL_FORMAT_NCL);\n acl_tensor_ptr acl_weight = ggml_cann_create_tensor(src0, src0->ne, src0->nb, 3, ACL_FORMAT_NCL);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, dst->ne, dst->nb, 3, ACL_FORMAT_NCL);\n\n // get base information of input and kernel\n int64_t input_len = *(src1->ne);\n int64_t dst_len = *(dst->ne);\n int64_t kernel_size = *(src0->ne);\n\n // set the max kernel size for each conv\n int64_t max_kernel_size = 255;\n\n // compute the partition of kernel\n int64_t part_num = 1;\n part_num = (kernel_size + max_kernel_size - 1) / max_kernel_size;\n\n int64_t strideVal[1];\n strideVal[0] = s0;\n acl_int_array_ptr stride = ggml_cann_create_int_array(strideVal, 1);\n int64_t paddingVal[] = { 0 };\n acl_int_array_ptr padding = ggml_cann_create_int_array(paddingVal, 1);\n int64_t dilationVal[] = { 1 };\n acl_int_array_ptr dilation = ggml_cann_create_int_array(dilationVal, 1);\n bool transposed = true;\n int64_t groups = 1;\n int8_t cubeMathType = 0;\n\n#ifdef ASCEND_310P\n cubeMathType = 1;\n#endif\n\n auto weight_type = ggml_cann_type_mapping(src0->type);\n auto dst_type = ggml_cann_type_mapping(dst->type);\n\n // slice the kernel to make each conv available\n int64_t slice_dim = -1;\n int64_t slice_start = 0;\n int64_t slice_end = max_kernel_size;\n int64_t slice_step = 1;\n int64_t interval = max_kernel_size;\n\n int64_t left_pad_len = dilationVal[0] * (max_kernel_size - 1) + 1 - 2 * paddingVal[0];\n int64_t right_pad_len = 0;\n\n acl_scalar_ptr alpha = nullptr;\n float alphaValue = 1.0;\n alpha = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n // set zero to destination\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, acl_dst.get());\n\n for (int k = 0; k < part_num; k++) {\n // create part kernel tensor and slice from big kernel\n slice_start = max_kernel_size * k;\n if (k == part_num - 1) {\n slice_end = kernel_size;\n interval = kernel_size - max_kernel_size * k;\n } else {\n slice_end = max_kernel_size * (k + 1);\n }\n\n int64_t part_ne[4];\n for (int i = 0; i < 4; i++) {\n part_ne[i] = *(src0->ne + i);\n }\n part_ne[0] = interval;\n\n size_t part_nb[4];\n part_nb[0] = sizeof(weight_type);\n for (int i = 1; i < 4; i++) {\n part_nb[i] = part_nb[i - 1] * part_ne[i - 1];\n }\n\n ggml_cann_pool_alloc part_kernel_allocator;\n part_kernel_allocator.alloc(ctx.pool(), part_nb[3]);\n void * part_kernel_buf = part_kernel_allocator.get();\n\n acl_tensor_ptr part_kernel = ggml_cann_create_tensor(part_kernel_buf, weight_type, ggml_element_size(src0),\n part_ne, part_nb, 3, ACL_FORMAT_NCL);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Slice, acl_weight.get(), slice_dim, slice_start, slice_end, slice_step,\n part_kernel.get());\n\n // create the part conv result tensor\n int64_t part_dst_ne[4];\n for (int i = 0; i < 4; i++) {\n part_dst_ne[i] = *(dst->ne + i);\n }\n part_dst_ne[0] = (input_len - 1) * strideVal[0] - 2 * paddingVal[0] + dilationVal[0] * (part_ne[0] - 1) + 1;\n\n size_t part_dst_nb[4];\n part_dst_nb[0] = sizeof(weight_type);\n for (int i = 1; i < 4; i++) {\n part_dst_nb[i] = part_dst_nb[i - 1] * part_dst_ne[i - 1];\n }\n ggml_cann_pool_alloc part_dst_allocator;\n part_dst_allocator.alloc(ctx.pool(), part_dst_nb[3]);\n void * part_dst_buf = part_dst_allocator.get();\n\n acl_tensor_ptr acl_part_dst = ggml_cann_create_tensor(part_dst_buf, dst_type, ggml_element_size(dst),\n part_dst_ne, part_dst_nb, 3, ACL_FORMAT_NCL);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, acl_part_dst.get());\n\n // compute part conv transpose 1d\n GGML_CANN_CALL_ACLNN_OP(ctx, Convolution, acl_input.get(), part_kernel.get(), nullptr, stride.get(),\n padding.get(), dilation.get(), transposed, padding.get(), groups, acl_part_dst.get(),\n cubeMathType);\n\n // compute the position of part result in final result\n int64_t global_start = slice_start;\n int64_t global_end = std::min((input_len - 1) * strideVal[0] + slice_end, dst_len);\n\n left_pad_len = global_start;\n right_pad_len = dst_len - global_end;\n\n std::vector padDataVal = { left_pad_len, right_pad_len };\n acl_int_array_ptr padData = ggml_cann_create_int_array(padDataVal.data(), 2);\n\n acl_scalar_ptr pad_value = nullptr;\n float pad_valueVal = 0.0;\n pad_value = ggml_cann_create_scalar(&pad_valueVal, aclDataType::ACL_FLOAT);\n\n int64_t conv_result_ne[4];\n for (int i = 0; i < 4; i++) {\n conv_result_ne[i] = *(dst->ne + i);\n }\n\n size_t conv_result_nb[4];\n conv_result_nb[0] = sizeof(weight_type);\n for (int i = 1; i < 4; i++) {\n conv_result_nb[i] = conv_result_nb[i - 1] * conv_result_ne[i - 1];\n }\n\n ggml_cann_pool_alloc conv_result_allocator;\n conv_result_allocator.alloc(ctx.pool(), conv_result_nb[3]);\n void * conv_result_buf = conv_result_allocator.get();\n\n acl_tensor_ptr conv_result = ggml_cann_create_tensor(conv_result_buf, dst_type, ggml_element_size(dst),\n conv_result_ne, conv_result_nb, 3, ACL_FORMAT_NCL);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, conv_result.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, ConstantPadNd, acl_part_dst.get(), padData.get(), pad_value.get(),\n conv_result.get());\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst.get(), conv_result.get(), alpha.get());\n }\n}\n\nvoid ggml_cann_elu(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n\n acl_tensor_ptr acl_input = ggml_cann_create_tensor(src0);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n float alphaValue = 1.0f;\n acl_scalar_ptr alpha = nullptr;\n alpha = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Elu, acl_input.get(), alpha.get(), alpha.get(), alpha.get(), acl_dst.get());\n}\n\nvoid ggml_cann_mean(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n int64_t reduceDimValue[] = { 3 };\n acl_int_array_ptr reduceDim = ggml_cann_create_int_array(reduceDimValue, 1);\n bool keepDim = true;\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Mean, acl_src.get(), reduceDim.get(), keepDim, ACL_FLOAT, acl_dst.get());\n}\n\nvoid ggml_cann_pad_reflect_1d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n int32_t * opts = (int32_t *) dst->op_params;\n int64_t paddingsArray[2] = { opts[0], opts[1] };\n acl_int_array_ptr paddings = ggml_cann_create_int_array(paddingsArray, 2);\n\n for (int64_t i = 0; i < src0->ne[3]; i++) {\n acl_tensor_ptr acl_src =\n ggml_cann_create_tensor((char *) src0->data + i * src0->ne[3], ggml_cann_type_mapping(src0->type),\n ggml_element_size(src0), src0->ne, src0->nb, 3);\n\n acl_tensor_ptr acl_dst =\n ggml_cann_create_tensor((char *) dst->data + i * src0->ne[3], ggml_cann_type_mapping(dst->type),\n ggml_element_size(dst), dst->ne, dst->nb, 3);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, ReflectionPad1d, acl_src.get(), paddings.get(), acl_dst.get());\n }\n}\n\nvoid ggml_cann_count_equal(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n ggml_tensor * src1 = dst->src[1];\n\n acl_tensor_ptr acl_self = ggml_cann_create_tensor(src0);\n acl_tensor_ptr acl_other = ggml_cann_create_tensor(src1);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceEqTensor, acl_self.get(), acl_other.get());\n\n ggml_cann_sum(ctx, dst);\n}\n\nvoid ggml_cann_step(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n float alphaValue = 0.0f;\n acl_scalar_ptr alpha = nullptr;\n alpha = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, GtScalar, acl_src.get(), alpha.get(), acl_dst.get());\n}\n\n/**\n * @brief Performs expert-specific matrix multiplication (MoE) with\n * floating-point precision using the CANN backend.\n *\n * This function executes a matrix multiplication operation tailored for\n * Mixture of Experts (MoE) models, where the input tensor is multiplied\n * with expert-specific weight matrices. It uses the CANN backend for\n * efficient computation and stores the result in the destination tensor `dst`.\n * The operation may leverage identity-based optimizations or routing masks\n * as part of sparse expert selection.\n *\n * @param ctx The context for executing CANN backend operations.\n * @param dst The destination tensor where the MoE multiplication result\n * will be stored.\n *\n * @note This function assumes floating-point data types and is designed for\n * MoE architectures, possibly involving sparse expert routing.\n */\nstatic void ggml_cann_mul_mat_id_fp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n //dst [M, K, N, 1]\n ggml_tensor * src0 = dst->src[0]; //src0\t[D, M, A, 1] -> [D, M, K, 1]\n ggml_tensor * src1 = dst->src[1]; //src1\t[D, B, N, 1], B = K or B = 1 -> [D, 1, K, 1]\n ggml_tensor * ids = dst->src[2]; //ids\t[K, N]\n\n GGML_ASSERT(src0->ne[3] == 1);\n GGML_ASSERT(src1->ne[3] == 1);\n GGML_ASSERT(dst->ne[3] == 1);\n\n int64_t batch = src1->ne[2];\n GGML_ASSERT(batch == ids->ne[1]);\n\n ggml_cann_pool_alloc export_allocator(ctx.pool(), src0->ne[0] * src0->ne[1] * ids->ne[0] * ggml_element_size(src0));\n void * export_ptr = export_allocator.get();\n for (int64_t i = 0; i < batch; i++) {\n acl_tensor_ptr select_index = ggml_cann_create_tensor(ids, ids->ne, ids->nb, 1, ACL_FORMAT_ND, i * ids->nb[1]);\n acl_tensor_ptr export_weight = ggml_cann_create_tensor(src0, src0->ne, src0->nb, 3);\n\n int64_t select_export_ne[] = { src0->ne[0], src0->ne[1], ids->ne[0] };\n size_t select_export_nb[3];\n select_export_nb[0] = src0->nb[0];\n for (int k = 1; k < 3; k++) {\n select_export_nb[k] = select_export_nb[k - 1] * select_export_ne[k - 1];\n }\n\n acl_tensor_ptr select_export =\n ggml_cann_create_tensor(export_ptr, ggml_cann_type_mapping(src0->type), ggml_element_size(src0),\n select_export_ne, select_export_nb, 3);\n GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, export_weight.get(), 0, select_index.get(), select_export.get());\n\n int64_t select_transpose_ne[] = { select_export_ne[1], select_export_ne[0], select_export_ne[2] };\n size_t select_transpose_nb[] = { select_export_nb[1], select_export_nb[0], select_export_nb[2] };\n acl_tensor_ptr select_export_transpose =\n ggml_cann_create_tensor(export_ptr, ggml_cann_type_mapping(src0->type), ggml_element_size(src0),\n select_transpose_ne, select_transpose_nb, 3);\n\n int64_t active_tensor_ne[] = { src1->ne[0], 1, src1->ne[1] };\n size_t active_tensor_nb[] = { src1->nb[0], src1->nb[1], src1->nb[1] };\n acl_tensor_ptr active_tensor =\n ggml_cann_create_tensor(src1, active_tensor_ne, active_tensor_nb, 3, ACL_FORMAT_ND, i * src1->nb[2]);\n\n int64_t dst_ne[] = { dst->ne[0], 1, dst->ne[1] };\n size_t dst_nb[] = { dst->nb[0], dst->nb[1], dst->nb[1] };\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, dst_ne, dst_nb, 3, ACL_FORMAT_ND, i * dst->nb[2]);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, BatchMatMul, active_tensor.get(), select_export_transpose.get(), acl_dst.get(), 2);\n }\n}\n\n/**\n * @brief Performs quantized matrix multiplication for Mixture of Experts (MoE)\n * models using the CANN backend.\n *\n * This function implements MUL_MAT_ID operation for quantized weight matrices\n * (Q4_0 and Q8_0 formats). It selects expert-specific weight matrices based on\n * the provided expert indices, and computes matrix multiplication using CANN's\n * WeightQuantBatchMatmulV2 operator.\n *\n * The function performs the following steps:\n * 1. Converts input/output tensors to F16 format if necessary\n * 2. Uses IndexSelect to extract expert-specific weights and scales based on indices\n * 3. Performs quantized matrix multiplication for each expert using WeightQuantBatchMatmulV2\n * 4. Converts output back to the target type if needed\n *\n * Tensor shapes:\n * - dst: [M, K, N, 1] - output tensor\n * - src0: [D, M, A, 1] - quantized weight matrices (Q4_0 or Q8_0)\n * - src1: [D, B, N, 1] - input activations (B = K for per-expert input, or B = 1 for broadcast)\n * - ids: [K, N] - expert indices for routing\n *\n * @param ctx The CANN backend context for operation execution.\n * @param dst The destination tensor where the multiplication result will be stored.\n *\n * @note Only Q4_0 and Q8_0 quantization formats are supported.\n * @note The function handles automatic type conversion to/from F16 as needed by the hardware.\n */\nstatic void ggml_cann_mul_mat_id_quant(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n // dst: [M, K, N, 1]\n // src0: [D, M, A, 1] - quantized weights\n // src1: [D, B, N, 1] - input activations, B = K or B = 1\n // ids: [K, N] - expert indices\n ggml_tensor * src0 = dst->src[0];\n ggml_tensor * src1 = dst->src[1];\n ggml_tensor * ids = dst->src[2];\n\n GGML_ASSERT(src0->ne[3] == 1);\n GGML_ASSERT(src1->ne[3] == 1);\n GGML_ASSERT(dst->ne[3] == 1);\n GGML_ASSERT(src1->ne[2] == ids->ne[1]);\n\n const int64_t n_batches = ids->ne[1];\n const int64_t n_select_experts = ids->ne[0];\n const enum ggml_type type = src0->type;\n\n const int32_t group_size = QK8_0; // Both Q4_0 and Q8_0 use group size of 32\n GGML_ASSERT(group_size == QK4_0);\n\n // Calculate element size for quantized weights\n const float weight_elem_size =\n (type == GGML_TYPE_Q4_0) ? 0.5f :\n (type == GGML_TYPE_Q8_0) ? 1.0f :\n (GGML_ABORT(\"MUL_MAT_ID only supports Q4_0 and Q8_0\"), 0.0f);\n\n // Calculate scale offset in memory\n const size_t weight_size = src0->ne[0] * src0->ne[1] * src0->ne[2] * weight_elem_size;\n const size_t scale_elem_size = sizeof(uint16_t);\n char * scale_data = (char *) src0->data + weight_size;\n\n // Allocate buffers for selected expert weights and scales\n const size_t selected_weight_size = src0->ne[0] * src0->ne[1] * n_select_experts * weight_elem_size;\n ggml_cann_pool_alloc selected_weight_alloc(ctx.pool(), selected_weight_size);\n void * selected_weight_buffer = selected_weight_alloc.get();\n\n const size_t selected_scale_size = (src0->ne[0] / group_size) * src0->ne[1] * n_select_experts * scale_elem_size;\n ggml_cann_pool_alloc selected_scale_alloc(ctx.pool(), selected_scale_size);\n void * selected_scale_buffer = selected_scale_alloc.get();\n\n // Helper lambda to allocate and cast tensor to F16 if needed\n constexpr size_t f16_elem_size = sizeof(uint16_t);\n auto prepare_f16_buffer = [&](ggml_tensor * tensor, ggml_cann_pool_alloc & allocator,\n bool need_cast = false) -> void * {\n if (tensor->type == GGML_TYPE_F16) {\n return tensor->data;\n }\n\n size_t total_size = f16_elem_size;\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n total_size *= tensor->ne[i];\n }\n void * buffer = allocator.alloc(total_size);\n\n if (need_cast == false) {\n return buffer;\n }\n\n int64_t ne[GGML_MAX_DIMS];\n size_t nb[GGML_MAX_DIMS] = { f16_elem_size };\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n ne[i] = tensor->ne[i];\n if (i > 0) {\n nb[i] = nb[i - 1] * ne[i - 1];\n }\n }\n\n acl_tensor_ptr src_tensor = ggml_cann_create_tensor(tensor);\n acl_tensor_ptr f16_tensor = ggml_cann_create_tensor(buffer, ACL_FLOAT16, f16_elem_size, ne, nb, GGML_MAX_DIMS);\n aclnn_cast(ctx, src_tensor.get(), f16_tensor.get(), ACL_FLOAT16);\n\n return buffer;\n };\n\n // Prepare input and output buffers\n ggml_cann_pool_alloc input_alloc(ctx.pool());\n void * input_buffer = prepare_f16_buffer(src1, input_alloc, true);\n\n ggml_cann_pool_alloc output_alloc(ctx.pool());\n void * output_buffer = prepare_f16_buffer(dst, output_alloc, false);\n\n // Process each batch\n for (int64_t batch_idx = 0; batch_idx < n_batches; batch_idx++) {\n // Create index tensor for current batch\n const size_t index_offset = batch_idx * ids->nb[1];\n acl_tensor_ptr batch_indices = ggml_cann_create_tensor(ids, ids->ne, ids->nb, 1, ACL_FORMAT_ND, index_offset);\n\n // Select quantized weights using expert indices\n // Q4_0 stores 2 values per byte, Q8_0 stores 1 value per byte\n const int64_t weight_d = (type == GGML_TYPE_Q4_0) ? src0->ne[0] / 2 : src0->ne[0];\n const int64_t weight_m = src0->ne[1];\n const int64_t weight_n_experts = src0->ne[2];\n\n int64_t weight_ne[3] = { weight_d, weight_m, weight_n_experts };\n size_t weight_nb[3] = { sizeof(int8_t), weight_d * sizeof(int8_t), weight_d * weight_m * sizeof(int8_t) };\n\n acl_tensor_ptr all_weights =\n ggml_cann_create_tensor(src0->data, ACL_INT8, sizeof(int8_t), weight_ne, weight_nb, 3);\n\n int64_t selected_weight_ne[3] = { weight_d, weight_m, n_select_experts };\n size_t selected_weight_nb[3] = { sizeof(int8_t), weight_d * sizeof(int8_t),\n weight_d * weight_m * sizeof(int8_t) };\n\n acl_tensor_ptr selected_weights = ggml_cann_create_tensor(selected_weight_buffer, ACL_INT8, sizeof(int8_t),\n selected_weight_ne, selected_weight_nb, 3);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, all_weights.get(), 0, batch_indices.get(), selected_weights.get());\n\n // Select scales using the same expert indices\n const int64_t scale_d = src0->ne[0] / group_size;\n int64_t scale_ne[3] = { scale_d, weight_m, weight_n_experts };\n size_t scale_nb[3] = { scale_elem_size, scale_d * scale_elem_size, scale_d * weight_m * scale_elem_size };\n\n acl_tensor_ptr all_scales =\n ggml_cann_create_tensor(scale_data, ACL_FLOAT16, scale_elem_size, scale_ne, scale_nb, 3);\n\n int64_t selected_scale_ne[3] = { scale_d, weight_m, n_select_experts };\n size_t selected_scale_nb[3] = { scale_elem_size, scale_d * scale_elem_size,\n scale_d * weight_m * scale_elem_size };\n\n acl_tensor_ptr selected_scales = ggml_cann_create_tensor(selected_scale_buffer, ACL_FLOAT16, scale_elem_size,\n selected_scale_ne, selected_scale_nb, 3);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, all_scales.get(), 0, batch_indices.get(), selected_scales.get());\n\n // Process each expert for current batch\n // IndexSelect output layout: [D, M, K] in contiguous format\n // WeightQuantBatchMatmulV2 expects: [M, D] with row-major stride\n for (int64_t expert_idx = 0; expert_idx < n_select_experts; expert_idx++) {\n // Determine input offset: broadcast if src1->ne[1]==1, otherwise use per-expert input\n const size_t input_offset =\n (batch_idx * src1->ne[1] + (src1->ne[1] == 1 ? 0 : expert_idx)) * src1->ne[0] * f16_elem_size;\n const size_t output_offset = (batch_idx * dst->ne[1] + expert_idx) * dst->ne[0] * f16_elem_size;\n\n // Create weight view for current expert: [D, M, K] -> [M, D]\n int64_t weight_view_ne[2] = { weight_m, src0->ne[0] };\n float weight_view_nb[2] = { src0->ne[0] * weight_elem_size, weight_elem_size };\n const size_t weight_view_offset = expert_idx * selected_weight_nb[2];\n\n acl_tensor_ptr weight_view =\n ggml_cann_create_tensor(selected_weight_buffer, ggml_cann_type_mapping(type), weight_elem_size,\n weight_view_ne, weight_view_nb, 2, ACL_FORMAT_ND, weight_view_offset);\n\n // Create scale view for current expert: [D, M, K] -> [M, D]\n int64_t scale_view_ne[2] = { weight_m, scale_d };\n size_t scale_view_nb[2] = { selected_scale_nb[1], selected_scale_nb[0] };\n const size_t scale_view_offset = expert_idx * selected_scale_nb[2];\n\n acl_tensor_ptr scale_view =\n ggml_cann_create_tensor(selected_scale_buffer, ACL_FLOAT16, scale_elem_size, scale_view_ne,\n scale_view_nb, 2, ACL_FORMAT_ND, scale_view_offset);\n\n // Create input activation tensor [D, 1]\n int64_t input_ne[2] = { src1->ne[0], 1 };\n size_t input_nb[2] = { f16_elem_size, src1->ne[0] * f16_elem_size };\n\n acl_tensor_ptr input_tensor = ggml_cann_create_tensor(input_buffer, ACL_FLOAT16, f16_elem_size, input_ne,\n input_nb, 2, ACL_FORMAT_ND, input_offset);\n\n // Create output tensor [M, 1]\n int64_t output_ne[2] = { dst->ne[0], 1 };\n size_t output_nb[2] = { f16_elem_size, dst->ne[0] * f16_elem_size };\n\n acl_tensor_ptr output_tensor = ggml_cann_create_tensor(output_buffer, ACL_FLOAT16, f16_elem_size, output_ne,\n output_nb, 2, ACL_FORMAT_ND, output_offset);\n\n // Perform quantized matrix multiplication\n GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, input_tensor.get(), weight_view.get(),\n scale_view.get(), nullptr, nullptr, nullptr, nullptr, group_size,\n output_tensor.get());\n }\n }\n\n // Cast output back to original type if we used a temporary F16 buffer\n if (dst->type != GGML_TYPE_F16) {\n int64_t ne[GGML_MAX_DIMS];\n size_t nb[GGML_MAX_DIMS] = { f16_elem_size };\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n ne[i] = dst->ne[i];\n if (i > 0) {\n nb[i] = nb[i - 1] * ne[i - 1];\n }\n }\n\n acl_tensor_ptr f16_output =\n ggml_cann_create_tensor(output_buffer, ACL_FLOAT16, f16_elem_size, ne, nb, GGML_MAX_DIMS);\n acl_tensor_ptr dst_tensor = ggml_cann_create_tensor(dst);\n\n aclnn_cast(ctx, f16_output.get(), dst_tensor.get(), ggml_cann_type_mapping(dst->type));\n }\n}\n\nvoid ggml_cann_mul_mat_id(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n const enum ggml_type type = dst->src[0]->type;\n switch (type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n ggml_cann_mul_mat_id_fp(ctx, dst);\n break;\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q8_0:\n ggml_cann_mul_mat_id_quant(ctx, dst);\n break;\n default:\n GGML_ABORT(\"Unsupported type for mul_mat_id\");\n break;\n }\n}\n\nvoid ggml_cann_flash_attn_ext(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0]; // q, fp32 | B, N, S, D (uncont) -> B, S, N, D (cont)\n ggml_tensor * src1 = dst->src[1]; // k, fp16 | B, N, S, D (uncont) -> B, S, N, D (cont)\n ggml_tensor * src2 = dst->src[2]; // v, fp16 | B, N, S, D (uncont) -> B, S, N, D (cont)\n ggml_tensor * src3 = dst->src[3]; // mask, fp16\n\n // B, N, S, D (uncont) -> B, S, N, D (cont)\n int64_t src0_bsnd_ne[GGML_MAX_DIMS];\n memcpy(src0_bsnd_ne, src0->ne, GGML_MAX_DIMS * sizeof(int64_t));\n size_t src0_bsnd_nb[GGML_MAX_DIMS];\n memcpy(src0_bsnd_nb, src0->nb, GGML_MAX_DIMS * sizeof(size_t));\n int64_t src1_bsnd_ne[GGML_MAX_DIMS];\n memcpy(src1_bsnd_ne, src1->ne, GGML_MAX_DIMS * sizeof(int64_t));\n size_t src1_bsnd_nb[GGML_MAX_DIMS];\n memcpy(src1_bsnd_nb, src1->nb, GGML_MAX_DIMS * sizeof(size_t));\n int64_t src2_bsnd_ne[GGML_MAX_DIMS];\n memcpy(src2_bsnd_ne, src2->ne, GGML_MAX_DIMS * sizeof(int64_t));\n size_t src2_bsnd_nb[GGML_MAX_DIMS];\n memcpy(src2_bsnd_nb, src2->nb, GGML_MAX_DIMS * sizeof(size_t));\n\n auto transpose12 = [](int64_t * ne, size_t * nb) {\n int64_t ne_tmp = ne[1];\n size_t nb_tmp = nb[1];\n ne[1] = ne[2];\n nb[1] = nb[2];\n ne[2] = ne_tmp;\n nb[2] = nb_tmp;\n };\n\n transpose12(src0_bsnd_ne, src0_bsnd_nb);\n transpose12(src1_bsnd_ne, src1_bsnd_nb);\n transpose12(src2_bsnd_ne, src2_bsnd_nb);\n\n float maxBias = 0.0f;\n float scaleValue = 1.0f;\n float logitSoftcap = 0.0f;\n memcpy(&scaleValue, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&maxBias, (float *) dst->op_params + 1, sizeof(float));\n memcpy(&logitSoftcap, (float *) dst->op_params + 2, sizeof(float));\n\n if (logitSoftcap == 0.0f) {\n size_t faElemSize = sizeof(uint16_t);\n auto faDataType = ACL_FLOAT16; //ACL_BF16;\n\n acl_tensor_ptr acl_q_tensor = nullptr;\n acl_tensor_ptr acl_k_tensor = nullptr;\n acl_tensor_ptr acl_v_tensor = nullptr;\n\n // Step 1: cast the src0 (Query) to fp16 if needed\n ggml_cann_pool_alloc src0_f16_allocator(ctx.pool());\n void * src0_f16_buffer = nullptr;\n\n if (ggml_cann_type_mapping(src0->type) != faDataType) {\n acl_tensor_ptr acl_src0_f32_tensor =\n ggml_cann_create_tensor(src0, src0_bsnd_ne, src0_bsnd_nb, GGML_MAX_DIMS);\n src0_f16_buffer = src0_f16_allocator.alloc(ggml_nelements(src0) * faElemSize);\n\n int64_t * src0_f16_ne = src0_bsnd_ne;\n size_t src0_f16_nb[GGML_MAX_DIMS];\n src0_f16_nb[0] = sizeof(uint16_t);\n for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n src0_f16_nb[i] = src0_f16_nb[i - 1] * src0_f16_ne[i - 1];\n }\n\n acl_q_tensor = ggml_cann_create_tensor(src0_f16_buffer, faDataType, faElemSize, src0_f16_ne, src0_f16_nb,\n GGML_MAX_DIMS);\n aclnn_cast(ctx, acl_src0_f32_tensor.get(), acl_q_tensor.get(), faDataType);\n } else {\n acl_q_tensor = ggml_cann_create_tensor(src0, src0_bsnd_ne, src0_bsnd_nb, GGML_MAX_DIMS);\n }\n\n // Step 2: create the acl tensors for src1 (Key), src2 (Value),\n // and the direct output from FusedInferAttention\n\n acl_k_tensor = ggml_cann_create_tensor(src1, src1_bsnd_ne, src1_bsnd_nb, GGML_MAX_DIMS);\n acl_v_tensor = ggml_cann_create_tensor(src2, src2_bsnd_ne, src2_bsnd_nb, GGML_MAX_DIMS);\n\n // Step 3: create the PSEShift tensor if needed\n // this tensor is considered as mask (f16) in the llama.cpp\n acl_tensor_ptr bcast_pse_tensor;\n ggml_cann_pool_alloc bcast_pse_allocator(ctx.pool());\n if (src3 != nullptr) {\n // Construct the truncated pse tensor (common for prefill/decode)\n int64_t trunc_pse_ne[GGML_MAX_DIMS] = {\n src3->ne[0], // D\n src0->ne[1], // S (number of Q tokens)\n src3->ne[2], // mask N\n src3->ne[3] // B\n };\n size_t * trunc_pse_nb = src3->nb;\n\n acl_tensor_ptr acl_mask_f16_trunc_tensor = ggml_cann_create_tensor(\n src3->data, ACL_FLOAT16, sizeof(uint16_t), trunc_pse_ne, trunc_pse_nb, GGML_MAX_DIMS);\n\n int64_t bcast_pse_ne[GGML_MAX_DIMS];\n size_t bcast_pse_nb[GGML_MAX_DIMS];\n bcast_pse_ne[0] = src3->ne[0]; // D\n bcast_pse_ne[1] = src0->ne[1]; // S\n bcast_pse_ne[2] = src0->ne[2]; // N (num_heads)\n bcast_pse_ne[3] = src3->ne[3]; // B\n if (maxBias == 0.0f) {\n // When maxBias == 0.0f, use nb = 0 reduce once repeat (Qwen2)\n // Construct the bcast tensor (simulate repeat on the head dimension using stride=0)\n bcast_pse_nb[0] = sizeof(uint16_t);\n bcast_pse_nb[1] = bcast_pse_nb[0] * bcast_pse_ne[0];\n bcast_pse_nb[2] = 0; // <---- the head dimension shares the same data\n bcast_pse_nb[3] = src3->nb[3];\n\n bcast_pse_tensor = ggml_cann_create_tensor(src3->data, ACL_FLOAT16, sizeof(uint16_t), bcast_pse_ne,\n bcast_pse_nb, GGML_MAX_DIMS);\n\n } else {\n bcast_pse_nb[0] = sizeof(uint16_t);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n bcast_pse_nb[i] = bcast_pse_nb[i - 1] * bcast_pse_ne[i - 1];\n }\n\n void * bcast_pse_buffer =\n bcast_pse_allocator.alloc(ggml_nelements(src3) * src0->ne[2] * sizeof(uint16_t));\n\n bcast_pse_tensor = ggml_cann_create_tensor(bcast_pse_buffer, ACL_FLOAT16, sizeof(uint16_t),\n bcast_pse_ne, bcast_pse_nb, GGML_MAX_DIMS);\n\n int64_t repeats[] = { 1, src0->ne[2], 1, 1 };\n aclnn_repeat(ctx, acl_mask_f16_trunc_tensor.get(), bcast_pse_tensor.get(), repeats);\n\n // alibi\n // Compute the slope if needed. Derived from ggml_cann_softmax().\n const int64_t n_heads = src0->ne[2];\n ggml_cann_pool_alloc slope_allocator(ctx.pool(), n_heads * sizeof(uint16_t));\n void * slope_buffer = slope_allocator.get();\n aclnn_get_slope(ctx, n_heads, slope_buffer, maxBias, GGML_TYPE_F16);\n\n int64_t slope_ne[] = { 1, 1, n_heads, 1 };\n size_t slope_nb[GGML_MAX_DIMS];\n slope_nb[0] = sizeof(uint16_t);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n slope_nb[i] = slope_nb[i - 1] * slope_ne[0];\n }\n\n acl_tensor_ptr slope_tensor = ggml_cann_create_tensor(slope_buffer, ACL_FLOAT16, sizeof(uint16_t),\n slope_ne, slope_nb, GGML_MAX_DIMS);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, bcast_pse_tensor.get(), slope_tensor.get());\n }\n }\n\n // Step 4: set the inputs for FusedInferAttention.\n acl_tensor_list_ptr acl_k_tensor_list = ggml_cann_create_tensor_list(acl_k_tensor);\n acl_tensor_list_ptr acl_v_tensor_list = ggml_cann_create_tensor_list(acl_v_tensor);\n\n int64_t numHeads = src0->ne[2]; // N\n int64_t numKeyValueHeads = src1->ne[2];\n // double scaleValue = 1 / sqrt(src0->ne[0]); // 1/sqrt(d)\n int64_t preTokens = 65535;\n int64_t nextTokens = 65535;\n char layout[5] = { 'B', 'S', 'N', 'D', 0 };\n int64_t sparseMode = 0;\n int64_t innerPrecise = (src0->ne[1] == 1) ? 0 : 2;\n int64_t blockSize = 0;\n int64_t antiquantMode = 0;\n bool softmaxLseFlag = false;\n int64_t keyAntiquantMode = 0;\n int64_t valueAntiquantMode = 0;\n\n GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n acl_tensor_ptr fa_dst_tensor;\n acl_tensor_ptr acl_dst_tensor;\n ggml_cann_pool_alloc out_f16_allocator(ctx.pool());\n if (dst->type == GGML_TYPE_F32) {\n void * out_f16_buffer = out_f16_allocator.alloc(ggml_nelements(dst) * faElemSize);\n\n int64_t * out_f16_ne = src0_bsnd_ne;\n size_t out_f16_nb[GGML_MAX_DIMS];\n out_f16_nb[0] = faElemSize;\n for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n out_f16_nb[i] = out_f16_nb[i - 1] * out_f16_ne[i - 1];\n }\n\n fa_dst_tensor =\n ggml_cann_create_tensor(out_f16_buffer, faDataType, faElemSize, out_f16_ne, out_f16_nb, GGML_MAX_DIMS);\n } else {\n fa_dst_tensor = ggml_cann_create_tensor(dst);\n }\n\n GGML_CANN_CALL_ACLNN_OP(ctx, FusedInferAttentionScoreV2, acl_q_tensor.get(), acl_k_tensor_list.get(),\n acl_v_tensor_list.get(), // q, k, v\n bcast_pse_tensor.get(), nullptr, // pse, mask\n nullptr, nullptr, // actSeqLen, actSeqLenkv\n nullptr, nullptr, // deqScale1, quantScale1\n nullptr, nullptr, nullptr, // deqScale2, quantScale2, quantOffset2\n nullptr, nullptr, // antiquantScale, antiquantOffset\n nullptr, // blockTable\n nullptr, nullptr, // qPadSize, kvPadSize\n nullptr, nullptr, // kAntiquantScale, kAntiQuantOffset\n nullptr, nullptr, // vAntiquantScale, vAntiQuantOffset\n nullptr, nullptr, nullptr, // kSharedPrefix, vSharedPrefix, actSharedLen\n numHeads, scaleValue, // heads, scaleValue\n preTokens, nextTokens, // preTokens, nextTokens\n layout, // inputLayout\n numKeyValueHeads, // numKVHeads\n sparseMode, innerPrecise, // sparseMode, innerPrecise\n blockSize, antiquantMode, // blockSize, antiquantMode\n softmaxLseFlag, // softmaxLseFlag\n keyAntiquantMode, valueAntiquantMode, // keyAntiqMode, valueAntiqMode\n fa_dst_tensor.get(), // attentionOut\n nullptr // softmaxLse\n );\n\n if (dst->type == GGML_TYPE_F32) {\n // Step 6: post-processing, permute and cast to f32\n acl_tensor_ptr acl_dst_tensor = ggml_cann_create_tensor(dst);\n aclnn_cast(ctx, fa_dst_tensor.get(), acl_dst_tensor.get(), ggml_cann_type_mapping(dst->type));\n }\n } else {\n GGML_ABORT(\"Function is not implemented.\");\n }\n}\n\nstatic void ggml_cann_out_prod_fp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0]; // weight\n ggml_tensor * src1 = dst->src[1]; // input\n GGML_TENSOR_BINARY_OP_LOCALS\n\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, acl_dst.get());\n\n const int64_t dps2 = ne2 / ne02;\n const int64_t dps3 = ne3 / ne03;\n for (int64_t i3 = 0; i3 < ne3; i3++) {\n for (int64_t i2 = 0; i2 < ne2; i2++) {\n const int64_t i02 = i2 / dps2;\n const int64_t i03 = i3 / dps3;\n\n const int64_t i12 = i2;\n const int64_t i13 = i3;\n acl_tensor_ptr accumulator =\n ggml_cann_create_tensor((char *) dst->data + i2 * nb2 + i3 * nb3, ggml_cann_type_mapping(dst->type),\n ggml_type_size(dst->type), dst->ne, dst->nb, 2);\n\n // The outer product needs to be accumulated in this dimension.\n for (int64_t i1 = 0; i1 < ne11; i1++) {\n acl_tensor_ptr acl_input = ggml_cann_create_tensor(\n (char *) src1->data + i1 * nb11 + i12 * nb12 + i13 * nb13, ggml_cann_type_mapping(src0->type),\n ggml_type_size(src0->type), src1->ne, src1->nb, 1);\n\n acl_tensor_ptr acl_weight = ggml_cann_create_tensor(\n (char *) src0->data + i1 * nb01 + i02 * nb02 + i03 * nb03, ggml_cann_type_mapping(src0->type),\n ggml_type_size(src0->type), src0->ne, src0->nb, 1);\n\n ggml_cann_pool_alloc output_allocator(ctx.pool());\n void * output_buffer = output_allocator.alloc(ggml_nbytes(dst));\n acl_tensor_ptr acl_out = ggml_cann_create_tensor(output_buffer, ggml_cann_type_mapping(dst->type),\n ggml_type_size(dst->type), dst->ne, dst->nb, 2);\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Ger, acl_input.get(), acl_weight.get(), acl_out.get());\n float alpha_value = 1.0f;\n aclScalar * alpha = aclCreateScalar(&alpha_value, ACL_FLOAT);\n GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, accumulator.get(), acl_out.get(), alpha);\n }\n }\n }\n}\n\nvoid ggml_cann_out_prod(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n\n const enum ggml_type type = src0->type;\n\n switch (type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n ggml_cann_out_prod_fp(ctx, dst);\n break;\n default:\n GGML_ABORT(\"Unsupport type for GGML_OP_OUT_PROD\");\n break;\n }\n}\n\nvoid ggml_cann_ssm_conv(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0]; // conv_x\n ggml_tensor * src1 = dst->src[1]; // conv1d.weight\n\n // This op is currently defined only for F32 in ggml_cpu\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n // Shapes follow ggml_compute_forward_ssm_conv_f32\n const int64_t nc = src1->ne[0]; // d_conv\n const int64_t ncs = src0->ne[0]; // d_conv - 1 + n_t\n const int64_t nr = src0->ne[1]; // d_inner\n const int64_t n_s = src0->ne[2]; // n_seqs\n\n const int64_t n_t = dst->ne[1]; // tokens per sequence\n\n GGML_ASSERT(dst->ne[0] == nr); // dst: {d_inner, n_t, n_s}\n GGML_ASSERT(src1->ne[1] == nr); // weight: {d_conv, d_inner}\n GGML_ASSERT(ncs == nc - 1 + n_t); // conv_x: {d_conv - 1 + n_t, d_inner, n_s}\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n GGML_ASSERT(src1->nb[0] == sizeof(float));\n\n // --- Build CANN tensors ---\n\n // 1) Input: conv_x as NCL\n //\n // src0->ne = { ncs, nr, n_s, 1 } // {L_in, C, N}\n // Passing ACL_FORMAT_NCL here means:\n // reversed dims -> [N, C, L_in] = [n_s, nr, ncs]\n acl_tensor_ptr acl_x = ggml_cann_create_tensor(src0, src0->ne, src0->nb, 3, ACL_FORMAT_NCL);\n\n // 2) Weights: depthwise conv kernel, view src1 as {K, 1, C}\n //\n // src1 original: ne = { nc, nr, 1, 1 } // [K, C, 1, 1]\n // we want a view: ne_w = { nc, 1, nr } // [K, 1, C]\n // so that reversed dims -> [C, 1, K] which matches\n // [out_channels, in_channels/groups, kernel_size]\n int64_t w_ne[GGML_MAX_DIMS] = { nc, 1, nr, 1 }; // [K, 1 input ch. per group, C groups]\n // Layout: src1 data is [K, C] with\n // offset(k, c) = k*nb0 + c*nb1\n // We want offset_w(k, 0, c) = k*nb0 + c*nb1,\n // so we can reuse nb0 and nb1, and set nb2 = nb1.\n size_t w_nb[GGML_MAX_DIMS] = { src1->nb[0], src1->nb[1], src1->nb[1], src1->nb[3] }; // same as src1\n\n acl_tensor_ptr acl_w = ggml_cann_create_tensor(src1->data, ggml_cann_type_mapping(src1->type),\n ggml_type_size(src1->type), w_ne, w_nb, 3, ACL_FORMAT_NCL);\n\n // 3) Output: dst is { d_inner, n_t, n_s } (CLN)\n //\n // We need an NCL view of the same buffer:\n // desired NCL logical shape: { L_out = n_t, C = nr, N = n_s }\n //\n // Original CLN layout:\n // dst->ne = { nr, n_t, n_s }\n // dst->nb[0] = sizeof(float)\n // dst->nb[1] = nr * sizeof(float)\n // dst->nb[2] = nr * n_t * sizeof(float)\n //\n // We want offset_new(L, C, N) = offset_orig(C, L, N).\n // Choose:\n // nb_y[0] = nr * sizeof(float); // step in L\n // nb_y[1] = sizeof(float); // step in C\n // nb_y[2] = nr * n_t * sizeof(float); // step in N\n int64_t y_ne[GGML_MAX_DIMS] = { n_t, nr, n_s, 1 }; // [L_out, C, N]\n size_t y_nb[GGML_MAX_DIMS] = { dst->ne[0] * sizeof(float), sizeof(float), dst->ne[0] * dst->ne[1] * sizeof(float),\n dst->nb[3] }; // [nr, 1, nr * n_t]\n\n acl_tensor_ptr acl_y = ggml_cann_create_tensor(dst->data, ggml_cann_type_mapping(dst->type),\n ggml_type_size(dst->type), y_ne, y_nb, 3, ACL_FORMAT_NCL);\n\n // --- Conv1d parameters: depthwise, stride 1, no padding (\"valid\") ---\n int64_t strideVal[1] = { 1 };\n int64_t paddingVal[1] = { 0 };\n int64_t dilationVal[1] = { 1 };\n\n acl_int_array_ptr stride = ggml_cann_create_int_array(strideVal, 1);\n acl_int_array_ptr padding = ggml_cann_create_int_array(paddingVal, 1);\n acl_int_array_ptr dilation = ggml_cann_create_int_array(dilationVal, 1);\n\n const bool transposed = false;\n const int64_t groups = nr; // depthwise: one group per inner dim\n int8_t cubeMathType = 0;\n\n#ifdef ASCEND_310P\n cubeMathType = 1;\n#endif\n\n GGML_CANN_CALL_ACLNN_OP(ctx, Convolution,\n acl_x.get(), // input: N, C, L_in = ncs\n acl_w.get(), // weight: [C, 1, K] with groups=nr\n nullptr, // bias\n stride.get(), padding.get(), dilation.get(), transposed,\n padding.get(), // output padding (unused for non-transposed)\n groups, acl_y.get(), cubeMathType);\n}\n\nvoid ggml_cann_op_add_rms_norm_fused(ggml_backend_cann_context & ctx,\n ggml_tensor * add_node,\n ggml_tensor * rms_norm_node) {\n // Get the two input tensors for ADD operation\n ggml_tensor * x1 = add_node->src[0];\n ggml_tensor * x2 = add_node->src[1];\n\n // Create ACL tensors for the two ADD inputs\n acl_tensor_ptr acl_x1 = ggml_cann_create_tensor(x1);\n acl_tensor_ptr acl_x2 = ggml_cann_create_tensor(x2);\n\n // Get epsilon parameter from rms_norm_tensor\n float eps;\n memcpy(&eps, rms_norm_node->op_params, sizeof(float));\n\n // Build gamma tensor (RMS normalization scaling factor)\n // Gamma should match the normalized dimensions (last dimension of x1)\n size_t acl_gamma_nb[GGML_MAX_DIMS];\n acl_gamma_nb[0] = ggml_type_size(rms_norm_node->type);\n for (int i = 1; i < GGML_MAX_DIMS; i++) {\n acl_gamma_nb[i] = acl_gamma_nb[i - 1] * x1->ne[i - 1];\n }\n acl_tensor_ptr acl_gamma =\n get_cache_acl_tensor(ctx, &ctx.rms_norm_one_tensor_cache.cache, ctx.rms_norm_one_tensor_cache.size, x1->ne,\n acl_gamma_nb, rms_norm_node->type,\n 1, // dims - only the last dimension\n 1.0f // value\n );\n\n // Build rstdOut tensor (output for normalized standard deviation)\n // Shape should be the dimensions that are NOT normalized\n int64_t acl_rstd_ne[] = { 1, x1->ne[1], x1->ne[2], x1->ne[3] };\n size_t acl_rstd_nb[GGML_MAX_DIMS - 1];\n acl_rstd_nb[0] = sizeof(float);\n for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n acl_rstd_nb[i] = acl_rstd_nb[i - 1] * acl_rstd_ne[i - 1];\n }\n acl_tensor_ptr acl_rstd =\n get_cache_acl_tensor(ctx, &ctx.rms_norm_zero_tensor_cache.cache, ctx.rms_norm_zero_tensor_cache.size,\n acl_rstd_ne, acl_rstd_nb, GGML_TYPE_F32, GGML_MAX_DIMS,\n 0.0f // value\n );\n\n acl_tensor_ptr acl_xout = ggml_cann_create_tensor(add_node);\n\n // Create yOut tensor (final output after RMS normalization)\n acl_tensor_ptr acl_yout = ggml_cann_create_tensor(rms_norm_node);\n\n // Call fused ADD + RMS_NORM operator\n GGML_CANN_CALL_ACLNN_OP(ctx, AddRmsNorm, acl_x1.get(), acl_x2.get(), acl_gamma.get(),\n eps, // double type\n acl_yout.get(), acl_rstd.get(), acl_xout.get());\n}\n\nvoid ggml_cann_gated_linear_attn(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * k = dst->src[0];\n ggml_tensor * v = dst->src[1];\n ggml_tensor * q = dst->src[2];\n ggml_tensor * g = dst->src[3];\n ggml_tensor * s = dst->src[4];\n\n int64_t B = dst->src[4]->ne[1];\n int64_t T = dst->src[0]->ne[2];\n int64_t H = dst->src[0]->ne[1];\n int64_t C = dst->ne[0];\n int64_t D = C / H;\n int64_t L = T / B;\n\n int64_t ne_qkg[2] = { 1, D };\n int64_t ne_s[2] = { D, D };\n int64_t ne_st[2] = { ne_s[1], ne_s[0] };\n int64_t ne_vo[2] = { D, 1 };\n int64_t ne_q[1] = { D };\n size_t nb_base = ggml_type_size(k->type);\n size_t nb_qkg[2] = { nb_base, nb_base };\n size_t nb_s[2] = { nb_base, D * nb_base };\n size_t nb_st[2] = { nb_s[1], nb_s[0] };\n size_t nb_vo[2] = { nb_base, D * nb_base };\n size_t nb_q[1] = { nb_base };\n\n const float scale = ggml_get_op_params_f32(dst, 0);\n\n acl_tensor_ptr acl_s = ggml_cann_create_tensor(s, s->ne, s->nb, 2, ACL_FORMAT_ND);\n acl_tensor_ptr new_state = ggml_cann_create_tensor(dst, s->ne, s->nb, 2, ACL_FORMAT_ND, (B * L * H * D) * nb_base);\n cann_copy(ctx, acl_s.get(), new_state.get());\n\n for (int64_t b = 0; b < B; b++) {\n for (int64_t h = 0; h < H; h++) {\n size_t s_offset = (b * (H * D * D) + h * (D * D)) * nb_base;\n // D * D\n acl_tensor_ptr acl_s_new =\n ggml_cann_create_tensor(dst, ne_s, nb_s, 2, ACL_FORMAT_ND, (B * L * H * D) * nb_base + s_offset);\n acl_tensor_ptr acl_s_new_t =\n ggml_cann_create_tensor(dst, ne_st, nb_st, 2, ACL_FORMAT_ND, (B * L * H * D) * nb_base + s_offset);\n for (int64_t l = 0; l < L; l++) {\n size_t qkvgo_offset = (b * (L * H * D) + l * (H * D) + h * (D)) * nb_base;\n // D * 1\n acl_tensor_ptr acl_k = ggml_cann_create_tensor(k, ne_qkg, nb_qkg, 2, ACL_FORMAT_ND, qkvgo_offset);\n acl_tensor_ptr acl_g = ggml_cann_create_tensor(g, ne_qkg, nb_qkg, 2, ACL_FORMAT_ND, qkvgo_offset);\n // D\n acl_tensor_ptr acl_q = ggml_cann_create_tensor(q, ne_q, nb_q, 1, ACL_FORMAT_ND, qkvgo_offset);\n // 1 * D\n acl_tensor_ptr acl_v = ggml_cann_create_tensor(v, ne_vo, nb_vo, 2, ACL_FORMAT_ND, qkvgo_offset);\n // D\n acl_tensor_ptr acl_o = ggml_cann_create_tensor(dst, ne_q, nb_q, 1, ACL_FORMAT_ND, qkvgo_offset);\n // k ⊗ v\n size_t buf_size = D * D * nb_base;\n ggml_cann_pool_alloc buffer_allocator(ctx.pool(), buf_size);\n acl_tensor_ptr tmp_tensor = ggml_cann_create_tensor(\n buffer_allocator.get(), ggml_cann_type_mapping(k->type), nb_base, ne_s, nb_s, 2);\n aclnn_mul(ctx, acl_k.get(), acl_v.get(), tmp_tensor.get());\n //s_new = g ⊗ s_old + k ⊗ v\n aclnn_mul(ctx, acl_s_new.get(), acl_g.get(), nullptr);\n aclnn_add(ctx, acl_s_new.get(), tmp_tensor.get(), nullptr);\n // compute output\n GGML_CANN_CALL_ACLNN_OP(ctx, Mv, acl_s_new_t.get(), acl_q.get(), acl_o.get(), 1);\n aclnn_muls(ctx, acl_o.get(), scale, nullptr, true);\n }\n }\n }\n}\n" }, { "path": "src/ggml-cann/aclnn_ops.h", "content": "/**\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#ifndef CANN_ACLNN_OPS\n#define CANN_ACLNN_OPS\n\n#include \"acl_tensor.h\"\n#include \"common.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n\n/**\n * @brief Repeats a ggml tensor along each dimension to match the dimensions\n * of another tensor.\n *\n * @details This function repeats the elements of a source ggml tensor along\n * each dimension to create a destination tensor with the specified\n * dimensions. The operation is performed using the ACL backend and\n * executed asynchronously on the device.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The ggml tensor representing the destination, which op is\n * GGML_OP_REPEAT and specifies the desired dimensions.\n */\nvoid ggml_cann_repeat(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies the Leaky ReLU activation function to a tensor using the CANN\n * backend.\n *\n * @details This function computes the Leaky ReLU activation for each element of\n * the input tensor. The Leaky ReLU function allows a small gradient\n * when the unit is not active (i.e., when the input is negative). The\n * Leaky ReLU function is defined as:\n * \\f[\n * \\text{dst} = \\max(0, src) + \\text{negativeSlope} \\cdot \\min(0,\n * src)\n * \\f]\n * `negativeSlope` is in dst->params.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the result of the Leaky ReLU\n * activation is stored, which op is `GGML_OP_LEAKY_RELU`\n */\nvoid ggml_cann_leaky_relu(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Concatenates multiple tensors along a specified dimension using the\n * CANN backend.\n *\n * @param ctx The CANN context used for operations.\n * @param tensorList A pointer to the list of tensors to be concatenated.\n * @param dst The destination tensor where the result of the\n * concatenation is stored. dst->op is `GGML_OP_CONCAT`.\n * @param concat_dim The dimension along which the tensors are concatenated.\n *\n * @attention tensorList length should be 2 and the dimension using for concat\n * default to 1.\n */\nvoid ggml_cann_concat(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Generates a sequence of evenly spaced values within a specified\n * interval for a ggml tensor using the CANN backend.\n *\n * @details This function creates a sequence of numbers over a specified i\n * nterval, starting from `start`, ending before `stop`, and\n * incrementing by `step`. The sequence is stored in the destination\n * tensor `dst`.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the generated sequence will be stored.\n * `start`, 'stop' and 'step' are in dst->op_params and dst->op is\n * `GGML_OP_ARANGE`.\n */\nvoid ggml_cann_arange(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies a clamp operation to the elements of a ggml tensor using the\n * CANN backend.\n *\n * @details This function clamps the elements of the input tensor `src` to a\n * specified range defined by `min` and `max` values. The result is\n * stored in the destination tensor `dst`. The operation is defined as:\n * \\f[\n * y = \\max(\\min(x, max\\_value), min\\_value)\n * \\f]\n * where `x` is an element of the input tensor, and `y` is the\n * corresponding element in the output tensor.\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the clamped values will be stored.\n * dst->op is `GGML_OP_CLAMP`, `min` and `max` value is in dst->params.\n */\nvoid ggml_cann_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Scales the elements of a ggml tensor by a constant factor using the\n * CANN backend.\n *\n * @details This function multiplies each element of the input tensor `src` by\n * a scaling factor `scale`, storing the result in the destination\n * tensor `dst`. The operation is defined as:\n * \\f[\n * dst = src \\times scale\n * \\f]\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the scaled values will be stored.\n * dst->op is `GGML_OP_SCALE` and `scale` value is in dst->params.\n */\nvoid ggml_cann_scale(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Sorts the elements of a ggml tensor and returns the indices that\n * would sort the tensor using the CANN backend.\n *\n * @details This function performs an argsort operation on the input tensor\n * `src`. It sorts the elements of `src` in either ascending or\n * descending order, depending on the `GGML_SORT_ORDER_DESC`,\n * and returns the indices that would sort the original tensor.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the sorted indices will be stored.\n * dst->op is `GGML_OP_ARGSORT`.\n */\nvoid ggml_cann_argsort(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the Layer Normalization for a ggml tensor using the CANN\n * backend.\n *\n * @details This function applies the Layer Normalization operation on the\n * input tensor `src` and stores the result in the destination tensor\n * `dst`. Layer Normalization normalizes the features at each sample in\n * a mini-batch independently. It is commonly used in neural networks\n * to normalize the activations of a layer by adjusting and scaling\n * the outputs.\n * The operation is defined as:\n * \\f[\n * \\text { out }=\\frac{x-\\mathrm{E}[x]}{\\sqrt{\\text{Var}[x]+eps}}\n * \\f]\n * `Var` defaults dst->ne[0]. `eps` is in dst->params.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the normalized values will be stored.\n * @attention `Var` defaults to dst->ne[0].\n */\nvoid ggml_cann_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the L2 Normalization for a ggml tensor using the CANN\n * backend.\n *\n * @details This function applies the L2 Normalization operation on the\n * input tensor `src` and stores the result in the destination tensor\n * `dst`. L2 Normalization scales the input tensor such that the\n * L2 norm along the specified dimension equals 1. This operation\n * is commonly used in neural networks for feature normalization\n * and vector scaling.\n * The operation is defined as:\n * \\f[\n * \\text{out} = \\frac{x}{\\sqrt{\\sum{x^2}}}\n * \\f]\n * The normalization is performed along the last dimension by default.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the normalized values will be stored.\n * @attention The normalization is performed along the last dimension of the\n * input tensor by default.\n */\nvoid ggml_cann_l2_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the Cross Entropy Loss for a ggml tensor using the CANN\n * backend.\n *\n * @details This function computes the cross entropy loss between the predicted\n * logits and target probability distributions. The operation follows\n * the same computation pattern as the CPU implementation:\n * 1. Applies log_softmax to the logits along the class dimension\n * 2. Element-wise multiplication with target distributions\n * 3. Summation along the class dimension to get per-sample losses\n * 4. Global summation and scaling by -1/nr to get final loss\n *\n * The computation can be expressed as:\n * \\f[\n * \\text{loss} = -\\frac{1}{N} \\sum_{i=1}^{N} \\sum_{j=1}^{C} y_{ij} \\cdot \\log(\\text{softmax}(x_{ij}))\n * \\f]\n * where \\f$N\\f$ is the total number of samples, \\f$C\\f$ is the number\n * of classes, \\f$x\\f$ are the logits, and \\f$y\\f$ are the target\n * probability distributions.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the computed loss will be stored.\n * This should be a scalar tensor containing the final loss value.\n *\n * @note This implementation computes cross entropy between probability\n * distributions, not the typical classification cross entropy that\n * expects class indices as targets. Both input tensors (src0 and src1)\n * should have the same shape and represent probability distributions\n * over the class dimension.\n * @note The function expects two source tensors:\n * - dst->src[0]: Logits tensor (before softmax)\n * - dst->src[1]: Target probability distributions tensor\n * @note The computation is performed using CANN backend operators including\n * LogSoftmax, Mul, ReduceSum, and Muls for the final scaling.\n */\nvoid ggml_cann_cross_entropy_loss(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the Group Normalization for a ggml tensor using the CANN\n * backend.\n *\n * @brief This function applies the Group Normalization operation on the input\n * tensor `src` and stores the result in the destination tensor `dst`.\n * Group Normalization divides the channels into groups and normalizes\n * the features within each group across spatial locations.\n * It is commonly used in convolutional neural networks to improve\n * training stability and performance.\n * The operation is defined as:\n * \\f[\n * \\text { out }=\\frac{x-\\mathrm{E}[x]}{\\sqrt{\\text{Var}[x]+eps}}\n * \\f]\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the normalized values will be stored.\n * `n_groups` is in dst->params, which split C channel to `n_groups`.\n * dst->op is `GGML_OP_GROUP_NORM`.\n *\n * @attention eps defaults to 1e-6f.\n */\nvoid ggml_cann_group_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the accumulation of tensors using the CANN backend.\n *\n * @details This function performs an accumulation operation on two tensors.\n * Depending on the `inplace` flag, it either updates the destination\n * tensor `dst` in place by adding `alpha * src1` to it, or it creates\n * a new tensor as the result of `src0 + alpha * src1` and stores it in\n * `dst`.\n * The operation is defined as:\n * \\f[\n * dst = src0 + alpha \\times src1\n * \\f]\n * if `inplace` is `true`, `src0` is equal to 'dst'.\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the accumulated values will be stored.\n * `inplace` is in dst->params, and dst->op is `GGML_OP_ACC`.\n */\nvoid ggml_cann_acc(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the sum of elements along the last dimension of a ggml tensor\n * using the CANN backend.\n *\n * @details This function performs a reduction sum operation along the last\n * dimension of the input tensor `src`. The result of the sum is stored\n * in the destination tensor `dst`.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the reduced values will be stored。\n * dst->op is `GGML_OP_SUM_ROWS`.\n *\n * @attention `reduce_dims` defaults to 3, which means the last dimension.\n */\nvoid ggml_cann_sum_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the sum of elements in a ggml tensor.\n *\n * @details This function performs a reduction sum operation along the last\n * dimension of the input tensor `src`. The result of the sum is stored\n * in the destination tensor `dst`.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the reduced values will be stored。\n *\n */\n\nvoid ggml_cann_sum(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Upsamples a ggml tensor using nearest neighbor interpolation using\n * the CANN backend.\n *\n * @details This function performs upsampling of the input tensor `src` using\n * nearest neighbor interpolation. The upsampling is applied to the\n * height and width dimensions (last two dimensions) of the tensor. The\n * result is stored in the destination tensor `dst`, which must have\n * the appropriate dimensions for the upsampled output.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the upsampled values will be stored.\n * dst->op is `GGML_OP_UPSCALE`.\n */\nvoid ggml_cann_upsample_nearest2d(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Pads a ggml tensor to match the dimensions of the destination tensor\n * using the CANN backend.\n *\n * @details This function pads the input tensor `src` so that it matches the\n * dimensions of the destination tensor `dst`. The amount of padding\n * is calculated based on the difference in sizes between `src` and\n * `dst` along each dimension. The padded tensor is stored in `dst`.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor, which specifies the target dimensions for\n * padding. dst->op is `GGML_OP_PAD`.\n */\nvoid ggml_cann_pad(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Executes a 2D pooling operation on a ggml tensor using the CANN\n * backend.\n *\n * @details This function dispatches the execution of a 2D pooling operation on\n * the input tensor `dst`. The type of pooling (average or max) is\n * determined by the `op` parameter, which is read from the operation\n * parameters of `dst`. The function supports average pooling\n * (`GGML_OP_POOL_AVG`) and max pooling (`GGML_OP_POOL_MAX`). If an\n * invalid operation is encountered, the function asserts a failure.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor on which the pooling operation is to be\n * performed. dst->op is `GGML_OP_POOL_2D`.\n */\nvoid ggml_cann_pool2d(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Duplicates a ggml tensor using the CANN backend.\n *\n * @details This function duplicates the contents of the source tensor `src` to\n * the destination tensor `dst`. The function supports various tensor\n * types and configurations, including handling of extra data, type\n * conversions, and special cases for contiguous and non-contiguous\n * tensors.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the duplicated data will be stored.\n * dst->op is `GGML_OP_DUP`\n *\n * @attention Only support Fp16/FP32. Not support when src and dst have\n * different shape and dst is no-contiguous.\n * @note: This func need to simplify.\n */\nvoid ggml_cann_dup(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the Root Mean Square (RMS) normalization of a ggml tensor\n * using the CANN backend.\n *\n * @details This function applies RMS normalization to the input tensor `src`\n * and stores the result in the destination tensor `dst`. RMS\n * normalization involves computing the root mean square of the input\n * tensor along a specified dimension and then dividing each element of\n * the tensor by this value, adjusted by a small epsilon value to\n * prevent division by zero.\n * The operation is defined as:\n * \\f[\n * \\text{RmsNorm}\\left(x_i\\right)=\\frac{x_i}{\\text{Rms}(\\mathbf{x})} g_i,\n * \\quad \\text { where } \\text{Rms}(\\mathbf{x})=\\sqrt{\\frac{1}{n} \\sum_{i=1}^n x_i^2+e p s}\n * \\f]\n * `eps` is in dst->op_params.\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the normalized values will be stored.\n * dst->op is `GGML_OP_RMS_NORM`.\n */\nvoid ggml_cann_rms_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies a diagonal mask to the tensor with a specified value.\n *\n * @details This function creates a mask tensor filled with ones, then applies\n * an upper triangular and lower triangular operation to it based on\n * the number of past elements specified. Afterward, it adds the masked\n * tensor to the destination tensor in-place.\n *\n * @param ctx The backend CANN context used for operations.\n * @param dst The destination tensor where the result will be stored. dst->op is\n * `GGML_OP_DIAG_MASK`\n * @param value The value to use for masking.\n */\nvoid ggml_cann_diag_mask(ggml_backend_cann_context & ctx, ggml_tensor * dst, float value);\n\n/**\n * @brief Performs an image-to-column transformation on the input tensor.\n *\n * @details This function takes an input tensor and applies an image-to-column\n * operation, converting spatial dimensions into column-like\n * structures suitable for convolutional operations. It supports both\n * half-precision (F16) and single-precision (F32) floating-point data\n * types.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor that stores the result of the operation.\n * dst->op is `GGML_OP_IM2COL`.\n */\nvoid ggml_cann_im2col(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes time step embeddings using sine and cosine functions.\n *\n * @details This function calculates time step embeddings by applying sine and\n * cosine transformations to a given input tensor, which is typically\n * used in temporal models like diffusion models or transformers to\n * encode time information effectively.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the result of the embedding operation\n * will be stored. dst->op is `GGML_OP_TIMESTEP_EMBEDDING`.\n */\nvoid ggml_cann_timestep_embedding(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n// @see ggml_cann_dup.\nvoid ggml_cann_cpy(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the softmax activation with optional masking.\n *\n * @details This function computes the softmax activation over the input tensor,\n * optionally applying a mask and scaling factor. It supports both FP16\n * and FP32 data types and can handle masking by broadcasting the mask\n * across rows if necessary.\n * The function performs the following steps:\n * 1. Multiplies the input tensor by a scale factor.\n * 2. Optionally casts the mask tensor to FP32 if it is in FP16 format.\n * 3. Broadcasts the mask tensor if its dimensions do not match the\n * input tensor's dimensions.\n * 4. Adds the mask to the scaled input tensor.\n * 5. Applies the softmax activation function along the specified\n * dimension.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the result will be stored. dst->op is\n * `GGML_OP_SOFTMAX`.\n */\nvoid ggml_cann_softmax(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Extracts specific rows from a tensor based on indices.\n *\n * @details This function retrieves rows from a source tensor src0 according to\n * the indices provided in another tensor src1 and stores the result in\n * a destination tensor (\\p dst).\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the extracted rows will be stored.\n */\nvoid ggml_cann_get_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Writes specific rows into a tensor at positions specified by indices.\n *\n * @details This function copies rows from a source tensor into a destination\n * tensor (\\p dst) at the positions indicated by the indices in another\n * tensor.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the specified rows will be updated.\n */\nvoid ggml_cann_set_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Executes matrix multiplication for the given tensor.\n *\n * @details This function performs matrix multiplication on the source tensors\n * associated with the destination tensor. It supports matrix\n * multiplication F32, F16, and Q8_0.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor for storing the result of the matrix\n * multiplication. dst->op is `GGML_OP_MUL_MAT`.\n */\nvoid ggml_cann_mul_mat(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies Rotary Positional Embedding (RoPE) to the input tensor.\n *\n * @details This function implements the RoPE mechanism, which is a method to\n * encode positional information into sequence data, particularly\n * useful in transformer models. It supports both F32 and F16 data\n * types.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the RoPE-transformed data will be\n * stored. dst->op is `GGML_OP_ROPE`.\n *\n * @note The function currently does not support cases where the n_dims is less\n * than the input tensor's first dimension.\n * @note The function currently does not support cases where the freq_factors is\n * not NULL.\n * @note The function currently does not support cases where the ext_factor is\n * not equal 0.\n * @note The function currently does not support cases where the freq_scale is\n * not equal 1.\n */\nvoid ggml_cann_rope(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the index of the maximum value along the specified dimension\n * of a ggml tensor using the CANN backend.\n *\n * @details This function performs an argmax operation on the input tensor.\n * It finds the index of the maximum value along the specified axis\n * and stores these indices in the destination tensor `dst`. The\n * operation is executed using the CANN backend for optimized performance.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the indices of the maximum values will\n * be stored. dst->op is `GGML_OP_ARGMAX`.\n */\nvoid ggml_cann_argmax(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Adds two tensors element-wise and stores the result in a destination\n * tensor.\n *\n * This function performs the operation:\n * \\f[\n * dst = acl\\_src0 + alpha \\times acl\\_src1\n * \\f]\n * where alpha is a scalar value and defaults to 1.0f.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src0 The first source tensor.\n * @param acl_src1 The second source tensor.\n * @param acl_dst The destination tensor where the result will be stored.\n */\nvoid aclnn_add(ggml_backend_cann_context & ctx,\n aclTensor * acl_src0,\n aclTensor * acl_src1,\n aclTensor * acl_dst = nullptr);\n\n/**\n * @brief Sub two tensors element-wise and stores the result in a destination\n * tensor.\n *\n * This function performs the operation:\n * \\f[\n * dst = acl\\_src0 - alpha \\times acl\\_src1\n * \\f]\n * where alpha is a scalar value and defaults to 1.0f.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src0 The first source tensor.\n * @param acl_src1 The second source tensor.\n * @param acl_dst The destination tensor where the result will be stored.\n */\nvoid aclnn_sub(ggml_backend_cann_context & ctx,\n aclTensor * acl_src0,\n aclTensor * acl_src1,\n aclTensor * acl_dst = nullptr);\n\n/**\n * @brief Performs element-wise multiplication of two tensors and stores the\n * result in a destination tensor.\n *\n * This function performs element-wise multiplication of the tensors `acl_src`\n * and `acl_other` and stores the result in the destination tensor `acl_dst`.\n * The operation is defined as:\n * \\f[\n * \\text {acl_dst }_i=\\text {acl_src }_i \\times \\text {acl_other }_i\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The first tensor for element-wise multiplication.\n * @param acl_other The second tensor for element-wise multiplication.\n * @param acl_dst The destination tensor where the result will be stored.\n */\nvoid aclnn_mul(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n aclTensor * acl_other,\n aclTensor * acl_dst = nullptr);\n\n/**\n * @brief Matrix division, optionally in-place.\n *\n * This function division each element of the source tensor `acl_src` by the\n * tensor `acl_other` and stores the result in the destination tensor `acl_dst`.\n * If `inplace` is true, `acl_dst` will not be used and the operation is\n * performed in-place on `acl_src`. The operation is defined as: \\f[\n * \\text{dst}_i = \\frac{\\text{acl_src}_i}{\\text{acl_other}_i}\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src Numerator tensor..\n * @param acl_other Denominator tensor.\n * @param acl_dst The destination tensor where the result will be stored if\n * `inplace` is false.\n * @param inplace Flag indicating whether to perform the operation in-place on\n * `acl_src`.\n */\nvoid aclnn_div(ggml_backend_cann_context & ctx,\n aclTensor * acl_src,\n aclTensor * acl_other,\n aclTensor * acl_dst = nullptr);\n\n/**\n * @brief Applies element-wise cosine function to the elements of a tensor.\n *\n * This function computes the cosine of each element in the source tensor\n * `acl_src` and stores the result in the destination tensor `acl_dst`. The\n * operation is defined as: \\f[ \\text {acl_dst }_i=\\cos \\left(\\text {acl_src\n * }_i\\right) \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor on which the cosine function will be\n * applied.\n * @param acl_dst The destination tensor where the cosine results will be\n * stored.\n */\nvoid aclnn_cos(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst);\n\n/**\n * @brief Applies element-wise sine function to the elements of a tensor.\n *\n * This function computes the sine of each element in the source tensor\n `acl_src`\n * and stores the result in the destination tensor `acl_dst`.\n * The operation is defined as:\n * \\f[\n * \\text {acl_dst }_i=\\sin \\left(\\text {acl_src }_i\\right)\n * \\f]\n\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor on which the sine function will be applied.\n * @param acl_dst The destination tensor where the sine results will be stored.\n */\nvoid aclnn_sin(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst);\n\n/**\n * @brief Prepares broadcast-compatible ACL tensors for two input tensors and one\n * output tensor.\n *\n * This function checks whether broadcasting is needed between `src0` and `src1`.\n * If broadcasting is required, it calculates the proper shapes and creates\n * ACL tensors with broadcast parameters. Otherwise, it directly creates ACL tensors\n * based on the original tensor shapes.\n *\n * @param src0 The first input tensor (reference shape).\n * @param src1 The second input tensor (possibly broadcasted).\n * @param dst The destination/output tensor.\n * @param acl_src0 Output pointer to the created ACL tensor corresponding to src0.\n * @param acl_src1 Output pointer to the created ACL tensor corresponding to src1.\n * @param acl_dst Output pointer to the created ACL tensor corresponding to dst.\n */\nvoid bcast_shape(ggml_tensor * src0,\n ggml_tensor * src1,\n ggml_tensor * dst,\n acl_tensor_ptr & acl_src0,\n acl_tensor_ptr & acl_src1,\n acl_tensor_ptr & acl_dst);\n\n/**\n * @brief Computes the 1D transposed convolution (deconvolution) of a ggml\n * tensor using the CANN backend.\n *\n * @details This function performs a 1D transposed convolution (also known as\n * deconvolution) operation on the input tensor. The computed result is stored\n * in the destination tensor `dst`. The operation is optimized using the CANN\n * backend for improved performance.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the transposed convolution result\n * will be stored. dst->op is `GGML_OP_CONV_TRANSPOSE_1D`.\n */\nvoid ggml_cann_conv_transpose_1d(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies the ELU (Exponential Linear Unit) activation to a ggml tensor\n * using the CANN backend.\n *\n * @details This function performs an element-wise ELU activation on the input\n * tensor.\n * The result is written to the destination tensor `dst` in-place.\n * The ELU function is defined as:\n *\n * \\text{ELU}(x) =\n * \\begin{cases}\n * x, & \\text{if } x > 0 \\\\\n * \\alpha \\left( \\exp(x) - 1 \\right), & \\text{if } x \\leq 0\n * \\end{cases}\n *\n * where α (alpha) is a hyperparameter, typically set to 1.0.\n * This operation is optimized using the CANN backend for high-performance\n * inference or training.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the ELU-activated result will be stored.\n * dst->op is expected to be `GGML_OP_ELU`.\n */\nvoid ggml_cann_elu(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Computes the mean of a ggml tensor element-wise using the CANN backend.\n *\n * @details This function calculates the element-wise mean of the input tensor.\n * The result is written to the destination tensor `dst`.\n * The mean is computed by averaging the values across the entire tensor.\n *\n * This operation is optimized using the CANN backend for high-performance inference or training.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the mean result will be stored.\n * dst->op is expected to be `GGML_OP_MEAN`.\n */\nvoid ggml_cann_mean(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies 1D reflect padding to a ggml tensor using the CANN backend.\n *\n * @details This function performs 1D reflect padding on the input tensor.\n * The amount of padding on each side is specified by parameters stored in `dst->op_params`.\n * The operation reflects the values at the borders of the tensor to generate the padded output.\n *\n * This operation is optimized using the CANN backend for high-performance inference or training.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the padded result will be stored.\n * dst->op is expected to be `GGML_OP_PAD_REFLECT_1D`.\n */\nvoid ggml_cann_pad_reflect_1d(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Counts the number of equal elements in two ggml tensors using the CANN backend.\n *\n * @details This function performs an element-wise comparison between two input tensors,\n * and counts the number of positions where the elements are equal. The result is\n * stored in the destination tensor `dst` as a scalar.\n *\n * The operation is optimized using the CANN backend, making it suitable for\n * high-performance inference or training scenarios.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the result will be stored.\n * dst->op is expected to be `GGML_OP_COUNT_EQUAL`.\n */\nvoid ggml_cann_count_equal(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies the Step activation function to a ggml tensor using the CANN backend.\n *\n * @details This function applies a step function element-wise to the input tensor, where\n * each element is transformed to 1.0 if it is greater than 0, and 0.0 otherwise.\n * The result is stored in the destination tensor `dst`.\n *\n * This operation is accelerated using the CANN backend to improve runtime performance.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the result will be stored.\n * dst->op is expected to be `GGML_OP_STEP`.\n */\nvoid ggml_cann_step(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Performs the Flash Attention extended operator using the CANN backend.\n *\n * @details This function implements the memory-efficient Flash Attention algorithm\n * for computing scaled dot-product attention with hardware acceleration.\n * The result is stored in the destination tensor `dst`.\n *\n * This operation is accelerated using the CANN backend to improve runtime performance.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the result will be stored.\n * dst->op is expected to be `GGML_OP_FLASH_ATTN_EXT`.\n */\nvoid ggml_cann_flash_attn_ext(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Forward Gated Linear Attention on the CANN backend.\n *\n * Expects dst->src[0..4] = {k, v, q, g, s} with shape conventions:\n * k, v, q, g: [D] with outer dims T x H batched as ne[2]=T, ne[1]=H\n * s: initial state [B, H, D, D], where B is batch and D=C/H\n * dst holds both outputs (o) and updated state; a scale factor is read from op params.\n *\n * The kernel updates per time step l: S_new = g ⊗ S_old + k ⊗ v, then computes o = (S_new^T q) * scale.\n *\n * @param ctx Backend context providing stream/allocator utilities.\n * @param dst Output tensor; src deps are k, v, q, g, s as above.\n */\nvoid ggml_cann_gated_linear_attn(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Launches an asynchronous task using the memory allocator.\n *\n * This macro submit an asynchronous task on the specified stream.\n * The task uses memory allocated by the allocator. It is guaranteed\n * that the memory will not be accessed by other tasks until this task\n * completes, due to the sequential execution order within the same stream.\n *\n * @param OP_NAME aclnn operator name.\n * @param args Additional arguments required by the task.\n *\n * @note\n * Memory from the allocator will be \"freed\" immediately and can be\n * reallocated to other pointers. However, it won't be accessed by any\n * other task before this asynchronous task ends, because all tasks in the\n * same stream are executed in queue order.\n */\n\n# define GGML_CANN_CALL_ACLNN_OP(CTX, OP_NAME, ...) \\\n do { \\\n uint64_t workspaceSize = 0; \\\n aclOpExecutor * executor; \\\n void * workspaceAddr = nullptr; \\\n ACL_CHECK(aclnn##OP_NAME##GetWorkspaceSize(__VA_ARGS__, &workspaceSize, &executor)); \\\n /* workspace should alloced in main thread to keep malloc order when using vmm. */ \\\n if (workspaceSize > 0) { \\\n ggml_cann_pool_alloc workspace_allocator(CTX.pool(), workspaceSize); \\\n workspaceAddr = workspace_allocator.get(); \\\n } \\\n ACL_CHECK(aclnn##OP_NAME(workspaceAddr, workspaceSize, executor, CTX.stream())); \\\n } while (0)\n\n/**\n * @brief Performs sparse expert-based matrix multiplication using the CANN backend.\n *\n * @details This function implements a MoE-style batched matrix multiplication, where each input token\n * is routed to one or more experts, and each expert corresponds to a specific [D, M] weight matrix\n * in the source tensor `src0`. The routing indices are provided via the `ids` tensor.\n *\n * For each token (from `src1`), the function selects the corresponding expert(s) as specified by `ids`,\n * performs the matrix multiplication with the selected expert's weight submatrix (from `src0`),\n * and stores the results in `dst`. This operation is optimized and executed on the CANN backend.\n *\n * Dimensions:\n * - src0: [D, M, A, 1], where A is the number of experts\n * - src1: [D, B, N, 1], where N is batch size and B is the slot count per sample\n * - ids : [K, N], where K is the number of experts each token is routed to\n * - dst : [M, K, N, 1], output tensor storing the result of expert × token multiplication\n *\n * The function handles two main modes:\n * - If `ne12 == 1`, a simpler per-token loop is used.\n * - TODO: If `ne12 > 1`, grouped multiplication and memory copying is used for efficiency.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the expert-weighted token outputs are stored.\n * Expected to be of shape [M, K, N, 1].\n */\nvoid ggml_cann_mul_mat_id(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Performs fused ADD + RMS_NORM operation using the CANN backend.\n *\n * This function fuses the ADD and RMS_NORM operations into a single kernel call\n * for better performance. It first adds two input tensors (x1 + x2), then applies\n * RMS normalization to the result.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The ADD operation node, contains the two input tensors to be added.\n * @param rms_norm_tensor The RMS_NORM operation node, contains the gamma weights\n * and epsilon parameter.\n */\nvoid ggml_cann_op_add_rms_norm_fused(ggml_backend_cann_context & ctx,\n ggml_tensor * add_node,\n ggml_tensor * rms_norm_node);\n\n/**\n * @brief Check whether a tensor is a weight tensor for matrix multiplication.\n *\n * @details Checks whether the given tensor serves as weight parameters in matrix multiplication operations,\n * typically within neural network layers. The function maintains a static set of canonical weight\n * naming suffixes from Transformer-based architectures. Uses substring matching to identify weight\n * tensors even with hierarchical naming patterns.\n *\n * @param tensor Pointer to the target ggml_tensor object (const-qualified).\n */\nstatic bool is_matmul_weight(const ggml_tensor * tensor) {\n std::string name = ggml_get_name(tensor);\n static const std::unordered_set weight_suffixes{ \"output.weight\", \"attn_q.weight\",\n \"attn_k.weight\", \"attn_v.weight\",\n \"attn_output.weight\", \"ffn_gate.weight\",\n \"ffn_up.weight\", \"ffn_down.weight\" };\n\n for (const auto & suffix : weight_suffixes) {\n if (name.find(suffix) != std::string::npos) {\n return true;\n }\n }\n return false;\n}\n\n/**\n * @brief Applies a element-wise operation to two input tensors using the CANN\n * backend.\n *\n * This templated function takes a binary operator and applies it to two source\n * tensors\n * associated with the destination tensor. The function handles broadcasting as\n * needed.\n *\n * @tparam binary_op A callable object (e.g., lambda or function pointer) representing\n * the binary operation to be performed. It must take three arguments:\n * (ggml_backend_cann_context&, aclTensor*, aclTensor*, aclTensor*).\n *\n * @param ctx The CANN backend context used to manage execution and resources.\n * @param dst The destination tensor.\n */\ntemplate void ggml_cann_binary_op(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src0 = dst->src[0];\n ggml_tensor * src1 = dst->src[1];\n\n acl_tensor_ptr acl_src0, acl_src1, acl_dst;\n\n // Need bcast\n bcast_shape(src0, src1, dst, acl_src0, acl_src1, acl_dst);\n binary_op(ctx, acl_src0.get(), acl_src1.get(), acl_dst.get());\n}\n\n/**\n * @brief Applies a unary operation to an input tensor using the CANN backend.\n *\n * This templated function applies a unary operator to the source tensor of `dst`\n * and stores the result in the destination tensor.\n *\n * @tparam unary_op A callable with the signature:\n * void(ggml_backend_cann_context&, aclTensor *, aclTensor *)\n * where the first aclTensor is the source and the second is the destination.\n * @param ctx The CANN backend context for managing resources and execution.\n * @param dst The destination tensor. Its src[0] is treated as the input tensor.\n */\ntemplate \nvoid ggml_cann_op_unary(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src = dst->src[0];\n\n acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n unary_op(ctx, acl_src.get(), acl_dst.get());\n}\n\n/**\n * @brief Applies a unary operation to a ggml tensor using the CANN backend.\n *\n * @details This function applies a unary operation to the input tensor using\n * a user-provided lambda or callable `unary_op`. The lambda receives the\n * CANN backend context and two ACL tensors: the source and the destination.\n *\n * Internally, this function handles the conversion from GGML tensors to ACL tensors,\n * calls the provided unary op, and manages resource cleanup. The input is assumed\n * to be `dst->src[0]`, and the result is written to `dst`.\n *\n * This utility simplifies writing unary op wrappers by abstracting tensor preparation.\n *\n * @param unary_op A callable that performs the unary operation using CANN ACL APIs.\n * @param ctx The CANN context for operation execution.\n * @param dst The destination ggml_tensor where the result will be stored.\n * The input tensor is assumed to be `dst->src[0]`.\n *\n * @see GGML_CANN_CALL_OP_UNARY\n */\nvoid ggml_cann_op_unary(std::function unary_op,\n ggml_backend_cann_context & ctx,\n ggml_tensor * dst);\n\nvoid ggml_cann_ssm_conv(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies a gated (GLU-style) unary operation using the CANN backend.\n *\n * @details This function performs a gated activation such as GEGLU or ReGLU.\n * It supports two input modes:\n *\n * 1. **Dual input mode**: `dst->src[0]` and `dst->src[1]` are both valid tensors.\n * These are used directly as the value and gate tensors.\n *\n * 2. **Packed input mode**: Only `dst->src[0]` is valid, and it is assumed to\n * contain a concatenation of value and gate along the first dimension. This tensor\n * will be split into two equal halves to form the value and gate inputs.\n *\n * The function applies a user-provided unary operation (e.g., GELU) to the value tensor,\n * then multiplies the result in-place with the gate tensor:\n *\n * @code\n * dst = unary_op(value) * gate;\n * @endcode\n *\n * The `swapped` parameter (from `dst->op_params[1]`) allows flipping the\n * order of value/gate in the packed input case.\n *\n * @param unary_op A callable that performs the unary operation using CANN ACL APIs.\n * It receives (ctx, acl_value_tensor, acl_output_tensor).\n * @param ctx The CANN context used for execution.\n * @param dst The destination ggml_tensor. Source tensors are in `dst->src[0]` and optionally `src[1]`.\n *\n * @see GGML_CANN_CALL_OP_UNARY_GATED\n */\nvoid ggml_cann_op_unary_gated(std::function unary_op,\n ggml_backend_cann_context & ctx,\n ggml_tensor * dst);\n\n/**\n * @brief Helper macro to call a unary ACL operator via ggml_cann_op_unary.\n *\n * This macro wraps the specified ACLNN unary operator name into a lambda expression,\n * and passes it to `ggml_cann_op_unary`, which handles the common logic for executing\n * unary ops in the CANN backend.\n *\n * Internally, this macro expands to a lambda like:\n * @code\n * [](ggml_backend_cann_context& ctx, aclTensor* acl_src, aclTensor* acl_dst) {\n * GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst);\n * };\n * @endcode\n *\n * This lambda is then passed to `ggml_cann_op_unary`, which applies the operation.\n *\n * @param OP_NAME The name of the ACL unary operator to invoke via GGML_CANN_CALL_ACLNN_OP.\n *\n * @see ggml_cann_op_unary\n * @see GGML_CANN_CALL_ACLNN_OP\n */\n# define GGML_CANN_CALL_OP_UNARY(OP_NAME) \\\n do { \\\n auto lambda = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) { \\\n GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst); \\\n }; \\\n ggml_cann_op_unary(lambda, ctx, dst); \\\n } while (0)\n\n/**\n * @brief Helper macro to call a gated unary ACL operator via ggml_cann_op_unary_gated.\n *\n * This macro wraps the specified ACLNN unary operator name into a lambda expression,\n * and passes it to `ggml_cann_op_unary_gated`, which handles the common logic for\n * executing gated unary ops in the CANN backend.\n *\n * Internally, this macro expands to a lambda like:\n * @code\n * [](ggml_backend_cann_context& ctx, aclTensor* acl_src, aclTensor* acl_dst) {\n * GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst);\n * };\n * @endcode\n *\n * This lambda is then passed to `ggml_cann_op_unary_gated`, which applies the operation.\n *\n * @param OP_NAME The name of the ACL unary operator to invoke via GGML_CANN_CALL_ACLNN_OP.\n *\n * @see ggml_cann_op_unary_gated\n * @see GGML_CANN_CALL_ACLNN_OP\n */\n# define GGML_CANN_CALL_OP_UNARY_GATED(OP_NAME) \\\n do { \\\n auto lambda = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) { \\\n GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst); \\\n }; \\\n ggml_cann_op_unary_gated(lambda, ctx, dst); \\\n } while (0)\n\n#endif // CANN_ACLNN_OPS\n\n/**\n * @brief Performs outer product operation on two ggml tensors using the CANN backend.\n *\n * @details This function computes the outer product of two input tensors (src0 and src1)\n * and stores the result in the destination tensor. The outer product operation is defined as:\n * dst[i,j,k,l] = sum_m (src0[i,m,k,l] * src1[j,m,k,l])\n *\n * The function supports multiple data types including F32, F16. For floating-point\n * types, it uses batch matrix multiplication for efficient computation.\n *\n * The implementation handles 4D tensor broadcasting and batch processing automatically.\n *\n * @param ctx The CANN backend context for operation execution and memory management.\n * @param dst The destination ggml_tensor where the outer product result will be stored.\n * The input tensors are assumed to be `dst->src[0]` and `dst->src[1]`.\n *\n * @see GGML_CANN_CALL_ACLNN_OP for CANN operator invocation\n */\nvoid ggml_cann_out_prod(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cann/common.h", "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#ifndef CANN_COMMON_H\n#define CANN_COMMON_H\n\n#include \"../ggml-impl.h\"\n#include \"../include/ggml-cann.h\"\n#include \"../include/ggml.h\"\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#define MATRIX_ROW_PADDING 512\n#define GGML_CANN_MAX_STREAMS 8\n\n/**\n * @brief Handles CANN-related errors by printing an error message and\n * terminating the program.\n * @param stmt The statement that caused the error.\n * @param func The function in which the error occurred.\n * @param file The file in which the error occurred.\n * @param line The line number at which the error occurred.\n * @param msg The error message.\n */\n[[noreturn]] void ggml_cann_error(const char * stmt, const char * func, const char * file, int line, const char * msg);\n\n/**\n * @brief Checks the result of a CANN function call and invokes the error\n * handler if the call fails.\n * @param stmt The CANN function call to check.\n * @param success The success code that indicates the call was successful.\n * @param error_fn The function to call to retrieve the error message.\n */\n#define ACL_CHECK_GEN(stmt, success, error_fn) \\\n do { \\\n int err_code = (stmt); \\\n if (err_code != (success)) { \\\n ggml_cann_error(#stmt, __func__, __FILE__, __LINE__, error_fn()); \\\n } \\\n } while (0);\n\n#define ACL_CHECK(stmt) ACL_CHECK_GEN(stmt, 0, aclGetRecentErrMsg)\n\n/**\n * @brief Contains information about CANN devices.\n */\nstruct ggml_cann_device_info {\n /**\n * @brief Number of CANN devices available.\n */\n int32_t device_count;\n\n /**\n * @brief Information about a single CANN device.\n */\n struct cann_device_info {\n int cc; /**< Compute capability. */\n size_t smpb; /**< Maximum shared memory per block. */\n bool vmm; /**< Virtual memory support. */\n size_t vmm_granularity; /**< Granularity of virtual memory. */\n size_t total_vram; /**< Total video RAM available on the device. */\n };\n\n cann_device_info devices[GGML_CANN_MAX_DEVICES] = {}; /**< Array of CANN device information. */\n};\n\nconst ggml_cann_device_info & ggml_cann_info();\n\nvoid ggml_cann_set_device(int32_t device);\n\nstd::optional get_env_as_lowercase(const std::string & name);\nbool parse_bool(const std::string & value);\nint parse_integer(const std::string & value);\n\n/**\n * @brief Abstract base class for memory pools used by CANN.\n */\nstruct ggml_cann_pool {\n /**\n * @brief Virtual destructor for the memory pool.\n */\n virtual ~ggml_cann_pool() = default;\n\n /**\n * @brief Allocates memory from the pool.\n *\n * @param size The size of the memory block to allocate.\n * @param actual_size Pointer to a variable where the actual allocated size\n * will be stored.\n * @return Pointer to the allocated memory block.\n */\n virtual void * alloc(size_t size, size_t * actual_size) = 0;\n\n /**\n * @brief Frees a previously allocated memory block.\n *\n * @param ptr Pointer to the memory block to free.\n * @param size Size of the memory block to free.\n * @note Note that all CANN opertors are running async. Make sure memory is\n * still avaiable before this operator finished.\n */\n virtual void free(void * ptr, size_t size) = 0;\n};\n\n/**\n * @brief RAII wrapper for managing memory allocations from a CANN memory pool.\n */\nstruct ggml_cann_pool_alloc {\n ggml_cann_pool * pool = nullptr; /**< Pointer to the memory pool. */\n void * ptr = nullptr; /**< Pointer to the allocated memory block. */\n size_t actual_size = 0; /**< Actual size of the allocated memory block. */\n\n /**\n * @brief Default constructor.\n */\n ggml_cann_pool_alloc() = default;\n\n /**\n * @brief Constructor that initializes the memory pool.\n * @param pool Reference to the memory pool.\n */\n explicit ggml_cann_pool_alloc(ggml_cann_pool & pool) : pool(&pool) {}\n\n /**\n * @brief Constructor that initializes the memory pool and allocates memory.\n * @param pool Reference to the memory pool.\n * @param size Size of the memory block to allocate.\n */\n ggml_cann_pool_alloc(ggml_cann_pool & pool, size_t size) : pool(&pool) { alloc(size); }\n\n /**\n * @brief Destructor that frees the allocated memory block.\n */\n ~ggml_cann_pool_alloc() {\n if (ptr != nullptr) {\n pool->free(ptr, actual_size);\n }\n }\n\n /**\n * @brief Allocates memory from the pool.\n * @param size Size of the memory block to allocate.\n * @return Pointer to the allocated memory block.\n */\n void * alloc(size_t size) {\n GGML_ASSERT(pool != nullptr);\n GGML_ASSERT(ptr == nullptr);\n ptr = pool->alloc(size, &this->actual_size);\n return ptr;\n }\n\n /**\n * @brief Allocates memory from a specific memory pool.\n * @param pool Reference to the memory pool.\n * @param size Size of the memory block to allocate.\n * @return Pointer to the allocated memory block.\n */\n void * alloc(ggml_cann_pool & pool, size_t size) {\n this->pool = &pool;\n return alloc(size);\n }\n\n /**\n * @brief Gets the pointer to the allocated memory block.\n * @return Pointer to the allocated memory block.\n */\n void * get() { return ptr; }\n\n // Deleted copy constructor\n ggml_cann_pool_alloc(const ggml_cann_pool_alloc &) = delete;\n\n // Deleted move constructor\n ggml_cann_pool_alloc(ggml_cann_pool_alloc &&) = delete;\n\n // Deleted copy assignment operator\n ggml_cann_pool_alloc & operator=(const ggml_cann_pool_alloc &) = delete;\n\n // Deleted move assignment operator\n ggml_cann_pool_alloc & operator=(ggml_cann_pool_alloc &&) = delete;\n};\n\n#ifdef USE_ACL_GRAPH\nstruct ggml_graph_node_properties {\n // dst tensor\n void * node_address;\n int64_t ne[GGML_MAX_DIMS];\n size_t nb[GGML_MAX_DIMS];\n\n // src tensor\n void * src_address[GGML_MAX_SRC];\n int64_t src_ne[GGML_MAX_SRC][GGML_MAX_DIMS];\n size_t src_nb[GGML_MAX_SRC][GGML_MAX_DIMS];\n\n // op\n ggml_op node_op;\n int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];\n\n /**\n * @brief Check if a ggml tensor node matches this property set.\n *\n * This function compares all relevant fields (address, op type, shape, source inputs, op params)\n * to determine whether the current node matches these previously recorded properties.\n *\n * @param node The current ggml tensor node.\n * @return true if all fields match (excluding GGML_OP_VIEW); false otherwise.\n */\n bool has_matching_properties(ggml_tensor * node) {\n if (node->data != this->node_address && node->op != GGML_OP_VIEW) {\n return false;\n }\n\n if (node->op != this->node_op) {\n return false;\n }\n\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n if (node->ne[i] != this->ne[i]) {\n return false;\n }\n if (node->nb[i] != this->nb[i]) {\n return false;\n }\n }\n\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (node->src[i]) {\n if (node->src[i]->data != this->src_address[i] && node->op != GGML_OP_VIEW) {\n return false;\n }\n\n for (int d = 0; d < GGML_MAX_DIMS; d++) {\n if (node->src[i]->ne[d] != this->src_ne[i][d]) {\n return false;\n }\n if (node->src[i]->nb[d] != this->src_nb[i][d]) {\n return false;\n }\n }\n } else {\n if (this->src_address[i] != nullptr) {\n return false;\n }\n }\n }\n\n if (node->op == GGML_OP_SCALE || node->op == GGML_OP_UNARY || node->op == GGML_OP_GLU) {\n return memcmp(this->op_params, node->op_params, GGML_MAX_OP_PARAMS) == 0;\n }\n return true;\n }\n};\n\nstruct ggml_cann_graph {\n ~ggml_cann_graph() {\n if (graph != nullptr) {\n ACL_CHECK(aclmdlRIDestroy(graph));\n }\n }\n\n aclmdlRI graph = nullptr;\n\n std::vector ggml_graph_properties;\n\n /**\n * @brief Create a new CANN graph from a ggml computation graph.\n *\n * This function creates a new ggml_cann_graph object and fills its node properties\n * (operation type, dimensions, strides, input sources, and operation parameters)\n * based on the current ggml computation graph.\n *\n * Each node in the ggml graph is mapped to a property entry in the new CANN graph:\n * - node address\n * - operation type\n * - shape (ne) and strides (nb)\n * - source tensor addresses\n * - operation parameters\n *\n * @param cgraph The current ggml computation graph.\n * @return Pointer to the newly created ggml_cann_graph object.\n */\n static ggml_cann_graph * create_from_cgraph(ggml_cgraph * cgraph) {\n ggml_cann_graph * new_graph = new ggml_cann_graph();\n new_graph->ggml_graph_properties.resize(cgraph->n_nodes);\n\n for (int node_idx = 0; node_idx < cgraph->n_nodes; ++node_idx) {\n ggml_tensor * node = cgraph->nodes[node_idx];\n auto & prop = new_graph->ggml_graph_properties[node_idx];\n\n prop.node_address = node->data;\n prop.node_op = node->op;\n\n std::copy_n(node->ne, GGML_MAX_DIMS, prop.ne);\n std::copy_n(node->nb, GGML_MAX_DIMS, prop.nb);\n\n for (int src = 0; src < GGML_MAX_SRC; ++src) {\n if (node->src[src]) {\n prop.src_address[src] = node->src[src]->data;\n std::copy_n(node->src[src]->ne, GGML_MAX_DIMS, prop.src_ne[src]);\n std::copy_n(node->src[src]->nb, GGML_MAX_DIMS, prop.src_nb[src]);\n } else {\n prop.src_address[src] = nullptr;\n std::fill_n(prop.src_ne[src], GGML_MAX_DIMS, 0);\n std::fill_n(prop.src_nb[src], GGML_MAX_DIMS, 0);\n }\n }\n\n memcpy(prop.op_params, node->op_params, GGML_MAX_OP_PARAMS);\n }\n\n return new_graph;\n }\n\n /**\n * @brief Check whether this CANN graph matches the given ggml computation graph.\n *\n * This function compares the number of nodes and each node's properties\n * (operation type, dimensions, strides, inputs, and operation parameters)\n * to determine whether this CANN graph matches the given ggml graph.\n *\n * @param cgraph The current ggml computation graph.\n * @return true if this CANN graph matches the ggml graph; false otherwise.\n */\n bool matches_cgraph(ggml_cgraph * cgraph) {\n if (this->ggml_graph_properties.size() != static_cast(cgraph->n_nodes)) {\n return false;\n }\n\n for (int i = 0; i < cgraph->n_nodes; ++i) {\n if (!this->ggml_graph_properties[i].has_matching_properties(cgraph->nodes[i])) {\n return false;\n }\n }\n\n return true;\n }\n};\n\n/**\n * @brief LRU cache for managing ggml_cann_graph objects.\n *\n * This class maintains a list of shared_ptr to ggml_cann_graph objects\n * and enforces a maximum capacity. It provides methods to push new graphs,\n * move existing graphs to the front (most recently used), and clear the cache.\n */\nstruct ggml_cann_graph_lru_cache {\n size_t capacity; /**< Maximum number of graphs in the cache. */\n\n std::list cache_list; /**< List storing cached graphs as raw pointers. */\n\n ggml_cann_graph_lru_cache() { capacity = parse_integer(get_env_as_lowercase(\"GGML_CANN_GRAPH_CACHE_CAPACITY\").value_or(\"12\")); }\n\n /**\n * @brief Push a new graph to the front of the cache.\n * If the cache exceeds capacity, the least recently used graph is deleted.\n * @param new_node Pointer to the new ggml_cann_graph to cache.\n * Ownership is transferred to the cache (cache will delete it).\n */\n void push(ggml_cann_graph * new_node) {\n if (cache_list.size() >= capacity) {\n ggml_cann_graph * old = cache_list.back();\n cache_list.pop_back();\n delete old; // free the old graph\n }\n cache_list.push_front(new_node);\n }\n\n /**\n * @brief Clear all graphs from the cache (also frees memory).\n */\n void clear() {\n for (auto ptr : cache_list) {\n delete ptr;\n }\n cache_list.clear();\n }\n\n /**\n * @brief Destructor that clears the cache and frees all cached graphs.\n */\n ~ggml_cann_graph_lru_cache() { clear(); }\n\n /**\n * @brief Find a cached CANN graph that matches the given ggml graph and move it to front.\n *\n * This function iterates through the cached CANN graphs stored in the LRU cache and\n * compares them against the given ggml computation graph. If a matching graph is found,\n * it is promoted to the front of the LRU cache and returned. Otherwise, the function\n * returns nullptr.\n *\n * @param cgraph The current ggml computation graph.\n * @return true if found; false otherwise.\n */\n bool find_and_move_to_front(ggml_cgraph * cgraph) {\n for (auto & graph_ptr : this->cache_list) {\n if (graph_ptr->matches_cgraph(cgraph)) {\n cache_list.remove(graph_ptr);\n cache_list.push_front(graph_ptr);\n return true;\n }\n }\n return false;\n }\n};\n#endif // USE_ACL_GRAPH\n\nstruct ggml_cann_rope_cache {\n ~ggml_cann_rope_cache() {\n if (theta_scale_cache) {\n ACL_CHECK(aclrtFree(theta_scale_cache));\n }\n if (sin_cache) {\n ACL_CHECK(aclrtFree(sin_cache));\n }\n if (cos_cache) {\n ACL_CHECK(aclrtFree(cos_cache));\n }\n if (position_select_index) {\n ACL_CHECK(aclrtFree(position_select_index));\n }\n if (theta_scale_exp_host) {\n free(theta_scale_exp_host);\n }\n if (position_select_index_host) {\n free(position_select_index_host);\n }\n if (yarn_ramp_cache) {\n ACL_CHECK(aclrtFree(yarn_ramp_cache));\n }\n }\n\n bool equal(int64_t theta_scale_length,\n int64_t position_length,\n float ext_factor,\n float theta_scale,\n float freq_scale,\n float attn_factor,\n bool is_neox,\n bool indep_sects,\n bool mrope_used,\n bool is_imrope,\n int sections[4]) {\n return this->theta_scale_length == theta_scale_length && this->position_length == position_length &&\n this->ext_factor == ext_factor && this->theta_scale == theta_scale && this->freq_scale == freq_scale &&\n this->attn_factor == attn_factor && this->is_neox == is_neox && this->indep_sects == indep_sects &&\n this->mrope_used == mrope_used && this->is_imrope == is_imrope && this->sections[0] == sections[0] &&\n this->sections[1] == sections[1] && this->sections[2] == sections[2] && this->sections[3] == sections[3];\n }\n\n void set(int64_t theta_scale_length,\n int64_t position_length,\n float ext_factor,\n float theta_scale,\n float freq_scale,\n float attn_factor,\n bool is_neox,\n bool indep_sects,\n bool mrope_used,\n bool is_imrope,\n int sections[4]) {\n this->theta_scale_length = theta_scale_length;\n this->position_length = position_length;\n this->ext_factor = ext_factor;\n this->theta_scale = theta_scale;\n this->freq_scale = freq_scale;\n this->attn_factor = attn_factor;\n this->is_neox = is_neox;\n this->indep_sects = indep_sects;\n this->mrope_used = mrope_used;\n this->is_imrope = is_imrope;\n this->sections[0] = sections[0];\n this->sections[1] = sections[1];\n this->sections[2] = sections[2];\n this->sections[3] = sections[3];\n }\n\n // memory cache, prepare before inferencing.\n void * theta_scale_cache = nullptr;\n float * theta_scale_exp_host = nullptr;\n int * position_select_index_host = nullptr;\n void * position_select_index = nullptr;\n void * yarn_ramp_cache = nullptr;\n // sin/cos cache, used only to accelerate first layer on each device\n void * sin_cache = nullptr;\n void * cos_cache = nullptr;\n // Properties to check before reusing the sincos cache\n int64_t theta_scale_length = 0;\n int64_t position_length = 0;\n bool cached = false;\n float ext_factor = 0.0f;\n float theta_scale = 0.0f;\n float freq_scale = 0.0f;\n float attn_factor = 0.0f;\n bool is_neox = false;\n bool indep_sects = false;\n bool mrope_used = false;\n int sections[4] = { 0, 0, 0, 0 };\n bool is_imrope = false;\n};\n\nstruct ggml_cann_tensor_cache {\n ~ggml_cann_tensor_cache() {\n if (cache != nullptr) {\n ACL_CHECK(aclrtFree(cache));\n }\n }\n\n void * cache = nullptr;\n int64_t size = 0;\n};\n\n/**\n * @brief Context for managing CANN backend operations.\n */\nstruct ggml_backend_cann_context {\n int32_t device; /**< Device ID. */\n std::string name; /**< Name of the device. */\n std::string description; /**< Description of the device. */\n aclrtEvent copy_event = nullptr; /**< Event for managing copy operations. */\n#ifdef USE_ACL_GRAPH\n /// Cached CANN ACL graph used for executing the current ggml computation graph.\n ggml_cann_graph_lru_cache graph_lru_cache;\n bool acl_graph_mode = true;\n#endif\n bool async_mode;\n // Rope Cache\n ggml_cann_rope_cache rope_cache;\n // Constant Pool\n ggml_cann_tensor_cache rms_norm_one_tensor_cache;\n ggml_cann_tensor_cache rms_norm_zero_tensor_cache;\n\n aclrtStream streams[GGML_CANN_MAX_STREAMS] = { nullptr }; /**< Array of streams for the device. */\n\n /**\n * @brief Constructor for initializing the context with a given device.\n * @param device Device ID.\n */\n explicit ggml_backend_cann_context(int device) : device(device), name(\"CANN\" + std::to_string(device)) {\n ggml_cann_set_device(device);\n description = aclrtGetSocName();\n\n#ifdef USE_ACL_GRAPH\n acl_graph_mode = parse_bool(get_env_as_lowercase(\"GGML_CANN_ACL_GRAPH\").value_or(\"on\"));\n GGML_LOG_INFO(\"%s: device %d execution mode is %s (%s)\\n\", __func__, device, acl_graph_mode ? \"GRAPH\" : \"EAGER\",\n acl_graph_mode ? \"acl graph enabled\" : \"acl graph disabled\");\n#endif\n }\n\n /**\n * @brief Destructor for cleaning up resources.\n */\n ~ggml_backend_cann_context() {\n ggml_cann_set_device(device);\n if (copy_event != nullptr) {\n ACL_CHECK(aclrtDestroyEvent(copy_event));\n }\n for (int i = 0; i < GGML_CANN_MAX_STREAMS; ++i) {\n if (streams[i] != nullptr) {\n ACL_CHECK(aclrtDestroyStream(streams[i]));\n }\n }\n }\n\n /**\n * @brief Get or create a stream for a given index.\n * @param stream Index of the stream.\n * @return The stream corresponding to the given index.\n */\n aclrtStream stream(int stream) {\n if (streams[stream] == nullptr) {\n // If the device is not set here, destroying the stream later may cause a mismatch\n // between the thread contexts where the stream was created and destroyed.\n // However, I printed the device_id, thread_id, and stream, and they are all consistent.\n ACL_CHECK(aclrtSetDevice(device));\n ACL_CHECK(aclrtCreateStream(&streams[stream]));\n }\n return streams[stream];\n }\n\n /**\n * @brief Get or create the default stream (index 0).\n * @return The default stream.\n */\n aclrtStream stream() { return stream(0); }\n\n // TODO: each stream should have a memory pool.\n std::unique_ptr mem_pool; /**< Memory pool for the device. */\n\n /**\n * @brief Create a new memory pool for a given device.\n * @param device Device ID.\n * @return A unique pointer to the new memory pool.\n */\n static std::unique_ptr new_pool_for_device(int device);\n\n /**\n * @brief Get or create the memory pool for the context.\n * @return Reference to the memory pool.\n */\n ggml_cann_pool & pool() {\n if (mem_pool == nullptr) {\n mem_pool = new_pool_for_device(device);\n }\n return *mem_pool;\n }\n};\n\n#endif // CANN_COMMON_H\n" }, { "path": "src/ggml-cann/ggml-cann.cpp", "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#include \"ggml-cann.h\"\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-cann/aclnn_ops.h\"\n#include \"ggml-cann/common.h\"\n#include \"ggml-impl.h\"\n#include \"ggml.h\"\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#define GGML_COMMON_DECL_C\n\n#include \"ggml-common.h\"\n\n#define GGML_CANN_NAME \"CANN\"\n\n/**\n * @brief Handles CANN errors by printing an error message and aborting.\n *\n * @param stmt The statement that caused the error.\n * @param func The function in which the error occurred.\n * @param file The file in which the error occurred.\n * @param line The line number where the error occurred.\n * @param msg The error message.\n */\n[[noreturn]] void ggml_cann_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {\n int32_t id = -1;\n aclrtGetDevice(&id);\n\n GGML_LOG_ERROR(\"CANN error: %s\\n\", msg);\n GGML_LOG_ERROR(\" current device: %d, in function %s at %s:%d\\n\", id, func, file, line);\n GGML_LOG_ERROR(\" %s\\n\", stmt);\n // abort with GGML_ASSERT to get a stack trace\n GGML_ABORT(\"CANN error\");\n}\n\n// Thread-local variable to record the current device of this thread.\nthread_local int g_current_cann_device = -1;\n\n/**\n * @brief Set the CANN device to be used.\n *\n * @param device The target device ID to set.\n */\nvoid ggml_cann_set_device(const int32_t device) {\n // int current_device = -1;\n // Note: In some CANN versions, if no device has been set yet,\n // aclrtGetDevice(¤t_device) may return 0 by default.\n // aclrtGetDevice(¤t_device);\n\n // If the current device is already the target one, no need to switch.\n if (device == g_current_cann_device) {\n return;\n }\n\n // Switch to the new device.\n ACL_CHECK(aclrtSetDevice(device));\n\n // Update the global device record.\n g_current_cann_device = device;\n}\n\n/**\n * @brief Get the value of the specified environment variable (name) as lowercase.\n * if not empty, return a std::string object\n */\nstd::optional get_env_as_lowercase(const std::string & name) {\n const char * val = std::getenv(name.c_str());\n if (!val) {\n return std::nullopt;\n }\n std::string res = std::string(val);\n std::transform(res.begin(), res.end(), res.begin(), ::tolower);\n return res;\n}\n\n/**\n * @brief Verify whether the environment variable is a valid value.\n */\nbool parse_bool(const std::string & value) {\n static const std::unordered_set valid_values = { \"on\", \"1\", \"yes\", \"y\", \"enable\", \"true\" };\n return valid_values.find(value) != valid_values.end();\n}\n\n/**\n * @brief Parse a string as an integer, returning 0 if invalid.\n *\n * This function attempts to convert the input string `value` to an `int`.\n * If the string is not a valid integer or is out of the `int` range,\n * it returns 0.\n *\n * @param value The string to parse.\n * @return The parsed integer, or 0 if conversion fails.\n */\nint parse_integer(const std::string & value) {\n try {\n return std::stoi(value);\n } catch (...) {\n return 0;\n }\n}\n\n/**\n * @brief Initialize the CANN device information.\n *\n * This function initializes the CANN device information by obtaining the\n * device count and setting the memory allocation granularity for each device.\n *\n * @return A structure containing the device information.\n */\nstatic ggml_cann_device_info ggml_cann_init() {\n ggml_cann_device_info info = {};\n\n aclError err = aclrtGetDeviceCount((uint32_t *) &info.device_count);\n\n if (err != ACL_SUCCESS) {\n GGML_LOG_ERROR(\"%s: failed to initialize CANN: %s\\n\", __func__, aclGetRecentErrMsg());\n return info;\n }\n\n GGML_ASSERT(info.device_count <= GGML_CANN_MAX_DEVICES);\n\n for (int id = 0; id < info.device_count; ++id) {\n aclrtPhysicalMemProp prop = {};\n prop.handleType = ACL_MEM_HANDLE_TYPE_NONE;\n prop.allocationType = ACL_MEM_ALLOCATION_TYPE_PINNED;\n prop.memAttr = ACL_HBM_MEM_HUGE;\n prop.location.type = ACL_MEM_LOCATION_TYPE_DEVICE;\n prop.location.id = id;\n prop.reserve = 0;\n err = aclrtMemGetAllocationGranularity(&prop, ACL_RT_MEM_ALLOC_GRANULARITY_RECOMMENDED,\n &info.devices[id].vmm_granularity);\n info.devices[id].vmm = err == ACL_SUCCESS;\n\n size_t free, total;\n ggml_backend_cann_get_device_memory(id, &free, &total);\n info.devices[id].total_vram = free;\n }\n\n // TODO: add more device info later.\n return info;\n}\n\n/**\n * @brief Retrieve the CANN device information.\n *\n * This function returns a reference to a structure containing the CANN device\n * information. The device information is initialized once and reused on\n * subsequent calls.\n *\n * @return A reference to the structure containing the device information.\n */\nconst ggml_cann_device_info & ggml_cann_info() {\n static ggml_cann_device_info info = ggml_cann_init();\n return info;\n}\n\n//#define DEBUG_CANN_MALLOC\n/**\n * @brief A pool of CANN buffers(priority segment buffer).\n *\n * This class manages a pool of CANN buffers for a specific device.\n */\nstruct ggml_cann_pool_buf_prio : public ggml_cann_pool {\n /**\n * @brief The maximum reuse margin for a buffer.\n */\n static const size_t max_reuse_margin = 1ull << 22; // 4MB\n\n /**\n * @brief The minimum free margin for a buffer.\n */\n static const size_t min_free_margin = 1ull << 20; // 1MB\n\n /**\n * @brief The alignment for buffer allocation.\n */\n static const size_t alignment = 128;\n\n /**\n * @brief The device ID associated with this buffer pool.\n */\n int device;\n\n /**\n * @brief Whether to disable clean during buffer allocation.\n */\n bool disable_clean = false;\n\n /**\n * @brief Structure representing a CANN buffer.\n */\n struct ggml_cann_buffer {\n void * ptr = nullptr; ///< Pointer to the buffer.\n size_t size = 0; ///< Size of the buffer.\n std::chrono::steady_clock::time_point last_used; ///< Last used time.\n\n bool operator>(const ggml_cann_buffer & other) const { return size > other.size; }\n };\n\n /**\n * @brief Array of CANN buffers in the pool.\n */\n std::unordered_map buffer_pool;\n std::priority_queue, std::greater<>> free_buffers;\n\n /**\n * @brief Total size of all buffers in the pool.\n */\n size_t pool_size = 0;\n\n /**\n * @brief Constructor to initialize the buffer pool for a specific device.\n *\n * @param device The device ID to associate with this buffer pool.\n */\n explicit ggml_cann_pool_buf_prio(int device) : device(device) {\n disable_clean = parse_bool(get_env_as_lowercase(\"GGML_CANN_DISABLE_BUF_POOL_CLEAN\").value_or(\"\"));\n }\n\n /**\n * @brief Destructor to free all buffers in the pool.\n */\n ~ggml_cann_pool_buf_prio() {\n ggml_cann_set_device(device);\n for (auto & [b_ptr, b_size] : buffer_pool) {\n aclrtFree(b_ptr);\n pool_size -= b_size;\n }\n buffer_pool.clear();\n GGML_ASSERT(pool_size == 0);\n }\n\n /**\n * @brief Allocate a buffer of the given size.\n *\n * @param size The size of the buffer to allocate.\n * @param actual_size A pointer to a variable to receive the actual size of\n * the allocated buffer.\n * @return A pointer to the allocated buffer.\n */\n void * alloc(size_t size, size_t * actual_size) override {\n size = GGML_PAD(size, alignment);\n if (size == 0) {\n size = alignment;\n }\n\n void * ptr = nullptr;\n auto now = std::chrono::steady_clock::now();\n\n std::vector free_buffers_rest;\n free_buffers_rest.reserve(free_buffers.size());\n while (!free_buffers.empty()) {\n auto b = free_buffers.top();\n free_buffers.pop();\n\n if (b.size >= size) {\n // reuse the buffer if the size is enough\n const size_t margin = b.size - size;\n if (margin <= max_reuse_margin) {\n *actual_size = b.size;\n ptr = b.ptr;\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\n \"cann pool[%d]: reused %p, \"\n \"pool_size = %5u MB, \"\n \"size = %5u MB, \"\n \"margin = %5u MB\\n\",\n device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n (uint32_t) (GGML_PAD(size, 1048576) / 1048576),\n (uint32_t) (GGML_PAD(margin, 1048576) / 1048576));\n#endif\n break;\n }\n }\n\n bool should_clean = !disable_clean && b.size > min_free_margin &&\n std::chrono::duration_cast(now - b.last_used).count() > 100;\n if (should_clean) {\n // free the buffer if the size is needed to be freed\n ACL_CHECK(aclrtFree(b.ptr));\n pool_size -= b.size;\n buffer_pool.erase(b.ptr);\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\n \"cann pool[%d]: clean %p, \"\n \"pool_size = %5u MB, \"\n \"size = %5u MB\\n\",\n device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n (uint32_t) (GGML_PAD(b.size, 1048576) / 1048576));\n#endif\n continue;\n }\n free_buffers_rest.push_back(b);\n }\n for (ggml_cann_buffer & b : free_buffers_rest) {\n free_buffers.push(std::move(b));\n }\n\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\"cann pool[%d] free pool_size = %5u MB\\n\\n\", device,\n (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576));\n#endif\n if (ptr != nullptr) {\n return ptr;\n }\n\n // allocate a new buffer if no buffer can be reused\n ggml_cann_set_device(device);\n ACL_CHECK(aclrtMalloc(&ptr, size, ACL_MEM_MALLOC_HUGE_FIRST));\n *actual_size = size;\n pool_size += size;\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\n \"cann pool[%d]: allocate %p, \"\n \"pool_size = %5u MB, \"\n \"size = %5u MB\\n\",\n device, ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n (uint32_t) (GGML_PAD(size, 1048576) / 1048576));\n#endif\n buffer_pool.emplace(ptr, size);\n return ptr;\n }\n\n /**\n * @brief Free a buffer and return it to the pool.\n *\n * @param ptr Pointer to the buffer to free.\n * @param size Size of the buffer to free.\n */\n void free(void * ptr, size_t size) override {\n GGML_UNUSED(size);\n auto it = buffer_pool.find(ptr);\n if (it == buffer_pool.end()) {\n GGML_ABORT(\"cann pool[%d]: buffer %p not found in pool\\n\", device, ptr);\n }\n\n auto now = std::chrono::steady_clock::now();\n free_buffers.emplace(ggml_cann_buffer{ ptr, it->second, now });\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\n \"cann pool[%d]: return %p, \"\n \"pool_size = %5u MB\\n\",\n device, ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576));\n#endif\n }\n};\n\n/**\n * @brief A pool of CANN buffers(segment buffer).\n *\n * This class manages a pool of CANN buffers for a specific device.\n */\nstruct ggml_cann_pool_buf : public ggml_cann_pool {\n /**\n * @brief The maximum reuse margin for a buffer.\n */\n static const size_t max_reuse_margin = 1ull << 22; // 4MB\n\n /**\n * @brief The minimum free margin for a buffer.\n */\n static const size_t min_free_margin = 1ull << 20; // 1MB\n\n /**\n * @brief The alignment for buffer allocation.\n */\n static const size_t alignment = 128;\n\n /**\n * @brief The maximum number of buffers in the pool.\n */\n static const int MAX_BUFFERS = 256;\n\n /**\n * @brief The device ID associated with this buffer pool.\n */\n int device;\n\n /**\n * @brief Whether to disable clean during buffer allocation.\n */\n bool disable_clean = false;\n\n /**\n * @brief Structure representing a CANN buffer.\n */\n struct ggml_cann_buffer {\n void * ptr = nullptr; ///< Pointer to the buffer memory.\n size_t size = 0; ///< Size of the buffer.\n bool used = false; ///< Whether the buffer is currently in use.\n std::chrono::steady_clock::time_point last_used; ///< Last used time.\n };\n\n /**\n * @brief Array of CANN buffers in the pool.\n */\n ggml_cann_buffer buffer_pool[MAX_BUFFERS] = {};\n\n /**\n * @brief Total size of all buffers in the pool.\n */\n size_t pool_size = 0;\n\n /**\n * @brief Constructor to initialize the buffer pool for a specific device.\n *\n * @param device The device ID to associate with this buffer pool.\n */\n explicit ggml_cann_pool_buf(int device) : device(device) {\n disable_clean = parse_bool(get_env_as_lowercase(\"GGML_CANN_DISABLE_BUF_POOL_CLEAN\").value_or(\"\"));\n }\n\n /**\n * @brief Destructor to free all buffers in the pool.\n */\n ~ggml_cann_pool_buf() {\n ggml_cann_set_device(device);\n for (int i = 0; i < MAX_BUFFERS; ++i) {\n ggml_cann_buffer & b = buffer_pool[i];\n if (b.ptr != nullptr) {\n aclrtFree(b.ptr);\n pool_size -= b.size;\n }\n }\n GGML_ASSERT(pool_size == 0);\n }\n\n /**\n * @brief Allocate a buffer of the given size.\n *\n * @param size The size of the buffer to allocate.\n * @param actual_size A pointer to a variable to receive the actual size of\n * the allocated buffer.\n * @return A pointer to the allocated buffer.\n */\n void * alloc(size_t size, size_t * actual_size) override {\n size = GGML_PAD(size, alignment);\n if (size == 0) {\n size = alignment;\n }\n\n void * ptr = nullptr;\n auto now = std::chrono::steady_clock::now();\n\n int i = 0;\n for (; i < MAX_BUFFERS; ++i) {\n ggml_cann_buffer & b = buffer_pool[i];\n if (b.ptr == nullptr) {\n break;\n }\n if (b.used) {\n continue;\n }\n if (b.size >= size) {\n // reuse the buffer if the size is enough\n const size_t margin = b.size - size;\n if (margin <= max_reuse_margin) {\n *actual_size = b.size;\n b.used = true;\n ptr = b.ptr;\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\n \"cann pool[%d]: reused %p, \"\n \"pool_size = %5u MB, \"\n \"size = %5u MB, \"\n \"margin = %5u MB\\n\",\n device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n (uint32_t) (GGML_PAD(size, 1048576) / 1048576),\n (uint32_t) (GGML_PAD(margin, 1048576) / 1048576));\n#endif\n break;\n }\n }\n\n bool should_clean = !disable_clean && b.size > min_free_margin &&\n std::chrono::duration_cast(now - b.last_used).count() > 100;\n if (should_clean) {\n // free the buffer if the size is needed to be freed\n ACL_CHECK(aclrtFree(b.ptr));\n pool_size -= b.size;\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\n \"cann pool[%d]: clean %p, \"\n \"pool_size = %5u MB, \"\n \"size = %5u MB\\n\",\n device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n (uint32_t) (GGML_PAD(b.size, 1048576) / 1048576));\n#endif\n b.ptr = nullptr;\n }\n }\n if (ptr != nullptr) {\n return ptr;\n }\n\n if (i < MAX_BUFFERS) {\n // allocate a new buffer if no buffer can be reused\n ggml_cann_buffer & b = buffer_pool[i];\n ggml_cann_set_device(device);\n ACL_CHECK(aclrtMalloc(&b.ptr, size, ACL_MEM_MALLOC_HUGE_FIRST));\n pool_size += size;\n *actual_size = size;\n b.size = size;\n b.used = true;\n if (i >= MAX_BUFFERS - 8) {\n GGML_LOG_WARN(\"cann pool[%d]: slots almost full\\n\", device);\n }\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\n \"cann pool[%d]: allocate %p, \"\n \"pool_size = %5u MB, \"\n \"size = %5u MB\\n\",\n device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n (uint32_t) (GGML_PAD(b.size, 1048576) / 1048576));\n#endif\n return b.ptr;\n }\n\n GGML_ABORT(\"cann pool[%d]: slots full\\n\", device);\n }\n\n /**\n * @brief Free a buffer and return it to the pool.\n *\n * @param ptr Pointer to the buffer to free.\n * @param size Size of the buffer to free.\n */\n void free(void * ptr, size_t size) override {\n GGML_UNUSED(size);\n for (int i = 0; i < MAX_BUFFERS; ++i) {\n ggml_cann_buffer & b = buffer_pool[i];\n if (b.ptr != ptr) {\n continue;\n }\n b.used = false;\n b.last_used = std::chrono::steady_clock::now();\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\n \"cann pool[%d]: return %p, \"\n \"pool_size = %5u MB\\n\",\n device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576));\n#endif\n return;\n }\n GGML_ABORT(\"cann pool[%d]: slots full\\n\", device);\n }\n};\n\n/**\n * @brief A pool of CANN buffers with virtual memory.\n *\n * This class manages a pool of CANN buffers with virtual memory for a specific\n * device.\n */\nstruct ggml_cann_pool_vmm : public ggml_cann_pool {\n /**\n * @brief The maximum size of the virtual memory pool (32 GB).\n */\n size_t max_size;\n\n /**\n * @brief The device ID associated with this buffer pool.\n */\n int device;\n\n /**\n * @brief Pointer to the start of the virtual memory pool.\n */\n void * pool_addr = 0;\n\n /**\n * @brief Amount of virtual memory used in the pool.\n */\n size_t pool_used = 0;\n\n /**\n * @brief Total size of the virtual memory pool.\n */\n size_t pool_size = 0;\n\n /**\n * @brief Allocation granularity for the virtual memory pool.\n */\n size_t granularity;\n\n /**\n * @brief Handles for the physical memory allocated.\n */\n std::vector handles;\n\n /**\n * @brief Offsets for the mapped memory regions.\n */\n std::vector map_offsets;\n\n /**\n * @brief Constructor to initialize the buffer pool with virtual memory for\n * a specific device.\n *\n * @param device The device ID to associate with this buffer pool.\n */\n explicit ggml_cann_pool_vmm(int device) : device(device) {\n auto dev = ggml_cann_info().devices[device];\n granularity = dev.vmm_granularity;\n max_size = dev.total_vram;\n }\n\n /**\n * @brief Destructor to free all buffers in the virtual memory pool.\n */\n ~ggml_cann_pool_vmm() {\n if (pool_addr != 0) {\n for (auto & offset : map_offsets) {\n ACL_CHECK(aclrtUnmapMem(offset));\n }\n for (auto & handle : handles) {\n ACL_CHECK(aclrtFreePhysical(handle));\n }\n ACL_CHECK(aclrtReleaseMemAddress(pool_addr));\n }\n }\n\n /**\n * @brief Allocate a buffer of the given size in the virtual memory pool.\n *\n * @param size The size of the buffer to allocate.\n * @param actual_size A pointer to a variable to receive the actual size of\n * the allocated buffer.\n * @return A pointer to the allocated buffer.\n */\n void * alloc(size_t size, size_t * actual_size) override {\n // round up the allocation size to the alignment to ensure that all\n // allocations are aligned for all data types\n const size_t alignment = 128;\n size = GGML_PAD(size, alignment);\n if (size == 0) {\n size = alignment;\n }\n\n size_t avail = pool_size - pool_used;\n\n if (size > avail) {\n // round up to the next multiple of the granularity\n size_t reserve_size = size - avail;\n reserve_size = GGML_PAD(reserve_size, granularity);\n\n GGML_ASSERT(pool_size + reserve_size <= max_size);\n\n // allocate more physical memory\n aclrtPhysicalMemProp prop = {};\n prop.handleType = ACL_MEM_HANDLE_TYPE_NONE;\n prop.allocationType = ACL_MEM_ALLOCATION_TYPE_PINNED;\n prop.memAttr = ACL_HBM_MEM_HUGE;\n prop.location.type = ACL_MEM_LOCATION_TYPE_DEVICE;\n prop.location.id = device;\n prop.reserve = 0;\n aclrtDrvMemHandle handle;\n ACL_CHECK(aclrtMallocPhysical(&handle, reserve_size, &prop, 0));\n\n // reserve virtual address space (if not already reserved)\n if (pool_addr == 0) {\n ACL_CHECK(aclrtReserveMemAddress(&pool_addr, max_size, 0, NULL, 1));\n }\n\n // map at the end of the pool\n ACL_CHECK(aclrtMapMem((char *) pool_addr + pool_size, reserve_size, 0, handle, 0));\n\n handles.push_back(handle);\n map_offsets.push_back((char *) pool_addr + pool_size);\n\n // add to the pool\n pool_size += reserve_size;\n\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\"cann pool[%d]: size increased to %llu MB (reserved %llu MB)\\n\", device,\n (unsigned long long) (pool_size / 1024 / 1024),\n (unsigned long long) (reserve_size / 1024 / 1024));\n#endif\n }\n\n GGML_ASSERT(pool_addr != 0);\n\n void * ptr = (void *) ((char *) pool_addr + pool_used);\n *actual_size = size;\n pool_used += size;\n\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\"cann pool[%d]: allocated %llu bytes at %llx\\n\", device, (unsigned long long) size,\n (unsigned long long) ptr);\n#endif\n return ptr;\n }\n\n /**\n * @brief Free a buffer and return it to the virtual memory pool.\n *\n * @param ptr Pointer to the buffer to free.\n * @param size Size of the buffer to free.\n */\n void free(void * ptr, size_t size) override {\n#ifdef DEBUG_CANN_MALLOC\n GGML_LOG_INFO(\"cann pool[%d]: freed %llu bytes at %llx\\n\", device, (unsigned long long) size,\n (unsigned long long) ptr);\n#endif\n\n pool_used -= size;\n\n // all deallocations must be in reverse order of the allocations\n GGML_ASSERT(ptr == (void *) ((char *) pool_addr + pool_used));\n }\n};\n\n/**\n * @brief Create a new CANN pool for a specific device.\n *\n * Factory method to create a new CANN pool object based on the device type.\n *\n * @param device The device ID for which to create the pool.\n * @return A unique pointer to the created CANN pool.\n */\nstd::unique_ptr ggml_backend_cann_context::new_pool_for_device(int device) {\n std::string mem_pool_type = get_env_as_lowercase(\"GGML_CANN_MEM_POOL\").value_or(\"\");\n\n if (mem_pool_type == \"prio\") {\n GGML_LOG_INFO(\"%s: device %d use buffer pool with priority queue\\n\", __func__, device);\n return std::unique_ptr(new ggml_cann_pool_buf_prio(device));\n }\n\n if (ggml_cann_info().devices[device].vmm && mem_pool_type != \"leg\") {\n GGML_LOG_INFO(\"%s: device %d use vmm pool\\n\", __func__, device);\n return std::unique_ptr(new ggml_cann_pool_vmm(device));\n }\n\n GGML_LOG_INFO(\"%s: device %d use buffer pool\\n\", __func__, device);\n return std::unique_ptr(new ggml_cann_pool_buf(device));\n}\n\n// cann buffer\n/**\n * @brief Context for managing a CANN buffer associated with a specific device.\n *\n * This structure holds information about a CANN buffer, including the device\n * ID, device pointer, and a name derived from GGML_CANN_NAME and the device ID.\n */\nstruct ggml_backend_cann_buffer_context {\n int32_t device; ///< The device ID associated with this buffer context.\n void * dev_ptr = nullptr; ///< Pointer to the device memory allocated for the buffer.\n\n /**\n * @brief Constructor to initialize the CANN buffer context.\n *\n * @param device The device ID associated with this buffer context.\n * @param dev_ptr Pointer to the device memory allocated for the buffer.\n */\n ggml_backend_cann_buffer_context(int32_t device, void * dev_ptr) : device(device), dev_ptr(dev_ptr) {}\n\n /**\n * @brief Destructor to free the device memory allocated for the buffer.\n */\n ~ggml_backend_cann_buffer_context() { ACL_CHECK(aclrtFree(dev_ptr)); }\n};\n\n// cann buffer type\n/**\n * @brief Structure representing context information for a specific backend\n * buffer type.\n */\nstruct ggml_backend_cann_buffer_type_context {\n int32_t device; /**< Device identifier associated with the buffer context. */\n std::string name; /**< Name associated with the buffer context. */\n};\n\n/**\n * @brief Retrieves the name associated with a CANN buffer type.\n *\n * This function returns the descriptive name associated with the specified\n * CANN buffer type context.\n *\n * @param buft Pointer to the buffer type context.\n * @return Const pointer to the C-style string containing the name.\n */\nstatic const char * ggml_backend_cann_buffer_type_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context;\n\n return buft_ctx->name.c_str();\n}\n\n/**\n * @brief Checks if the backend buffer type is associated with the CANN backend.\n *\n * This function checks whether the provided backend buffer type is associated\n * with the CANN backend based on the comparison of its name retrieval function\n * pointer.\n *\n * @param buft Pointer to the backend buffer type to check.\n * @return bool Returns true if the buffer type is associated with the CANN\n * backend, otherwise false.\n */\nstatic bool ggml_backend_buft_is_cann(ggml_backend_buffer_type_t buft) {\n return buft->iface.get_name == ggml_backend_cann_buffer_type_name;\n}\n\n/**\n * @brief Free resources associated with a CANN buffer.\n *\n * This function frees the resources associated with a CANN buffer, including\n * its context.\n *\n * @param buffer The CANN buffer to free.\n */\nstatic void ggml_backend_cann_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n delete ctx;\n}\n\n/**\n * @brief Retrieve the base pointer of a CANN buffer.\n *\n * This function returns the base pointer of a CANN buffer, which points to the\n * device memory allocated for the buffer.\n *\n * @param buffer The CANN buffer whose base pointer is to be retrieved.\n * @return A pointer to the base of the device memory allocated for the buffer.\n */\nstatic void * ggml_backend_cann_buffer_get_base(ggml_backend_buffer_t buffer) {\n ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n return ctx->dev_ptr;\n}\n\n/**\n * @brief Transform quantized Q4.0 tensor data into a format suitable for CANN\n * processing.\n *\n * This function transforms quantized Q4.0 tensor data into a format suitable\n * for CANN processing. It extracts quantization values and scales from the\n * source data and prepares them in a format expected by CANN operations.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source data in Q4.0 format.\n * @param dst Pointer to the destination buffer where transformed data will be\n * stored.\n */\nstatic void ggml_backend_cann_transform_q4_0(ggml_tensor * tensor, const void * src, void * dst) {\n int64_t n_elems = ggml_nelements(tensor);\n int64_t groups = n_elems / QK4_0;\n size_t quant_bytes = n_elems * sizeof(uint8_t) / 2;\n\n uint8_t * quant_offset = (uint8_t *) dst;\n uint16_t * scale_offset = (uint16_t *) ((char *) dst + quant_bytes);\n\n for (int i = 0; i < groups; i++) {\n const block_q4_0 * group = (const block_q4_0 *) ((const char *) src + i * sizeof(block_q4_0));\n *scale_offset = group->d;\n scale_offset++;\n\n // 0-15\n for (int j = 0; j < QK4_0 / 2; j += 2) {\n (*quant_offset) = (group->qs[j] & 0x0F);\n (*quant_offset) |= ((group->qs[j + 1] << 4));\n quant_offset++;\n }\n\n // 16-31\n for (int j = 0; j < QK4_0 / 2; j += 2) {\n (*quant_offset) = (group->qs[j] >> 4);\n (*quant_offset) |= (group->qs[j + 1] & 0xF0);\n quant_offset++;\n }\n }\n\n // put (uint4b_t -8) into int4b_t\n for (quant_offset = (uint8_t *) dst; quant_offset < (uint8_t *) dst + quant_bytes; quant_offset++) {\n (*quant_offset) ^= 0x88;\n }\n}\n\n/**\n * @brief Transform CANN processed data back into quantized Q4.0 format.\n *\n * This function transforms CANN processed data back into quantized Q4.0 format.\n * It reverses the transformation performed by\n * ggml_backend_cann_transform_q4_0(), converting the data back into its\n * original quantized form.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source buffer containing transformed data.\n * @param dst Pointer to the destination buffer where the Q4.0 formatted data\n * will be stored.\n */\nstatic void ggml_backend_cann_transform_back_q4_0(const ggml_tensor * tensor, void * src, void * dst) {\n int64_t n_elems = ggml_nelements(tensor);\n int64_t groups = n_elems / QK4_0;\n size_t quant_bytes = n_elems * sizeof(uint8_t) / 2;\n\n uint8_t * quant_offset = (uint8_t *) src;\n uint16_t * scale_offset = (uint16_t *) ((char *) src + quant_bytes);\n\n for (; quant_offset < (uint8_t *) src + quant_bytes; quant_offset++) {\n (*quant_offset) ^= 0x88;\n }\n quant_offset = (uint8_t *) src;\n\n for (int i = 0; i < groups; i++) {\n block_q4_0 * group = (block_q4_0 *) ((char *) dst + i * sizeof(block_q4_0));\n group->d = *scale_offset;\n scale_offset++;\n\n // 0-15\n for (int j = 0; j < QK4_0 / 2; j += 2) {\n group->qs[j] = ((*quant_offset) & 0x0F);\n group->qs[j + 1] = ((*quant_offset) >> 4);\n quant_offset++;\n }\n\n // 16-31\n for (int j = 0; j < QK4_0 / 2; j += 2) {\n group->qs[j] |= ((*quant_offset) << 4);\n group->qs[j + 1] |= ((*quant_offset) & 0xF0);\n quant_offset++;\n }\n }\n}\n\n/**\n * @brief Transform quantized Q8.0 tensor data into a format suitable for CANN\n * processing.\n *\n * This function transforms quantized Q8.0 tensor data into a format suitable\n * for CANN processing. It extracts quantization values and scales from the\n * source data and prepares them in a format expected by CANN operations.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source data in Q8.0 format.\n * @param dst Pointer to the destination buffer where transformed data will be\n * stored.\n */\nstatic void ggml_backend_cann_transform_q8_0(ggml_tensor * tensor, const void * src, void * dst) {\n int64_t n_elems = ggml_nelements(tensor);\n int64_t groups = n_elems / QK8_0;\n size_t quant_bytes = n_elems * sizeof(uint8_t);\n\n uint8_t * quant_offset = (uint8_t *) dst;\n uint16_t * scale_offset = (uint16_t *) ((char *) dst + quant_bytes);\n\n for (int i = 0; i < groups; i++) {\n const block_q8_0 * group = (const block_q8_0 *) ((const char *) src + i * sizeof(block_q8_0));\n *scale_offset = group->d;\n scale_offset++;\n size_t group_quant_size = QK8_0 * sizeof(uint8_t);\n memcpy(quant_offset, group->qs, group_quant_size);\n quant_offset += group_quant_size;\n }\n}\n\n/**\n * @brief Transform CANN processed data back into quantized Q8.0 format.\n *\n * This function transforms CANN processed data back into quantized Q8.0 format.\n * It reverses the transformation performed by\n * ggml_backend_cann_transform_q8_0(), converting the data back into its\n * original quantized form.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source buffer containing transformed data.\n * @param dst Pointer to the destination buffer where the Q8.0 formatted data\n * will be stored.\n */\nstatic void ggml_backend_cann_transform_back_q8_0(const ggml_tensor * tensor, const void * src, void * dst) {\n int64_t n_elems = ggml_nelements(tensor);\n int64_t groups = n_elems / QK8_0;\n size_t quant_bytes = n_elems * sizeof(uint8_t);\n\n const uint8_t * quant_offset = (const uint8_t *) src;\n const uint16_t * scale_offset = (const uint16_t *) ((const char *) src + quant_bytes);\n\n for (int i = 0; i < groups; i++) {\n block_q8_0 * group = (block_q8_0 *) ((char *) dst + i * sizeof(block_q8_0));\n group->d = *scale_offset;\n scale_offset++;\n size_t group_quant_size = QK8_0 * sizeof(uint8_t);\n memcpy(group->qs, quant_offset, group_quant_size);\n quant_offset += group_quant_size;\n }\n}\n\n/**\n * @brief Transform tensor data based on its type for CANN processing.\n *\n * This function transforms tensor data based on its quantization type for CANN\n * processing. It dispatches the transformation based on the tensor's type to\n * specialized functions handling Q4.0 and Q8.0 formats.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source data to be transformed.\n * @param dst Pointer to the destination buffer where transformed data will be\n * stored.\n */\nstatic void ggml_backend_cann_transform(ggml_tensor * tensor, const void * src, void * dst) {\n switch (tensor->type) {\n case GGML_TYPE_Q4_0:\n ggml_backend_cann_transform_q4_0(tensor, src, dst);\n break;\n case GGML_TYPE_Q8_0:\n ggml_backend_cann_transform_q8_0(tensor, src, dst);\n break;\n default:\n break;\n }\n}\n\n/**\n * @brief Transform CANN processed data back into tensor data based on its type.\n *\n * This function transforms CANN processed data back into tensor data based on\n * its quantization type for Q4.0 and Q8.0 formats. It dispatches the\n * transformation based on the tensor's type to specialized functions.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source data containing CANN processed data.\n * @param dst Pointer to the destination buffer where transformed tensor data\n * will be stored.\n */\nstatic void ggml_backend_cann_transform_back(const ggml_tensor * tensor, void * src, void * dst) {\n switch (tensor->type) {\n case GGML_TYPE_Q4_0:\n ggml_backend_cann_transform_back_q4_0(tensor, src, dst);\n break;\n case GGML_TYPE_Q8_0:\n ggml_backend_cann_transform_back_q8_0(tensor, src, dst);\n break;\n default:\n break;\n }\n}\n\n/**\n * @brief Check if transformation is needed for a given tensor type.\n *\n * This function checks if transformation is needed for a given tensor type\n * to prepare data for CANN processing.\n *\n * @param type The tensor type to check.\n * @return true if transformation is needed, false otherwise.\n */\nstatic bool need_transform(ggml_type type) {\n switch (type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q8_0:\n return true;\n default:\n return false;\n }\n}\n\n/**\n * @brief Initialize a tensor using data from a CANN buffer.\n *\n * This function initializes a tensor using data from a CANN buffer.\n * It handles special cases such as views and quantization.\n *\n * @param buffer The CANN buffer from which to initialize the tensor.\n * @param tensor Pointer to the tensor to be initialized.\n */\nstatic enum ggml_status ggml_backend_cann_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {\n if (tensor->view_src != NULL && tensor->view_offs == 0) {\n GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);\n return GGML_STATUS_SUCCESS;\n }\n\n // TODO: cann backend doesn't support quantized yet. Just leave the code\n // here.\n if (ggml_is_quantized(tensor->type)) {\n // Initialize padding to 0 to avoid possible NaN values\n size_t original_size = ggml_nbytes(tensor);\n size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);\n\n if (padded_size > original_size && tensor->view_src == nullptr) {\n size_t memset_size = padded_size - original_size;\n ACL_CHECK(aclrtMemset((char *) tensor->data + original_size, memset_size, 0, memset_size));\n }\n }\n return GGML_STATUS_SUCCESS;\n}\n\n/**\n * @brief Workspace for caching NZ buffers per device.\n *\n * This struct manages a device buffer used in NZ computations. It supports\n * allocation, reallocation, and clearing of cached memory. The struct is\n * designed to be used with a global array, one per device.\n */\nstruct ggml_cann_nz_workspace {\n void * ptr; // Pointer to allocated device buffer\n size_t allocated; // Size of currently allocated buffer in bytes\n\n /**\n * @brief Constructor. Initializes the workspace with no allocated memory.\n */\n ggml_cann_nz_workspace() : ptr(nullptr), allocated(0) {}\n\n /**\n * @brief Free cached memory and reset the workspace.\n *\n * If a buffer has been allocated, this function releases it using\n * aclrtFree and resets internal state.\n */\n void clear() {\n if (ptr) {\n ACL_CHECK(aclrtFree(ptr));\n ptr = nullptr;\n allocated = 0;\n }\n }\n\n /**\n * @brief Allocate or reallocate the workspace buffer.\n *\n * If the requested size is larger than the currently allocated size,\n * the old buffer will be freed and a new buffer of the requested size\n * will be allocated on the device.\n *\n * @param new_size Size in bytes to allocate for the workspace.\n */\n void realloc(size_t new_size) {\n if (new_size > allocated) {\n clear();\n ACL_CHECK(aclrtMalloc(&ptr, new_size, ACL_MEM_MALLOC_HUGE_FIRST));\n allocated = new_size;\n }\n }\n\n /**\n * @brief Get the device buffer pointer.\n *\n * @return Pointer to the allocated buffer, or nullptr if not allocated.\n */\n void * get() const { return ptr; }\n};\n\n/**\n * @brief Global array of NZ workspaces, one per device.\n */\nstatic ggml_cann_nz_workspace g_nz_workspaces[GGML_CANN_MAX_DEVICES];\n\n/**\n * @brief Convert tensor weights to NZ format using Ascend CANN API.\n *\n * This function creates a transposed tensor descriptor and performs the\n * TransMatmulWeight operation. Converting tensor formats can significantly\n * improve performance on certain hardware.\n *\n * @param tensor Pointer to the input ggml_tensor containing the weights.\n * @param offset Byte offset within the tensor data buffer where weights start.\n * @param device device id.\n *\n * @note The workspace buffer used in this function is managed globally and reused\n * across calls. This reduces overhead from repeated memory allocation and deallocation.\n */\nstatic void weight_format_to_nz(ggml_tensor * tensor, size_t offset, int device) {\n acl_tensor_ptr weightTransposed = ggml_cann_create_tensor(tensor, tensor->ne, tensor->nb, 2, ACL_FORMAT_ND, offset);\n uint64_t workspaceSize = 0;\n aclOpExecutor * executor;\n\n // TransMatmulWeight\n ACL_CHECK(aclnnTransMatmulWeightGetWorkspaceSize(weightTransposed.get(), &workspaceSize, &executor));\n // Avoid frequent malloc/free of the workspace.\n g_nz_workspaces[device].realloc(workspaceSize);\n\n void * g_nz_workspace = g_nz_workspaces[device].get();\n\n ACL_CHECK(aclnnTransMatmulWeight(g_nz_workspace, workspaceSize, executor, nullptr));\n}\n\n// TODO: need handle tensor which has paddings.\n/**\n * @brief Set tensor data in a CANN buffer.\n *\n * This function sets tensor data in a CANN buffer, handling transformations\n * if needed based on the tensor's type.\n *\n * @param buffer The CANN buffer where the tensor data will be set.\n * @param tensor Pointer to the tensor whose data will be set.\n * @param data Pointer to the source data to be copied into the tensor.\n * @param offset Offset in the source data from where to start copying.\n * @param size Size of the data to be copied, in bytes.\n */\nstatic void ggml_backend_cann_buffer_set_tensor(ggml_backend_buffer_t buffer,\n ggml_tensor * tensor,\n const void * data,\n size_t offset,\n size_t size) {\n ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n\n ggml_cann_set_device(ctx->device);\n // TODO: refer to cann(#6017), it use thread's default stream.\n // For acl, synchronous functions use this default stream.\n // Why aclrtSynchronizeDevice?\n\n // Only check env once.\n static bool weight_to_nz = parse_bool(get_env_as_lowercase(\"GGML_CANN_WEIGHT_NZ\").value_or(\"on\"));\n if (!need_transform(tensor->type)) {\n ACL_CHECK(aclrtMemcpy((char *) tensor->data + offset, size, data, size, ACL_MEMCPY_HOST_TO_DEVICE));\n if (weight_to_nz && is_matmul_weight((const ggml_tensor *) tensor)) {\n GGML_ASSERT(tensor->ne[2] == 1);\n GGML_ASSERT(tensor->ne[3] == 1);\n weight_format_to_nz(tensor, offset, ctx->device);\n }\n } else {\n void * transform_buffer = malloc(size);\n ggml_backend_cann_transform(tensor, data, transform_buffer);\n\n ACL_CHECK(aclrtMemcpy((char *) tensor->data + offset, size, transform_buffer, size, ACL_MEMCPY_HOST_TO_DEVICE));\n free(transform_buffer);\n }\n}\n\n/**\n * @brief Get tensor data from a CANN buffer.\n *\n * This function retrieves tensor data from a CANN buffer, handling\n * transformations if needed based on the tensor's type.\n *\n * @param buffer The CANN buffer from which to retrieve tensor data.\n * @param tensor Pointer to the tensor whose data will be retrieved.\n * @param data Pointer to the destination buffer where the tensor data will be\n * copied.\n * @param offset Offset in the destination buffer where to start copying.\n * @param size Size of the data to be copied, in bytes.\n */\nstatic void ggml_backend_cann_buffer_get_tensor(ggml_backend_buffer_t buffer,\n const ggml_tensor * tensor,\n void * data,\n size_t offset,\n size_t size) {\n ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n\n ggml_cann_set_device(ctx->device);\n\n if (!need_transform(tensor->type)) {\n ACL_CHECK(aclrtMemcpy(data, size, (char *) tensor->data + offset, size, ACL_MEMCPY_DEVICE_TO_HOST));\n } else {\n void * transform_buffer = malloc(size);\n ACL_CHECK(aclrtMemcpy(transform_buffer, size, (char *) tensor->data + offset, size, ACL_MEMCPY_DEVICE_TO_HOST));\n ggml_backend_cann_transform_back(tensor, transform_buffer, data);\n free(transform_buffer);\n }\n}\n\n/**\n * @brief Copy tensor data between CANN buffers if possible.\n *\n * This function copies tensor data between CANN buffers if the source and\n * destination buffers are CANN buffers and they meet the necessary conditions\n * (same device or devices can access each other).\n *\n * @param buffer The destination CANN buffer where the tensor data will be\n * copied.\n * @param src Pointer to the source tensor whose data will be copied.\n * @param dst Pointer to the destination tensor where the data will be copied.\n * @return true if the copy operation succeeded, false otherwise.\n */\nstatic bool ggml_backend_cann_buffer_cpy_tensor(ggml_backend_buffer_t buffer,\n const ggml_tensor * src,\n ggml_tensor * dst) {\n if (ggml_backend_buft_is_cann(src->buffer->buft)) {\n ggml_backend_cann_buffer_context * src_ctx = (ggml_backend_cann_buffer_context *) src->buffer->context;\n ggml_backend_cann_buffer_context * dst_ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n\n size_t memcpy_size = ggml_nbytes(src);\n // Same device.\n if (src_ctx->device == dst_ctx->device) {\n ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size,\n ACL_MEMCPY_DEVICE_TO_DEVICE));\n return true;\n } else {\n#ifdef ASCEND_310P\n // TODO: Support 310p P2P copy\n return false;\n#endif\n // Different device but can access by peer.\n int32_t canAccessPeer = 0;\n ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, src_ctx->device, dst_ctx->device));\n if (canAccessPeer) {\n ggml_cann_set_device(src_ctx->device);\n ACL_CHECK(aclrtDeviceEnablePeerAccess(dst_ctx->device, 0));\n ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size,\n ACL_MEMCPY_DEVICE_TO_DEVICE));\n return true;\n }\n }\n }\n return false;\n}\n\n/**\n * @brief Clear a CANN buffer by setting all its memory to a specified value.\n *\n * This function clears a CANN buffer by setting all its memory to a specified\n * value.\n *\n * @param buffer The CANN buffer to be cleared.\n * @param value The value to which each byte in the buffer will be set.\n */\nstatic void ggml_backend_cann_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n\n ggml_cann_set_device(ctx->device);\n ACL_CHECK(aclrtMemset(ctx->dev_ptr, buffer->size, value, buffer->size));\n}\n\n/**\n * @brief Interface for a CANN buffer in the backend.\n *\n * This structure defines function pointers to operations that can be performed\n * on a CANN buffer within the backend.\n */\nstatic const ggml_backend_buffer_i ggml_backend_cann_buffer_interface = {\n /* .free_buffer = */ ggml_backend_cann_buffer_free_buffer,\n /* .get_base = */ ggml_backend_cann_buffer_get_base,\n /* .init_tensor = */ ggml_backend_cann_buffer_init_tensor,\n /* .memset_tensor = */ NULL,\n /* .set_tensor = */ ggml_backend_cann_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_cann_buffer_get_tensor,\n /* .cpy_tensor = */ ggml_backend_cann_buffer_cpy_tensor,\n /* .clear = */ ggml_backend_cann_buffer_clear,\n /* .reset = */ NULL,\n};\n\n/**\n * @brief Allocates a new CANN buffer of the specified type and size.\n *\n * This function allocates a new CANN buffer on the specified device with the\n * given size.\n *\n * @param buft Pointer to the buffer type context.\n * @param size Size in bytes of the buffer to allocate.\n * @return Pointer to the allocated buffer, or nullptr if allocation fails.\n */\nstatic ggml_backend_buffer_t ggml_backend_cann_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context;\n\n ggml_cann_set_device(buft_ctx->device);\n\n const size_t alignment = 128;\n size = GGML_PAD(size, alignment);\n if (size == 0) {\n size = alignment;\n }\n void * dev_ptr;\n aclError err = aclrtMalloc(&dev_ptr, size, ACL_MEM_MALLOC_HUGE_FIRST);\n if (err != ACL_SUCCESS) {\n GGML_LOG_ERROR(\"%s: allocating %.2f MiB on device %d: aclrtMalloc failed: %s\\n\", __func__,\n size / 1024.0 / 1024.0, buft_ctx->device, aclGetRecentErrMsg());\n return nullptr;\n }\n\n ggml_backend_cann_buffer_context * ctx = new ggml_backend_cann_buffer_context(buft_ctx->device, dev_ptr);\n\n return ggml_backend_buffer_init(buft, ggml_backend_cann_buffer_interface, ctx, size);\n}\n\n/**\n * @brief Retrieves the memory alignment requirement for CANN buffers of this\n * type.\n *\n * This function returns the alignment requirement in bytes for memory allocated\n * by the CANN buffer type.\n *\n * @param buft Pointer to the buffer type context (unused in this\n * implementation).\n * @return The alignment requirement in bytes (fixed at 128 bytes for CANN\n * buffers).\n */\nstatic size_t ggml_backend_cann_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n return 128;\n\n GGML_UNUSED(buft);\n}\n\n/**\n * @brief Calculates the allocation size required for a tensor in a CANN buffer.\n *\n * Computes the total allocation size needed for storing the tensor's data in a\n * CANN buffer, considering any necessary padding or adjustments for quantized\n * types.\n *\n * @param buft Pointer to the buffer type context (unused in this\n * implementation).\n * @param tensor Pointer to the tensor for which the allocation size is\n * calculated.\n * @return The total allocation size in bytes required for the tensor in the\n * CANN buffer.\n */\nstatic size_t ggml_backend_cann_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft,\n const ggml_tensor * tensor) {\n size_t size = ggml_nbytes(tensor);\n int64_t ne0 = tensor->ne[0];\n\n // Only check env once.\n static bool weight_to_nz = parse_bool(get_env_as_lowercase(\"GGML_CANN_WEIGHT_NZ\").value_or(\"on\"));\n\n // last line must bigger than 32, because every single op deal at\n // least 32 bytes.\n // TODO: quantized type?\n // int64_t line_size = ne0 * ggml_element_size(tensor);\n // int64_t line_size_align_32 = (line_size + 31) & ~31;\n // size += (line_size_align_32 - line_size);\n if (ggml_is_quantized(tensor->type)) {\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n } else if (weight_to_nz && is_matmul_weight((const ggml_tensor *) tensor)) {\n // NZ format weight are not support quantized yet.\n // If ND tensor transform to NZ, size may changed.\n int64_t shape[] = { tensor->ne[1], tensor->ne[0] };\n GGML_ASSERT(tensor->ne[2] == 1);\n GGML_ASSERT(tensor->ne[3] == 1);\n const aclIntArray * acl_shape = aclCreateIntArray(shape, 2);\n size_t new_size;\n ACL_CHECK(aclnnCalculateMatmulWeightSizeV2(acl_shape, ggml_cann_type_mapping(tensor->type), &new_size));\n ACL_CHECK(aclDestroyIntArray(acl_shape));\n size = std::max(size, new_size);\n }\n\n return size;\n\n GGML_UNUSED(buft);\n}\n\nstatic bool ggml_backend_cann_buffer_type_is_host(ggml_backend_buffer_type_t buft) {\n return false;\n\n GGML_UNUSED(buft);\n}\n\n/**\n * @brief Interface for managing CANN buffer types in the GGML backend.\n *\n * Provides function pointers for allocating, querying properties, and managing\n * memory for CANN buffer types in the GGML backend.\n */\nstatic const ggml_backend_buffer_type_i ggml_backend_cann_buffer_type_interface = {\n /* .get_name = */ ggml_backend_cann_buffer_type_name,\n /* .alloc_buffer = */ ggml_backend_cann_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cann_buffer_type_get_alignment,\n /* .get_max_size = */ NULL, // defaults to SIZE_MAX\n /* .get_alloc_size = */ ggml_backend_cann_buffer_type_get_alloc_size,\n /* .is_host = */ ggml_backend_cann_buffer_type_is_host,\n};\n\n/**\n * @brief Retrieves the CANN buffer type for a specified device.\n *\n * This function initializes and returns the buffer type interface associated\n * with the given device. It ensures thread-safe access using a mutex.\n *\n * @param device The device index for which to retrieve the buffer type.\n * @return A pointer to the buffer type interface for the specified device, or\n * nullptr if the device index is out of range.\n */\nggml_backend_buffer_type_t ggml_backend_cann_buffer_type(int32_t device) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n if (device >= ggml_backend_cann_get_device_count()) {\n return nullptr;\n }\n\n static ggml_backend_buffer_type ggml_backend_cann_buffer_types[GGML_CANN_MAX_DEVICES];\n\n static bool ggml_backend_cann_buffer_type_initialized = false;\n\n if (!ggml_backend_cann_buffer_type_initialized) {\n for (int32_t i = 0; i < ggml_cann_info().device_count; i++) {\n ggml_backend_cann_buffer_types[i] = {\n /* .iface = */ ggml_backend_cann_buffer_type_interface,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cann_reg(), i),\n /* .context = */\n new ggml_backend_cann_buffer_type_context{ i, \"CANN\" + std::to_string(i) },\n };\n }\n ggml_backend_cann_buffer_type_initialized = true;\n }\n\n return &ggml_backend_cann_buffer_types[device];\n}\n\n/**\n * @brief Retrieves the name associated with a CANN host buffer type.\n *\n * This function returns the descriptive name associated with the specified\n * CANN host buffer type context.\n *\n * @param buft Pointer to the host buffer type context.\n * @return Const pointer to the C-style string containing the name.\n */\nstatic const char * ggml_backend_cann_host_buffer_type_name(ggml_backend_buffer_type_t buft) {\n return \"CANN_Host\";\n\n GGML_UNUSED(buft);\n}\n\n/**\n * @brief Retrieves the name associated with a CANN host buffer.\n *\n * This function returns the descriptive name associated with the specified\n * CANN host buffer context.\n *\n * @param buft Pointer to the host buffer context.\n * @return Const pointer to the C-style string containing the name.\n */\nstatic const char * ggml_backend_cann_host_buffer_name(ggml_backend_buffer_t buffer) {\n return \"CANN_Host\";\n\n GGML_UNUSED(buffer);\n}\n\n/**\n * @brief Free resources associated with a CANN host buffer.\n *\n * This function frees the resources associated with a CANN host buffer, including\n * its context.\n *\n * @param buffer The CANN host buffer to free.\n */\nstatic void ggml_backend_cann_host_buffer_free(ggml_backend_buffer_t buffer) {\n ACL_CHECK(aclrtFreeHost(buffer->context));\n}\n\n/**\n * @brief Allocates a new CANN host buffer of the specified size.\n *\n * This function allocates a new CANN host buffer with the given size.\n * @param size Size in bytes of the host buffer to allocate.\n * @return Pointer to the allocated host buffer, or nullptr if allocation fails.\n */\nstatic void * ggml_cann_host_malloc(size_t size) {\n if (getenv(\"GGML_CANN_NO_PINNED\") != nullptr) {\n return nullptr;\n }\n\n const size_t alignment = 128;\n size = GGML_PAD(size, alignment);\n if (size == 0) {\n size = alignment;\n }\n\n void * hostPtr = nullptr;\n aclError err = aclrtMallocHost((void **) &hostPtr, size);\n if (err != ACL_SUCCESS) {\n GGML_LOG_WARN(\"%s: failed to allocate %.2f MiB of pinned memory: %s\\n\", __func__, size / 1024.0 / 1024.0,\n aclGetRecentErrMsg());\n return nullptr;\n }\n return hostPtr;\n}\n\n/**\n * @brief Allocates a new CANN host buffer of the specified type and size.\n *\n * @param buft Pointer to the host buffer type context.\n * @param size Size in bytes of the host buffer to allocate.\n * @return Pointer to the allocated host buffer, or CPU buffer pointer if allocation fails.\n */\nstatic ggml_backend_buffer_t ggml_backend_cann_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,\n size_t size) {\n void * hostPtr = ggml_cann_host_malloc(size);\n\n if (hostPtr == nullptr) {\n // fallback to cpu buffer\n return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n }\n\n ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(hostPtr, size);\n buffer->buft = buft;\n buffer->iface.free_buffer = ggml_backend_cann_host_buffer_free;\n\n return buffer;\n}\n\n/**\n * @brief Interface for managing CANN host buffer types in the GGML backend.\n *\n * Provides function pointers for allocating, querying properties, and managing\n * memory for CANN buffer types in the GGML backend.\n */\nggml_backend_buffer_type_t ggml_backend_cann_host_buffer_type() {\n static struct ggml_backend_buffer_type ggml_backend_cann_buffer_type_host = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_cann_host_buffer_type_name,\n /* .alloc_buffer = */ ggml_backend_cann_host_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,\n /* .get_max_size = */ NULL, // defaults to SIZE_MAX\n /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,\n /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,\n },\n /* .device = */\n ggml_backend_reg_dev_get(ggml_backend_cann_reg(), 0),\n /* .context = */ nullptr,\n };\n\n return &ggml_backend_cann_buffer_type_host;\n}\n\n/**\n * @brief Computes the forward operation for a given tensor using CANN\n * operations.\n *\n * This function selects the appropriate CANN operation based on the type of\n * operation specified in the tensor and performs the computation.\n *\n * @param ctx The CANN context containing necessary resources and\n * configurations.\n * @param dst The destination tensor where the result of the computation will be\n * stored.\n * @return true if the computation was successful; false otherwise.\n */\nstatic bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct ggml_tensor * dst) {\n switch (dst->op) {\n case GGML_OP_REPEAT:\n ggml_cann_repeat(ctx, dst);\n break;\n case GGML_OP_GET_ROWS:\n ggml_cann_get_rows(ctx, dst);\n break;\n case GGML_OP_SET_ROWS:\n ggml_cann_set_rows(ctx, dst);\n break;\n case GGML_OP_DUP:\n ggml_cann_dup(ctx, dst);\n break;\n case GGML_OP_ADD:\n case GGML_OP_ADD1:\n ggml_cann_binary_op(ctx, dst);\n break;\n case GGML_OP_SUB:\n ggml_cann_binary_op(ctx, dst);\n break;\n case GGML_OP_ACC:\n ggml_cann_acc(ctx, dst);\n break;\n case GGML_OP_MUL:\n ggml_cann_binary_op(ctx, dst);\n break;\n case GGML_OP_DIV:\n ggml_cann_binary_op(ctx, dst);\n break;\n case GGML_OP_UNARY:\n switch (ggml_get_unary_op(dst)) {\n case GGML_UNARY_OP_ABS:\n GGML_CANN_CALL_OP_UNARY(Abs);\n break;\n case GGML_UNARY_OP_NEG:\n GGML_CANN_CALL_OP_UNARY(Neg);\n break;\n case GGML_UNARY_OP_GELU:\n case GGML_UNARY_OP_GELU_ERF:\n // aclnnGelu internally uses the erf-based approximation.\n GGML_CANN_CALL_OP_UNARY(Gelu);\n break;\n case GGML_UNARY_OP_SILU:\n GGML_CANN_CALL_OP_UNARY(Silu);\n break;\n case GGML_UNARY_OP_GELU_QUICK:\n {\n auto lambda = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n GGML_CANN_CALL_ACLNN_OP(ctx, GeluV2, acl_src, 0, acl_dst);\n };\n ggml_cann_op_unary(lambda, ctx, dst);\n }\n break;\n case GGML_UNARY_OP_TANH:\n GGML_CANN_CALL_OP_UNARY(Tanh);\n break;\n case GGML_UNARY_OP_RELU:\n GGML_CANN_CALL_OP_UNARY(Relu);\n break;\n case GGML_UNARY_OP_SIGMOID:\n GGML_CANN_CALL_OP_UNARY(Sigmoid);\n break;\n case GGML_UNARY_OP_HARDSIGMOID:\n GGML_CANN_CALL_OP_UNARY(Hardsigmoid);\n break;\n case GGML_UNARY_OP_HARDSWISH:\n GGML_CANN_CALL_OP_UNARY(Hardswish);\n break;\n case GGML_UNARY_OP_EXP:\n GGML_CANN_CALL_OP_UNARY(Exp);\n break;\n case GGML_UNARY_OP_ELU:\n ggml_cann_elu(ctx, dst);\n break;\n case GGML_UNARY_OP_SGN:\n GGML_CANN_CALL_OP_UNARY(Sign);\n break;\n case GGML_UNARY_OP_STEP:\n ggml_cann_step(ctx, dst);\n break;\n default:\n return false;\n }\n break;\n case GGML_OP_GLU:\n switch (ggml_get_glu_op(dst)) {\n case GGML_GLU_OP_REGLU:\n GGML_CANN_CALL_OP_UNARY_GATED(Relu);\n break;\n case GGML_GLU_OP_GEGLU:\n case GGML_GLU_OP_GEGLU_ERF:\n // aclnnGelu internally uses the erf-based approximation.\n GGML_CANN_CALL_OP_UNARY_GATED(Gelu);\n break;\n case GGML_GLU_OP_SWIGLU:\n GGML_CANN_CALL_OP_UNARY_GATED(Silu);\n break;\n case GGML_GLU_OP_GEGLU_QUICK:\n {\n auto lambda = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n GGML_CANN_CALL_ACLNN_OP(ctx, GeluV2, acl_src, 0, acl_dst);\n };\n ggml_cann_op_unary_gated(lambda, ctx, dst);\n }\n break;\n default:\n return false;\n }\n break;\n case GGML_OP_NORM:\n ggml_cann_norm(ctx, dst);\n break;\n case GGML_OP_GROUP_NORM:\n ggml_cann_group_norm(ctx, dst);\n break;\n case GGML_OP_L2_NORM:\n ggml_cann_l2_norm(ctx, dst);\n break;\n case GGML_OP_CROSS_ENTROPY_LOSS:\n ggml_cann_cross_entropy_loss(ctx, dst);\n break;\n case GGML_OP_CONCAT:\n ggml_cann_concat(ctx, dst);\n break;\n case GGML_OP_UPSCALE:\n ggml_cann_upsample_nearest2d(ctx, dst);\n break;\n case GGML_OP_PAD:\n ggml_cann_pad(ctx, dst);\n break;\n case GGML_OP_ARANGE:\n ggml_cann_arange(ctx, dst);\n break;\n case GGML_OP_TIMESTEP_EMBEDDING:\n ggml_cann_timestep_embedding(ctx, dst);\n break;\n case GGML_OP_LEAKY_RELU:\n ggml_cann_leaky_relu(ctx, dst);\n break;\n case GGML_OP_RMS_NORM:\n ggml_cann_rms_norm(ctx, dst);\n break;\n case GGML_OP_MUL_MAT:\n ggml_cann_mul_mat(ctx, dst);\n break;\n case GGML_OP_MUL_MAT_ID:\n ggml_cann_mul_mat_id(ctx, dst);\n break;\n case GGML_OP_SCALE:\n ggml_cann_scale(ctx, dst);\n break;\n case GGML_OP_SQR:\n GGML_ASSERT(dst->src[1] == nullptr);\n dst->src[1] = dst->src[0];\n ggml_cann_binary_op(ctx, dst);\n break;\n case GGML_OP_SQRT:\n GGML_CANN_CALL_OP_UNARY(Sqrt);\n break;\n case GGML_OP_CLAMP:\n ggml_cann_clamp(ctx, dst);\n break;\n case GGML_OP_CPY:\n ggml_cann_cpy(ctx, dst);\n break;\n case GGML_OP_CONT:\n ggml_cann_dup(ctx, dst);\n break;\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n break;\n case GGML_OP_DIAG_MASK_INF:\n ggml_cann_diag_mask(ctx, dst, -INFINITY);\n break;\n case GGML_OP_SOFT_MAX:\n ggml_cann_softmax(ctx, dst);\n break;\n case GGML_OP_ROPE:\n ggml_cann_rope(ctx, dst);\n break;\n case GGML_OP_IM2COL:\n ggml_cann_im2col(ctx, dst);\n break;\n case GGML_OP_POOL_2D:\n ggml_cann_pool2d(ctx, dst);\n break;\n case GGML_OP_SUM:\n ggml_cann_sum(ctx, dst);\n break;\n case GGML_OP_SUM_ROWS:\n ggml_cann_sum_rows(ctx, dst);\n break;\n case GGML_OP_ARGSORT:\n ggml_cann_argsort(ctx, dst);\n break;\n case GGML_OP_ARGMAX:\n ggml_cann_argmax(ctx, dst);\n break;\n case GGML_OP_COS:\n ggml_cann_op_unary(ctx, dst);\n break;\n case GGML_OP_SIN:\n ggml_cann_op_unary(ctx, dst);\n break;\n case GGML_OP_CONV_TRANSPOSE_1D:\n ggml_cann_conv_transpose_1d(ctx, dst);\n break;\n case GGML_OP_LOG:\n GGML_CANN_CALL_OP_UNARY(Log);\n break;\n case GGML_OP_MEAN:\n ggml_cann_mean(ctx, dst);\n break;\n case GGML_OP_PAD_REFLECT_1D:\n ggml_cann_pad_reflect_1d(ctx, dst);\n break;\n case GGML_OP_COUNT_EQUAL:\n ggml_cann_count_equal(ctx, dst);\n break;\n case GGML_OP_FLASH_ATTN_EXT:\n ggml_cann_flash_attn_ext(ctx, dst);\n break;\n case GGML_OP_OUT_PROD:\n ggml_cann_out_prod(ctx, dst);\n break;\n case GGML_OP_GATED_LINEAR_ATTN:\n ggml_cann_gated_linear_attn(ctx, dst);\n break;\n case GGML_OP_SSM_CONV:\n ggml_cann_ssm_conv(ctx, dst);\n break;\n default:\n return false;\n }\n\n return true;\n}\n\n// backend\n/**\n * @brief Retrieves the name associated with the CANN backend.\n *\n * This function returns the name assigned to the CANN backend, which is stored\n * in the context of the provided backend structure.\n *\n * @param backend Pointer to the CANN backend structure.\n * @return A pointer to a constant string representing the backend name.\n */\nstatic const char * ggml_backend_cann_name(ggml_backend_t backend) {\n ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n\n return cann_ctx->name.c_str();\n}\n\n/**\n * @brief Frees resources associated with the CANN backend.\n *\n * This function releases resources associated with the CANN backend context\n * and resets the device associated with the backend to its initial state.\n *\n * @param backend Pointer to the CANN backend structure to be freed.\n */\nstatic void ggml_backend_cann_free(ggml_backend_t backend) {\n ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n ACL_CHECK(aclrtSynchronizeDevice());\n ACL_CHECK(aclrtResetDevice(cann_ctx->device));\n\n delete cann_ctx;\n delete backend;\n}\n\n/**\n * @brief Sets tensor data asynchronously in the CANN backend.\n *\n * This function asynchronously sets tensor data in the CANN backend.\n *\n * @param backend Pointer to the CANN backend structure.\n * @param tensor Pointer to the tensor structure to set data for.\n * @param data Pointer to the host data to copy to the tensor.\n * @param offset Offset in bytes within the host data.\n * @param size Size of the data to copy in bytes.\n */\nstatic void ggml_backend_cann_set_tensor_async(ggml_backend_t backend,\n ggml_tensor * tensor,\n const void * data,\n size_t offset,\n size_t size) {\n ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n GGML_ASSERT(buf->buft == ggml_backend_cann_buffer_type(cann_ctx->device) && \"unsupported buffer type\");\n GGML_ASSERT(!ggml_is_quantized(tensor->type));\n\n ACL_CHECK(aclrtMemcpyAsync((char *) tensor->data + offset, size, data, size, ACL_MEMCPY_HOST_TO_DEVICE,\n cann_ctx->stream()));\n}\n\n/**\n * @brief Gets tensor data asynchronously in the CANN backend.\n *\n * This function asynchronously gets tensor data in the CANN backend.\n *\n * @param backend Pointer to the CANN backend structure.\n * @param tensor Pointer to the tensor structure to get data from.\n * @param data Pointer to the host data to copy from the tensor.\n * @param offset Offset in bytes within the host data.\n * @param size Size of the data to copy in bytes.\n */\nstatic void ggml_backend_cann_get_tensor_async(ggml_backend_t backend,\n const ggml_tensor * tensor,\n void * data,\n size_t offset,\n size_t size) {\n ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n GGML_ASSERT(buf->buft == ggml_backend_cann_buffer_type(cann_ctx->device) && \"unsupported buffer type\");\n GGML_ASSERT(!ggml_is_quantized(tensor->type));\n\n ACL_CHECK(aclrtMemcpyAsync(data, size, (char *) tensor->data + offset, size, ACL_MEMCPY_DEVICE_TO_HOST,\n cann_ctx->stream()));\n}\n\n/**\n * @brief Asynchronously copies tensor data between CANN backends.\n *\n * This function copies tensor data asynchronously between two CANN backends. It\n * checks if both tensors reside in CANN buffers and whether the devices support\n * peer-to-peer access for direct copying. If not, it returns false.\n *\n * @param backend_src Pointer to the source CANN backend structure.\n * @param backend_dst Pointer to the destination CANN backend structure.\n * @param src Pointer to the source tensor to copy data from.\n * @param dst Pointer to the destination tensor to copy data to.\n * @return true if the copy operation succeeds, false otherwise.\n */\nstatic bool ggml_backend_cann_cpy_tensor_async(ggml_backend_t backend_src,\n ggml_backend_t backend_dst,\n const ggml_tensor * src,\n ggml_tensor * dst) {\n GGML_ASSERT(ggml_backend_is_cann(backend_src) || ggml_backend_is_cann(backend_dst));\n\n GGML_ASSERT(!is_matmul_weight((const ggml_tensor *) src));\n\n if (!ggml_backend_buft_is_cann(src->buffer->buft) || !ggml_backend_buft_is_cann(dst->buffer->buft)) {\n return false;\n }\n\n ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;\n ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;\n\n ggml_backend_cann_context * cann_ctx_src = (ggml_backend_cann_context *) backend_src->context;\n ggml_backend_cann_context * cann_ctx_dst = (ggml_backend_cann_context *) backend_dst->context;\n\n size_t copy_size = ggml_nbytes(dst);\n if (copy_size == 0) {\n return true;\n }\n if (backend_src != backend_dst) {\n#ifdef ASCEND_310P\n // TODO: Support 310p P2P copy\n return false;\n#endif\n ggml_backend_cann_buffer_context * buf_ctx_src = (ggml_backend_cann_buffer_context *) buf_src->context;\n ggml_backend_cann_buffer_context * buf_ctx_dst = (ggml_backend_cann_buffer_context *) buf_dst->context;\n\n GGML_ASSERT(cann_ctx_src->device == buf_ctx_src->device);\n GGML_ASSERT(cann_ctx_dst->device == buf_ctx_dst->device);\n\n int32_t canAccessPeer = 0;\n ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, cann_ctx_src->device, cann_ctx_dst->device));\n if (!canAccessPeer) {\n return false;\n }\n\n // need open both directions for memcpyasync between devices.\n ACL_CHECK(aclrtDeviceEnablePeerAccess(cann_ctx_src->device, 0));\n ggml_cann_set_device(cann_ctx_src->device);\n ACL_CHECK(aclrtDeviceEnablePeerAccess(cann_ctx_dst->device, 0));\n\n // wait for task_queue empty to keep task order.\n ACL_CHECK(aclrtMemcpyAsync(dst->data, copy_size, src->data, copy_size, ACL_MEMCPY_DEVICE_TO_DEVICE,\n cann_ctx_src->stream()));\n // record event on src stream after the copy\n // TODO: this event is not effective with acl graph mode, change to use aclrtSynchronizeStream\n // if (!cann_ctx_src->copy_event) {\n // ACL_CHECK(aclrtCreateEventWithFlag(&cann_ctx_src->copy_event, ACL_EVENT_SYNC));\n // }\n // ACL_CHECK(aclrtRecordEvent(cann_ctx_src->copy_event, cann_ctx_src->stream()));\n\n // // wait on dst stream for the copy to complete\n // ggml_cann_set_device(cann_ctx_dst->device);\n // ACL_CHECK(aclrtStreamWaitEvent(cann_ctx_dst->stream(), cann_ctx_src->copy_event));\n ACL_CHECK(aclrtSynchronizeStream(cann_ctx_src->stream()));\n } else {\n // src and dst are on the same backend\n ACL_CHECK(aclrtMemcpyAsync(dst->data, copy_size, src->data, copy_size, ACL_MEMCPY_DEVICE_TO_DEVICE,\n cann_ctx_dst->stream()));\n }\n\n return true;\n}\n\n/**\n * @brief Synchronizes a CANN backend.\n *\n * This function synchronizes the specified CANN backend by waiting for all\n * operations in its associated stream to complete.\n *\n * @param backend Pointer to the CANN backend structure to synchronize.\n */\nstatic void ggml_backend_cann_synchronize(ggml_backend_t backend) {\n ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n ggml_cann_set_device(cann_ctx->device);\n ACL_CHECK(aclrtSynchronizeStream(cann_ctx->stream()));\n}\n\n/**\n * @brief Check if CANN backend can fuse the specified operation sequence\n *\n * This function determines whether an operation sequence starting from the specified node\n * can be fused into an optimized operation in the CANN backend. Operation fusion can reduce\n * memory access overhead and improve computational efficiency.\n *\n * @param cgraph Pointer to the computation graph\n * @param node_idx Index of the starting node in the computation graph\n * @param ops Sequence of operation types to check for fusion\n * @return true if the operations can be fused\n * @return false if the operations cannot be fused\n */\nstatic bool ggml_cann_can_fuse(const struct ggml_cgraph * cgraph,\n int node_idx,\n std::initializer_list ops) {\n if (!ggml_can_fuse(cgraph, node_idx, ops)) {\n return false;\n }\n\n // CANN backend supports fusing ADD + RMS_NORM operations\n if ((ops.size() == 2) && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_RMS_NORM) {\n ggml_tensor * add_node = cgraph->nodes[node_idx];\n // TODO: support broadcast for ADD + RMS_NORM\n if (add_node->src[0]->ne[0] != add_node->src[1]->ne[0] || add_node->src[0]->ne[1] != add_node->src[1]->ne[1] ||\n add_node->src[0]->ne[2] != add_node->src[1]->ne[2] || add_node->src[0]->ne[3] != add_node->src[1]->ne[3]) {\n return false;\n }\n return true;\n }\n\n return false;\n}\n\n/**\n * @brief Evaluate the computation graph and optionally capture or execute it using CANN graph API.\n *\n * If CANN graph execution is enabled and graph capture is required, this function begins\n * graph capture, runs the graph, ends capture, and stores the captured graph.\n *\n * Otherwise, it falls back to op-by-op execution using the CANN compute kernel dispatcher.\n *\n * @param cann_ctx The CANN backend context.\n * @param cgraph The ggml computation graph.\n * @param use_cann_graph Whether to use CANN graph execution.\n * @param cann_graph_capture_required Whether graph capture is needed due to graph changes.\n */\nstatic void evaluate_and_capture_cann_graph(ggml_backend_cann_context * cann_ctx,\n ggml_cgraph * cgraph,\n bool use_cann_graph,\n bool cann_graph_capture_required) {\n#ifdef USE_ACL_GRAPH\n if (use_cann_graph && cann_graph_capture_required) { // Begin CANN graph capture\n ACL_CHECK(aclmdlRICaptureBegin(cann_ctx->stream(), ACL_MODEL_RI_CAPTURE_MODE_GLOBAL));\n }\n#endif // USE_ACL_GRAPH\n // Only perform the graph execution if CANN graphs are not enabled, or we are capturing the graph.\n // With the use of CANN graphs, the execution will be performed by the graph launch.\n static bool opt_fusion = parse_bool(get_env_as_lowercase(\"GGML_CANN_OPERATOR_FUSION\").value_or(\"\"));\n\n if (!use_cann_graph || cann_graph_capture_required) {\n for (int i = 0; i < cgraph->n_nodes; i++) {\n ggml_tensor * node = cgraph->nodes[i];\n if (opt_fusion) {\n if (ggml_cann_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_RMS_NORM })) {\n ggml_cann_op_add_rms_norm_fused(*cann_ctx, node, cgraph->nodes[i + 1]);\n i++;\n continue;\n }\n }\n\n if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE ||\n node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {\n continue;\n }\n\n if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n continue;\n }\n\n bool ok = ggml_cann_compute_forward(*cann_ctx, node);\n if (!ok) {\n GGML_LOG_ERROR(\"%s: op not supported %s (%s)\\n\", __func__, node->name, ggml_op_name(node->op));\n }\n GGML_ASSERT(ok);\n }\n }\n\n#ifdef USE_ACL_GRAPH\n if (use_cann_graph) {\n GGML_ASSERT(!cann_ctx->graph_lru_cache.cache_list.empty());\n ggml_cann_graph * matched_graph = cann_ctx->graph_lru_cache.cache_list.front();\n\n if (cann_graph_capture_required) { // End CANN graph capture\n ACL_CHECK(aclmdlRICaptureEnd(cann_ctx->stream(), &matched_graph->graph));\n }\n\n // Execute CANN graph\n ACL_CHECK(aclmdlRIExecuteAsync(matched_graph->graph, cann_ctx->stream()));\n }\n#endif // USE_ACL_GRAPH\n}\n\n/**\n * @brief Computes a computational graph using a CANN backend.\n *\n * This function computes the operations defined in the computational graph\n * using the specified CANN backend.\n *\n * @param backend Pointer to the CANN backend structure to use for computation.\n * @param cgraph Pointer to the computational graph structure containing nodes\n * representing operations to be computed.\n * @return enum ggml_status Returns GGML_STATUS_SUCCESS if computation\n * completes successfully, otherwise an appropriate error status.\n */\nstatic enum ggml_status ggml_backend_cann_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {\n ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n ggml_cann_set_device(cann_ctx->device);\n g_nz_workspaces[cann_ctx->device].clear();\n\n // calculate rope cache for fist layer in current device.\n cann_ctx->rope_cache.cached = false;\n\n bool graph_capture_required = false;\n#ifdef USE_ACL_GRAPH\n bool use_cann_graph = true;\n\n static bool prefill_use_graph = parse_bool(get_env_as_lowercase(\"GGML_CANN_PREFILL_USE_GRAPH\").value_or(\"\"));\n if (!prefill_use_graph) {\n // Do not use acl_graph for prefill.\n for (int i = 0; i < cgraph->n_nodes; i++) {\n ggml_tensor * node = cgraph->nodes[i];\n // TODO: Optimize here. Currently, we can only\n // get seq_len by FA's input.\n if (node->op == GGML_OP_FLASH_ATTN_EXT) {\n // Q -> src[0], shape: [B, S, N, D]\n use_cann_graph = (node->src[0]->ne[1] == 1);\n break;\n }\n }\n }\n\n if (!cann_ctx->acl_graph_mode) {\n use_cann_graph = false;\n }\n\n if (use_cann_graph) {\n // If no matching graph is found, the graph needs to be recaptured.\n graph_capture_required = !cann_ctx->graph_lru_cache.find_and_move_to_front(cgraph);\n if (graph_capture_required) {\n // If no matching graph is found, add a new ACL graph.\n ggml_cann_graph * new_graph = ggml_cann_graph::create_from_cgraph(cgraph);\n cann_ctx->graph_lru_cache.push(new_graph);\n }\n }\n#else\n bool use_cann_graph = false;\n#endif // USE_ACL_GRAPH\n evaluate_and_capture_cann_graph(cann_ctx, cgraph, use_cann_graph, graph_capture_required);\n\n return GGML_STATUS_SUCCESS;\n}\n\n/**\n * @brief Checks if the CANN backend supports a specific operation.\n *\n * This function checks whether the specified operation is supported by the\n * CANN backend.\n *\n * @param backend Pointer to the CANN backend structure to check support for\n * the operation.\n * @param op Pointer to the tensor representing the operation to check.\n * @return bool Returns true if the operation is supported by the backend,\n * otherwise false.\n */\nstatic bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {\n switch (op->op) {\n case GGML_OP_UNARY:\n switch (ggml_get_unary_op(op)) {\n case GGML_UNARY_OP_ABS:\n case GGML_UNARY_OP_NEG:\n case GGML_UNARY_OP_GELU:\n case GGML_UNARY_OP_SILU:\n case GGML_UNARY_OP_RELU:\n case GGML_UNARY_OP_SIGMOID:\n case GGML_UNARY_OP_HARDSIGMOID:\n case GGML_UNARY_OP_HARDSWISH:\n case GGML_UNARY_OP_GELU_QUICK:\n case GGML_UNARY_OP_TANH:\n case GGML_UNARY_OP_EXP:\n case GGML_UNARY_OP_ELU:\n case GGML_UNARY_OP_SGN:\n case GGML_UNARY_OP_STEP:\n case GGML_UNARY_OP_GELU_ERF:\n return true;\n default:\n return false;\n }\n case GGML_OP_GLU:\n switch (ggml_get_glu_op(op)) {\n case GGML_GLU_OP_REGLU:\n case GGML_GLU_OP_GEGLU:\n case GGML_GLU_OP_SWIGLU:\n case GGML_GLU_OP_GEGLU_ERF:\n case GGML_GLU_OP_GEGLU_QUICK:\n return true;\n default:\n return false;\n }\n break;\n case GGML_OP_MUL_MAT:\n {\n switch (op->src[0]->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_F32:\n return true;\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q4_0:\n#ifdef ASCEND_310P\n // Q4 && Q8 per group is not support on 310p device\n return false;\n#endif\n // only support contiguous for quantized types.\n return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);\n default:\n return false;\n }\n }\n case GGML_OP_MUL_MAT_ID:\n switch (op->src[0]->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_F32:\n return true;\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q4_0:\n#ifdef ASCEND_310P\n // Q4 && Q8 per group is not support on 310p device\n return false;\n#endif\n // only support contiguous for quantized types.\n return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);\n default:\n return false;\n }\n // embedding\n case GGML_OP_GET_ROWS:\n {\n switch (op->src[0]->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n case GGML_TYPE_Q8_0:\n return true;\n default:\n return false;\n }\n }\n break;\n case GGML_OP_SET_ROWS:\n {\n switch (op->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n return true;\n default:\n return false;\n }\n }\n break;\n case GGML_OP_CPY:\n {\n ggml_tensor * src = op->src[0];\n if ((op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) ||\n (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_F16)) {\n // only support F32 and F16.\n return false;\n }\n return true;\n }\n break;\n case GGML_OP_CONT:\n {\n // TODO: support GGML_TYPE_BF16\n switch (op->src[0]->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n return true;\n default:\n return false;\n }\n }\n case GGML_OP_ROPE:\n {\n if (op->src[0]->ne[0] > 896) {\n return false;\n }\n#ifdef ASCEND_310P\n // TODO: Support rope_dim < ne00(dim)\n if (op->src[0]->ne[0] != op->op_params[1]) {\n return false;\n }\n if (!ggml_is_contiguous(op->src[0])) {\n return false;\n }\n#endif\n return true;\n }\n case GGML_OP_UPSCALE:\n {\n // aclnnUpsampleNearest2dGetWorkspaceSize not support\n // selfDimN[2]/outDimN[2] or selfDimC[3]/outDimC[3] not equal\n if (op->src[0]->ne[2] * op->ne[3] != op->src[0]->ne[3] * op->ne[2]) {\n return false;\n }\n if (op->op_params[0] != GGML_SCALE_MODE_NEAREST) {\n return false;\n }\n if (op->op_params[0] & GGML_SCALE_FLAG_ANTIALIAS) {\n return false;\n }\n return true;\n }\n case GGML_OP_POOL_2D:\n {\n const int32_t * opts = (const int32_t *) op->op_params;\n#ifdef ASCEND_310P\n enum ggml_op_pool opt = static_cast(opts[0]);\n if (opt == GGML_OP_POOL_MAX) {\n return false;\n }\n#endif\n const int k0 = opts[1];\n const int k1 = opts[2];\n const int p0 = opts[5];\n const int p1 = opts[6];\n // value of paddingH should be at most half of kernelH\n // value of paddingW should be at most half of kernelW\n return (p0 <= (k0 / 2)) && (p1 <= (k1 / 2));\n }\n case GGML_OP_SUM:\n return ggml_is_contiguous_rows(op->src[0]);\n case GGML_OP_L2_NORM:\n case GGML_OP_CROSS_ENTROPY_LOSS:\n case GGML_OP_DUP:\n case GGML_OP_IM2COL:\n case GGML_OP_CONCAT:\n case GGML_OP_REPEAT:\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n case GGML_OP_NORM:\n case GGML_OP_ADD:\n case GGML_OP_ADD1:\n case GGML_OP_SUB:\n case GGML_OP_MUL:\n case GGML_OP_DIV:\n case GGML_OP_RMS_NORM:\n case GGML_OP_SQR:\n case GGML_OP_SQRT:\n case GGML_OP_CLAMP:\n case GGML_OP_DIAG_MASK_INF:\n case GGML_OP_SUM_ROWS:\n case GGML_OP_ARGSORT:\n case GGML_OP_ACC:\n case GGML_OP_GROUP_NORM:\n return true;\n case GGML_OP_PAD:\n // TODO: add circular padding support for cann, see https://github.com/ggml-org/llama.cpp/pull/16985\n return ggml_get_op_params_i32(op, 8) == 0;\n case GGML_OP_ARANGE:\n case GGML_OP_TIMESTEP_EMBEDDING:\n case GGML_OP_LEAKY_RELU:\n case GGML_OP_ARGMAX:\n case GGML_OP_COS:\n case GGML_OP_SIN:\n case GGML_OP_LOG:\n case GGML_OP_MEAN:\n case GGML_OP_PAD_REFLECT_1D:\n case GGML_OP_COUNT_EQUAL:\n case GGML_OP_GATED_LINEAR_ATTN:\n return true;\n case GGML_OP_OUT_PROD:\n {\n#ifdef ASCEND_310P\n // Ger is not supported on 310p device\n return false;\n#endif\n switch (op->src[0]->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_F32:\n return true;\n default:\n return false;\n }\n }\n case GGML_OP_CONV_TRANSPOSE_1D:\n return true;\n case GGML_OP_SCALE:\n float bias;\n memcpy(&bias, (const float *) (op->op_params) + 1, sizeof(float));\n return bias == 0.0f; // TODO: support bias != 0.0f\n case GGML_OP_SOFT_MAX:\n // TODO: support attention sinks [TAG_ATTN_SINKS]\n if (op->src[2]) {\n return false;\n }\n return true;\n case GGML_OP_FLASH_ATTN_EXT:\n {\n#ifdef ASCEND_310P\n // FA not support on 310p device\n return false;\n#endif\n // derived from [ggml-cuda.cu]\n if (op->src[1]->type != GGML_TYPE_F16 || op->src[2]->type != GGML_TYPE_F16) {\n return false;\n }\n if (op->src[1]->type != GGML_TYPE_F16 && op->src[1]->type != GGML_TYPE_F32 &&\n op->src[1]->type != GGML_TYPE_BF16) {\n return false;\n }\n if (op->type != GGML_TYPE_F16 && op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_BF16) {\n return false;\n }\n // TODO: support attention sinks [TAG_ATTN_SINKS]\n if (op->src[4]) {\n return false;\n }\n if (op->src[1]->ne[0] != op->src[2]->ne[0]) {\n // different head sizes of K and V are not supported yet\n return false;\n }\n if (op->src[0]->ne[0] % 16 != 0) {\n // TODO: padding to support\n return false;\n }\n float logitSoftcap = 0.0f;\n memcpy(&logitSoftcap, (const float *) (op->op_params) + 2, sizeof(float));\n if (logitSoftcap != 0.0f) {\n return false;\n }\n return true;\n }\n case GGML_OP_SSM_CONV:\n return true;\n default:\n return false;\n }\n\n GGML_UNUSED(dev);\n}\n\n/**\n * @brief Records an event on the CANN backend stream.\n *\n * This function records the given event on the ACL runtime stream associated\n * with the backend context.\n *\n * @param event Pointer to the event structure to be recorded.\n */\nstatic void ggml_backend_cann_event_record(ggml_backend_t backend, ggml_backend_event_t event) {\n ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n ACL_CHECK(aclrtRecordEvent((aclrtEvent) event->context, cann_ctx->stream()));\n}\n\n/**\n * @brief Waits for a recorded event to complete on the CANN backend stream.\n *\n * This function makes the given backend wait for the event to complete on its\n * ACL runtime stream.\n *\n * @param backend Pointer to the backend structure.\n * @param event Pointer to the event structure that the backend needs to wait\n * for.\n */\nstatic void ggml_backend_cann_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {\n ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n if (ggml_backend_is_cann(backend)) {\n ACL_CHECK(aclrtStreamWaitEvent(cann_ctx->stream(), (aclrtEvent) event->context));\n } else {\n GGML_ABORT(\"fatal error\");\n }\n}\n\n/**\n * @brief Structure defining the interface for the CANN backend.\n *\n * This structure contains function pointers for various operations\n * supported by the CANN backend, including name retrieval, memory\n * management, tensor operations, synchronization, and event handling.\n */\nstatic const ggml_backend_i ggml_backend_cann_interface = {\n /* .get_name = */ ggml_backend_cann_name,\n /* .free = */ ggml_backend_cann_free,\n /* .set_tensor_async = */ ggml_backend_cann_set_tensor_async,\n /* .get_tensor_async = */ ggml_backend_cann_get_tensor_async,\n /* .cpy_tensor_async = */ ggml_backend_cann_cpy_tensor_async,\n /* .synchronize = */ ggml_backend_cann_synchronize,\n /* .graph_plan_create = */ NULL,\n /* .graph_plan_free = */ NULL,\n /* .graph_plan_update = */ NULL,\n /* .graph_plan_compute = */ NULL,\n /* .graph_compute = */ ggml_backend_cann_graph_compute,\n /* .event_record = */ ggml_backend_cann_event_record,\n /* .event_wait = */ ggml_backend_cann_event_wait,\n /* .graph_optimize = */ NULL,\n};\n\n/**\n * @brief Return the hardcoded GUID for the CANN backend.\n *\n * This function returns a static GUID which uniquely identifies the CANN\n * backend.\n *\n * @return A pointer to the static GUID.\n */\nstatic ggml_guid_t ggml_backend_cann_guid() {\n static ggml_guid guid = { 0xa1, 0x94, 0xaf, 0xac, 0xbd, 0x4f, 0x47, 0x34,\n 0xbe, 0x1a, 0x9e, 0x71, 0x1f, 0x9e, 0xed, 0x64 };\n return &guid;\n}\n\n// backend device\nstruct ggml_backend_cann_device_context {\n int device;\n std::string name;\n std::string description;\n int op_offload_min_batch_size;\n};\n\nstatic const char * ggml_backend_cann_device_get_name(ggml_backend_dev_t dev) {\n ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n return ctx->name.c_str();\n}\n\nstatic const char * ggml_backend_cann_device_get_description(ggml_backend_dev_t dev) {\n ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n return ctx->description.c_str();\n}\n\nstatic void ggml_backend_cann_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n ggml_backend_cann_get_device_memory(ctx->device, free, total);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_cann_device_get_type(ggml_backend_dev_t dev) {\n GGML_UNUSED(dev);\n return GGML_BACKEND_DEVICE_TYPE_GPU;\n}\n\nstatic void ggml_backend_cann_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {\n props->name = ggml_backend_cann_device_get_name(dev);\n props->description = ggml_backend_cann_device_get_description(dev);\n props->type = ggml_backend_cann_device_get_type(dev);\n ggml_backend_cann_device_get_memory(dev, &props->memory_free, &props->memory_total);\n\n bool host_buffer = getenv(\"GGML_CANN_NO_PINNED\") == nullptr;\n\n props->caps = {\n /* .async = */ false,\n /* .host_buffer = */ host_buffer,\n /* .buffer_from_host_ptr = */ false,\n /* .events = */ true,\n };\n}\n\nstatic ggml_backend_t ggml_backend_cann_device_init(ggml_backend_dev_t dev, const char * params) {\n GGML_UNUSED(params);\n ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n return ggml_backend_cann_init(ctx->device);\n}\n\n/**\n * @brief Checks if the CANN backend supports a specific backend buffer type.\n *\n * This function determines whether the CANN backend supports the given backend\n * buffer type by comparing the device context of the backend and buffer type.\n * It returns true if the devices are same between the backend context and\n * buffer type context.\n *\n * @param backend Pointer to the CANN backend.\n * @param buft Pointer to the backend buffer type to check.\n * @return bool Returns true if the CANN backend supports the buffer type,\n * otherwise false.\n */\nstatic bool ggml_backend_cann_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n if (ggml_backend_buft_is_cann(buft)) {\n ggml_backend_cann_device_context * dev_ctx = (ggml_backend_cann_device_context *) dev->context;\n ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context;\n return buft_ctx->device == dev_ctx->device;\n }\n return false;\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cann_device_get_buffer_type(ggml_backend_dev_t dev) {\n ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n return ggml_backend_cann_buffer_type(ctx->device);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cann_device_get_host_buffer_type(ggml_backend_dev_t dev) {\n GGML_UNUSED(dev);\n return ggml_backend_cann_host_buffer_type();\n}\n\n/**\n * @brief Determines if a tensor operation should be offloaded to the CANN\n * backend.\n *\n * This function checks if a given tensor operation should be offloaded to the\n * CANN backend based on the operation type and the size of the tensor. It\n * returns true if the second dimension (ne[1]) of the tensor is greater than or\n * equal to the minimum batch size and the operation is not GGML_OP_GET_ROWS.\n *\n * @param backend Pointer to the CANN backend.\n * @param op Pointer to the tensor operation to check.\n * @return bool Returns true if the operation should be offloaded, otherwise\n * false.\n */\nstatic bool ggml_backend_cann_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) {\n ggml_backend_cann_device_context * dev_ctx = (ggml_backend_cann_device_context *)dev->context;\n\n return op->ne[1] >= dev_ctx->op_offload_min_batch_size && op->op != GGML_OP_GET_ROWS;\n}\n\n/**\n * @brief Creates a new event for the CANN backend device.\n *\n * This function initializes a new event for the CANN backend by setting the\n * device and creating an ACL runtime event. The created event is then wrapped\n * in a ggml_backend_event structure and returned.\n *\n * @param backend Pointer to the CANN backend.\n * @return ggml_backend_event_t Returns a pointer to the new event structure.\n */\nstatic ggml_backend_event_t ggml_backend_cann_device_event_new(ggml_backend_dev_t dev) {\n ggml_backend_cann_device_context * dev_ctx = (ggml_backend_cann_device_context *) dev->context;\n\n ggml_cann_set_device(dev_ctx->device);\n\n aclrtEvent event;\n ACL_CHECK(aclrtCreateEvent(&event));\n\n return new ggml_backend_event{\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cann_reg(), dev_ctx->device),\n /* .context = */ event,\n };\n}\n\n/**\n * @brief Frees a CANN backend event.\n *\n * This function destroys the ACL runtime event associated with the given CANN\n * backend event and then deletes the event structure itself.\n *\n * @param event Pointer to the event structure to be freed.\n */\nstatic void ggml_backend_cann_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) {\n ACL_CHECK(aclrtDestroyEvent((aclrtEvent) event->context));\n\n delete event;\n GGML_UNUSED(dev);\n}\n\n/**\n * @brief Synchronizes the given event on the CANN backend.\n *\n * This function waits for the specified event to complete on the ACL runtime.\n *\n * @param event Pointer to the event structure to be synchronized.\n */\nstatic void ggml_backend_cann_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) {\n ACL_CHECK(aclrtSynchronizeEvent((aclrtEvent) event->context));\n\n GGML_UNUSED(dev);\n}\n\nstatic const ggml_backend_device_i ggml_backend_cann_device_interface = {\n /* .get_name = */ ggml_backend_cann_device_get_name,\n /* .get_description = */ ggml_backend_cann_device_get_description,\n /* .get_memory = */ ggml_backend_cann_device_get_memory,\n /* .get_type = */ ggml_backend_cann_device_get_type,\n /* .get_props = */ ggml_backend_cann_device_get_props,\n /* .init_backend = */ ggml_backend_cann_device_init, // called for every card\n /* .get_buffer_type = */ ggml_backend_cann_device_get_buffer_type,\n /* .get_host_buffer_type = */ ggml_backend_cann_device_get_host_buffer_type,\n /* .buffer_from_host_ptr = */ NULL, // not supported for CANN\n /* .supports_op = */ ggml_backend_cann_supports_op,\n /* .supports_buft = */ ggml_backend_cann_supports_buft,\n /* .offload_op = */ ggml_backend_cann_offload_op,\n /* .event_new = */ ggml_backend_cann_device_event_new,\n /* .event_free = */ ggml_backend_cann_device_event_free,\n /* .event_synchronize = */ ggml_backend_cann_device_event_synchronize,\n};\n\n// backend reg\nstruct ggml_backend_cann_reg_context {\n std::vector devices;\n};\n\nstatic const char * ggml_backend_cann_reg_get_name(ggml_backend_reg_t reg) {\n GGML_UNUSED(reg);\n return GGML_CANN_NAME;\n}\n\nstatic size_t ggml_backend_cann_reg_get_device_count(ggml_backend_reg_t reg) {\n ggml_backend_cann_reg_context * ctx = (ggml_backend_cann_reg_context *) reg->context;\n return ctx->devices.size();\n}\n\nstatic ggml_backend_dev_t ggml_backend_cann_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n ggml_backend_cann_reg_context * ctx = (ggml_backend_cann_reg_context *) reg->context;\n GGML_ASSERT(index < ctx->devices.size());\n return ctx->devices[index];\n}\n\nstatic void * ggml_backend_cann_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n GGML_UNUSED(reg);\n GGML_UNUSED(name);\n // reserved for future use\n return nullptr;\n}\n\nstatic const ggml_backend_reg_i ggml_backend_cann_reg_interface = {\n /* .get_name = */ ggml_backend_cann_reg_get_name,\n /* .get_device_count = */ ggml_backend_cann_reg_get_device_count,\n /* .get_device = */ ggml_backend_cann_reg_get_device,\n /* .get_proc_address = */ ggml_backend_cann_reg_get_proc_address,\n};\n\n// backend registry, called only once for cann backend\nggml_backend_reg_t ggml_backend_cann_reg() {\n static ggml_backend_reg reg;\n static bool initialized = false;\n\n {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n if (!initialized) {\n aclInit(nullptr);\n ggml_backend_cann_reg_context * ctx = new ggml_backend_cann_reg_context;\n const int min_batch_size = getenv(\"GGML_OP_OFFLOAD_MIN_BATCH\") ? atoi(getenv(\"GGML_OP_OFFLOAD_MIN_BATCH\")) : 32;\n\n for (int i = 0; i < ggml_cann_info().device_count; i++) {\n ggml_backend_cann_device_context * dev_ctx = new ggml_backend_cann_device_context();\n dev_ctx->description = aclrtGetSocName();\n dev_ctx->device = i;\n dev_ctx->name = GGML_CANN_NAME + std::to_string(i);\n dev_ctx->op_offload_min_batch_size = min_batch_size;\n ggml_cann_set_device(i);\n ggml_backend_dev_t dev = new ggml_backend_device{ /* .iface = */ ggml_backend_cann_device_interface,\n /* .reg = */ ®,\n /* .context = */ dev_ctx };\n ctx->devices.push_back(dev);\n }\n\n reg = ggml_backend_reg{ /* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_cann_reg_interface,\n /* .context = */ ctx };\n }\n\n initialized = true;\n }\n\n return ®\n}\n\nggml_backend_t ggml_backend_cann_init(int32_t device) {\n aclInit(nullptr);\n if (device < 0 || device >= ggml_backend_cann_get_device_count()) {\n GGML_LOG_ERROR(\"%s: error: invalid device %d\\n\", __func__, device);\n return nullptr;\n }\n\n ggml_backend_cann_context * ctx = new ggml_backend_cann_context(device);\n if (ctx == nullptr) {\n GGML_LOG_ERROR(\"%s: error: failed to allocate context\\n\", __func__);\n return nullptr;\n }\n ggml_cann_set_device(ctx->device);\n ggml_backend_t cann_backend =\n new ggml_backend{ /* .guid = */ ggml_backend_cann_guid(),\n /* .interface = */ ggml_backend_cann_interface,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cann_reg(), device),\n /* .context = */ ctx };\n\n return cann_backend;\n}\n\nbool ggml_backend_is_cann(ggml_backend_t backend) {\n return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cann_guid());\n}\n\nint32_t ggml_backend_cann_get_device_count() {\n return ggml_cann_info().device_count;\n}\n\nvoid ggml_backend_cann_get_device_description(int32_t device, char * description, size_t description_size) {\n ggml_cann_set_device(device);\n const char * soc_name = aclrtGetSocName();\n snprintf(description, description_size, \"%s\", soc_name);\n}\n\nvoid ggml_backend_cann_get_device_memory(int32_t device, size_t * free, size_t * total) {\n ggml_cann_set_device(device);\n ACL_CHECK(aclrtGetMemInfo(ACL_HBM_MEM, free, total));\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_cann_reg)\n" }, { "path": "src/ggml-common.h", "content": "#ifndef GGML_COMMON_DECL\n\n#if defined(GGML_COMMON_DECL_C)\n#include \n\ntypedef uint16_t ggml_half;\ntypedef uint32_t ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_CPP)\n#include \n\ntypedef uint16_t ggml_half;\ntypedef uint32_t ggml_half2;\n\n// std-c++ allow anonymous unions but some compiler warn on it\n#define GGML_COMMON_AGGR_U data\n// std-c++ do not allow it.\n#define GGML_COMMON_AGGR_S data\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_METAL)\n#include \n\ntypedef half ggml_half;\ntypedef half2 ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_CUDA)\n#if defined(GGML_COMMON_DECL_MUSA)\n#include \n#else\n#include \n#endif\n#include \n\ntypedef half ggml_half;\ntypedef half2 ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S data\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_HIP)\n#include \n#include \n\ntypedef half ggml_half;\ntypedef half2 ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S data\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_SYCL)\n#include \n#include \n\ntypedef sycl::half ggml_half;\ntypedef sycl::half2 ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S data\n\n#define GGML_COMMON_DECL\n#endif\n\n#if defined(GGML_COMMON_DECL)\n\n#ifndef __cplusplus\n#ifndef static_assert\n#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)\n#define static_assert(cond, msg) _Static_assert(cond, msg)\n#else\n#define static_assert(cond, msg) struct global_scope_noop_trick\n#endif\n#endif\n#endif // __cplusplus\n\n// QK = number of values after dequantization\n// QK_K = super-block size\n\n#define QK_K 256\n#define K_SCALE_SIZE 12\n\n#if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL)\n// QR = QK / number of values before dequantization\n// QI = number of 32 bit integers before dequantization\n\n#define QI4_0 (QK4_0 / (4 * QR4_0))\n#define QR4_0 2\n\n#define QI4_1 (QK4_1 / (4 * QR4_1))\n#define QR4_1 2\n\n#define QI_MXFP4 (QK_MXFP4 / (4 * QR_MXFP4))\n#define QR_MXFP4 2\n\n#define QI_NVFP4 (QK_NVFP4 / (4 * QR_NVFP4))\n#define QR_NVFP4 2\n\n#define QI5_0 (QK5_0 / (4 * QR5_0))\n#define QR5_0 2\n\n#define QI5_1 (QK5_1 / (4 * QR5_1))\n#define QR5_1 2\n\n#define QI8_0 (QK8_0 / (4 * QR8_0))\n#define QR8_0 1\n\n#define QI8_1 (QK8_1 / (4 * QR8_1))\n#define QR8_1 1\n\n#define QI2_K (QK_K / (4*QR2_K))\n#define QR2_K 4\n\n#define QI3_K (QK_K / (4*QR3_K))\n#define QR3_K 4\n\n#define QI4_K (QK_K / (4*QR4_K))\n#define QR4_K 2\n\n#define QI5_K (QK_K / (4*QR5_K))\n#define QR5_K 2\n\n#define QI6_K (QK_K / (4*QR6_K))\n#define QR6_K 2\n\n#define QI2_XXS (QK_K / (4*QR2_XXS))\n#define QR2_XXS 4\n\n#define QI2_XS (QK_K / (4*QR2_XS))\n#define QR2_XS 4\n\n#define QI2_S (QK_K / (4*QR2_S))\n#define QR2_S 4\n\n#define QI3_XXS (QK_K / (4*QR3_XXS))\n#define QR3_XXS 4\n\n#define QI3_XS (QK_K / (4*QR3_XS))\n#define QR3_XS 4\n\n#define QI1_S (QK_K / (4*QR1_S))\n#define QR1_S 8\n\n#define QI1_M (QK_K / (4*QR1_M))\n#define QR1_M 8\n\n#define QI4_NL (QK4_NL / (4*QR4_NL))\n#define QR4_NL 2\n\n#define QI4_XS (QK_K / (4*QR4_XS))\n#define QR4_XS 2\n\n#define QI3_S (QK_K / (4*QR3_S))\n#define QR3_S 4\n\n#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP\n\n#ifdef _MSC_VER\n#define GGML_EXTENSION\n#else // _MSC_VER\n#define GGML_EXTENSION __extension__\n#endif // _MSC_VER\n\n#define QK4_0 32\ntypedef struct {\n ggml_half d; // delta\n uint8_t qs[QK4_0 / 2]; // nibbles / quants\n} block_q4_0;\nstatic_assert(sizeof(block_q4_0) == sizeof(ggml_half) + QK4_0 / 2, \"wrong q4_0 block size/padding\");\n\n#define QK4_1 32\ntypedef struct {\n GGML_EXTENSION union {\n struct {\n ggml_half d; // delta\n ggml_half m; // min\n } GGML_COMMON_AGGR_S;\n ggml_half2 dm;\n } GGML_COMMON_AGGR_U;\n uint8_t qs[QK4_1 / 2]; // nibbles / quants\n} block_q4_1;\nstatic_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, \"wrong q4_1 block size/padding\");\n\n#define QK_MXFP4 32\ntypedef struct {\n uint8_t e; // E8M0\n uint8_t qs[QK_MXFP4/2];\n} block_mxfp4;\nstatic_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + QK_MXFP4/2, \"wrong mxfp4 block size/padding\");\n\n#define QK_NVFP4 64\n#define QK_NVFP4_SUB 16 // sub-block size for per-group scales\ntypedef struct {\n uint8_t d[QK_NVFP4/QK_NVFP4_SUB]; // UE4M3 scales (4 bytes, one per 16-element sub-block)\n uint8_t qs[QK_NVFP4/2]; // packed 4-bit E2M1 values (32 bytes)\n} block_nvfp4;\nstatic_assert(sizeof(block_nvfp4) == sizeof(uint8_t)*(QK_NVFP4/QK_NVFP4_SUB) + QK_NVFP4/2, \"wrong nvfp4 block size/padding\");\n\n#define QK5_0 32\ntypedef struct {\n ggml_half d; // delta\n uint8_t qh[4]; // 5-th bit of quants\n uint8_t qs[QK5_0 / 2]; // nibbles / quants\n} block_q5_0;\nstatic_assert(sizeof(block_q5_0) == sizeof(ggml_half) + sizeof(uint32_t) + QK5_0 / 2, \"wrong q5_0 block size/padding\");\n\n#define QK5_1 32\ntypedef struct {\n GGML_EXTENSION union {\n struct {\n ggml_half d; // delta\n ggml_half m; // min\n } GGML_COMMON_AGGR_S;\n ggml_half2 dm;\n } GGML_COMMON_AGGR_U;\n uint8_t qh[4]; // 5-th bit of quants\n uint8_t qs[QK5_1 / 2]; // nibbles / quants\n} block_q5_1;\nstatic_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_half) + sizeof(uint32_t) + QK5_1 / 2, \"wrong q5_1 block size/padding\");\n\n#define QK8_0 32\ntypedef struct {\n ggml_half d; // delta\n int8_t qs[QK8_0]; // quants\n} block_q8_0;\nstatic_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, \"wrong q8_0 block size/padding\");\n\n#define QK8_1 32\ntypedef struct {\n GGML_EXTENSION union {\n struct {\n ggml_half d; // delta\n ggml_half s; // d * sum(qs[i])\n } GGML_COMMON_AGGR_S;\n ggml_half2 ds;\n } GGML_COMMON_AGGR_U;\n int8_t qs[QK8_1]; // quants\n} block_q8_1;\nstatic_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, \"wrong q8_1 block size/padding\");\n\n//\n// Ternary quantization\n//\n\n// 1.6875 bpw\ntypedef struct {\n uint8_t qs[(QK_K - 4 * QK_K / 64) / 5]; // 5 elements per byte (3^5 = 243 < 256)\n uint8_t qh[QK_K/64]; // 4 elements per byte\n ggml_half d;\n} block_tq1_0;\nstatic_assert(sizeof(block_tq1_0) == sizeof(ggml_half) + QK_K / 64 + (QK_K - 4 * QK_K / 64) / 5, \"wrong tq1_0 block size/padding\");\n\n// 2.0625 bpw\ntypedef struct {\n uint8_t qs[QK_K/4]; // 2 bits per element\n ggml_half d;\n} block_tq2_0;\nstatic_assert(sizeof(block_tq2_0) == sizeof(ggml_half) + QK_K / 4, \"wrong tq2_0 block size/padding\");\n\n//\n// Super-block quantization structures\n//\n\n// 2-bit quantization\n// weight is represented as x = a * q + b\n// 16 blocks of 16 elements each\n// Effectively 2.625 bits per weight\ntypedef struct {\n uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits\n uint8_t qs[QK_K/4]; // quants\n GGML_EXTENSION union {\n struct {\n ggml_half d; // super-block scale for quantized scales\n ggml_half dmin; // super-block scale for quantized mins\n } GGML_COMMON_AGGR_S;\n ggml_half2 dm;\n } GGML_COMMON_AGGR_U;\n} block_q2_K;\nstatic_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, \"wrong q2_K block size/padding\");\n\n// 3-bit quantization\n// weight is represented as x = a * q\n// 16 blocks of 16 elements each\n// Effectively 3.4375 bits per weight\ntypedef struct {\n uint8_t hmask[QK_K/8]; // quants - high bit\n uint8_t qs[QK_K/4]; // quants - low 2 bits\n uint8_t scales[12]; // scales, quantized with 6 bits\n ggml_half d; // super-block scale\n} block_q3_K;\nstatic_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, \"wrong q3_K block size/padding\");\n\n// 4-bit quantization\n// 8 blocks of 32 elements each\n// weight is represented as x = a * q + b\n// Effectively 4.5 bits per weight\ntypedef struct {\n GGML_EXTENSION union {\n struct {\n ggml_half d; // super-block scale for quantized scales\n ggml_half dmin; // super-block scale for quantized mins\n } GGML_COMMON_AGGR_S;\n ggml_half2 dm;\n } GGML_COMMON_AGGR_U;\n uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits\n uint8_t qs[QK_K/2]; // 4--bit quants\n} block_q4_K;\nstatic_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, \"wrong q4_K block size/padding\");\n\n// 5-bit quantization\n// 8 blocks of 32 elements each\n// weight is represented as x = a * q + b\n// Effectively 5.5 bits per weight\ntypedef struct {\n GGML_EXTENSION union {\n struct {\n ggml_half d; // super-block scale for quantized scales\n ggml_half dmin; // super-block scale for quantized mins\n } GGML_COMMON_AGGR_S;\n ggml_half2 dm;\n } GGML_COMMON_AGGR_U;\n uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits\n uint8_t qh[QK_K/8]; // quants, high bit\n uint8_t qs[QK_K/2]; // quants, low 4 bits\n} block_q5_K;\nstatic_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, \"wrong q5_K block size/padding\");\n\n// 6-bit quantization\n// weight is represented as x = a * q\n// 16 blocks of 16 elements each\n// Effectively 6.5625 bits per weight\ntypedef struct {\n uint8_t ql[QK_K/2]; // quants, lower 4 bits\n uint8_t qh[QK_K/4]; // quants, upper 2 bits\n int8_t scales[QK_K/16]; // scales, quantized with 8 bits\n ggml_half d; // super-block scale\n} block_q6_K;\nstatic_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, \"wrong q6_K block size/padding\");\n\n// This is only used for intermediate quantization and dot products\ntypedef struct {\n float d; // delta\n int8_t qs[QK_K]; // quants\n int16_t bsums[QK_K/16]; // sum of quants in groups of 16\n} block_q8_K;\nstatic_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), \"wrong q8_K block size/padding\");\n\n// (Almost) \"true\" 2-bit quantization.\n// Due to the need to use blocks as per ggml design, it ends up using\n// 2.0625 bpw because of the 16-bit scale for each block of 256.\ntypedef struct {\n ggml_half d;\n uint16_t qs[QK_K/8];\n} block_iq2_xxs;\nstatic_assert(sizeof(block_iq2_xxs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t), \"wrong iq2_xxs block size/padding\");\n\n// 2.3125 bpw quants\ntypedef struct {\n ggml_half d;\n uint16_t qs[QK_K/8];\n uint8_t scales[QK_K/32];\n} block_iq2_xs;\nstatic_assert(sizeof(block_iq2_xs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t) + QK_K/32, \"wrong iq2_xs block size/padding\");\n\n// 2.5625 bpw quants\ntypedef struct {\n ggml_half d;\n uint8_t qs[QK_K/4];\n uint8_t qh[QK_K/32];\n uint8_t scales[QK_K/32];\n} block_iq2_s;\nstatic_assert(sizeof(block_iq2_s) == sizeof(ggml_half) + QK_K/4 + QK_K/16, \"wrong iq2_s block size/padding\");\n\n// (Almost) \"true\" 3-bit quantization.\n// Due to the need to use blocks as per ggml design, it ends up using\n// 3.0625 bpw because of the 16-bit scale for each block of 256.\ntypedef struct {\n ggml_half d;\n uint8_t qs[3*QK_K/8];\n} block_iq3_xxs;\nstatic_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), \"wrong iq3_xxs block size/padding\");\n\n// 3.4375 bpw\n#define IQ3S_N_SCALE QK_K/64\ntypedef struct {\n ggml_half d;\n uint8_t qs[QK_K/4];\n uint8_t qh[QK_K/32];\n uint8_t signs[QK_K/8];\n uint8_t scales[IQ3S_N_SCALE];\n} block_iq3_s;\nstatic_assert(sizeof(block_iq3_s) == sizeof(ggml_half) + 13*(QK_K/32) + IQ3S_N_SCALE, \"wrong iq3_s block size/padding\");\n\n// 1.5625 bpw\ntypedef struct {\n ggml_half d;\n uint8_t qs[QK_K/8];\n uint16_t qh[QK_K/32];\n} block_iq1_s;\nstatic_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, \"wrong iq1_s block size/padding\");\n\n// 1.75 bpw\ntypedef struct {\n uint8_t qs[QK_K/8]; // grid index, low 8 bits\n uint8_t qh[QK_K/16]; // grid index, high 3 bits + grid shift bit (for two groups of 8)\n uint8_t scales[QK_K/32]; // 3-bit block scales (4-bit if QK_K == 64)\n} block_iq1_m;\nstatic_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32, \"wrong iq1_m block size/padding\");\n\n// Used by IQ1_M quants\ntypedef union {\n ggml_half f16;\n uint16_t u16;\n} iq1m_scale_t;\n\n// Non-linear quants\n#define QK4_NL 32\ntypedef struct {\n ggml_half d;\n uint8_t qs[QK4_NL/2];\n} block_iq4_nl;\nstatic_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, \"wrong iq4_nl block size/padding\");\n\ntypedef struct {\n ggml_half d;\n uint16_t scales_h;\n uint8_t scales_l[QK_K/64];\n uint8_t qs[QK_K/2];\n} block_iq4_xs;\nstatic_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, \"wrong iq4_xs block size/padding\");\n\n#endif // GGML_COMMON_DECL\n#endif // GGML_COMMON_DECL\n\n////////////////////////////////////////////////////////////////////////////////\n\n#ifndef GGML_COMMON_IMPL\n\n#if defined(GGML_COMMON_IMPL_C)\n#include \n\n#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#elif defined(GGML_COMMON_IMPL_CPP)\n#include \n\n#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#elif defined(GGML_COMMON_IMPL_METAL)\n#include \n\n#define GGML_TABLE_BEGIN(type, name, size) static const constant type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#elif defined(GGML_COMMON_IMPL_CUDA) || defined(GGML_COMMON_IMPL_HIP) || defined(GGML_COMMON_IMPL_MUSA)\n#include \n\n#define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#elif defined(GGML_COMMON_IMPL_SYCL)\n\n#include \n\n#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#endif\n\n#if defined(GGML_COMMON_IMPL)\n\nGGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8)\n 1, 2, 4, 8, 16, 32, 64, 128\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)\n 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,\n 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,\n 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,\n 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,\n 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,\n 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,\n 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,\n 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint64_t, ksigns64, 128)\n 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,\n 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,\n 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,\n 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,\n 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,\n 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,\n 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,\n 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,\n 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,\n 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,\n 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,\n 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,\n 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,\n 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,\n 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,\n 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,\n 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,\n 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,\n 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,\n 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,\n 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,\n 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,\n 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,\n 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,\n 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,\n 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,\n 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,\n 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,\n 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,\n 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,\n 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,\n 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,\nGGML_TABLE_END()\n\n\nGGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256)\n 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,\n 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,\n 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,\n 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,\n 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,\n 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,\n 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,\n 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,\n 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,\n 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,\n 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,\n 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,\n 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,\n 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,\n 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,\n 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,\n 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,\n 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,\n 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,\n 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,\n 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,\n 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,\n 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,\n 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,\n 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,\n 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,\n 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,\n 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,\n 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,\n 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,\n 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,\n 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,\n 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,\n 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,\n 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,\n 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,\n 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,\n 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,\n 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,\n 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,\n 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,\n 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,\n 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,\n 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,\n 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,\n 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,\n 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,\n 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,\n 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,\n 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,\n 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,\n 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,\n 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,\n 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,\n 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,\n 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,\n 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,\n 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,\n 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,\n 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,\n 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,\n 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,\n 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,\n 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint64_t, iq2xs_grid, 512)\n 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,\n 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,\n 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,\n 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,\n 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,\n 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,\n 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,\n 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,\n 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,\n 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,\n 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,\n 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,\n 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,\n 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,\n 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,\n 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,\n 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,\n 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,\n 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,\n 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,\n 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,\n 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,\n 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,\n 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,\n 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,\n 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,\n 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,\n 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,\n 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,\n 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,\n 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,\n 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,\n 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,\n 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,\n 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,\n 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,\n 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,\n 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,\n 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,\n 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,\n 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,\n 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,\n 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,\n 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,\n 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,\n 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,\n 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,\n 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,\n 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,\n 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,\n 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,\n 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,\n 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,\n 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,\n 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,\n 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,\n 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,\n 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,\n 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,\n 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,\n 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,\n 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,\n 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,\n 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,\n 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,\n 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,\n 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,\n 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,\n 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,\n 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,\n 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,\n 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,\n 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,\n 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,\n 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,\n 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,\n 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,\n 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,\n 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,\n 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,\n 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,\n 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,\n 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,\n 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,\n 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,\n 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,\n 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,\n 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,\n 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,\n 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,\n 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,\n 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,\n 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,\n 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,\n 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,\n 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,\n 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,\n 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,\n 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,\n 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,\n 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,\n 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,\n 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,\n 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,\n 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,\n 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,\n 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,\n 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,\n 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,\n 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,\n 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,\n 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,\n 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,\n 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,\n 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,\n 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,\n 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,\n 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,\n 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,\n 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,\n 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,\n 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,\n 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,\n 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,\n 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,\n 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,\n 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,\n 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024)\n 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,\n 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,\n 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,\n 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,\n 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,\n 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,\n 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,\n 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,\n 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,\n 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,\n 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,\n 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,\n 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,\n 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,\n 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,\n 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,\n 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,\n 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,\n 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,\n 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,\n 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,\n 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,\n 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,\n 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,\n 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,\n 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,\n 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,\n 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,\n 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,\n 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,\n 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,\n 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,\n 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,\n 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,\n 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,\n 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,\n 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,\n 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,\n 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,\n 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,\n 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,\n 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,\n 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,\n 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,\n 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,\n 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,\n 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,\n 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,\n 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,\n 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,\n 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,\n 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,\n 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,\n 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,\n 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,\n 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,\n 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,\n 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,\n 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,\n 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,\n 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,\n 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,\n 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,\n 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,\n 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,\n 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,\n 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,\n 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,\n 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,\n 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,\n 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,\n 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,\n 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,\n 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,\n 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,\n 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,\n 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,\n 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,\n 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,\n 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,\n 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,\n 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,\n 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,\n 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,\n 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,\n 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,\n 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,\n 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,\n 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,\n 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,\n 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,\n 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,\n 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,\n 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,\n 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,\n 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,\n 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,\n 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,\n 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,\n 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,\n 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,\n 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,\n 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,\n 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,\n 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,\n 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,\n 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,\n 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,\n 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,\n 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,\n 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,\n 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,\n 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,\n 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,\n 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,\n 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,\n 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,\n 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,\n 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,\n 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,\n 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,\n 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,\n 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,\n 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,\n 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,\n 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,\n 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,\n 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,\n 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,\n 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,\n 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,\n 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,\n 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,\n 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,\n 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,\n 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,\n 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,\n 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,\n 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,\n 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,\n 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,\n 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,\n 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,\n 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,\n 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,\n 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,\n 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,\n 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,\n 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,\n 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,\n 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,\n 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,\n 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,\n 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,\n 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,\n 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,\n 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,\n 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,\n 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,\n 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,\n 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,\n 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,\n 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,\n 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,\n 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,\n 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,\n 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,\n 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,\n 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,\n 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,\n 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,\n 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,\n 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,\n 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,\n 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,\n 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,\n 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,\n 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,\n 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,\n 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,\n 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,\n 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,\n 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,\n 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,\n 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,\n 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,\n 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,\n 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,\n 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,\n 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,\n 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,\n 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,\n 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,\n 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,\n 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,\n 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,\n 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,\n 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,\n 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,\n 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,\n 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,\n 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,\n 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,\n 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,\n 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,\n 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,\n 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,\n 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,\n 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,\n 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,\n 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,\n 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,\n 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,\n 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,\n 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,\n 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,\n 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,\n 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,\n 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,\n 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,\n 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,\n 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,\n 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,\n 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,\n 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,\n 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,\n 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,\n 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,\n 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,\n 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,\n 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,\n 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,\n 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,\n 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,\n 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,\n 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,\n 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,\n 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,\n 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,\n 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,\n 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,\n 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,\n 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,\n 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,\n 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,\n 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,\n 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,\n 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,\n 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,\n 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,\n 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,\n 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,\n 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,\n 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,\n 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,\n 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint32_t, iq3xxs_grid, 256)\n 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,\n 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,\n 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,\n 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,\n 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,\n 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,\n 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,\n 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,\n 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,\n 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,\n 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,\n 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,\n 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,\n 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,\n 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,\n 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,\n 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,\n 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,\n 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,\n 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,\n 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,\n 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,\n 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,\n 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,\n 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,\n 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,\n 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,\n 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,\n 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,\n 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,\n 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,\n 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512)\n 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305,\n 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905,\n 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09,\n 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b,\n 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b,\n 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d,\n 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03,\n 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505,\n 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03,\n 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901,\n 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d,\n 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303,\n 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501,\n 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105,\n 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505,\n 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101,\n 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707,\n 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b,\n 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01,\n 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f,\n 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305,\n 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103,\n 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509,\n 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503,\n 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b,\n 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f,\n 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f,\n 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f,\n 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109,\n 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f,\n 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509,\n 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501,\n 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303,\n 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f,\n 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907,\n 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703,\n 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03,\n 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01,\n 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01,\n 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903,\n 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505,\n 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b,\n 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107,\n 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509,\n 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303,\n 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103,\n 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05,\n 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b,\n 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f,\n 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701,\n 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909,\n 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305,\n 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d,\n 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b,\n 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d,\n 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307,\n 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09,\n 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309,\n 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709,\n 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f,\n 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303,\n 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503,\n 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b,\n 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101,\nGGML_TABLE_END()\n\n// TODO: fix name to kvalues_iq4_nl\nGGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16)\n -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113,\nGGML_TABLE_END()\n\n// e2m1 values (doubled)\n// ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf\nGGML_TABLE_BEGIN(int8_t, kvalues_mxfp4, 16)\n 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12,\nGGML_TABLE_END()\n\n#define NGRID_IQ1S 2048\n#define IQ1S_DELTA 0.125f\n#define IQ1M_DELTA 0.125f\n#if defined(GGML_COMMON_IMPL_C)\nGGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)\n 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,\n 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,\n 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000,\n 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000,\n 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101,\n 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff,\n 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00,\n 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101,\n 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100,\n 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00,\n 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000,\n 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000,\n 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101,\n 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff,\n 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100,\n 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01,\n 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000,\n 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff,\n 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001,\n 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000,\n 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00,\n 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff,\n 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff,\n 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000,\n 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff,\n 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff,\n 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101,\n 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff,\n 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000,\n 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100,\n 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01,\n 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00,\n 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00,\n 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01,\n 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff,\n 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000,\n 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff,\n 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000,\n 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101,\n 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100,\n 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff,\n 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff,\n 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001,\n 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01,\n 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff,\n 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000,\n 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000,\n 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff,\n 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01,\n 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00,\n 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000,\n 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000,\n 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000,\n 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001,\n 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff,\n 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01,\n 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff,\n 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff,\n 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000,\n 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000,\n 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00,\n 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001,\n 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100,\n 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101,\n 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00,\n 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00,\n 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000,\n 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001,\n 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff,\n 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000,\n 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000,\n 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff,\n 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100,\n 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000,\n 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101,\n 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001,\n 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000,\n 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff,\n 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01,\n 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100,\n 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000,\n 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01,\n 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101,\n 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000,\n 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff,\n 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff,\n 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001,\n 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001,\n 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000,\n 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000,\n 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00,\n 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100,\n 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01,\n 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00,\n 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff,\n 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000,\n 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00,\n 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000,\n 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff,\n 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00,\n 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000,\n 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000,\n 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff,\n 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00,\n 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00,\n 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001,\n 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001,\n 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000,\n 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100,\n 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101,\n 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101,\n 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000,\n 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00,\n 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff,\n 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000,\n 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff,\n 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff,\n 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff,\n 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101,\n 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001,\n 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100,\n 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101,\n 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01,\n 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00,\n 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00,\n 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000,\n 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101,\n 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100,\n 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001,\n 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff,\n 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00,\n 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000,\n 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100,\n 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00,\n 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01,\n 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff,\n 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101,\n 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100,\n 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100,\n 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000,\n 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff,\n 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff,\n 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000,\n 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101,\n 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000,\n 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101,\n 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101,\n 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100,\n 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01,\n 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001,\n 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001,\n 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff,\n 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff,\n 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000,\n 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000,\n 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101,\n 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff,\n 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001,\n 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000,\n 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff,\n 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00,\n 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff,\n 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000,\n 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00,\n 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff,\n 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000,\n 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000,\n 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff,\n 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000,\n 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000,\n 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000,\n 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101,\n 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff,\n 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100,\n 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101,\n 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001,\n 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff,\n 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100,\n 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff,\n 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01,\n 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff,\n 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff,\n 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff,\n 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff,\n 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001,\n 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff,\n 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01,\n 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00,\n 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100,\n 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101,\n 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff,\n 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00,\n 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01,\n 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00,\n 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100,\n 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00,\n 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000,\n 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000,\n 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000,\n 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000,\n 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001,\n 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff,\n 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00,\n 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff,\n 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00,\n 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000,\n 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff,\n 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff,\n 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101,\n 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000,\n 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff,\n 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff,\n 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff,\n 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff,\n 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000,\n 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000,\n 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100,\n 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00,\n 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100,\n 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100,\n 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00,\n 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff,\n 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff,\n 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100,\n 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff,\n 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000,\n 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00,\n 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,\n 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00,\n 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100,\n 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff,\n 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff,\n 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001,\n 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00,\n 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101,\n 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001,\n 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000,\n 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100,\n 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000,\n 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001,\n 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000,\n 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff,\n 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101,\n 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,\n 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101,\n 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000,\n 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff,\n 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000,\n 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff,\n 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01,\n 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100,\n 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff,\n 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101,\n 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001,\n 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01,\n 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff,\n 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,\n 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff,\n 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00,\n 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff,\n 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff,\n 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff,\n 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001,\n 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff,\n 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff,\n 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001,\n 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff,\n 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff,\n 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000,\n 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00,\n 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100,\n 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01,\n 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff,\n 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff,\n 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000,\n 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101,\n 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01,\n 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100,\n 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100,\n 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00,\n 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101,\n 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001,\n 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01,\n 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100,\n 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff,\n 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01,\n 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff,\n 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000,\n 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001,\n 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01,\n 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff,\n 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff,\n 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00,\n 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff,\n 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100,\n 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000,\n 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001,\n 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100,\n 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff,\n 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff,\n 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01,\n 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101,\n 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001,\n 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff,\n 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101,\n 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100,\n 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00,\n 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01,\n 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001,\n 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00,\n 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000,\n 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101,\n 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001,\n 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100,\n 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000,\n 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000,\n 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00,\n 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101,\n 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff,\n 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101,\n 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001,\n 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01,\n 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100,\n 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000,\n 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00,\n 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff,\n 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001,\n 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001,\n 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff,\n 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00,\n 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100,\n 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00,\n 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101,\n 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff,\n 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff,\n 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff,\n 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00,\n 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff,\n 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff,\n 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101,\n 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100,\n 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101,\n 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00,\n 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00,\n 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff,\n 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff,\n 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001,\n 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000,\n 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff,\n 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff,\n 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff,\n 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff,\n 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001,\n 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100,\n 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101,\n 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001,\n 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001,\n 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101,\n 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101,\n 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff,\n 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff,\n 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000,\n 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101,\n 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001,\n 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff,\n 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000,\n 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff,\n 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100,\n 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000,\n 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101,\n 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff,\n 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100,\n 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff,\n 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01,\n 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100,\n 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000,\n 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100,\n 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100,\n 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001,\n 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000,\n 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000,\n 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000,\n 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00,\n 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff,\n 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001,\n 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000,\n 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101,\n 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101,\n 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000,\n 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff,\n 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000,\n 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101,\n 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff,\n 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff,\n 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000,\n 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101,\n 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff,\n 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff,\n 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01,\n 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff,\n 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000,\n 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101,\n 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff,\n 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff,\n 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff,\n 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01,\n 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00,\n 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000,\n 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000,\n 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101,\n 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001,\n 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff,\n 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000,\n 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff,\n 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000,\n 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000,\n 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000,\n 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000,\n 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001,\n 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff,\n 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001,\n 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000,\n 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000,\n 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00,\n 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000,\n 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101,\n 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001,\n 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00,\n 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001,\n 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff,\n 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000,\n 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00,\n 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100,\n 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101,\n 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001,\n 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01,\n 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff,\n 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff,\n 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00,\n 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01,\n 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100,\n 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000,\n 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff,\n 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff,\n 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff,\n 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000,\n 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff,\n 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101,\n 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff,\n 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001,\n 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001,\n 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001,\n 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000,\n 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000,\n 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff,\n 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101,\n 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001,\n 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff,\n 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000,\n 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000,\n 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff,\n 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101,\n 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000,\n 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100,\n 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00,\n 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000,\n 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff,\n 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01,\n 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00,\n 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff,\n 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000,\n 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101,\n 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff,\n 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001,\n 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff,\n 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000,\n 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100,\n 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff,\n 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101,\n 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100,\n 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff,\n 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01,\n 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000,\n 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff,\n 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00,\n 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100,\n 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff,\n 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001,\n 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00,\n 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100,\n 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff,\n 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01,\n 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00,\n 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000,\n 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01,\n 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff,\n 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000,\n 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff,\n 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff,\n 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff,\n 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001,\n 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff,\n 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01,\n 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff,\n 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00,\n 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00,\n 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001,\n 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101,\n 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101,\n 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff,\n 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000,\n 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101,\nGGML_TABLE_END()\n#else\nGGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S)\n 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,\n 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,\n 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,\n 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,\n 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,\n 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,\n 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,\n 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,\n 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,\n 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,\n 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,\n 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,\n 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,\n 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,\n 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,\n 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,\n 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,\n 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,\n 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,\n 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,\n 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,\n 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,\n 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,\n 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,\n 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,\n 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,\n 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,\n 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,\n 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,\n 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,\n 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,\n 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,\n 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,\n 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,\n 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,\n 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,\n 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,\n 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,\n 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,\n 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,\n 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,\n 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,\n 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,\n 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,\n 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,\n 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,\n 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,\n 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,\n 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,\n 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,\n 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,\n 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,\n 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,\n 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,\n 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,\n 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,\n 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,\n 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,\n 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,\n 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,\n 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,\n 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,\n 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,\n 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,\n 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,\n 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,\n 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,\n 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,\n 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,\n 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,\n 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,\n 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,\n 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,\n 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,\n 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,\n 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,\n 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,\n 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,\n 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,\n 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,\n 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,\n 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,\n 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,\n 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,\n 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,\n 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,\n 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,\n 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,\n 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,\n 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,\n 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,\n 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,\n 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,\n 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,\n 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,\n 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,\n 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,\n 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,\n 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,\n 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,\n 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,\n 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,\n 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,\n 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,\n 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,\n 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,\n 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,\n 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,\n 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,\n 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,\n 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,\n 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,\n 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,\n 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,\n 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,\n 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,\n 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,\n 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,\n 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,\n 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,\n 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,\n 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,\n 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,\n 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,\n 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,\n 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,\n 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,\n 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,\n 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,\n 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,\n 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,\n 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,\n 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,\n 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,\n 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,\n 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,\n 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,\n 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,\n 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,\n 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,\n 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,\n 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,\n 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,\n 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,\n 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,\n 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,\n 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,\n 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,\n 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,\n 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,\n 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,\n 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,\n 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,\n 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,\n 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,\n 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,\n 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,\n 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,\n 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,\n 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,\n 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,\n 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,\n 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,\n 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,\n 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,\n 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,\n 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,\n 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,\n 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,\n 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,\n 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,\n 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,\n 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,\n 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,\n 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,\n 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,\n 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,\n 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,\n 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,\n 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,\n 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,\n 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,\n 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,\n 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,\n 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,\n 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,\n 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,\n 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,\n 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,\n 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,\n 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,\n 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,\n 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,\n 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,\n 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,\n 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,\n 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,\n 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,\n 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,\n 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,\n 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,\n 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,\n 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,\n 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,\n 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,\n 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,\n 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,\n 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,\n 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,\n 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,\n 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,\n 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,\n 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,\n 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,\n 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,\n 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,\n 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,\n 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,\n 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,\n 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,\n 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,\n 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,\n 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,\n 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,\n 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,\n 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,\n 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,\n 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,\n 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,\n 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,\n 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,\n 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,\n 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,\n 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,\n 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,\n 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,\n 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,\n 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,\n 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,\n 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,\n 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,\n 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,\n 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,\n 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,\n 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,\n 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,\n 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,\n 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,\n 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,\n 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,\n 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,\n 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,\n 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,\n 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,\n 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,\n 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,\nGGML_TABLE_END()\n#endif\n\n#endif // GGML_COMMON_IMPL\n#endif // GGML_COMMON_IMPL\n" }, { "path": "src/ggml-cpu/CMakeLists.txt", "content": "function(ggml_add_cpu_backend_features cpu_name arch)\n # The feature detection code is compiled as a separate target so that\n # it can be built without the architecture flags\n # Since multiple variants of the CPU backend may be included in the same\n # build, using set_source_files_properties() to set the arch flags is not possible\n set(GGML_CPU_FEATS_NAME ${cpu_name}-feats)\n add_library(${GGML_CPU_FEATS_NAME} OBJECT ggml-cpu/arch/${arch}/cpu-feats.cpp)\n target_include_directories(${GGML_CPU_FEATS_NAME} PRIVATE . ../include)\n target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE ${ARGN})\n target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE GGML_BACKEND_DL GGML_BACKEND_BUILD GGML_BACKEND_SHARED)\n set_target_properties(${GGML_CPU_FEATS_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)\n # Disable LTO for the feature detection code to prevent cross-module optimization\n # from inlining architecture-specific instructions into the score function.\n # Without this, LTO can cause SIGILL when loading backends on older CPUs\n # (e.g., loading power10 backend on power9 crashes before feature check runs).\n target_compile_options(${GGML_CPU_FEATS_NAME} PRIVATE -fno-lto)\n target_link_libraries(${cpu_name} PRIVATE ${GGML_CPU_FEATS_NAME})\nendfunction()\n\nfunction(ggml_add_cpu_backend_variant_impl tag_name)\n if (tag_name)\n set(GGML_CPU_NAME ggml-cpu-${tag_name})\n else()\n set(GGML_CPU_NAME ggml-cpu)\n endif()\n\n ggml_add_backend_library(${GGML_CPU_NAME})\n\n list (APPEND GGML_CPU_SOURCES\n ggml-cpu/ggml-cpu.c\n ggml-cpu/ggml-cpu.cpp\n ggml-cpu/repack.cpp\n ggml-cpu/repack.h\n ggml-cpu/hbm.cpp\n ggml-cpu/hbm.h\n ggml-cpu/quants.c\n ggml-cpu/quants.h\n ggml-cpu/traits.cpp\n ggml-cpu/traits.h\n ggml-cpu/amx/amx.cpp\n ggml-cpu/amx/amx.h\n ggml-cpu/amx/mmq.cpp\n ggml-cpu/amx/mmq.h\n ggml-cpu/ggml-cpu-impl.h\n ggml-cpu/common.h\n ggml-cpu/binary-ops.h\n ggml-cpu/binary-ops.cpp\n ggml-cpu/unary-ops.h\n ggml-cpu/unary-ops.cpp\n ggml-cpu/simd-mappings.h\n ggml-cpu/vec.h\n ggml-cpu/vec.cpp\n ggml-cpu/ops.h\n ggml-cpu/ops.cpp\n )\n\n target_compile_features(${GGML_CPU_NAME} PRIVATE c_std_11 cxx_std_17)\n target_include_directories(${GGML_CPU_NAME} PRIVATE . ggml-cpu)\n\n if (APPLE AND GGML_ACCELERATE)\n find_library(ACCELERATE_FRAMEWORK Accelerate)\n if (ACCELERATE_FRAMEWORK)\n message(STATUS \"Accelerate framework found\")\n\n target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_ACCELERATE)\n target_compile_definitions(${GGML_CPU_NAME} PRIVATE ACCELERATE_NEW_LAPACK)\n target_compile_definitions(${GGML_CPU_NAME} PRIVATE ACCELERATE_LAPACK_ILP64)\n\n target_link_libraries(${GGML_CPU_NAME} PRIVATE ${ACCELERATE_FRAMEWORK})\n else()\n message(WARNING \"Accelerate framework not found\")\n endif()\n endif()\n\n if (GGML_OPENMP)\n find_package(OpenMP)\n if (OpenMP_FOUND)\n set(GGML_OPENMP_ENABLED \"ON\" CACHE INTERNAL \"\")\n target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP)\n\n target_link_libraries(${GGML_CPU_NAME} PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX)\n else()\n set(GGML_OPENMP_ENABLED \"OFF\" CACHE INTERNAL \"\")\n message(WARNING \"OpenMP not found\")\n endif()\n endif()\n\n if (GGML_LLAMAFILE)\n target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_LLAMAFILE)\n\n list(APPEND GGML_CPU_SOURCES\n ggml-cpu/llamafile/sgemm.cpp\n ggml-cpu/llamafile/sgemm.h)\n endif()\n\n if (GGML_CPU_HBM)\n find_library(memkind memkind REQUIRED)\n\n message(STATUS \"Using memkind for CPU HBM\")\n\n target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_HBM)\n\n target_link_libraries(${GGML_CPU_NAME} PUBLIC memkind)\n endif()\n\n if (GGML_SYSTEM_ARCH STREQUAL \"ARM\")\n message(STATUS \"ARM detected\")\n list(APPEND GGML_CPU_SOURCES\n ggml-cpu/arch/arm/quants.c\n ggml-cpu/arch/arm/repack.cpp\n )\n\n if (MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL \"Clang\")\n message(FATAL_ERROR \"MSVC is not supported for ARM, use clang\")\n else()\n check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)\n if (NOT \"${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}\" STREQUAL \"\")\n list(APPEND ARCH_FLAGS -mfp16-format=ieee)\n endif()\n\n if (GGML_NATIVE)\n # -mcpu=native does not always enable all the features in some compilers,\n # so we check for them manually and enable them if available\n\n execute_process(\n COMMAND ${CMAKE_C_COMPILER} -mcpu=native -E -v -\n INPUT_FILE \"/dev/null\"\n OUTPUT_QUIET\n ERROR_VARIABLE ARM_MCPU\n RESULT_VARIABLE ARM_MCPU_RESULT\n )\n if (NOT ARM_MCPU_RESULT)\n string(REGEX MATCH \"-mcpu=[^ ']+\" ARM_MCPU_FLAG \"${ARM_MCPU}\")\n string(REGEX MATCH \"-march=[^ ']+\" ARM_MARCH_FLAG \"${ARM_MCPU}\")\n\n # on some old GCC we need to read -march=\n if (ARM_MARCH_FLAG AND NOT \"${ARM_MARCH_FLAG}\" STREQUAL \"-march=native\")\n set(ARM_NATIVE_FLAG \"${ARM_MARCH_FLAG}\")\n elseif(ARM_MCPU_FLAG AND NOT \"${ARM_MCPU_FLAG}\" STREQUAL \"-mcpu=native\")\n set(ARM_NATIVE_FLAG \"${ARM_MCPU_FLAG}\")\n endif()\n endif()\n\n if (\"${ARM_NATIVE_FLAG}\" STREQUAL \"\")\n set(ARM_NATIVE_FLAG -mcpu=native)\n message(WARNING \"ARM -march/-mcpu not found, -mcpu=native will be used\")\n else()\n message(STATUS \"ARM detected flags: ${ARM_NATIVE_FLAG}\")\n endif()\n\n include(CheckCXXSourceRuns)\n\n macro(check_arm_feature tag feature code)\n set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})\n set(CMAKE_REQUIRED_FLAGS \"${ARM_NATIVE_FLAG}+${tag}\")\n check_cxx_source_runs(\"${code}\" GGML_MACHINE_SUPPORTS_${tag})\n if (GGML_MACHINE_SUPPORTS_${tag})\n set(ARM_NATIVE_FLAG_FIX \"${ARM_NATIVE_FLAG_FIX}+${tag}\")\n else()\n set(CMAKE_REQUIRED_FLAGS \"${ARM_NATIVE_FLAG}+no${tag}\")\n check_cxx_source_compiles(\"int main() { return 0; }\" GGML_MACHINE_SUPPORTS_no${tag})\n if (GGML_MACHINE_SUPPORTS_no${tag})\n set(ARM_NATIVE_FLAG_FIX \"${ARM_NATIVE_FLAG_FIX}+no${tag}\")\n list(APPEND ARCH_FLAGS -U__ARM_FEATURE_${feature})\n endif()\n endif()\n set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})\n endmacro()\n\n check_arm_feature(dotprod DOTPROD \"#include \\nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }\")\n check_arm_feature(i8mm MATMUL_INT8 \"#include \\nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vmmlaq_s32(_s, _a, _b); return 0; }\")\n check_arm_feature(sve SVE \"#include \\nint main() { svfloat32_t _a, _b; volatile svfloat32_t _c = svadd_f32_z(svptrue_b8(), _a, _b); return 0; }\")\n check_arm_feature(sme SME \"#include \\n__arm_locally_streaming int main() { __asm__ volatile(\\\"smstart; smstop;\\\"); return 0; }\")\n\n list(APPEND ARCH_FLAGS \"${ARM_NATIVE_FLAG}${ARM_NATIVE_FLAG_FIX}\")\n else()\n if (GGML_CPU_ARM_ARCH)\n list(APPEND ARCH_FLAGS -march=${GGML_CPU_ARM_ARCH})\n elseif(GGML_CPU_ALL_VARIANTS)\n # Begin with the lowest baseline\n set(ARM_MCPU \"armv8-a\")\n set(ARCH_TAGS \"\")\n set(ARCH_DEFINITIONS \"\")\n\n # When a feature is selected, bump the MCPU to the first\n # version that supported it\n if (GGML_INTERNAL_DOTPROD)\n set(ARM_MCPU \"armv8.2-a\")\n set(ARCH_TAGS \"${ARCH_TAGS}+dotprod\")\n list(APPEND ARCH_DEFINITIONS GGML_USE_DOTPROD)\n endif()\n if (GGML_INTERNAL_FP16_VECTOR_ARITHMETIC)\n set(ARM_MCPU \"armv8.2-a\")\n set(ARCH_TAGS \"${ARCH_TAGS}+fp16\")\n list(APPEND ARCH_DEFINITIONS GGML_USE_FP16_VECTOR_ARITHMETIC)\n endif()\n if (GGML_INTERNAL_SVE)\n set(ARM_MCPU \"armv8.2-a\")\n set(ARCH_TAGS \"${ARCH_TAGS}+sve\")\n list(APPEND ARCH_DEFINITIONS GGML_USE_SVE)\n endif()\n if (GGML_INTERNAL_MATMUL_INT8)\n set(ARM_MCPU \"armv8.6-a\")\n set(ARCH_TAGS \"${ARCH_TAGS}+i8mm\")\n list(APPEND ARCH_DEFINITIONS GGML_USE_MATMUL_INT8)\n endif()\n if (GGML_INTERNAL_SVE2)\n set(ARM_MCPU \"armv8.6-a\")\n set(ARCH_TAGS \"${ARCH_TAGS}+sve2\")\n list(APPEND ARCH_DEFINITIONS GGML_USE_SVE2)\n endif()\n if (GGML_INTERNAL_NOSVE)\n set(ARCH_TAGS \"${ARCH_TAGS}+nosve\")\n endif()\n if (GGML_INTERNAL_SME)\n set(ARM_MCPU \"armv9.2-a\")\n set(ARCH_TAGS \"${ARCH_TAGS}+sme\")\n list(APPEND ARCH_DEFINITIONS GGML_USE_SME)\n endif()\n list(APPEND ARCH_FLAGS \"-march=${ARM_MCPU}${ARCH_TAGS}\")\n ggml_add_cpu_backend_features(${GGML_CPU_NAME} arm ${ARCH_DEFINITIONS})\n endif()\n endif()\n\n message(STATUS \"Checking for ARM features using flags:\")\n foreach(flag IN LISTS ARCH_FLAGS)\n message(STATUS \" ${flag}\")\n endforeach()\n\n include(CheckCXXSourceCompiles)\n set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})\n string(REPLACE \";\" \" \" ARCH_FLAGS_STR \"${ARCH_FLAGS}\")\n set(CMAKE_REQUIRED_FLAGS \"${ARCH_FLAGS_STR}\")\n foreach(feature DOTPROD SVE MATMUL_INT8 FMA FP16_VECTOR_ARITHMETIC SME)\n set(ARM_FEATURE \"HAVE_${feature}\")\n check_cxx_source_compiles(\n \"\n #if !defined(__ARM_FEATURE_${feature})\n # error \\\"Feature ${feature} is not defined\\\"\n #endif\n int main() { return 0; }\n \"\n ${ARM_FEATURE}\n )\n endforeach()\n set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})\n endif()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"x86\")\n message(STATUS \"x86 detected\")\n list(APPEND GGML_CPU_SOURCES\n ggml-cpu/arch/x86/quants.c\n ggml-cpu/arch/x86/repack.cpp\n )\n\n if (MSVC)\n # instruction set detection for MSVC only\n if (GGML_NATIVE)\n include(ggml-cpu/cmake/FindSIMD.cmake)\n endif ()\n if (GGML_AVX512)\n list(APPEND ARCH_FLAGS /arch:AVX512)\n # /arch:AVX512 includes: __AVX512F__, __AVX512CD__, __AVX512BW__, __AVX512DQ__, and __AVX512VL__\n # MSVC has no compile-time flags enabling specific\n # AVX512 extensions, neither it defines the\n # macros corresponding to the extensions.\n # Do it manually.\n list(APPEND ARCH_DEFINITIONS GGML_AVX512)\n if (GGML_AVX512_VBMI)\n list(APPEND ARCH_DEFINITIONS __AVX512VBMI__)\n if (CMAKE_C_COMPILER_ID STREQUAL \"Clang\")\n list(APPEND ARCH_FLAGS -mavx512vbmi)\n endif()\n endif()\n if (GGML_AVX512_VNNI)\n list(APPEND ARCH_DEFINITIONS __AVX512VNNI__ GGML_AVX512_VNNI)\n if (CMAKE_C_COMPILER_ID STREQUAL \"Clang\")\n list(APPEND ARCH_FLAGS -mavx512vnni)\n endif()\n endif()\n if (GGML_AVX512_BF16)\n list(APPEND ARCH_DEFINITIONS __AVX512BF16__ GGML_AVX512_BF16)\n if (CMAKE_C_COMPILER_ID STREQUAL \"Clang\")\n list(APPEND ARCH_FLAGS -mavx512bf16)\n endif()\n endif()\n if (GGML_AMX_TILE)\n list(APPEND ARCH_DEFINITIONS __AMX_TILE__ GGML_AMX_TILE)\n endif()\n if (GGML_AMX_INT8)\n list(APPEND ARCH_DEFINITIONS __AMX_INT8__ GGML_AMX_INT8)\n endif()\n if (GGML_AMX_BF16)\n list(APPEND ARCH_DEFINITIONS __AMX_BF16__ GGML_AMX_BF16)\n endif()\n elseif (GGML_AVX2)\n list(APPEND ARCH_FLAGS /arch:AVX2)\n list(APPEND ARCH_DEFINITIONS GGML_AVX2 GGML_FMA GGML_F16C)\n elseif (GGML_AVX)\n list(APPEND ARCH_FLAGS /arch:AVX)\n list(APPEND ARCH_DEFINITIONS GGML_AVX)\n elseif (GGML_SSE42)\n list(APPEND ARCH_FLAGS /arch:SSE4.2)\n list(APPEND ARCH_DEFINITIONS GGML_SSE42)\n endif()\n if (GGML_AVX_VNNI)\n list(APPEND ARCH_DEFINITIONS __AVXVNNI__ GGML_AVX_VNNI)\n endif()\n if (GGML_BMI2)\n # MSVC does not define macro __BMI2__\n list(APPEND ARCH_DEFINITIONS __BMI2__ GGML_BMI2)\n endif()\n else ()\n if (GGML_NATIVE)\n list(APPEND ARCH_FLAGS -march=native)\n else ()\n if (GGML_SSE42)\n list(APPEND ARCH_FLAGS -msse4.2)\n list(APPEND ARCH_DEFINITIONS GGML_SSE42)\n endif()\n if (GGML_F16C)\n list(APPEND ARCH_FLAGS -mf16c)\n list(APPEND ARCH_DEFINITIONS GGML_F16C)\n endif()\n if (GGML_FMA)\n list(APPEND ARCH_FLAGS -mfma)\n list(APPEND ARCH_DEFINITIONS GGML_FMA)\n endif()\n if (GGML_BMI2)\n list(APPEND ARCH_FLAGS -mbmi2)\n list(APPEND ARCH_DEFINITIONS GGML_BMI2)\n endif()\n if (GGML_AVX)\n list(APPEND ARCH_FLAGS -mavx)\n list(APPEND ARCH_DEFINITIONS GGML_AVX)\n endif()\n if (GGML_AVX2)\n list(APPEND ARCH_FLAGS -mavx2)\n list(APPEND ARCH_DEFINITIONS GGML_AVX2)\n endif()\n if (GGML_AVX_VNNI)\n list(APPEND ARCH_FLAGS -mavxvnni)\n list(APPEND ARCH_DEFINITIONS GGML_AVX_VNNI)\n endif()\n if (GGML_AVX512)\n list(APPEND ARCH_FLAGS -mavx512f)\n list(APPEND ARCH_FLAGS -mavx512cd)\n list(APPEND ARCH_FLAGS -mavx512vl)\n list(APPEND ARCH_FLAGS -mavx512dq)\n list(APPEND ARCH_FLAGS -mavx512bw)\n list(APPEND ARCH_DEFINITIONS GGML_AVX512)\n endif()\n if (GGML_AVX512_VBMI)\n list(APPEND ARCH_FLAGS -mavx512vbmi)\n list(APPEND ARCH_DEFINITIONS GGML_AVX512_VBMI)\n endif()\n if (GGML_AVX512_VNNI)\n list(APPEND ARCH_FLAGS -mavx512vnni)\n list(APPEND ARCH_DEFINITIONS GGML_AVX512_VNNI)\n endif()\n if (GGML_AVX512_BF16)\n list(APPEND ARCH_FLAGS -mavx512bf16)\n list(APPEND ARCH_DEFINITIONS GGML_AVX512_BF16)\n endif()\n if (GGML_AMX_TILE)\n list(APPEND ARCH_FLAGS -mamx-tile)\n list(APPEND ARCH_DEFINITIONS GGML_AMX_TILE)\n endif()\n if (GGML_AMX_INT8)\n list(APPEND ARCH_FLAGS -mamx-int8)\n list(APPEND ARCH_DEFINITIONS GGML_AMX_INT8)\n endif()\n if (GGML_AMX_BF16)\n list(APPEND ARCH_FLAGS -mamx-bf16)\n list(APPEND ARCH_DEFINITIONS GGML_AMX_BF16)\n endif()\n endif()\n endif()\n\n if (GGML_BACKEND_DL)\n if (GGML_NATIVE)\n # the feature check relies on ARCH_DEFINITIONS, but it is not set with GGML_NATIVE\n message(FATAL_ERROR \"GGML_NATIVE is not compatible with GGML_BACKEND_DL, consider using GGML_CPU_ALL_VARIANTS\")\n endif()\n ggml_add_cpu_backend_features(${GGML_CPU_NAME} x86 ${ARCH_DEFINITIONS})\n endif()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"PowerPC\")\n message(STATUS \"PowerPC detected\")\n list(APPEND GGML_CPU_SOURCES ggml-cpu/arch/powerpc/quants.c)\n if (GGML_NATIVE)\n if (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"ppc64\")\n file(READ \"/proc/cpuinfo\" POWER10_M)\n elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"powerpc\")\n execute_process(COMMAND bash -c \"prtconf |grep 'Implementation' | head -n 1\" OUTPUT_VARIABLE POWER10_M)\n endif()\n\n string(TOUPPER \"${POWER10_M}\" POWER10_M_UPPER)\n string(REGEX MATCHALL \"POWER *([0-9]+)\" MATCHED_STRING \"${POWER10_M_UPPER}\")\n string(REGEX REPLACE \"POWER *([0-9]+)\" \"\\\\1\" EXTRACTED_NUMBER \"${MATCHED_STRING}\")\n\n if (EXTRACTED_NUMBER GREATER_EQUAL 10)\n list(APPEND ARCH_FLAGS -mcpu=power10)\n elseif (EXTRACTED_NUMBER EQUAL 9)\n list(APPEND ARCH_FLAGS -mcpu=power9)\n elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"ppc64le\")\n list(APPEND ARCH_FLAGS -mcpu=powerpc64le -mtune=native)\n else()\n list(APPEND ARCH_FLAGS -mcpu=native -mtune=native -mpowerpc64)\n endif()\n elseif(GGML_CPU_ALL_VARIANTS)\n # Begin with the lowest baseline\n set(ARCH_DEFINITIONS \"\")\n\n # When a feature is selected, bump the MCPU to the first\n # version that supported it\n foreach(PVER RANGE 7 11)\n if(DEFINED GGML_INTERNAL_POWER${PVER})\n set(POWERPC_MCPU \"power${PVER}\")\n list(APPEND ARCH_DEFINITIONS GGML_USE_POWER${PVER})\n endif()\n endforeach()\n if (GGML_INTERNAL_VSX)\n list(APPEND ARCH_DEFINITIONS GGML_USE_VSX)\n list(APPEND ARCH_FLAGS -mvsx)\n endif()\n\n if (DEFINED POWERPC_MCPU)\n list(APPEND ARCH_FLAGS -mcpu=${POWERPC_MCPU})\n endif()\n ggml_add_cpu_backend_features(${GGML_CPU_NAME} powerpc ${ARCH_DEFINITIONS})\n else()\n if (GGML_CPU_POWERPC_CPUTYPE)\n list(APPEND ARCH_FLAGS -mcpu=${GGML_CPU_POWERPC_CPUTYPE})\n endif()\n endif()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"loongarch64\")\n message(STATUS \"loongarch64 detected\")\n list(APPEND GGML_CPU_SOURCES ggml-cpu/arch/loongarch/quants.c)\n\n list(APPEND ARCH_FLAGS -march=loongarch64)\n if (GGML_LASX)\n list(APPEND ARCH_FLAGS -mlasx)\n endif()\n if (GGML_LSX)\n list(APPEND ARCH_FLAGS -mlsx)\n endif()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"riscv64\")\n message(STATUS \"riscv64 detected\")\n list(APPEND GGML_CPU_SOURCES\n ggml-cpu/arch/riscv/quants.c\n ggml-cpu/arch/riscv/repack.cpp\n )\n if (GGML_CPU_RISCV64_SPACEMIT)\n target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_RISCV64_SPACEMIT ${RISCV64_SPACEMIT_IME_SPEC})\n list(APPEND GGML_CPU_SOURCES\n ggml-cpu/spacemit/ime.cpp\n ggml-cpu/spacemit/ime.h\n ggml-cpu/spacemit/ime1_kernels.cpp\n ggml-cpu/spacemit/ime_kernels.h\n )\n endif()\n if(NOT GGML_CPU_ALL_VARIANTS)\n set(MARCH_STR \"rv64gc\")\n if (GGML_RV_ZFH)\n string(APPEND MARCH_STR \"_zfh\")\n endif()\n\n if (GGML_XTHEADVECTOR)\n string(APPEND MARCH_STR \"_xtheadvector\")\n elseif (GGML_RVV)\n string(APPEND MARCH_STR \"_v\")\n if (GGML_RV_ZVFH)\n string(APPEND MARCH_STR \"_zvfh\")\n endif()\n if (GGML_RV_ZVFBFWMA)\n string(APPEND MARCH_STR \"_zvfbfwma\")\n endif()\n endif()\n if (GGML_RV_ZICBOP)\n string(APPEND MARCH_STR \"_zicbop\")\n endif()\n if (GGML_RV_ZIHINTPAUSE)\n string(APPEND MARCH_STR \"_zihintpause\")\n endif()\n list(APPEND ARCH_FLAGS \"-march=${MARCH_STR}\" -mabi=lp64d)\n else()\n # Begin with the lowest baseline\n set(ARCH_DEFINITIONS \"\")\n\n if (GGML_INTERNAL_RVV)\n message(STATUS \"RVV enabled\")\n list(APPEND ARCH_DEFINITIONS GGML_USE_RVV)\n list(APPEND ARCH_FLAGS -march=rv64gc_v -mabi=lp64d)\n endif()\n\n ggml_add_cpu_backend_features(${GGML_CPU_NAME} riscv ${ARCH_DEFINITIONS})\n endif()\n elseif (GGML_SYSTEM_ARCH STREQUAL \"s390x\")\n message(STATUS \"s390x detected\")\n list(APPEND GGML_CPU_SOURCES\n ggml-cpu/arch/s390/quants.c)\n\n # for native compilation\n if (GGML_NATIVE)\n # check machine level to determine target\n file(READ \"/proc/cpuinfo\" CPUINFO_CONTENTS)\n string(REGEX REPLACE \"machine[ \\t\\r\\n]*=[ \\t\\r\\n]*([0-9]+)\" \"\\\\1\" S390X_M ${CPUINFO_CONTENTS})\n\n # TODO: Separation to determine activation of VX/VXE/VXE2\n if (${S390X_M} MATCHES \"8561|8562\")\n message(STATUS \"z15 target\")\n list(APPEND ARCH_FLAGS -march=z15)\n elseif (${S390X_M} MATCHES \"3931\")\n message(STATUS \"z16 target\")\n list(APPEND ARCH_FLAGS -march=z16)\n elseif (${S390X_M} MATCHES \"9175|9176\")\n # NOTE: Only available from GCC 15.1.0 onwards. Any z17 machine with compile issues must first verify their GCC version.\n # binutils must also be updated to the latest for the -march=z17 flag to work. Otherwise, use -march=arch15.\n message(STATUS \"z17 target\")\n list(APPEND ARCH_FLAGS -march=arch15)\n else()\n message(STATUS \"Unknown target\")\n message(WARNING \"Unknown target. If you are compiling for z14 and earlier, you might have to add -DGGML_VXE=OFF.\")\n list(APPEND ARCH_FLAGS -march=native -mtune=native)\n endif()\n # for cross-compilation\n elseif(GGML_CPU_ALL_VARIANTS)\n # range through IBM z15 to z17\n # NOTE: update when a new hardware level is released\n foreach (ZHW RANGE 15 17)\n if(DEFINED GGML_INTERNAL_Z${ZHW})\n message(STATUS \"z${ZHW} cross-compile target\")\n list(APPEND ARCH_FLAGS -march=z${ZHW})\n endif()\n endforeach()\n endif()\n\n if (GGML_VXE OR GGML_INTERNAL_VXE2)\n message(STATUS \"VXE2 enabled\")\n list(APPEND ARCH_FLAGS -mvx -mzvector)\n list(APPEND ARCH_DEFINITIONS GGML_USE_VXE2)\n endif()\n\n if (GGML_INTERNAL_NNPA)\n message(STATUS \"NNPA enabled\")\n list(APPEND ARCH_DEFINITIONS GGML_USE_NNPA)\n endif()\n\n ggml_add_cpu_backend_features(${GGML_CPU_NAME} s390 ${ARCH_DEFINITIONS})\n elseif (CMAKE_SYSTEM_PROCESSOR MATCHES \"wasm\")\n message(STATUS \"Wasm detected\")\n list (APPEND GGML_CPU_SOURCES ggml-cpu/arch/wasm/quants.c)\n else()\n message(WARNING \"Unknown CPU architecture. Falling back to generic implementations.\")\n list(APPEND ARCH_FLAGS -DGGML_CPU_GENERIC)\n endif()\n\n if (GGML_CPU_REPACK)\n target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_REPACK)\n endif()\n\n if (GGML_CPU_KLEIDIAI)\n message(STATUS \"Using KleidiAI optimized kernels if applicable\")\n\n # Disable the KleidiAI tests\n set(KLEIDIAI_BUILD_TESTS OFF)\n\n # Fetch KleidiAI sources:\n include(FetchContent)\n set(KLEIDIAI_COMMIT_TAG \"v1.22.0\")\n set(KLEIDIAI_DOWNLOAD_URL \"https://github.com/ARM-software/kleidiai/archive/refs/tags/${KLEIDIAI_COMMIT_TAG}.tar.gz\")\n set(KLEIDIAI_ARCHIVE_MD5 \"54049037570ab0ee0a0d126b2ba5ece1\")\n\n if (POLICY CMP0135)\n cmake_policy(SET CMP0135 NEW)\n endif()\n\n # TODO: Use FetchContent_MakeAvailable with EXCLUDE_FROM_ALL after bumping minimum CMake version to 3.28+\n # Using FetchContent_Populate instead to avoid EXCLUDE_FROM_ALL which requires CMake 3.28\n FetchContent_Declare(KleidiAI_Download\n URL ${KLEIDIAI_DOWNLOAD_URL}\n DOWNLOAD_EXTRACT_TIMESTAMP NEW\n URL_HASH MD5=${KLEIDIAI_ARCHIVE_MD5})\n\n FetchContent_GetProperties(KleidiAI_Download\n SOURCE_DIR KLEIDIAI_SRC\n POPULATED KLEIDIAI_POPULATED)\n\n if (NOT KLEIDIAI_POPULATED)\n FetchContent_Populate(KleidiAI_Download)\n FetchContent_GetProperties(KleidiAI_Download SOURCE_DIR KLEIDIAI_SRC)\n endif()\n\n add_compile_definitions(GGML_USE_CPU_KLEIDIAI)\n\n list(APPEND GGML_CPU_SOURCES\n ggml-cpu/kleidiai/kleidiai.cpp\n ggml-cpu/kleidiai/kernels.cpp\n ggml-cpu/kleidiai/kleidiai.h\n ggml-cpu/kleidiai/kernels.h\n )\n\n # KleidiAI\n include_directories(\n ${KLEIDIAI_SRC}/\n ${KLEIDIAI_SRC}/kai/\n ${KLEIDIAI_SRC}/kai/ukernels/\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/)\n\n set(ARCH_FLAGS_TEMP \"${ARCH_FLAGS}\")\n if (NOT ARCH_FLAGS_TEMP)\n string(REGEX MATCH \"-march=[^ ]+\" ARCH_FLAGS_TEMP \"${CMAKE_C_FLAGS}\")\n endif()\n string(FIND \"${ARCH_FLAGS_TEMP}\" \"+dotprod\" DOTPROD_ENABLED)\n string(FIND \"${ARCH_FLAGS_TEMP}\" \"+i8mm\" I8MM_ENABLED)\n string(FIND \"${ARCH_FLAGS_TEMP}\" \"+sme\" SME_ENABLED)\n string(FIND \"${ARCH_FLAGS_TEMP}\" \"+sve\" SVE_ENABLED)\n\n set(PRIVATE_ARCH_FLAGS ${ARCH_FLAGS_TEMP})\n\n list(APPEND GGML_KLEIDIAI_SOURCES\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.c)\n\n if (NOT DOTPROD_ENABLED MATCHES -1)\n list(APPEND GGML_KLEIDIAI_SOURCES\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.c)\n endif()\n\n if (NOT I8MM_ENABLED MATCHES -1)\n list(APPEND GGML_KLEIDIAI_SOURCES\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.c)\n endif()\n\n if (NOT SME_ENABLED MATCHES -1)\n list(APPEND GGML_KLEIDIAI_SOURCES\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa_asm.S\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot_asm.S\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa_asm.S\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa_asm.S\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_bf16p2vlx2_f32_sme.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_f16pmrx2_f32_neon.c\n ${KLEIDIAI_SRC}/kai/kai_common_sme_asm.S)\n set(PRIVATE_ARCH_FLAGS \"-fno-tree-vectorize;${PRIVATE_ARCH_FLAGS}+sve+sve2+sme2+fp16\")\n endif()\n\n if (NOT SVE_ENABLED MATCHES -1)\n list(APPEND GGML_KLEIDIAI_SOURCES\n ${KLEIDIAI_SRC}/kai/kai_common_sve_asm.S\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod_asm.S\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.c\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm_asm.S\n ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.c)\n endif()\n\n set_source_files_properties(${GGML_KLEIDIAI_SOURCES} PROPERTIES COMPILE_OPTIONS \"${PRIVATE_ARCH_FLAGS}\")\n list(APPEND GGML_CPU_SOURCES ${GGML_KLEIDIAI_SOURCES})\n endif()\n\n message(STATUS \"Adding CPU backend variant ${GGML_CPU_NAME}: ${ARCH_FLAGS} ${ARCH_DEFINITIONS}\")\n target_sources(${GGML_CPU_NAME} PRIVATE ${GGML_CPU_SOURCES})\n target_compile_options(${GGML_CPU_NAME} PRIVATE ${ARCH_FLAGS})\n target_compile_definitions(${GGML_CPU_NAME} PRIVATE ${ARCH_DEFINITIONS})\n\n if (EMSCRIPTEN)\n set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS \"-msimd128\")\n endif()\n\n if (CMAKE_CXX_COMPILER_ID STREQUAL \"IntelLLVM\")\n # The compiler automatically enables \"-ffast-math\" which can cause NaNs in tests due to \"-fassociative-math\"\n target_compile_options(${GGML_CPU_NAME} PRIVATE \"-fno-associative-math\")\n endif()\nendfunction()\n" }, { "path": "src/ggml-cpu/amx/amx.cpp", "content": "#include \"amx.h\"\n#include \"common.h\"\n#include \"mmq.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"traits.h\"\n\n#if defined(__linux__)\n#include \n#include \n#endif\n\n#include \n#include \n#include \n\n#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n\n// AMX type_trais\nnamespace ggml::cpu::amx {\nclass tensor_traits : public ggml::cpu::tensor_traits {\n bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n size = ggml_backend_amx_desired_wsize(op);\n return true;\n }\n\n bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override {\n if (op->op == GGML_OP_MUL_MAT) {\n ggml_backend_amx_mul_mat(params, op);\n return true;\n }\n return false;\n }\n};\n\nstatic ggml::cpu::tensor_traits * get_tensor_traits(ggml_backend_buffer_t, struct ggml_tensor *) {\n static tensor_traits traits;\n return &traits;\n}\n} // namespace ggml::cpu::amx\n\n// AMX buffer interface\nstatic void ggml_backend_amx_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n free(buffer->context);\n}\n\nstatic void * ggml_backend_amx_buffer_get_base(ggml_backend_buffer_t buffer) {\n return (void *) (buffer->context);\n}\n\nstatic enum ggml_status ggml_backend_amx_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {\n tensor->extra = (void *) ggml::cpu::amx::get_tensor_traits(buffer, tensor);\n\n GGML_UNUSED(buffer);\n return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_amx_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,\n uint8_t value, size_t offset, size_t size) {\n memset((char *) tensor->data + offset, value, size);\n\n GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_amx_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,\n const void * data, size_t offset, size_t size) {\n if (qtype_has_amx_kernels(tensor->type)) {\n GGML_LOG_DEBUG(\"%s: amx repack tensor %s of type %s\\n\", __func__, tensor->name, ggml_type_name(tensor->type));\n ggml_backend_amx_convert_weight(tensor, data, offset, size);\n } else {\n memcpy((char *) tensor->data + offset, data, size);\n }\n\n GGML_UNUSED(buffer);\n}\n\n/*\n// need to figure what we need to do with buffer->extra.\nstatic void ggml_backend_amx_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n GGML_ASSERT(!qtype_has_amx_kernels(tensor->type));\n memcpy(data, (const char *)tensor->data + offset, size);\n\n GGML_UNUSED(buffer);\n}\n\nstatic bool ggml_backend_amx_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {\n if (ggml_backend_buffer_is_host(src->buffer)) {\n if (qtype_has_amx_kernels(src->type)) {\n ggml_backend_amx_convert_weight(dst, src->data, 0, ggml_nbytes(dst));\n } else {\n memcpy(dst->data, src->data, ggml_nbytes(src));\n }\n return true;\n }\n return false;\n\n GGML_UNUSED(buffer);\n}\n*/\n\nstatic void ggml_backend_amx_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n memset(buffer->context, value, buffer->size);\n}\n\nstatic ggml_backend_buffer_i ggml_backend_amx_buffer_interface = {\n /* .free_buffer = */ ggml_backend_amx_buffer_free_buffer,\n /* .get_base = */ ggml_backend_amx_buffer_get_base,\n /* .init_tensor = */ ggml_backend_amx_buffer_init_tensor,\n /* .memset_tensor = */ ggml_backend_amx_buffer_memset_tensor,\n /* .set_tensor = */ ggml_backend_amx_buffer_set_tensor,\n /* .get_tensor = */ nullptr,\n /* .cpy_tensor = */ nullptr,\n /* .clear = */ ggml_backend_amx_buffer_clear,\n /* .reset = */ nullptr,\n};\n\nstatic const char * ggml_backend_amx_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n return \"AMX\";\n\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_amx_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n void * data = ggml_aligned_malloc(size);\n if (data == NULL) {\n fprintf(stderr, \"%s: failed to allocate buffer of size %zu\\n\", __func__, size);\n return NULL;\n }\n\n return ggml_backend_buffer_init(buft, ggml_backend_amx_buffer_interface, data, size);\n}\n\nstatic size_t ggml_backend_amx_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n return TENSOR_ALIGNMENT;\n\n GGML_UNUSED(buft);\n}\n\nnamespace ggml::cpu::amx {\nclass extra_buffer_type : ggml::cpu::extra_buffer_type {\n bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {\n if (op->op != GGML_OP_MUL_MAT) {\n return false;\n }\n auto * src0 = op->src[0];\n auto * src1 = op->src[1];\n\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {\n return false;\n }\n if (!src0->buffer || src0->buffer->buft != ggml_backend_amx_buffer_type()) {\n return false;\n }\n if (src1->buffer && !ggml_backend_buft_is_host(src1->buffer->buft)) {\n return false;\n }\n if (op->ne[0] % (TILE_N * 2)) {\n return false;\n }\n int alignment;\n switch (src0->type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q8_0:\n alignment = TILE_K;\n break;\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_IQ4_XS:\n alignment = 256; // QK_K\n break;\n case GGML_TYPE_F16:\n alignment = 16;\n break;\n default:\n return false;\n }\n if (src0->ne[0] % alignment) {\n return false;\n }\n if (src1->type != GGML_TYPE_F32) {\n return false;\n }\n return true;\n }\n\n ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {\n if (op->op == GGML_OP_MUL_MAT && op->src[0]->buffer &&\n op->src[0]->buffer->buft == ggml_backend_amx_buffer_type()) {\n return (ggml::cpu::tensor_traits *) op->src[0]->extra;\n }\n\n return nullptr;\n }\n};\n} // namespace ggml::cpu::amx\n\nstatic size_t ggml_backend_amx_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n return ggml_backend_amx_get_alloc_size(tensor);\n\n GGML_UNUSED(buft);\n}\n\n#define ARCH_GET_XCOMP_PERM 0x1022\n#define ARCH_REQ_XCOMP_PERM 0x1023\n#define XFEATURE_XTILECFG 17\n#define XFEATURE_XTILEDATA 18\n\nstatic bool ggml_amx_init() {\n#if defined(__linux__)\n if (syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA)) {\n fprintf(stderr, \"AMX is not ready to be used!\\n\");\n return false;\n }\n return true;\n#elif defined(_WIN32)\n return true;\n#else\n return false;\n#endif\n}\n\nggml_backend_buffer_type_t ggml_backend_amx_buffer_type() {\n static struct ggml_backend_buffer_type ggml_backend_buffer_type_amx = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_amx_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_amx_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_amx_buffer_type_get_alignment,\n /* .get_max_size = */ nullptr, // defaults to SIZE_MAX\n /* .get_alloc_size = */ ggml_backend_amx_buffer_type_get_alloc_size,\n /* .is_host = */ nullptr,\n },\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n /* .context = */ new ggml::cpu::amx::extra_buffer_type(),\n };\n\n if (!ggml_amx_init()) {\n return nullptr;\n }\n\n return &ggml_backend_buffer_type_amx;\n}\n\n#endif // defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n" }, { "path": "src/ggml-cpu/amx/amx.h", "content": "#include \"ggml-backend.h\"\n#include \"ggml-cpu-impl.h\"\n\n// GGML internal header\n\n#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\nggml_backend_buffer_type_t ggml_backend_amx_buffer_type(void);\n#endif\n" }, { "path": "src/ggml-cpu/amx/common.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-cpu-impl.h\"\n\n#include \n#include \n#include \n\n#if defined(GGML_USE_OPENMP)\n#include \n#else\n#include \n#endif\n\n#define TILE_M 16\n#define TILE_N 16\n#define TILE_K 32\n#define VNNI_BLK 4\n\n#define AMX_BLK_SIZE 32\n\n#define TMM0 0\n#define TMM1 1\n#define TMM2 2\n#define TMM3 3\n#define TMM4 4\n#define TMM5 5\n#define TMM6 6\n#define TMM7 7\n\n// parallel routines\ntemplate ::value, int>::type = 0>\ninline T div_up(T x, T y) { return (x + y - 1) / y; }\n\ntemplate \ninline void balance211(T n, T nth, T ith, T& n_start, T& n_end) {\n#if 0\n // onednn partition pattern\n T& n_my = n_end;\n if (nth <= 1 || n == 0) {\n n_start = 0;\n n_my = n;\n } else {\n T n1 = div_up(n, nth);\n T n2 = n1 - 1;\n T T1 = n - n2 * nth;\n n_my = ith < T1 ? n1 : n2;\n n_start = ith <= T1 ? ith*n1 : T1 * n1 + (ith - T1) * n2;\n }\n n_end += n_start;\n#else\n // pytorch aten partition pattern\n T n_my = div_up(n, nth);\n n_start = ith * n_my;\n n_end = std::min(n_start + n_my, n);\n#endif\n}\n\ntemplate \ninline void parallel_for(int n, const func_t & f) {\n if (n <= 0) {\n return;\n }\n#if defined(GGML_USE_OPENMP)\n #pragma omp parallel\n {\n int nth = omp_get_num_threads();\n int ith = omp_get_thread_num();\n int tbegin, tend;\n balance211(n, nth, ith, tbegin, tend);\n f(tbegin, tend);\n }\n#else\n int nth = std::thread::hardware_concurrency();\n if (nth <= 1) {\n f(0, n);\n return;\n }\n if (nth > n) {\n nth = n;\n }\n std::vector threads;\n threads.reserve(nth);\n for (int ith = 0; ith < nth; ++ith) {\n threads.emplace_back([&f, n, ith, nth] {\n int tbegin, tend;\n balance211(n, nth, ith, tbegin, tend);\n f(tbegin, tend);\n });\n }\n for (auto & t : threads) {\n t.join();\n }\n#endif\n}\n\ntemplate \ninline void parallel_for_ggml(const ggml_compute_params * params, int n, const func_t & f) {\n int tbegin, tend;\n balance211(n, params->nth, params->ith, tbegin, tend);\n f(tbegin, tend);\n}\n\n// quantized types that have AMX support\ninline bool qtype_has_amx_kernels(const enum ggml_type type) {\n // TODO: fix padding for vnni format\n return (type == GGML_TYPE_Q4_0) ||\n (type == GGML_TYPE_Q4_1) ||\n (type == GGML_TYPE_Q8_0) ||\n (type == GGML_TYPE_Q4_K) ||\n (type == GGML_TYPE_Q5_K) ||\n (type == GGML_TYPE_Q6_K) ||\n (type == GGML_TYPE_IQ4_XS);\n}\n" }, { "path": "src/ggml-cpu/amx/mmq.cpp", "content": "#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Wpedantic\"\n#pragma GCC diagnostic ignored \"-Wunused-local-typedefs\"\n#endif\n\n#include \"amx.h\"\n#include \"mmq.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"quants.h\"\n#include \"ggml-quants.h\"\n#include \n#include \n\n#if defined(__gnu_linux__)\n#include \n#include \n#endif\n\n#if (defined(_WIN32) || defined(_WIN64))\n#define RESTRICT __restrict\n#else\n#define RESTRICT __restrict__\n#endif\n\n#if (defined(_WIN32) || defined(_WIN64))\n#define ALWAYS_INLINE __forceinline\n#elif __has_attribute(always_inline) || defined(__GNUC__)\n#define ALWAYS_INLINE __attribute__((__always_inline__)) inline\n#else\n#define ALWAYS_INLINE inline\n#endif\n\n#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n\nnamespace {\n\n// Forced unrolling\ntemplate \nstruct Unroll {\n template \n ALWAYS_INLINE void operator()(const Func& f, Args... args) const {\n Unroll{}(f, args...);\n f(std::integral_constant{}, args...);\n }\n};\n\ntemplate <>\nstruct Unroll<1> {\n template \n ALWAYS_INLINE void operator()(const Func& f, Args... args) const {\n f(std::integral_constant{}, args...);\n }\n};\n\n// type traits\ntemplate struct PackedTypes {};\ntemplate <> struct PackedTypes { using type = int8_t; };\ntemplate <> struct PackedTypes { using type = uint8_t; };\ntemplate <> struct PackedTypes { using type = int8_t; };\ntemplate using packed_B_type = typename PackedTypes::type;\n\ntemplate \nstruct do_compensate : std::integral_constant::value> {};\n\ntemplate \nstruct do_unpack : std::integral_constant::value ||\n std::is_same::value> {};\n\ntemplate \nstruct is_type_qkk : std::integral_constant::value ||\n std::is_same::value ||\n std::is_same::value ||\n std::is_same::value> {};\n\n#define GGML_DISPATCH_FLOATING_TYPES(TYPE, ...) \\\n [&] { \\\n switch (TYPE) { \\\n case GGML_TYPE_F16: { \\\n using type = ggml_fp16_t; \\\n constexpr int blck_size = 16; \\\n return __VA_ARGS__(); \\\n } \\\n case GGML_TYPE_BF16: { \\\n using type = ggml_bf16_t; \\\n constexpr int blck_size = 32; \\\n return __VA_ARGS__(); \\\n } \\\n default: \\\n fprintf(stderr, \"Unsupported floating data type\\n\"); \\\n } \\\n }()\n\n#define GGML_DISPATCH_QTYPES(QT, ...) \\\n [&] { \\\n switch (QT) { \\\n case GGML_TYPE_Q4_0: { \\\n using type = block_q4_0; \\\n using vec_dot_type = block_q8_0; \\\n constexpr int blck_size = QK4_0; \\\n return __VA_ARGS__(); \\\n } \\\n case GGML_TYPE_Q4_1: { \\\n using type = block_q4_1; \\\n using vec_dot_type = block_q8_1; \\\n constexpr int blck_size = QK4_1; \\\n return __VA_ARGS__(); \\\n } \\\n case GGML_TYPE_Q8_0: { \\\n using type = block_q8_0; \\\n using vec_dot_type = block_q8_0; \\\n constexpr int blck_size = QK8_0; \\\n return __VA_ARGS__(); \\\n } \\\n case GGML_TYPE_Q4_K: { \\\n using type = block_q4_K; \\\n using vec_dot_type = block_q8_K; \\\n constexpr int blck_size = QK_K; \\\n return __VA_ARGS__(); \\\n } \\\n case GGML_TYPE_Q5_K: { \\\n using type = block_q5_K; \\\n using vec_dot_type = block_q8_K; \\\n constexpr int blck_size = QK_K; \\\n return __VA_ARGS__(); \\\n } \\\n case GGML_TYPE_Q6_K: { \\\n using type = block_q6_K; \\\n using vec_dot_type = block_q8_K; \\\n constexpr int blck_size = QK_K; \\\n return __VA_ARGS__(); \\\n } \\\n case GGML_TYPE_IQ4_XS: { \\\n using type = block_iq4_xs; \\\n using vec_dot_type = block_q8_K; \\\n constexpr int blck_size = QK_K; \\\n return __VA_ARGS__(); \\\n } \\\n default: \\\n fprintf(stderr, \"Unsupported quantized data type: %d\\n\", int(TYPE)); \\\n } \\\n }()\n\n#define GGML_DISPATCH_BOOL(BOOL_V, BOOL_NAME, ...) \\\n [&] { \\\n if (BOOL_V) { \\\n constexpr bool BOOL_NAME = true; \\\n return __VA_ARGS__(); \\\n } else { \\\n constexpr bool BOOL_NAME = false; \\\n return __VA_ARGS__(); \\\n } \\\n }()\n\n// define amx tile config data structure\nstruct tile_config_t{\n uint8_t palette_id = 0;\n uint8_t start_row = 0;\n uint8_t reserved_0[14] = {0};\n uint16_t colsb[16] = {0};\n uint8_t rows[16] = {0};\n};\n\n// Notes: amx tile config\n//\n// Typically, TMUL calculates A and B of size 16 x 64 containing INT8 values,\n// and accumulate the result to a 16 x 16 matrix C containing INT32 values,\n//\n// As many GGUF quantized types as `block_size` of 32, so a 16-16-32 config is used\n// instead of the normally used 16-16-64 config.\n//\n// Block A: {16, 32}, dtype = int8_t\n// Block B: {16, 32}, dtype = uint8_t/int8_t\n// Block C: {16, 16}, dtype = int32_t\n//\n// Block B needs to be prepacked to vnni format before feeding into TMUL:\n// packed_B: from {n, k} to {k/vnni_blk, n, vnni_blck}, viewed in 2d, we get {8, 64}\n//\n// Therefore, we get tileconfig:\n// A B C\n// rows 16 8 16\n// colsb 32 64 16\n//\n// For tile distribution, follow a 2-2-4 pattern, e.g. A used TMM2-TMM3, B used TMM0-TMM1,\n// C used TMM4-TMM7:\n// B TMM0 B TMM1\n// A TMM2 C TMM4 C TMM6\n// A TMM3 C TMM5 C TMM7\n//\n// Each `amx` kernel handles 4 blocks at a time: 2MB * 2NB, when m < 2 * BLOCK_M, unpack A\n// will be needed.\n//\n// Here another commonly used pattern 1-3-3 is skipped, as it is mostly used when m <=16;\n// and the single batch gemm (m=1) has a special fast path with `avx512-vnni`.\n//\n// ref: https://www.intel.com/content/www/us/en/developer/articles/code-sample/\n// advanced-matrix-extensions-intrinsics-functions.html\n//\n\ninline void ggml_tile_config_init(void) {\n static thread_local bool done = false;\n\n if (done) {\n return;\n }\n\n alignas(64) tile_config_t tc = {};\n tc.palette_id = 1;\n tc.start_row = 0;\n tc.rows[0] = 8; tc.colsb[0] = 64;\n tc.rows[1] = 8; tc.colsb[1] = 64;\n tc.rows[2] = 16; tc.colsb[2] = 32;\n tc.rows[3] = 16; tc.colsb[3] = 32;\n tc.rows[4] = 16; tc.colsb[4] = 64;\n tc.rows[5] = 16; tc.colsb[5] = 64;\n tc.rows[6] = 16; tc.colsb[6] = 64;\n tc.rows[7] = 16; tc.colsb[7] = 64;\n\n _tile_loadconfig(&tc);\n done = true;\n}\n\n// we need an extra 16 * 4B (TILE_N * int32_t) for each NB/KB block for compensation.\n// See the notes `s8s8 igemm compensation in avx512-vnni` for detail.\ntemplate \nint get_tile_size() {\n int tile_size = TILE_N * sizeof(TB);\n if (do_compensate::value) {\n tile_size += TILE_N * sizeof(int32_t);\n }\n if (std::is_same::value ||\n std::is_same::value) {\n tile_size += TILE_N * 4;\n }\n if (std::is_same::value) {\n tile_size += TILE_N * 2;\n }\n return tile_size;\n}\n\ntemplate \nint get_row_size(int K) {\n int KB = K / BLOCK_K;\n int row_size = KB * sizeof(TB);\n if (do_compensate::value) {\n row_size += KB * sizeof(int32_t);\n }\n if (std::is_same::value ||\n std::is_same::value) {\n row_size += KB * 4;\n }\n if (std::is_same::value) {\n row_size += KB * 2;\n }\n return row_size;\n}\n\n// transpose utils\n#define SHUFFLE_EPI32(a, b, mask) \\\n _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), mask))\ninline void transpose_8x8_32bit(__m256i * v, __m256i * v1) {\n // unpacking and 32-bit elements\n v1[0] = _mm256_unpacklo_epi32(v[0], v[1]);\n v1[1] = _mm256_unpackhi_epi32(v[0], v[1]);\n v1[2] = _mm256_unpacklo_epi32(v[2], v[3]);\n v1[3] = _mm256_unpackhi_epi32(v[2], v[3]);\n v1[4] = _mm256_unpacklo_epi32(v[4], v[5]);\n v1[5] = _mm256_unpackhi_epi32(v[4], v[5]);\n v1[6] = _mm256_unpacklo_epi32(v[6], v[7]);\n v1[7] = _mm256_unpackhi_epi32(v[6], v[7]);\n\n // shuffling the 32-bit elements\n v[0] = SHUFFLE_EPI32(v1[0], v1[2], 0x44);\n v[1] = SHUFFLE_EPI32(v1[0], v1[2], 0xee);\n v[2] = SHUFFLE_EPI32(v1[4], v1[6], 0x44);\n v[3] = SHUFFLE_EPI32(v1[4], v1[6], 0xee);\n v[4] = SHUFFLE_EPI32(v1[1], v1[3], 0x44);\n v[5] = SHUFFLE_EPI32(v1[1], v1[3], 0xee);\n v[6] = SHUFFLE_EPI32(v1[5], v1[7], 0x44);\n v[7] = SHUFFLE_EPI32(v1[5], v1[7], 0xee);\n\n // shuffling 128-bit elements\n v1[0] = _mm256_permute2f128_si256(v[2], v[0], 0x02);\n v1[1] = _mm256_permute2f128_si256(v[3], v[1], 0x02);\n v1[2] = _mm256_permute2f128_si256(v[6], v[4], 0x02);\n v1[3] = _mm256_permute2f128_si256(v[7], v[5], 0x02);\n v1[4] = _mm256_permute2f128_si256(v[2], v[0], 0x13);\n v1[5] = _mm256_permute2f128_si256(v[3], v[1], 0x13);\n v1[6] = _mm256_permute2f128_si256(v[6], v[4], 0x13);\n v1[7] = _mm256_permute2f128_si256(v[7], v[5], 0x13);\n}\n\ninline void transpose_16x4_32bit(__m512i * r, __m512i * d) {\n\n static const __m512i index1 = _mm512_set_epi32(\n 0x0f, 0x0b, 0x07, 0x03,\n 0x0e, 0x0a, 0x06, 0x02,\n 0x0d, 0x09, 0x05, 0x01,\n 0x0c, 0x08, 0x04, 0x00);\n\n d[0] = _mm512_permutexvar_epi32(index1, r[0]);\n d[1] = _mm512_permutexvar_epi32(index1, r[1]);\n d[2] = _mm512_permutexvar_epi32(index1, r[2]);\n d[3] = _mm512_permutexvar_epi32(index1, r[3]);\n\n r[0] = _mm512_shuffle_i32x4(d[0], d[1], 0x44);\n r[1] = _mm512_shuffle_i32x4(d[0], d[1], 0xee);\n r[2] = _mm512_shuffle_i32x4(d[2], d[3], 0x44);\n r[3] = _mm512_shuffle_i32x4(d[2], d[3], 0xee);\n\n d[0] = _mm512_shuffle_i32x4(r[0], r[2], 0x88);\n d[1] = _mm512_shuffle_i32x4(r[0], r[2], 0xdd);\n d[2] = _mm512_shuffle_i32x4(r[1], r[3], 0x88);\n d[3] = _mm512_shuffle_i32x4(r[1], r[3], 0xdd);\n}\n\ninline void transpose_16x16_32bit(__m512i * v) {\n __m512i v1[16];\n v1[0] = _mm512_unpacklo_epi32(v[0], v[1]);\n v1[1] = _mm512_unpackhi_epi32(v[0], v[1]);\n v1[2] = _mm512_unpacklo_epi32(v[2], v[3]);\n v1[3] = _mm512_unpackhi_epi32(v[2], v[3]);\n v1[4] = _mm512_unpacklo_epi32(v[4], v[5]);\n v1[5] = _mm512_unpackhi_epi32(v[4], v[5]);\n v1[6] = _mm512_unpacklo_epi32(v[6], v[7]);\n v1[7] = _mm512_unpackhi_epi32(v[6], v[7]);\n v1[8] = _mm512_unpacklo_epi32(v[8], v[9]);\n v1[9] = _mm512_unpackhi_epi32(v[8], v[9]);\n v1[10] = _mm512_unpacklo_epi32(v[10], v[11]);\n v1[11] = _mm512_unpackhi_epi32(v[10], v[11]);\n v1[12] = _mm512_unpacklo_epi32(v[12], v[13]);\n v1[13] = _mm512_unpackhi_epi32(v[12], v[13]);\n v1[14] = _mm512_unpacklo_epi32(v[14], v[15]);\n v1[15] = _mm512_unpackhi_epi32(v[14], v[15]);\n\n v[0] = _mm512_unpacklo_epi64(v1[0], v1[2]);\n v[1] = _mm512_unpackhi_epi64(v1[0], v1[2]);\n v[2] = _mm512_unpacklo_epi64(v1[1], v1[3]);\n v[3] = _mm512_unpackhi_epi64(v1[1], v1[3]);\n v[4] = _mm512_unpacklo_epi64(v1[4], v1[6]);\n v[5] = _mm512_unpackhi_epi64(v1[4], v1[6]);\n v[6] = _mm512_unpacklo_epi64(v1[5], v1[7]);\n v[7] = _mm512_unpackhi_epi64(v1[5], v1[7]);\n v[8] = _mm512_unpacklo_epi64(v1[8], v1[10]);\n v[9] = _mm512_unpackhi_epi64(v1[8], v1[10]);\n v[10] = _mm512_unpacklo_epi64(v1[9], v1[11]);\n v[11] = _mm512_unpackhi_epi64(v1[9], v1[11]);\n v[12] = _mm512_unpacklo_epi64(v1[12], v1[14]);\n v[13] = _mm512_unpackhi_epi64(v1[12], v1[14]);\n v[14] = _mm512_unpacklo_epi64(v1[13], v1[15]);\n v[15] = _mm512_unpackhi_epi64(v1[13], v1[15]);\n\n v1[0] = _mm512_shuffle_i32x4(v[0], v[4], 0x88);\n v1[1] = _mm512_shuffle_i32x4(v[1], v[5], 0x88);\n v1[2] = _mm512_shuffle_i32x4(v[2], v[6], 0x88);\n v1[3] = _mm512_shuffle_i32x4(v[3], v[7], 0x88);\n v1[4] = _mm512_shuffle_i32x4(v[0], v[4], 0xdd);\n v1[5] = _mm512_shuffle_i32x4(v[1], v[5], 0xdd);\n v1[6] = _mm512_shuffle_i32x4(v[2], v[6], 0xdd);\n v1[7] = _mm512_shuffle_i32x4(v[3], v[7], 0xdd);\n v1[8] = _mm512_shuffle_i32x4(v[8], v[12], 0x88);\n v1[9] = _mm512_shuffle_i32x4(v[9], v[13], 0x88);\n v1[10] = _mm512_shuffle_i32x4(v[10], v[14], 0x88);\n v1[11] = _mm512_shuffle_i32x4(v[11], v[15], 0x88);\n v1[12] = _mm512_shuffle_i32x4(v[8], v[12], 0xdd);\n v1[13] = _mm512_shuffle_i32x4(v[9], v[13], 0xdd);\n v1[14] = _mm512_shuffle_i32x4(v[10], v[14], 0xdd);\n v1[15] = _mm512_shuffle_i32x4(v[11], v[15], 0xdd);\n\n v[0] = _mm512_shuffle_i32x4(v1[0], v1[8], 0x88);\n v[1] = _mm512_shuffle_i32x4(v1[1], v1[9], 0x88);\n v[2] = _mm512_shuffle_i32x4(v1[2], v1[10], 0x88);\n v[3] = _mm512_shuffle_i32x4(v1[3], v1[11], 0x88);\n v[4] = _mm512_shuffle_i32x4(v1[4], v1[12], 0x88);\n v[5] = _mm512_shuffle_i32x4(v1[5], v1[13], 0x88);\n v[6] = _mm512_shuffle_i32x4(v1[6], v1[14], 0x88);\n v[7] = _mm512_shuffle_i32x4(v1[7], v1[15], 0x88);\n v[8] = _mm512_shuffle_i32x4(v1[0], v1[8], 0xdd);\n v[9] = _mm512_shuffle_i32x4(v1[1], v1[9], 0xdd);\n v[10] = _mm512_shuffle_i32x4(v1[2], v1[10], 0xdd);\n v[11] = _mm512_shuffle_i32x4(v1[3], v1[11], 0xdd);\n v[12] = _mm512_shuffle_i32x4(v1[4], v1[12], 0xdd);\n v[13] = _mm512_shuffle_i32x4(v1[5], v1[13], 0xdd);\n v[14] = _mm512_shuffle_i32x4(v1[6], v1[14], 0xdd);\n v[15] = _mm512_shuffle_i32x4(v1[7], v1[15], 0xdd);\n}\n\nvoid quantize_row_q8_K_vnni(const float * RESTRICT x, void * RESTRICT vy, int64_t k) {\n assert(k % QK_K == 0);\n const int KB = k / QK_K;\n constexpr int kVecs = QK_K / 16;\n\n block_q8_K * y = reinterpret_cast(vy);\n\n // hold 16 float vecs from x\n __m512 v[kVecs];\n\n // hold the quants vecs\n __m512i vq[kVecs / 4];\n\n // hold the packed quants vecs\n __m512i vq_packed[kVecs / 4];\n\n const __m512 signBit = _mm512_set1_ps(-0.f);\n\n for (int i = 0; i < KB; ++i) {\n // Compute max(abs(e)) for the block\n __m512 vamax = _mm512_set1_ps(0.f);\n for (int j = 0; j < kVecs; ++j) {\n v[j] = _mm512_loadu_ps(x); x += 16;\n vamax = _mm512_max_ps(vamax, _mm512_andnot_ps(signBit, v[j]));\n }\n const float amax = _mm512_reduce_max_ps(vamax);\n\n // Quantize these floats\n const float iscale = 127.f / amax;\n y[i].d = GGML_CPU_FP32_TO_FP16(1 / iscale);\n const float id = ( amax != 0.0f ) ? iscale : 0.f;\n const __m512 vscale = _mm512_set1_ps(id);\n\n // Apply multiplier and round to nearest integer\n for (int j = 0; j < kVecs; ++j) {\n v[j] = _mm512_mul_ps(v[j], vscale);\n v[j] = _mm512_roundscale_ps(v[j], (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC));\n }\n\n // Pack to epi8 vecs\n for (int j = 0; j < kVecs / 4; ++j) {\n __m128i q8_0 = _mm512_cvtepi32_epi8(_mm512_cvtps_epi32(v[j * 4 + 0]));\n __m128i q8_1 = _mm512_cvtepi32_epi8(_mm512_cvtps_epi32(v[j * 4 + 1]));\n __m128i q8_2 = _mm512_cvtepi32_epi8(_mm512_cvtps_epi32(v[j * 4 + 2]));\n __m128i q8_3 = _mm512_cvtepi32_epi8(_mm512_cvtps_epi32(v[j * 4 + 3]));\n\n __m256i q8_01 = _mm256_insertf128_si256(_mm256_castsi128_si256(q8_0), (q8_1), 1);\n __m256i q8_23 = _mm256_insertf128_si256(_mm256_castsi128_si256(q8_2), (q8_3), 1);\n\n vq[j] = _mm512_inserti32x8(_mm512_castsi256_si512(q8_01), q8_23, 1);\n _mm512_storeu_si512((__m512i *)(y[i].qs + j * 64), vq[j]);\n }\n\n // Compute the bsums with vnni\n transpose_16x4_32bit(vq, vq_packed);\n\n const __m512i one = _mm512_set1_epi8(1);\n __m512i sum = _mm512_setzero_si512();\n for (int k = 0; k < 4; ++k) {\n sum = _mm512_dpbusd_epi32(sum, one, vq_packed[k]);\n }\n _mm256_storeu_si256((__m256i *)(y[i].bsums), _mm512_cvtepi32_epi16(sum));\n }\n}\n\n// quantize A from float to `vec_dot_type`\ntemplate \ninline void from_float(const float * x, char * vy, int64_t k);\n\ntemplate <>\ninline void from_float(const float * x, char * vy, int64_t k) {\n quantize_row_q8_0(x, (block_q8_0 *)vy, k);\n}\n\ntemplate <>\ninline void from_float(const float * x, char * vy, int64_t k) {\n quantize_row_q8_1(x, (block_q8_1 *)vy, k);\n}\n\ntemplate <>\ninline void from_float(const float * x, char * vy, int64_t k) {\n#if 1\n // TODO: this is reference impl!\n quantize_row_q8_K_ref(x, (block_q8_K *)vy, k);\n#else\n quantize_row_q8_K_vnni(x, vy, k);\n#endif\n}\n\n// load A from memory to array when nrows can not fill in whole tile\nvoid unpack_A(int8_t * RESTRICT tile, const block_q8_0 * RESTRICT A, int lda, int nr) {\n assert(nr != TILE_M);\n for (int m = 0; m < nr; ++m) {\n const __m256i v = _mm256_loadu_si256((const __m256i *)(A[m * lda].qs));\n _mm256_storeu_si256((__m256i *)(tile + m * TILE_K), v);\n }\n}\n\nvoid unpack_A(int8_t * RESTRICT tile, const block_q8_1 * RESTRICT A, int lda, int nr) {\n assert(nr != TILE_M);\n for (int m = 0; m < nr; ++m) {\n const __m256i v = _mm256_loadu_si256((const __m256i *)(A[m * lda].qs));\n _mm256_storeu_si256((__m256i *)(tile + m * TILE_K), v);\n }\n}\n\ntemplate \nvoid unpack_A(int8_t * RESTRICT tile, const block_q8_K * RESTRICT A, int lda, int k, int nr) {\n assert(nr <= TILE_M);\n for (int m = 0; m < nr; ++m) {\n const __m256i v = _mm256_loadu_si256((const __m256i *)(A[m * lda].qs + k * 32));\n _mm256_storeu_si256((__m256i *)(tile + m * TILE_K), v);\n }\n}\n\ntemplate <>\nvoid unpack_A(int8_t * RESTRICT tile, const block_q8_K * RESTRICT A, int lda, int k, int nr) {\n assert(nr <= TILE_M);\n // zero padding k from 16 to 32, so that we don't have to re-config amx\n const __m128i zero = _mm_setzero_si128();\n for (int m = 0; m < nr; ++m) {\n const __m128i v = _mm_loadu_si128((const __m128i *)(A[m * lda].qs + k * 16));\n const __m256i r = _mm256_insertf128_si256(_mm256_castsi128_si256(v), zero, 1);\n _mm256_storeu_si256((__m256i *)(tile + m * TILE_K), r);\n }\n}\n\n#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)\ninline __m256i bytes_from_nibbles_32(const uint8_t * rsi) {\n const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);\n const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);\n const __m256i lowMask = _mm256_set1_epi8(0xF);\n return _mm256_and_si256(lowMask, bytes);\n}\n\n// used for block_q4_K\ninline __m512i bytes_from_nibbles_64(const uint8_t * rsi) {\n const __m256i tmp = _mm256_loadu_si256((const __m256i *)rsi);\n const __m256i lowMask = _mm256_set1_epi8(0xF);\n const __m256i q4l = _mm256_and_si256(tmp, lowMask);\n const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(tmp, 4), lowMask);\n return _mm512_inserti32x8(_mm512_castsi256_si512(q4l), q4h, 1);\n}\n\n// used for block_q5_K\ninline __m512i bytes_from_nibbles_64(const uint8_t * qs, const uint8_t * qh, int k) {\n const __m256i lowMask = _mm256_set1_epi8(0xF);\n __m256i hmask = _mm256_set1_epi8(1);\n hmask = _mm256_slli_epi16(hmask, k);\n\n const __m256i q5bits = _mm256_loadu_si256((const __m256i *)qs);\n const __m256i hbits = _mm256_loadu_si256((const __m256i *)qh);\n\n const __m256i q5l_0 = _mm256_and_si256(q5bits, lowMask);\n const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), k + 0), 4);\n const __m256i q5_0 = _mm256_add_epi8(q5l_0, q5h_0);\n hmask = _mm256_slli_epi16(hmask, 1);\n\n const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), lowMask);\n const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), k + 1), 4);\n const __m256i q5_1 = _mm256_add_epi8(q5l_1, q5h_1);\n\n return _mm512_inserti32x8(_mm512_castsi256_si512(q5_0), q5_1, 1);\n}\n\n// used for block_q6_K\ninline void bytes_from_nibbles_128(__m512i& r0, __m512i& r1, const uint8_t * qs, const uint8_t * qh) {\n const __m256i m4 = _mm256_set1_epi8(0xF);\n const __m256i m2 = _mm256_set1_epi8(0x3);\n\n const __m256i q6bits1 = _mm256_loadu_si256((const __m256i *)qs);\n const __m256i q6bits2 = _mm256_loadu_si256((const __m256i *)(qs + 32));\n const __m256i q6bitsH = _mm256_loadu_si256((const __m256i *)qh);\n\n const __m256i q6h_0 = _mm256_slli_epi16(_mm256_and_si256( q6bitsH, m2), 4);\n const __m256i q6h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q6bitsH, 2), m2), 4);\n const __m256i q6h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q6bitsH, 4), m2), 4);\n const __m256i q6h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q6bitsH, 6), m2), 4);\n\n const __m256i q6_0 = _mm256_or_si256(_mm256_and_si256(q6bits1, m4), q6h_0);\n const __m256i q6_1 = _mm256_or_si256(_mm256_and_si256(q6bits2, m4), q6h_1);\n const __m256i q6_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q6bits1, 4), m4), q6h_2);\n const __m256i q6_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q6bits2, 4), m4), q6h_3);\n\n r0 = _mm512_inserti32x8(_mm512_castsi256_si512(q6_0), q6_1, 1);\n r1 = _mm512_inserti32x8(_mm512_castsi256_si512(q6_2), q6_3, 1);\n}\n\ninline __m512i packNibbles(__m512i r0, __m512i r1) {\n return _mm512_or_si512(r0, _mm512_slli_epi16(r1, 4));\n}\n\ntemplate \ninline void pack_qs(void * RESTRICT packed_B, const TB * RESTRICT B, int KB) {\n int8_t tmp[8 * 64];\n __m256i v[8], v2[8];\n for (int n = 0; n < 8; ++n) {\n v[n] = bytes_from_nibbles_32(B[n * KB].qs);\n }\n transpose_8x8_32bit(v, v2);\n for (int n = 0; n < 8; ++n) {\n _mm256_storeu_si256((__m256i *)(tmp + n * 64), v2[n]);\n }\n for (int n = 0; n < 8; ++n) {\n v[n] = bytes_from_nibbles_32(B[(n + 8) * KB].qs);\n }\n transpose_8x8_32bit(v, v2);\n for (int n = 0; n < 8; ++n) {\n _mm256_storeu_si256((__m256i *)(tmp + n * 64 + 32), v2[n]);\n }\n\n // pack again with 128 to fully utilize vector length\n for (int n = 0; n < 8; n += 2) {\n __m512i r0 = _mm512_loadu_si512((const __m512i *)(tmp + n * 64));\n __m512i r1 = _mm512_loadu_si512((const __m512i *)(tmp + n * 64 + 64));\n __m512i r1r0 = packNibbles(r0, r1);\n _mm512_storeu_si512((__m512i *)((char *)packed_B + n * 32), r1r0);\n }\n}\n\ntemplate <>\ninline void pack_qs(void * RESTRICT packed_B, const block_q8_0 * RESTRICT B, int KB) {\n __m256i v[8], v2[8];\n for (int n = 0; n < 8; ++n) {\n v[n] = _mm256_loadu_si256((const __m256i *)(B[n * KB].qs));\n }\n transpose_8x8_32bit(v, v2);\n for (int n = 0; n < 8; ++n) {\n _mm256_storeu_si256((__m256i *)((char *)packed_B + n * 64), v2[n]);\n }\n for (int n = 0; n < 8; ++n) {\n v[n] = _mm256_loadu_si256((const __m256i *)(B[(n + 8) * KB].qs));\n }\n transpose_8x8_32bit(v, v2);\n for (int n = 0; n < 8; ++n) {\n _mm256_storeu_si256((__m256i *)((char *)packed_B + n * 64 + 32), v2[n]);\n }\n}\n\ntemplate <>\ninline void pack_qs(void * RESTRICT packed_B, const block_q4_K * RESTRICT B, int KB) {\n __m512i v[16];\n // QK_K 256 with 8 groups, handle 2 groups at a time\n char * pb = (char *)packed_B;\n for (int k = 0; k < QK_K / 64; ++k) {\n // pack 2 groups { n, g, k} to {g, k/4, 4n}\n // e.g. {16, 2, 32} to {2, 8, 64}\n for (int n = 0; n < TILE_N; ++n) {\n v[n] = bytes_from_nibbles_64(B[n * KB].qs + k * 32);\n }\n\n transpose_16x16_32bit(v);\n\n // pack again with 128 to fully utilize vector length\n for (int n = 0; n < TILE_N; n += 2) {\n _mm512_storeu_si512((__m512i *)pb, packNibbles(v[n], v[n + 1]));\n pb += 64;\n }\n }\n}\n\ntemplate <>\ninline void pack_qs(void * RESTRICT packed_B, const block_q5_K * RESTRICT B, int KB) {\n __m512i v[16];\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n // QK_K 256 with 8 groups, handle 2 groups at a time\n char * pb = (char *)packed_B;\n char * ph = (char *)packed_B + (QK_K / 2) * TILE_N;\n for (int k = 0; k < QK_K / 64; ++k) {\n // pack 2 groups { n, g, k} to {g, k/4, 4n}\n // e.g. {16, 2, 32} to {2, 8, 64}\n for (int n = 0; n < TILE_N; ++n) {\n v[n] = bytes_from_nibbles_64(B[n * KB].qs + k * 32, B[n * KB].qh, /* group */2 * k);\n }\n\n transpose_16x16_32bit(v);\n\n // 1. pack lower 4bits with 2 groups\n for (int n = 0; n < TILE_N; n += 2) {\n // get lower 4 bits\n const __m512i r0 = _mm512_and_si512(v[n], lowMask);\n const __m512i r1 = _mm512_and_si512(v[n + 1], lowMask);\n _mm512_storeu_si512((__m512i *)pb, packNibbles(r0, r1)); pb += 64;\n }\n\n // 2. pack higher 1bit with 2 groups\n const __m512i hmask = _mm512_set1_epi8(0x10);\n for (int g = 0; g < 2; ++g) {\n __m512i hbits = _mm512_setzero_si512();\n hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 8 + 0], hmask), 4));\n hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 8 + 1], hmask), 3));\n hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 8 + 2], hmask), 2));\n hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 8 + 3], hmask), 1));\n hbits = _mm512_add_epi8(hbits, _mm512_and_si512(v[g * 8 + 4], hmask) );\n hbits = _mm512_add_epi8(hbits, _mm512_slli_epi16(_mm512_and_si512(v[g * 8 + 5], hmask), 1));\n hbits = _mm512_add_epi8(hbits, _mm512_slli_epi16(_mm512_and_si512(v[g * 8 + 6], hmask), 2));\n hbits = _mm512_add_epi8(hbits, _mm512_slli_epi16(_mm512_and_si512(v[g * 8 + 7], hmask), 3));\n _mm512_storeu_si512((__m512i *)ph, hbits); ph += 64;\n }\n }\n}\n\ntemplate <>\ninline void pack_qs(void * RESTRICT packed_B, const block_q6_K * RESTRICT B, int KB) {\n __m512i v[32];\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n // QK_K 256 with 8 groups, handle 4 groups at a time\n char * pb = (char *)packed_B;\n char * ph = (char *)packed_B + (QK_K / 2) * TILE_N;\n for (int k = 0; k < QK_K / 128; ++k) {\n for (int n = 0; n < TILE_N; ++n) {\n bytes_from_nibbles_128(v[n], v[n + 16], B[n * KB].ql + k * 64, B[n * KB].qh + k * 32);\n }\n\n // top half: group 0,1 or 4,5; bottom half: group 2,3 or 6,7\n transpose_16x16_32bit(v);\n transpose_16x16_32bit(v + 16);\n\n // 1. pack lower 4bits with 4 groups\n for (int n = 0; n < 32; n += 2) {\n const __m512i r0 = _mm512_and_si512(v[n], lowMask);\n const __m512i r1 = _mm512_and_si512(v[n + 1], lowMask);\n _mm512_storeu_si512((__m512i *)pb, packNibbles(r0, r1)); pb += 64;\n }\n\n // 2. pack higher 2bit with 4 groups\n const __m512i hmask = _mm512_set1_epi8(0x30);\n for (int g = 0; g < 8; ++g) {\n __m512i hbits = _mm512_setzero_si512();\n hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 4 + 0], hmask), 4));\n hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 4 + 1], hmask), 2));\n hbits = _mm512_add_epi8(hbits, _mm512_and_si512(v[g * 4 + 2], hmask) );\n hbits = _mm512_add_epi8(hbits, _mm512_slli_epi16(_mm512_and_si512(v[g * 4 + 3], hmask), 2));\n _mm512_storeu_si512((__m512i *)ph, hbits); ph += 64;\n }\n }\n}\n\ntemplate <>\ninline void pack_qs(void * RESTRICT packed_B, const block_iq4_xs * RESTRICT B, int KB) {\n __m512i v[16];\n char * pb = (char *)packed_B;\n for (int k = 0; k < QK_K / 64; ++k) {\n for (int n = 0; n < TILE_N; ++n) {\n __m256i r0 = bytes_from_nibbles_32(B[n * KB].qs + k * 32 + 0);\n __m256i r1 = bytes_from_nibbles_32(B[n * KB].qs + k * 32 + 16);\n v[n] = _mm512_inserti32x8(_mm512_castsi256_si512(r0), r1, 1);\n }\n\n transpose_16x16_32bit(v);\n\n // pack again with 128 to fully utilize vector length\n for (int n = 0; n < TILE_N; n += 2) {\n _mm512_storeu_si512((__m512i *)pb, packNibbles(v[n], v[n + 1]));\n pb += 64;\n }\n }\n}\n\n// pack B to vnni formats in 4bits or 8 bits\nvoid pack_B(void * RESTRICT packed_B, const block_q4_0 * RESTRICT B, int KB) {\n pack_qs(packed_B, B, KB);\n ggml_half * d0 = reinterpret_cast((char *)packed_B + TILE_N * TILE_K / 2);\n for (int n = 0; n < TILE_N; ++n) {\n d0[n] = B[n * KB].d;\n }\n}\n\nvoid pack_B(void * RESTRICT packed_B, const block_q4_1 * RESTRICT B, int KB) {\n pack_qs(packed_B, B, KB);\n ggml_half * d0 = reinterpret_cast((char *)packed_B + TILE_N * TILE_K / 2);\n ggml_half * m0 = d0 + TILE_N;\n for (int n = 0; n < TILE_N; ++n) {\n d0[n] = B[n * KB].d;\n m0[n] = B[n * KB].m;\n }\n}\n\ninline void s8s8_compensation(void * RESTRICT packed_B) {\n // packed_B layout:\n // quants {TILE_N, TILEK} int8_t\n // d0 {TILE_N} ggml_half\n // comp {TILE_N} int32_t\n const int offset = TILE_N * TILE_K + TILE_N * sizeof(ggml_half);\n __m512i vcomp = _mm512_setzero_si512();\n const __m512i off = _mm512_set1_epi8(static_cast(0x80));\n for (int k = 0; k < 8; ++k) {\n __m512i vb = _mm512_loadu_si512((const __m512i *)((const char *)packed_B + k * 64));\n vcomp = _mm512_dpbusd_epi32(vcomp, off, vb);\n }\n _mm512_storeu_si512((__m512i *)((char *)(packed_B) + offset), vcomp);\n}\n\nvoid pack_B(void * RESTRICT packed_B, const block_q8_0 * RESTRICT B, int KB) {\n pack_qs(packed_B, B, KB);\n ggml_half * d0 = reinterpret_cast((char *)packed_B + TILE_N * TILE_K);\n for (int n = 0; n < TILE_N; ++n) {\n d0[n] = B[n * KB].d;\n }\n s8s8_compensation(packed_B);\n}\n\n// convert 8 * {min, scale} from int6 to int8\ninline void unpack_mins_and_scales(const uint8_t * scales, uint32_t * utmp) {\n const uint32_t kmask1 = 0x3f3f3f3f;\n const uint32_t kmask2 = 0x0f0f0f0f;\n const uint32_t kmask3 = 0x03030303;\n\n memcpy(utmp, scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n}\n\n// packed_B layout:\n// quants {8, TILE_N, 16} uint8\n// scales {8, TILE_N} uint8\n// mins {8, TILE_N} uint8\n// d {TILE_N} ggml_half\n// dmin {TILE_N} ggml_half\nvoid pack_B(void * RESTRICT packed_B, const block_q4_K * RESTRICT B, int KB) {\n pack_qs(packed_B, B, KB);\n\n uint8_t * scales = reinterpret_cast((char *)packed_B + (QK_K / 2) * TILE_N);\n uint8_t * mins = scales + 8 * TILE_N;\n ggml_half * d = reinterpret_cast(mins + 8 * TILE_N);\n ggml_half * dmin = d + TILE_N;\n\n union {\n uint32_t u32[4];\n uint8_t u8[16];\n } s;\n\n for (int n = 0; n < TILE_N; ++n) {\n unpack_mins_and_scales(B[n * KB].scales, s.u32);\n for (int k = 0; k < 8; ++k) {\n scales[k * TILE_N + n] = s.u8[k];\n mins[(k >> 1) * TILE_N * 2 + n * 2 + (k & 0x1)] = s.u8[k + 8];\n }\n d[n] = B[n * KB].d;\n dmin[n] = B[n * KB].dmin;\n }\n}\n\n// packed_B layout:\n// quants {8, TILE_N, 16} uint8\n// qh {8, TILE_N, 4} uint8\n// scales {8, TILE_N} uint8\n// mins {8, TILE_N} uint8\n// d {TILE_N} ggml_half\n// dmin {TILE_N} ggml_half\nvoid pack_B(void * RESTRICT packed_B, const block_q5_K * RESTRICT B, int KB) {\n pack_qs(packed_B, B, KB);\n\n uint8_t * scales = reinterpret_cast((char *)packed_B + (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N);\n uint8_t * mins = scales + 8 * TILE_N;\n ggml_half * d = reinterpret_cast(mins + 8 * TILE_N);\n ggml_half * dmin = d + TILE_N;\n\n union {\n uint32_t u32[4];\n uint8_t u8[16];\n } s;\n\n for (int n = 0; n < TILE_N; ++n) {\n unpack_mins_and_scales(B[n * KB].scales, s.u32);\n for (int k = 0; k < 8; ++k) {\n scales[k * TILE_N + n] = s.u8[k];\n mins[(k >> 1) * TILE_N * 2 + n * 2 + (k & 0x1)] = s.u8[k + 8];\n }\n d[n] = B[n * KB].d;\n dmin[n] = B[n * KB].dmin;\n }\n}\n\n// packed_B layout:\n// quants {16, TILE_N, 8} uint8\n// qh {16, TILE_N, 4} uint8\n// scales {16, TILE_N} uint8\n// d {TILE_N} ggml_half\nvoid pack_B(void * RESTRICT packed_B, const block_q6_K * RESTRICT B, int KB) {\n pack_qs(packed_B, B, KB);\n\n uint8_t * scales = reinterpret_cast((char *)packed_B + (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N);\n ggml_half * d = reinterpret_cast(scales + 16 * TILE_N);\n for (int n = 0; n < TILE_N; ++n) {\n const int8_t * ps = B[n * KB].scales;\n for (int k = 0; k < 16; ++k) {\n scales[k * TILE_N + n] = ps[k];\n }\n d[n] = B[n * KB].d;\n }\n}\n\n// packed_B layout:\n// quants {8, TILE_N, 16} uint8\n// scales {8, TILE_N} int8\n// d {TILE_N} ggml_half\nvoid pack_B(void * RESTRICT packed_B, const block_iq4_xs * RESTRICT B, int KB) {\n pack_qs(packed_B, B, KB);\n\n int8_t * scales = reinterpret_cast((char *)packed_B + (QK_K / 2) * TILE_N);\n ggml_half * d = reinterpret_cast(scales + 8 * TILE_N);\n\n // pack the scales\n for (int n = 0; n < TILE_N; ++n) {\n uint16_t sh = B[n * KB].scales_h;\n for (int k = 0; k < 8; k += 2) {\n const int16_t ls1 = ((B[n * KB].scales_l[k / 2] & 0xf) | ((sh << 4) & 0x30)) - 32;\n const int16_t ls2 = ((B[n * KB].scales_l[k / 2] >> 4) | ((sh << 2) & 0x30)) - 32;\n scales[(k + 0) * TILE_N + n] = ls1;\n scales[(k + 1) * TILE_N + n] = ls2;\n sh >>= 4;\n }\n d[n] = B[n * KB].d;\n }\n}\n\ntemplate>\nvoid unpack_B(packed_B_t * RESTRICT tile, const void * RESTRICT packed_B) {\n GGML_UNUSED(tile);\n GGML_UNUSED(packed_B);\n}\n\ntemplate <>\nvoid unpack_B(int8_t * RESTRICT tile, const void * RESTRICT packed_B) {\n const __m512i off = _mm512_set1_epi8(8);\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n for (int n = 0; n < 8; n += 2) {\n __m512i bytes = _mm512_loadu_si512((const __m512i *)((const char *)packed_B + n * 32));\n const __m512i r0 = _mm512_sub_epi8(_mm512_and_si512(bytes, lowMask), off);\n const __m512i r1 = _mm512_sub_epi8(_mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask), off);\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 0), r0);\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n }\n}\n\ntemplate <>\nvoid unpack_B(uint8_t * RESTRICT tile, const void * RESTRICT packed_B) {\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n for (int n = 0; n < 8; n += 2) {\n __m512i bytes = _mm512_loadu_si512((const __m512i *)((const char *)packed_B + n * 32));\n const __m512i r0 = _mm512_and_si512(bytes, lowMask);\n const __m512i r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 0), r0);\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n }\n}\n\n// packed_B_t for QKK is int8_t\ntemplate \nvoid unpack_B(int8_t * RESTRICT tile, const void * RESTRICT packed_B, int k) {\n const int packed_B_group_size = QK_K / 2 * TILE_N / 8;\n const char * packed_B_group = (const char *)packed_B + k * packed_B_group_size;\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n for (int n = 0; n < 8; n += 2) {\n __m512i bytes = _mm512_loadu_si512(packed_B_group + n * 32);\n const __m512i r0 = _mm512_and_si512(bytes, lowMask);\n const __m512i r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 0), r0);\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n }\n}\n\ntemplate <>\nvoid unpack_B(int8_t * RESTRICT tile, const void * RESTRICT packed_B, int k) {\n // lower 4bits, stride 256 bytes\n const int packed_l4_group_size = QK_K / 2 * TILE_N / 8;\n const char * pb = (const char *)packed_B + k * packed_l4_group_size;\n\n // higher 1bit, stride 64 bytes\n const int packed_h1_group_size = QK_K / 8 * TILE_N / 8;\n const char * ph = (const char *)packed_B + (QK_K / 2) * TILE_N + k * packed_h1_group_size;\n const __m512i hbits = _mm512_loadu_si512(ph);\n\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n __m512i hmask0 = _mm512_set1_epi8(0x1);\n __m512i hmask1 = _mm512_set1_epi8(0x2);\n\n for (int n = 0; n < 8; n += 2) {\n __m512i bytes = _mm512_loadu_si512(pb + n * 32);\n __m512i r0 = _mm512_and_si512(bytes, lowMask);\n __m512i r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n __m512i h0 = _mm512_slli_epi16(_mm512_srli_epi16(_mm512_and_si512(hbits, hmask0), n), 4);\n __m512i h1 = _mm512_slli_epi16(_mm512_srli_epi16(_mm512_and_si512(hbits, hmask1), n + 1), 4);\n\n hmask0 = _mm512_slli_epi16(hmask0, 2);\n hmask1 = _mm512_slli_epi16(hmask1, 2);\n r0 = _mm512_add_epi8(r0, h0);\n r1 = _mm512_add_epi8(r1, h1);\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 0), r0);\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n }\n}\n\ntemplate <>\nvoid unpack_B(int8_t * RESTRICT tile, const void * RESTRICT packed_B, int k) {\n // lower 4bits, stride 128 bytes\n const int packed_l4_group_size = QK_K / 2 * TILE_N / 16;\n const char * pb = (const char *)packed_B + k * packed_l4_group_size;\n\n // higher 2bits, stride 64 bytes\n const int packed_h2_group_size = QK_K / 4 * TILE_N / 16;\n const char * ph = (const char *)packed_B + (QK_K / 2) * TILE_N + k * packed_h2_group_size;\n const __m512i hbits = _mm512_loadu_si512(ph);\n\n const __m512i off = _mm512_set1_epi8(32);\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n __m512i hmask0 = _mm512_set1_epi8(0x3); // 0011\n __m512i hmask1 = _mm512_set1_epi8(0xC); // 1100\n\n // notes: skip zero padding from row4 to row7 as we have done so in `unpack_A`\n __m512i bytes = _mm512_loadu_si512(pb);\n __m512i r0 = _mm512_and_si512(bytes, lowMask);\n __m512i r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n __m512i h0 = _mm512_slli_epi16(_mm512_and_si512(hbits, hmask0), 4);\n __m512i h1 = _mm512_slli_epi16(_mm512_and_si512(hbits, hmask1), 2);\n _mm512_storeu_si512((__m512i *)(tile + 0), _mm512_sub_epi8(_mm512_add_epi8(r0, h0), off));\n _mm512_storeu_si512((__m512i *)(tile + 64), _mm512_sub_epi8(_mm512_add_epi8(r1, h1), off));\n\n hmask0 = _mm512_slli_epi16(hmask0, 4);\n hmask1 = _mm512_slli_epi16(hmask1, 4);\n\n bytes = _mm512_loadu_si512(pb + 64);\n r0 = _mm512_and_si512(bytes, lowMask);\n r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n h0 = _mm512_and_si512(hbits, hmask0);\n h1 = _mm512_srli_epi16(_mm512_and_si512(hbits, hmask1), 2);\n _mm512_storeu_si512((__m512i *)(tile + 128), _mm512_sub_epi8(_mm512_add_epi8(r0, h0), off));\n _mm512_storeu_si512((__m512i *)(tile + 192), _mm512_sub_epi8(_mm512_add_epi8(r1, h1), off));\n}\n\ntemplate <>\nvoid unpack_B(int8_t * RESTRICT tile, const void * RESTRICT packed_B, int k) {\n static const __m512i values128 = _mm512_set_epi8(\n 113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n 113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n 113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n 113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127\n );\n\n const int packed_B_group_size = QK_K / 2 * TILE_N / 8;\n const char * pb = (const char *)packed_B + k * packed_B_group_size;\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n for (int n = 0; n < 8; n += 2) {\n __m512i bytes = _mm512_loadu_si512(pb + n * 32);\n const __m512i r0 = _mm512_shuffle_epi8(values128, _mm512_and_si512(bytes, lowMask));\n const __m512i r1 = _mm512_shuffle_epi8(values128, _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask));\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 0), r0);\n _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n }\n}\n\ntemplate \nstruct acc_C {};\n\ntemplate \nstruct acc_C {\n static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_0 * A, int lda, const void * packed_B, int nr) {\n const int offset = TILE_N * TILE_K / 2;\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));\n\n for (int m = 0; m < nr; ++m) {\n const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));\n const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n __m512 vsum;\n if (is_acc) {\n vsum = _mm512_loadu_ps(C + m * ldc);\n } else {\n vsum = _mm512_set1_ps(0.f);\n }\n vsum = _mm512_fmadd_ps(vtile, _mm512_mul_ps(vd0, vd1), vsum);\n _mm512_storeu_ps(C + m * ldc, vsum);\n }\n }\n};\n\ntemplate \nstruct acc_C {\n static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_1 * A, int lda, const void * packed_B, int nr) {\n const int offset = TILE_N * TILE_K / 2;\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));\n const __m512 vm0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset + TILE_N * sizeof(ggml_half))));\n\n for (int m = 0; m < nr; ++m) {\n const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));\n const __m512 vs1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].s));\n const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n __m512 vsum;\n if (is_acc) {\n vsum = _mm512_loadu_ps(C + m * ldc);\n } else {\n vsum = _mm512_set1_ps(0.f);\n }\n vsum = _mm512_fmadd_ps(vtile, _mm512_mul_ps(vd0, vd1), vsum);\n vsum = _mm512_fmadd_ps(vm0, vs1, vsum);\n _mm512_storeu_ps(C + m * ldc, vsum);\n }\n }\n};\n\ntemplate \nstruct acc_C {\n static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_0 * A, int lda, const void * packed_B, int nr) {\n const int offset = TILE_N * TILE_K;\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));\n\n for (int m = 0; m < nr; ++m) {\n const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));\n const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n __m512 vsum;\n if (is_acc) {\n vsum = _mm512_loadu_ps(C + m * ldc);\n } else {\n vsum = _mm512_set1_ps(0.f);\n }\n vsum = _mm512_fmadd_ps(vtile, _mm512_mul_ps(vd0, vd1), vsum);\n _mm512_storeu_ps(C + m * ldc, vsum);\n }\n }\n};\n\ntemplate \nstruct acc_C {\n static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_K * A, int lda, const void * packed_B, int nr) {\n const uint8_t * scales = reinterpret_cast((const char *)packed_B + (QK_K / 2) * TILE_N);\n const uint8_t * mins = scales + 8 * TILE_N;\n const ggml_half * d0 = reinterpret_cast(mins + 8 * TILE_N);\n const ggml_half * dmin = d0 + TILE_N;\n\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)d0));\n const __m512 vdmin = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)dmin));\n\n for (int m = 0; m < nr; ++m) {\n const float d1 = A[m * lda].d;\n const __m512 vd = _mm512_mul_ps(_mm512_set1_ps(d1), vd0);\n const __m512 vdm = _mm512_mul_ps(_mm512_set1_ps(-d1), vdmin);\n const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n __m512 vsum;\n if (is_acc) {\n vsum = _mm512_loadu_ps(C + m * ldc);\n } else {\n vsum = _mm512_set1_ps(0.f);\n }\n\n const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[m * lda].bsums);\n const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n\n __m512i acc_m = _mm512_setzero_si512();\n for (int k = 0; k < 4; ++k) {\n __m512i vmask = _mm512_set1_epi32(k);\n __m512i va = _mm512_permutexvar_epi32(vmask, _mm512_castsi128_si512(q8s));\n __m512i vb = _mm512_cvtepi8_epi16(_mm256_loadu_si256((const __m256i *)(mins + k * 32)));\n acc_m = _mm512_dpwssds_epi32(acc_m, va, vb);\n }\n\n vsum = _mm512_fmadd_ps(vtile, vd, vsum);\n vsum = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc_m), vdm, vsum);\n _mm512_storeu_ps(C + m * ldc, vsum);\n }\n }\n};\n\ntemplate \nstruct acc_C {\n static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_K * A, int lda, const void * packed_B, int nr) {\n const uint8_t * scales = reinterpret_cast((const char *)packed_B + (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N);\n const uint8_t * mins = scales + 8 * TILE_N;\n const ggml_half * d0 = reinterpret_cast(mins + 8 * TILE_N);\n const ggml_half * dmin = d0 + TILE_N;\n\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)d0));\n const __m512 vdmin = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)dmin));\n\n for (int m = 0; m < nr; ++m) {\n const float d1 = A[m * lda].d;\n const __m512 vd = _mm512_mul_ps(_mm512_set1_ps(d1), vd0);\n const __m512 vdm = _mm512_mul_ps(_mm512_set1_ps(-d1), vdmin);\n const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n __m512 vsum;\n if (is_acc) {\n vsum = _mm512_loadu_ps(C + m * ldc);\n } else {\n vsum = _mm512_set1_ps(0.f);\n }\n\n const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[m * lda].bsums);\n const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n\n __m512i acc_m = _mm512_setzero_si512();\n for (int k = 0; k < 4; ++k) {\n __m512i vmask = _mm512_set1_epi32(k);\n __m512i va = _mm512_permutexvar_epi32(vmask, _mm512_castsi128_si512(q8s));\n __m512i vb = _mm512_cvtepi8_epi16(_mm256_loadu_si256((const __m256i *)(mins + k * 32)));\n acc_m = _mm512_dpwssds_epi32(acc_m, va, vb);\n }\n\n vsum = _mm512_fmadd_ps(vtile, vd, vsum);\n vsum = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc_m), vdm, vsum);\n _mm512_storeu_ps(C + m * ldc, vsum);\n }\n }\n};\n\ntemplate \nstruct acc_C {\n static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_K * A, int lda, const void * packed_B, int nr) {\n const uint8_t * scales = reinterpret_cast((const char *)packed_B + (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N);\n const ggml_half * d0 = reinterpret_cast(scales + 16 * TILE_N);\n\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)d0));\n\n for (int m = 0; m < nr; ++m) {\n const float d1 = A[m * lda].d;\n const __m512 vd = _mm512_mul_ps(_mm512_set1_ps(d1), vd0);\n const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n __m512 vsum;\n if (is_acc) {\n vsum = _mm512_loadu_ps(C + m * ldc);\n } else {\n vsum = _mm512_set1_ps(0.f);\n }\n\n vsum = _mm512_fmadd_ps(vtile, vd, vsum);\n _mm512_storeu_ps(C + m * ldc, vsum);\n }\n }\n};\n\ntemplate \nstruct acc_C {\n static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_K * A, int lda, const void * packed_B, int nr) {\n const int8_t * scales = reinterpret_cast((const char *)packed_B + (QK_K / 2) * TILE_N);\n const ggml_half * d0 = reinterpret_cast(scales + 8 * TILE_N);\n\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)d0));\n\n for (int m = 0; m < nr; ++m) {\n const float d1 = A[m * lda].d;\n const __m512 vd = _mm512_mul_ps(_mm512_set1_ps(d1), vd0);\n const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n __m512 vsum;\n if (is_acc) {\n vsum = _mm512_loadu_ps(C + m * ldc);\n } else {\n vsum = _mm512_set1_ps(0.f);\n }\n\n vsum = _mm512_fmadd_ps(vtile, vd, vsum);\n _mm512_storeu_ps(C + m * ldc, vsum);\n }\n }\n};\n\ntemplate constexpr int get_quants_size();\ntemplate <> constexpr int get_quants_size() { return (QK_K / 2) * TILE_N; }\ntemplate <> constexpr int get_quants_size() { return (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N; }\ntemplate <> constexpr int get_quants_size() { return (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N; }\ntemplate <> constexpr int get_quants_size() { return (QK_K / 2) * TILE_N; }\n\n// used for QKK format\ntemplate ::value, int>::type = 0>\ninline void scale_C(const int32_t * RESTRICT tile, int32_t * RESTRICT sumi, const void * packed_B, int k, int nr) {\n const uint8_t * scales = reinterpret_cast((const char *)packed_B + get_quants_size());\n const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(scales + k * TILE_N)));\n\n for (int m = 0; m < nr; ++m) {\n __m512i vsumi;\n if (is_acc) {\n vsumi = _mm512_loadu_si512(sumi + m * TILE_N);\n } else {\n vsumi = _mm512_setzero_si512();\n }\n __m512i vtile = _mm512_loadu_si512(tile + m * TILE_N);\n vsumi = _mm512_add_epi32(vsumi, _mm512_mullo_epi32(vtile, vscale));\n _mm512_storeu_si512((__m512i *)(sumi + m * TILE_N), vsumi);\n }\n}\n\ntemplate \nstruct tinygemm_kernel_avx {\n static void apply(int K, const TA * RESTRICT A, const TB * RESTRICT B, TC * RESTRICT C, int ldc) {\n GGML_UNUSED(K);\n GGML_UNUSED(A);\n GGML_UNUSED(B);\n GGML_UNUSED(C);\n GGML_UNUSED(ldc);\n }\n};\n\ntemplate \nstruct tinygemm_kernel_avx {\n static void apply(int K, const float * RESTRICT A, const ggml_fp16_t * RESTRICT B, float * RESTRICT C, int ldc) {\n constexpr int ROWS = BLOCK_M;\n constexpr int COLS = BLOCK_N;\n assert(BLOCK_K == 16);\n\n __m512 va;\n __m512 vb[COLS];\n __m512 vc[ROWS * COLS];\n\n auto loadc = [&](auto idx) {\n vc[idx] = _mm512_setzero_ps();\n };\n Unroll{}(loadc);\n\n auto compute = [&](auto idx, auto k) {\n constexpr int row = idx / COLS;\n constexpr int col = idx % COLS;\n\n if constexpr (col == 0) {\n va = _mm512_loadu_ps(A + row * K + k);\n }\n if constexpr (row == 0) {\n vb[col] = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(B + col * K + k)));\n }\n vc[idx] = _mm512_fmadd_ps(va, vb[col], vc[idx]);\n };\n\n for (int k = 0; k < K; k += 16) {\n Unroll{}(compute, k);\n }\n\n auto storec = [&](auto idx) {\n constexpr int row = idx / COLS;\n constexpr int col = idx % COLS;\n C[row * ldc + col] = _mm512_reduce_add_ps(vc[idx]);\n };\n Unroll{}(storec);\n }\n};\n\n#define LAUNCH_TINYGEMM_KERNEL_AVX(MB_SIZE, NB_SIZE) \\\n tinygemm_kernel_avx::apply( \\\n K, (const float *)src1->data + src1_offset + mb_start * K, \\\n (const type *)src0->data + src0_offset + nb_start * K, \\\n (float *)dst->data + dst_offset + mb_start * ldc + nb_start, ldc)\n\n\n// re-organize in the format {NB, KB, TILE_SIZE}:\n#define PACKED_INDEX(n, k, KB, tile_size) (n * KB + k) * tile_size\n\ntemplate\nvoid convert_B_packed_format(void * RESTRICT packed_B, const TB * RESTRICT B, int N, int K) {\n const int NB = N / TILE_N;\n const int KB = K / BLOCK_K;\n const int TILE_SIZE = get_tile_size();\n\n // parallel on NB should be enough\n parallel_for(NB, [&](int begin, int end) {\n for (int n = begin; n < end; ++n) {\n for (int k = 0; k < KB; ++k) {\n int n0 = n * TILE_N;\n pack_B((char *)packed_B + PACKED_INDEX(n, k, KB, TILE_SIZE), &B[n0 * KB + k], KB);\n }\n }\n });\n}\n\ntemplate \nstruct tinygemm_kernel_vnni {};\n\ntemplate \nstruct tinygemm_kernel_vnni {\n static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n constexpr int COLS = BLOCK_N / 16;\n const int TILE_SIZE = TILE_N * sizeof(block_q4_0);\n\n const block_q8_0 * RESTRICT A = static_cast(_A);\n const char * RESTRICT B = static_cast(_B);\n\n __m512i va[8];\n __m512 vc[COLS];\n __m512 vd1;\n\n // sum of offsets, shared across COLS\n //\n // avx512-vnni does not have `_mm512_dpbssd_epi32`,\n // need to transform ss to us:\n // a * (b - 8) is equivalent to b * a - 8 * a\n // s u u u s u s\n //\n __m512i vcomp;\n\n const __m512i off = _mm512_set1_epi8(8);\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n auto loadc = [&](auto col) {\n vc[col] = _mm512_setzero_ps();\n };\n Unroll{}(loadc);\n\n auto compute = [&](auto col, auto i) {\n // load a and compute compensation\n if constexpr (col == 0) {\n const int32_t * a_ptr = reinterpret_cast(A[0 * KB + i].qs);\n vcomp = _mm512_setzero_si512();\n for (int k = 0; k < 8; ++k) {\n va[k] = _mm512_set1_epi32(a_ptr[k]);\n vcomp = _mm512_dpbusd_epi32(vcomp, off, va[k]);\n }\n vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));\n }\n\n // load b\n __m512i vsum = _mm512_setzero_si512();\n const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n for (int k = 0; k < 8; k += 2) {\n __m512i bytes = _mm512_loadu_si512((const __m512i *)(b_ptr + k * 32));\n __m512i vb0 = _mm512_and_si512(bytes, lowMask);\n vsum = _mm512_dpbusd_epi32(vsum, vb0, va[k + 0]);\n __m512i vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n vsum = _mm512_dpbusd_epi32(vsum, vb1, va[k + 1]);\n }\n const int offset = TILE_N * TILE_K / 2;\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset)));\n vsum = _mm512_sub_epi32(vsum, vcomp);\n\n vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(vsum), _mm512_mul_ps(vd0, vd1), vc[col]);\n };\n\n for (int i = 0; i < KB; ++i) {\n Unroll{}(compute, i);\n }\n\n //store to C\n auto storec = [&](auto col) {\n _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n };\n Unroll{}(storec);\n }\n};\n\ntemplate \nstruct tinygemm_kernel_vnni {\n static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n constexpr int COLS = BLOCK_N / 16;\n const int TILE_SIZE = TILE_N * sizeof(block_q4_1);\n\n const block_q8_1 * RESTRICT A = static_cast(_A);\n const char * RESTRICT B = static_cast(_B);\n\n __m512i va[8];\n __m512i vb[8];\n __m512 vc[COLS];\n __m512 vd1, vs1;\n\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n auto loadc = [&](auto col) {\n vc[col] = _mm512_setzero_ps();\n };\n Unroll{}(loadc);\n\n auto compute = [&](auto col, auto i) {\n // load a\n if constexpr (col == 0) {\n const int32_t * a_ptr = reinterpret_cast(A[0 * KB + i].qs);\n for (int k = 0; k < 8; ++k) {\n va[k] = _mm512_set1_epi32(a_ptr[k]);\n }\n vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));\n vs1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].s));\n }\n\n // load b\n const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n for (int k = 0; k < 8; k += 2) {\n __m512i bytes = _mm512_loadu_si512((const __m512i *)(b_ptr + k * 32));\n vb[k + 0] = _mm512_and_si512(bytes, lowMask);\n vb[k + 1] = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n }\n const int offset = TILE_N * TILE_K / 2;\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset)));\n const __m512 vm0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset + TILE_N * sizeof(ggml_half))));\n\n __m512i vsum = _mm512_setzero_si512();\n for (int k = 0; k < 8; ++k) {\n vsum = _mm512_dpbusd_epi32(vsum, vb[k], va[k]);\n }\n\n vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(vsum), _mm512_mul_ps(vd0, vd1), vc[col]);\n vc[col] = _mm512_fmadd_ps(vm0, vs1, vc[col]);\n };\n\n for (int i = 0; i < KB; ++i) {\n Unroll{}(compute, i);\n }\n\n //store to C\n auto storec = [&](auto col) {\n _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n };\n Unroll{}(storec);\n }\n};\n\ntemplate \nstruct tinygemm_kernel_vnni {\n static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n constexpr int COLS = BLOCK_N / 16;\n const int TILE_SIZE = TILE_N * sizeof(block_q8_0) + TILE_N * sizeof(int32_t);\n\n const block_q8_0 * RESTRICT A = static_cast(_A);\n const char * RESTRICT B = static_cast(_B);\n\n __m512i va[8];\n __m512i vb[8];\n __m512 vc[COLS];\n __m512 vd1;\n\n // Notes: s8s8 igemm compensation in avx512-vnni\n // change s8s8 to u8s8 with compensate\n // a * b = (a + 128) * b - 128 * b\n // s s u s u s\n //\n // (128 * b is pre-computed when packing B to vnni formats)\n //\n const __m512i off = _mm512_set1_epi8(static_cast(0x80));\n\n auto loadc = [&](auto col) {\n vc[col] = _mm512_setzero_ps();\n };\n Unroll{}(loadc);\n\n auto compute = [&](auto col, auto i) {\n // load a and add offset 128\n if constexpr (col == 0) {\n const int32_t * a_ptr = reinterpret_cast(A[0 * KB + i].qs);\n for (int k = 0; k < 8; ++k) {\n va[k] = _mm512_set1_epi32(a_ptr[k]);\n va[k] = _mm512_add_epi8(va[k], off);\n }\n vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));\n }\n\n // load b\n const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n for (int k = 0; k < 8; ++k) {\n vb[k] = _mm512_loadu_si512((const __m512i *)(b_ptr + k * 64));\n }\n const int offset = TILE_N * TILE_K;\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset)));\n const int offset2 = TILE_N * TILE_K + TILE_N * sizeof(ggml_half);\n const __m512i vcomp = _mm512_loadu_si512((const __m512i *)(b_ptr + offset2));\n\n __m512i vsum = _mm512_setzero_si512();\n for (int k = 0; k < 8; ++k) {\n vsum = _mm512_dpbusd_epi32(vsum, va[k], vb[k]);\n }\n vsum = _mm512_sub_epi32(vsum, vcomp);\n\n vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(vsum), _mm512_mul_ps(vd0, vd1), vc[col]);\n };\n\n for (int i = 0; i < KB; ++i) {\n Unroll{}(compute, i);\n }\n\n //store to C\n auto storec = [&](auto col) {\n _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n };\n Unroll{}(storec);\n }\n};\n\ntemplate \nstruct tinygemm_kernel_vnni {\n static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n constexpr int COLS = BLOCK_N / 16;\n const int TILE_SIZE = TILE_N * sizeof(block_q4_K) + TILE_N * 4;\n\n const block_q8_K * RESTRICT A = static_cast(_A);\n const char * RESTRICT B = static_cast(_B);\n\n // a.qs: 8 groups, 32 bytes each group (m256i)\n __m512i va[8];\n // a.bsum: 8 groups, 2 bytes each group (m128i)\n __m512i va_bsum;\n __m512 vc[COLS];\n __m512 vd1;\n\n // packed_B:\n const int offset_scales = (QK_K / 2) * TILE_N;\n const int offset_mins = (QK_K / 2) * TILE_N + 8 * TILE_N;\n const int offset_d0 = (QK_K / 2) * TILE_N + 16 * TILE_N;\n const int offset_dmin = (QK_K / 2) * TILE_N + 16 * TILE_N + TILE_N * sizeof(ggml_half);\n\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n auto loadc = [&](auto col) {\n vc[col] = _mm512_setzero_ps();\n };\n Unroll{}(loadc);\n\n // Notes: vnni formats in QK_K\n // a) quants vnni format\n // int8 {k/4, n, 4}, viewed as 2d {k/4, 4n}, k = 32\n // from {16, 32} to {8, 64}\n //\n // b) min vnni format\n // int16 {k/2, n, 2}, viewed as 2d {k/2, 2n}, k = 8\n // from {16, 8} to {4, 32}\n //\n auto compute = [&](auto col, auto i) {\n // load a\n if constexpr (col == 0) {\n for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n va[k_group] = _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)(A[0 * KB + i].qs + k_group * 32)));\n }\n const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[0 * KB + i].bsums);\n const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n va_bsum = _mm512_castsi128_si512(q8s);\n vd1 = _mm512_set1_ps(A[0 * KB + i].d);\n }\n\n // step 1: accumultate the quants\n __m512i acc = _mm512_setzero_si512();\n const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n const char * b_qs = b_ptr;\n for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n __m512i vsum = _mm512_setzero_si512();\n for (int k = 0; k < 8; k += 2) {\n __m512i va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(k + 0), va[k_group]);\n __m512i va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(k + 1), va[k_group]);\n\n __m512i bytes = _mm512_loadu_si512((const __m512i *)b_qs);\n __m512i vb0 = _mm512_and_si512(bytes, lowMask);\n vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n __m512i vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n\n b_qs += 64;\n }\n // vacc += scale * (q8 @ q4)\n const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(b_ptr + offset_scales + k_group * TILE_N)));\n acc = _mm512_add_epi32(acc, _mm512_mullo_epi32(vsum, vscale));\n }\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_d0)));\n vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc), _mm512_mul_ps(vd0, vd1), vc[col]);\n\n // step 2: accumulate the mins\n __m512i acc_m = _mm512_setzero_si512();\n for (int k = 0; k < 4; ++k) {\n __m512i vmask = _mm512_set1_epi32(k);\n __m512i va = _mm512_permutexvar_epi32(vmask, va_bsum);\n __m512i vb = _mm512_cvtepi8_epi16(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_mins + k * 32)));\n acc_m = _mm512_dpwssds_epi32(acc_m, va, vb);\n }\n const __m512 vdmin = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_dmin)));\n vc[col] = _mm512_fnmadd_ps(_mm512_cvtepi32_ps(acc_m), _mm512_mul_ps(vdmin, vd1), vc[col]);\n };\n\n for (int i = 0; i < KB; ++i) {\n Unroll{}(compute, i);\n }\n\n //store to C\n auto storec = [&](auto col) {\n _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n };\n Unroll{}(storec);\n }\n};\n\ntemplate \nstruct tinygemm_kernel_vnni {\n static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n constexpr int COLS = BLOCK_N / 16;\n const int TILE_SIZE = TILE_N * sizeof(block_q5_K) + TILE_N * 4;\n\n const block_q8_K * RESTRICT A = static_cast(_A);\n const char * RESTRICT B = static_cast(_B);\n\n // a.qs: 8 groups, 32 bytes each group (m256i)\n __m512i va[8];\n // a.bsum: 8 groups, 2 bytes each group (m128i)\n __m512i va_bsum;\n __m512 vc[COLS];\n __m512 vd1;\n\n // packed_B:\n const int offset_qh = (QK_K / 2) * TILE_N;\n const int offset_scales = (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N;\n const int offset_mins = (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N + 8 * TILE_N;\n const int offset_d0 = (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N + 16 * TILE_N;\n const int offset_dmin = (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N + 16 * TILE_N + TILE_N * sizeof(ggml_half);\n\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n auto loadc = [&](auto col) {\n vc[col] = _mm512_setzero_ps();\n };\n Unroll{}(loadc);\n\n // Q5_K and Q4_K shares the same vnni formats, refer to notes above.\n auto compute = [&](auto col, auto i) {\n // load a\n if constexpr (col == 0) {\n for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n va[k_group] = _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)(A[0 * KB + i].qs + k_group * 32)));\n }\n const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[0 * KB + i].bsums);\n const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n va_bsum = _mm512_castsi128_si512(q8s);\n vd1 = _mm512_set1_ps(A[0 * KB + i].d);\n }\n\n // step 1: accumultate the quants\n __m512i acc = _mm512_setzero_si512();\n const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n const char * b_qs = b_ptr;\n const char * b_qh = b_ptr + offset_qh;\n for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n __m512i vsum = _mm512_setzero_si512();\n __m512i hmask0 = _mm512_set1_epi8(0x1);\n __m512i hmask1 = _mm512_set1_epi8(0x2);\n __m512i hbits = _mm512_loadu_si512((const __m512i *)(b_qh + k_group * 64));\n for (int k = 0; k < 8; k += 2) {\n __m512i va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(k + 0), va[k_group]);\n __m512i va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(k + 1), va[k_group]);\n\n __m512i bytes = _mm512_loadu_si512((const __m512i *)b_qs);\n __m512i vb0 = _mm512_and_si512(bytes, lowMask);\n __m512i vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n\n __m512i vh0 = _mm512_slli_epi16(_mm512_srli_epi16(_mm512_and_si512(hbits, hmask0), k), 4);\n __m512i vh1 = _mm512_slli_epi16(_mm512_srli_epi16(_mm512_and_si512(hbits, hmask1), k + 1), 4);\n\n hmask0 = _mm512_slli_epi16(hmask0, 2);\n hmask1 = _mm512_slli_epi16(hmask1, 2);\n vb0 = _mm512_add_epi8(vb0, vh0);\n vb1 = _mm512_add_epi8(vb1, vh1);\n\n vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n\n b_qs += 64;\n }\n // vacc += scale * (q8 @ q5)\n const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(b_ptr + offset_scales + k_group * TILE_N)));\n acc = _mm512_add_epi32(acc, _mm512_mullo_epi32(vsum, vscale));\n }\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_d0)));\n vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc), _mm512_mul_ps(vd0, vd1), vc[col]);\n\n // step 2: accumulate the mins\n __m512i acc_m = _mm512_setzero_si512();\n for (int k = 0; k < 4; ++k) {\n __m512i vmask = _mm512_set1_epi32(k);\n __m512i va = _mm512_permutexvar_epi32(vmask, va_bsum);\n __m512i vb = _mm512_cvtepi8_epi16(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_mins + k * 32)));\n acc_m = _mm512_dpwssds_epi32(acc_m, va, vb);\n }\n const __m512 vdmin = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_dmin)));\n vc[col] = _mm512_fnmadd_ps(_mm512_cvtepi32_ps(acc_m), _mm512_mul_ps(vdmin, vd1), vc[col]);\n };\n\n for (int i = 0; i < KB; ++i) {\n Unroll{}(compute, i);\n }\n\n //store to C\n auto storec = [&](auto col) {\n _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n };\n Unroll{}(storec);\n }\n};\n\ntemplate \nstruct tinygemm_kernel_vnni {\n static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n constexpr int COLS = BLOCK_N / 16;\n const int TILE_SIZE = TILE_N * sizeof(block_q6_K);\n\n const block_q8_K * RESTRICT A = static_cast(_A);\n const char * RESTRICT B = static_cast(_B);\n\n // load the 256 bytes from A to 4 avx512 vectors\n __m512i va[4];\n __m512 vc[COLS];\n __m512 vd1;\n\n // packed_B:\n const int offset_qh = (QK_K / 2) * TILE_N;\n const int offset_scales = (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N;\n const int offset_d0 = (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N + 16 * TILE_N;\n\n // compensation\n __m512i vcomp;\n\n const __m512i m32s = _mm512_set1_epi32(32);\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n auto loadc = [&](auto col) {\n vc[col] = _mm512_setzero_ps();\n };\n Unroll{}(loadc);\n\n auto compute = [&](auto col, auto i) {\n if constexpr (col == 0) {\n // load a\n va[0] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 0));\n va[1] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 64));\n va[2] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 128));\n va[3] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 192));\n\n const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[0 * KB + i].bsums);\n vcomp = _mm512_mullo_epi32(_mm512_cvtepi16_epi32(q8sums), m32s);\n vd1 = _mm512_set1_ps(A[0 * KB + i].d);\n }\n\n // accmulate the quants\n __m512i acc = _mm512_setzero_si512();\n const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n const char * b_qs = b_ptr;\n const char * b_qh = b_ptr + offset_qh;\n int mask = 0;\n for (int k_group = 0; k_group < QK_K / 16; ++k_group) {\n int r = k_group >> 2;\n __m512i va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n __m512i va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n\n __m512i vsum = _mm512_setzero_si512();\n __m512i hmask = _mm512_set1_epi8(0x3);\n\n __m512i bytes = _mm512_loadu_si512(b_qs);\n __m512i hbits = _mm512_loadu_si512(b_qh);\n __m512i vb0 = _mm512_and_si512(bytes, lowMask);\n __m512i vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n __m512i vh0 = _mm512_slli_epi16(_mm512_and_si512(hbits, hmask), 4);\n __m512i vh1 = _mm512_slli_epi16(_mm512_and_si512(hbits, _mm512_slli_epi16(hmask, 2)), 2);\n\n vb0 = _mm512_add_epi8(vb0, vh0);\n vb1 = _mm512_add_epi8(vb1, vh1);\n vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n b_qs += 64;\n\n va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n\n bytes = _mm512_loadu_si512(b_qs);\n vb0 = _mm512_and_si512(bytes, lowMask);\n vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n vh0 = _mm512_and_si512(hbits, _mm512_slli_epi16(hmask, 4));\n vh1 = _mm512_srli_epi16(_mm512_and_si512(hbits, _mm512_slli_epi16(hmask, 6)), 2);\n vb0 = _mm512_add_epi8(vb0, vh0);\n vb1 = _mm512_add_epi8(vb1, vh1);\n vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n b_qs += 64;\n b_qh += 64;\n\n // B * A - 32 * A\n __m512i vmask = _mm512_set1_epi32(k_group);\n vsum = _mm512_sub_epi32(vsum, _mm512_permutexvar_epi32(vmask, vcomp));\n\n // vacc += scale * (q8 @ q6)\n const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(b_ptr + offset_scales + k_group * TILE_N)));\n acc = _mm512_add_epi32(acc, _mm512_mullo_epi32(vsum, vscale));\n }\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_d0)));\n vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc), _mm512_mul_ps(vd0, vd1), vc[col]);\n };\n\n for (int i = 0; i < KB; ++i) {\n Unroll{}(compute, i);\n }\n\n //store to C\n auto storec = [&](int col) {\n _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n };\n Unroll{}(storec);\n }\n};\n\ntemplate \nstruct tinygemm_kernel_vnni {\n static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n constexpr int COLS = BLOCK_N / 16;\n const int TILE_SIZE = TILE_N * sizeof(block_iq4_xs) + TILE_N * 2;\n\n const block_q8_K * RESTRICT A = static_cast(_A);\n const char * RESTRICT B = static_cast(_B);\n\n // load the 256 bytes from A to 4 avx512 vectors\n __m512i va[4];\n __m512 vc[COLS];\n __m512 vd1;\n\n // packed_B:\n const int offset_scales = (QK_K / 2) * TILE_N ;\n const int offset_d0 = (QK_K / 2) * TILE_N + 8 * TILE_N;\n\n // compensation\n __m512i vcomp;\n\n const __m256i m128s = _mm256_set1_epi16(128);\n const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n const __m512i values128 = _mm512_set_epi8(\n 113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n 113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n 113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n 113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127\n );\n const __m512i off = _mm512_set1_epi8(static_cast(0x80));\n const __m512i values256 = _mm512_add_epi8(values128, off);\n\n auto loadc = [&](auto col) {\n vc[col] = _mm512_setzero_ps();\n };\n Unroll{}(loadc);\n\n auto compute = [&](auto col, auto i) {\n if constexpr (col == 0) {\n // load a\n va[0] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 0));\n va[1] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 64));\n va[2] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 128));\n va[3] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 192));\n\n // compensation: 128 * A\n const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[0 * KB + i].bsums);\n vcomp = _mm512_castsi256_si512(_mm256_madd_epi16(q8sums, m128s));\n vd1 = _mm512_set1_ps(A[0 * KB + i].d);\n }\n\n // accmulate the quants\n __m512i acc = _mm512_setzero_si512();\n const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n const char * b_qs = b_ptr;\n int mask = 0;\n for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n int r = k_group >> 1;\n __m512i vmask = _mm512_set1_epi32(k_group);\n __m512i vsum = _mm512_setzero_si512();\n for (int k = 0; k < 8; k += 2) {\n __m512i va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n __m512i va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n\n __m512i bytes = _mm512_loadu_si512(b_qs);\n __m512i vb0 = _mm512_shuffle_epi8(values256, _mm512_and_si512(bytes, lowMask));\n __m512i vb1 = _mm512_shuffle_epi8(values256, _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask));\n\n vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n b_qs += 64;\n }\n // (B + 128) * A - 128 * A\n vsum = _mm512_sub_epi32(vsum, _mm512_permutexvar_epi32(vmask, vcomp));\n\n // vacc += scale * (q8 @ q4)\n const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(b_ptr + offset_scales + k_group * TILE_N)));\n acc = _mm512_add_epi32(acc, _mm512_mullo_epi32(vsum, vscale));\n }\n const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_d0)));\n vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc), _mm512_mul_ps(vd0, vd1), vc[col]);\n };\n\n for (int i = 0; i < KB; ++i) {\n Unroll{}(compute, i);\n }\n\n //store to C\n auto storec = [&](auto col) {\n _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n };\n Unroll{}(storec);\n }\n};\n\n#define LAUNCH_TINYGEMM_KERNEL_VNNI(NB_SIZE) \\\n tinygemm_kernel_vnni::apply( \\\n KB, wdata_batch, \\\n (const char *)src0->data + src0_offset + PACKED_INDEX(nb * kTilesN, 0, KB, TILE_SIZE), \\\n (float *) dst->data + dst_offset + nb_start, ldc)\n\ntemplate ::value, int>::type = 0>\nvoid tinygemm_kernel_amx(int M, int N, int KB, const void * RESTRICT _A, const void * RESTRICT _B, TC * RESTRICT C, int ldc) {\n using packed_B_t = packed_B_type;\n const int TILE_SIZE = get_tile_size();\n const bool need_unpack = do_unpack::value;\n\n GGML_ASSERT(M <= 2 * TILE_M && N == 2 * TILE_N);\n const TA * RESTRICT A = static_cast(_A);\n const char * RESTRICT B = static_cast(_B);\n\n const int m0 = std::min(M, TILE_M);\n const int m1 = std::max(M - TILE_M, 0);\n const int lda = KB * sizeof(TA);\n //const int ldb = KB * sizeof(TB);\n\n static thread_local packed_B_t Tile0[TILE_N * TILE_K];\n static thread_local packed_B_t Tile1[TILE_N * TILE_K];\n static thread_local int8_t Tile23[TILE_M * TILE_K];\n\n static thread_local int32_t TileC0[TILE_M * TILE_N * 4];\n static thread_local int32_t TileC1[TILE_M * TILE_N * 4];\n\n // double buffering C to interleave avx512 and amx\n int32_t * C_cur = TileC0;\n int32_t * C_pre = TileC1;\n\n auto Tile4 = [&](int32_t * base) { return base; };\n auto Tile5 = [&](int32_t * base) { return base + TILE_M * TILE_N; };\n auto Tile6 = [&](int32_t * base) { return base + 2 * TILE_M * TILE_N; };\n auto Tile7 = [&](int32_t * base) { return base + 3 * TILE_M * TILE_N; };\n\n if (M == 2 * TILE_M) {\n // i = 0\n const char * B_blk0 = B + PACKED_INDEX(0, 0, KB, TILE_SIZE);\n const char * B_blk1 = B + PACKED_INDEX(1, 0, KB, TILE_SIZE);\n if (need_unpack) {\n unpack_B(Tile0, B_blk0);\n _tile_loadd(TMM0, Tile0, TILE_N * VNNI_BLK);\n } else {\n _tile_loadd(TMM0, B_blk0, TILE_N * VNNI_BLK);\n }\n\n _tile_zero(TMM4);\n _tile_loadd(TMM2, A[0].qs, lda);\n _tile_dpbssd(TMM4, TMM2, TMM0);\n _tile_stored(TMM4, Tile4(C_pre), TILE_N * sizeof(int32_t));\n\n _tile_zero(TMM5);\n _tile_loadd(TMM3, A[TILE_M * KB + 0].qs, lda);\n _tile_dpbssd(TMM5, TMM3, TMM0);\n _tile_stored(TMM5, Tile5(C_pre), TILE_N * sizeof(int32_t));\n\n if (need_unpack) {\n unpack_B(Tile1, B_blk1);\n _tile_loadd(TMM1, Tile1, TILE_N * VNNI_BLK);\n } else {\n _tile_loadd(TMM1, B_blk1, TILE_N * VNNI_BLK);\n }\n\n _tile_zero(TMM6);\n _tile_dpbssd(TMM6, TMM2, TMM1);\n _tile_stored(TMM6, Tile6(C_pre), TILE_N * sizeof(int32_t));\n\n _tile_zero(TMM7);\n _tile_dpbssd(TMM7, TMM3, TMM1);\n _tile_stored(TMM7, Tile7(C_pre), TILE_N * sizeof(int32_t));\n\n for (int i = 1; i < KB; ++i) {\n // index of previous iter\n const int ii = i - 1;\n const char * B_blk0 = B + PACKED_INDEX(0, i, KB, TILE_SIZE);\n const char * B_blk1 = B + PACKED_INDEX(1, i, KB, TILE_SIZE);\n GGML_DISPATCH_BOOL(ii > 0, is_acc, [&] {\n if (need_unpack) {\n unpack_B(Tile0, B_blk0);\n _tile_loadd(TMM0, Tile0, TILE_N * VNNI_BLK);\n } else {\n _tile_loadd(TMM0, B_blk0, TILE_N * VNNI_BLK);\n }\n _tile_zero(TMM4);\n _tile_loadd(TMM2, A[i].qs, lda);\n acc_C::apply(C, ldc, Tile4(C_pre), &A[ii], KB, B + PACKED_INDEX(0, ii, KB, TILE_SIZE), TILE_M);\n\n _tile_dpbssd(TMM4, TMM2, TMM0);\n _tile_stored(TMM4, Tile4(C_cur), TILE_N * sizeof(int32_t));\n\n _tile_zero(TMM5);\n _tile_loadd(TMM3, A[TILE_M * KB + i].qs, lda);\n acc_C::apply(C + TILE_M * ldc, ldc, Tile5(C_pre), &A[TILE_M * KB + ii], KB, B + PACKED_INDEX(0, ii, KB, TILE_SIZE), TILE_M);\n\n _tile_dpbssd(TMM5, TMM3, TMM0);\n _tile_stored(TMM5, Tile5(C_cur), TILE_N * sizeof(int32_t));\n\n if (need_unpack) {\n unpack_B(Tile1, B_blk1);\n _tile_loadd(TMM1, Tile1, TILE_N * VNNI_BLK);\n } else {\n _tile_loadd(TMM1, B_blk1, TILE_N * VNNI_BLK);\n }\n _tile_zero(TMM6);\n acc_C::apply(C + TILE_N, ldc, Tile6(C_pre), &A[ii], KB, B + PACKED_INDEX(1, ii, KB, TILE_SIZE), TILE_M);\n\n _tile_dpbssd(TMM6, TMM2, TMM1);\n _tile_stored(TMM6, Tile6(C_cur), TILE_N * sizeof(int32_t));\n\n _tile_zero(TMM7);\n acc_C::apply(C + TILE_M * ldc + TILE_N, ldc, Tile7(C_pre), &A[TILE_M * KB + ii], KB, B + PACKED_INDEX(1, ii, KB, TILE_SIZE), TILE_M);\n\n _tile_dpbssd(TMM7, TMM3, TMM1);\n _tile_stored(TMM7, Tile7(C_cur), TILE_N * sizeof(int32_t));\n\n std::swap(C_cur, C_pre);\n });\n }\n // final accumulation\n {\n int ii = KB - 1;\n acc_C::apply(C, ldc, Tile4(C_pre), &A[ii], KB, B + PACKED_INDEX(0, ii, KB, TILE_SIZE), TILE_M);\n acc_C::apply(C + TILE_M * ldc, ldc, Tile5(C_pre), &A[TILE_M * KB + ii], KB, B + PACKED_INDEX(0, ii, KB, TILE_SIZE), TILE_M);\n acc_C::apply(C + TILE_N, ldc, Tile6(C_pre), &A[ii], KB, B + PACKED_INDEX(1, ii, KB, TILE_SIZE), TILE_M);\n acc_C::apply(C + TILE_M * ldc + TILE_N, ldc, Tile7(C_pre), &A[TILE_M * KB + ii], KB, B + PACKED_INDEX(1, ii, KB, TILE_SIZE), TILE_M);\n }\n } else {\n for (int i = 0; i < KB; ++i) {\n _tile_zero(TMM4);\n _tile_zero(TMM6);\n if (m1 != 0) {\n _tile_zero(TMM5);\n _tile_zero(TMM7);\n }\n\n const char * B_blk0 = B + PACKED_INDEX(0, i, KB, TILE_SIZE);\n const char * B_blk1 = B + PACKED_INDEX(1, i, KB, TILE_SIZE);\n if (need_unpack) {\n unpack_B(Tile0, B_blk0);\n _tile_loadd(TMM0, Tile0, TILE_N * VNNI_BLK);\n } else {\n _tile_loadd(TMM0, B_blk0, TILE_N * VNNI_BLK);\n }\n\n if (need_unpack) {\n unpack_B(Tile1, B_blk1);\n _tile_loadd(TMM1, Tile1, TILE_N * VNNI_BLK);\n } else {\n _tile_loadd(TMM1, B_blk1, TILE_N * VNNI_BLK);\n }\n\n if (m0 == TILE_M) {\n _tile_loadd(TMM2, A[i].qs, lda);\n } else {\n unpack_A(Tile23, &A[i], KB, m0);\n _tile_loadd(TMM2, Tile23, TILE_K);\n }\n\n _tile_dpbssd(TMM4, TMM2, TMM0);\n _tile_dpbssd(TMM6, TMM2, TMM1);\n\n _tile_stored(TMM4, Tile4(C_cur), TILE_N * sizeof(int32_t));\n _tile_stored(TMM6, Tile6(C_cur), TILE_N * sizeof(int32_t));\n\n GGML_DISPATCH_BOOL(i > 0, is_acc, [&] {\n acc_C::apply(C, ldc, Tile4(C_cur), &A[i], KB, B + PACKED_INDEX(0, i, KB, TILE_SIZE), m0);\n acc_C::apply(C + TILE_N, ldc, Tile6(C_cur), &A[i], KB, B + PACKED_INDEX(1, i, KB, TILE_SIZE), m0);\n });\n\n if (m1 != 0) {\n unpack_A(Tile23, &A[TILE_M * KB + i], KB, m1);\n _tile_loadd(TMM3, Tile23, TILE_K);\n\n _tile_dpbssd(TMM5, TMM3, TMM0);\n _tile_dpbssd(TMM7, TMM3, TMM1);\n _tile_stored(TMM5, Tile5(C_cur), TILE_N * sizeof(int32_t));\n _tile_stored(TMM7, Tile7(C_cur), TILE_N * sizeof(int32_t));\n GGML_DISPATCH_BOOL(i > 0, is_acc, [&] {\n acc_C::apply(C + TILE_M * ldc, ldc, Tile5(C_cur), &A[TILE_M * KB + i], KB, B + PACKED_INDEX(0, i, KB, TILE_SIZE), m1);\n acc_C::apply(C + TILE_M * ldc + TILE_N, ldc, Tile7(C_cur), &A[TILE_M * KB + i], KB, B + PACKED_INDEX(1, i, KB, TILE_SIZE), m1);\n });\n }\n }\n }\n return;\n}\n\ntemplate ::value, int>::type = 0>\nvoid tinygemm_kernel_amx(int M, int N, int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n static_assert(std::is_same::value);\n const int TILE_SIZE = get_tile_size();\n\n GGML_ASSERT(M <= 2 * TILE_M && N == 2 * TILE_N);\n const TA * RESTRICT A = static_cast(_A);\n const char * RESTRICT B = static_cast(_B);\n\n const int m0 = std::min(M, TILE_M);\n const int m1 = std::max(M - TILE_M, 0);\n //const int lda = KB * sizeof(TA);\n\n static thread_local int8_t Tile0[TILE_N * TILE_K];\n static thread_local int8_t Tile1[TILE_N * TILE_K];\n static thread_local int8_t Tile23[TILE_M * TILE_K];\n\n // mat mul result for each group\n static thread_local int32_t Tile4[TILE_M * TILE_N];\n static thread_local int32_t Tile5[TILE_M * TILE_N];\n static thread_local int32_t Tile6[TILE_M * TILE_N];\n static thread_local int32_t Tile7[TILE_M * TILE_N];\n\n // sum of each QK_K block, contains 8 groups, int32\n static thread_local int32_t Sumi4[TILE_M * TILE_N];\n static thread_local int32_t Sumi5[TILE_M * TILE_N];\n static thread_local int32_t Sumi6[TILE_M * TILE_N];\n static thread_local int32_t Sumi7[TILE_M * TILE_N];\n\n const int k_group_size = std::is_same::value ? 16 : 32;\n for (int i = 0; i < KB; ++i) {\n // step 1: accumulate the quants across 8 groups, each group with 32\n for (int k = 0; k < QK_K / k_group_size; ++k) {\n GGML_DISPATCH_BOOL(k > 0, is_acc, [&] {\n _tile_zero(TMM4);\n _tile_zero(TMM6);\n\n unpack_B(Tile0, B + PACKED_INDEX(0, i, KB, TILE_SIZE), k);\n _tile_loadd(TMM0, Tile0, TILE_N * VNNI_BLK);\n\n unpack_B(Tile1, B + PACKED_INDEX(1, i, KB, TILE_SIZE), k);\n _tile_loadd(TMM1, Tile1, TILE_N * VNNI_BLK);\n\n unpack_A(Tile23, &A[i], KB, k, m0);\n _tile_loadd(TMM2, Tile23, TILE_K);\n\n _tile_dpbssd(TMM4, TMM2, TMM0);\n _tile_dpbssd(TMM6, TMM2, TMM1);\n\n _tile_stored(TMM4, Tile4, TILE_N * sizeof(int32_t));\n _tile_stored(TMM6, Tile6, TILE_N * sizeof(int32_t));\n\n scale_C(Tile4, Sumi4, B + PACKED_INDEX(0, i, KB, TILE_SIZE), k, m0);\n scale_C(Tile6, Sumi6, B + PACKED_INDEX(1, i, KB, TILE_SIZE), k, m0);\n\n if (m1 != 0) {\n _tile_zero(TMM5);\n _tile_zero(TMM7);\n\n unpack_A(Tile23, &A[TILE_M * KB + i], KB, k, m1);\n _tile_loadd(TMM3, Tile23, TILE_K);\n\n _tile_dpbssd(TMM5, TMM3, TMM0);\n _tile_dpbssd(TMM7, TMM3, TMM1);\n\n _tile_stored(TMM5, Tile5, TILE_N * sizeof(int32_t));\n _tile_stored(TMM7, Tile7, TILE_N * sizeof(int32_t));\n\n scale_C(Tile5, Sumi5, B + PACKED_INDEX(0, i, KB, TILE_SIZE), k, m1);\n scale_C(Tile7, Sumi7, B + PACKED_INDEX(1, i, KB, TILE_SIZE), k, m1);\n }\n });\n }\n\n // step 2: accmulate the mins\n GGML_DISPATCH_BOOL(i > 0, is_acc, [&] {\n acc_C::apply(C, ldc, Sumi4, &A[i], KB, B + PACKED_INDEX(0, i, KB, TILE_SIZE), m0);\n acc_C::apply(C + TILE_N, ldc, Sumi6, &A[i], KB, B + PACKED_INDEX(1, i, KB, TILE_SIZE), m0);\n if (m1 != 0) {\n acc_C::apply(C + TILE_M * ldc, ldc, Sumi5, &A[TILE_M * KB + i], KB, B + PACKED_INDEX(0, i, KB, TILE_SIZE), m1);\n acc_C::apply(C + TILE_M * ldc + TILE_N, ldc, Sumi7, &A[TILE_M * KB + i], KB, B + PACKED_INDEX(1, i, KB, TILE_SIZE), m1);\n }\n });\n }\n return;\n}\n\n} // anonymous namespace\n\n// get the packed tensor size for quantized weights\nsize_t ggml_backend_amx_get_alloc_size(const struct ggml_tensor * tensor) {\n const enum ggml_type TYPE = tensor->type;\n\n const int K = tensor->ne[0]; // ne0: in_features\n const int N = tensor->ne[1]; // ne1: out_features\n\n auto get_tensor_size = [&] {\n size_t row_size_B{0};\n GGML_DISPATCH_QTYPES(TYPE, [&] {\n row_size_B = get_row_size(K);\n });\n return N * row_size_B;\n };\n\n if (qtype_has_amx_kernels(TYPE)) {\n return get_tensor_size();\n } else {\n // for f16, bf16 we don't do packing\n return ggml_nbytes(tensor);\n }\n}\n\n// pack weight to vnni format\nvoid ggml_backend_amx_convert_weight(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); // only full tensor conversion is supported for now\n\n const enum ggml_type TYPE = tensor->type;\n\n const int K = tensor->ne[0]; // ne0: in_features\n const int N = tensor->ne[1]; // ne1: out_features\n\n GGML_DISPATCH_QTYPES(TYPE, [&] {\n convert_B_packed_format((void *)((char *)tensor->data + offset), (const type *)data, N, K);\n });\n}\n\n// ne2 is passed explicitly to help compiler optimize repeated calls\ninline int64_t ggml_batch_offset(const ggml_tensor * t, int64_t batch_idx, int64_t ne2) {\n const int64_t i2 = batch_idx % ne2;\n const int64_t i3 = batch_idx / ne2;\n return i3 * t->nb[3] + i2 * t->nb[2];\n}\n\nsize_t ggml_backend_amx_desired_wsize(const struct ggml_tensor * dst) {\n struct ggml_tensor * src0 = dst->src[0];\n\n const enum ggml_type TYPE = src0->type;\n\n const bool is_floating_type = TYPE == GGML_TYPE_F16;\n if (is_floating_type) {\n return 0;\n }\n\n const int M = dst->ne[1];\n const int K = src0->ne[0];\n const int64_t n_batch = dst->ne[2] * dst->ne[3];\n\n size_t desired_wsize = 0;\n\n GGML_DISPATCH_QTYPES(TYPE, [&] {\n const size_t row_size_A = K / blck_size * sizeof(vec_dot_type);\n desired_wsize = n_batch * M * row_size_A;\n });\n\n return desired_wsize;\n}\n\n// NB: mixed dtype gemm with Advanced Matrix Extensions (Intel AMX)\n//\n// src0: weight in shape of {N, K}, quantized\n// src1: input in shape of {M, K}, float32\n// dst: output in shape of {M, N}, float32\n//\n// the function performs: dst = src1 @ src0.T for each batch\n//\nvoid ggml_backend_amx_mul_mat(const ggml_compute_params * params, struct ggml_tensor * dst) {\n struct ggml_tensor * src0 = dst->src[0];\n struct ggml_tensor * src1 = dst->src[1];\n\n const enum ggml_type TYPE = src0->type;\n\n // f16 only has avx512 kernels for now,\n // amx kernels will be added once 6th gen xeon is released.\n const bool is_floating_type = TYPE == GGML_TYPE_F16;\n\n const int M = dst->ne[1];\n const int N = dst->ne[0];\n const int K = src0->ne[0];\n const int ldc = dst->nb[1] / dst->nb[0];\n\n const int64_t ne2 = dst->ne[2];\n const int64_t n_batch = ne2 * dst->ne[3];\n\n if (is_floating_type) {\n constexpr int BLOCK_M = 4;\n constexpr int BLOCK_N = 6;\n const int MB = div_up(M, BLOCK_M);\n const int NB = div_up(N, BLOCK_N);\n\n parallel_for_ggml(params, n_batch * MB * NB, [&](int begin, int end) {\n GGML_DISPATCH_FLOATING_TYPES(TYPE, [&] {\n for (int i = begin; i < end; ++i) {\n int batch_idx = i / (MB * NB);\n int remaining = i % (MB * NB);\n int mb = remaining / NB;\n int nb = remaining % NB;\n\n int64_t src0_offset = ggml_batch_offset(src0, batch_idx, ne2);\n int64_t src1_offset = ggml_batch_offset(src1, batch_idx, ne2);\n int64_t dst_offset = ggml_batch_offset(dst, batch_idx, ne2);\n\n int mb_start = mb * BLOCK_M;\n int mb_size = std::min(BLOCK_M, M - mb_start);\n int nb_start = nb * BLOCK_N;\n int nb_size = std::min(BLOCK_N, N - nb_start);\n\n switch (mb_size << 4 | nb_size) {\n case 0x12: LAUNCH_TINYGEMM_KERNEL_AVX(1, 2); break;\n case 0x14: LAUNCH_TINYGEMM_KERNEL_AVX(1, 4); break;\n case 0x16: LAUNCH_TINYGEMM_KERNEL_AVX(1, 6); break;\n case 0x22: LAUNCH_TINYGEMM_KERNEL_AVX(2, 2); break;\n case 0x24: LAUNCH_TINYGEMM_KERNEL_AVX(2, 4); break;\n case 0x26: LAUNCH_TINYGEMM_KERNEL_AVX(2, 6); break;\n case 0x32: LAUNCH_TINYGEMM_KERNEL_AVX(3, 2); break;\n case 0x34: LAUNCH_TINYGEMM_KERNEL_AVX(3, 4); break;\n case 0x36: LAUNCH_TINYGEMM_KERNEL_AVX(3, 6); break;\n case 0x42: LAUNCH_TINYGEMM_KERNEL_AVX(4, 2); break;\n case 0x44: LAUNCH_TINYGEMM_KERNEL_AVX(4, 4); break;\n case 0x46: LAUNCH_TINYGEMM_KERNEL_AVX(4, 6); break;\n default: fprintf(stderr, \"Unexpected block size!\\n\");\n }\n }\n });\n });\n return;\n }\n\n // pointer to work space, used convert A from float to quantized type\n void * wdata = params->wdata;\n\n //TODO: performance improvement: merge quant A\n // if (params->ith == 0) {\n GGML_DISPATCH_QTYPES(TYPE, [&] {\n const size_t row_size_A = K / blck_size * sizeof(vec_dot_type);\n const size_t desired_wsize = n_batch * M * row_size_A;\n if (params->wsize < desired_wsize) {\n GGML_ABORT(\"insufficient work space size\");\n }\n\n // Q4_0, Q4_1, Q8_0 handles 1 TILE_K per blck_size\n // Q4_K, Q5_K, Q6_K, IQ4_XS handles 8 TILE_K per blck_size\n GGML_ASSERT(TILE_K == blck_size || TILE_K * 8 == blck_size);\n\n parallel_for_ggml(params, n_batch, [&](int begin, int end) {\n for (int batch_idx = begin; batch_idx < end; ++batch_idx) {\n int64_t src1_offset = ggml_batch_offset(src1, batch_idx, ne2);\n const float * A_data = (const float *)((const char *)src1->data + src1_offset);\n char * wdata_batch = (char *)wdata + batch_idx * M * row_size_A;\n\n for (int m = 0; m < M; ++m) {\n from_float(A_data + m * K, wdata_batch + m * row_size_A, K);\n }\n }\n });\n });\n // }\n\n ggml_barrier(params->threadpool);\n\n if (M == 1) {\n // MB = 1 and handle 8 tiles in each block\n constexpr int kTilesN = 4;\n constexpr int BLOCK_N = TILE_N * kTilesN;\n const int NB = div_up(N, BLOCK_N);\n\n parallel_for_ggml(params, n_batch * NB, [&](int begin, int end) {\n GGML_DISPATCH_QTYPES(TYPE, [&] {\n const int KB = K / blck_size;\n const int TILE_SIZE = get_tile_size();\n const int row_size_A = KB * sizeof(vec_dot_type);\n for (int i = begin; i < end; ++i) {\n int batch_idx = i / NB;\n int nb = i % NB;\n\n int64_t src0_offset = ggml_batch_offset(src0, batch_idx, ne2);\n int64_t dst_offset = ggml_batch_offset(dst, batch_idx, ne2);\n const char * wdata_batch = (const char *)wdata + batch_idx * row_size_A;\n\n int nb_start = nb * BLOCK_N;\n int nb_size = std::min(BLOCK_N, N - nb_start); // 32, 64, 96\n\n switch (nb_size) {\n //case 160: LAUNCH_TINYGEMM_KERNEL_VNNI(160); break;\n case 128: LAUNCH_TINYGEMM_KERNEL_VNNI(128); break;\n case 96: LAUNCH_TINYGEMM_KERNEL_VNNI(96); break;\n case 64: LAUNCH_TINYGEMM_KERNEL_VNNI(64); break;\n case 32: LAUNCH_TINYGEMM_KERNEL_VNNI(32); break;\n default: fprintf(stderr, \"Unexpected n block size!\\n\");\n }\n }\n });\n });\n return;\n }\n\n // handle 4 tiles at a tile\n constexpr int BLOCK_M = TILE_M * 2;\n constexpr int BLOCK_N = TILE_N * 2;\n const int MB = div_up(M, BLOCK_M);\n const int NB = div_up(N, BLOCK_N);\n\n parallel_for_ggml(params, n_batch * MB * NB, [&](int begin, int end) {\n // init tile config for each thread\n ggml_tile_config_init();\n\n GGML_DISPATCH_QTYPES(TYPE, [&] {\n const int KB = K / blck_size;\n const int TILE_SIZE = get_tile_size();\n const int row_size_A = KB * sizeof(vec_dot_type);\n\n for (int i = begin; i < end; ++i) {\n int batch_idx = i / (MB * NB);\n int remaining = i % (MB * NB);\n int mb = remaining / NB;\n int nb = remaining % NB;\n\n int64_t src0_offset = ggml_batch_offset(src0, batch_idx, ne2);\n int64_t dst_offset = ggml_batch_offset(dst, batch_idx, ne2);\n const char * wdata_batch = (const char *)wdata + batch_idx * M * row_size_A;\n\n int mb_start = mb * BLOCK_M;\n int mb_size = std::min(BLOCK_M, M - mb_start);\n int nb_start = nb * BLOCK_N;\n int nb_size = BLOCK_N;\n\n tinygemm_kernel_amx(\n mb_size, nb_size, KB,\n wdata_batch + mb_start * row_size_A,\n (const char *)src0->data + src0_offset + PACKED_INDEX(nb * 2, 0, KB, TILE_SIZE),\n (float *) dst->data + dst_offset + mb_start * N + nb_start, ldc);\n }\n });\n });\n}\n\n#endif // if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n" }, { "path": "src/ggml-cpu/amx/mmq.h", "content": "#pragma once\n#include \"common.h\"\n\nsize_t ggml_backend_amx_desired_wsize(const struct ggml_tensor * dst);\n\nsize_t ggml_backend_amx_get_alloc_size(const struct ggml_tensor * tensor);\n\nvoid ggml_backend_amx_convert_weight(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n\nvoid ggml_backend_amx_mul_mat(const struct ggml_compute_params * params, struct ggml_tensor * dst);\n" }, { "path": "src/ggml-cpu/arch/arm/cpu-feats.cpp", "content": "#include \"ggml-backend-impl.h\"\n\n#if defined(__aarch64__)\n\n#if defined(__linux__)\n#include \n#elif defined(__APPLE__)\n#include \n#endif\n\n#if !defined(HWCAP2_SVE2)\n#define HWCAP2_SVE2 (1 << 1)\n#endif\n\n#if !defined(HWCAP2_I8MM)\n#define HWCAP2_I8MM (1 << 13)\n#endif\n\n#if !defined(HWCAP2_SME)\n#define HWCAP2_SME (1 << 23)\n#endif\n\nstruct aarch64_features {\n // has_neon not needed, aarch64 has NEON guaranteed\n bool has_dotprod = false;\n bool has_fp16_va = false;\n bool has_sve = false;\n bool has_sve2 = false;\n bool has_i8mm = false;\n bool has_sme = false;\n\n aarch64_features() {\n#if defined(__linux__)\n uint32_t hwcap = getauxval(AT_HWCAP);\n uint32_t hwcap2 = getauxval(AT_HWCAP2);\n\n has_dotprod = !!(hwcap & HWCAP_ASIMDDP);\n has_fp16_va = !!(hwcap & HWCAP_FPHP);\n has_sve = !!(hwcap & HWCAP_SVE);\n has_sve2 = !!(hwcap2 & HWCAP2_SVE2);\n has_i8mm = !!(hwcap2 & HWCAP2_I8MM);\n has_sme = !!(hwcap2 & HWCAP2_SME);\n#elif defined(__APPLE__)\n int oldp = 0;\n size_t size = sizeof(oldp);\n\n if (sysctlbyname(\"hw.optional.arm.FEAT_DotProd\", &oldp, &size, NULL, 0) == 0) {\n has_dotprod = static_cast(oldp);\n }\n\n if (sysctlbyname(\"hw.optional.arm.FEAT_I8MM\", &oldp, &size, NULL, 0) == 0) {\n has_i8mm = static_cast(oldp);\n }\n\n if (sysctlbyname(\"hw.optional.arm.FEAT_SME\", &oldp, &size, NULL, 0) == 0) {\n has_sme = static_cast(oldp);\n }\n\n // Apple apparently does not implement SVE yet\n#endif\n }\n};\n\nstatic int ggml_backend_cpu_aarch64_score() {\n int score = 1;\n aarch64_features af;\n\n#ifdef GGML_USE_DOTPROD\n if (!af.has_dotprod) { return 0; }\n score += 1<<1;\n#endif\n#ifdef GGML_USE_FP16_VECTOR_ARITHMETIC\n if (!af.has_fp16_va) { return 0; }\n score += 1<<2;\n#endif\n#ifdef GGML_USE_SVE\n if (!af.has_sve) { return 0; }\n score += 1<<3;\n#endif\n#ifdef GGML_USE_MATMUL_INT8\n if (!af.has_i8mm) { return 0; }\n score += 1<<4;\n#endif\n#ifdef GGML_USE_SVE2\n if (!af.has_sve2) { return 0; }\n score += 1<<5;\n#endif\n#ifdef GGML_USE_SME\n if (!af.has_sme) { return 0; }\n score += 1<<6;\n#endif\n\n return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_aarch64_score)\n\n# endif // defined(__aarch64__)\n" }, { "path": "src/ggml-cpu/arch/arm/quants.c", "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include \n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__ARM_NEON)\n#define B1(c,s,n) 0x ## n ## c , 0x ## n ## s\n#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)\n#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)\n#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)\n#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)\n#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)\n#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)\n#define B8(c,s ) B7(c,s, c), B7(c,s, s)\n\n// precomputed tables for expanding 8bits to 8 bytes:\nstatic const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4\nstatic const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4\n#endif\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_NEON)\n for (int i = 0; i < nb; i++) {\n float32x4_t srcv [8];\n float32x4_t asrcv[8];\n float32x4_t amaxv[8];\n\n for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j);\n for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);\n\n for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);\n for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);\n for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);\n\n const float amax = vmaxvq_f32(amaxv[0]);\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n for (int j = 0; j < 8; j++) {\n const float32x4_t v = vmulq_n_f32(srcv[j], id);\n const int32x4_t vi = vcvtnq_s32_f32(v);\n\n y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);\n y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);\n y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);\n y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);\n }\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK8_1 == 0);\n const int nb = k / QK8_1;\n\n block_q8_1 * GGML_RESTRICT y = vy;\n#if defined(__ARM_NEON)\n for (int i = 0; i < nb; i++) {\n float32x4_t srcv [8];\n float32x4_t asrcv[8];\n float32x4_t amaxv[8];\n\n for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j);\n for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);\n\n for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);\n for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);\n for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);\n\n const float amax = vmaxvq_f32(amaxv[0]);\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n int32x4_t accv = vdupq_n_s32(0);\n\n for (int j = 0; j < 8; j++) {\n const float32x4_t v = vmulq_n_f32(srcv[j], id);\n const int32x4_t vi = vcvtnq_s32_f32(v);\n\n y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);\n y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);\n y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);\n y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);\n\n accv = vaddq_s32(accv, vi);\n }\n\n y[i].s = GGML_CPU_FP32_TO_FP16(d * vaddvq_s32(accv));\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n// placeholder implementation for Apple targets\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_q8_K_ref(x, y, k);\n}\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n assert((nrc == 2) || (nrc == 1));\n#else\n assert(nrc == 1);\n#endif\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n if (nrc == 2) {\n const block_q4_0 * GGML_RESTRICT vx0 = vx;\n const block_q4_0 * GGML_RESTRICT vx1 = (const block_q4_0 *) ((const uint8_t*)vx + bx);\n const block_q8_0 * GGML_RESTRICT vy0 = vy;\n const block_q8_0 * GGML_RESTRICT vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by);\n\n float32x4_t sumv0 = vdupq_n_f32(0.0f);\n\n for (int i = 0; i < nb; i++) {\n const block_q4_0 * GGML_RESTRICT b_x0 = &vx0[i];\n const block_q4_0 * GGML_RESTRICT b_x1 = &vx1[i];\n const block_q8_0 * GGML_RESTRICT b_y0 = &vy0[i];\n const block_q8_0 * GGML_RESTRICT b_y1 = &vy1[i];\n\n const uint8x16_t m4b = vdupq_n_u8(0x0F);\n const int8x16_t s8b = vdupq_n_s8(0x8);\n\n const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);\n const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);\n\n // 4-bit -> 8-bit\n const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));\n const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));\n const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n // sub 8\n const int8x16_t x0_l = vsubq_s8(v0_0l, s8b);\n const int8x16_t x0_h = vsubq_s8(v0_0h, s8b);\n const int8x16_t x1_l = vsubq_s8(v0_1l, s8b);\n const int8x16_t x1_h = vsubq_s8(v0_1h, s8b);\n\n // load y\n const int8x16_t y0_l = vld1q_s8(b_y0->qs);\n const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);\n const int8x16_t y1_l = vld1q_s8(b_y1->qs);\n const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);\n\n float32_t _scale[4] = {\n GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y1->d),\n GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y1->d)\n };\n float32x4_t scale = vld1q_f32(_scale);\n\n int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n\n int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n\n int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n\n int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n\n sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),\n l1, r1)), l2, r2)), l3, r3))), scale);\n }\n\n float32x4_t sumv1 = vextq_f32 (sumv0, sumv0, 2);\n float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);\n\n vst1_f32(s, vget_low_f32 (sumv2));\n vst1_f32(s + bs, vget_high_f32(sumv2));\n\n return;\n }\n#endif\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__ARM_FEATURE_SVE)\n svfloat32_t sumv0 = svdup_n_f32(0.0f);\n svfloat32_t sumv1 = svdup_n_f32(0.0f);\n\n const int vector_length = ggml_cpu_get_sve_cnt()*8;\n\n // VLA Implementation using switch case\n switch (vector_length) {\n case 128:\n {\n // predicate for activating higher lanes for 4 float32 elements\n const svbool_t ph4 = svptrue_pat_b32(SV_VL4);\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q4_0 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n // load x\n const svuint8_t qx0r = svld1rq_u8(svptrue_b8(), x0->qs);\n const svuint8_t qx1r = svld1rq_u8(svptrue_b8(), x1->qs);\n\n // 4-bit -> 8-bit\n const svint8_t qx0l = svreinterpret_s8_u8(svand_n_u8_m(svptrue_b8(), qx0r, 0x0F));\n const svint8_t qx0h = svreinterpret_s8_u8(svlsr_n_u8_m(svptrue_b8(), qx0r, 0x04));\n const svint8_t qx1l = svreinterpret_s8_u8(svand_n_u8_m(svptrue_b8(), qx1r, 0x0F));\n const svint8_t qx1h = svreinterpret_s8_u8(svlsr_n_u8_m(svptrue_b8(), qx1r, 0x04));\n\n // sub 8\n const svint8_t qx0ls = svsub_n_s8_x(svptrue_b8(), qx0h, 8);\n const svint8_t qx0hs = svsub_n_s8_x(svptrue_b8(), qx0l, 8);\n const svint8_t qx1ls = svsub_n_s8_x(svptrue_b8(), qx1h, 8);\n const svint8_t qx1hs = svsub_n_s8_x(svptrue_b8(), qx1l, 8);\n\n // load y\n const svint8_t qy0h = svld1_s8(svptrue_b8(), y0->qs);\n const svint8_t qy0l = svld1_s8(svptrue_b8(), y0->qs + 16);\n const svint8_t qy1h = svld1_s8(svptrue_b8(), y1->qs);\n const svint8_t qy1l = svld1_s8(svptrue_b8(), y1->qs + 16);\n\n // dot product\n sumv0 = svmla_n_f32_x(ph4, sumv0, svcvt_f32_s32_x(ph4, svadd_x(ph4,\n svdot_s32(svdup_n_s32(0), qx0ls, qy0l),\n svdot_s32(svdup_n_s32(0), qx0hs, qy0h))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n sumv1 = svmla_n_f32_x(ph4, sumv1, svcvt_f32_s32_x(ph4, svadd_x(ph4,\n svdot_s32(svdup_n_s32(0), qx1ls, qy1l),\n svdot_s32(svdup_n_s32(0), qx1hs, qy1h))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1));\n } break;\n case 256:\n {\n // predicate for activating higher lanes for 16 int8 elements\n const svbool_t ph16 = svptrue_pat_b8(SV_VL16);\n // predicate for activating lower lanes for 16 int8 elements\n const svbool_t pl16 = svnot_b_z(svptrue_b8(), ph16);\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q4_0 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n // load x\n const svuint8_t qx0r = svld1rq_u8(svptrue_b8(), x0->qs);\n const svuint8_t qx1r = svld1rq_u8(svptrue_b8(), x1->qs);\n\n // 4-bit -> 8-bit\n const svint8_t qx0 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_n_u8_m(ph16, qx0r, 0x0F), 0x04));\n const svint8_t qx1 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_n_u8_m(ph16, qx1r, 0x0F), 0x04));\n\n // sub 8\n const svint8_t qx0s = svsub_n_s8_x(svptrue_b8(), qx0, 8);\n const svint8_t qx1s = svsub_n_s8_x(svptrue_b8(), qx1, 8);\n\n // load y\n const svint8_t qy0 = svld1_s8(svptrue_b8(), y0->qs);\n const svint8_t qy1 = svld1_s8(svptrue_b8(), y1->qs);\n\n // dot product\n sumv0 = svmla_n_f32_x(svptrue_b32(), sumv0, svcvt_f32_s32_x(svptrue_b32(),\n svdot_s32(svdup_n_s32(0), qx0s, qy0)), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n sumv1 = svmla_n_f32_x(svptrue_b32(), sumv1, svcvt_f32_s32_x(svptrue_b32(),\n svdot_s32(svdup_n_s32(0), qx1s, qy1)), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1));\n } break;\n case 512:\n {\n // predicate for activating higher lanes for 32 int8 elements\n const svbool_t ph32 = svptrue_pat_b8(SV_VL32);\n\n // predicate for activating higher lanes for 16 int8 elements\n const svbool_t ph16 = svptrue_pat_b8(SV_VL16);\n // predicate for activating lower lanes for 16 int8 elements from first 32 int8 activated lanes\n const svbool_t pl16 = svnot_b_z(ph32, ph16);\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q4_0 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n // load x\n const svuint8_t qx0r = svld1rq_u8(ph32, x0->qs);\n const svuint8_t qx1r = svld1rq_u8(ph32, x1->qs);\n\n // 4-bit -> 8-bit\n const svint8_t qx0 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_n_u8_m(ph16, qx0r, 0x0F), 0x04));\n const svint8_t qx1 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_n_u8_m(ph16, qx1r, 0x0F), 0x04));\n\n // sub 8\n const svint8_t qx0s = svsub_n_s8_x(ph32, qx0, 8);\n const svint8_t qx1s = svsub_n_s8_x(ph32, qx1, 8);\n\n // load y\n const svint8_t qy0 = svld1_s8(ph32, y0->qs);\n const svint8_t qy1 = svld1_s8(ph32, y1->qs);\n\n // dot product\n sumv0 = svmla_n_f32_x(ph32, sumv0, svcvt_f32_s32_x(ph32,\n svdot_s32(svdup_n_s32(0), qx0s, qy0)), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n sumv1 = svmla_n_f32_x(ph32, sumv1, svcvt_f32_s32_x(ph32,\n svdot_s32(svdup_n_s32(0), qx1s, qy1)), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = svaddv_f32(ph32, svadd_f32_x(ph32, sumv0, sumv1));\n } break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n\n#elif defined(__ARM_NEON)\n float32x4_t sumv0 = vdupq_n_f32(0.0f);\n float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q4_0 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n const uint8x16_t m4b = vdupq_n_u8(0x0F);\n const int8x16_t s8b = vdupq_n_s8(0x8);\n\n const uint8x16_t v0_0 = vld1q_u8(x0->qs);\n const uint8x16_t v0_1 = vld1q_u8(x1->qs);\n\n // 4-bit -> 8-bit\n const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));\n const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));\n const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n // sub 8\n const int8x16_t v0_0ls = vsubq_s8(v0_0l, s8b);\n const int8x16_t v0_0hs = vsubq_s8(v0_0h, s8b);\n const int8x16_t v0_1ls = vsubq_s8(v0_1l, s8b);\n const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);\n\n // load y\n const int8x16_t v1_0l = vld1q_s8(y0->qs);\n const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);\n const int8x16_t v1_1l = vld1q_s8(y1->qs);\n const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);\n\n // dot product into int32x4_t\n const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);\n const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);\n\n sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);\n#endif\n for (; ib < nb; ++ib) {\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n const int v1 = (x[ib].qs[j] >> 4) - 8;\n\n sumi0 += (v0 * y[ib].qs[j]);\n sumi1 += (v1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n assert((nrc == 2) || (nrc == 1));\n#else\n assert(nrc == 1);\n#endif\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n if (nrc == 2) {\n const block_q4_1 * GGML_RESTRICT vx0 = vx;\n const block_q4_1 * GGML_RESTRICT vx1 = (const block_q4_1 *) ((const uint8_t*)vx + bx);\n const block_q8_1 * GGML_RESTRICT vy0 = vy;\n const block_q8_1 * GGML_RESTRICT vy1 = (const block_q8_1 *) ((const uint8_t*)vy + by);\n\n float32x4_t sumv0 = vdupq_n_f32(0.0f);\n float32x4_t summs0 = vdupq_n_f32(0.0f);\n\n for (int i = 0; i < nb; i++) {\n const block_q4_1 * GGML_RESTRICT b_x0 = &vx0[i];\n const block_q4_1 * GGML_RESTRICT b_x1 = &vx1[i];\n const block_q8_1 * GGML_RESTRICT b_y0 = &vy0[i];\n const block_q8_1 * GGML_RESTRICT b_y1 = &vy1[i];\n\n float32_t summs_t[4] = {\n GGML_CPU_FP16_TO_FP32(b_x0->m) * GGML_CPU_FP16_TO_FP32(b_y0->s),\n GGML_CPU_FP16_TO_FP32(b_x1->m) * GGML_CPU_FP16_TO_FP32(b_y0->s),\n GGML_CPU_FP16_TO_FP32(b_x0->m) * GGML_CPU_FP16_TO_FP32(b_y1->s),\n GGML_CPU_FP16_TO_FP32(b_x1->m) * GGML_CPU_FP16_TO_FP32(b_y1->s)\n };\n summs0 = vaddq_f32(summs0, vld1q_f32(summs_t));\n\n const uint8x16_t m4b = vdupq_n_u8(0x0F);\n\n const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);\n const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);\n\n // 4-bit -> 8-bit\n const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));\n const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));\n const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n // load y\n const int8x16_t y0_l = vld1q_s8(b_y0->qs);\n const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);\n const int8x16_t y1_l = vld1q_s8(b_y1->qs);\n const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);\n\n // mmla into int32x4_t\n float32_t _scale[4] = {\n GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y1->d),\n GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y1->d)\n };\n float32x4_t scale = vld1q_f32(_scale);\n\n int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n\n int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n\n int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n\n int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),\n l1, r1)), l2, r2)), l3, r3))), scale);\n }\n\n float32x4_t sumv1 = vextq_f32 (sumv0, sumv0, 2);\n float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);\n\n sumv2 = vaddq_f32(sumv2, summs0);\n\n vst1_f32(s, vget_low_f32 (sumv2));\n vst1_f32(s + bs, vget_high_f32(sumv2));\n\n return;\n }\n#endif\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__ARM_NEON)\n float32x4_t sumv0 = vdupq_n_f32(0.0f);\n float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n float summs = 0;\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q4_1 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q4_1 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_1 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_1 * GGML_RESTRICT y1 = &y[ib + 1];\n\n summs += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s) + GGML_CPU_FP16_TO_FP32(x1->m) * GGML_CPU_FP16_TO_FP32(y1->s);\n\n const uint8x16_t m4b = vdupq_n_u8(0x0F);\n\n const uint8x16_t v0_0 = vld1q_u8(x0->qs);\n const uint8x16_t v0_1 = vld1q_u8(x1->qs);\n\n // 4-bit -> 8-bit\n const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));\n const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));\n const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n // load y\n const int8x16_t v1_0l = vld1q_s8(y0->qs);\n const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);\n const int8x16_t v1_1l = vld1q_s8(y1->qs);\n const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);\n\n // dot product into int32x4_t\n const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);\n const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);\n\n sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;\n\n#endif\n for (; ib < nb; ++ib) {\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const int v0 = (x[ib].qs[j] & 0x0F);\n const int v1 = (x[ib].qs[j] >> 4);\n\n sumi0 += (v0 * y[ib].qs[j]);\n sumi1 += (v1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_MXFP4 == 0);\n static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n const block_mxfp4 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_MXFP4;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined __ARM_NEON\n const int8x16_t values = vld1q_s8(kvalues_mxfp4);\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n uint8x16x2_t q4bits;\n int8x16x4_t q4b;\n int8x16x4_t q8b;\n int32x4_t prod_1;\n int32x4_t prod_2;\n\n for (; ib + 1 < nb; ib += 2) {\n q4bits.val[0] = vld1q_u8(x[ib + 0].qs);\n q4bits.val[1] = vld1q_u8(x[ib + 1].qs);\n q8b.val[0] = vld1q_s8(y[ib + 0].qs);\n q8b.val[1] = vld1q_s8(y[ib + 0].qs + 16);\n q8b.val[2] = vld1q_s8(y[ib + 1].qs);\n q8b.val[3] = vld1q_s8(y[ib + 1].qs + 16);\n\n q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[0], m4b));\n q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));\n q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[1], m4b));\n q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));\n\n prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);\n prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);\n\n sumf +=\n GGML_E8M0_TO_FP32_HALF(x[ib + 0].e) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * vaddvq_s32(prod_1) +\n GGML_E8M0_TO_FP32_HALF(x[ib + 1].e) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * vaddvq_s32(prod_2);\n }\n\n#endif\n for (; ib < nb; ++ib) {\n const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_E8M0_TO_FP32_HALF(x[ib].e);\n int sumi1 = 0;\n int sumi2 = 0;\n for (int j = 0; j < QK_MXFP4/2; ++j) {\n sumi1 += y[ib].qs[j + 0] * kvalues_mxfp4[x[ib].qs[j] & 0xf];\n sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >> 4];\n }\n sumf += d * (sumi1 + sumi2);\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_NVFP4 == 0);\n\n const block_nvfp4 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n // Each NVFP4 super-block (64 elements) spans 2 q8_0 blocks\n const int nb = n / QK_NVFP4;\n\n float sumf = 0;\n\n#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)\n const int8x16_t values = vld1q_s8(kvalues_mxfp4);\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n float32x4_t acc = vdupq_n_f32(0.0f);\n\n for (int ib = 0; ib < nb; ++ib) {\n const uint8x16_t q4bits_0 = vld1q_u8(x[ib].qs);\n const uint8x16_t q4bits_1 = vld1q_u8(x[ib].qs + 16);\n\n const int8x16_t q4_lo_0 = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits_0, m4b));\n const int8x16_t q4_hi_0 = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits_0, 4));\n const int8x16_t q4_lo_1 = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits_1, m4b));\n const int8x16_t q4_hi_1 = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits_1, 4));\n\n const int8x16_t q8_0a = vld1q_s8(y[2*ib].qs);\n const int8x16_t q8_0b = vld1q_s8(y[2*ib].qs + 16);\n const int8x16_t q8_lo_0 = vcombine_s8(vget_low_s8(q8_0a), vget_low_s8(q8_0b));\n const int8x16_t q8_hi_0 = vcombine_s8(vget_high_s8(q8_0a), vget_high_s8(q8_0b));\n\n const int8x16_t q8_1a = vld1q_s8(y[2*ib+1].qs);\n const int8x16_t q8_1b = vld1q_s8(y[2*ib+1].qs + 16);\n const int8x16_t q8_lo_1 = vcombine_s8(vget_low_s8(q8_1a), vget_low_s8(q8_1b));\n const int8x16_t q8_hi_1 = vcombine_s8(vget_high_s8(q8_1a), vget_high_s8(q8_1b));\n\n const int32x4_t p0 = vaddq_s32(\n ggml_vdotq_s32(vdupq_n_s32(0), q4_lo_0, q8_lo_0),\n ggml_vdotq_s32(vdupq_n_s32(0), q4_hi_0, q8_hi_0));\n const int32x4_t p1 = vaddq_s32(\n ggml_vdotq_s32(vdupq_n_s32(0), q4_lo_1, q8_lo_1),\n ggml_vdotq_s32(vdupq_n_s32(0), q4_hi_1, q8_hi_1));\n\n const int32x4_t sums = vpaddq_s32(p0, p1);\n\n // Decode 4 UE4M3 scales to f32 and multiply with q8 scales\n const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d);\n const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d);\n const float32x4_t nvsc = {\n ggml_ue4m3_to_fp32(x[ib].d[0]),\n ggml_ue4m3_to_fp32(x[ib].d[1]),\n ggml_ue4m3_to_fp32(x[ib].d[2]),\n ggml_ue4m3_to_fp32(x[ib].d[3])\n };\n const float32x4_t scales = vmulq_f32(nvsc, (float32x4_t){dy0, dy0, dy1, dy1});\n\n acc = vfmaq_f32(acc, vcvtq_f32_s32(sums), scales);\n }\n sumf = vaddvq_f32(acc);\n#else\n for (int ib = 0; ib < nb; ++ib) {\n for (int si = 0; si < 4; ++si) {\n const float d = ggml_ue4m3_to_fp32(x[ib].d[si]);\n const int q8b = si / 2;\n const int q8o = (si % 2) * QK_NVFP4_SUB;\n const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8b].d);\n\n int sumi_lo = 0, sumi_hi = 0;\n for (int j = 0; j < QK_NVFP4_SUB/2; ++j) {\n const uint8_t qv = x[ib].qs[si*(QK_NVFP4_SUB/2) + j];\n sumi_lo += y[2*ib + q8b].qs[q8o + j + 0] * kvalues_mxfp4[qv & 0xf];\n sumi_hi += y[2*ib + q8b].qs[q8o + j + QK_NVFP4_SUB/2] * kvalues_mxfp4[qv >> 4];\n }\n sumf += dy * d * (sumi_lo + sumi_hi);\n }\n }\n#endif\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_NEON)\n float32x4_t sumv0 = vdupq_n_f32(0.0f);\n float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n uint32_t qh0;\n uint32_t qh1;\n\n uint64_t tmp0[4];\n uint64_t tmp1[4];\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q5_0 * GGML_RESTRICT x0 = &x[ib];\n const block_q5_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n const uint8x16_t m4b = vdupq_n_u8(0x0F);\n\n // extract the 5th bit via lookup table ((!b) << 4)\n memcpy(&qh0, x0->qh, sizeof(qh0));\n memcpy(&qh1, x1->qh, sizeof(qh1));\n\n tmp0[0] = table_b2b_1[(qh0 >> 0) & 0xFF];\n tmp0[1] = table_b2b_1[(qh0 >> 8) & 0xFF];\n tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];\n tmp0[3] = table_b2b_1[(qh0 >> 24) ];\n\n tmp1[0] = table_b2b_1[(qh1 >> 0) & 0xFF];\n tmp1[1] = table_b2b_1[(qh1 >> 8) & 0xFF];\n tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];\n tmp1[3] = table_b2b_1[(qh1 >> 24) ];\n\n const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));\n const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));\n const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));\n const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));\n\n const uint8x16_t v0_0 = vld1q_u8(x0->qs);\n const uint8x16_t v0_1 = vld1q_u8(x1->qs);\n\n // 4-bit -> 8-bit\n int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));\n int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));\n int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)\n const int8x16_t v0_0lf = vsubq_s8(v0_0l, qhl0);\n const int8x16_t v0_0hf = vsubq_s8(v0_0h, qhh0);\n const int8x16_t v0_1lf = vsubq_s8(v0_1l, qhl1);\n const int8x16_t v0_1hf = vsubq_s8(v0_1h, qhh1);\n\n // load y\n const int8x16_t v1_0l = vld1q_s8(y0->qs);\n const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);\n const int8x16_t v1_1l = vld1q_s8(y1->qs);\n const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);\n\n sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(\n ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),\n ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(\n ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),\n ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);\n\n#endif\n for (; ib < nb; ++ib) {\n uint32_t qh;\n memcpy(&qh, x[ib].qh, sizeof(qh));\n\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;\n const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));\n\n const int32_t x0 = (int8_t)(((x[ib].qs[j] & 0x0F) | xh_0) - 16);\n const int32_t x1 = (int8_t)(((x[ib].qs[j] >> 4) | xh_1) - 16);\n\n sumi0 += (x0 * y[ib].qs[j]);\n sumi1 += (x1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d)) * sumi;\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_1);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_NEON)\n float32x4_t sumv0 = vdupq_n_f32(0.0f);\n float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n float summs0 = 0.0f;\n float summs1 = 0.0f;\n\n uint32_t qh0;\n uint32_t qh1;\n\n uint64_t tmp0[4];\n uint64_t tmp1[4];\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q5_1 * GGML_RESTRICT x0 = &x[ib];\n const block_q5_1 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_1 * GGML_RESTRICT y0 = &y[ib];\n const block_q8_1 * GGML_RESTRICT y1 = &y[ib + 1];\n\n const uint8x16_t m4b = vdupq_n_u8(0x0F);\n\n summs0 += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);\n summs1 += GGML_CPU_FP16_TO_FP32(x1->m) * GGML_CPU_FP16_TO_FP32(y1->s);\n\n // extract the 5th bit via lookup table ((b) << 4)\n memcpy(&qh0, x0->qh, sizeof(qh0));\n memcpy(&qh1, x1->qh, sizeof(qh1));\n\n tmp0[0] = table_b2b_0[(qh0 >> 0) & 0xFF];\n tmp0[1] = table_b2b_0[(qh0 >> 8) & 0xFF];\n tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];\n tmp0[3] = table_b2b_0[(qh0 >> 24) ];\n\n tmp1[0] = table_b2b_0[(qh1 >> 0) & 0xFF];\n tmp1[1] = table_b2b_0[(qh1 >> 8) & 0xFF];\n tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];\n tmp1[3] = table_b2b_0[(qh1 >> 24) ];\n\n const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));\n const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));\n const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));\n const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));\n\n const uint8x16_t v0_0 = vld1q_u8(x0->qs);\n const uint8x16_t v0_1 = vld1q_u8(x1->qs);\n\n // 4-bit -> 8-bit\n const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));\n const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));\n const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n // add high bit\n const int8x16_t v0_0lf = vorrq_s8(v0_0l, qhl0);\n const int8x16_t v0_0hf = vorrq_s8(v0_0h, qhh0);\n const int8x16_t v0_1lf = vorrq_s8(v0_1l, qhl1);\n const int8x16_t v0_1hf = vorrq_s8(v0_1h, qhh1);\n\n // load y\n const int8x16_t v1_0l = vld1q_s8(y0->qs);\n const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);\n const int8x16_t v1_1l = vld1q_s8(y1->qs);\n const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);\n\n sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(\n ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),\n ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(\n ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),\n ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;\n\n#endif\n for (; ib < nb; ++ib) {\n uint32_t qh;\n memcpy(&qh, x[ib].qh, sizeof(qh));\n\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;\n const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;\n\n const int32_t x0 = (x[ib].qs[j] & 0xF) | xh_0;\n const int32_t x1 = (x[ib].qs[j] >> 4) | xh_1;\n\n sumi0 += (x0 * y[ib].qs[j]);\n sumi1 += (x1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n assert((nrc == 2) || (nrc == 1));\n#else\n assert(nrc == 1);\n#endif\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q8_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n if (nrc == 2) {\n const block_q8_0 * GGML_RESTRICT vx0 = vx;\n const block_q8_0 * GGML_RESTRICT vx1 = (const block_q8_0 *) ((const uint8_t*)vx + bx);\n const block_q8_0 * GGML_RESTRICT vy0 = vy;\n const block_q8_0 * GGML_RESTRICT vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by);\n\n float32x4_t sumv0 = vdupq_n_f32(0.0f);\n\n for (int i = 0; i < nb; i++) {\n const block_q8_0 * GGML_RESTRICT b_x0 = &vx0[i];\n const block_q8_0 * GGML_RESTRICT b_y0 = &vy0[i];\n\n const block_q8_0 * GGML_RESTRICT b_x1 = &vx1[i];\n const block_q8_0 * GGML_RESTRICT b_y1 = &vy1[i];\n\n const int8x16_t x0_l = vld1q_s8(b_x0->qs);\n const int8x16_t x0_h = vld1q_s8(b_x0->qs + 16);\n const int8x16_t x1_l = vld1q_s8(b_x1->qs);\n const int8x16_t x1_h = vld1q_s8(b_x1->qs + 16);\n\n // load y\n const int8x16_t y0_l = vld1q_s8(b_y0->qs);\n const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);\n const int8x16_t y1_l = vld1q_s8(b_y1->qs);\n const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);\n\n float32_t _scale[4] = {\n GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y1->d),\n GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y1->d)\n };\n float32x4_t scale = vld1q_f32(_scale);\n\n int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n\n int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n\n int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n\n int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n\n sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),\n l1, r1)), l2, r2)), l3, r3))), scale);\n }\n\n float32x4_t sumv1 = vextq_f32 (sumv0, sumv0, 2);\n float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);\n\n vst1_f32(s, vget_low_f32 (sumv2));\n vst1_f32(s + bs, vget_high_f32(sumv2));\n\n return;\n }\n#endif\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__ARM_FEATURE_SVE)\n svfloat32_t sumv0 = svdup_n_f32(0.0f);\n svfloat32_t sumv1 = svdup_n_f32(0.0f);\n\n const int vector_length = ggml_cpu_get_sve_cnt()*8;\n\n //VLA Implementation for SVE\n switch (vector_length) {\n case 128:\n {\n // predicate for activating lanes for 16 Int8 elements\n const svbool_t ph16 = svptrue_pat_b8 (SV_VL16);\n const svbool_t pl16 = svptrue_pat_b32(SV_VL4);\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q8_0 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q8_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n // load x\n const svint8_t qx0_0 = svld1_s8(ph16, x0->qs);\n const svint8_t qx0_1 = svld1_s8(ph16, x0->qs+16);\n const svint8_t qx1_0 = svld1_s8(ph16, x1->qs);\n const svint8_t qx1_1 = svld1_s8(ph16, x1->qs+16);\n\n // load y\n const svint8_t qy0_0 = svld1_s8(ph16, y0->qs);\n const svint8_t qy0_1 = svld1_s8(ph16, y0->qs+16);\n const svint8_t qy1_0 = svld1_s8(ph16, y1->qs);\n const svint8_t qy1_1 = svld1_s8(ph16, y1->qs+16);\n\n sumv0 = svmla_n_f32_x(pl16, sumv0, svcvt_f32_s32_x(pl16, svadd_x(pl16,\n svdot_s32(svdup_n_s32(0), qx0_0, qy0_0),\n svdot_s32(svdup_n_s32(0), qx0_1, qy0_1))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n sumv1 = svmla_n_f32_x(pl16, sumv1, svcvt_f32_s32_x(pl16, svadd_x(pl16,\n svdot_s32(svdup_n_s32(0), qx1_0, qy1_0),\n svdot_s32(svdup_n_s32(0), qx1_1, qy1_1))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = svaddv_f32(pl16, svadd_f32_x(pl16, sumv0, sumv1));\n } break;\n case 256:\n {\n //printf(\"sve256\");\n for (; ib + 1 < nb; ib += 2) {\n const block_q8_0 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q8_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n // load x\n const svint8_t qx0 = svld1_s8(svptrue_b8(), x0->qs);\n const svint8_t qx1 = svld1_s8(svptrue_b8(), x1->qs);\n\n // load y\n const svint8_t qy0 = svld1_s8(svptrue_b8(), y0->qs);\n const svint8_t qy1 = svld1_s8(svptrue_b8(), y1->qs);\n\n sumv0 = svmla_n_f32_x(svptrue_b32(), sumv0, svcvt_f32_s32_x(svptrue_b32(),\n svdot_s32(svdup_n_s32(0), qx0, qy0)), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n sumv1 = svmla_n_f32_x(svptrue_b32(), sumv1, svcvt_f32_s32_x(svptrue_b32(),\n svdot_s32(svdup_n_s32(0), qx1, qy1)), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1));\n } break;\n case 512:\n {\n // predicate for activating high 256 bit\n const svbool_t ph32 = svptrue_pat_b8(SV_VL32);\n // predicate for activating low 256 bit\n const svbool_t pl32 = svnot_b_z(svptrue_b8(), ph32);\n\n // predicate for activating high lanes for 8 float32 elements\n const svbool_t ph8 = svptrue_pat_b32(SV_VL8);\n // predicate for activating low lanes for 8 float32 elements\n const svbool_t pl8 = svnot_b_z(svptrue_b32(), ph8);\n\n svfloat32_t sumv00 = svdup_n_f32(0.0f);\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q8_0 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q8_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n //load 32 int8_t in first half of vector and put another 32 int8_t in second vector lower bits\n // and add them to make one 64 element vector\n // load x\n const svint8_t qx_32 = svld1_s8(ph32, x0->qs);\n svint8_t qx_64 = svld1_s8(pl32, x0->qs + 2);\n\n qx_64 = svadd_s8_x(svptrue_b8(), qx_32, qx_64);\n\n // load y\n const svint8_t qy_32 = svld1_s8(ph32, y0->qs);\n svint8_t qy_64 = svld1_s8(pl32, y0->qs + 2);\n\n qy_64 = svadd_s8_x(svptrue_b8(), qy_32, qy_64);\n\n // scale creation\n const float32_t deq1 = GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d);\n const float32_t deq2 = GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d);\n\n // duplicate deq1 in first half of vector and deq2 in second half of vector\n const svfloat32_t temp = svdup_f32_m(svdup_f32_z(ph8, deq1), pl8, deq2);\n\n const svfloat32_t sumvt = svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx_64, qy_64));\n\n sumv00 = svmla_f32_m(svptrue_b32(), sumv00, sumvt, temp);\n }\n\n sumf = svaddv_f32(svptrue_b32(), sumv00);\n break;\n }\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n#elif defined(__ARM_NEON)\n float32x4_t sumv0 = vdupq_n_f32(0.0f);\n float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q8_0 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q8_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n const int8x16_t x0_0 = vld1q_s8(x0->qs);\n const int8x16_t x0_1 = vld1q_s8(x0->qs + 16);\n const int8x16_t x1_0 = vld1q_s8(x1->qs);\n const int8x16_t x1_1 = vld1q_s8(x1->qs + 16);\n\n // load y\n const int8x16_t y0_0 = vld1q_s8(y0->qs);\n const int8x16_t y0_1 = vld1q_s8(y0->qs + 16);\n const int8x16_t y1_0 = vld1q_s8(y1->qs);\n const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);\n\n sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(\n ggml_vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),\n ggml_vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n\n sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(\n ggml_vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),\n ggml_vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n }\n\n sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);\n#endif\n for (; ib < nb; ++ib) {\n int sumi = 0;\n\n for (int j = 0; j < qk; j++) {\n sumi += x[ib].qs[j]*y[ib].qs[j];\n }\n\n sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_tq1_0 * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n float sumf = 0.0f;\n\n uint8_t k_shift[16] = {1, 1, 1, 1, 3, 3, 3, 3, 9, 9, 9, 9, 27, 27, 27, 27};\n\n const uint8x16_t shift = vld1q_u8(k_shift);\n\n for (int i = 0; i < nb; ++i) {\n#if defined(__ARM_FEATURE_DOTPROD)\n int32x4_t sumi0 = vdupq_n_s32(0);\n int32x4_t sumi1 = vdupq_n_s32(0);\n#else\n int16x8_t sumi0 = vdupq_n_s16(0);\n int16x8_t sumi1 = vdupq_n_s16(0);\n#endif\n\n // first 32 bytes of 5 elements\n {\n uint8x16_t qx0 = vld1q_u8(x[i].qs + 0);\n uint8x16_t qx1 = vld1q_u8(x[i].qs + 16);\n uint8x16_t qx2 = vmulq_u8(qx0, vdupq_n_u8(3));\n uint8x16_t qx3 = vmulq_u8(qx1, vdupq_n_u8(3));\n uint8x16_t qx4 = vmulq_u8(qx0, vdupq_n_u8(9));\n uint8x16_t qx5 = vmulq_u8(qx1, vdupq_n_u8(9));\n uint8x16_t qx6 = vmulq_u8(qx0, vdupq_n_u8(27));\n uint8x16_t qx7 = vmulq_u8(qx1, vdupq_n_u8(27));\n uint8x16_t qx8 = vmulq_u8(qx0, vdupq_n_u8(81));\n uint8x16_t qx9 = vmulq_u8(qx1, vdupq_n_u8(81));\n\n // multiply by 3 and keep the 2 bits above 8 bits\n int8x16_t sqx0 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx0, vshrq_n_u8(qx0, 1)), 6));\n int8x16_t sqx1 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx1, vshrq_n_u8(qx1, 1)), 6));\n int8x16_t sqx2 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx2, vshrq_n_u8(qx2, 1)), 6));\n int8x16_t sqx3 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx3, vshrq_n_u8(qx3, 1)), 6));\n int8x16_t sqx4 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx4, vshrq_n_u8(qx4, 1)), 6));\n int8x16_t sqx5 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx5, vshrq_n_u8(qx5, 1)), 6));\n int8x16_t sqx6 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx6, vshrq_n_u8(qx6, 1)), 6));\n int8x16_t sqx7 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx7, vshrq_n_u8(qx7, 1)), 6));\n int8x16_t sqx8 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx8, vshrq_n_u8(qx8, 1)), 6));\n int8x16_t sqx9 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx9, vshrq_n_u8(qx9, 1)), 6));\n\n const int8x16_t qy0 = vld1q_s8(y[i].qs + 0);\n const int8x16_t qy1 = vld1q_s8(y[i].qs + 16);\n const int8x16_t qy2 = vld1q_s8(y[i].qs + 32);\n const int8x16_t qy3 = vld1q_s8(y[i].qs + 48);\n const int8x16_t qy4 = vld1q_s8(y[i].qs + 64);\n const int8x16_t qy5 = vld1q_s8(y[i].qs + 80);\n const int8x16_t qy6 = vld1q_s8(y[i].qs + 96);\n const int8x16_t qy7 = vld1q_s8(y[i].qs + 112);\n const int8x16_t qy8 = vld1q_s8(y[i].qs + 128);\n const int8x16_t qy9 = vld1q_s8(y[i].qs + 144);\n\n#if defined(__ARM_FEATURE_DOTPROD)\n sumi0 = vdotq_s32(sumi0, sqx0, qy0);\n sumi1 = vdotq_s32(sumi1, sqx1, qy1);\n sumi0 = vdotq_s32(sumi0, sqx2, qy2);\n sumi1 = vdotq_s32(sumi1, sqx3, qy3);\n sumi0 = vdotq_s32(sumi0, sqx4, qy4);\n sumi1 = vdotq_s32(sumi1, sqx5, qy5);\n sumi0 = vdotq_s32(sumi0, sqx6, qy6);\n sumi1 = vdotq_s32(sumi1, sqx7, qy7);\n sumi0 = vdotq_s32(sumi0, sqx8, qy8);\n sumi1 = vdotq_s32(sumi1, sqx9, qy9);\n#else\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx0), vget_low_s8(qy0));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx0), vget_high_s8(qy0));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx1), vget_low_s8(qy1));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx1), vget_high_s8(qy1));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx2), vget_low_s8(qy2));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx2), vget_high_s8(qy2));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx3), vget_low_s8(qy3));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx3), vget_high_s8(qy3));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx4), vget_low_s8(qy4));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx4), vget_high_s8(qy4));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx5), vget_low_s8(qy5));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx5), vget_high_s8(qy5));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx6), vget_low_s8(qy6));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx6), vget_high_s8(qy6));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx7), vget_low_s8(qy7));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx7), vget_high_s8(qy7));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx8), vget_low_s8(qy8));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx8), vget_high_s8(qy8));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx9), vget_low_s8(qy9));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx9), vget_high_s8(qy9));\n#endif\n }\n\n // last 16 bytes of 5-element, along with the 4 bytes of 4 elements\n {\n uint8x16_t qx0 = vld1q_u8(x[i].qs + 32);\n uint8x16_t qx1 = vmulq_u8(qx0, vdupq_n_u8(3));\n uint8x16_t qx2 = vmulq_u8(qx0, vdupq_n_u8(9));\n uint8x16_t qx3 = vmulq_u8(qx0, vdupq_n_u8(27));\n uint8x16_t qx4 = vmulq_u8(qx0, vdupq_n_u8(81));\n uint32_t qh;\n memcpy(&qh, x[i].qh, sizeof(qh)); // potentially unaligned\n uint8x16_t qx5 = vreinterpretq_u8_u32(vdupq_n_u32(qh));\n qx5 = vmulq_u8(qx5, shift);\n\n // multiply by 3 and keep the 2 bits above 8 bits\n int8x16_t sqx0 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx0, vshrq_n_u8(qx0, 1)), 6));\n int8x16_t sqx1 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx1, vshrq_n_u8(qx1, 1)), 6));\n int8x16_t sqx2 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx2, vshrq_n_u8(qx2, 1)), 6));\n int8x16_t sqx3 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx3, vshrq_n_u8(qx3, 1)), 6));\n int8x16_t sqx4 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx4, vshrq_n_u8(qx4, 1)), 6));\n int8x16_t sqx5 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx5, vshrq_n_u8(qx5, 1)), 6));\n\n const int8x16_t qy0 = vld1q_s8(y[i].qs + 160);\n const int8x16_t qy1 = vld1q_s8(y[i].qs + 176);\n const int8x16_t qy2 = vld1q_s8(y[i].qs + 192);\n const int8x16_t qy3 = vld1q_s8(y[i].qs + 208);\n const int8x16_t qy4 = vld1q_s8(y[i].qs + 224);\n const int8x16_t qy5 = vld1q_s8(y[i].qs + 240);\n\n#if defined(__ARM_FEATURE_DOTPROD)\n sumi0 = vdotq_s32(sumi0, sqx0, qy0);\n sumi1 = vdotq_s32(sumi1, sqx1, qy1);\n sumi0 = vdotq_s32(sumi0, sqx2, qy2);\n sumi1 = vdotq_s32(sumi1, sqx3, qy3);\n sumi0 = vdotq_s32(sumi0, sqx4, qy4);\n sumi1 = vdotq_s32(sumi1, sqx5, qy5);\n#else\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx0), vget_low_s8(qy0));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx0), vget_high_s8(qy0));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx1), vget_low_s8(qy1));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx1), vget_high_s8(qy1));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx2), vget_low_s8(qy2));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx2), vget_high_s8(qy2));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx3), vget_low_s8(qy3));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx3), vget_high_s8(qy3));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx4), vget_low_s8(qy4));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx4), vget_high_s8(qy4));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx5), vget_low_s8(qy5));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx5), vget_high_s8(qy5));\n#endif\n }\n\n const int16x8_t ysum0 = vld1q_s16(y[i].bsums);\n const int16x8_t ysum1 = vld1q_s16(y[i].bsums + 8);\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n#if defined(__ARM_FEATURE_DOTPROD)\n sumi0 = vaddq_s32(sumi0, sumi1);\n sumi0 = vsubq_s32(sumi0, vpaddlq_s16(vaddq_s16(ysum0, ysum1)));\n\n sumf += d * (float) vaddvq_s32(sumi0);\n#else\n sumi0 = vaddq_s16(sumi0, sumi1);\n sumi0 = vsubq_s16(sumi0, vaddq_s16(ysum0, ysum1));\n\n sumf += d * (float) vaddlvq_s16(sumi0);\n#endif\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_tq2_0 * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n float sumf = 0.0f;\n\n const uint8x16_t m3 = vdupq_n_u8(3);\n\n for (int i = 0; i < nb; ++i) {\n#if defined(__ARM_FEATURE_DOTPROD)\n int32x4_t sumi0 = vdupq_n_s32(0);\n int32x4_t sumi1 = vdupq_n_s32(0);\n#else\n int16x8_t sumi0 = vdupq_n_s16(0);\n int16x8_t sumi1 = vdupq_n_s16(0);\n#endif\n\n for (size_t j = 0; j < sizeof(x->qs); j += 32) {\n uint8x16_t qx0 = vld1q_u8(x[i].qs + j);\n uint8x16_t qx1 = vld1q_u8(x[i].qs + j + 16);\n uint8x16_t qx2 = vshrq_n_u8(qx0, 2);\n uint8x16_t qx3 = vshrq_n_u8(qx1, 2);\n uint8x16_t qx4 = vshrq_n_u8(qx0, 4);\n uint8x16_t qx5 = vshrq_n_u8(qx1, 4);\n uint8x16_t qx6 = vshrq_n_u8(qx0, 6);\n uint8x16_t qx7 = vshrq_n_u8(qx1, 6);\n\n int8x16_t sqx0 = vreinterpretq_s8_u8(vandq_u8(qx0, m3));\n int8x16_t sqx1 = vreinterpretq_s8_u8(vandq_u8(qx1, m3));\n int8x16_t sqx2 = vreinterpretq_s8_u8(vandq_u8(qx2, m3));\n int8x16_t sqx3 = vreinterpretq_s8_u8(vandq_u8(qx3, m3));\n int8x16_t sqx4 = vreinterpretq_s8_u8(vandq_u8(qx4, m3));\n int8x16_t sqx5 = vreinterpretq_s8_u8(vandq_u8(qx5, m3));\n int8x16_t sqx6 = vreinterpretq_s8_u8(vandq_u8(qx6, m3));\n int8x16_t sqx7 = vreinterpretq_s8_u8(vandq_u8(qx7, m3));\n\n const int8x16_t qy0 = vld1q_s8(y[i].qs + j*4 + 0);\n const int8x16_t qy1 = vld1q_s8(y[i].qs + j*4 + 16);\n const int8x16_t qy2 = vld1q_s8(y[i].qs + j*4 + 32);\n const int8x16_t qy3 = vld1q_s8(y[i].qs + j*4 + 48);\n const int8x16_t qy4 = vld1q_s8(y[i].qs + j*4 + 64);\n const int8x16_t qy5 = vld1q_s8(y[i].qs + j*4 + 80);\n const int8x16_t qy6 = vld1q_s8(y[i].qs + j*4 + 96);\n const int8x16_t qy7 = vld1q_s8(y[i].qs + j*4 + 112);\n\n#if defined(__ARM_FEATURE_DOTPROD)\n sumi0 = vdotq_s32(sumi0, sqx0, qy0);\n sumi1 = vdotq_s32(sumi1, sqx1, qy1);\n sumi0 = vdotq_s32(sumi0, sqx2, qy2);\n sumi1 = vdotq_s32(sumi1, sqx3, qy3);\n sumi0 = vdotq_s32(sumi0, sqx4, qy4);\n sumi1 = vdotq_s32(sumi1, sqx5, qy5);\n sumi0 = vdotq_s32(sumi0, sqx6, qy6);\n sumi1 = vdotq_s32(sumi1, sqx7, qy7);\n#else\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx0), vget_low_s8(qy0));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx0), vget_high_s8(qy0));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx1), vget_low_s8(qy1));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx1), vget_high_s8(qy1));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx2), vget_low_s8(qy2));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx2), vget_high_s8(qy2));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx3), vget_low_s8(qy3));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx3), vget_high_s8(qy3));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx4), vget_low_s8(qy4));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx4), vget_high_s8(qy4));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx5), vget_low_s8(qy5));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx5), vget_high_s8(qy5));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx6), vget_low_s8(qy6));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx6), vget_high_s8(qy6));\n sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx7), vget_low_s8(qy7));\n sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx7), vget_high_s8(qy7));\n#endif\n }\n\n const int16x8_t ysum0 = vld1q_s16(y[i].bsums);\n const int16x8_t ysum1 = vld1q_s16(y[i].bsums + 8);\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n#if defined(__ARM_FEATURE_DOTPROD)\n sumi0 = vaddq_s32(sumi0, sumi1);\n sumi0 = vsubq_s32(sumi0, vpaddlq_s16(vaddq_s16(ysum0, ysum1)));\n\n sumf += d * (float) vaddvq_s32(sumi0);\n#else\n sumi0 = vaddq_s16(sumi0, sumi1);\n sumi0 = vsubq_s16(sumi0, vaddq_s16(ysum0, ysum1));\n\n sumf += d * (float) vaddlvq_s16(sumi0);\n#endif\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q2_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#ifdef __ARM_FEATURE_SVE\n const int vector_length = svcntb()*8;\n const svuint8_t m3s = svdup_n_u8(0x3);\n const svuint32_t m4s = svdup_n_u32(0xF);\n const svint32_t vzero_sv = svdup_n_s32(0);\n svfloat32_t acc_sum = svdup_n_f32(0);\n svbool_t pred_s32 = svptrue_pat_b32(SV_VL4);\n\n switch (vector_length) {\n case 128:\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n svfloat32_t d_broad = svdup_n_f32((float32_t)d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n svfloat32_t dmin_broad = svdup_n_f32((float32_t)dmin);\n\n const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8_sv = y[i].qs;\n const uint8_t * GGML_RESTRICT sc = x[i].scales;\n\n svuint32_t mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc);\n const svint32_t mins_sv_1 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4));\n\n mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc+4);\n const svint32_t mins_sv_2 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4));\n\n svint32_t q8sums_sv_1 = svld1sh_s32(svptrue_b32(), y[i].bsums);\n svint32_t q8sums_sv_2 = svld1sh_s32(svptrue_b32(), y[i].bsums+4);\n\n const svint32_t s0 = svadd_s32_x(svptrue_b32(), svmul_s32_x(svptrue_b32(), mins_sv_1, q8sums_sv_1), svmul_s32_x(svptrue_b32(), mins_sv_2, q8sums_sv_2));\n\n mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc+8);\n const svint32_t mins_sv_3 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4));\n\n mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc+12);\n const svint32_t mins_sv_4 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4));\n\n q8sums_sv_1 = svld1sh_s32(svptrue_b32(), y[i].bsums+8);\n q8sums_sv_2 = svld1sh_s32(svptrue_b32(), y[i].bsums+12);\n\n svint32_t s1 = svadd_s32_x(svptrue_b32(), svmul_s32_x(svptrue_b32(), mins_sv_3, q8sums_sv_1), svmul_s32_x(svptrue_b32(), mins_sv_4, q8sums_sv_2));\n\n svfloat32_t temp = svcvt_f32_s32_x(svptrue_b32(), svadd_s32_x(svptrue_b32(), s0, s1));\n\n acc_sum = svmla_f32_m(svptrue_b32(), acc_sum, temp, dmin_broad);\n\n svint32_t sumi1 = svdup_n_s32(0);\n\n {\n const svuint8_t q2bits_1 = svld1_u8(svptrue_b8(), q2);\n svint8_t q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_1, m3s));\n svint8_t q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n const svint32_t scales_sv = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc), m4s));\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 0));\n\n const svuint8_t q2bits_3 = svld1_u8(svptrue_b8(), q2+16);\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_3, m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 1));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_1, 2), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 2));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_3, 2), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 3));\n\n\n const svint32_t scales_sv_1 = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc+4), m4s));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_1, 4), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 0));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_3, 4), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 1));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_1, 6), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 2));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_3, 6), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 3));\n\n //-------------------------------\n\n q2 += 32;\n const svint32_t scales_sv_2 = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc+8), m4s));\n const svuint8_t q2bits_2 = svld1_u8(svptrue_b8(), q2);\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_2, m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 0));\n\n const svuint8_t q2bits_4 = svld1_u8(svptrue_b8(), q2+16);\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_4, m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 1));\n\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_2, 2), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 2));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_4, 2), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 3));\n\n\n const svint32_t scales_sv_3 = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc+12), m4s));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_2, 4), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 0));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_4, 4), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 1));\n\n\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_2, 6), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 2));\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_4, 6), m3s));\n q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 3));\n }\n acc_sum = svmla_f32_m(svptrue_b32(), acc_sum, svcvt_f32_s32_x(svptrue_b32(), sumi1), d_broad);\n }\n *s = svaddv_f32(svptrue_b32(), acc_sum);\n break;\n\n case 256:\n case 512:\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n svfloat32_t d_broad = svdup_n_f32((float32_t)d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n svfloat32_t dmin_broad = svdup_n_f32((float32_t)dmin);\n\n const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8_sv = y[i].qs;\n const uint8_t * GGML_RESTRICT sc = x[i].scales;\n\n const svuint32_t mins_and_scales_sve = svld1ub_u32(svptrue_pat_b32(SV_VL8), sc); sc += 8;\n const svint32_t scales_sv = svreinterpret_s32_u32(svand_u32_m(svptrue_pat_b32(SV_VL8), mins_and_scales_sve, m4s));\n const svint32_t mins_sv_1 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_pat_b32(SV_VL8), mins_and_scales_sve, 4));\n svint32_t q8sums_sv_1 = svld1sh_s32(svptrue_pat_b32(SV_VL8), y[i].bsums);\n\n const svuint32_t mins_and_scales_sve_1 = svld1ub_u32(svptrue_pat_b32(SV_VL8), sc);\n const svint32_t scales_sv_1 = svreinterpret_s32_u32(svand_u32_m(svptrue_pat_b32(SV_VL8), mins_and_scales_sve_1, m4s));\n const svint32_t mins_sv_2 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_pat_b32(SV_VL8), mins_and_scales_sve_1, 4));\n\n svint32_t q8sums_sv_2 = svld1sh_s32(svptrue_pat_b32(SV_VL8), y[i].bsums+8);\n\n svfloat32_t temp = svcvt_f32_s32_x(svptrue_pat_b32(SV_VL8), svadd_s32_x(svptrue_pat_b32(SV_VL8), svmul_s32_x(svptrue_pat_b32(SV_VL8), mins_sv_1, q8sums_sv_1), svmul_s32_x(svptrue_pat_b32(SV_VL8), mins_sv_2, q8sums_sv_2)));\n\n acc_sum = svmla_f32_m(svptrue_pat_b32(SV_VL8), acc_sum, temp, dmin_broad);\n\n svint32_t sumi1 = svdup_n_s32(0);\n\n {\n const svuint8_t q2bits_1 = svld1_u8(svptrue_pat_b8(SV_VL32), q2);\n svint8_t q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), q2bits_1, m3s));\n svint8_t q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n svint32_t scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv, 0), svdup_lane_s32(scales_sv, 1));\n sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1);\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_1, 2), m3s));\n q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n svint32_t scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv, 2), svdup_lane_s32(scales_sv, 3));\n sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(svdup_n_s32(0), q2bytes_sv, q8bytes_sv), scale_2);\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_1, 4), m3s));\n q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv, 4), svdup_lane_s32(scales_sv, 5));\n sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1);\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_1, 6), m3s));\n q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv, 6), svdup_lane_s32(scales_sv, 7));\n sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_2);\n\n q2 += 32;\n\n const svuint8_t q2bits_2 = svld1_u8(svptrue_pat_b8(SV_VL32), q2);\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), q2bits_2, m3s));\n q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 0), svdup_lane_s32(scales_sv_1, 1));\n sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1);\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_2, 2), m3s));\n q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 2), svdup_lane_s32(scales_sv_1, 3));\n sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_2);\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_2, 4), m3s));\n q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 4), svdup_lane_s32(scales_sv_1, 5));\n sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1);\n\n q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_2, 6), m3s));\n q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 6), svdup_lane_s32(scales_sv_1, 7));\n sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_2);\n }\n acc_sum = svmla_f32_m(svptrue_pat_b32(SV_VL8), acc_sum, svcvt_f32_s32_x(svptrue_pat_b32(SV_VL8), sumi1), d_broad);\n }\n *s = svaddv_f32(svptrue_pat_b32(SV_VL8), acc_sum);\n break;\n\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n\n#elif __ARM_NEON\n const uint8x16_t m3 = vdupq_n_u8(0x3);\n const uint8x16_t m4 = vdupq_n_u8(0xF);\n\n const int32x4_t vzero = vdupq_n_s32(0);\n\n ggml_int8x16x2_t q2bytes;\n uint8_t aux[16];\n\n float sum = 0;\n\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n const uint8_t * GGML_RESTRICT sc = x[i].scales;\n\n const uint8x16_t mins_and_scales = vld1q_u8(sc);\n const uint8x16_t scales = vandq_u8(mins_and_scales, m4);\n vst1q_u8(aux, scales);\n\n const uint8x16_t mins = vshrq_n_u8(mins_and_scales, 4);\n const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);\n const ggml_int16x8x2_t mins16 = {{vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))}};\n const int32x4_t s0 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[0]), vget_low_s16 (q8sums.val[0])),\n vmull_s16(vget_high_s16(mins16.val[0]), vget_high_s16(q8sums.val[0])));\n const int32x4_t s1 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[1]), vget_low_s16 (q8sums.val[1])),\n vmull_s16(vget_high_s16(mins16.val[1]), vget_high_s16(q8sums.val[1])));\n sum += dmin * vaddvq_s32(vaddq_s32(s0, s1));\n\n int isum = 0;\n int is = 0;\n\n// We use this macro instead of a function call because for some reason\n// the code runs 2-3% slower, even if the function is declared inline\n#define MULTIPLY_ACCUM_WITH_SCALE(index)\\\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * aux[is+(index)];\\\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * aux[is+1+(index)];\n\n#define SHIFT_MULTIPLY_ACCUM_WITH_SCALE(shift, index)\\\n q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\\\n q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[0], (shift)), m3));\\\n q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[1], (shift)), m3));\\\n MULTIPLY_ACCUM_WITH_SCALE((index));\n\n for (int j = 0; j < QK_K/128; ++j) {\n const ggml_uint8x16x2_t q2bits = ggml_vld1q_u8_x2(q2); q2 += 32;\n\n ggml_int8x16x2_t q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\n q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[0], m3));\n q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[1], m3));\n\n MULTIPLY_ACCUM_WITH_SCALE(0);\n\n SHIFT_MULTIPLY_ACCUM_WITH_SCALE(2, 2);\n SHIFT_MULTIPLY_ACCUM_WITH_SCALE(4, 4);\n SHIFT_MULTIPLY_ACCUM_WITH_SCALE(6, 6);\n\n is += 8;\n }\n\n sum += d * isum;\n }\n\n *s = sum;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const uint32_t kmask1 = 0x03030303;\n const uint32_t kmask2 = 0x0f0f0f0f;\n\n const block_q3_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__ARM_FEATURE_SVE)\n\n uint32_t aux[3];\n uint32_t utmp[4];\n\n const int8_t m32 = 32;\n const int vector_length = svcntb()*8;\n const svuint8_t m3b_sv = svdup_n_u8(0x3);\n const svint32_t vzero_sv = svdup_n_s32(0);\n\n const svuint8_t m0_sv = svdup_n_u8(1);\n const svuint8_t m1_sv = svlsl_n_u8_x(svptrue_b8(), m0_sv, 1);\n const svuint8_t m2_sv = svlsl_n_u8_x(svptrue_b8(), m0_sv, 2);\n const svuint8_t m3_sv = svlsl_n_u8_x(svptrue_b8(), m0_sv, 3);\n\n float sum = 0;\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q3_sv = x[i].qs;\n const uint8_t * GGML_RESTRICT qh_sv = x[i].hmask;\n const int8_t * GGML_RESTRICT q8_sv = y[i].qs;\n\n // Set up scales\n memcpy(aux, x[i].scales, 12);\n utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);\n utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);\n utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);\n utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);\n\n int8_t * scale = (int8_t *)utmp;\n\n for (int j = 0; j < 16; ++j) scale[j] -= m32;\n\n switch (vector_length) {\n case 128:\n {\n svuint8_t qhbits_sv_1 = svld1_u8(svptrue_b8(), qh_sv);\n svuint8_t qhbits_sv_2 = svld1_u8(svptrue_b8(), qh_sv+16);\n svuint8_t q3h_sv;\n\n svint32_t sumi1_1 = svdup_n_s32(0);\n svint8_t q3bytes_sv;\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n const svuint8_t q3bits_sv = svld1_u8(svptrue_b8(), q3_sv); q3_sv += 16;\n const svuint8_t q3bits_sv_1 = svld1_u8(svptrue_b8(), q3_sv); q3_sv += 16;\n svint8_t q8bytes_1_sv_1 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n svint8_t q8bytes_1_sv_2 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n q3h_sv = svlsl_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m0_sv, qhbits_sv_1), 2);\n q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), q3bits_sv, m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), svdup_n_s32((int32_t)scale[0]));\n\n q3h_sv = svlsl_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m0_sv, qhbits_sv_2), 2);\n q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), q3bits_sv_1, m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), svdup_n_s32((int32_t)scale[1]));\n\n q8bytes_1_sv_1 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n q8bytes_1_sv_2 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n q3h_sv = svlsl_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m1_sv, qhbits_sv_1), 1);\n q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv, 2), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), svdup_n_s32((int32_t)scale[2]));\n\n q3h_sv = svlsl_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m1_sv, qhbits_sv_2), 1);\n q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv_1, 2), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), svdup_n_s32((int32_t)scale[3]));\n\n\n scale += 4;\n q8bytes_1_sv_1 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n q8bytes_1_sv_2 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n q3h_sv = svbic_u8_x(svptrue_b8(), m2_sv, qhbits_sv_1);\n q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv, 4), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), svdup_n_s32((int32_t)scale[0]));\n\n q3h_sv = svbic_u8_x(svptrue_b8(), m2_sv, qhbits_sv_2);\n q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv_1, 4), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), svdup_n_s32((int32_t)scale[1]));\n\n\n q8bytes_1_sv_1 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n q8bytes_1_sv_2 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n q3h_sv = svlsr_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m3_sv, qhbits_sv_1), 1);\n q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv, 6), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), svdup_n_s32((int32_t)scale[2]));\n\n q3h_sv = svlsr_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m3_sv, qhbits_sv_2), 1);\n q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv_1, 6), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), svdup_n_s32((int32_t)scale[3]));\n\n if (j == 0) {\n qhbits_sv_1 = svlsr_n_u8_x(svptrue_b8(), qhbits_sv_1, 4);\n qhbits_sv_2 = svlsr_n_u8_x(svptrue_b8(), qhbits_sv_2, 4);\n }\n\n scale += 4;\n }\n\n sum += d * (svaddv_s32(svptrue_b32(), sumi1_1));\n } break;\n case 256:\n case 512:\n {\n svuint8_t qhbits_sv = svld1_u8(svptrue_pat_b8(SV_VL32), qh_sv);\n svuint8_t q3h_sv;\n\n svint32_t sumi1_1 = svdup_n_s32(0);\n svint8_t q3bytes_sv;\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n const svuint8_t q3bits_sv = svld1_u8(svptrue_pat_b8(SV_VL32), q3_sv); q3_sv += 32;\n svint8_t q8bytes_1_sv_1 = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n svint8_t q8bytes_1_sv_2 = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n q3h_sv = svlsl_n_u8_x(svptrue_pat_b8(SV_VL32), svbic_u8_x(svptrue_pat_b8(SV_VL32), m0_sv, qhbits_sv), 2);\n q3bytes_sv = svsub_s8_x(svptrue_pat_b8(SV_VL32), svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), q3bits_sv, m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n\n svint32_t scale_1 = svsel_s32(svptrue_pat_b32(SV_VL4), svdup_n_s32((int32_t)scale[0]), svdup_n_s32((int32_t)scale[1]));\n sumi1_1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), scale_1);\n\n q3h_sv = svlsl_n_u8_x(svptrue_pat_b8(SV_VL32), svbic_u8_x(svptrue_pat_b8(SV_VL32), m1_sv, qhbits_sv), 1);\n q3bytes_sv = svsub_s8_x(svptrue_pat_b8(SV_VL32), svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q3bits_sv, 2), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n scale_1 = svsel_s32(svptrue_pat_b32(SV_VL4), svdup_n_s32((int32_t)scale[2]), svdup_n_s32((int32_t)scale[3]));\n sumi1_1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), scale_1);\n\n scale += 4;\n q8bytes_1_sv_1 = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n q8bytes_1_sv_2 = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n q3h_sv = svbic_u8_x(svptrue_pat_b8(SV_VL32), m2_sv, qhbits_sv);\n q3bytes_sv = svsub_s8_x(svptrue_pat_b8(SV_VL32), svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q3bits_sv, 4), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n scale_1 = svsel_s32(svptrue_pat_b32(SV_VL4), svdup_n_s32((int32_t)scale[0]), svdup_n_s32((int32_t)scale[1]));\n sumi1_1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), scale_1);\n\n q3h_sv = svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), svbic_u8_x(svptrue_pat_b8(SV_VL32), m3_sv, qhbits_sv), 1);\n q3bytes_sv = svsub_s8_x(svptrue_pat_b8(SV_VL32), svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q3bits_sv, 6), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n scale_1 = svsel_s32(svptrue_pat_b32(SV_VL4), svdup_n_s32((int32_t)scale[2]), svdup_n_s32((int32_t)scale[3]));\n sumi1_1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), scale_1);\n\n if (j == 0) {\n qhbits_sv = svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), qhbits_sv, 4);\n }\n\n scale += 4;\n }\n\n sum += d * (svaddv_s32(svptrue_pat_b32(SV_VL8), sumi1_1));\n } break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n }\n *s = sum;\n\n#elif __ARM_NEON\n\n uint32_t aux[3];\n uint32_t utmp[4];\n\n const uint8x16_t m3b = vdupq_n_u8(0x3);\n const int32x4_t vzero = vdupq_n_s32(0);\n\n const uint8x16_t m0 = vdupq_n_u8(1);\n const uint8x16_t m1 = vshlq_n_u8(m0, 1);\n const uint8x16_t m2 = vshlq_n_u8(m0, 2);\n const uint8x16_t m3 = vshlq_n_u8(m0, 3);\n const int8_t m32 = 32;\n\n ggml_int8x16x4_t q3bytes;\n\n float sum = 0;\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].hmask;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);\n\n ggml_uint8x16x4_t q3h;\n\n int32_t isum = 0;\n\n // Set up scales\n memcpy(aux, x[i].scales, 12);\n utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);\n utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);\n utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);\n utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);\n\n int8_t * scale = (int8_t *)utmp;\n for (int j = 0; j < 16; ++j) scale[j] -= m32;\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n const ggml_uint8x16x2_t q3bits = ggml_vld1q_u8_x2(q3); q3 += 32;\n const ggml_int8x16x4_t q8bytes_1 = ggml_vld1q_s8_x4(q8); q8 += 64;\n const ggml_int8x16x4_t q8bytes_2 = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n q3h.val[0] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[0]), 2);\n q3h.val[1] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[1]), 2);\n q3h.val[2] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[0]), 1);\n q3h.val[3] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[1]), 1);\n\n q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[0], m3b)), vreinterpretq_s8_u8(q3h.val[0]));\n q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[1], m3b)), vreinterpretq_s8_u8(q3h.val[1]));\n q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 2), m3b)), vreinterpretq_s8_u8(q3h.val[2]));\n q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 2), m3b)), vreinterpretq_s8_u8(q3h.val[3]));\n\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_1.val[0])) * scale[0];\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_1.val[1])) * scale[1];\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_1.val[2])) * scale[2];\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_1.val[3])) * scale[3];\n\n scale += 4;\n\n q3h.val[0] = vbicq_u8(m2, qhbits.val[0]);\n q3h.val[1] = vbicq_u8(m2, qhbits.val[1]);\n q3h.val[2] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[0]), 1);\n q3h.val[3] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[1]), 1);\n\n q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 4), m3b)), vreinterpretq_s8_u8(q3h.val[0]));\n q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 4), m3b)), vreinterpretq_s8_u8(q3h.val[1]));\n q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 6), m3b)), vreinterpretq_s8_u8(q3h.val[2]));\n q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 6), m3b)), vreinterpretq_s8_u8(q3h.val[3]));\n\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_2.val[0])) * scale[0];\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_2.val[1])) * scale[1];\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_2.val[2])) * scale[2];\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_2.val[3])) * scale[3];\n\n scale += 4;\n\n if (j == 0) {\n qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 4);\n qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 4);\n }\n\n }\n sum += d * isum;\n\n }\n\n *s = sum;\n\n#else\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n\n}\n\n#ifdef __ARM_FEATURE_SVE\nstatic inline svuint32_t ggml_decode_q4scales_and_mins_for_mmla(const uint32_t * vx_scales) {\n const svbool_t pg_all = svptrue_pat_b32(SV_VL4);\n const svbool_t pg_false = svpfalse_b(); // 0x0000\n const svbool_t pg_lo_8 = svwhilelt_b8_s32(0, 8); // 0x00ff\n const svbool_t pg_odd = svzip1_b32(pg_false, pg_lo_8);\n\n svuint32_t vutmp_hi, vutmp_lo;\n svuint32_t vx01 = svld1_u32(pg_lo_8, vx_scales);\n vutmp_hi = svzip1_u32(vx01, vx01);\n vutmp_hi = svlsr_n_u32_m(pg_odd, vutmp_hi, 2);\n vutmp_hi = svreinterpret_u32_u64(svand_n_u64_x(pg_all, svreinterpret_u64_u32(vutmp_hi), UINT64_C(0x303030303f3f3f3f)));\n const svuint32_t vx2 = svdup_u32(vx_scales[2]);\n vutmp_lo = svlsr_u32_x(pg_all, vx2, svreinterpret_u32_s32(svindex_s32(-2, 2)));\n vutmp_lo = svand_n_u32_z(pg_odd, vutmp_lo, UINT32_C(0x0f0f0f0f));\n svuint32_t vutmp = svorr_u32_z(pg_all, vutmp_hi, vutmp_lo);\n return vutmp;\n}\n#endif\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n#ifdef __ARM_FEATURE_MATMUL_INT8\n assert((nrc == 2) || (nrc == 1));\n#else\n assert(nrc == 1);\n#endif\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n#ifdef __ARM_FEATURE_SVE\n const int vector_length = ggml_cpu_get_sve_cnt()*8;\n#endif\n\n#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n if (nrc == 2) {\n svbool_t pg32_2 = svptrue_pat_b32(SV_VL2);\n\n const block_q4_K * GGML_RESTRICT vx0 = vx;\n const block_q8_K * GGML_RESTRICT vy0 = vy;\n const block_q4_K * GGML_RESTRICT vx1 = (const block_q4_K *) ((const uint8_t*)vx + bx);\n const block_q8_K * GGML_RESTRICT vy1 = (const block_q8_K *) ((const uint8_t*)vy + by);\n\n union {\n uint32_t u32[8];\n uint64_t u64[4];\n } new_utmp;\n\n svfloat32_t sumf1 = svdup_n_f32(0);\n\n switch (vector_length) {\n case 128:\n {\n svbool_t pg_false = svpfalse_b();\n svbool_t pg_lo_8 = svwhilelt_b8_s32(0, 8);\n svbool_t vmins_mask1= svzip1_b32(pg_lo_8, pg_false);\n svbool_t vmins_mask2 = svzip1_b32(pg_false, pg_lo_8);\n svbool_t pg128_all = svptrue_pat_b8(SV_VL16);\n for (int i = 0; i < nb; ++i) {\n svfloat32_t vy_d = svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d));\n svfloat32_t vx_d = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].d)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].d)));\n svfloat32_t svsuper_block_scales = svmul_f32_x(pg128_all, vy_d, vx_d);\n svfloat32_t vx_dmins = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].dmin)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].dmin)));\n svfloat32_t vy_dmins = svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d));\n svfloat32_t svdmins = svmul_n_f32_x(pg128_all, svmul_f32_x(pg128_all, vy_dmins, vx_dmins), -1);\n const uint8_t * GGML_RESTRICT q4_0 = vx0[i].qs;\n const int8_t * GGML_RESTRICT q8_0 = vy0[i].qs;\n const uint8_t * GGML_RESTRICT q4_1 = vx1[i].qs;\n const int8_t * GGML_RESTRICT q8_1 = vy1[i].qs;\n svint16_t lo = svld1_s16(pg128_all, vy0[i].bsums + 0);\n svint16_t hi = svld1_s16(pg128_all, vy0[i].bsums + 8);\n svint16_t sum_tmp1 = svuzp1_s16(lo, hi);\n svint16_t sum_tmp2 = svuzp2_s16(lo, hi);\n svint16_t svq8sums_0 = svadd_s16_x(pg128_all, sum_tmp1, sum_tmp2);\n lo = svld1_s16(pg128_all, vy1[i].bsums + 0);\n hi = svld1_s16(pg128_all, vy1[i].bsums + 8);\n sum_tmp1 = svuzp1(lo, hi);\n sum_tmp2 = svuzp2(lo, hi);\n svint16_t svq8sums_1 = svadd_s16_x(pg128_all, sum_tmp1, sum_tmp2);\n svuint32_t decoded_scales0 = ggml_decode_q4scales_and_mins_for_mmla((const uint32_t *)vx0[i].scales);\n svuint32_t decoded_scales1 = ggml_decode_q4scales_and_mins_for_mmla((const uint32_t *)vx1[i].scales);\n svuint32x2_t decoded_scales = svcreate2_u32(decoded_scales0, decoded_scales1);\n svst2_u32(pg128_all, new_utmp.u32, decoded_scales);\n svint16_t svmins8_0 = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u32(svuzp1_u32(svld1_u32(vmins_mask1, new_utmp.u32+4), svdup_n_u32(0)))));\n svint16_t svmins8_1 = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u32(svuzp2_u32(svld1_u32(vmins_mask2, new_utmp.u32+4), svdup_n_u32(0)))));\n svint32_t svsumfs_tmp1 = svreinterpret_s32_s64(svdot_s64(svdup_n_s64(0), svq8sums_0, svmins8_0));\n svint32_t svsumfs_tmp2 = svreinterpret_s32_s64(svdot_s64(svdup_n_s64(0), svq8sums_0, svmins8_1));\n svint32_t svsumfs_tmp3 = svtrn1_s32(svsumfs_tmp1, svsumfs_tmp2);\n svint32_t svsumfs_tmp4 = svreinterpret_s32_s64(svdot_s64(svdup_n_s64(0), svq8sums_1, svmins8_0));\n svint32_t svsumfs_tmp5 = svreinterpret_s32_s64(svdot_s64(svdup_n_s64(0), svq8sums_1, svmins8_1));\n svint32_t svsumfs_tmp6 = svtrn1_s32(svsumfs_tmp4, svsumfs_tmp5);\n svint32_t svsumfs_tmp7 = svreinterpret_s32_s64(svtrn2_s64(svreinterpret_s64_s32(svsumfs_tmp3), svreinterpret_s64_s32(svsumfs_tmp6)));\n svint32_t svsumfs_tmp8 = svreinterpret_s32_s64(svtrn1_s64(svreinterpret_s64_s32(svsumfs_tmp3), svreinterpret_s64_s32(svsumfs_tmp6)));\n svint32_t svsumfs_tmp = svadd_s32_x(pg128_all, svsumfs_tmp7, svsumfs_tmp8);\n svint32_t svscales, sumi1, sumi2;\n svint32_t acc_sumif1 = svdup_n_s32(0);\n svint32_t acc_sumif2 = svdup_n_s32(0);\n svint8_t q4bytes_0_l, q4bytes_0_h, q4bytes_1_l, q4bytes_1_h, l0, l1, l2, l3,\n q8bytes_0_h, q8bytes_0_l, q8bytes_1_h, q8bytes_1_l, r0, r1, r2, r3;\n#pragma GCC unroll 1\n for (int j = 0; j < QK_K/64; ++j) {\n q4bytes_0_l = svreinterpret_s8_u8(svand_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_0), 0xf));\n q4bytes_1_l = svreinterpret_s8_u8(svand_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_1), 0xf));\n q4bytes_0_h = svreinterpret_s8_u8(svand_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_0+16), 0xf));\n q4bytes_1_h = svreinterpret_s8_u8(svand_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_1+16), 0xf));\n l0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q4bytes_0_l), svreinterpret_s64_s8(q4bytes_1_l)));\n l1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q4bytes_0_l), svreinterpret_s64_s8(q4bytes_1_l)));\n l2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q4bytes_0_h), svreinterpret_s64_s8(q4bytes_1_h)));\n l3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q4bytes_0_h), svreinterpret_s64_s8(q4bytes_1_h)));\n q8bytes_0_h = svld1_s8(pg128_all, q8_0);\n q8bytes_1_h = svld1_s8(pg128_all, q8_1);\n q8bytes_0_l = svld1_s8(pg128_all, q8_0+16);\n q8bytes_1_l = svld1_s8(pg128_all, q8_1+16);\n r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0_h), svreinterpret_s64_s8(q8bytes_1_h)));\n r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0_h), svreinterpret_s64_s8(q8bytes_1_h)));\n r2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0_l), svreinterpret_s64_s8(q8bytes_1_l)));\n r3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0_l), svreinterpret_s64_s8(q8bytes_1_l)));\n sumi1 = svmmla_s32(svmmla_s32(svmmla_s32(svmmla_s32(svdup_n_s32(0), r0, l0), r1, l1), r2, l2), r3, l3);\n svscales = svreinterpret_s32_u32(svlsr_n_u32_x(pg128_all, svlsl_n_u32_x(pg128_all, svreinterpret_u32_u64(svdup_n_u64(new_utmp.u64[j/2])), 8*(4-2*(j%2)-1)), 24));\n acc_sumif1 = svmla_s32_x(pg128_all, acc_sumif1, svscales, sumi1);\n\n q4bytes_0_l = svreinterpret_s8_u8(svlsr_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_0), 4));\n q4bytes_1_l = svreinterpret_s8_u8(svlsr_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_1), 4));\n q4bytes_0_h = svreinterpret_s8_u8(svlsr_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_0+16), 4));\n q4bytes_1_h = svreinterpret_s8_u8(svlsr_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_1+16), 4));\n l0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q4bytes_0_l), svreinterpret_s64_s8(q4bytes_1_l)));\n l1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q4bytes_0_l), svreinterpret_s64_s8(q4bytes_1_l)));\n l2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q4bytes_0_h), svreinterpret_s64_s8(q4bytes_1_h)));\n l3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q4bytes_0_h), svreinterpret_s64_s8(q4bytes_1_h)));\n q8bytes_0_h = svld1_s8(pg128_all, q8_0+32);\n q8bytes_1_h = svld1_s8(pg128_all, q8_1+32);\n q8bytes_0_l = svld1_s8(pg128_all, q8_0+48);\n q8bytes_1_l = svld1_s8(pg128_all, q8_1+48);\n r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0_h), svreinterpret_s64_s8(q8bytes_1_h)));\n r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0_h), svreinterpret_s64_s8(q8bytes_1_h)));\n r2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0_l), svreinterpret_s64_s8(q8bytes_1_l)));\n r3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0_l), svreinterpret_s64_s8(q8bytes_1_l)));\n sumi2 = svmmla_s32(svmmla_s32(svmmla_s32(svmmla_s32(svdup_n_s32(0), r0, l0), r1, l1), r2, l2), r3, l3);\n svscales = svreinterpret_s32_u32(svlsr_n_u32_x(pg128_all, svlsl_n_u32_x(pg128_all, svreinterpret_u32_u64(svdup_n_u64(new_utmp.u64[j/2])), 8*(4-2*(j%2)-2)), 24));\n acc_sumif2 = svmla_s32_x(pg128_all, acc_sumif2, svscales, sumi2);\n q4_0 += 32; q4_1 += 32; q8_0 += 64; q8_1 += 64;\n }\n sumf1 = svmla_f32_x(pg128_all,\n svmla_f32_x(pg128_all,\n sumf1,\n svcvt_f32_x(pg128_all,\n svadd_s32_x(pg128_all, acc_sumif1, acc_sumif2)),\n svsuper_block_scales),\n svdmins,\n svcvt_f32_s32_x(pg128_all, svsumfs_tmp));\n } //end of for nb\n } // end of case 128\n break;\n case 256:\n case 512:\n {\n const svbool_t pg32_4 = svptrue_pat_b32(SV_VL4);\n const svbool_t pg8_16 = svptrue_pat_b8(SV_VL16);\n const svbool_t pg256_all = svptrue_pat_b8(SV_ALL);\n for (int i = 0; i < nb; ++i) {\n const uint8_t * GGML_RESTRICT q4_0 = vx0[i].qs;\n const int8_t * GGML_RESTRICT q8_0 = vy0[i].qs;\n const uint8_t * GGML_RESTRICT q4_1 = vx1[i].qs;\n const int8_t * GGML_RESTRICT q8_1 = vy1[i].qs;\n svint32_t svscales, sumi1, sumi2;\n svint32_t acc_sumif1 = svdup_n_s32(0);\n svint32_t acc_sumif2 = svdup_n_s32(0);\n svint8_t l0, l1, l2, l3, r0, r1, r2, r3;\n svfloat32_t vx_d = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].d)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].d)));\n svfloat64_t vy_d_tmp = svreinterpret_f64_f32(svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d)));\n svfloat32_t vy_d = svreinterpret_f32_f64(svuzp1_f64(vy_d_tmp, vy_d_tmp));\n svfloat32_t svsuper_block_scales = svmul_f32_z(pg32_4, vy_d, vx_d);\n svfloat32_t vx_dmins = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].dmin)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].dmin)));\n svfloat64_t vy_dmins_tmp = svreinterpret_f64_f32(svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d)));\n svfloat32_t vy_dmins = svreinterpret_f32_f64(svuzp1_f64(vy_dmins_tmp, vy_dmins_tmp));\n svfloat32_t svdmins = svmul_n_f32_x(pg32_4, svmul_f32_x(pg32_4, vx_dmins, vy_dmins), -1);\n svint16_t rc1 = svuzp1_s16(svld1_s16(pg256_all, vy0[i].bsums), svld1_s16(pg256_all, vy1[i].bsums));\n svint16_t rc2 = svuzp2_s16(svld1_s16(pg256_all, vy0[i].bsums), svld1_s16(pg256_all, vy1[i].bsums));\n svint16_t svq8sums = svadd_s16_x(pg256_all, rc1, rc2);\n svuint32_t decoded_scales0 = ggml_decode_q4scales_and_mins_for_mmla((const uint32_t *)vx0[i].scales);\n svuint32_t decoded_scales1 = ggml_decode_q4scales_and_mins_for_mmla((const uint32_t *)vx1[i].scales);\n svuint32x2_t decoded_scales = svcreate2_u32(decoded_scales0, decoded_scales1);\n svst2_u32(pg8_16, new_utmp.u32, decoded_scales);\n svint16_t new_svq8sums_0 = svreinterpret_s16_u64(svtrn1_u64(svreinterpret_u64_s16(svq8sums), svreinterpret_u64_s16(svq8sums)));\n svint16_t new_svq8sums_1 = svreinterpret_s16_u64(svtrn2_u64(svreinterpret_u64_s16(svq8sums), svreinterpret_u64_s16(svq8sums)));\n svuint64_t new_mins_0 = svdup_u64(new_utmp.u64[2]);\n svuint64_t new_mins_1 = svdup_u64(new_utmp.u64[3]);\n svint16_t new_svmins8_0 = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u64(new_mins_0)));\n svint16_t new_svmins8_1 = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u64(new_mins_1)));\n svint64_t dot_prod_0 = svdot_s64(svdup_s64(0), new_svmins8_0, new_svq8sums_0);\n svint64_t dot_prod_1 = svdot_s64(dot_prod_0, new_svmins8_1, new_svq8sums_1);\n svfloat32_t converted_dot_prod_1 = svcvt_f32_s64_x(pg256_all, dot_prod_1);\n svfloat32_t svsumfs_tmp = svuzp1_f32(converted_dot_prod_1, converted_dot_prod_1);\n\n#pragma GCC unroll 1\n for (int j = 0; j < QK_K/64; ++j) {\n svuint8_t q4bytes_0 = svand_n_u8_x(pg256_all, svld1_u8(pg256_all, q4_0), 0xf);\n svuint8_t q4bytes_1 = svand_n_u8_x(pg256_all, svld1_u8(pg256_all, q4_1), 0xf);\n svuint8_t q4bytes_2 = svlsr_n_u8_x(pg256_all, svld1_u8(pg256_all, q4_0), 4);\n svuint8_t q4bytes_3 = svlsr_n_u8_x(pg256_all, svld1_u8(pg256_all, q4_1), 4);\n l0 = svreinterpret_s8_u64(svzip1_u64(svreinterpret_u64_u8(q4bytes_0), svreinterpret_u64_u8(q4bytes_1)));\n l1 = svreinterpret_s8_u64(svzip2_u64(svreinterpret_u64_u8(q4bytes_0), svreinterpret_u64_u8(q4bytes_1)));\n l2 = svreinterpret_s8_u64(svzip1_u64(svreinterpret_u64_u8(q4bytes_2), svreinterpret_u64_u8(q4bytes_3)));\n l3 = svreinterpret_s8_u64(svzip2_u64(svreinterpret_u64_u8(q4bytes_2), svreinterpret_u64_u8(q4bytes_3)));\n svint8_t q8bytes_0 = svld1_s8(pg256_all, q8_0);\n svint8_t q8bytes_1 = svld1_s8(pg256_all, q8_1);\n svint8_t q8bytes_2 = svld1_s8(pg256_all, q8_0+32);\n svint8_t q8bytes_3 = svld1_s8(pg256_all, q8_1+32);\n r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n r2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_2), svreinterpret_s64_s8(q8bytes_3)));\n r3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_2), svreinterpret_s64_s8(q8bytes_3)));\n sumi1 = svmmla(svmmla(svdup_n_s32(0), r0, l0), r1, l1);\n svscales = svreinterpret_s32_u32(svlsr_n_u32_x(pg256_all, svlsl_n_u32_x(pg256_all, svreinterpret_u32_u64(svdup_n_u64(new_utmp.u64[j/2])), 8*(4-2*(j%2)-1)), 24));\n acc_sumif1 = svmla_s32_x(pg256_all, acc_sumif1, svscales, sumi1);\n sumi2 = svmmla(svmmla(svdup_n_s32(0), r2, l2), r3, l3);\n svscales = svreinterpret_s32_u32(svlsr_n_u32_x(pg256_all, svlsl_n_u32_x(pg256_all, svreinterpret_u32_u64(svdup_n_u64(new_utmp.u64[j/2])), 8*(4-2*(j%2)-2)), 24));\n acc_sumif2 = svmla_s32_x(pg256_all, acc_sumif2, svscales, sumi2);\n q4_0 += 32; q4_1 += 32; q8_0 += 64; q8_1 += 64;\n }\n svint32_t acc_sumif = svadd_s32_x(pg256_all, acc_sumif1, acc_sumif2);\n svint32_t swap_acc_sumif = svext_s32(acc_sumif, acc_sumif, 4);\n acc_sumif = svadd_s32_x(pg32_4, acc_sumif, swap_acc_sumif);\n sumf1 = svmla_f32_x(pg32_4,\n svmla_f32_x(pg32_4,\n sumf1,\n svcvt_f32_x(pg32_4, acc_sumif),\n svsuper_block_scales),\n svdmins,\n svsumfs_tmp);\n } // end of for nb\n } // end of case 256-512\n break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n\n svst1_f32(pg32_2, s, sumf1);\n svst1_f32(pg32_2, s + bs, svreinterpret_f32_u8(svext_u8(svreinterpret_u8_f32(sumf1), svdup_n_u8(0), 8)));\n\n return;\n }\n#elif defined(__ARM_FEATURE_MATMUL_INT8)\n if (nrc == 2) {\n const block_q4_K * GGML_RESTRICT x0 = x;\n const block_q4_K * GGML_RESTRICT x1 = (const block_q4_K *) ((const uint8_t *)vx + bx);\n const block_q8_K * GGML_RESTRICT y0 = y;\n const block_q8_K * GGML_RESTRICT y1 = (const block_q8_K *) ((const uint8_t *)vy + by);\n\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n\n float32x4_t vfsum = vdupq_n_f32(0.0f);\n\n for (int i = 0; i < nb; ++i, ++x0, ++x1, ++y0, ++y1) {\n const uint8_t * GGML_RESTRICT qx0 = x0->qs;\n const uint8_t * GGML_RESTRICT qx1 = x1->qs;\n const int8_t * GGML_RESTRICT qy0 = y0->qs;\n const int8_t * GGML_RESTRICT qy1 = y1->qs;\n\n // decode scales and mins\n int8_t x0_scales[8], x1_scales[8];\n int16x8_t x0_mins, x1_mins;\n {\n uint32_t scales_mins[3];\n memcpy(scales_mins, x0->scales, 12);\n const uint32_t mins_0_3 = scales_mins[1] & kmask1;\n const uint32_t mins_4_7 = ((scales_mins[2] >> 4) & kmask2) | (((scales_mins[1] >> 6) & kmask3) << 4);\n const uint32x2_t mins = {mins_0_3, mins_4_7};\n x0_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins)));\n uint32_t scales[2];\n scales[0] = scales_mins[0] & kmask1; // scales 0~3\n scales[1] = (scales_mins[2] & kmask2) | (((scales_mins[0] >> 6) & kmask3) << 4); // scales 4~7\n memcpy(x0_scales, scales, 8);\n }\n {\n uint32_t scales_mins[3];\n memcpy(scales_mins, x1->scales, 12);\n const uint32_t mins_0_3 = scales_mins[1] & kmask1;\n const uint32_t mins_4_7 = ((scales_mins[2] >> 4) & kmask2) | (((scales_mins[1] >> 6) & kmask3) << 4);\n const uint32x2_t mins = {mins_0_3, mins_4_7};\n x1_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins)));\n uint32_t scales[2];\n scales[0] = scales_mins[0] & kmask1; // scales 0~3\n scales[1] = (scales_mins[2] & kmask2) | (((scales_mins[0] >> 6) & kmask3) << 4); // scales 4~7\n memcpy(x1_scales, scales, 8);\n }\n\n int32x4_t visum = {0};\n\n // process 64 data points per iteration, totally 256 data points\n for (int j = 0; j < QK_K / 64; ++j, qx0 += 32, qx1 += 32, qy0 += 64, qy1 += 64) {\n const int8x16x4_t vy0 = vld1q_s8_x4(qy0);\n const int8x16x4_t vy1 = vld1q_s8_x4(qy1);\n\n int8x16_t vx0[4], vx1[4];\n {\n const uint8x16x2_t vv = vld1q_u8_x2(qx0);\n vx0[0] = vreinterpretq_s8_u8(vandq_u8(vv.val[0], m4b));\n vx0[1] = vreinterpretq_s8_u8(vandq_u8(vv.val[1], m4b));\n vx0[2] = vreinterpretq_s8_u8(vshrq_n_u8(vv.val[0], 4));\n vx0[3] = vreinterpretq_s8_u8(vshrq_n_u8(vv.val[1], 4));\n }\n {\n const uint8x16x2_t vv = vld1q_u8_x2(qx1);\n vx1[0] = vreinterpretq_s8_u8(vandq_u8(vv.val[0], m4b));\n vx1[1] = vreinterpretq_s8_u8(vandq_u8(vv.val[1], m4b));\n vx1[2] = vreinterpretq_s8_u8(vshrq_n_u8(vv.val[0], 4));\n vx1[3] = vreinterpretq_s8_u8(vshrq_n_u8(vv.val[1], 4));\n }\n\n // process 32 data points (share same block scale) per iteration\n for (int k = 0; k < 2; ++k) {\n const int blk = j * 2 + k;\n const int32x4_t block_scale = {\n x0_scales[blk],\n x0_scales[blk],\n x1_scales[blk],\n x1_scales[blk],\n };\n\n int32x4_t vr = {0};\n for (int l = 0; l < 2; ++l) {\n const int idx = k * 2 + l;\n const int64x2_t vx0_s64 = vreinterpretq_s64_s8(vx0[idx]);\n const int64x2_t vx1_s64 = vreinterpretq_s64_s8(vx1[idx]);\n const int64x2_t vy0_s64 = vreinterpretq_s64_s8(vy0.val[idx]);\n const int64x2_t vy1_s64 = vreinterpretq_s64_s8(vy1.val[idx]);\n const int8x16_t vx_l = vreinterpretq_s8_s64(vzip1q_s64(vx0_s64, vx1_s64));\n const int8x16_t vx_h = vreinterpretq_s8_s64(vzip2q_s64(vx0_s64, vx1_s64));\n const int8x16_t vy_l = vreinterpretq_s8_s64(vzip1q_s64(vy0_s64, vy1_s64));\n const int8x16_t vy_h = vreinterpretq_s8_s64(vzip2q_s64(vy0_s64, vy1_s64));\n vr = vmmlaq_s32(vr, vx_l, vy_l);\n vr = vmmlaq_s32(vr, vx_h, vy_h);\n }\n // apply block scale, will NOT overflow\n // block_scale * sum_256(int4*int8) <= 2^(8+8+4+8) = 28 bits\n visum = vmlaq_s32(visum, vr, block_scale);\n }\n }\n\n // adjust bias, apply superblock scale\n {\n int32_t bias[4];\n // no obvious uplift from sve sdot-16, just use neon mul add\n const int16x8_t y0_sums = vpaddq_s16(vld1q_s16(y0->bsums), vld1q_s16(y0->bsums+8));\n const int16x8_t y1_sums = vpaddq_s16(vld1q_s16(y1->bsums), vld1q_s16(y1->bsums+8));\n bias[0] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y0_sums), vget_low_s16(x0_mins)),\n vmull_s16(vget_high_s16(y0_sums), vget_high_s16(x0_mins))));\n bias[1] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y1_sums), vget_low_s16(x0_mins)),\n vmull_s16(vget_high_s16(y1_sums), vget_high_s16(x0_mins))));\n bias[2] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y0_sums), vget_low_s16(x1_mins)),\n vmull_s16(vget_high_s16(y0_sums), vget_high_s16(x1_mins))));\n bias[3] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y1_sums), vget_low_s16(x1_mins)),\n vmull_s16(vget_high_s16(y1_sums), vget_high_s16(x1_mins))));\n const float32x4_t dmins = {\n GGML_CPU_FP16_TO_FP32(x0->dmin) * y0->d,\n GGML_CPU_FP16_TO_FP32(x0->dmin) * y1->d,\n GGML_CPU_FP16_TO_FP32(x1->dmin) * y0->d,\n GGML_CPU_FP16_TO_FP32(x1->dmin) * y1->d,\n };\n vfsum = vmlsq_f32(vfsum, vcvtq_f32_s32(vld1q_s32(bias)), dmins);\n\n const float32x4_t superblock_scale = {\n GGML_CPU_FP16_TO_FP32(x0->d) * y0->d,\n GGML_CPU_FP16_TO_FP32(x0->d) * y1->d,\n GGML_CPU_FP16_TO_FP32(x1->d) * y0->d,\n GGML_CPU_FP16_TO_FP32(x1->d) * y1->d,\n };\n vfsum = vmlaq_f32(vfsum, vcvtq_f32_s32(visum), superblock_scale);\n }\n }\n\n // vfsum = ABCD -> ACBD\n // AC -> s, BD -> (s+bs)\n vfsum = vzip1q_f32(vfsum, vextq_f32(vfsum, vfsum, 2));\n vst1_f32(s, vget_low_f32 (vfsum));\n vst1_f32(s + bs, vget_high_f32(vfsum));\n\n return;\n }\n#endif\n\n#ifdef __ARM_FEATURE_SVE\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));\n\n memcpy(utmp, x[i].scales, K_SCALE_SIZE);\n\n uint32x2_t mins8 = { 0 };\n mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);\n mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);\n\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[0] &= kmask1;\n\n const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));\n const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),\n vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));\n sumf -= dmin * vaddvq_s32(prod);\n\n const uint8_t * scales = (const uint8_t *)utmp;\n\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const svuint8_t m4b = svdup_n_u8(0xf);\n const svint32_t mzero = svdup_n_s32(0);\n svint32_t sumi1 = svdup_n_s32(0);\n svint32_t sumi1_1 = svdup_n_s32(0);\n svint32_t sumi1_2 = svdup_n_s32(0);\n svint32_t sumi2 = svdup_n_s32(0);\n svint32_t sumi2_1 = svdup_n_s32(0);\n svint32_t sumi2_2 = svdup_n_s32(0);\n switch (vector_length) {\n case 128:\n {\n for (int j = 0; j < QK_K/64; ++j) {\n svint8_t q4bytes = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svld1_u8(svptrue_b8(), q4), m4b));\n svint8_t q8bytes = svld1_s8(svptrue_b8(), q8); q8 += 16;\n sumi1_1 = svmla_n_s32_x(svptrue_b32(), sumi1_1, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+0]);\n q4bytes = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svld1_u8(svptrue_b8(), q4+16), m4b));\n q8bytes = svld1_s8(svptrue_b8(), q8); q8 += 16;\n sumi1_2 = svmla_n_s32_x(svptrue_b32(), sumi1_2, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+0]);\n\n q4bytes = svreinterpret_s8_u8(svlsr_n_u8_x(svptrue_b8(), svld1_u8(svptrue_b8(), q4), 4));\n q8bytes = svld1_s8(svptrue_b8(), q8); q8 += 16;\n sumi2_1 = svmla_n_s32_x(svptrue_b32(), sumi2_1, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+1]);\n q4bytes = svreinterpret_s8_u8(svlsr_n_u8_x(svptrue_b8(), svld1_u8(svptrue_b8(), q4+16), 4));\n q8bytes = svld1_s8(svptrue_b8(), q8); q8 += 16;\n sumi2_2 = svmla_n_s32_x(svptrue_b32(), sumi2_2, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+1]);\n q4 += 32;\n }\n sumi1 = svadd_s32_x(svptrue_b32(), sumi1_1, sumi1_2);\n sumi2 = svadd_s32_x(svptrue_b32(), sumi2_1, sumi2_2);\n sumf += d * (svaddv_s32(svptrue_b32(), svadd_s32_x(svptrue_b32(), sumi1, sumi2)));\n } break;\n case 256:\n case 512:\n {\n for (int j = 0; j < QK_K/64; ++j) {\n const svuint8_t q4bits = svld1_u8(svptrue_pat_b8(SV_VL32), q4); q4 += 32;\n svint8_t q4bytes = svreinterpret_s8_u8(svand_u8_x(svptrue_pat_b8(SV_VL32), q4bits, m4b));\n svint8_t q8bytes = svld1_s8(svptrue_pat_b8(SV_VL32), q8); q8 += 32;\n sumi1 = svmla_n_s32_x(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+0]);\n\n q4bytes = svreinterpret_s8_u8(svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q4bits, 4));\n q8bytes = svld1_s8(svptrue_pat_b8(SV_VL32), q8); q8 += 32;\n sumi2 = svmla_n_s32_x(svptrue_pat_b32(SV_VL8), sumi2, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+1]);\n }\n sumf += d * (svaddv_s32(svptrue_pat_b32(SV_VL8), svadd_s32_x(svptrue_pat_b32(SV_VL8), sumi1, sumi2)));\n } break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n }\n *s = sumf;\n#elif defined __ARM_NEON\n const uint8x16_t m4b = vdupq_n_u8(0xf);\n const int32x4_t mzero = vdupq_n_s32(0);\n\n ggml_int8x16x2_t q4bytes;\n ggml_int8x16x2_t q8bytes;\n\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));\n\n memcpy(utmp, x[i].scales, 12);\n\n uint32x2_t mins8 = { 0 };\n mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);\n mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);\n\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[0] &= kmask1;\n\n const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));\n const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),\n vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));\n sumf -= dmin * vaddvq_s32(prod);\n\n const uint8_t * scales = (const uint8_t *)utmp;\n\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n int32_t sumi1 = 0;\n int32_t sumi2 = 0;\n\n for (int j = 0; j < QK_K/64; ++j) {\n const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;\n\n q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\n q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));\n q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));\n\n const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);\n sumi1 += vaddvq_s32(p1) * scales[2*j+0];\n\n q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\n q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));\n q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));\n\n const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);\n\n sumi2 += vaddvq_s32(p2) * scales[2*j+1];\n }\n\n sumf += d * (sumi1 + sumi2);\n\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n\n#ifdef __ARM_NEON\n const uint8x16_t m4b = vdupq_n_u8(0xf);\n const uint8x16_t mone = vdupq_n_u8(1);\n const uint8x16_t mtwo = vdupq_n_u8(2);\n const int32x4_t mzero = vdupq_n_s32(0);\n\n ggml_int8x16x4_t q5bytes;\n\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n const uint8x8_t mins8 = vld1_u8((const uint8_t*)utmp + 8);\n const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(mins8));\n const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),\n vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));\n int32_t sumi_mins = vaddvq_s32(prod);\n\n const uint8_t * scales = (const uint8_t *)utmp;\n\n const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);\n\n ggml_uint8x16x4_t q5h;\n\n int32_t sumi = 0;\n\n for (int j = 0; j < QK_K/64; ++j) {\n\n const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5); q5 += 32;\n const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n q5h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);\n q5h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);\n q5h.val[2] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[0]), 3);\n q5h.val[3] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[1]), 3);\n qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 2);\n qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 2);\n\n q5bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[0], m4b), q5h.val[0]));\n q5bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[1], m4b), q5h.val[1]));\n q5bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[0], 4), q5h.val[2]));\n q5bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[1], 4), q5h.val[3]));\n\n sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]), q5bytes.val[1], q8bytes.val[1])) * *scales++;\n sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]), q5bytes.val[3], q8bytes.val[3])) * *scales++;\n }\n\n sumf += d * sumi - dmin * sumi_mins;\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n#ifdef __ARM_FEATURE_MATMUL_INT8\n assert((nrc == 2) || (nrc == 1));\n#else\n assert(nrc == 1);\n#endif\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q6_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#ifdef __ARM_FEATURE_SVE\n const int vector_length = ggml_cpu_get_sve_cnt()*8;\n#endif\n#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n if (nrc == 2) {\n const svbool_t pg32_2 = svptrue_pat_b32(SV_VL2);\n\n svfloat32_t sum = svdup_n_f32(0);\n\n const block_q6_K * GGML_RESTRICT vx0 = vx;\n const block_q8_K * GGML_RESTRICT vy0 = vy;\n const block_q6_K * GGML_RESTRICT vx1 = (const block_q6_K *) ((const uint8_t*)vx + bx);\n const block_q8_K * GGML_RESTRICT vy1 = (const block_q8_K *) ((const uint8_t*)vy + by);\n\n switch (vector_length) {\n case 128:\n {\n const svbool_t pg128_all = svptrue_pat_b8(SV_ALL);\n for (int i = 0; i < nb; ++i) {\n const uint8_t * GGML_RESTRICT ql0 = vx0[i].ql;\n const uint8_t * GGML_RESTRICT qh0 = vx0[i].qh;\n const uint8_t * GGML_RESTRICT ql1 = vx1[i].ql;\n const uint8_t * GGML_RESTRICT qh1 = vx1[i].qh;\n const int8_t * GGML_RESTRICT q80 = vy0[i].qs;\n const int8_t * GGML_RESTRICT q81 = vy1[i].qs;\n\n const int8_t * GGML_RESTRICT scale0 = vx0[i].scales;\n const int8_t * GGML_RESTRICT scale1 = vx1[i].scales;\n\n svfloat32_t vy_d = svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d));\n svfloat32_t vx_d = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].d)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].d)));\n svfloat32_t svsuper_block_scales = svmul_f32_x(pg128_all, vy_d, vx_d);\n // process q8sum summation 128 bit route\n const svint16_t q8sums_01 = svld1_s16(pg128_all, vy0[i].bsums);\n const svint16_t q8sums_02 = svld1_s16(pg128_all, vy0[i].bsums + 8);\n const svint16_t q8sums_11 = svld1_s16(pg128_all, vy1[i].bsums);\n const svint16_t q8sums_12 = svld1_s16(pg128_all, vy1[i].bsums + 8);\n const svint64x2_t q6scales_0_tmp = svld2_s64(pg128_all, (const int64_t *)scale0);\n const svint16_t q6scales_01 = svunpklo_s16(svreinterpret_s8_s64(svget2_s64(q6scales_0_tmp, 0)));\n const svint16_t q6scales_02 = svunpklo_s16(svreinterpret_s8_s64(svget2_s64(q6scales_0_tmp, 1)));\n const svint64x2_t q6scales_1_tmp = svld2_s64(pg128_all, (const int64_t *)scale1);\n const svint16_t q6scales_11 = svunpklo_s16(svreinterpret_s8_s64(svget2_s64(q6scales_1_tmp, 0)));\n const svint16_t q6scales_12 = svunpklo_s16(svreinterpret_s8_s64(svget2_s64(q6scales_1_tmp, 1)));\n const svint64_t prod = svdup_n_s64(0);\n\n svint32_t isum_tmp1 = svreinterpret_s32_s64(svdot_s64(svdot_s64(prod, q8sums_01, q6scales_01), q8sums_02, q6scales_02));\n svint32_t isum_tmp2 = svreinterpret_s32_s64(svdot_s64(svdot_s64(prod, q8sums_01, q6scales_11), q8sums_02, q6scales_12));\n svint32_t isum_tmp3 = svtrn1_s32(isum_tmp1, isum_tmp2);\n svint32_t isum_tmp4 = svreinterpret_s32_s64(svdot_s64(svdot_s64(prod, q8sums_11, q6scales_01), q8sums_12, q6scales_02));\n svint32_t isum_tmp5 = svreinterpret_s32_s64(svdot_s64(svdot_s64(prod, q8sums_11, q6scales_11), q8sums_12, q6scales_12));\n svint32_t isum_tmp6 = svtrn1_s32(isum_tmp4, isum_tmp5);\n svint32_t isum_tmp7 = svreinterpret_s32_s64(svtrn2_s64(svreinterpret_s64_s32(isum_tmp3), svreinterpret_s64_s32(isum_tmp6)));\n svint32_t isum_tmp8 = svreinterpret_s32_s64(svtrn1_s64(svreinterpret_s64_s32(isum_tmp3), svreinterpret_s64_s32(isum_tmp6)));\n svint32_t svisum_mins = svadd_s32_x(pg128_all, isum_tmp7, isum_tmp8);\n\n // process mmla\n svint8_t l0, l1, r0, r1;\n svint32_t isum_tmp = svdup_n_s32(0);\n for (int j = 0; j < QK_K/128; ++j) {\n for (int k = 0; k < 8; ++k) {\n svuint8_t qhbits_0 = svld1_u8(pg128_all, qh0+16*(k%2));\n svuint8_t qhbits_1 = svld1_u8(pg128_all, qh1+16*(k%2));\n svuint8_t q6bits_0 = svld1_u8(pg128_all, ql0+16*(k%4));\n svuint8_t q6bits_1 = svld1_u8(pg128_all, ql1+16*(k%4));\n const int ql_pos = (k/4)*4;\n svuint8_t q6bytes_0_lo = (ql_pos < 4) ? svand_n_u8_x(pg128_all, q6bits_0, 0xf) : svlsr_n_u8_x(pg128_all, q6bits_0, 4);\n svuint8_t q6bytes_1_lo = (ql_pos < 4) ? svand_n_u8_x(pg128_all, q6bits_1, 0xf) : svlsr_n_u8_x(pg128_all, q6bits_1, 4);\n const int qh_pos = (k/2)*2;\n svuint8_t q6bytes_0_hi = svand_n_u8_x(pg128_all, qhbits_0, 0x3 << qh_pos);\n svuint8_t q6bytes_1_hi = svand_n_u8_x(pg128_all, qhbits_1, 0x3 << qh_pos);\n svint8_t q6bytes_0, q6bytes_1;\n if (qh_pos <= 4) {\n q6bytes_0 = svreinterpret_s8_u8(svmla_n_u8_x(pg128_all, q6bytes_0_lo, q6bytes_0_hi, 1 << (4 - qh_pos)));\n q6bytes_1 = svreinterpret_s8_u8(svmla_n_u8_x(pg128_all, q6bytes_1_lo, q6bytes_1_hi, 1 << (4 - qh_pos)));\n } else {\n q6bytes_0 = svreinterpret_s8_u8(svorr_u8_x(pg128_all, q6bytes_0_lo, svlsr_n_u8_x(pg128_all, q6bytes_0_hi, (qh_pos - 4))));\n q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg128_all, q6bytes_1_lo, svlsr_n_u8_x(pg128_all, q6bytes_1_hi, (qh_pos - 4))));\n }\n svint8_t q8bytes_0 = svld1_s8(pg128_all, q80+16*(k%8));\n svint8_t q8bytes_1 = svld1_s8(pg128_all, q81+16*(k%8));\n l0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q6bytes_0), svreinterpret_s64_s8(q6bytes_1)));\n l1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q6bytes_0), svreinterpret_s64_s8(q6bytes_1)));\n r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n svint32_t svscale = svzip1_s32(svdup_n_s32(scale0[k]), svdup_n_s32(scale1[k]));\n isum_tmp = svmla_s32_x(pg128_all, isum_tmp, svmmla_s32(svmmla_s32(svdup_n_s32(0), r0, l0), r1, l1), svscale);\n }\n qh0 += 32; qh1 += 32;\n ql0 += 64; ql1 += 64;\n q80 += 128; q81 += 128;\n scale0 += 8; scale1 += 8;\n }\n sum = svmla_f32_x(pg128_all, sum,\n svcvt_f32_x(pg128_all, svmla_s32_x(pg128_all, isum_tmp,\n svisum_mins, svdup_n_s32(-32))),\n svsuper_block_scales);\n }\n } // end of case 128\n break;\n case 256:\n case 512:\n {\n const svbool_t pg256_all = svptrue_pat_b8(SV_ALL);\n const svbool_t pg32_4 = svptrue_pat_b32(SV_VL4);\n for (int i = 0; i < nb; ++i) {\n const uint8_t * GGML_RESTRICT ql0 = vx0[i].ql;\n const uint8_t * GGML_RESTRICT qh0 = vx0[i].qh;\n const uint8_t * GGML_RESTRICT ql1 = vx1[i].ql;\n const uint8_t * GGML_RESTRICT qh1 = vx1[i].qh;\n const int8_t * GGML_RESTRICT q80 = vy0[i].qs;\n const int8_t * GGML_RESTRICT q81 = vy1[i].qs;\n\n const int8_t * GGML_RESTRICT scale0 = vx0[i].scales;\n const int8_t * GGML_RESTRICT scale1 = vx1[i].scales;\n svfloat32_t vx_d = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].d)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].d)));\n svfloat64_t vy_d_tmp = svreinterpret_f64_f32(svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d)));\n svfloat32_t vy_d = svreinterpret_f32_f64(svuzp1_f64(vy_d_tmp, vy_d_tmp));\n svfloat32_t svsuper_block_scales = svmul_f32_x(pg32_4, vy_d, vx_d);\n // process q8sum summation 256 bit route\n const svint16_t q8sums_0 = svld1_s16(pg256_all, vy0[i].bsums);\n const svint16_t q8sums_1 = svld1_s16(pg256_all, vy1[i].bsums);\n const svint16_t q6scales_0 = svunpklo_s16(svld1_s8(pg256_all, scale0));\n const svint16_t q6scales_1 = svunpklo_s16(svld1_s8(pg256_all, scale1));\n const svint64_t prod = svdup_n_s64(0);\n svint32_t isum_tmp1 = svreinterpret_s32_s64(svdot_s64(prod, q8sums_0, q6scales_0));\n svint32_t isum_tmp2 = svreinterpret_s32_s64(svdot_s64(prod, q8sums_0, q6scales_1));\n svint32_t isum_tmp3 = svreinterpret_s32_s64(svdot_s64(prod, q8sums_1, q6scales_0));\n svint32_t isum_tmp4 = svreinterpret_s32_s64(svdot_s64(prod, q8sums_1, q6scales_1));\n svint32_t isum_tmp5 = svtrn1_s32(isum_tmp1, isum_tmp2);\n svint32_t isum_tmp6 = svtrn1_s32(isum_tmp3, isum_tmp4);\n svint32_t isum_tmp7 = svreinterpret_s32_s64(svtrn2_s64(svreinterpret_s64_s32(isum_tmp5), svreinterpret_s64_s32(isum_tmp6)));\n svint32_t isum_tmp8 = svreinterpret_s32_s64(svtrn1_s64(svreinterpret_s64_s32(isum_tmp5), svreinterpret_s64_s32(isum_tmp6)));\n svint32_t isum_tmp9 = svadd_s32_x(pg256_all, isum_tmp7, isum_tmp8);\n svint32_t isum_tmp10 = svreinterpret_s32_u8(svext_u8(svreinterpret_u8_s32(isum_tmp9), svreinterpret_u8_s32(isum_tmp9), 16));\n svint32_t svisum_mins = svadd_s32_z(pg32_4, isum_tmp9, isum_tmp10);\n\n // process mmla\n svint8_t l0, l1, r0, r1;\n svint32_t isum_tmp = svdup_n_s32(0);\n for (int j = 0; j < QK_K/128; ++j) {\n for (int k = 0; k < 8; k+=2) { // process 2 block\n svuint8_t qhbits_0 = svld1_u8(pg256_all, qh0);\n svuint8_t qhbits_1 = svld1_u8(pg256_all, qh1);\n svuint8_t q6bits_0 = svld1_u8(pg256_all, ql0+32*((k%4)/2));\n svuint8_t q6bits_1 = svld1_u8(pg256_all, ql1+32*((k%4)/2));\n const int ql_pos = (k/4)*4;\n svuint8_t q6bytes_0_lo = (ql_pos < 4) ? svand_n_u8_x(pg256_all, q6bits_0, 0xf) : svlsr_n_u8_x(pg256_all, q6bits_0, 4);\n svuint8_t q6bytes_1_lo = (ql_pos < 4) ? svand_n_u8_x(pg256_all, q6bits_1, 0xf) : svlsr_n_u8_x(pg256_all, q6bits_1, 4);\n const int qh_pos = (k/2)*2;\n svuint8_t q6bytes_0_hi = svand_n_u8_x(pg256_all, qhbits_0, 0x3 << qh_pos);\n svuint8_t q6bytes_1_hi = svand_n_u8_x(pg256_all, qhbits_1, 0x3 << qh_pos);\n svint8_t q6bytes_0, q6bytes_1;\n if (qh_pos <= 4) {\n q6bytes_0 = svreinterpret_s8_u8(svmla_n_u8_x(pg256_all, q6bytes_0_lo, q6bytes_0_hi, 1 << (4 - qh_pos)));\n q6bytes_1 = svreinterpret_s8_u8(svmla_n_u8_x(pg256_all, q6bytes_1_lo, q6bytes_1_hi, 1 << (4 - qh_pos)));\n } else {\n q6bytes_0 = svreinterpret_s8_u8(svorr_u8_x(pg256_all, q6bytes_0_lo, svlsr_n_u8_x(pg256_all, q6bytes_0_hi, (qh_pos - 4))));\n q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg256_all, q6bytes_1_lo, svlsr_n_u8_x(pg256_all, q6bytes_1_hi, (qh_pos - 4))));\n }\n svint8_t q8bytes_0 = svld1_s8(pg256_all, q80+32*(k/2));\n svint8_t q8bytes_1 = svld1_s8(pg256_all, q81+32*(k/2));\n l0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q6bytes_0), svreinterpret_s64_s8(q6bytes_1)));\n l1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q6bytes_0), svreinterpret_s64_s8(q6bytes_1)));\n r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n svint32_t svscale0 = svzip1_s32(svdup_n_s32(scale0[k]), svdup_n_s32(scale1[k]));\n svint32_t svscale1 = svzip1_s32(svdup_n_s32(scale0[k+1]), svdup_n_s32(scale1[k+1]));\n isum_tmp = svmla_s32_x(pg256_all, isum_tmp, svmmla_s32(svdup_n_s32(0), r0, l0), svscale0);\n isum_tmp = svmla_s32_x(pg256_all, isum_tmp, svmmla_s32(svdup_n_s32(0), r1, l1), svscale1);\n }\n qh0 += 32; qh1 += 32;\n ql0 += 64; ql1 += 64;\n q80 += 128; q81 += 128;\n scale0 += 8; scale1 += 8;\n } // end of for\n svint32_t swap_isum_tmp = svext_s32(isum_tmp, isum_tmp, 4);\n isum_tmp = svadd_s32_x(pg32_4, isum_tmp, swap_isum_tmp);\n sum = svmla_f32_x(pg32_4, sum,\n svcvt_f32_x(pg32_4, svmla_s32_x(pg32_4, isum_tmp,\n svisum_mins, svdup_n_s32(-32))),\n svsuper_block_scales);\n }\n } // end of case 256\n break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n } // end of switch\n\n svst1_f32(pg32_2, s, sum);\n svst1_f32(pg32_2, s + bs, svreinterpret_f32_u8(svext_u8(svreinterpret_u8_f32(sum), svdup_n_u8(0), 8)));\n\n return;\n }\n#elif defined(__ARM_FEATURE_MATMUL_INT8)\n if (nrc == 2) {\n const block_q6_K * GGML_RESTRICT x0 = x;\n const block_q6_K * GGML_RESTRICT x1 = (const block_q6_K *) ((const uint8_t *)vx + bx);\n const block_q8_K * GGML_RESTRICT y0 = y;\n const block_q8_K * GGML_RESTRICT y1 = (const block_q8_K *) ((const uint8_t *)vy + by);\n\n float32x4_t vfsum = vdupq_n_f32(0.0f);\n\n for (int i = 0; i < nb; ++i, ++x0, ++x1, ++y0, ++y1) {\n const uint8_t * GGML_RESTRICT ql0 = x0->ql;\n const uint8_t * GGML_RESTRICT ql1 = x1->ql;\n const uint8_t * GGML_RESTRICT qh0 = x0->qh;\n const uint8_t * GGML_RESTRICT qh1 = x1->qh;\n const int8_t * GGML_RESTRICT qy0 = y0->qs;\n const int8_t * GGML_RESTRICT qy1 = y1->qs;\n\n const uint8x16_t mone = vdupq_n_u8(0x30);\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n\n int32x4_t visum = vdupq_n_s32(0);\n\n // process 8 blocks per iteration, totally 16 blocks\n for (int j = 0; j < 2; ++j, qh0 += 32, ql0 += 64, qh1 += 32, ql1 += 64) {\n int8x16_t vx0[8], vx1[8];\n\n // de-quantize vx0[8]\n {\n const uint8x16x2_t qh_bits = vld1q_u8_x2(qh0);\n const uint8x16x4_t ql_bits = vld1q_u8_x4(ql0);\n\n uint8x16_t q6h_0 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[0], 4));\n uint8x16_t q6h_1 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[1], 4));\n uint8x16_t q6h_2 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[0], 2));\n uint8x16_t q6h_3 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[1], 2));\n\n vx0[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[0], m4b), q6h_0));\n vx0[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[1], m4b), q6h_1));\n vx0[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[2], m4b), q6h_2));\n vx0[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[3], m4b), q6h_3));\n\n q6h_0 = vandq_u8(mone, qh_bits.val[0]);\n q6h_1 = vandq_u8(mone, qh_bits.val[1]);\n q6h_2 = vandq_u8(mone, vshrq_n_u8(qh_bits.val[0], 2));\n q6h_3 = vandq_u8(mone, vshrq_n_u8(qh_bits.val[1], 2));\n\n vx0[4] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[0], 4), q6h_0));\n vx0[5] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[1], 4), q6h_1));\n vx0[6] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[2], 4), q6h_2));\n vx0[7] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[3], 4), q6h_3));\n }\n\n // de-quantize vx1[8]\n {\n const uint8x16x2_t qh_bits = vld1q_u8_x2(qh1);\n const uint8x16x4_t ql_bits = vld1q_u8_x4(ql1);\n\n uint8x16_t q6h_0 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[0], 4));\n uint8x16_t q6h_1 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[1], 4));\n uint8x16_t q6h_2 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[0], 2));\n uint8x16_t q6h_3 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[1], 2));\n\n vx1[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[0], m4b), q6h_0));\n vx1[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[1], m4b), q6h_1));\n vx1[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[2], m4b), q6h_2));\n vx1[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[3], m4b), q6h_3));\n\n q6h_0 = vandq_u8(mone, qh_bits.val[0]);\n q6h_1 = vandq_u8(mone, qh_bits.val[1]);\n q6h_2 = vandq_u8(mone, vshrq_n_u8(qh_bits.val[0], 2));\n q6h_3 = vandq_u8(mone, vshrq_n_u8(qh_bits.val[1], 2));\n\n vx1[4] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[0], 4), q6h_0));\n vx1[5] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[1], 4), q6h_1));\n vx1[6] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[2], 4), q6h_2));\n vx1[7] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[3], 4), q6h_3));\n }\n\n // process 16 elements (one block with same scale) per iteration\n // - vx = concat(ql, qh) - 32\n // - r1,r2,r3,r4 = smmla(vx, vy)\n for (int k = 0; k < 8; ++k) {\n const int blk = j * 8 + k;\n\n const int8x16_t vy0 = vld1q_s8(qy0);\n const int8x16_t vy1 = vld1q_s8(qy1);\n qy0 += 16;\n qy1 += 16;\n\n const int32x4_t block_scale = {\n x0->scales[blk],\n x0->scales[blk],\n x1->scales[blk],\n x1->scales[blk],\n };\n\n // calculate four results at once with outer product\n const int8x16_t vx_l = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(vx0[k]), vreinterpretq_s64_s8(vx1[k])));\n const int8x16_t vx_h = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(vx0[k]), vreinterpretq_s64_s8(vx1[k])));\n const int8x16_t vy_l = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(vy0), vreinterpretq_s64_s8(vy1)));\n const int8x16_t vy_h = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(vy0), vreinterpretq_s64_s8(vy1)));\n int32x4_t vr = vdupq_n_s32(0);\n vr = vmmlaq_s32(vr, vx_l, vy_l);\n vr = vmmlaq_s32(vr, vx_h, vy_h);\n\n // apply block scale, will NOT overflow\n // block_scale * sum_256(int6*int8) <= 2^(8+8+6+8) = 30 bits\n visum = vmlaq_s32(visum, vr, block_scale);\n }\n }\n\n // adjust bias, apply superblock scale\n {\n int32_t bias[4];\n // NEON doesn't support int16 dot product, fallback to separated mul and add\n const int16x8x2_t q8sums0 = vld1q_s16_x2(y0->bsums);\n const int16x8x2_t q8sums1 = vld1q_s16_x2(y1->bsums);\n\n int8x16_t scales_s8 = vld1q_s8(x0->scales);\n const int16x8x2_t q6scales0 = {{vmovl_s8(vget_low_s8(scales_s8)), vmovl_s8(vget_high_s8(scales_s8))}};\n scales_s8 = vld1q_s8(x1->scales);\n const int16x8x2_t q6scales1 = {{vmovl_s8(vget_low_s8(scales_s8)), vmovl_s8(vget_high_s8(scales_s8))}};\n\n int32x4_t prod;\n prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums0.val[0]), vget_low_s16 (q6scales0.val[0])),\n vmull_s16(vget_high_s16(q8sums0.val[0]), vget_high_s16(q6scales0.val[0]))),\n vaddq_s32(vmull_s16(vget_low_s16 (q8sums0.val[1]), vget_low_s16 (q6scales0.val[1])),\n vmull_s16(vget_high_s16(q8sums0.val[1]), vget_high_s16(q6scales0.val[1]))));\n bias[0] = vaddvq_s32(prod);\n prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums1.val[0]), vget_low_s16 (q6scales0.val[0])),\n vmull_s16(vget_high_s16(q8sums1.val[0]), vget_high_s16(q6scales0.val[0]))),\n vaddq_s32(vmull_s16(vget_low_s16 (q8sums1.val[1]), vget_low_s16 (q6scales0.val[1])),\n vmull_s16(vget_high_s16(q8sums1.val[1]), vget_high_s16(q6scales0.val[1]))));\n bias[1] = vaddvq_s32(prod);\n prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums0.val[0]), vget_low_s16 (q6scales1.val[0])),\n vmull_s16(vget_high_s16(q8sums0.val[0]), vget_high_s16(q6scales1.val[0]))),\n vaddq_s32(vmull_s16(vget_low_s16 (q8sums0.val[1]), vget_low_s16 (q6scales1.val[1])),\n vmull_s16(vget_high_s16(q8sums0.val[1]), vget_high_s16(q6scales1.val[1]))));\n bias[2] = vaddvq_s32(prod);\n prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums1.val[0]), vget_low_s16 (q6scales1.val[0])),\n vmull_s16(vget_high_s16(q8sums1.val[0]), vget_high_s16(q6scales1.val[0]))),\n vaddq_s32(vmull_s16(vget_low_s16 (q8sums1.val[1]), vget_low_s16 (q6scales1.val[1])),\n vmull_s16(vget_high_s16(q8sums1.val[1]), vget_high_s16(q6scales1.val[1]))));\n bias[3] = vaddvq_s32(prod);\n\n const int32x4_t vibias = vmulq_n_s32(vld1q_s32(bias), 32);\n\n const float32x4_t superblock_scale = {\n GGML_CPU_FP16_TO_FP32(x0->d) * y0->d,\n GGML_CPU_FP16_TO_FP32(x0->d) * y1->d,\n GGML_CPU_FP16_TO_FP32(x1->d) * y0->d,\n GGML_CPU_FP16_TO_FP32(x1->d) * y1->d,\n };\n\n visum = vsubq_s32(visum, vibias);\n vfsum = vmlaq_f32(vfsum, vcvtq_f32_s32(visum), superblock_scale);\n }\n }\n\n // vfsum = ABCD -> ACBD\n // AC -> s, BD -> (s+bs)\n vfsum = vzip1q_f32(vfsum, vextq_f32(vfsum, vfsum, 2));\n vst1_f32(s, vget_low_f32 (vfsum));\n vst1_f32(s + bs, vget_high_f32(vfsum));\n\n return;\n }\n#endif\n\n#ifdef __ARM_FEATURE_SVE\n float sum = 0;\n svuint8_t m4b = svdup_n_u8(0xf);\n svint32_t vzero = svdup_n_s32(0);\n svuint8_t mone = svdup_n_u8(0x30);\n svint8_t q6bytes_1, q6bytes_2, q6bytes_3, q6bytes_4;\n svuint8_t q6h_1, q6h_2, q6h_3, q6h_4;\n\n for (int i = 0; i < nb; ++i) {\n const float d_all = GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q6 = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const int8_t * GGML_RESTRICT scale = x[i].scales;\n\n const svbool_t pg16_8 = svptrue_pat_b16(SV_VL8);\n const svint16_t q8sums_1 = svld1_s16(pg16_8, y[i].bsums);\n const svint16_t q8sums_2 = svld1_s16(pg16_8, y[i].bsums + 8);\n const svint16_t q6scales_1 = svunpklo_s16(svld1_s8(svptrue_pat_b8(SV_VL8), scale));\n const svint16_t q6scales_2 = svunpklo_s16(svld1_s8(svptrue_pat_b8(SV_VL8), scale + 8));\n const svint64_t prod = svdup_n_s64(0);\n int32_t isum_mins = svaddv_s64(svptrue_b64(), svadd_s64_x(svptrue_b64(), svdot_s64(prod, q8sums_1, q6scales_1),\n svdot_s64(prod, q8sums_2, q6scales_2)));\n int32_t isum = 0;\n\n switch (vector_length) {\n case 128:\n {\n const svbool_t pg32_4 = svptrue_pat_b32(SV_VL4);\n const svbool_t pg8_16 = svptrue_pat_b8(SV_VL16);\n svint32_t isum_tmp = svdup_n_s32(0);\n for (int j = 0; j < QK_K/128; ++j) {\n svuint8_t qhbits_1 = svld1_u8(pg8_16, qh);\n svuint8_t qhbits_2 = svld1_u8(pg8_16, qh+16);\n qh += 32;\n svuint8_t q6bits_1 = svld1_u8(pg8_16, q6);\n svuint8_t q6bits_2 = svld1_u8(pg8_16, q6+16);\n svuint8_t q6bits_3 = svld1_u8(pg8_16, q6+32);\n svuint8_t q6bits_4 = svld1_u8(pg8_16, q6+48);\n q6 += 64;\n svint8_t q8bytes_1 = svld1_s8(pg8_16, q8);\n svint8_t q8bytes_2 = svld1_s8(pg8_16, q8+16);\n svint8_t q8bytes_3 = svld1_s8(pg8_16, q8+32);\n svint8_t q8bytes_4 = svld1_s8(pg8_16, q8+48);\n q8 += 64;\n\n q6h_1 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_1, 4));\n q6h_2 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_2, 4));\n q6h_3 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_1, 2));\n q6h_4 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_2, 2));\n q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_1, m4b), q6h_1));\n q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_2, m4b), q6h_2));\n q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_3, m4b), q6h_3));\n q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_4, m4b), q6h_4));\n isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale[0]);\n isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale[1]);\n isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale[2]);\n isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale[3]);\n\n scale += 4;\n q8bytes_1 = svld1_s8(pg8_16, q8);\n q8bytes_2 = svld1_s8(pg8_16, q8+16);\n q8bytes_3 = svld1_s8(pg8_16, q8+32);\n q8bytes_4 = svld1_s8(pg8_16, q8+48);\n q8 += 64;\n\n q6h_1 = svand_u8_x(pg16_8, mone, qhbits_1);\n q6h_2 = svand_u8_x(pg16_8, mone, qhbits_2);\n q6h_3 = svand_u8_x(pg16_8, mone, svlsr_n_u8_x(pg16_8, qhbits_1, 2));\n q6h_4 = svand_u8_x(pg16_8, mone, svlsr_n_u8_x(pg16_8, qhbits_2, 2));\n q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_1, 4), q6h_1));\n q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_2, 4), q6h_2));\n q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_3, 4), q6h_3));\n q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_4, 4), q6h_4));\n isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale[0]);\n isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale[1]);\n isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale[2]);\n isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale[3]);\n scale += 4;\n }\n isum += svaddv_s32(pg32_4, isum_tmp);\n sum += d_all * y[i].d * (isum - 32 * isum_mins);\n }\n break;\n case 256:\n case 512:\n {\n const svbool_t pg8_2 = svptrue_pat_b8(SV_VL2);\n const svbool_t pg32_8 = svptrue_pat_b32(SV_VL8);\n const svbool_t pg8_32 = svptrue_pat_b8(SV_VL32);\n svint32_t isum_tmp = svdup_n_s32(0);\n for (int j = 0; j < QK_K/128; j++) {\n svuint8_t qhbits_1 = svld1_u8(pg8_32, qh);\n qh += 32;\n svuint8_t q6bits_1 = svld1_u8(pg8_32, q6);\n svuint8_t q6bits_2 = svld1_u8(pg8_32, q6+32);\n q6 += 64;\n svint8_t q8bytes_1 = svld1_s8(pg8_32, q8);\n svint8_t q8bytes_2 = svld1_s8(pg8_32, q8+32);\n svint8_t q8bytes_3 = svld1_s8(pg8_32, q8+64);\n svint8_t q8bytes_4 = svld1_s8(pg8_32, q8+96);\n q8 += 128;\n q6h_1 = svand_u8_x(pg8_32, mone, svlsl_n_u8_x(pg8_32, qhbits_1, 4));\n q6h_2 = svand_u8_x(pg8_32, mone, svlsl_n_u8_x(pg8_32, qhbits_1, 2));\n q6h_3 = svand_u8_x(pg8_32, mone, qhbits_1);\n q6h_4 = svand_u8_x(pg8_32, mone, svlsr_n_u8_x(pg8_32, qhbits_1, 2));\n q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svand_u8_x(pg8_32, q6bits_1, m4b), q6h_1));\n q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svand_u8_x(pg8_32, q6bits_2, m4b), q6h_2));\n q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svlsr_n_u8_x(pg8_32, q6bits_1, 4), q6h_3));\n q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svlsr_n_u8_x(pg8_32, q6bits_2, 4), q6h_4));\n\n svint8_t scale_lane_1_tmp = svld1_s8(pg8_2, scale);\n scale_lane_1_tmp= svzip1_s8(scale_lane_1_tmp, scale_lane_1_tmp);\n scale_lane_1_tmp= svzip1_s8(scale_lane_1_tmp, scale_lane_1_tmp);\n svint8_t scale_lane_2_tmp = svld1_s8(pg8_2, scale+2);\n scale_lane_2_tmp = svzip1_s8(scale_lane_2_tmp, scale_lane_2_tmp);\n scale_lane_2_tmp = svzip1_s8(scale_lane_2_tmp, scale_lane_2_tmp);\n svint8_t scale_lane_3_tmp = svld1_s8(pg8_2, scale+4);\n scale_lane_3_tmp = svzip1_s8(scale_lane_3_tmp, scale_lane_3_tmp);\n scale_lane_3_tmp = svzip1_s8(scale_lane_3_tmp, scale_lane_3_tmp);\n svint8_t scale_lane_4_tmp = svld1_s8(pg8_2, scale+6);\n scale_lane_4_tmp = svzip1_s8(scale_lane_4_tmp, scale_lane_4_tmp);\n scale_lane_4_tmp = svzip1_s8(scale_lane_4_tmp, scale_lane_4_tmp);\n svint32_t scale_lane_1 = svunpklo_s32(svunpklo_s16(scale_lane_1_tmp));\n svint32_t scale_lane_2 = svunpklo_s32(svunpklo_s16(scale_lane_2_tmp));\n svint32_t scale_lane_3 = svunpklo_s32(svunpklo_s16(scale_lane_3_tmp));\n svint32_t scale_lane_4 = svunpklo_s32(svunpklo_s16(scale_lane_4_tmp));\n\n isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale_lane_1);\n isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale_lane_2);\n isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale_lane_3);\n isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale_lane_4);\n scale += 8;\n }\n isum += svaddv_s32(pg32_8, isum_tmp);\n sum += d_all * y[i].d * (isum - 32 * isum_mins);\n }\n break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n }\n\n *s = sum;\n\n#elif __ARM_NEON\n float sum = 0;\n\n const uint8x16_t m4b = vdupq_n_u8(0xF);\n const int32x4_t vzero = vdupq_n_s32(0);\n //const int8x16_t m32s = vdupq_n_s8(32);\n\n const uint8x16_t mone = vdupq_n_u8(3);\n\n ggml_int8x16x4_t q6bytes;\n ggml_uint8x16x4_t q6h;\n\n for (int i = 0; i < nb; ++i) {\n\n const float d_all = GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q6 = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const int8_t * GGML_RESTRICT scale = x[i].scales;\n\n const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);\n const int8x16_t scales = vld1q_s8(scale);\n const ggml_int16x8x2_t q6scales = {{vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))}};\n\n const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),\n vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),\n vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[1]), vget_low_s16 (q6scales.val[1])),\n vmull_s16(vget_high_s16(q8sums.val[1]), vget_high_s16(q6scales.val[1]))));\n int32_t isum_mins = vaddvq_s32(prod);\n\n int32_t isum = 0;\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh); qh += 32;\n ggml_uint8x16x4_t q6bits = ggml_vld1q_u8_x4(q6); q6 += 64;\n ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);\n q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);\n uint8x16_t shifted = vshrq_n_u8(qhbits.val[0], 2);\n q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n shifted = vshrq_n_u8(qhbits.val[1], 2);\n q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n\n //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s);\n //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s);\n //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2])), m32s);\n //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3])), m32s);\n q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0]));\n q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1]));\n q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));\n q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));\n\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +\n vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +\n vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +\n vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];\n\n scale += 4;\n\n q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n shifted = vshrq_n_u8(qhbits.val[0], 4);\n q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n shifted = vshrq_n_u8(qhbits.val[1], 4);\n q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n shifted = vshrq_n_u8(qhbits.val[0], 6);\n q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n shifted = vshrq_n_u8(qhbits.val[1], 6);\n q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n\n //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0])), m32s);\n //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1])), m32s);\n //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2])), m32s);\n //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3])), m32s);\n q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0]));\n q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1]));\n q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));\n q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));\n\n isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +\n vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +\n vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +\n vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];\n scale += 4;\n }\n //sum += isum * d_all * y[i].d;\n sum += d_all * y[i].d * (isum - 32 * isum_mins);\n\n }\n *s = sum;\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined (__ARM_NEON)\nstatic const int8_t keven_signs_q2xs[1024] = {\n 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,\n 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,\n 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,\n 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,\n 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,\n 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,\n 1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, 1,\n 1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, -1,\n 1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, -1,\n 1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, 1,\n 1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, 1,\n 1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1,\n 1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, 1,\n 1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, -1,\n 1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, -1,\n 1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1,\n 1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, -1,\n 1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1,\n 1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, 1,\n 1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, -1,\n 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1,\n 1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, -1,\n 1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, -1,\n 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1,\n 1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, 1,\n 1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, -1,\n 1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1,\n 1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, 1,\n 1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1,\n 1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, 1,\n 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, 1,\n 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n uint32_t aux32[4];\n const uint8_t * aux8 = (const uint8_t *)aux32;\n\n ggml_int8x16x4_t q2u;\n ggml_int8x16x4_t q2s;\n ggml_int8x16x4_t q8b;\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n float sumf1 = 0, sumf2 = 0;\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;\n q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 0])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 1])));\n q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 2])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 3])));\n q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 8])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 9])));\n q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[10])), vld1_s8((const void *)(iq2xxs_grid + aux8[11])));\n q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 7) & 127))));\n q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 21) & 127))));\n q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >> 7) & 127))));\n q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >> 21) & 127))));\n q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);\n q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);\n q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);\n q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);\n const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]), q2u.val[1], q8b.val[1]);\n const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]), q2u.val[3], q8b.val[3]);\n sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[1] >> 28));\n sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[3] >> 28));\n }\n sumf += d*(sumf1 + sumf2);\n }\n *s = 0.25f * sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n ggml_int8x16x4_t q2u;\n ggml_int8x16x4_t q2s;\n ggml_int8x16x4_t q8b;\n\n int32x4x4_t scales32;\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n const uint8x8_t scales8 = vld1_u8(x[i].scales);\n const uint8x8_t scales_l = vand_u8(scales8, vdup_n_u8(0xf));\n const uint8x8_t scales_h = vshr_n_u8(scales8, 4);\n uint8x16_t scales = vcombine_u8(vzip1_u8(scales_l, scales_h), vzip2_u8(scales_l, scales_h));\n scales = vaddq_u8(vshlq_n_u8(scales, 1), vdupq_n_u8(1));\n const uint16x8_t scales1 = vmovl_u8(vget_low_u8(scales));\n const uint16x8_t scales2 = vmovl_u8(vget_high_u8(scales));\n scales32.val[0] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales1)));\n scales32.val[1] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales1)));\n scales32.val[2] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales2)));\n scales32.val[3] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales2)));\n int32x4_t sumi = vdupq_n_s32(0);\n for (int ib64 = 0; ib64 < QK_K/64; ++ib64) {\n q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[0] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[1] & 511))));\n q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[2] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[3] & 511))));\n q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[4] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[5] & 511))));\n q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[6] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[7] & 511))));\n q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[0] >> 9))), vld1_s8((const void *)(signs64 + (q2[1] >> 9))));\n q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[2] >> 9))), vld1_s8((const void *)(signs64 + (q2[3] >> 9))));\n q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[4] >> 9))), vld1_s8((const void *)(signs64 + (q2[5] >> 9))));\n q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[6] >> 9))), vld1_s8((const void *)(signs64 + (q2[7] >> 9))));\n q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);\n q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);\n q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);\n q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);\n const int32x4_t p1 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]);\n const int32x4_t p2 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[1], q8b.val[1]);\n const int32x4_t p3 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]);\n const int32x4_t p4 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[3], q8b.val[3]);\n const int32x4_t p = vpaddq_s32(vpaddq_s32(p1, p2), vpaddq_s32(p3, p4));\n sumi = vmlaq_s32(sumi, p, scales32.val[ib64]);\n q2 += 8;\n }\n sumf += d*vaddvq_s32(sumi);\n }\n *s = 0.125f * sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};\n\n const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1);\n const uint8x16_t mask2 = vld1q_u8(k_mask2);\n const uint8x16_t m1 = vdupq_n_u8(1);\n const int32x4_t vzero = vdupq_n_s32(0);\n\n uint8x16x2_t vs;\n ggml_int8x16x4_t q2s;\n ggml_int8x16x4_t q8b;\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n int sumi1 = 0, sumi2 = 0;\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n q2s.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[0] | ((qh[ib32+0] << 8) & 0x300)))),\n vld1_s8((const int8_t *)(iq2s_grid + (qs[1] | ((qh[ib32+0] << 6) & 0x300)))));\n q2s.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[2] | ((qh[ib32+0] << 4) & 0x300)))),\n vld1_s8((const int8_t *)(iq2s_grid + (qs[3] | ((qh[ib32+0] << 2) & 0x300)))));\n q2s.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[4] | ((qh[ib32+1] << 8) & 0x300)))),\n vld1_s8((const int8_t *)(iq2s_grid + (qs[5] | ((qh[ib32+1] << 6) & 0x300)))));\n q2s.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[6] | ((qh[ib32+1] << 4) & 0x300)))),\n vld1_s8((const int8_t *)(iq2s_grid + (qs[7] | ((qh[ib32+1] << 2) & 0x300)))));\n qs += 8;\n\n vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16)));\n vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);\n vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);\n vs.val[0] = vceqq_u8(vs.val[0], mask2);\n vs.val[1] = vceqq_u8(vs.val[1], mask2);\n\n q2s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[0]);\n q2s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[1]);\n\n vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16)));\n vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);\n vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);\n vs.val[0] = vceqq_u8(vs.val[0], mask2);\n vs.val[1] = vceqq_u8(vs.val[1], mask2);\n\n signs += 4;\n\n q2s.val[2] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[2]);\n q2s.val[3] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[3]);\n\n const int32x4_t p1 = ggml_vdotq_s32(vzero, q2s.val[0], q8b.val[0]);\n const int32x4_t p2 = ggml_vdotq_s32(vzero, q2s.val[1], q8b.val[1]);\n const int32x4_t p3 = ggml_vdotq_s32(vzero, q2s.val[2], q8b.val[2]);\n const int32x4_t p4 = ggml_vdotq_s32(vzero, q2s.val[3], q8b.val[3]);\n\n sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32+0] & 0xf));\n sumi2 += vaddvq_s32(p2) * (1 + 2*(x[i].scales[ib32+0] >> 4));\n sumi1 += vaddvq_s32(p3) * (1 + 2*(x[i].scales[ib32+1] & 0xf));\n sumi2 += vaddvq_s32(p4) * (1 + 2*(x[i].scales[ib32+1] >> 4));\n }\n sumf += d*(sumi1 + sumi2);\n }\n\n *s = 0.125f * sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n uint32_t aux32[2];\n\n ggml_int8x16x4_t q3s;\n ggml_int8x16x4_t q8b;\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n float sumf1 = 0, sumf2 = 0;\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n memcpy(aux32, gas, 2*sizeof(uint32_t)); gas += 2*sizeof(uint32_t);\n const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]);\n const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]);\n const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]);\n const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]);\n q3 += 16;\n q3s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >> 7) & 127))));\n q3s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >> 21) & 127))));\n q3s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 7) & 127))));\n q3s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 21) & 127))));\n q3s.val[0] = vmulq_s8(q3s.val[0], vreinterpretq_s8_u32(aux32x4_0));\n q3s.val[1] = vmulq_s8(q3s.val[1], vreinterpretq_s8_u32(aux32x4_1));\n q3s.val[2] = vmulq_s8(q3s.val[2], vreinterpretq_s8_u32(aux32x4_2));\n q3s.val[3] = vmulq_s8(q3s.val[3], vreinterpretq_s8_u32(aux32x4_3));\n const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]);\n const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]);\n sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[0] >> 28));\n sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[1] >> 28));\n }\n sumf += d*(sumf1 + sumf2);\n }\n *s = 0.5f * sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n typedef union {\n uint16x8_t vec_index;\n uint16_t index[8];\n } vec_index_t;\n\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};\n\n static const int16_t k_shift[8] = {8, 7, 6, 5, 4, 3, 2, 1};\n\n const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1);\n const uint8x16_t mask2 = vld1q_u8(k_mask2);\n\n const int16x8_t hshift = vld1q_s16(k_shift);\n const uint16x8_t m256 = vdupq_n_u16(256);\n const uint8x16_t m1 = vdupq_n_u8(1);\n\n uint8x16x2_t vs;\n ggml_int8x16x4_t q3s;\n ggml_int8x16x4_t q8b;\n vec_index_t idx;\n\n uint32_t scales32[2];\n const uint8_t * scales8 = (const uint8_t *)scales32;\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n memcpy(scales32, x[i].scales, 4);\n scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101;\n scales32[0] = ((scales32[0] & 0x0f0f0f0f) << 1) | 0x01010101;\n\n int sumi1 = 0, sumi2 = 0;\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n const uint8x16_t idx_l = vld1q_u8(qs); qs += 16;\n idx.vec_index = vorrq_u16(vmovl_u8(vget_low_u8 (idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+0]), hshift), m256));\n const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]],\n iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]);\n const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]],\n iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]);\n idx.vec_index = vorrq_u16(vmovl_u8(vget_high_u8(idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+1]), hshift), m256));\n const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]],\n iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]);\n const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]],\n iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]);\n\n\n vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16)));\n vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);\n vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);\n vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1);\n vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1);\n\n q3s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_0));\n q3s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_1));\n\n vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16)));\n vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);\n vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);\n vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1);\n vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1);\n\n signs += 4;\n\n q3s.val[2] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_2));\n q3s.val[3] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_3));\n\n const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]);\n const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]);\n\n sumi1 += vaddvq_s32(p1) * scales8[ib32/2+0];\n sumi2 += vaddvq_s32(p2) * scales8[ib32/2+4];\n }\n sumf += d*(sumi1 + sumi2);\n }\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __ARM_NEON\n\n ggml_int8x16x4_t q1b;\n ggml_int8x16x4_t q8b;\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint16_t * qh = x[i].qh;\n\n int sumi1 = 0, sumi2 = 0, sumi3 = 0;\n\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n\n q1b.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[0] | ((qh[ib+0] << 8) & 0x700)))),\n vld1_s8((const int8_t *)(iq1s_grid + (qs[1] | ((qh[ib+0] << 5) & 0x700)))));\n q1b.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[2] | ((qh[ib+0] << 2) & 0x700)))),\n vld1_s8((const int8_t *)(iq1s_grid + (qs[3] | ((qh[ib+0] >> 1) & 0x700)))));\n q1b.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[4] | ((qh[ib+1] << 8) & 0x700)))),\n vld1_s8((const int8_t *)(iq1s_grid + (qs[5] | ((qh[ib+1] << 5) & 0x700)))));\n q1b.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[6] | ((qh[ib+1] << 2) & 0x700)))),\n vld1_s8((const int8_t *)(iq1s_grid + (qs[7] | ((qh[ib+1] >> 1) & 0x700)))));\n qs += 8;\n\n q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[0], q8b.val[0]), q1b.val[1], q8b.val[1]);\n const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[2], q8b.val[2]), q1b.val[3], q8b.val[3]);\n\n const int ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;\n const int ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;\n sumi1 += vaddvq_s32(p1) * ls1;\n sumi2 += vaddvq_s32(p2) * ls2;\n sumi3 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * ls1 * (qh[ib+0] & 0x8000 ? -1 : 1)\n + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * ls2 * (qh[ib+1] & 0x8000 ? -1 : 1);\n\n }\n\n sumf += y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d) * (sumi1 + sumi2 + IQ1S_DELTA * sumi3);\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_m * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n iq1m_scale_t scale;\n\n#if defined __ARM_NEON\n const int32x4_t mask = vdupq_n_s32(0x7);\n const int32x4_t mone = vdupq_n_s32(1);\n const int32x4_t mzero = vdupq_n_s32(0);\n\n ggml_int8x16x4_t deltas;\n deltas.val[0] = vcombine_s8(vdup_n_s8(+1), vdup_n_s8(+1));\n deltas.val[1] = vcombine_s8(vdup_n_s8(-1), vdup_n_s8(+1));\n deltas.val[2] = vcombine_s8(vdup_n_s8(+1), vdup_n_s8(-1));\n deltas.val[3] = vcombine_s8(vdup_n_s8(-1), vdup_n_s8(-1));\n\n ggml_int8x16x4_t q1b;\n ggml_int8x16x4_t q8b;\n\n uint32_t aux32;\n const uint8_t * aux8 = (const uint8_t *)&aux32;\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint8_t * qh = x[i].qh;\n const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n\n int32x4_t sumi1 = mzero;\n int32x4_t sumi2 = mzero;\n\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n\n q1b.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[0] | ((qh[0] << 8) & 0x700)))),\n vld1_s8((const int8_t *)(iq1s_grid + (qs[1] | ((qh[0] << 4) & 0x700)))));\n q1b.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[2] | ((qh[1] << 8) & 0x700)))),\n vld1_s8((const int8_t *)(iq1s_grid + (qs[3] | ((qh[1] << 4) & 0x700)))));\n q1b.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[4] | ((qh[2] << 8) & 0x700)))),\n vld1_s8((const int8_t *)(iq1s_grid + (qs[5] | ((qh[2] << 4) & 0x700)))));\n q1b.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[6] | ((qh[3] << 8) & 0x700)))),\n vld1_s8((const int8_t *)(iq1s_grid + (qs[7] | ((qh[3] << 4) & 0x700)))));\n\n q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n const int32x4_t p1 = vpaddq_s32(ggml_vdotq_s32(mzero, q1b.val[0], q8b.val[0]), ggml_vdotq_s32(mzero, q1b.val[1], q8b.val[1]));\n const int32x4_t p2 = vpaddq_s32(ggml_vdotq_s32(mzero, q1b.val[2], q8b.val[2]), ggml_vdotq_s32(mzero, q1b.val[3], q8b.val[3]));\n const int32x4_t p12 = vpaddq_s32(p1, p2);\n\n const uint32_t * qh32 = (const uint32_t *)qh; // we are 4-byte aligned, so we can do that\n aux32 = ((qh32[0] >> 3) & 0x01010101) | ((qh32[0] >> 6) & 0x02020202);\n\n const int32x4_t p3 = vpaddq_s32(ggml_vdotq_s32(mzero, deltas.val[aux8[0]], q8b.val[0]), ggml_vdotq_s32(mzero, deltas.val[aux8[1]], q8b.val[1]));\n const int32x4_t p4 = vpaddq_s32(ggml_vdotq_s32(mzero, deltas.val[aux8[2]], q8b.val[2]), ggml_vdotq_s32(mzero, deltas.val[aux8[3]], q8b.val[3]));\n const int32x4_t p34 = vpaddq_s32(p3, p4);\n\n int32x4_t scales_4 = ggml_vld1q_u32(sc[ib/2] >> 0, sc[ib/2] >> 3, sc[ib/2] >> 6, sc[ib/2] >> 9);\n\n scales_4 = vaddq_s32(vshlq_n_s32(vandq_s32(scales_4, mask), 1), mone);\n\n sumi1 = vmlaq_s32(sumi1, scales_4, p12);\n sumi2 = vmlaq_s32(sumi2, scales_4, p34);\n\n qs += 8; qh += 4;\n\n }\n\n sumf += y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16) * (vaddvq_s32(sumi1) + IQ1M_DELTA * vaddvq_s32(sumi2));\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(scale);\n ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK4_NL == 0);\n static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n const block_iq4_nl * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK4_NL;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined __ARM_NEON\n const int8x16_t values = vld1q_s8(kvalues_iq4nl);\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n uint8x16x2_t q4bits;\n int8x16x4_t q4b;\n int8x16x4_t q8b;\n int32x4_t prod_1, prod_2;\n\n for (; ib + 1 < nb; ib += 2) {\n\n q4bits.val[0] = vld1q_u8(x[ib + 0].qs);\n q4bits.val[1] = vld1q_u8(x[ib + 1].qs);\n q8b.val[0] = vld1q_s8(y[ib + 0].qs);\n q8b.val[1] = vld1q_s8(y[ib + 0].qs + 16);\n q8b.val[2] = vld1q_s8(y[ib + 1].qs);\n q8b.val[3] = vld1q_s8(y[ib + 1].qs + 16);\n\n q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[0], m4b));\n q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));\n q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[1], m4b));\n q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));\n\n prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);\n prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);\n\n sumf +=\n GGML_CPU_FP16_TO_FP32(x[ib+0].d) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * vaddvq_s32(prod_1) +\n GGML_CPU_FP16_TO_FP32(x[ib+1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * vaddvq_s32(prod_2);\n }\n\n#endif\n for (; ib < nb; ++ib) {\n const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);\n int sumi1 = 0, sumi2 = 0;\n for (int j = 0; j < QK4_NL/2; ++j) {\n sumi1 += y[ib].qs[j+ 0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];\n sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >> 4];\n }\n sumf += d * (sumi1 + sumi2);\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_K == 0);\n\n const block_iq4_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __ARM_NEON\n const int8x16_t values = vld1q_s8(kvalues_iq4nl);\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n ggml_uint8x16x2_t q4bits;\n ggml_int8x16x4_t q4b;\n ggml_int8x16x4_t q8b;\n int32x4_t prod_1, prod_2;\n\n float sumf = 0;\n\n for (int ibl = 0; ibl < nb; ++ibl) {\n\n const int8_t * q8 = y[ibl].qs;\n const uint8_t * q4 = x[ibl].qs;\n uint16_t h = x[ibl].scales_h;\n\n int sumi1 = 0, sumi2 = 0;\n for (int ib = 0; ib < QK_K/64; ++ib) {\n\n q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;\n q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[0], m4b));\n q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));\n q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[1], m4b));\n q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));\n\n prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);\n prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);\n\n int ls1 = ((x[ibl].scales_l[ib] & 0xf) | ((h << 4) & 0x30)) - 32;\n int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32;\n h >>= 4;\n sumi1 += vaddvq_s32(prod_1) * ls1;\n sumi2 += vaddvq_s32(prod_2) * ls2;\n\n }\n\n sumf += GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2);\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n" }, { "path": "src/ggml-cpu/arch/arm/repack.cpp", "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"traits.h\"\n\n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GGML_CPU_CLANG_WORKAROUND\n#include \"../../repack.h\"\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#endif\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_FEATURE_MATMUL_INT8) || defined(__ARM_FEATURE_DOTPROD))\n// Helper for decoding scales and mins of Q4_K and Q5_K block formats\nstatic inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t * out_mins, int8_t * out_scales) {\n constexpr uint32_t kmask1 = 0x3f3f3f3f;\n constexpr uint32_t kmask2 = 0x0f0f0f0f;\n constexpr uint32_t kmask3 = 0x03030303;\n constexpr uint8_t scales_size = 12;\n\n uint32_t sm[3];\n memcpy(sm, scales_in, scales_size);\n\n const uint32_t mins_0_3 = sm[1] & kmask1;\n const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4);\n const uint32x2_t mins_u32 = { mins_0_3, mins_4_7 };\n\n *out_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins_u32)));\n\n uint32_t scales_u32[2];\n scales_u32[0] = sm[0] & kmask1;\n scales_u32[1] = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4);\n memcpy(out_scales, scales_u32, 8);\n}\n#endif\n\nvoid ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n#if defined(__ARM_NEON)\n float32x4_t srcv[4][8];\n float id[4];\n\n for (int i = 0; i < nb; i++) {\n float32x4_t asrcv[8];\n float32x4_t amaxv[8];\n\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n for (int j = 0; j < 8; j++) srcv[row_iter][j] = vld1q_f32(x + row_iter * k + i * 32 + 4 * j);\n for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[row_iter][j]);\n\n for (int j = 0; j < 4; j++) amaxv[2 * j] = vmaxq_f32(asrcv[2 * j], asrcv[2 * j + 1]);\n for (int j = 0; j < 2; j++) amaxv[4 * j] = vmaxq_f32(amaxv[4 * j], amaxv[4 * j + 2]);\n for (int j = 0; j < 1; j++) amaxv[8 * j] = vmaxq_f32(amaxv[8 * j], amaxv[8 * j + 4]);\n\n const float amax = vmaxvq_f32(amaxv[0]);\n\n const float d = amax / ((1 << 7) - 1);\n id[row_iter] = d ? 1.0f / d : 0.0f;\n\n y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n }\n\n for (int j = 0; j < 8; j++) {\n float32x4_t v = vmulq_n_f32(srcv[0][j], id[0]);\n int32x4_t vi = vcvtnq_s32_f32(v);\n y[i].qs[16 * j + 0] = vgetq_lane_s32(vi, 0);\n y[i].qs[16 * j + 1] = vgetq_lane_s32(vi, 1);\n y[i].qs[16 * j + 2] = vgetq_lane_s32(vi, 2);\n y[i].qs[16 * j + 3] = vgetq_lane_s32(vi, 3);\n\n v = vmulq_n_f32(srcv[1][j], id[1]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[16 * j + 4] = vgetq_lane_s32(vi, 0);\n y[i].qs[16 * j + 5] = vgetq_lane_s32(vi, 1);\n y[i].qs[16 * j + 6] = vgetq_lane_s32(vi, 2);\n y[i].qs[16 * j + 7] = vgetq_lane_s32(vi, 3);\n\n v = vmulq_n_f32(srcv[2][j], id[2]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[16 * j + 8] = vgetq_lane_s32(vi, 0);\n y[i].qs[16 * j + 9] = vgetq_lane_s32(vi, 1);\n y[i].qs[16 * j + 10] = vgetq_lane_s32(vi, 2);\n y[i].qs[16 * j + 11] = vgetq_lane_s32(vi, 3);\n\n v = vmulq_n_f32(srcv[3][j], id[3]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[16 * j + 12] = vgetq_lane_s32(vi, 0);\n y[i].qs[16 * j + 13] = vgetq_lane_s32(vi, 1);\n y[i].qs[16 * j + 14] = vgetq_lane_s32(vi, 2);\n y[i].qs[16 * j + 15] = vgetq_lane_s32(vi, 3);\n }\n }\n#else\n UNUSED(nb);\n UNUSED(y);\n ggml_quantize_mat_q8_0_4x4_generic(x, vy, k);\n#endif\n}\n\nvoid ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n#if defined(__ARM_NEON)\n float32x4_t srcv[4][8];\n float id[4];\n\n for (int i = 0; i < nb; i++) {\n float32x4_t asrcv[8];\n float32x4_t amaxv[8];\n\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n for (int j = 0; j < 8; j++) srcv[row_iter][j] = vld1q_f32(x + row_iter * k + i * 32 + 4 * j);\n for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[row_iter][j]);\n\n for (int j = 0; j < 4; j++) amaxv[2 * j] = vmaxq_f32(asrcv[2 * j], asrcv[2 * j + 1]);\n for (int j = 0; j < 2; j++) amaxv[4 * j] = vmaxq_f32(amaxv[4 * j], amaxv[4 * j + 2]);\n for (int j = 0; j < 1; j++) amaxv[8 * j] = vmaxq_f32(amaxv[8 * j], amaxv[8 * j + 4]);\n\n const float amax = vmaxvq_f32(amaxv[0]);\n\n const float d = amax / ((1 << 7) - 1);\n id[row_iter] = d ? 1.0f / d : 0.0f;\n\n y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n }\n\n for (int j = 0; j < 4; j++) {\n float32x4_t v = vmulq_n_f32(srcv[0][2 * j], id[0]);\n int32x4_t vi = vcvtnq_s32_f32(v);\n y[i].qs[32 * j + 0] = vgetq_lane_s32(vi, 0);\n y[i].qs[32 * j + 1] = vgetq_lane_s32(vi, 1);\n y[i].qs[32 * j + 2] = vgetq_lane_s32(vi, 2);\n y[i].qs[32 * j + 3] = vgetq_lane_s32(vi, 3);\n v = vmulq_n_f32(srcv[0][2 * j + 1], id[0]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[32 * j + 4] = vgetq_lane_s32(vi, 0);\n y[i].qs[32 * j + 5] = vgetq_lane_s32(vi, 1);\n y[i].qs[32 * j + 6] = vgetq_lane_s32(vi, 2);\n y[i].qs[32 * j + 7] = vgetq_lane_s32(vi, 3);\n\n v = vmulq_n_f32(srcv[1][2 * j], id[1]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[32 * j + 8] = vgetq_lane_s32(vi, 0);\n y[i].qs[32 * j + 9] = vgetq_lane_s32(vi, 1);\n y[i].qs[32 * j + 10] = vgetq_lane_s32(vi, 2);\n y[i].qs[32 * j + 11] = vgetq_lane_s32(vi, 3);\n v = vmulq_n_f32(srcv[1][2 * j + 1], id[1]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[32 * j + 12] = vgetq_lane_s32(vi, 0);\n y[i].qs[32 * j + 13] = vgetq_lane_s32(vi, 1);\n y[i].qs[32 * j + 14] = vgetq_lane_s32(vi, 2);\n y[i].qs[32 * j + 15] = vgetq_lane_s32(vi, 3);\n\n v = vmulq_n_f32(srcv[2][2 * j], id[2]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[32 * j + 16] = vgetq_lane_s32(vi, 0);\n y[i].qs[32 * j + 17] = vgetq_lane_s32(vi, 1);\n y[i].qs[32 * j + 18] = vgetq_lane_s32(vi, 2);\n y[i].qs[32 * j + 19] = vgetq_lane_s32(vi, 3);\n v = vmulq_n_f32(srcv[2][2 * j + 1], id[2]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[32 * j + 20] = vgetq_lane_s32(vi, 0);\n y[i].qs[32 * j + 21] = vgetq_lane_s32(vi, 1);\n y[i].qs[32 * j + 22] = vgetq_lane_s32(vi, 2);\n y[i].qs[32 * j + 23] = vgetq_lane_s32(vi, 3);\n\n v = vmulq_n_f32(srcv[3][2 * j], id[3]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[32 * j + 24] = vgetq_lane_s32(vi, 0);\n y[i].qs[32 * j + 25] = vgetq_lane_s32(vi, 1);\n y[i].qs[32 * j + 26] = vgetq_lane_s32(vi, 2);\n y[i].qs[32 * j + 27] = vgetq_lane_s32(vi, 3);\n v = vmulq_n_f32(srcv[3][2 * j + 1], id[3]);\n vi = vcvtnq_s32_f32(v);\n y[i].qs[32 * j + 28] = vgetq_lane_s32(vi, 0);\n y[i].qs[32 * j + 29] = vgetq_lane_s32(vi, 1);\n y[i].qs[32 * j + 30] = vgetq_lane_s32(vi, 2);\n y[i].qs[32 * j + 31] = vgetq_lane_s32(vi, 3);\n }\n }\n\n#else\n UNUSED(nb);\n UNUSED(y);\n ggml_quantize_mat_q8_0_4x8_generic(x, vy, k);\n#endif\n}\n\nvoid ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx;\n\n for (int c = 0; c < nc; c += ncols_interleaved) {\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n float32x4_t acc = vdupq_n_f32(0);\n for (int b = 0; b < nb; b++) {\n int8x16_t b0 = vld1q_s8((const int8_t *) b_ptr->qs);\n int8x16_t b1 = vld1q_s8((const int8_t *) b_ptr->qs + 16);\n int8x16_t b2 = vld1q_s8((const int8_t *) b_ptr->qs + 32);\n int8x16_t b3 = vld1q_s8((const int8_t *) b_ptr->qs + 48);\n float16x4_t bd = vld1_f16((const __fp16 *) b_ptr->d);\n\n int8x16_t a0 = vld1q_s8(a_ptr->qs);\n int8x16_t a1 = vld1q_s8(a_ptr->qs + qk/2);\n float16x4_t ad = vld1_dup_f16((const __fp16 *) &a_ptr->d);\n\n int32x4_t ret = vdupq_n_s32(0);\n\n ret = vdotq_laneq_s32(ret, b0 << 4, a0, 0);\n ret = vdotq_laneq_s32(ret, b1 << 4, a0, 1);\n ret = vdotq_laneq_s32(ret, b2 << 4, a0, 2);\n ret = vdotq_laneq_s32(ret, b3 << 4, a0, 3);\n\n ret = vdotq_laneq_s32(ret, b0 & 0xf0U, a1, 0);\n ret = vdotq_laneq_s32(ret, b1 & 0xf0U, a1, 1);\n ret = vdotq_laneq_s32(ret, b2 & 0xf0U, a1, 2);\n ret = vdotq_laneq_s32(ret, b3 & 0xf0U, a1, 3);\n\n acc = vfmaq_f32(acc, vcvtq_n_f32_s32(ret, 4),\n vmulq_f32(vcvt_f32_f16(ad), vcvt_f32_f16(bd)));\n a_ptr++;\n b_ptr++;\n }\n vst1q_f32(s, acc);\n s += ncols_interleaved;\n }\n return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx;\n\n for (int c = 0; c < nc; c += ncols_interleaved) {\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n float32x4_t acc = vdupq_n_f32(0);\n for (int b = 0; b < nb; b++) {\n int8x16_t b0 = vld1q_s8((const int8_t *) b_ptr->qs);\n int8x16_t b1 = vld1q_s8((const int8_t *) b_ptr->qs + 16);\n int8x16_t b2 = vld1q_s8((const int8_t *) b_ptr->qs + 32);\n int8x16_t b3 = vld1q_s8((const int8_t *) b_ptr->qs + 48);\n float16x4_t bd = vld1_f16((const __fp16 *) b_ptr->d);\n\n int8x16_t a0 = (int8x16_t) vld1q_dup_s64((const int64_t *) a_ptr->qs);\n int8x16_t a1 = (int8x16_t) vld1q_dup_s64((const int64_t *) a_ptr->qs + 1);\n int8x16_t a2 = (int8x16_t) vld1q_dup_s64((const int64_t *) a_ptr->qs + 2);\n int8x16_t a3 = (int8x16_t) vld1q_dup_s64((const int64_t *) a_ptr->qs + 3);\n float16x4_t ad = vld1_dup_f16((const __fp16 *) &a_ptr->d);\n\n int32x4_t ret0 = vdupq_n_s32(0);\n int32x4_t ret1 = vdupq_n_s32(0);\n\n ret0 = vdotq_s32(ret0, b0 << 4, a0);\n ret1 = vdotq_s32(ret1, b1 << 4, a0);\n ret0 = vdotq_s32(ret0, b2 << 4, a1);\n ret1 = vdotq_s32(ret1, b3 << 4, a1);\n\n ret0 = vdotq_s32(ret0, b0 & 0xf0U, a2);\n ret1 = vdotq_s32(ret1, b1 & 0xf0U, a2);\n ret0 = vdotq_s32(ret0, b2 & 0xf0U, a3);\n ret1 = vdotq_s32(ret1, b3 & 0xf0U, a3);\n\n int32x4_t ret = vpaddq_s32(ret0, ret1);\n\n acc = vfmaq_f32(acc, vcvtq_n_f32_s32(ret, 4),\n vmulq_f32(vcvt_f32_f16(ad), vcvt_f32_f16(bd)));\n a_ptr++;\n b_ptr++;\n }\n vst1q_f32(s, acc);\n s += ncols_interleaved;\n }\n return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q4_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)\n#if defined(__ARM_FEATURE_SVE)\n if (ggml_cpu_get_sve_cnt() == QK8_0) {\n const void * b_ptr = vx;\n const void * a_ptr = vy;\n float * res_ptr = s;\n\n __asm__ __volatile__(\n \"ptrue p0.b\\n\"\n \"add %x[b_ptr], %x[b_ptr], #0x10\\n\"\n \"1:\" // Column loop\n \"add x22, %x[a_ptr], #0x2\\n\"\n \"mov z31.b, #0x0\\n\"\n \"mov x21, %x[nb]\\n\"\n \"2:\" // Block loop\n \"ld1b { z30.b }, p0/Z, [%x[b_ptr]]\\n\"\n \"ld1b { z29.b }, p0/Z, [%x[b_ptr], #1, MUL VL]\\n\"\n \"mov z28.s, #0x0\\n\"\n \"mov z27.s, #0x0\\n\"\n \"ld1rd { z26.d }, p0/Z, [x22]\\n\"\n \"ld1b { z25.b }, p0/Z, [%x[b_ptr], #2, MUL VL]\\n\"\n \"sub x20, x22, #0x2\\n\"\n \"sub x21, x21, #0x1\\n\"\n \"ld1b { z24.b }, p0/Z, [%x[b_ptr], #3, MUL VL]\\n\"\n \"ld1rd { z23.d }, p0/Z, [x22, #8]\\n\"\n \"lsl z22.b, z30.b, #0x4\\n\"\n \"lsl z16.b, z29.b, #0x4\\n\"\n \"and z30.b, z30.b, #0xf0\\n\"\n \"and z29.b, z29.b, #0xf0\\n\"\n \"ld1rd { z21.d }, p0/Z, [x22, #16]\\n\"\n \"ld1rd { z20.d }, p0/Z, [x22, #24]\\n\"\n \"lsl z19.b, z25.b, #0x4\\n\"\n \"and z25.b, z25.b, #0xf0\\n\"\n \"ld1rh { z17.h }, p0/Z, [x20]\\n\"\n \"ld1h { z18.s }, p0/Z, [%x[b_ptr], #-1, MUL VL]\\n\"\n \"sdot z28.s, z22.b, z26.b\\n\"\n \"sdot z27.s, z16.b, z26.b\\n\"\n \"lsl z16.b, z24.b, #0x4\\n\"\n \"add x22, x22, #0x22\\n\"\n \"and z24.b, z24.b, #0xf0\\n\"\n \"add %x[b_ptr], %x[b_ptr], #0x90\\n\"\n \"fcvt z17.s, p0/m, z17.h\\n\"\n \"fcvt z18.s, p0/m, z18.h\\n\"\n \"sdot z28.s, z19.b, z23.b\\n\"\n \"sdot z27.s, z16.b, z23.b\\n\"\n \"fmul z18.s, z18.s, z17.s\\n\"\n \"sdot z28.s, z30.b, z21.b\\n\"\n \"sdot z27.s, z29.b, z21.b\\n\"\n \"sdot z28.s, z25.b, z20.b\\n\"\n \"sdot z27.s, z24.b, z20.b\\n\"\n \"uzp1 z17.s, z28.s, z27.s\\n\"\n \"uzp2 z16.s, z28.s, z27.s\\n\"\n \"add z17.s, z17.s, z16.s\\n\"\n \"asr z17.s, z17.s, #0x4\\n\"\n \"scvtf z17.s, p0/m, z17.s\\n\"\n \"fmla z31.s, p0/M, z17.s, z18.s\\n\"\n \"cbnz x21, 2b\\n\"\n \"sub %x[nc], %x[nc], #0x8\\n\"\n \"st1w { z31.s }, p0, [%x[res_ptr]]\\n\"\n \"add %x[res_ptr], %x[res_ptr], #0x20\\n\"\n \"cbnz %x[nc], 1b\\n\"\n : [b_ptr] \"+&r\" (b_ptr), [res_ptr] \"+&r\" (res_ptr), [nc] \"+&r\" (nc)\n : [a_ptr] \"r\" (a_ptr), [nb] \"r\" (nb)\n : \"memory\", \"p0\", \"x20\", \"x21\", \"x22\", \"z16\", \"z17\", \"z18\", \"z19\", \"z20\", \"z21\", \"z22\", \"z23\", \"z24\", \"z25\", \"z26\", \"z27\", \"z28\", \"z29\", \"z30\", \"z31\"\n );\n return;\n }\n#endif // #if defined(__ARM_FEATURE_SVE)\n\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)\n ggml_gemv_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n const int8x16_t kvalues = vld1q_s8(kvalues_iq4nl);\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n float * res_ptr = s;\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx4 * b_ptr = (const block_iq4_nlx4 *) vx + (x * nb);\n\n float32x4_t sumf = vdupq_n_f32(0);\n for (int l = 0; l < nb; l++) {\n uint8x16_t b_0 = vld1q_u8(b_ptr[l].qs + 0);\n uint8x16_t b_1 = vld1q_u8(b_ptr[l].qs + 16);\n uint8x16_t b_2 = vld1q_u8(b_ptr[l].qs + 32);\n uint8x16_t b_3 = vld1q_u8(b_ptr[l].qs + 48);\n\n int8x16_t b_0_hi = vqtbl1q_s8(kvalues, b_0 >> 4);\n int8x16_t b_0_lo = vqtbl1q_s8(kvalues, b_0 & 0x0F);\n int8x16_t b_1_hi = vqtbl1q_s8(kvalues, b_1 >> 4);\n int8x16_t b_1_lo = vqtbl1q_s8(kvalues, b_1 & 0x0F);\n int8x16_t b_2_hi = vqtbl1q_s8(kvalues, b_2 >> 4);\n int8x16_t b_2_lo = vqtbl1q_s8(kvalues, b_2 & 0x0F);\n int8x16_t b_3_hi = vqtbl1q_s8(kvalues, b_3 >> 4);\n int8x16_t b_3_lo = vqtbl1q_s8(kvalues, b_3 & 0x0F);\n\n int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 0);\n int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16);\n\n int32x4_t sumi = vdupq_n_s32(0);\n sumi = vdotq_laneq_s32(sumi, b_0_lo, a_0, 0);\n sumi = vdotq_laneq_s32(sumi, b_0_hi, a_1, 0);\n sumi = vdotq_laneq_s32(sumi, b_1_lo, a_0, 1);\n sumi = vdotq_laneq_s32(sumi, b_1_hi, a_1, 1);\n sumi = vdotq_laneq_s32(sumi, b_2_lo, a_0, 2);\n sumi = vdotq_laneq_s32(sumi, b_2_hi, a_1, 2);\n sumi = vdotq_laneq_s32(sumi, b_3_lo, a_0, 3);\n sumi = vdotq_laneq_s32(sumi, b_3_hi, a_1, 3);\n\n float32x4_t a_d = vcvt_f32_f16(vld1_dup_f16((const float16_t *)&a_ptr[l].d));\n float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *)b_ptr[l].d));\n float32x4_t d = a_d * b_d;\n\n sumf = vmlaq_f32(sumf, d, vcvtq_f32_s32(sumi));\n }\n\n vst1q_f32(res_ptr + x * 4, sumf);\n }\n return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n ggml_gemv_iq4_nl_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n const int8x16_t kvalues = vld1q_s8(kvalues_mxfp4);\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n float * res_ptr = s;\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);\n\n float32x4_t sumf = vdupq_n_f32(0);\n for (int l = 0; l < nb; l++) {\n uint8x16_t b_0 = vld1q_u8(b_ptr[l].qs + 0);\n uint8x16_t b_1 = vld1q_u8(b_ptr[l].qs + 16);\n uint8x16_t b_2 = vld1q_u8(b_ptr[l].qs + 32);\n uint8x16_t b_3 = vld1q_u8(b_ptr[l].qs + 48);\n\n int8x16_t b_0_hi = vqtbl1q_s8(kvalues, b_0 >> 4);\n int8x16_t b_0_lo = vqtbl1q_s8(kvalues, b_0 & 0x0F);\n int8x16_t b_1_hi = vqtbl1q_s8(kvalues, b_1 >> 4);\n int8x16_t b_1_lo = vqtbl1q_s8(kvalues, b_1 & 0x0F);\n int8x16_t b_2_hi = vqtbl1q_s8(kvalues, b_2 >> 4);\n int8x16_t b_2_lo = vqtbl1q_s8(kvalues, b_2 & 0x0F);\n int8x16_t b_3_hi = vqtbl1q_s8(kvalues, b_3 >> 4);\n int8x16_t b_3_lo = vqtbl1q_s8(kvalues, b_3 & 0x0F);\n\n int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 0);\n int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16);\n\n int32x4_t sumi = vdupq_n_s32(0);\n sumi = vdotq_laneq_s32(sumi, b_0_lo, a_0, 0);\n sumi = vdotq_laneq_s32(sumi, b_0_hi, a_1, 0);\n sumi = vdotq_laneq_s32(sumi, b_1_lo, a_0, 1);\n sumi = vdotq_laneq_s32(sumi, b_1_hi, a_1, 1);\n sumi = vdotq_laneq_s32(sumi, b_2_lo, a_0, 2);\n sumi = vdotq_laneq_s32(sumi, b_2_hi, a_1, 2);\n sumi = vdotq_laneq_s32(sumi, b_3_lo, a_0, 3);\n sumi = vdotq_laneq_s32(sumi, b_3_hi, a_1, 3);\n\n float32x4_t a_d = vcvt_f32_f16(vld1_dup_f16((const float16_t *)&a_ptr[l].d));\n float32x4_t b_d = {\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[0]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[1]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[2]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[3]),\n };\n float32x4_t d = a_d * b_d;\n\n sumf = vmlaq_f32(sumf, d, vcvtq_f32_s32(sumi));\n }\n\n vst1q_f32(res_ptr + x * 4, sumf);\n }\n return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n ggml_gemv_mxfp4_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n constexpr int col_groups = ncols_interleaved / 4; // 0123 and 4567\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n\n // 1x8 tile = 2 x 4\n float32x4_t acc_f32[col_groups];\n\n const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < col_groups; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n float32x4_t q4_d_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d)); // d0 d1 d2 d3\n float32x4_t q4_d_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d + 4)); // d4 d5 d6 d7\n float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d);\n float32x4_t sb_scale_0123 = vmulq_f32(q4_d_0, q8_d);\n float32x4_t sb_scale_4567 = vmulq_f32(q4_d_1, q8_d);\n float32x4_t q4_dmin_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin)); // dmin 0..3\n float32x4_t q4_dmin_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin + 4)); // dmin 4..7\n float32x4_t sb_min_0123 = vmulq_f32(q4_dmin_0, q8_d);\n float32x4_t sb_min_4567 = vmulq_f32(q4_dmin_1, q8_d);\n\n // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567\n int32x4_t bias_acc[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };\n int32x4_t acc_lo[col_groups];\n int32x4_t acc_hi[col_groups];\n\n // Each bsum is 16 elements, pairwise add leaves us with the 8 bsums of the entire block\n const int16x8_t bsums = vpaddq_s16(vld1q_s16(q8_ptr[b].bsums), vld1q_s16(q8_ptr[b].bsums + 8));\n int16_t bsums_arr[8];\n vst1q_s16(bsums_arr, bsums);\n for (int sb = 0; sb < QK_K / 64; sb++) {\n for (int i = 0; i < col_groups; i++) {\n acc_lo[i] = vdupq_n_s32(0);\n acc_hi[i] = vdupq_n_s32(0);\n }\n // Need scales for the low and high nibbles\n // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n int16x8_t q4sb_mins[2];\n int16x8_t q4sb_scales[2];\n for (int i = 0; i < 2; i++) {\n int8_t aux_q4sb[8];\n const int offset = sb * 24 + i * 12;\n decode_q_Kx8_6bit_scales(&q4_ptr[b].scales[offset], &q4sb_mins[i], aux_q4sb);\n q4sb_scales[i] = vmovl_s8(vld1_s8(aux_q4sb));\n }\n\n int8x16_t q8_qs[64 / 16];\n for (int i = 0; i < 64 / 16; i++) {\n q8_qs[i] = vld1q_s8(q8_ptr[b].qs + sb * 64 + i * 16);\n }\n\n for (int c = 0; c < col_groups; c++) {\n uint8x16_t q4_cols[8];\n for (int i = 0; i < 8; i++) {\n q4_cols[i] = vld1q_u8(q4_ptr[b].qs + sb * QK_K + i * 32 + 16 * c);\n }\n\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[0], m4b)), q8_qs[0], 0);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[1], m4b)), q8_qs[0], 1);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[2], m4b)), q8_qs[0], 2);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[3], m4b)), q8_qs[0], 3);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[4], m4b)), q8_qs[1], 0);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[5], m4b)), q8_qs[1], 1);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[6], m4b)), q8_qs[1], 2);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[7], m4b)), q8_qs[1], 3);\n\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[0], 4)), q8_qs[2], 0);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[1], 4)), q8_qs[2], 1);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[2], 4)), q8_qs[2], 2);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[3], 4)), q8_qs[2], 3);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[4], 4)), q8_qs[3], 0);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[5], 4)), q8_qs[3], 1);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[6], 4)), q8_qs[3], 2);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[7], 4)), q8_qs[3], 3);\n }\n\n // Scales\n // row c0123 blk0 and blk1\n const int16x4_t sc_0123_lo = vget_low_s16(q4sb_scales[0]);\n const int16x4_t sc_0123_hi = vget_low_s16(q4sb_scales[1]);\n const float32x4_t sumf_0123 = vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_0123_lo), acc_lo[0]),\n vmulq_s32(vmovl_s16(sc_0123_hi), acc_hi[0])));\n acc_f32[0] = vfmaq_f32(acc_f32[0], sb_scale_0123, sumf_0123);\n // row c4567 blk0 and blk1\n const int16x4_t sc_4567_lo = vget_high_s16(q4sb_scales[0]);\n const int16x4_t sc_4567_hi = vget_high_s16(q4sb_scales[1]);\n const float32x4_t sumf_4567 = vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_4567_lo), acc_lo[1]),\n vmulq_s32(vmovl_s16(sc_4567_hi), acc_hi[1])));\n acc_f32[1] = vfmaq_f32(acc_f32[1], sb_scale_4567, sumf_4567);\n\n // Bias Correction\n const int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[2 * sb + 0]);\n const int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[2 * sb + 1]);\n\n bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_lo, vget_low_s16(q4sb_mins[0]));\n bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_hi, vget_low_s16(q4sb_mins[1]));\n bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_lo, vget_high_s16(q4sb_mins[0]));\n bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_hi, vget_high_s16(q4sb_mins[1]));\n } // for sb\n\n acc_f32[0] = vmlsq_f32(acc_f32[0], vcvtq_f32_s32(bias_acc[0]), sb_min_0123);\n acc_f32[1] = vmlsq_f32(acc_f32[1], vcvtq_f32_s32(bias_acc[1]), sb_min_4567);\n } // for b\n\n int base = x * ncols_interleaved;\n vst1q_f32(s + base, acc_f32[0]);\n vst1q_f32(s + base + 4, acc_f32[1]);\n } // for x\n return;\n#endif // #if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q4_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_K_8x8_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n constexpr int col_pairs = ncols_interleaved / 2;\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n\n // 1x8 tile = 2 x 4\n float32x4_t acc_f32[ncols_interleaved / 4];\n\n const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < ncols_interleaved / 4; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n float32x4_t q4_d_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d)); // d0 d1 d2 d3\n float32x4_t q4_d_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d + 4)); // d4 d5 d6 d7\n float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d);\n float32x4_t sb_scale_0 = vmulq_f32(q4_d_0, q8_d);\n float32x4_t sb_scale_1 = vmulq_f32(q4_d_1, q8_d);\n float32x4_t q4_dmin_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin)); // dmin 0..3\n float32x4_t q4_dmin_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin + 4)); // dmin 4..7\n float32x4_t sb_min_0 = vmulq_f32(q4_dmin_0, q8_d);\n float32x4_t sb_min_1 = vmulq_f32(q4_dmin_1, q8_d);\n\n // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567\n int32x4_t bias_acc[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };\n // 2 sb each iteration\n int32x4_t acc_lo[col_pairs];\n int32x4_t acc_hi[col_pairs];\n\n // Each bsum is 16 elements, pairwise add leaves us with the 8 bsums of the entire block\n const int16x8_t bsums = vpaddq_s16(vld1q_s16(q8_ptr[b].bsums), vld1q_s16(q8_ptr[b].bsums + 8));\n int16_t bsums_arr[8];\n vst1q_s16(bsums_arr, bsums);\n for (int sb = 0; sb < QK_K / 64; sb++) {\n for (int i = 0; i < col_pairs; i++) {\n acc_lo[i] = vdupq_n_s32(0);\n acc_hi[i] = vdupq_n_s32(0);\n }\n // Need scales for the low and high nibbles\n // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n int16x8_t q4sb_mins[2]; // int16 as its needed for bias_acc later\n int16x8_t q4sb_scales[2];\n for (int i = 0; i < 2; i++) {\n int8_t aux_q4sb[8];\n const int offset = sb * 24 + i * 12;\n decode_q_Kx8_6bit_scales(&q4_ptr[b].scales[offset], &q4sb_mins[i], aux_q4sb);\n q4sb_scales[i] = vmovl_s8(vld1_s8(aux_q4sb));\n }\n\n const uint8_t * q4_base = q4_ptr[b].qs + sb * QK_K;\n\n // Load the 64 quants from q8K duplicated to use vecdots with the interleaved columns\n // but still need the qs to use the low and hi bits from q4\n const int8_t * q8_base = q8_ptr[b].qs + sb * 64;\n int8x16_t q8_qs[8];\n for (int i = 0; i < 8; i++) {\n q8_qs[i] = (int8x16_t) vld1q_dup_s64((const int64_t *) (q8_base + i * 8));\n }\n\n // Q4s columns iterated in pairs (01, 23, 45, 67)\n for (int cp = 0; cp < col_pairs; cp++) {\n uint8x16_t q4_qs_cp_0 = vld1q_u8(q4_base + 16 * cp);\n uint8x16_t q4_qs_cp_1 = vld1q_u8(q4_base + 16 * cp + 64);\n uint8x16_t q4_qs_cp_2 = vld1q_u8(q4_base + 16 * cp + 128);\n uint8x16_t q4_qs_cp_3 = vld1q_u8(q4_base + 16 * cp + 192);\n\n acc_lo[cp] =\n ggml_vdotq_s32(acc_lo[cp], vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_0, m4b)), q8_qs[0]); // 0 .. 7\n acc_lo[cp] =\n ggml_vdotq_s32(acc_lo[cp], vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_1, m4b)), q8_qs[1]); // 8 ..15\n acc_lo[cp] =\n ggml_vdotq_s32(acc_lo[cp], vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_2, m4b)), q8_qs[2]); // 16..23\n acc_lo[cp] =\n ggml_vdotq_s32(acc_lo[cp], vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_3, m4b)), q8_qs[3]); // 24..31\n\n acc_hi[cp] =\n ggml_vdotq_s32(acc_hi[cp], vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_0, 4)), q8_qs[4]); // 32..39\n acc_hi[cp] =\n ggml_vdotq_s32(acc_hi[cp], vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_1, 4)), q8_qs[5]); // 40..47\n acc_hi[cp] =\n ggml_vdotq_s32(acc_hi[cp], vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_2, 4)), q8_qs[6]); // 48..55\n acc_hi[cp] =\n ggml_vdotq_s32(acc_hi[cp], vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_3, 4)), q8_qs[7]); // 56..63\n }\n\n // Iterates over a pair of column pairs (4 columns) to use a single 128 register\n // p = 0 -> 0123 p2 -> 4567\n for (int i = 0, p = 0; p < col_pairs; i++, p += 2) {\n int16x4_t group_scales_lo = p == 0 ? vget_low_s16(q4sb_scales[0]) : vget_high_s16(q4sb_scales[0]);\n int16x4_t group_scales_hi = p == 0 ? vget_low_s16(q4sb_scales[1]) : vget_high_s16(q4sb_scales[1]);\n float32x4_t sb_scale = p == 0 ? sb_scale_0 : sb_scale_1;\n\n // 0123 or 4567\n float32x4_t sumf_0 =\n vcvtq_f32_s32(vmulq_s32(vmovl_s16(group_scales_lo), vpaddq_s32(acc_lo[p], acc_lo[p + 1])));\n acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_0);\n\n float32x4_t sumf_1 =\n vcvtq_f32_s32(vmulq_s32(vmovl_s16(group_scales_hi), vpaddq_s32(acc_hi[p], acc_hi[p + 1])));\n acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_1);\n }\n\n // Multiply Acc bsum + mins\n // Each pair of subblocks share the same bsums\n // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)).\n int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[2 * sb + 0]);\n int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[2 * sb + 1]);\n\n // cols 0-3 bias\n bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_lo, vget_low_s16(q4sb_mins[0]));\n bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_hi, vget_low_s16(q4sb_mins[1]));\n\n // cols 4-7 bias\n bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_lo, vget_high_s16(q4sb_mins[0]));\n bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_hi, vget_high_s16(q4sb_mins[1]));\n } // for sb\n\n acc_f32[0] = vmlsq_f32(acc_f32[0], vcvtq_f32_s32(bias_acc[0]), sb_min_0);\n acc_f32[1] = vmlsq_f32(acc_f32[1], vcvtq_f32_s32(bias_acc[1]), sb_min_1);\n } // for b\n\n int base = x * ncols_interleaved;\n vst1q_f32(s + base, acc_f32[0]);\n vst1q_f32(s + base + 4, acc_f32[1]);\n } // for x\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q5_K_8x4_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 4;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n constexpr int col_groups = ncols_interleaved / 4; // 0123 and 4567\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n const uint8x16_t mone = vdupq_n_u8(1);\n const uint8x16_t mtwo = vdupq_n_u8(2);\n\n // 1x8 tile = 2 x 4\n float32x4_t acc_f32[col_groups];\n\n const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q5_Kx8 * GGML_RESTRICT q5_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < col_groups; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n float32x4_t q5_d_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d)); // d0 d1 d2 d3\n float32x4_t q5_d_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d + 4)); // d4 d5 d6 d7\n float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d);\n float32x4_t sb_scale_0123 = vmulq_f32(q5_d_0, q8_d);\n float32x4_t sb_scale_4567 = vmulq_f32(q5_d_1, q8_d);\n float32x4_t q5_dmin_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin)); // dmin 0..3\n float32x4_t q5_dmin_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin + 4)); // dmin 4..7\n float32x4_t sb_min_0123 = vmulq_f32(q5_dmin_0, q8_d);\n float32x4_t sb_min_4567 = vmulq_f32(q5_dmin_1, q8_d);\n\n // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567\n int32x4_t bias_acc[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };\n int32x4_t acc_lo[col_groups];\n int32x4_t acc_hi[col_groups];\n\n // Each bsum is 16 elements, pairwise add leaves us with the 8 bsums of the entire block\n const int16x8_t bsums = vpaddq_s16(vld1q_s16(q8_ptr[b].bsums), vld1q_s16(q8_ptr[b].bsums + 8));\n int16_t bsums_arr[8];\n vst1q_s16(bsums_arr, bsums);\n\n uint8x16_t qh[col_groups][8];\n for (int c = 0; c < col_groups; c++) {\n for (int i = 0; i < 8; i++) {\n qh[c][i] = vld1q_u8(q5_ptr[b].qh + i * 32 + 16 * c);\n }\n }\n\n for (int sb = 0; sb < QK_K / 64; sb++) {\n for (int i = 0; i < col_groups; i++) {\n acc_lo[i] = vdupq_n_s32(0);\n acc_hi[i] = vdupq_n_s32(0);\n }\n // Need scales for the low and high nibbles\n // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n int16x8_t q5sb_mins[2];\n int16x8_t q5sb_scales[2];\n for (int i = 0; i < 2; i++) {\n int8_t aux_q5sb[8];\n const int offset = sb * 24 + i * 12;\n decode_q_Kx8_6bit_scales(&q5_ptr[b].scales[offset], &q5sb_mins[i], aux_q5sb);\n q5sb_scales[i] = vmovl_s8(vld1_s8(aux_q5sb));\n }\n\n int8x16_t q8_qs[4];\n for (int i = 0; i < 4; i++) {\n q8_qs[i] = vld1q_s8(q8_ptr[b].qs + sb * 64 + i * 16);\n }\n\n for (int c = 0; c < col_groups; c++) {\n uint8x16_t q5_cols[8];\n uint8x16_t hbit_lo[8];\n uint8x16_t hbit_hi[8];\n int8x16_t q5_lo[8];\n int8x16_t q5_hi[8];\n\n for (int i = 0; i < 8; i++) {\n q5_cols[i] = vld1q_u8(q5_ptr[b].qs + sb * QK_K + i * 32 + 16 * c);\n hbit_lo[i] = vandq_u8(qh[c][i], mone);\n hbit_hi[i] = vshlq_n_u8(vandq_u8(qh[c][i], mtwo), 3);\n qh[c][i] = vshrq_n_u8(qh[c][i], 2);\n q5_lo[i] = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q5_cols[i], m4b), hbit_lo[i], 4));\n q5_hi[i] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5_cols[i], 4), hbit_hi[i]));\n }\n\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[0], q8_qs[0], 0);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[1], q8_qs[0], 1);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[2], q8_qs[0], 2);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[3], q8_qs[0], 3);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[4], q8_qs[1], 0);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[5], q8_qs[1], 1);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[6], q8_qs[1], 2);\n acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[7], q8_qs[1], 3);\n\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[0], q8_qs[2], 0);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[1], q8_qs[2], 1);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[2], q8_qs[2], 2);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[3], q8_qs[2], 3);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[4], q8_qs[3], 0);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[5], q8_qs[3], 1);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[6], q8_qs[3], 2);\n acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[7], q8_qs[3], 3);\n }\n\n // Scales\n // row c0123 blk0 and blk1\n const int16x4_t sc_0123_lo = vget_low_s16(q5sb_scales[0]);\n const int16x4_t sc_0123_hi = vget_low_s16(q5sb_scales[1]);\n const float32x4_t sumf_0123 = vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_0123_lo), acc_lo[0]),\n vmulq_s32(vmovl_s16(sc_0123_hi), acc_hi[0])));\n acc_f32[0] = vfmaq_f32(acc_f32[0], sb_scale_0123, sumf_0123);\n // row c4567 blk0 and blk1\n const int16x4_t sc_4567_lo = vget_high_s16(q5sb_scales[0]);\n const int16x4_t sc_4567_hi = vget_high_s16(q5sb_scales[1]);\n const float32x4_t sumf_4567 = vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_4567_lo), acc_lo[1]),\n vmulq_s32(vmovl_s16(sc_4567_hi), acc_hi[1])));\n acc_f32[1] = vfmaq_f32(acc_f32[1], sb_scale_4567, sumf_4567);\n\n // Bias Correction\n const int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[2 * sb + 0]);\n const int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[2 * sb + 1]);\n\n bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_lo, vget_low_s16(q5sb_mins[0]));\n bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_hi, vget_low_s16(q5sb_mins[1]));\n bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_lo, vget_high_s16(q5sb_mins[0]));\n bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_hi, vget_high_s16(q5sb_mins[1]));\n } // for sb\n\n acc_f32[0] = vmlsq_f32(acc_f32[0], vcvtq_f32_s32(bias_acc[0]), sb_min_0123);\n acc_f32[1] = vmlsq_f32(acc_f32[1], vcvtq_f32_s32(bias_acc[1]), sb_min_4567);\n } // for b\n\n int base = x * ncols_interleaved;\n vst1q_f32(s + base, acc_f32[0]);\n vst1q_f32(s + base + 4, acc_f32[1]);\n } // for x\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q5_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q5_K_8x8_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n constexpr int col_pairs = ncols_interleaved / 2;\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n const uint8x16_t mone = vdupq_n_u8(1);\n const uint8x16_t mtwo = vdupq_n_u8(2);\n\n // 1x8 tile = 2 x 4\n float32x4_t acc_f32[ncols_interleaved / 4];\n\n const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q5_Kx8 * GGML_RESTRICT q5_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < ncols_interleaved / 4; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n float32x4_t q5_d_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d)); // d0 d1 d2 d3\n float32x4_t q5_d_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d + 4)); // d4 d5 d6 d7\n float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d);\n float32x4_t sb_scale_0 = vmulq_f32(q5_d_0, q8_d);\n float32x4_t sb_scale_1 = vmulq_f32(q5_d_1, q8_d);\n float32x4_t q5_dmin_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin)); // dmin 0..3\n float32x4_t q5_dmin_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin + 4)); // dmin 4..7\n float32x4_t sb_min_0 = vmulq_f32(q5_dmin_0, q8_d);\n float32x4_t sb_min_1 = vmulq_f32(q5_dmin_1, q8_d);\n\n // 2 sb each iteration\n int32x4_t acc_lo[col_pairs];\n int32x4_t acc_hi[col_pairs];\n\n // Each bsum is 16 elements, pairwise add leaves us with the 8 bsums of the entire block\n const int16x8_t bsums = vpaddq_s16(vld1q_s16(q8_ptr[b].bsums), vld1q_s16(q8_ptr[b].bsums + 8));\n int16_t bsums_arr[8];\n vst1q_s16(bsums_arr, bsums);\n\n // Load qh once per block and shift after each subblock\n const uint8_t * qh_base = q5_ptr[b].qh;\n uint8x16_t qh[col_pairs][4];\n for (int cp = 0; cp < col_pairs; cp++) {\n qh[cp][0] = vld1q_u8(qh_base + 16 * cp);\n qh[cp][1] = vld1q_u8(qh_base + 16 * cp + 64);\n qh[cp][2] = vld1q_u8(qh_base + 16 * cp + 128);\n qh[cp][3] = vld1q_u8(qh_base + 16 * cp + 192);\n }\n\n for (int sb = 0; sb < QK_K / 64; sb++) {\n for (int i = 0; i < col_pairs; i++) {\n acc_lo[i] = vdupq_n_s32(0);\n acc_hi[i] = vdupq_n_s32(0);\n }\n // Need scales for the low and high nibbles\n // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n int16x8_t q5sb_mins[2]; // int16 as its needed for bias_acc later\n int16x8_t q5sb_scales[2];\n for (int i = 0; i < 2; i++) {\n int8_t aux_q5sb[8];\n const int offset = sb * 24 + i * 12;\n decode_q_Kx8_6bit_scales(&q5_ptr[b].scales[offset], &q5sb_mins[i], aux_q5sb);\n q5sb_scales[i] = vmovl_s8(vld1_s8(aux_q5sb));\n }\n\n const uint8_t * qs_base = q5_ptr[b].qs + sb * QK_K;\n\n // Load the 64 quants from q8K duplicated to use vecdots with the interleaved columns\n const int8_t * q8_base = q8_ptr[b].qs + sb * 64;\n int8x16_t q8_qs[8];\n for (int i = 0; i < 8; i++) {\n q8_qs[i] = (int8x16_t) vld1q_dup_s64((const int64_t *) (q8_base + i * 8));\n }\n\n // Q5s column pair loop unrolled\n {\n // Cols 01\n uint8x16_t qs_0 = vld1q_u8(qs_base);\n uint8x16_t qs_1 = vld1q_u8(qs_base + 64);\n uint8x16_t qs_2 = vld1q_u8(qs_base + 128);\n uint8x16_t qs_3 = vld1q_u8(qs_base + 192);\n\n uint8x16_t hbit_lo_0 = vandq_u8(qh[0][0], mone);\n uint8x16_t hbit_lo_1 = vandq_u8(qh[0][1], mone);\n uint8x16_t hbit_lo_2 = vandq_u8(qh[0][2], mone);\n uint8x16_t hbit_lo_3 = vandq_u8(qh[0][3], mone);\n uint8x16_t hbit_hi_0 = vshlq_n_u8(vandq_u8(qh[0][0], mtwo), 3);\n uint8x16_t hbit_hi_1 = vshlq_n_u8(vandq_u8(qh[0][1], mtwo), 3);\n uint8x16_t hbit_hi_2 = vshlq_n_u8(vandq_u8(qh[0][2], mtwo), 3);\n uint8x16_t hbit_hi_3 = vshlq_n_u8(vandq_u8(qh[0][3], mtwo), 3);\n\n qh[0][0] = vshrq_n_u8(qh[0][0], 2);\n qh[0][1] = vshrq_n_u8(qh[0][1], 2);\n qh[0][2] = vshrq_n_u8(qh[0][2], 2);\n qh[0][3] = vshrq_n_u8(qh[0][3], 2);\n\n acc_lo[0] = ggml_vdotq_s32(\n acc_lo[0], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_0, m4b), hbit_lo_0, 4)), q8_qs[0]);\n acc_lo[0] = ggml_vdotq_s32(\n acc_lo[0], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_1, m4b), hbit_lo_1, 4)), q8_qs[1]);\n acc_lo[0] = ggml_vdotq_s32(\n acc_lo[0], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_2, m4b), hbit_lo_2, 4)), q8_qs[2]);\n acc_lo[0] = ggml_vdotq_s32(\n acc_lo[0], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_3, m4b), hbit_lo_3, 4)), q8_qs[3]);\n acc_hi[0] = ggml_vdotq_s32(acc_hi[0], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_0, 4), hbit_hi_0)),\n q8_qs[4]);\n acc_hi[0] = ggml_vdotq_s32(acc_hi[0], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_1, 4), hbit_hi_1)),\n q8_qs[5]);\n acc_hi[0] = ggml_vdotq_s32(acc_hi[0], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_2, 4), hbit_hi_2)),\n q8_qs[6]);\n acc_hi[0] = ggml_vdotq_s32(acc_hi[0], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_3, 4), hbit_hi_3)),\n q8_qs[7]);\n\n // Cols 23\n qs_0 = vld1q_u8(qs_base + 16);\n qs_1 = vld1q_u8(qs_base + 80);\n qs_2 = vld1q_u8(qs_base + 144);\n qs_3 = vld1q_u8(qs_base + 208);\n\n hbit_lo_0 = vandq_u8(qh[1][0], mone);\n hbit_lo_1 = vandq_u8(qh[1][1], mone);\n hbit_lo_2 = vandq_u8(qh[1][2], mone);\n hbit_lo_3 = vandq_u8(qh[1][3], mone);\n hbit_hi_0 = vshlq_n_u8(vandq_u8(qh[1][0], mtwo), 3);\n hbit_hi_1 = vshlq_n_u8(vandq_u8(qh[1][1], mtwo), 3);\n hbit_hi_2 = vshlq_n_u8(vandq_u8(qh[1][2], mtwo), 3);\n hbit_hi_3 = vshlq_n_u8(vandq_u8(qh[1][3], mtwo), 3);\n\n qh[1][0] = vshrq_n_u8(qh[1][0], 2);\n qh[1][1] = vshrq_n_u8(qh[1][1], 2);\n qh[1][2] = vshrq_n_u8(qh[1][2], 2);\n qh[1][3] = vshrq_n_u8(qh[1][3], 2);\n\n acc_lo[1] = ggml_vdotq_s32(\n acc_lo[1], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_0, m4b), hbit_lo_0, 4)), q8_qs[0]);\n acc_lo[1] = ggml_vdotq_s32(\n acc_lo[1], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_1, m4b), hbit_lo_1, 4)), q8_qs[1]);\n acc_lo[1] = ggml_vdotq_s32(\n acc_lo[1], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_2, m4b), hbit_lo_2, 4)), q8_qs[2]);\n acc_lo[1] = ggml_vdotq_s32(\n acc_lo[1], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_3, m4b), hbit_lo_3, 4)), q8_qs[3]);\n acc_hi[1] = ggml_vdotq_s32(acc_hi[1], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_0, 4), hbit_hi_0)),\n q8_qs[4]);\n acc_hi[1] = ggml_vdotq_s32(acc_hi[1], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_1, 4), hbit_hi_1)),\n q8_qs[5]);\n acc_hi[1] = ggml_vdotq_s32(acc_hi[1], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_2, 4), hbit_hi_2)),\n q8_qs[6]);\n acc_hi[1] = ggml_vdotq_s32(acc_hi[1], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_3, 4), hbit_hi_3)),\n q8_qs[7]);\n\n // Cols 45\n qs_0 = vld1q_u8(qs_base + 32);\n qs_1 = vld1q_u8(qs_base + 96);\n qs_2 = vld1q_u8(qs_base + 160);\n qs_3 = vld1q_u8(qs_base + 224);\n\n hbit_lo_0 = vandq_u8(qh[2][0], mone);\n hbit_lo_1 = vandq_u8(qh[2][1], mone);\n hbit_lo_2 = vandq_u8(qh[2][2], mone);\n hbit_lo_3 = vandq_u8(qh[2][3], mone);\n hbit_hi_0 = vshlq_n_u8(vandq_u8(qh[2][0], mtwo), 3);\n hbit_hi_1 = vshlq_n_u8(vandq_u8(qh[2][1], mtwo), 3);\n hbit_hi_2 = vshlq_n_u8(vandq_u8(qh[2][2], mtwo), 3);\n hbit_hi_3 = vshlq_n_u8(vandq_u8(qh[2][3], mtwo), 3);\n\n qh[2][0] = vshrq_n_u8(qh[2][0], 2);\n qh[2][1] = vshrq_n_u8(qh[2][1], 2);\n qh[2][2] = vshrq_n_u8(qh[2][2], 2);\n qh[2][3] = vshrq_n_u8(qh[2][3], 2);\n\n acc_lo[2] = ggml_vdotq_s32(\n acc_lo[2], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_0, m4b), hbit_lo_0, 4)), q8_qs[0]);\n acc_lo[2] = ggml_vdotq_s32(\n acc_lo[2], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_1, m4b), hbit_lo_1, 4)), q8_qs[1]);\n acc_lo[2] = ggml_vdotq_s32(\n acc_lo[2], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_2, m4b), hbit_lo_2, 4)), q8_qs[2]);\n acc_lo[2] = ggml_vdotq_s32(\n acc_lo[2], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_3, m4b), hbit_lo_3, 4)), q8_qs[3]);\n acc_hi[2] = ggml_vdotq_s32(acc_hi[2], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_0, 4), hbit_hi_0)),\n q8_qs[4]);\n acc_hi[2] = ggml_vdotq_s32(acc_hi[2], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_1, 4), hbit_hi_1)),\n q8_qs[5]);\n acc_hi[2] = ggml_vdotq_s32(acc_hi[2], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_2, 4), hbit_hi_2)),\n q8_qs[6]);\n acc_hi[2] = ggml_vdotq_s32(acc_hi[2], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_3, 4), hbit_hi_3)),\n q8_qs[7]);\n\n // Cols 45\n qs_0 = vld1q_u8(qs_base + 48);\n qs_1 = vld1q_u8(qs_base + 112);\n qs_2 = vld1q_u8(qs_base + 176);\n qs_3 = vld1q_u8(qs_base + 240);\n\n hbit_lo_0 = vandq_u8(qh[3][0], mone);\n hbit_lo_1 = vandq_u8(qh[3][1], mone);\n hbit_lo_2 = vandq_u8(qh[3][2], mone);\n hbit_lo_3 = vandq_u8(qh[3][3], mone);\n hbit_hi_0 = vshlq_n_u8(vandq_u8(qh[3][0], mtwo), 3);\n hbit_hi_1 = vshlq_n_u8(vandq_u8(qh[3][1], mtwo), 3);\n hbit_hi_2 = vshlq_n_u8(vandq_u8(qh[3][2], mtwo), 3);\n hbit_hi_3 = vshlq_n_u8(vandq_u8(qh[3][3], mtwo), 3);\n\n qh[3][0] = vshrq_n_u8(qh[3][0], 2);\n qh[3][1] = vshrq_n_u8(qh[3][1], 2);\n qh[3][2] = vshrq_n_u8(qh[3][2], 2);\n qh[3][3] = vshrq_n_u8(qh[3][3], 2);\n\n acc_lo[3] = ggml_vdotq_s32(\n acc_lo[3], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_0, m4b), hbit_lo_0, 4)), q8_qs[0]);\n acc_lo[3] = ggml_vdotq_s32(\n acc_lo[3], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_1, m4b), hbit_lo_1, 4)), q8_qs[1]);\n acc_lo[3] = ggml_vdotq_s32(\n acc_lo[3], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_2, m4b), hbit_lo_2, 4)), q8_qs[2]);\n acc_lo[3] = ggml_vdotq_s32(\n acc_lo[3], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_3, m4b), hbit_lo_3, 4)), q8_qs[3]);\n acc_hi[3] = ggml_vdotq_s32(acc_hi[3], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_0, 4), hbit_hi_0)),\n q8_qs[4]);\n acc_hi[3] = ggml_vdotq_s32(acc_hi[3], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_1, 4), hbit_hi_1)),\n q8_qs[5]);\n acc_hi[3] = ggml_vdotq_s32(acc_hi[3], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_2, 4), hbit_hi_2)),\n q8_qs[6]);\n acc_hi[3] = ggml_vdotq_s32(acc_hi[3], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_3, 4), hbit_hi_3)),\n q8_qs[7]);\n }\n\n // Prepare bsum vectors for bias computation\n // Each pair of subblocks share the same bsums\n int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[2 * sb + 0]);\n int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[2 * sb + 1]);\n\n // Iterates over a pair of column pairs (4 columns) to use a single 128 register\n // p = 0 -> 0123 p2 -> 4567\n for (int i = 0, p = 0; p < col_pairs; i++, p += 2) {\n int16x4_t group_scales_lo = p == 0 ? vget_low_s16(q5sb_scales[0]) : vget_high_s16(q5sb_scales[0]);\n int16x4_t group_scales_hi = p == 0 ? vget_low_s16(q5sb_scales[1]) : vget_high_s16(q5sb_scales[1]);\n int16x4_t group_mins_lo = p == 0 ? vget_low_s16(q5sb_mins[0]) : vget_high_s16(q5sb_mins[0]);\n int16x4_t group_mins_hi = p == 0 ? vget_low_s16(q5sb_mins[1]) : vget_high_s16(q5sb_mins[1]);\n float32x4_t sb_scale = p == 0 ? sb_scale_0 : sb_scale_1;\n float32x4_t sb_min = p == 0 ? sb_min_0 : sb_min_1;\n\n // 0123 or 4567\n float32x4_t sumf_0 =\n vcvtq_f32_s32(vmulq_s32(vmovl_s16(group_scales_lo), vpaddq_s32(acc_lo[p], acc_lo[p + 1])));\n acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_0);\n\n float32x4_t sumf_1 =\n vcvtq_f32_s32(vmulq_s32(vmovl_s16(group_scales_hi), vpaddq_s32(acc_hi[p], acc_hi[p + 1])));\n acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_1);\n\n // FUSED BIAS: Compute and subtract bias immediately\n // bias = (bsums_lo * mins_lo + bsums_hi * mins_hi) * sb_min\n int32x4_t bias = vmull_s16(bsums_vec_lo, group_mins_lo);\n bias = vmlal_s16(bias, bsums_vec_hi, group_mins_hi);\n float32x4_t bias_f32 = vcvtq_f32_s32(bias);\n acc_f32[i] = vmlsq_f32(acc_f32[i], sb_min, bias_f32);\n }\n } // for sb\n } // for b\n\n int base = x * ncols_interleaved;\n vst1q_f32(s + base, acc_f32[0]);\n vst1q_f32(s + base + 4, acc_f32[1]);\n } // for x\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q5_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q6_K_8x4_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 4;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n constexpr int col_groups = ncols_interleaved / 4;\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n const uint8x16_t mask_lo = vdupq_n_u8(0x03);\n const uint8x16_t mask_hi = vdupq_n_u8(0x30);\n\n // 1x8 tile = 2 x 4\n float32x4_t acc_f32[2];\n\n const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < col_groups; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n float32x4_t q6_d_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d)); // d0 d1 d2 d3\n float32x4_t q6_d_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d + 4)); // d4 d5 d6 d7\n float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d);\n float32x4_t sb_scale_0 = vmulq_f32(q6_d_0, q8_d);\n float32x4_t sb_scale_1 = vmulq_f32(q6_d_1, q8_d);\n\n int32x4_t acc[col_groups];\n for (int i = 0; i < col_groups; i++) {\n acc[i] = vdupq_n_s32(0);\n }\n\n // Load all 16 scales once and widen to int16 (Q6_K has 16 scales per block)\n // Reused for bias and dequantization later\n int16_t q6_scales[16 * 8];\n for (int i = 0; i < 16; i++) {\n int16x8_t scales = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));\n vst1q_s16(q6_scales + i * 8, scales);\n }\n\n // Compute bias per column using q8 bsums and preloaded scales to skip the -32 shift\n int32x4_t bias_lo = vdupq_n_s32(0);\n int32x4_t bias_hi = vdupq_n_s32(0);\n\n // Load bsums in chunks of 4 to process with vectorized operations\n for (int i = 0; i < 16; i += 4) {\n int16x4_t bsums_vec = vld1_s16(q8_ptr[b].bsums + i);\n int16x4_t scales_lo_0 = vld1_s16(q6_scales + (i + 0) * 8);\n int16x4_t scales_hi_0 = vld1_s16(q6_scales + (i + 0) * 8 + 4);\n int16x4_t scales_lo_1 = vld1_s16(q6_scales + (i + 1) * 8);\n int16x4_t scales_hi_1 = vld1_s16(q6_scales + (i + 1) * 8 + 4);\n int16x4_t scales_lo_2 = vld1_s16(q6_scales + (i + 2) * 8);\n int16x4_t scales_hi_2 = vld1_s16(q6_scales + (i + 2) * 8 + 4);\n int16x4_t scales_lo_3 = vld1_s16(q6_scales + (i + 3) * 8);\n int16x4_t scales_hi_3 = vld1_s16(q6_scales + (i + 3) * 8 + 4);\n\n bias_lo = vmlal_lane_s16(bias_lo, scales_lo_0, bsums_vec, 0);\n bias_hi = vmlal_lane_s16(bias_hi, scales_hi_0, bsums_vec, 0);\n bias_lo = vmlal_lane_s16(bias_lo, scales_lo_1, bsums_vec, 1);\n bias_hi = vmlal_lane_s16(bias_hi, scales_hi_1, bsums_vec, 1);\n bias_lo = vmlal_lane_s16(bias_lo, scales_lo_2, bsums_vec, 2);\n bias_hi = vmlal_lane_s16(bias_hi, scales_hi_2, bsums_vec, 2);\n bias_lo = vmlal_lane_s16(bias_lo, scales_lo_3, bsums_vec, 3);\n bias_hi = vmlal_lane_s16(bias_hi, scales_hi_3, bsums_vec, 3);\n }\n bias_lo = vshlq_n_s32(bias_lo, 5);\n bias_hi = vshlq_n_s32(bias_hi, 5);\n\n // Process two 128-value halves per superblock\n for (int half = 0; half < 2; half++) {\n const uint8_t * ql_base = q6_ptr[b].ql + half * 512;\n const uint8_t * qh_base = q6_ptr[b].qh + half * 256;\n\n // A subblock (sb) is a set of weights that share the scale\n // Since q6_K scales are per 16 elements\n // num sbs -> 256 elements / (16 elements/scale * 2 elements/byte * 2 halves)\n for (int sb = 0; sb < QK_K / 64; sb++) {\n const int8_t * q8_base_l = q8_ptr[b].qs + half * 128 + sb * 16;\n const int8_t * q8_base_h = q8_base_l + 64;\n\n // Load and duplicate q8 values (each register covers four interleaved columns of q6)\n int8x16_t q8_l[4];\n int8x16_t q8_h[4];\n for (int i = 0; i < 4; i++) {\n q8_l[i] = (int8x16_t) vld1q_dup_s32((const int32_t *) (q8_base_l + i * 4));\n q8_h[i] = (int8x16_t) vld1q_dup_s32((const int32_t *) (q8_base_h + i * 4));\n }\n\n const int ql_off_base = sb * QK_K / 2;\n const int qh_off_base = ql_off_base & 255; // wraps after 256 bytes\n\n // Load 4 vectors at once (64 bytes each for ql_0, ql_1, qh_0, qh_1)\n uint8x16x4_t q6_ql_0 = vld1q_u8_x4(ql_base + ql_off_base);\n uint8x16x4_t q6_ql_1 = vld1q_u8_x4(ql_base + ql_off_base + 64);\n uint8x16x4_t q6_qh_0 = vld1q_u8_x4(qh_base + qh_off_base);\n uint8x16x4_t q6_qh_1 = vld1q_u8_x4(qh_base + qh_off_base + 64);\n\n // Adjust qh for subblocks 2 and 3 (shift right by 2)\n if (sb > 1) {\n q6_qh_0.val[0] = vshrq_n_u8(q6_qh_0.val[0], 2);\n q6_qh_0.val[1] = vshrq_n_u8(q6_qh_0.val[1], 2);\n q6_qh_0.val[2] = vshrq_n_u8(q6_qh_0.val[2], 2);\n q6_qh_0.val[3] = vshrq_n_u8(q6_qh_0.val[3], 2);\n q6_qh_1.val[0] = vshrq_n_u8(q6_qh_1.val[0], 2);\n q6_qh_1.val[1] = vshrq_n_u8(q6_qh_1.val[1], 2);\n q6_qh_1.val[2] = vshrq_n_u8(q6_qh_1.val[2], 2);\n q6_qh_1.val[3] = vshrq_n_u8(q6_qh_1.val[3], 2);\n }\n\n const uint8x16_t q6_ql[8] = { q6_ql_0.val[0], q6_ql_0.val[1], q6_ql_0.val[2], q6_ql_0.val[3],\n q6_ql_1.val[0], q6_ql_1.val[1], q6_ql_1.val[2], q6_ql_1.val[3] };\n const uint8x16_t q6_qh[8] = { q6_qh_0.val[0], q6_qh_0.val[1], q6_qh_0.val[2], q6_qh_0.val[3],\n q6_qh_1.val[0], q6_qh_1.val[1], q6_qh_1.val[2], q6_qh_1.val[3] };\n\n // Process column groups (0-3, 4-7)\n for (int g = 0; g < col_groups; g++) {\n int32x4_t sb_acc_l = vdupq_n_s32(0);\n int32x4_t sb_acc_h = vdupq_n_s32(0);\n\n for (int chunk = 0; chunk < 4; chunk++) {\n const int idx = chunk * 2 + g;\n\n const uint8x16_t q6_qs_l = q6_ql[idx];\n const uint8x16_t q6_qs_h = q6_qh[idx];\n\n // Extract high 2 bits for upper nibble reconstruction\n const uint8x16_t q6_qs_hh = vandq_u8(q6_qs_h, mask_hi);\n\n // q6 = (low4 | high2<<4), without -32 bias (handled via bsums)\n const int8x16_t q6_l =\n vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_qs_l, m4b), vandq_u8(q6_qs_h, mask_lo), 4));\n const int8x16_t q6_h = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_l, 4), q6_qs_hh));\n\n sb_acc_l = vdotq_s32(sb_acc_l, q6_l, q8_l[chunk]);\n sb_acc_h = vdotq_s32(sb_acc_h, q6_h, q8_h[chunk]);\n }\n\n const int scale_idx_l = half * 8 + sb;\n const int scale_idx_h = half * 8 + sb + 4;\n\n const int32x4_t scale_vec_l = vmovl_s16(vld1_s16(q6_scales + scale_idx_l * 8 + g * 4));\n const int32x4_t scale_vec_h = vmovl_s16(vld1_s16(q6_scales + scale_idx_h * 8 + g * 4));\n\n acc[g] = vmlaq_s32(acc[g], sb_acc_l, scale_vec_l);\n acc[g] = vmlaq_s32(acc[g], sb_acc_h, scale_vec_h);\n }\n }\n } // for half\n\n // Bias correction\n acc[0] = vsubq_s32(acc[0], bias_lo);\n acc[1] = vsubq_s32(acc[1], bias_hi);\n\n // Apply superblock scale (no mins for q6_K)\n // acc[g] has [c0, c1, c2, c3]\n float32x4_t w_0123 = vmulq_f32(vcvtq_f32_s32(acc[0]), sb_scale_0);\n float32x4_t w_4567 = vmulq_f32(vcvtq_f32_s32(acc[1]), sb_scale_1);\n\n acc_f32[0] = vaddq_f32(acc_f32[0], w_0123);\n acc_f32[1] = vaddq_f32(acc_f32[1], w_4567);\n } // for b\n\n int base = x * ncols_interleaved;\n vst1q_f32(s + base, acc_f32[0]);\n vst1q_f32(s + base + 4, acc_f32[1]);\n } // for x\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q6_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q6_K_8x8_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n constexpr int col_pairs = ncols_interleaved / 2;\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n const uint8x16_t mask_lo = vdupq_n_u8(0x03);\n const uint8x16_t mask_hi = vdupq_n_u8(0x30);\n\n // 1x8 tile = 2 x 4\n float32x4_t acc_f32[2];\n\n const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n acc_f32[0] = vdupq_n_f32(0);\n acc_f32[1] = vdupq_n_f32(0);\n\n for (int b = 0; b < nb; b++) {\n float32x4_t q6_d_0 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d)); // d0 d1 d2 d3\n float32x4_t q6_d_1 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d + 4)); // d4 d5 d6 d7\n float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d);\n float32x4_t sb_scale_0 = vmulq_f32(q6_d_0, q8_d);\n float32x4_t sb_scale_1 = vmulq_f32(q6_d_1, q8_d);\n\n int32x2_t acc[col_pairs];\n for (int i = 0; i < col_pairs; i++) {\n acc[i] = vdup_n_s32(0);\n }\n\n // Load all 16 scales once and widen to int16 (Q6_K has 16 scales per block)\n // Reused for bias and dequantization later\n int16_t q6_scales[16 * 8];\n for (int i = 0; i < 16; i++) {\n int16x8_t scales = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));\n vst1q_s16(q6_scales + i * 8, scales);\n }\n\n // Compute bias per column using q8 bsums and preloaded scales to skip the -32 shift\n int32x4_t bias_lo = vdupq_n_s32(0);\n int32x4_t bias_hi = vdupq_n_s32(0);\n\n // Load bsums in chunks of 4 to process with vectorized operations\n for (int i = 0; i < 16; i += 4) {\n int16x4_t bsums_vec = vld1_s16(q8_ptr[b].bsums + i);\n int16x4_t scales_lo_0 = vld1_s16(q6_scales + (i + 0) * 8);\n int16x4_t scales_hi_0 = vld1_s16(q6_scales + (i + 0) * 8 + 4);\n int16x4_t scales_lo_1 = vld1_s16(q6_scales + (i + 1) * 8);\n int16x4_t scales_hi_1 = vld1_s16(q6_scales + (i + 1) * 8 + 4);\n int16x4_t scales_lo_2 = vld1_s16(q6_scales + (i + 2) * 8);\n int16x4_t scales_hi_2 = vld1_s16(q6_scales + (i + 2) * 8 + 4);\n int16x4_t scales_lo_3 = vld1_s16(q6_scales + (i + 3) * 8);\n int16x4_t scales_hi_3 = vld1_s16(q6_scales + (i + 3) * 8 + 4);\n\n bias_lo = vmlal_lane_s16(bias_lo, scales_lo_0, bsums_vec, 0);\n bias_hi = vmlal_lane_s16(bias_hi, scales_hi_0, bsums_vec, 0);\n bias_lo = vmlal_lane_s16(bias_lo, scales_lo_1, bsums_vec, 1);\n bias_hi = vmlal_lane_s16(bias_hi, scales_hi_1, bsums_vec, 1);\n bias_lo = vmlal_lane_s16(bias_lo, scales_lo_2, bsums_vec, 2);\n bias_hi = vmlal_lane_s16(bias_hi, scales_hi_2, bsums_vec, 2);\n bias_lo = vmlal_lane_s16(bias_lo, scales_lo_3, bsums_vec, 3);\n bias_hi = vmlal_lane_s16(bias_hi, scales_hi_3, bsums_vec, 3);\n }\n bias_lo = vshlq_n_s32(bias_lo, 5);\n bias_hi = vshlq_n_s32(bias_hi, 5);\n\n // Process two 128-value halves per superblock\n for (int half = 0; half < 2; half++) {\n const uint8_t * ql_base = q6_ptr[b].ql + half * 512;\n const uint8_t * qh_base = q6_ptr[b].qh + half * 256;\n\n // A subblock (sb) is a set of weights that share the scale\n // Since q6_K scales are per 16 elements\n // num sbs -> 256 elements / (16 elements/scale * 2 elements/byte * 2 halves)\n for (int sb = 0; sb < QK_K / 64; sb++) {\n const int8_t * q8_base_l = q8_ptr[b].qs + half * 128 + sb * 16;\n const int8_t * q8_base_h = q8_base_l + 64;\n\n // Load and duplicate q8 values (each register covers two interleaved columns of q6)\n int8x16_t q8_l[2];\n int8x16_t q8_h[2];\n for (int i = 0; i < 2; i++) {\n q8_l[i] = (int8x16_t) vld1q_dup_s64((const int64_t *) (q8_base_l + i * 8));\n q8_h[i] = (int8x16_t) vld1q_dup_s64((const int64_t *) (q8_base_h + i * 8));\n }\n\n const int ql_off_base = sb * QK_K / 2;\n const int qh_off_base = ql_off_base & 255; // wraps after 256 bytes\n\n // Load 4 vectors at once (64 bytes each for ql_0, ql_1, qh_0, qh_1)\n uint8x16x4_t q6_ql_0 = vld1q_u8_x4(ql_base + ql_off_base);\n uint8x16x4_t q6_ql_1 = vld1q_u8_x4(ql_base + ql_off_base + 64);\n uint8x16x4_t q6_qh_0 = vld1q_u8_x4(qh_base + qh_off_base);\n uint8x16x4_t q6_qh_1 = vld1q_u8_x4(qh_base + qh_off_base + 64);\n\n // Adjust qh for subblocks 2 and 3 (shift right by 2)\n if (sb > 1) {\n q6_qh_0.val[0] = vshrq_n_u8(q6_qh_0.val[0], 2);\n q6_qh_0.val[1] = vshrq_n_u8(q6_qh_0.val[1], 2);\n q6_qh_0.val[2] = vshrq_n_u8(q6_qh_0.val[2], 2);\n q6_qh_0.val[3] = vshrq_n_u8(q6_qh_0.val[3], 2);\n q6_qh_1.val[0] = vshrq_n_u8(q6_qh_1.val[0], 2);\n q6_qh_1.val[1] = vshrq_n_u8(q6_qh_1.val[1], 2);\n q6_qh_1.val[2] = vshrq_n_u8(q6_qh_1.val[2], 2);\n q6_qh_1.val[3] = vshrq_n_u8(q6_qh_1.val[3], 2);\n }\n\n // Process column pairs (0-1, 2-3, 4-5, 6-7)\n for (int cp = 0; cp < col_pairs; cp++) {\n const uint8x16_t q6_qs_cp_0_l = q6_ql_0.val[cp];\n const uint8x16_t q6_qs_cp_1_l = q6_ql_1.val[cp];\n const uint8x16_t q6_qs_cp_0_h = q6_qh_0.val[cp];\n const uint8x16_t q6_qs_cp_1_h = q6_qh_1.val[cp];\n\n // Extract high 2 bits for upper nibble reconstruction\n const uint8x16_t q6_qs_cp_0_hh = vandq_u8(q6_qs_cp_0_h, mask_hi);\n const uint8x16_t q6_qs_cp_1_hh = vandq_u8(q6_qs_cp_1_h, mask_hi);\n\n // q6 = (low4 | high2<<4), without -32 bias (handled via bsums)\n const int8x16_t q6_l0 = vreinterpretq_s8_u8(\n vsliq_n_u8(vandq_u8(q6_qs_cp_0_l, m4b), vandq_u8(q6_qs_cp_0_h, mask_lo), 4));\n const int8x16_t q6_l1 = vreinterpretq_s8_u8(\n vsliq_n_u8(vandq_u8(q6_qs_cp_1_l, m4b), vandq_u8(q6_qs_cp_1_h, mask_lo), 4));\n const int8x16_t q6_h0 =\n vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_cp_0_l, 4), q6_qs_cp_0_hh));\n const int8x16_t q6_h1 =\n vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_cp_1_l, 4), q6_qs_cp_1_hh));\n\n int32x4_t sb_acc_l = vdupq_n_s32(0);\n sb_acc_l = vdotq_s32(sb_acc_l, q6_l0, q8_l[0]);\n sb_acc_l = vdotq_s32(sb_acc_l, q6_l1, q8_l[1]);\n\n int32x4_t sb_acc_h = vdupq_n_s32(0);\n sb_acc_h = vdotq_s32(sb_acc_h, q6_h0, q8_h[0]);\n sb_acc_h = vdotq_s32(sb_acc_h, q6_h1, q8_h[1]);\n\n // Pairwise add to get per-column sums: [col0, col1]\n int32x2_t sum_l = vpadd_s32(vget_low_s32(sb_acc_l), vget_high_s32(sb_acc_l));\n int32x2_t sum_h = vpadd_s32(vget_low_s32(sb_acc_h), vget_high_s32(sb_acc_h));\n\n const int scale_idx_l = half * 8 + sb;\n const int scale_idx_h = half * 8 + sb + 4;\n\n // Access scales using array indexing (scales are interleaved by column)\n const int32x2_t scale_vec_l = { (int32_t) q6_scales[scale_idx_l * 8 + cp * 2],\n (int32_t) q6_scales[scale_idx_l * 8 + cp * 2 + 1] };\n const int32x2_t scale_vec_h = { (int32_t) q6_scales[scale_idx_h * 8 + cp * 2],\n (int32_t) q6_scales[scale_idx_h * 8 + cp * 2 + 1] };\n\n // Accumulate scaled results\n acc[cp] = vmla_s32(acc[cp], sum_l, scale_vec_l);\n acc[cp] = vmla_s32(acc[cp], sum_h, scale_vec_h);\n }\n }\n } // for half\n\n // Bias correction\n acc[0] = vsub_s32(acc[0], vget_low_s32(bias_lo));\n acc[1] = vsub_s32(acc[1], vget_high_s32(bias_lo));\n acc[2] = vsub_s32(acc[2], vget_low_s32(bias_hi));\n acc[3] = vsub_s32(acc[3], vget_high_s32(bias_hi));\n\n // Apply superblock scale (no mins for q6_K)\n // acc[cp] has [c0, c1]\n float32x2_t w_01 = vmul_f32(vcvt_f32_s32(acc[0]), vget_low_f32(sb_scale_0));\n float32x2_t w_23 = vmul_f32(vcvt_f32_s32(acc[1]), vget_high_f32(sb_scale_0));\n float32x2_t w_45 = vmul_f32(vcvt_f32_s32(acc[2]), vget_low_f32(sb_scale_1));\n float32x2_t w_67 = vmul_f32(vcvt_f32_s32(acc[3]), vget_high_f32(sb_scale_1));\n\n acc_f32[0] = vaddq_f32(acc_f32[0], vcombine_f32(w_01, w_23));\n acc_f32[1] = vaddq_f32(acc_f32[1], vcombine_f32(w_45, w_67));\n } // for b\n\n int base = x * ncols_interleaved;\n vst1q_f32(s + base, acc_f32[0]);\n vst1q_f32(s + base + 4, acc_f32[1]);\n } // for x\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q6_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q8_0_4x4_q8_0(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx;\n\n for (int c = 0; c < nc; c += ncols_interleaved) {\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n float32x4_t acc = vdupq_n_f32(0);\n for (int b = 0; b < nb; b++) {\n int8x16x4_t b_low = vld1q_s8_x4((const int8_t *) b_ptr->qs);\n int8x16x4_t b_high = vld1q_s8_x4((const int8_t *) b_ptr->qs + 64);\n float16x4_t bd = vld1_f16((const __fp16 *) b_ptr->d);\n\n int8x16x2_t a = vld1q_s8_x2(a_ptr->qs);\n float16x4_t ad = vld1_dup_f16((const __fp16 *) &a_ptr->d);\n\n int32x4_t ret = vdupq_n_s32(0);\n\n ret = vdotq_laneq_s32(ret, b_low.val[0], a.val[0], 0);\n ret = vdotq_laneq_s32(ret, b_low.val[1], a.val[0], 1);\n ret = vdotq_laneq_s32(ret, b_low.val[2], a.val[0], 2);\n ret = vdotq_laneq_s32(ret, b_low.val[3], a.val[0], 3);\n\n ret = vdotq_laneq_s32(ret, b_high.val[0], a.val[1], 0);\n ret = vdotq_laneq_s32(ret, b_high.val[1], a.val[1], 1);\n ret = vdotq_laneq_s32(ret, b_high.val[2], a.val[1], 2);\n ret = vdotq_laneq_s32(ret, b_high.val[3], a.val[1], 3);\n\n acc = vfmaq_f32(acc, vcvtq_f32_s32(ret), vmulq_f32(vcvt_f32_f16(ad), vcvt_f32_f16(bd)));\n a_ptr++;\n b_ptr++;\n }\n vst1q_f32(s, acc);\n s += ncols_interleaved;\n }\n return;\n\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q8_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q8_0_4x8_q8_0(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx;\n\n for (int c = 0; c < nc; c += ncols_interleaved) {\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n float32x4_t acc = vdupq_n_f32(0);\n\n for (int b = 0; b < nb; b++) {\n int8x16x4_t b_low = vld1q_s8_x4((const int8_t *) b_ptr->qs);\n int8x16x4_t b_high = vld1q_s8_x4((const int8_t *) b_ptr->qs + 64);\n float16x4_t bd = vld1_f16((const __fp16 *) b_ptr->d);\n\n int8x8x4_t a_chunks = vld1_s8_x4(a_ptr->qs);\n int8x16_t a0 = vcombine_s8(a_chunks.val[0], a_chunks.val[0]);\n int8x16_t a1 = vcombine_s8(a_chunks.val[1], a_chunks.val[1]);\n int8x16_t a2 = vcombine_s8(a_chunks.val[2], a_chunks.val[2]);\n int8x16_t a3 = vcombine_s8(a_chunks.val[3], a_chunks.val[3]);\n float16x4_t ad = vld1_dup_f16((const __fp16 *) &a_ptr->d);\n\n int32x4_t ret0 = vdupq_n_s32(0);\n int32x4_t ret1 = vdupq_n_s32(0);\n\n // 0..7\n ret0 = vdotq_s32(ret0, b_low.val[0], a0);\n ret1 = vdotq_s32(ret1, b_low.val[1], a0);\n // 8..15\n ret0 = vdotq_s32(ret0, b_low.val[2], a1);\n ret1 = vdotq_s32(ret1, b_low.val[3], a1);\n // 16..23\n ret0 = vdotq_s32(ret0, b_high.val[0], a2);\n ret1 = vdotq_s32(ret1, b_high.val[1], a2);\n // 24..31\n ret0 = vdotq_s32(ret0, b_high.val[2], a3);\n ret1 = vdotq_s32(ret1, b_high.val[3], a3);\n\n int32x4_t ret = vpaddq_s32(ret0, ret1);\n\n acc = vfmaq_f32(acc, vcvtq_f32_s32(ret), vmulq_f32(vcvt_f32_f16(ad), vcvt_f32_f16(bd)));\n a_ptr++;\n b_ptr++;\n }\n vst1q_f32(s, acc);\n s += ncols_interleaved;\n }\n return;\n\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemv_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n const void * b_ptr = vx;\n const void * a_ptr = vy;\n float * res_ptr = s;\n size_t res_stride = bs * sizeof(float);\n\n __asm__ __volatile__(\n \"mov x10, %x[nr]\\n\"\n \"mov x9, #0x88\\n\"\n \"cmp x10, #0x10\\n\"\n \"mul x9, %x[nb], x9\\n\"\n \"blt 4f\\n\"\n \"1:\" // Row loop\n \"add x28, %x[b_ptr], #0x8\\n\"\n \"mov x27, %x[nc]\\n\"\n \"add x26, %x[res_ptr], %x[res_stride], LSL #4\\n\"\n \"2:\" // Column loop\n \"add x25, %x[a_ptr], #0x8\\n\"\n \"movi v15.16b, #0x0\\n\"\n \"movi v19.16b, #0x0\\n\"\n \"mov x24, %x[nb]\\n\"\n \"add x23, x25, x9\\n\"\n \"movi v18.16b, #0x0\\n\"\n \"movi v14.16b, #0x0\\n\"\n \"add x22, x23, x9\\n\"\n \"movi v11.16b, #0x0\\n\"\n \"movi v13.16b, #0x0\\n\"\n \"add x21, x22, x9\\n\"\n \"movi v23.16b, #0x0\\n\"\n \"movi v16.16b, #0x0\\n\"\n \"movi v25.16b, #0x0\\n\"\n \"movi v7.16b, #0x0\\n\"\n \"movi v0.16b, #0x0\\n\"\n \"movi v4.16b, #0x0\\n\"\n \"movi v5.16b, #0x0\\n\"\n \"movi v21.16b, #0x0\\n\"\n \"movi v8.16b, #0x0\\n\"\n \"movi v1.16b, #0x0\\n\"\n \"3:\" // Block loop\n \"ldr q3, [x28, #0x0]\\n\"\n \"ldr q31, [x25, #0x0]\\n\"\n \"movi v28.16b, #0x4\\n\"\n \"movi v10.4s, #0x0\\n\"\n \"ldr q22, [x28, #0x10]\\n\"\n \"ldr q6, [x25, #0x10]\\n\"\n \"movi v29.4s, #0x0\\n\"\n \"movi v9.4s, #0x0\\n\"\n \"ldr q27, [x28, #0x20]\\n\"\n \"ldr q30, [x28, #0x30]\\n\"\n \"movi v20.4s, #0x0\\n\"\n \"movi v24.16b, #0xf0\\n\"\n \"ldr d2, [x25, #-0x8]\\n\"\n \"ldr d26, [x23, #-0x8]\\n\"\n \"sshl v12.16b, v3.16b, v28.16b\\n\"\n \"sub x20, x28, #0x8\\n\"\n \"ldr d17, [x20, #0x0]\\n\"\n \"and v3.16b, v3.16b, v24.16b\\n\"\n \"subs x24, x24, #0x1\\n\"\n \"add x28, x28, #0x48\\n\"\n \".inst 0x4f9fe18a // sdot v10.4s, v12.16b, v31.4b[0]\\n\"\n \".inst 0x4fbfe19d // sdot v29.4s, v12.16b, v31.4b[1]\\n\"\n \".inst 0x4f9fe989 // sdot v9.4s, v12.16b, v31.4b[2]\\n\"\n \".inst 0x4fbfe994 // sdot v20.4s, v12.16b, v31.4b[3]\\n\"\n \"sshl v31.16b, v22.16b, v28.16b\\n\"\n \"and v22.16b, v22.16b, v24.16b\\n\"\n \"fcvtl v17.4s, v17.4h\\n\"\n \"fcvtl v2.4s, v2.4h\\n\"\n \"fcvtl v26.4s, v26.4h\\n\"\n \".inst 0x4f86e3ea // sdot v10.4s, v31.16b, v6.4b[0]\\n\"\n \".inst 0x4fa6e3fd // sdot v29.4s, v31.16b, v6.4b[1]\\n\"\n \".inst 0x4f86ebe9 // sdot v9.4s, v31.16b, v6.4b[2]\\n\"\n \".inst 0x4fa6ebf4 // sdot v20.4s, v31.16b, v6.4b[3]\\n\"\n \"sshl v6.16b, v27.16b, v28.16b\\n\"\n \"sshl v28.16b, v30.16b, v28.16b\\n\"\n \"and v27.16b, v27.16b, v24.16b\\n\"\n \"and v30.16b, v30.16b, v24.16b\\n\"\n \"ldr q24, [x25, #0x20]\\n\"\n \".inst 0x4f98e0ca // sdot v10.4s, v6.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e0dd // sdot v29.4s, v6.16b, v24.4b[1]\\n\"\n \".inst 0x4f98e8c9 // sdot v9.4s, v6.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e8d4 // sdot v20.4s, v6.16b, v24.4b[3]\\n\"\n \"ldr q24, [x25, #0x30]\\n\"\n \".inst 0x4f98e38a // sdot v10.4s, v28.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e39d // sdot v29.4s, v28.16b, v24.4b[1]\\n\"\n \".inst 0x4f98eb89 // sdot v9.4s, v28.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8eb94 // sdot v20.4s, v28.16b, v24.4b[3]\\n\"\n \"ldr q24, [x25, #0x40]\\n\"\n \".inst 0x4f98e06a // sdot v10.4s, v3.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e07d // sdot v29.4s, v3.16b, v24.4b[1]\\n\"\n \".inst 0x4f98e869 // sdot v9.4s, v3.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e874 // sdot v20.4s, v3.16b, v24.4b[3]\\n\"\n \"ldr q24, [x25, #0x50]\\n\"\n \".inst 0x4f98e2ca // sdot v10.4s, v22.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e2dd // sdot v29.4s, v22.16b, v24.4b[1]\\n\"\n \".inst 0x4f98eac9 // sdot v9.4s, v22.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8ead4 // sdot v20.4s, v22.16b, v24.4b[3]\\n\"\n \"ldr q24, [x25, #0x60]\\n\"\n \".inst 0x4f98e36a // sdot v10.4s, v27.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e37d // sdot v29.4s, v27.16b, v24.4b[1]\\n\"\n \".inst 0x4f98eb69 // sdot v9.4s, v27.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8eb74 // sdot v20.4s, v27.16b, v24.4b[3]\\n\"\n \"ldr q24, [x25, #0x70]\\n\"\n \"add x25, x25, #0x88\\n\"\n \".inst 0x4f98e3ca // sdot v10.4s, v30.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e3dd // sdot v29.4s, v30.16b, v24.4b[1]\\n\"\n \".inst 0x4f98ebc9 // sdot v9.4s, v30.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8ebd4 // sdot v20.4s, v30.16b, v24.4b[3]\\n\"\n \"fmul v24.4s, v17.4s, v2.s[0]\\n\"\n \"scvtf v10.4s, v10.4s, #0x4\\n\"\n \"scvtf v29.4s, v29.4s, #0x4\\n\"\n \"scvtf v9.4s, v9.4s, #0x4\\n\"\n \"scvtf v20.4s, v20.4s, #0x4\\n\"\n \"fmla v15.4s, v10.4s, v24.4s\\n\"\n \"ldr q24, [x23, #0x0]\\n\"\n \"fmul v10.4s, v17.4s, v2.s[1]\\n\"\n \"fmla v19.4s, v29.4s, v10.4s\\n\"\n \"ldr q10, [x23, #0x10]\\n\"\n \"fmul v29.4s, v17.4s, v2.s[2]\\n\"\n \"fmul v2.4s, v17.4s, v2.s[3]\\n\"\n \"fmla v18.4s, v9.4s, v29.4s\\n\"\n \"movi v9.4s, #0x0\\n\"\n \"movi v29.4s, #0x0\\n\"\n \".inst 0x4f98e189 // sdot v9.4s, v12.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e19d // sdot v29.4s, v12.16b, v24.4b[1]\\n\"\n \"fmla v14.4s, v20.4s, v2.4s\\n\"\n \"movi v20.4s, #0x0\\n\"\n \"movi v2.4s, #0x0\\n\"\n \".inst 0x4f98e994 // sdot v20.4s, v12.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e982 // sdot v2.4s, v12.16b, v24.4b[3]\\n\"\n \"ldr q24, [x23, #0x20]\\n\"\n \".inst 0x4f8ae3e9 // sdot v9.4s, v31.16b, v10.4b[0]\\n\"\n \".inst 0x4faae3fd // sdot v29.4s, v31.16b, v10.4b[1]\\n\"\n \".inst 0x4f8aebf4 // sdot v20.4s, v31.16b, v10.4b[2]\\n\"\n \".inst 0x4faaebe2 // sdot v2.4s, v31.16b, v10.4b[3]\\n\"\n \"ldr q10, [x23, #0x30]\\n\"\n \".inst 0x4f98e0c9 // sdot v9.4s, v6.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e0dd // sdot v29.4s, v6.16b, v24.4b[1]\\n\"\n \".inst 0x4f98e8d4 // sdot v20.4s, v6.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e8c2 // sdot v2.4s, v6.16b, v24.4b[3]\\n\"\n \"ldr q24, [x23, #0x40]\\n\"\n \".inst 0x4f8ae389 // sdot v9.4s, v28.16b, v10.4b[0]\\n\"\n \".inst 0x4faae39d // sdot v29.4s, v28.16b, v10.4b[1]\\n\"\n \".inst 0x4f8aeb94 // sdot v20.4s, v28.16b, v10.4b[2]\\n\"\n \".inst 0x4faaeb82 // sdot v2.4s, v28.16b, v10.4b[3]\\n\"\n \"ldr q10, [x23, #0x50]\\n\"\n \".inst 0x4f98e069 // sdot v9.4s, v3.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e07d // sdot v29.4s, v3.16b, v24.4b[1]\\n\"\n \".inst 0x4f98e874 // sdot v20.4s, v3.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e862 // sdot v2.4s, v3.16b, v24.4b[3]\\n\"\n \"ldr q24, [x23, #0x60]\\n\"\n \".inst 0x4f8ae2c9 // sdot v9.4s, v22.16b, v10.4b[0]\\n\"\n \".inst 0x4faae2dd // sdot v29.4s, v22.16b, v10.4b[1]\\n\"\n \".inst 0x4f8aead4 // sdot v20.4s, v22.16b, v10.4b[2]\\n\"\n \".inst 0x4faaeac2 // sdot v2.4s, v22.16b, v10.4b[3]\\n\"\n \"ldr q10, [x23, #0x70]\\n\"\n \"add x23, x23, #0x88\\n\"\n \".inst 0x4f98e369 // sdot v9.4s, v27.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e37d // sdot v29.4s, v27.16b, v24.4b[1]\\n\"\n \".inst 0x4f98eb74 // sdot v20.4s, v27.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8eb62 // sdot v2.4s, v27.16b, v24.4b[3]\\n\"\n \"ldr q24, [x22, #0x0]\\n\"\n \".inst 0x4f8ae3c9 // sdot v9.4s, v30.16b, v10.4b[0]\\n\"\n \".inst 0x4faae3dd // sdot v29.4s, v30.16b, v10.4b[1]\\n\"\n \".inst 0x4f8aebd4 // sdot v20.4s, v30.16b, v10.4b[2]\\n\"\n \".inst 0x4faaebc2 // sdot v2.4s, v30.16b, v10.4b[3]\\n\"\n \"fmul v10.4s, v17.4s, v26.s[0]\\n\"\n \"scvtf v9.4s, v9.4s, #0x4\\n\"\n \"scvtf v29.4s, v29.4s, #0x4\\n\"\n \"scvtf v20.4s, v20.4s, #0x4\\n\"\n \"scvtf v2.4s, v2.4s, #0x4\\n\"\n \"fmla v11.4s, v9.4s, v10.4s\\n\"\n \"ldr q9, [x22, #0x10]\\n\"\n \"fmul v10.4s, v17.4s, v26.s[1]\\n\"\n \"fmla v13.4s, v29.4s, v10.4s\\n\"\n \"ldr d29, [x22, #-0x8]\\n\"\n \"fmul v10.4s, v17.4s, v26.s[2]\\n\"\n \"fmul v26.4s, v17.4s, v26.s[3]\\n\"\n \"fcvtl v29.4s, v29.4h\\n\"\n \"fmla v23.4s, v20.4s, v10.4s\\n\"\n \"movi v20.4s, #0x0\\n\"\n \"movi v10.4s, #0x0\\n\"\n \"fmla v16.4s, v2.4s, v26.4s\\n\"\n \"movi v26.4s, #0x0\\n\"\n \"movi v2.4s, #0x0\\n\"\n \".inst 0x4f98e194 // sdot v20.4s, v12.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e18a // sdot v10.4s, v12.16b, v24.4b[1]\\n\"\n \".inst 0x4f98e99a // sdot v26.4s, v12.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e982 // sdot v2.4s, v12.16b, v24.4b[3]\\n\"\n \"ldr q24, [x22, #0x20]\\n\"\n \".inst 0x4f89e3f4 // sdot v20.4s, v31.16b, v9.4b[0]\\n\"\n \".inst 0x4fa9e3ea // sdot v10.4s, v31.16b, v9.4b[1]\\n\"\n \".inst 0x4f89ebfa // sdot v26.4s, v31.16b, v9.4b[2]\\n\"\n \".inst 0x4fa9ebe2 // sdot v2.4s, v31.16b, v9.4b[3]\\n\"\n \"ldr q9, [x22, #0x30]\\n\"\n \".inst 0x4f98e0d4 // sdot v20.4s, v6.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e0ca // sdot v10.4s, v6.16b, v24.4b[1]\\n\"\n \".inst 0x4f98e8da // sdot v26.4s, v6.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e8c2 // sdot v2.4s, v6.16b, v24.4b[3]\\n\"\n \"ldr q24, [x22, #0x40]\\n\"\n \".inst 0x4f89e394 // sdot v20.4s, v28.16b, v9.4b[0]\\n\"\n \".inst 0x4fa9e38a // sdot v10.4s, v28.16b, v9.4b[1]\\n\"\n \".inst 0x4f89eb9a // sdot v26.4s, v28.16b, v9.4b[2]\\n\"\n \".inst 0x4fa9eb82 // sdot v2.4s, v28.16b, v9.4b[3]\\n\"\n \"ldr q9, [x22, #0x50]\\n\"\n \".inst 0x4f98e074 // sdot v20.4s, v3.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e06a // sdot v10.4s, v3.16b, v24.4b[1]\\n\"\n \".inst 0x4f98e87a // sdot v26.4s, v3.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e862 // sdot v2.4s, v3.16b, v24.4b[3]\\n\"\n \"ldr q24, [x22, #0x60]\\n\"\n \".inst 0x4f89e2d4 // sdot v20.4s, v22.16b, v9.4b[0]\\n\"\n \".inst 0x4fa9e2ca // sdot v10.4s, v22.16b, v9.4b[1]\\n\"\n \".inst 0x4f89eada // sdot v26.4s, v22.16b, v9.4b[2]\\n\"\n \".inst 0x4fa9eac2 // sdot v2.4s, v22.16b, v9.4b[3]\\n\"\n \"ldr q9, [x22, #0x70]\\n\"\n \"add x22, x22, #0x88\\n\"\n \".inst 0x4f98e374 // sdot v20.4s, v27.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e36a // sdot v10.4s, v27.16b, v24.4b[1]\\n\"\n \".inst 0x4f98eb7a // sdot v26.4s, v27.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8eb62 // sdot v2.4s, v27.16b, v24.4b[3]\\n\"\n \"ldr q24, [x21, #0x0]\\n\"\n \".inst 0x4f89e3d4 // sdot v20.4s, v30.16b, v9.4b[0]\\n\"\n \".inst 0x4fa9e3ca // sdot v10.4s, v30.16b, v9.4b[1]\\n\"\n \".inst 0x4f89ebda // sdot v26.4s, v30.16b, v9.4b[2]\\n\"\n \".inst 0x4fa9ebc2 // sdot v2.4s, v30.16b, v9.4b[3]\\n\"\n \"fmul v9.4s, v17.4s, v29.s[0]\\n\"\n \"scvtf v20.4s, v20.4s, #0x4\\n\"\n \"scvtf v10.4s, v10.4s, #0x4\\n\"\n \"scvtf v26.4s, v26.4s, #0x4\\n\"\n \"scvtf v2.4s, v2.4s, #0x4\\n\"\n \"fmla v25.4s, v20.4s, v9.4s\\n\"\n \"ldr q9, [x21, #0x10]\\n\"\n \"fmul v20.4s, v17.4s, v29.s[1]\\n\"\n \"fmla v7.4s, v10.4s, v20.4s\\n\"\n \"ldr d20, [x21, #-0x8]\\n\"\n \"fmul v10.4s, v17.4s, v29.s[2]\\n\"\n \"fmul v29.4s, v17.4s, v29.s[3]\\n\"\n \"fcvtl v20.4s, v20.4h\\n\"\n \"fmla v0.4s, v26.4s, v10.4s\\n\"\n \"movi v26.4s, #0x0\\n\"\n \"movi v10.4s, #0x0\\n\"\n \"fmla v4.4s, v2.4s, v29.4s\\n\"\n \"movi v2.4s, #0x0\\n\"\n \"movi v29.4s, #0x0\\n\"\n \".inst 0x4f98e19a // sdot v26.4s, v12.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e18a // sdot v10.4s, v12.16b, v24.4b[1]\\n\"\n \".inst 0x4f98e982 // sdot v2.4s, v12.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e99d // sdot v29.4s, v12.16b, v24.4b[3]\\n\"\n \"ldr q12, [x21, #0x20]\\n\"\n \"fmul v24.4s, v17.4s, v20.s[0]\\n\"\n \".inst 0x4f89e3fa // sdot v26.4s, v31.16b, v9.4b[0]\\n\"\n \".inst 0x4fa9e3ea // sdot v10.4s, v31.16b, v9.4b[1]\\n\"\n \".inst 0x4f89ebe2 // sdot v2.4s, v31.16b, v9.4b[2]\\n\"\n \".inst 0x4fa9ebfd // sdot v29.4s, v31.16b, v9.4b[3]\\n\"\n \"ldr q9, [x21, #0x30]\\n\"\n \"fmul v31.4s, v17.4s, v20.s[1]\\n\"\n \".inst 0x4f8ce0da // sdot v26.4s, v6.16b, v12.4b[0]\\n\"\n \".inst 0x4face0ca // sdot v10.4s, v6.16b, v12.4b[1]\\n\"\n \".inst 0x4f8ce8c2 // sdot v2.4s, v6.16b, v12.4b[2]\\n\"\n \".inst 0x4face8dd // sdot v29.4s, v6.16b, v12.4b[3]\\n\"\n \"ldr q12, [x21, #0x40]\\n\"\n \"fmul v6.4s, v17.4s, v20.s[2]\\n\"\n \"fmul v20.4s, v17.4s, v20.s[3]\\n\"\n \".inst 0x4f89e39a // sdot v26.4s, v28.16b, v9.4b[0]\\n\"\n \".inst 0x4fa9e38a // sdot v10.4s, v28.16b, v9.4b[1]\\n\"\n \".inst 0x4f89eb82 // sdot v2.4s, v28.16b, v9.4b[2]\\n\"\n \".inst 0x4fa9eb9d // sdot v29.4s, v28.16b, v9.4b[3]\\n\"\n \"ldr q9, [x21, #0x50]\\n\"\n \".inst 0x4f8ce07a // sdot v26.4s, v3.16b, v12.4b[0]\\n\"\n \".inst 0x4face06a // sdot v10.4s, v3.16b, v12.4b[1]\\n\"\n \".inst 0x4f8ce862 // sdot v2.4s, v3.16b, v12.4b[2]\\n\"\n \".inst 0x4face87d // sdot v29.4s, v3.16b, v12.4b[3]\\n\"\n \"ldr q12, [x21, #0x60]\\n\"\n \".inst 0x4f89e2da // sdot v26.4s, v22.16b, v9.4b[0]\\n\"\n \".inst 0x4fa9e2ca // sdot v10.4s, v22.16b, v9.4b[1]\\n\"\n \".inst 0x4f89eac2 // sdot v2.4s, v22.16b, v9.4b[2]\\n\"\n \".inst 0x4fa9eadd // sdot v29.4s, v22.16b, v9.4b[3]\\n\"\n \"ldr q17, [x21, #0x70]\\n\"\n \"add x21, x21, #0x88\\n\"\n \".inst 0x4f8ce37a // sdot v26.4s, v27.16b, v12.4b[0]\\n\"\n \".inst 0x4face36a // sdot v10.4s, v27.16b, v12.4b[1]\\n\"\n \".inst 0x4f8ceb62 // sdot v2.4s, v27.16b, v12.4b[2]\\n\"\n \".inst 0x4faceb7d // sdot v29.4s, v27.16b, v12.4b[3]\\n\"\n \".inst 0x4f91e3da // sdot v26.4s, v30.16b, v17.4b[0]\\n\"\n \".inst 0x4fb1e3ca // sdot v10.4s, v30.16b, v17.4b[1]\\n\"\n \".inst 0x4f91ebc2 // sdot v2.4s, v30.16b, v17.4b[2]\\n\"\n \".inst 0x4fb1ebdd // sdot v29.4s, v30.16b, v17.4b[3]\\n\"\n \"scvtf v26.4s, v26.4s, #0x4\\n\"\n \"scvtf v10.4s, v10.4s, #0x4\\n\"\n \"fmla v5.4s, v26.4s, v24.4s\\n\"\n \"scvtf v2.4s, v2.4s, #0x4\\n\"\n \"scvtf v29.4s, v29.4s, #0x4\\n\"\n \"fmla v21.4s, v10.4s, v31.4s\\n\"\n \"fmla v8.4s, v2.4s, v6.4s\\n\"\n \"fmla v1.4s, v29.4s, v20.4s\\n\"\n \"bgt 3b\\n\"\n \"mov x20, %x[res_ptr]\\n\"\n \"subs x27, x27, #0x4\\n\"\n \"add %x[res_ptr], %x[res_ptr], #0x10\\n\"\n \"str q15, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q19, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q18, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q14, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q11, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q13, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q23, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q16, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q25, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q7, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q0, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q4, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q5, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q21, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q8, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q1, [x20, #0x0]\\n\"\n \"bne 2b\\n\"\n \"mov x20, #0x4\\n\"\n \"sub x10, x10, #0x10\\n\"\n \"cmp x10, #0x10\\n\"\n \"mov %x[res_ptr], x26\\n\"\n \"madd %x[a_ptr], x20, x9, %x[a_ptr]\\n\"\n \"bge 1b\\n\"\n \"4:\" // Row loop skip\n \"cbz x10, 9f\\n\"\n \"5:\" // Row tail: Row loop\n \"add x24, %x[b_ptr], #0x8\\n\"\n \"mov x23, %x[nc]\\n\"\n \"add x22, %x[res_ptr], %x[res_stride], LSL #2\\n\"\n \"6:\" // Row tail: Column loop\n \"movi v15.16b, #0x0\\n\"\n \"movi v19.16b, #0x0\\n\"\n \"add x25, %x[a_ptr], #0x8\\n\"\n \"mov x21, %x[nb]\\n\"\n \"movi v18.16b, #0x0\\n\"\n \"movi v14.16b, #0x0\\n\"\n \"7:\" // Row tail: Block loop\n \"ldr q7, [x24, #0x0]\\n\"\n \"ldr q5, [x25, #0x0]\\n\"\n \"movi v9.16b, #0x4\\n\"\n \"movi v4.4s, #0x0\\n\"\n \"ldr q3, [x24, #0x10]\\n\"\n \"ldr q2, [x25, #0x10]\\n\"\n \"movi v1.4s, #0x0\\n\"\n \"movi v0.4s, #0x0\\n\"\n \"ldr q13, [x24, #0x20]\\n\"\n \"ldr q31, [x25, #0x20]\\n\"\n \"movi v30.4s, #0x0\\n\"\n \"movi v29.16b, #0xf0\\n\"\n \"ldr q28, [x24, #0x30]\\n\"\n \"ldr q27, [x25, #0x30]\\n\"\n \"sshl v20.16b, v7.16b, v9.16b\\n\"\n \"sub x20, x24, #0x8\\n\"\n \"ldr q26, [x25, #0x40]\\n\"\n \"ldr q25, [x25, #0x50]\\n\"\n \"sshl v17.16b, v3.16b, v9.16b\\n\"\n \"and v7.16b, v7.16b, v29.16b\\n\"\n \"ldr q24, [x25, #0x60]\\n\"\n \"ldr q16, [x25, #0x70]\\n\"\n \"sshl v22.16b, v13.16b, v9.16b\\n\"\n \"and v3.16b, v3.16b, v29.16b\\n\"\n \"ldr d21, [x20, #0x0]\\n\"\n \"ldr d12, [x25, #-0x8]\\n\"\n \".inst 0x4f85e284 // sdot v4.4s, v20.16b, v5.4b[0]\\n\"\n \".inst 0x4fa5e281 // sdot v1.4s, v20.16b, v5.4b[1]\\n\"\n \".inst 0x4f85ea80 // sdot v0.4s, v20.16b, v5.4b[2]\\n\"\n \".inst 0x4fa5ea9e // sdot v30.4s, v20.16b, v5.4b[3]\\n\"\n \"sshl v9.16b, v28.16b, v9.16b\\n\"\n \"subs x21, x21, #0x1\\n\"\n \"and v13.16b, v13.16b, v29.16b\\n\"\n \"and v28.16b, v28.16b, v29.16b\\n\"\n \"add x25, x25, #0x88\\n\"\n \"add x24, x24, #0x48\\n\"\n \"fcvtl v21.4s, v21.4h\\n\"\n \"fcvtl v12.4s, v12.4h\\n\"\n \".inst 0x4f82e224 // sdot v4.4s, v17.16b, v2.4b[0]\\n\"\n \".inst 0x4fa2e221 // sdot v1.4s, v17.16b, v2.4b[1]\\n\"\n \".inst 0x4f82ea20 // sdot v0.4s, v17.16b, v2.4b[2]\\n\"\n \".inst 0x4fa2ea3e // sdot v30.4s, v17.16b, v2.4b[3]\\n\"\n \"fmul v11.4s, v21.4s, v12.s[0]\\n\"\n \"fmul v23.4s, v21.4s, v12.s[1]\\n\"\n \"fmul v17.4s, v21.4s, v12.s[2]\\n\"\n \".inst 0x4f9fe2c4 // sdot v4.4s, v22.16b, v31.4b[0]\\n\"\n \"fmul v6.4s, v21.4s, v12.s[3]\\n\"\n \".inst 0x4fbfe2c1 // sdot v1.4s, v22.16b, v31.4b[1]\\n\"\n \".inst 0x4f9feac0 // sdot v0.4s, v22.16b, v31.4b[2]\\n\"\n \".inst 0x4fbfeade // sdot v30.4s, v22.16b, v31.4b[3]\\n\"\n \".inst 0x4f9be124 // sdot v4.4s, v9.16b, v27.4b[0]\\n\"\n \".inst 0x4fbbe121 // sdot v1.4s, v9.16b, v27.4b[1]\\n\"\n \".inst 0x4f9be920 // sdot v0.4s, v9.16b, v27.4b[2]\\n\"\n \".inst 0x4fbbe93e // sdot v30.4s, v9.16b, v27.4b[3]\\n\"\n \".inst 0x4f9ae0e4 // sdot v4.4s, v7.16b, v26.4b[0]\\n\"\n \".inst 0x4fbae0e1 // sdot v1.4s, v7.16b, v26.4b[1]\\n\"\n \".inst 0x4f9ae8e0 // sdot v0.4s, v7.16b, v26.4b[2]\\n\"\n \".inst 0x4fbae8fe // sdot v30.4s, v7.16b, v26.4b[3]\\n\"\n \".inst 0x4f99e064 // sdot v4.4s, v3.16b, v25.4b[0]\\n\"\n \".inst 0x4fb9e061 // sdot v1.4s, v3.16b, v25.4b[1]\\n\"\n \".inst 0x4f99e860 // sdot v0.4s, v3.16b, v25.4b[2]\\n\"\n \".inst 0x4fb9e87e // sdot v30.4s, v3.16b, v25.4b[3]\\n\"\n \".inst 0x4f98e1a4 // sdot v4.4s, v13.16b, v24.4b[0]\\n\"\n \".inst 0x4fb8e1a1 // sdot v1.4s, v13.16b, v24.4b[1]\\n\"\n \".inst 0x4f98e9a0 // sdot v0.4s, v13.16b, v24.4b[2]\\n\"\n \".inst 0x4fb8e9be // sdot v30.4s, v13.16b, v24.4b[3]\\n\"\n \".inst 0x4f90e384 // sdot v4.4s, v28.16b, v16.4b[0]\\n\"\n \".inst 0x4fb0e381 // sdot v1.4s, v28.16b, v16.4b[1]\\n\"\n \".inst 0x4f90eb80 // sdot v0.4s, v28.16b, v16.4b[2]\\n\"\n \".inst 0x4fb0eb9e // sdot v30.4s, v28.16b, v16.4b[3]\\n\"\n \"scvtf v4.4s, v4.4s, #0x4\\n\"\n \"scvtf v1.4s, v1.4s, #0x4\\n\"\n \"scvtf v0.4s, v0.4s, #0x4\\n\"\n \"fmla v15.4s, v4.4s, v11.4s\\n\"\n \"scvtf v30.4s, v30.4s, #0x4\\n\"\n \"fmla v19.4s, v1.4s, v23.4s\\n\"\n \"fmla v18.4s, v0.4s, v17.4s\\n\"\n \"fmla v14.4s, v30.4s, v6.4s\\n\"\n \"bgt 7b\\n\"\n \"mov x20, %x[res_ptr]\\n\"\n \"cmp x10, #0x1\\n\"\n \"str q15, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"ble 8f\\n\"\n \"cmp x10, #0x2\\n\"\n \"str q19, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"ble 8f\\n\"\n \"cmp x10, #0x3\\n\"\n \"str q18, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"ble 8f\\n\"\n \"str q14, [x20, #0x0]\\n\"\n \"8:\" // Row tail: Accumulator store skip\n \"subs x23, x23, #0x4\\n\"\n \"add %x[res_ptr], %x[res_ptr], #0x10\\n\"\n \"bne 6b\\n\"\n \"subs x10, x10, #0x4\\n\"\n \"add %x[a_ptr], %x[a_ptr], x9\\n\"\n \"mov %x[res_ptr], x22\\n\"\n \"bgt 5b\\n\"\n \"9:\" // Row tail: Row loop skip\n : [a_ptr] \"+&r\" (a_ptr), [res_ptr] \"+&r\" (res_ptr)\n : [b_ptr] \"r\" (b_ptr), [nr] \"r\" (nr), [nb] \"r\" (nb), [res_stride] \"r\" (res_stride), [nc] \"r\" (nc)\n : \"cc\", \"memory\", \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\", \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\", \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\", \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\", \"x9\", \"x10\", \"x20\", \"x21\", \"x22\", \"x23\", \"x24\", \"x25\", \"x26\", \"x27\", \"x28\"\n );\n return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n ggml_gemm_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n const void * b_ptr = vx;\n const void * a_ptr = vy;\n float * res_ptr = s;\n size_t res_stride = bs * sizeof(float);\n\n __asm__ __volatile__(\n \"mov x10, %x[nr]\\n\"\n \"mov x9, #0x88\\n\"\n \"cmp x10, #0x10\\n\"\n \"mul x9, %x[nb], x9\\n\"\n \"blt 4f\\n\"\n \"1:\" // Row loop\n \"add x28, %x[b_ptr], #0x8\\n\"\n \"mov x27, %x[nc]\\n\"\n \"add x26, %x[res_ptr], %x[res_stride], LSL #4\\n\"\n \"2:\" // Column loop\n \"add x25, %x[a_ptr], #0x8\\n\"\n \"movi v2.16b, #0x0\\n\"\n \"movi v10.16b, #0x0\\n\"\n \"mov x24, %x[nb]\\n\"\n \"add x23, x25, x9\\n\"\n \"movi v12.16b, #0x0\\n\"\n \"movi v28.16b, #0x0\\n\"\n \"add x22, x23, x9\\n\"\n \"movi v11.16b, #0x0\\n\"\n \"movi v13.16b, #0x0\\n\"\n \"add x21, x22, x9\\n\"\n \"movi v22.16b, #0x0\\n\"\n \"movi v23.16b, #0x0\\n\"\n \"movi v25.16b, #0x0\\n\"\n \"movi v5.16b, #0x0\\n\"\n \"movi v7.16b, #0x0\\n\"\n \"movi v4.16b, #0x0\\n\"\n \"movi v6.16b, #0x0\\n\"\n \"movi v30.16b, #0x0\\n\"\n \"movi v24.16b, #0x0\\n\"\n \"movi v14.16b, #0x0\\n\"\n \"3:\" // Block loop\n \"ldr q21, [x28, #0x0]\\n\"\n \"ldr q16, [x28, #0x10]\\n\"\n \"movi v1.16b, #0x4\\n\"\n \"movi v19.4s, #0x0\\n\"\n \"ldr q27, [x25, #0x0]\\n\"\n \"ldr q15, [x25, #0x10]\\n\"\n \"movi v26.4s, #0x0\\n\"\n \"movi v18.4s, #0x0\\n\"\n \"ldr q29, [x28, #0x20]\\n\"\n \"ldr q3, [x28, #0x30]\\n\"\n \"movi v17.4s, #0x0\\n\"\n \"movi v0.16b, #0xf0\\n\"\n \"ldr d20, [x25, #-0x8]\\n\"\n \"ldr d9, [x23, #-0x8]\\n\"\n \"sshl v8.16b, v21.16b, v1.16b\\n\"\n \"sshl v31.16b, v16.16b, v1.16b\\n\"\n \"and v21.16b, v21.16b, v0.16b\\n\"\n \"and v16.16b, v16.16b, v0.16b\\n\"\n \"sub x20, x28, #0x8\\n\"\n \"subs x24, x24, #0x1\\n\"\n \"add x28, x28, #0x48\\n\"\n \".inst 0x4e88a773 // smmla v19.4s, v27.16b, v8.16b\\n\"\n \".inst 0x4e9fa77a // smmla v26.4s, v27.16b, v31.16b\\n\"\n \"ldr q27, [x25, #0x20]\\n\"\n \".inst 0x4e88a5f2 // smmla v18.4s, v15.16b, v8.16b\\n\"\n \".inst 0x4e9fa5f1 // smmla v17.4s, v15.16b, v31.16b\\n\"\n \"sshl v15.16b, v29.16b, v1.16b\\n\"\n \"sshl v1.16b, v3.16b, v1.16b\\n\"\n \"and v29.16b, v29.16b, v0.16b\\n\"\n \"and v3.16b, v3.16b, v0.16b\\n\"\n \"ldr q0, [x25, #0x30]\\n\"\n \"fcvtl v20.4s, v20.4h\\n\"\n \".inst 0x4e8fa773 // smmla v19.4s, v27.16b, v15.16b\\n\"\n \"fcvtl v9.4s, v9.4h\\n\"\n \".inst 0x4e81a77a // smmla v26.4s, v27.16b, v1.16b\\n\"\n \"ldr q27, [x25, #0x40]\\n\"\n \".inst 0x4e8fa412 // smmla v18.4s, v0.16b, v15.16b\\n\"\n \".inst 0x4e81a411 // smmla v17.4s, v0.16b, v1.16b\\n\"\n \"ldr q0, [x25, #0x50]\\n\"\n \".inst 0x4e95a773 // smmla v19.4s, v27.16b, v21.16b\\n\"\n \".inst 0x4e90a77a // smmla v26.4s, v27.16b, v16.16b\\n\"\n \"ldr q27, [x25, #0x60]\\n\"\n \".inst 0x4e95a412 // smmla v18.4s, v0.16b, v21.16b\\n\"\n \".inst 0x4e90a411 // smmla v17.4s, v0.16b, v16.16b\\n\"\n \"ldr q0, [x25, #0x70]\\n\"\n \"add x25, x25, #0x88\\n\"\n \".inst 0x4e9da773 // smmla v19.4s, v27.16b, v29.16b\\n\"\n \".inst 0x4e83a77a // smmla v26.4s, v27.16b, v3.16b\\n\"\n \"ldr d27, [x20, #0x0]\\n\"\n \".inst 0x4e9da412 // smmla v18.4s, v0.16b, v29.16b\\n\"\n \".inst 0x4e83a411 // smmla v17.4s, v0.16b, v3.16b\\n\"\n \"fcvtl v27.4s, v27.4h\\n\"\n \"uzp1 v0.2d, v19.2d, v26.2d\\n\"\n \"uzp2 v26.2d, v19.2d, v26.2d\\n\"\n \"fmul v19.4s, v27.4s, v20.s[0]\\n\"\n \"scvtf v0.4s, v0.4s, #0x4\\n\"\n \"scvtf v26.4s, v26.4s, #0x4\\n\"\n \"fmla v2.4s, v0.4s, v19.4s\\n\"\n \"ldr q19, [x23, #0x0]\\n\"\n \"uzp1 v0.2d, v18.2d, v17.2d\\n\"\n \"uzp2 v18.2d, v18.2d, v17.2d\\n\"\n \"fmul v17.4s, v27.4s, v20.s[1]\\n\"\n \"scvtf v0.4s, v0.4s, #0x4\\n\"\n \"scvtf v18.4s, v18.4s, #0x4\\n\"\n \"fmla v10.4s, v26.4s, v17.4s\\n\"\n \"ldr q17, [x23, #0x10]\\n\"\n \"fmul v26.4s, v27.4s, v20.s[2]\\n\"\n \"fmul v20.4s, v27.4s, v20.s[3]\\n\"\n \"fmla v12.4s, v0.4s, v26.4s\\n\"\n \"ldr d0, [x22, #-0x8]\\n\"\n \"ldr d26, [x21, #-0x8]\\n\"\n \"fcvtl v0.4s, v0.4h\\n\"\n \"fmla v28.4s, v18.4s, v20.4s\\n\"\n \"movi v20.4s, #0x0\\n\"\n \"movi v18.4s, #0x0\\n\"\n \".inst 0x4e88a674 // smmla v20.4s, v19.16b, v8.16b\\n\"\n \".inst 0x4e9fa672 // smmla v18.4s, v19.16b, v31.16b\\n\"\n \"ldr q19, [x23, #0x20]\\n\"\n \"fcvtl v26.4s, v26.4h\\n\"\n \".inst 0x4e8fa674 // smmla v20.4s, v19.16b, v15.16b\\n\"\n \".inst 0x4e81a672 // smmla v18.4s, v19.16b, v1.16b\\n\"\n \"ldr q19, [x23, #0x40]\\n\"\n \".inst 0x4e95a674 // smmla v20.4s, v19.16b, v21.16b\\n\"\n \".inst 0x4e90a672 // smmla v18.4s, v19.16b, v16.16b\\n\"\n \"ldr q19, [x23, #0x60]\\n\"\n \".inst 0x4e9da674 // smmla v20.4s, v19.16b, v29.16b\\n\"\n \".inst 0x4e83a672 // smmla v18.4s, v19.16b, v3.16b\\n\"\n \"uzp1 v19.2d, v20.2d, v18.2d\\n\"\n \"scvtf v19.4s, v19.4s, #0x4\\n\"\n \"uzp2 v20.2d, v20.2d, v18.2d\\n\"\n \"fmul v18.4s, v27.4s, v9.s[0]\\n\"\n \"scvtf v20.4s, v20.4s, #0x4\\n\"\n \"fmla v11.4s, v19.4s, v18.4s\\n\"\n \"ldr q18, [x22, #0x0]\\n\"\n \"fmul v19.4s, v27.4s, v9.s[1]\\n\"\n \"fmla v13.4s, v20.4s, v19.4s\\n\"\n \"movi v19.4s, #0x0\\n\"\n \"movi v20.4s, #0x0\\n\"\n \".inst 0x4e88a633 // smmla v19.4s, v17.16b, v8.16b\\n\"\n \".inst 0x4e9fa634 // smmla v20.4s, v17.16b, v31.16b\\n\"\n \"ldr q17, [x23, #0x30]\\n\"\n \".inst 0x4e8fa633 // smmla v19.4s, v17.16b, v15.16b\\n\"\n \".inst 0x4e81a634 // smmla v20.4s, v17.16b, v1.16b\\n\"\n \"ldr q17, [x23, #0x50]\\n\"\n \".inst 0x4e95a633 // smmla v19.4s, v17.16b, v21.16b\\n\"\n \".inst 0x4e90a634 // smmla v20.4s, v17.16b, v16.16b\\n\"\n \"ldr q17, [x23, #0x70]\\n\"\n \"add x23, x23, #0x88\\n\"\n \".inst 0x4e9da633 // smmla v19.4s, v17.16b, v29.16b\\n\"\n \".inst 0x4e83a634 // smmla v20.4s, v17.16b, v3.16b\\n\"\n \"uzp1 v17.2d, v19.2d, v20.2d\\n\"\n \"scvtf v17.4s, v17.4s, #0x4\\n\"\n \"uzp2 v20.2d, v19.2d, v20.2d\\n\"\n \"fmul v19.4s, v27.4s, v9.s[2]\\n\"\n \"fmul v9.4s, v27.4s, v9.s[3]\\n\"\n \"scvtf v20.4s, v20.4s, #0x4\\n\"\n \"fmla v22.4s, v17.4s, v19.4s\\n\"\n \"ldr q17, [x22, #0x10]\\n\"\n \"movi v19.4s, #0x0\\n\"\n \".inst 0x4e88a653 // smmla v19.4s, v18.16b, v8.16b\\n\"\n \"fmla v23.4s, v20.4s, v9.4s\\n\"\n \"movi v20.4s, #0x0\\n\"\n \"movi v9.4s, #0x0\\n\"\n \".inst 0x4e9fa654 // smmla v20.4s, v18.16b, v31.16b\\n\"\n \"ldr q18, [x22, #0x20]\\n\"\n \".inst 0x4e88a629 // smmla v9.4s, v17.16b, v8.16b\\n\"\n \".inst 0x4e8fa653 // smmla v19.4s, v18.16b, v15.16b\\n\"\n \".inst 0x4e81a654 // smmla v20.4s, v18.16b, v1.16b\\n\"\n \"ldr q18, [x22, #0x40]\\n\"\n \".inst 0x4e95a653 // smmla v19.4s, v18.16b, v21.16b\\n\"\n \".inst 0x4e90a654 // smmla v20.4s, v18.16b, v16.16b\\n\"\n \"ldr q18, [x22, #0x60]\\n\"\n \".inst 0x4e9da653 // smmla v19.4s, v18.16b, v29.16b\\n\"\n \".inst 0x4e83a654 // smmla v20.4s, v18.16b, v3.16b\\n\"\n \"movi v18.4s, #0x0\\n\"\n \".inst 0x4e9fa632 // smmla v18.4s, v17.16b, v31.16b\\n\"\n \"ldr q17, [x22, #0x30]\\n\"\n \".inst 0x4e8fa629 // smmla v9.4s, v17.16b, v15.16b\\n\"\n \".inst 0x4e81a632 // smmla v18.4s, v17.16b, v1.16b\\n\"\n \"ldr q17, [x22, #0x50]\\n\"\n \".inst 0x4e95a629 // smmla v9.4s, v17.16b, v21.16b\\n\"\n \".inst 0x4e90a632 // smmla v18.4s, v17.16b, v16.16b\\n\"\n \"ldr q17, [x22, #0x70]\\n\"\n \"add x22, x22, #0x88\\n\"\n \".inst 0x4e9da629 // smmla v9.4s, v17.16b, v29.16b\\n\"\n \".inst 0x4e83a632 // smmla v18.4s, v17.16b, v3.16b\\n\"\n \"uzp1 v17.2d, v19.2d, v20.2d\\n\"\n \"uzp2 v20.2d, v19.2d, v20.2d\\n\"\n \"fmul v19.4s, v27.4s, v0.s[0]\\n\"\n \"scvtf v17.4s, v17.4s, #0x4\\n\"\n \"scvtf v20.4s, v20.4s, #0x4\\n\"\n \"fmla v25.4s, v17.4s, v19.4s\\n\"\n \"ldr q19, [x21, #0x0]\\n\"\n \"fmul v17.4s, v27.4s, v0.s[1]\\n\"\n \"fmla v5.4s, v20.4s, v17.4s\\n\"\n \"ldr q17, [x21, #0x10]\\n\"\n \"uzp1 v20.2d, v9.2d, v18.2d\\n\"\n \"uzp2 v9.2d, v9.2d, v18.2d\\n\"\n \"fmul v18.4s, v27.4s, v0.s[2]\\n\"\n \"fmul v0.4s, v27.4s, v0.s[3]\\n\"\n \"scvtf v20.4s, v20.4s, #0x4\\n\"\n \"scvtf v9.4s, v9.4s, #0x4\\n\"\n \"fmla v7.4s, v20.4s, v18.4s\\n\"\n \"movi v20.4s, #0x0\\n\"\n \"movi v18.4s, #0x0\\n\"\n \".inst 0x4e88a674 // smmla v20.4s, v19.16b, v8.16b\\n\"\n \".inst 0x4e9fa672 // smmla v18.4s, v19.16b, v31.16b\\n\"\n \"ldr q19, [x21, #0x20]\\n\"\n \"fmla v4.4s, v9.4s, v0.4s\\n\"\n \"movi v9.4s, #0x0\\n\"\n \"movi v0.4s, #0x0\\n\"\n \".inst 0x4e88a629 // smmla v9.4s, v17.16b, v8.16b\\n\"\n \"fmul v8.4s, v27.4s, v26.s[0]\\n\"\n \".inst 0x4e9fa620 // smmla v0.4s, v17.16b, v31.16b\\n\"\n \"ldr q17, [x21, #0x30]\\n\"\n \".inst 0x4e8fa674 // smmla v20.4s, v19.16b, v15.16b\\n\"\n \"fmul v31.4s, v27.4s, v26.s[1]\\n\"\n \".inst 0x4e81a672 // smmla v18.4s, v19.16b, v1.16b\\n\"\n \"ldr q19, [x21, #0x40]\\n\"\n \".inst 0x4e8fa629 // smmla v9.4s, v17.16b, v15.16b\\n\"\n \"fmul v15.4s, v27.4s, v26.s[2]\\n\"\n \"fmul v27.4s, v27.4s, v26.s[3]\\n\"\n \".inst 0x4e81a620 // smmla v0.4s, v17.16b, v1.16b\\n\"\n \"ldr q1, [x21, #0x50]\\n\"\n \".inst 0x4e95a674 // smmla v20.4s, v19.16b, v21.16b\\n\"\n \".inst 0x4e90a672 // smmla v18.4s, v19.16b, v16.16b\\n\"\n \"ldr q26, [x21, #0x60]\\n\"\n \".inst 0x4e95a429 // smmla v9.4s, v1.16b, v21.16b\\n\"\n \".inst 0x4e90a420 // smmla v0.4s, v1.16b, v16.16b\\n\"\n \"ldr q21, [x21, #0x70]\\n\"\n \"add x21, x21, #0x88\\n\"\n \".inst 0x4e9da754 // smmla v20.4s, v26.16b, v29.16b\\n\"\n \".inst 0x4e83a752 // smmla v18.4s, v26.16b, v3.16b\\n\"\n \".inst 0x4e9da6a9 // smmla v9.4s, v21.16b, v29.16b\\n\"\n \".inst 0x4e83a6a0 // smmla v0.4s, v21.16b, v3.16b\\n\"\n \"uzp1 v29.2d, v20.2d, v18.2d\\n\"\n \"uzp2 v21.2d, v20.2d, v18.2d\\n\"\n \"scvtf v29.4s, v29.4s, #0x4\\n\"\n \"uzp1 v18.2d, v9.2d, v0.2d\\n\"\n \"uzp2 v16.2d, v9.2d, v0.2d\\n\"\n \"scvtf v21.4s, v21.4s, #0x4\\n\"\n \"fmla v6.4s, v29.4s, v8.4s\\n\"\n \"scvtf v18.4s, v18.4s, #0x4\\n\"\n \"scvtf v16.4s, v16.4s, #0x4\\n\"\n \"fmla v30.4s, v21.4s, v31.4s\\n\"\n \"fmla v24.4s, v18.4s, v15.4s\\n\"\n \"fmla v14.4s, v16.4s, v27.4s\\n\"\n \"bgt 3b\\n\"\n \"mov x20, %x[res_ptr]\\n\"\n \"subs x27, x27, #0x4\\n\"\n \"add %x[res_ptr], %x[res_ptr], #0x10\\n\"\n \"str q2, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q10, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q12, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q28, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q11, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q13, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q22, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q23, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q25, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q5, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q7, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q4, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q6, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q30, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q24, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"str q14, [x20, #0x0]\\n\"\n \"bne 2b\\n\"\n \"mov x20, #0x4\\n\"\n \"sub x10, x10, #0x10\\n\"\n \"cmp x10, #0x10\\n\"\n \"mov %x[res_ptr], x26\\n\"\n \"madd %x[a_ptr], x20, x9, %x[a_ptr]\\n\"\n \"bge 1b\\n\"\n \"4:\" // Row loop skip\n \"cbz x10, 9f\\n\"\n \"5:\" // Row tail: Row loop\n \"add x24, %x[b_ptr], #0x8\\n\"\n \"mov x23, %x[nc]\\n\"\n \"add x22, %x[res_ptr], %x[res_stride], LSL #2\\n\"\n \"6:\" // Row tail: Column loop\n \"movi v2.16b, #0x0\\n\"\n \"movi v10.16b, #0x0\\n\"\n \"add x25, %x[a_ptr], #0x8\\n\"\n \"mov x21, %x[nb]\\n\"\n \"movi v12.16b, #0x0\\n\"\n \"movi v28.16b, #0x0\\n\"\n \"7:\" // Row tail: Block loop\n \"ldr q6, [x24, #0x0]\\n\"\n \"ldr q5, [x24, #0x10]\\n\"\n \"movi v17.16b, #0x4\\n\"\n \"movi v8.4s, #0x0\\n\"\n \"ldr q4, [x25, #0x0]\\n\"\n \"ldr q13, [x25, #0x10]\\n\"\n \"movi v27.4s, #0x0\\n\"\n \"movi v0.4s, #0x0\\n\"\n \"ldr q31, [x24, #0x20]\\n\"\n \"ldr q14, [x24, #0x30]\\n\"\n \"movi v29.4s, #0x0\\n\"\n \"movi v22.16b, #0xf0\\n\"\n \"ldr q11, [x25, #0x20]\\n\"\n \"ldr q23, [x25, #0x30]\\n\"\n \"sshl v21.16b, v6.16b, v17.16b\\n\"\n \"sshl v16.16b, v5.16b, v17.16b\\n\"\n \"ldr q20, [x25, #0x40]\\n\"\n \"ldr q26, [x25, #0x50]\\n\"\n \"and v6.16b, v6.16b, v22.16b\\n\"\n \"and v5.16b, v5.16b, v22.16b\\n\"\n \"ldr q25, [x25, #0x60]\\n\"\n \"ldr q3, [x25, #0x70]\\n\"\n \"sshl v19.16b, v31.16b, v17.16b\\n\"\n \"sshl v18.16b, v14.16b, v17.16b\\n\"\n \"ldr d17, [x25, #-0x8]\\n\"\n \".inst 0x4e95a488 // smmla v8.4s, v4.16b, v21.16b\\n\"\n \".inst 0x4e90a49b // smmla v27.4s, v4.16b, v16.16b\\n\"\n \"and v31.16b, v31.16b, v22.16b\\n\"\n \".inst 0x4e95a5a0 // smmla v0.4s, v13.16b, v21.16b\\n\"\n \".inst 0x4e90a5bd // smmla v29.4s, v13.16b, v16.16b\\n\"\n \"and v14.16b, v14.16b, v22.16b\\n\"\n \"sub x20, x24, #0x8\\n\"\n \"ldr d16, [x20, #0x0]\\n\"\n \"subs x21, x21, #0x1\\n\"\n \"add x25, x25, #0x88\\n\"\n \"fcvtl v17.4s, v17.4h\\n\"\n \"add x24, x24, #0x48\\n\"\n \".inst 0x4e93a568 // smmla v8.4s, v11.16b, v19.16b\\n\"\n \".inst 0x4e92a57b // smmla v27.4s, v11.16b, v18.16b\\n\"\n \".inst 0x4e93a6e0 // smmla v0.4s, v23.16b, v19.16b\\n\"\n \".inst 0x4e92a6fd // smmla v29.4s, v23.16b, v18.16b\\n\"\n \"fcvtl v16.4s, v16.4h\\n\"\n \".inst 0x4e86a688 // smmla v8.4s, v20.16b, v6.16b\\n\"\n \".inst 0x4e85a69b // smmla v27.4s, v20.16b, v5.16b\\n\"\n \"fmul v23.4s, v16.4s, v17.s[0]\\n\"\n \"fmul v21.4s, v16.4s, v17.s[1]\\n\"\n \"fmul v1.4s, v16.4s, v17.s[2]\\n\"\n \"fmul v20.4s, v16.4s, v17.s[3]\\n\"\n \".inst 0x4e86a740 // smmla v0.4s, v26.16b, v6.16b\\n\"\n \".inst 0x4e85a75d // smmla v29.4s, v26.16b, v5.16b\\n\"\n \".inst 0x4e9fa728 // smmla v8.4s, v25.16b, v31.16b\\n\"\n \".inst 0x4e8ea73b // smmla v27.4s, v25.16b, v14.16b\\n\"\n \".inst 0x4e9fa460 // smmla v0.4s, v3.16b, v31.16b\\n\"\n \".inst 0x4e8ea47d // smmla v29.4s, v3.16b, v14.16b\\n\"\n \"uzp1 v19.2d, v8.2d, v27.2d\\n\"\n \"uzp2 v18.2d, v8.2d, v27.2d\\n\"\n \"scvtf v19.4s, v19.4s, #0x4\\n\"\n \"uzp1 v17.2d, v0.2d, v29.2d\\n\"\n \"uzp2 v16.2d, v0.2d, v29.2d\\n\"\n \"scvtf v18.4s, v18.4s, #0x4\\n\"\n \"fmla v2.4s, v19.4s, v23.4s\\n\"\n \"scvtf v17.4s, v17.4s, #0x4\\n\"\n \"scvtf v16.4s, v16.4s, #0x4\\n\"\n \"fmla v10.4s, v18.4s, v21.4s\\n\"\n \"fmla v12.4s, v17.4s, v1.4s\\n\"\n \"fmla v28.4s, v16.4s, v20.4s\\n\"\n \"bgt 7b\\n\"\n \"mov x20, %x[res_ptr]\\n\"\n \"cmp x10, #0x1\\n\"\n \"str q2, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"ble 8f\\n\"\n \"cmp x10, #0x2\\n\"\n \"str q10, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"ble 8f\\n\"\n \"cmp x10, #0x3\\n\"\n \"str q12, [x20, #0x0]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"ble 8f\\n\"\n \"str q28, [x20, #0x0]\\n\"\n \"8:\" // Row tail: Accumulator store skip\n \"subs x23, x23, #0x4\\n\"\n \"add %x[res_ptr], %x[res_ptr], #0x10\\n\"\n \"bne 6b\\n\"\n \"subs x10, x10, #0x4\\n\"\n \"add %x[a_ptr], %x[a_ptr], x9\\n\"\n \"mov %x[res_ptr], x22\\n\"\n \"bgt 5b\\n\"\n \"9:\" // Row tail: Row loop skip\n : [a_ptr] \"+&r\" (a_ptr), [res_ptr] \"+&r\" (res_ptr)\n : [b_ptr] \"r\" (b_ptr), [nr] \"r\" (nr), [nb] \"r\" (nb), [res_stride] \"r\" (res_stride), [nc] \"r\" (nc)\n : \"cc\", \"memory\", \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\", \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\", \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\", \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\", \"x9\", \"x10\", \"x20\", \"x21\", \"x22\", \"x23\", \"x24\", \"x25\", \"x26\", \"x27\", \"x28\"\n );\n return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n ggml_gemm_q4_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)\n#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n if (ggml_cpu_get_sve_cnt() == QK8_0) {\n const void * b_ptr = vx;\n const void * a_ptr = vy;\n float * res_ptr = s;\n size_t res_stride = bs * sizeof(float);\n\n __asm__ __volatile__(\n \"mov x20, #0x4\\n\"\n \"mov x13, %x[nr]\\n\"\n \"mov z28.s, #-0x4\\n\"\n \"mov x12, #0x88\\n\"\n \"ptrue p1.b\\n\"\n \"whilelt p0.s, XZR, x20\\n\"\n \"cmp x13, #0x10\\n\"\n \"mul x12, %x[nb], x12\\n\"\n \"blt 4f\\n\"\n \"1:\" // Row loop\n \"add x11, %x[b_ptr], #0x10\\n\"\n \"mov x10, %x[nc]\\n\"\n \"add x9, %x[res_ptr], %x[res_stride], LSL #4\\n\"\n \"2:\" // Column loop\n \"add x28, %x[a_ptr], #0x8\\n\"\n \"mov z24.b, #0x0\\n\"\n \"mov z15.b, #0x0\\n\"\n \"mov x27, %x[nb]\\n\"\n \"add x26, x28, x12\\n\"\n \"mov z12.b, #0x0\\n\"\n \"mov z0.b, #0x0\\n\"\n \"add x25, x26, x12\\n\"\n \"mov z13.b, #0x0\\n\"\n \"mov z1.b, #0x0\\n\"\n \"add x24, x25, x12\\n\"\n \"mov z20.b, #0x0\\n\"\n \"mov z25.b, #0x0\\n\"\n \"mov z11.b, #0x0\\n\"\n \"mov z16.b, #0x0\\n\"\n \"mov z19.b, #0x0\\n\"\n \"mov z26.b, #0x0\\n\"\n \"mov z8.b, #0x0\\n\"\n \"mov z29.b, #0x0\\n\"\n \"mov z27.b, #0x0\\n\"\n \"mov z10.b, #0x0\\n\"\n \"3:\" // Block loop\n \"ld1b { z30.b }, p1/Z, [x11]\\n\"\n \"ld1b { z21.b }, p1/Z, [x11, #1, MUL VL]\\n\"\n \"mov z18.s, #0x0\\n\"\n \"mov z7.s, #0x0\\n\"\n \"ld1rqb { z3.b }, p1/Z, [x28]\\n\"\n \"ld1rqb { z5.b }, p1/Z, [x28, #16]\\n\"\n \"mov z9.s, #0x0\\n\"\n \"mov z22.s, #0x0\\n\"\n \"ld1b { z4.b }, p1/Z, [x11, #2, MUL VL]\\n\"\n \"ld1b { z17.b }, p1/Z, [x11, #3, MUL VL]\\n\"\n \"sub x20, x11, #0x10\\n\"\n \"sub x23, x28, #0x8\\n\"\n \"lsl z31.b, z30.b, #0x4\\n\"\n \"lsl z6.b, z21.b, #0x4\\n\"\n \"ld1h { z23.s }, p1/Z, [x20]\\n\"\n \"sub x22, x26, #0x8\\n\"\n \"and z30.b, z30.b, #0xf0\\n\"\n \"and z21.b, z21.b, #0xf0\\n\"\n \"sub x21, x25, #0x8\\n\"\n \"sub x20, x24, #0x8\\n\"\n \"lsl z14.b, z4.b, #0x4\\n\"\n \"lsl z2.b, z17.b, #0x4\\n\"\n \"subs x27, x27, #0x1\\n\"\n \"add x11, x11, #0x90\\n\"\n \".inst 0x451f9872 // smmla z18.s, z3.b, z31.b\\n\"\n \".inst 0x45069867 // smmla z7.s, z3.b, z6.b\\n\"\n \"ld1rqb { z3.b }, p1/Z, [x28, #32]\\n\"\n \"and z4.b, z4.b, #0xf0\\n\"\n \".inst 0x451f98a9 // smmla z9.s, z5.b, z31.b\\n\"\n \".inst 0x450698b6 // smmla z22.s, z5.b, z6.b\\n\"\n \"ld1rqb { z5.b }, p1/Z, [x28, #48]\\n\"\n \"and z17.b, z17.b, #0xf0\\n\"\n \"fcvt z23.s, p1/m, z23.h\\n\"\n \".inst 0x450e9872 // smmla z18.s, z3.b, z14.b\\n\"\n \".inst 0x45029867 // smmla z7.s, z3.b, z2.b\\n\"\n \"ld1rqb { z3.b }, p1/Z, [x28, #64]\\n\"\n \".inst 0x450e98a9 // smmla z9.s, z5.b, z14.b\\n\"\n \".inst 0x450298b6 // smmla z22.s, z5.b, z2.b\\n\"\n \"ld1rqb { z5.b }, p1/Z, [x28, #80]\\n\"\n \"fscale z23.s, p1/m, z23.s, z28.s\\n\"\n \".inst 0x451e9872 // smmla z18.s, z3.b, z30.b\\n\"\n \".inst 0x45159867 // smmla z7.s, z3.b, z21.b\\n\"\n \"ld1rqb { z3.b }, p1/Z, [x28, #96]\\n\"\n \".inst 0x451e98a9 // smmla z9.s, z5.b, z30.b\\n\"\n \".inst 0x451598b6 // smmla z22.s, z5.b, z21.b\\n\"\n \"ld1rqb { z5.b }, p1/Z, [x28, #112]\\n\"\n \"add x28, x28, #0x88\\n\"\n \".inst 0x45049872 // smmla z18.s, z3.b, z4.b\\n\"\n \".inst 0x45119867 // smmla z7.s, z3.b, z17.b\\n\"\n \"ld1h { z3.s }, p0/Z, [x23]\\n\"\n \".inst 0x450498a9 // smmla z9.s, z5.b, z4.b\\n\"\n \".inst 0x451198b6 // smmla z22.s, z5.b, z17.b\\n\"\n \"fcvt z3.s, p1/m, z3.h\\n\"\n \"uzp1 z5.d, z18.d, z7.d\\n\"\n \"uzp2 z18.d, z18.d, z7.d\\n\"\n \"mov z3.q, z3.q[0]\\n\"\n \"uzp1 z7.d, z9.d, z22.d\\n\"\n \"uzp2 z22.d, z9.d, z22.d\\n\"\n \"fmul z9.s, z23.s, z3.s[0]\\n\"\n \"scvtf z5.s, p1/m, z5.s\\n\"\n \"scvtf z18.s, p1/m, z18.s\\n\"\n \"scvtf z7.s, p1/m, z7.s\\n\"\n \"scvtf z22.s, p1/m, z22.s\\n\"\n \"fmla z24.s, p1/M, z5.s, z9.s\\n\"\n \"ld1rqb { z5.b }, p1/Z, [x26]\\n\"\n \"fmul z9.s, z23.s, z3.s[1]\\n\"\n \"fmla z15.s, p1/M, z18.s, z9.s\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x26, #16]\\n\"\n \"fmul z9.s, z23.s, z3.s[2]\\n\"\n \"fmul z3.s, z23.s, z3.s[3]\\n\"\n \"fmla z12.s, p1/M, z7.s, z9.s\\n\"\n \"mov z9.s, #0x0\\n\"\n \"ld1h { z7.s }, p0/Z, [x22]\\n\"\n \".inst 0x451f98a9 // smmla z9.s, z5.b, z31.b\\n\"\n \"fmla z0.s, p1/M, z22.s, z3.s\\n\"\n \"mov z22.s, #0x0\\n\"\n \"ld1h { z3.s }, p0/Z, [x21]\\n\"\n \".inst 0x450698b6 // smmla z22.s, z5.b, z6.b\\n\"\n \"ld1rqb { z5.b }, p1/Z, [x26, #32]\\n\"\n \"fcvt z7.s, p1/m, z7.h\\n\"\n \"fcvt z3.s, p1/m, z3.h\\n\"\n \".inst 0x450e98a9 // smmla z9.s, z5.b, z14.b\\n\"\n \".inst 0x450298b6 // smmla z22.s, z5.b, z2.b\\n\"\n \"ld1rqb { z5.b }, p1/Z, [x26, #64]\\n\"\n \"mov z7.q, z7.q[0]\\n\"\n \"mov z3.q, z3.q[0]\\n\"\n \".inst 0x451e98a9 // smmla z9.s, z5.b, z30.b\\n\"\n \".inst 0x451598b6 // smmla z22.s, z5.b, z21.b\\n\"\n \"ld1rqb { z5.b }, p1/Z, [x26, #96]\\n\"\n \".inst 0x450498a9 // smmla z9.s, z5.b, z4.b\\n\"\n \".inst 0x451198b6 // smmla z22.s, z5.b, z17.b\\n\"\n \"uzp1 z5.d, z9.d, z22.d\\n\"\n \"scvtf z5.s, p1/m, z5.s\\n\"\n \"uzp2 z22.d, z9.d, z22.d\\n\"\n \"fmul z9.s, z23.s, z7.s[0]\\n\"\n \"scvtf z22.s, p1/m, z22.s\\n\"\n \"fmla z13.s, p1/M, z5.s, z9.s\\n\"\n \"ld1rqb { z9.b }, p1/Z, [x25]\\n\"\n \"fmul z5.s, z23.s, z7.s[1]\\n\"\n \"fmla z1.s, p1/M, z22.s, z5.s\\n\"\n \"mov z5.s, #0x0\\n\"\n \"mov z22.s, #0x0\\n\"\n \".inst 0x451f9a45 // smmla z5.s, z18.b, z31.b\\n\"\n \".inst 0x45069a56 // smmla z22.s, z18.b, z6.b\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x26, #48]\\n\"\n \".inst 0x450e9a45 // smmla z5.s, z18.b, z14.b\\n\"\n \".inst 0x45029a56 // smmla z22.s, z18.b, z2.b\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x26, #80]\\n\"\n \".inst 0x451e9a45 // smmla z5.s, z18.b, z30.b\\n\"\n \".inst 0x45159a56 // smmla z22.s, z18.b, z21.b\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x26, #112]\\n\"\n \"add x26, x26, #0x88\\n\"\n \".inst 0x45049a45 // smmla z5.s, z18.b, z4.b\\n\"\n \".inst 0x45119a56 // smmla z22.s, z18.b, z17.b\\n\"\n \"uzp1 z18.d, z5.d, z22.d\\n\"\n \"scvtf z18.s, p1/m, z18.s\\n\"\n \"uzp2 z22.d, z5.d, z22.d\\n\"\n \"fmul z5.s, z23.s, z7.s[2]\\n\"\n \"fmul z7.s, z23.s, z7.s[3]\\n\"\n \"scvtf z22.s, p1/m, z22.s\\n\"\n \"fmla z20.s, p1/M, z18.s, z5.s\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x25, #16]\\n\"\n \"ld1h { z5.s }, p0/Z, [x20]\\n\"\n \"fcvt z5.s, p1/m, z5.h\\n\"\n \"fmla z25.s, p1/M, z22.s, z7.s\\n\"\n \"mov z22.s, #0x0\\n\"\n \"mov z7.s, #0x0\\n\"\n \".inst 0x451f9936 // smmla z22.s, z9.b, z31.b\\n\"\n \".inst 0x45069927 // smmla z7.s, z9.b, z6.b\\n\"\n \"ld1rqb { z9.b }, p1/Z, [x25, #32]\\n\"\n \"mov z5.q, z5.q[0]\\n\"\n \".inst 0x450e9936 // smmla z22.s, z9.b, z14.b\\n\"\n \".inst 0x45029927 // smmla z7.s, z9.b, z2.b\\n\"\n \"ld1rqb { z9.b }, p1/Z, [x25, #64]\\n\"\n \".inst 0x451e9936 // smmla z22.s, z9.b, z30.b\\n\"\n \".inst 0x45159927 // smmla z7.s, z9.b, z21.b\\n\"\n \"ld1rqb { z9.b }, p1/Z, [x25, #96]\\n\"\n \".inst 0x45049936 // smmla z22.s, z9.b, z4.b\\n\"\n \".inst 0x45119927 // smmla z7.s, z9.b, z17.b\\n\"\n \"uzp1 z9.d, z22.d, z7.d\\n\"\n \"scvtf z9.s, p1/m, z9.s\\n\"\n \"uzp2 z22.d, z22.d, z7.d\\n\"\n \"fmul z7.s, z23.s, z3.s[0]\\n\"\n \"scvtf z22.s, p1/m, z22.s\\n\"\n \"fmla z11.s, p1/M, z9.s, z7.s\\n\"\n \"ld1rqb { z9.b }, p1/Z, [x24]\\n\"\n \"fmul z7.s, z23.s, z3.s[1]\\n\"\n \"fmla z16.s, p1/M, z22.s, z7.s\\n\"\n \"mov z22.s, #0x0\\n\"\n \"mov z7.s, #0x0\\n\"\n \".inst 0x451f9a56 // smmla z22.s, z18.b, z31.b\\n\"\n \".inst 0x45069a47 // smmla z7.s, z18.b, z6.b\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x25, #48]\\n\"\n \".inst 0x450e9a56 // smmla z22.s, z18.b, z14.b\\n\"\n \".inst 0x45029a47 // smmla z7.s, z18.b, z2.b\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x25, #80]\\n\"\n \".inst 0x451e9a56 // smmla z22.s, z18.b, z30.b\\n\"\n \".inst 0x45159a47 // smmla z7.s, z18.b, z21.b\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x25, #112]\\n\"\n \"add x25, x25, #0x88\\n\"\n \".inst 0x45049a56 // smmla z22.s, z18.b, z4.b\\n\"\n \".inst 0x45119a47 // smmla z7.s, z18.b, z17.b\\n\"\n \"uzp1 z18.d, z22.d, z7.d\\n\"\n \"scvtf z18.s, p1/m, z18.s\\n\"\n \"uzp2 z7.d, z22.d, z7.d\\n\"\n \"fmul z22.s, z23.s, z3.s[2]\\n\"\n \"fmul z3.s, z23.s, z3.s[3]\\n\"\n \"scvtf z7.s, p1/m, z7.s\\n\"\n \"fmla z19.s, p1/M, z18.s, z22.s\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x24, #16]\\n\"\n \"fmul z22.s, z23.s, z5.s[0]\\n\"\n \"fmla z26.s, p1/M, z7.s, z3.s\\n\"\n \"mov z3.s, #0x0\\n\"\n \"mov z7.s, #0x0\\n\"\n \".inst 0x451f9923 // smmla z3.s, z9.b, z31.b\\n\"\n \".inst 0x45069927 // smmla z7.s, z9.b, z6.b\\n\"\n \"ld1rqb { z9.b }, p1/Z, [x24, #32]\\n\"\n \".inst 0x450e9923 // smmla z3.s, z9.b, z14.b\\n\"\n \".inst 0x45029927 // smmla z7.s, z9.b, z2.b\\n\"\n \"mov z9.s, #0x0\\n\"\n \".inst 0x451f9a49 // smmla z9.s, z18.b, z31.b\\n\"\n \"mov z31.s, #0x0\\n\"\n \".inst 0x45069a5f // smmla z31.s, z18.b, z6.b\\n\"\n \"ld1rqb { z6.b }, p1/Z, [x24, #48]\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x24, #64]\\n\"\n \".inst 0x450e98c9 // smmla z9.s, z6.b, z14.b\\n\"\n \"fmul z14.s, z23.s, z5.s[1]\\n\"\n \".inst 0x450298df // smmla z31.s, z6.b, z2.b\\n\"\n \"ld1rqb { z6.b }, p1/Z, [x24, #80]\\n\"\n \"fmul z2.s, z23.s, z5.s[2]\\n\"\n \"fmul z23.s, z23.s, z5.s[3]\\n\"\n \".inst 0x451e9a43 // smmla z3.s, z18.b, z30.b\\n\"\n \".inst 0x45159a47 // smmla z7.s, z18.b, z21.b\\n\"\n \"ld1rqb { z5.b }, p1/Z, [x24, #96]\\n\"\n \".inst 0x451e98c9 // smmla z9.s, z6.b, z30.b\\n\"\n \".inst 0x451598df // smmla z31.s, z6.b, z21.b\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x24, #112]\\n\"\n \"add x24, x24, #0x88\\n\"\n \".inst 0x450498a3 // smmla z3.s, z5.b, z4.b\\n\"\n \".inst 0x451198a7 // smmla z7.s, z5.b, z17.b\\n\"\n \".inst 0x45049a49 // smmla z9.s, z18.b, z4.b\\n\"\n \".inst 0x45119a5f // smmla z31.s, z18.b, z17.b\\n\"\n \"uzp1 z18.d, z3.d, z7.d\\n\"\n \"uzp2 z5.d, z3.d, z7.d\\n\"\n \"scvtf z18.s, p1/m, z18.s\\n\"\n \"uzp1 z6.d, z9.d, z31.d\\n\"\n \"uzp2 z9.d, z9.d, z31.d\\n\"\n \"scvtf z5.s, p1/m, z5.s\\n\"\n \"fmla z8.s, p1/M, z18.s, z22.s\\n\"\n \"scvtf z6.s, p1/m, z6.s\\n\"\n \"scvtf z9.s, p1/m, z9.s\\n\"\n \"fmla z29.s, p1/M, z5.s, z14.s\\n\"\n \"fmla z27.s, p1/M, z6.s, z2.s\\n\"\n \"fmla z10.s, p1/M, z9.s, z23.s\\n\"\n \"bgt 3b\\n\"\n \"mov x20, %x[res_ptr]\\n\"\n \"subs x10, x10, #0x8\\n\"\n \"add %x[res_ptr], %x[res_ptr], #0x20\\n\"\n \"st1w { z24.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z15.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z12.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z0.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z13.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z1.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z20.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z25.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z11.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z16.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z19.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z26.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z8.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z29.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z27.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"st1w { z10.s }, p1, [x20]\\n\"\n \"bne 2b\\n\"\n \"mov x20, #0x4\\n\"\n \"sub x13, x13, #0x10\\n\"\n \"cmp x13, #0x10\\n\"\n \"mov %x[res_ptr], x9\\n\"\n \"madd %x[a_ptr], x20, x12, %x[a_ptr]\\n\"\n \"bge 1b\\n\"\n \"4:\" // Row loop skip\n \"cbz x13, 9f\\n\"\n \"5:\" // Row tail: Row loop\n \"add x25, %x[b_ptr], #0x10\\n\"\n \"mov x24, %x[nc]\\n\"\n \"add x23, %x[res_ptr], %x[res_stride], LSL #2\\n\"\n \"6:\" // Row tail: Column loop\n \"mov z24.b, #0x0\\n\"\n \"mov z15.b, #0x0\\n\"\n \"add x28, %x[a_ptr], #0x8\\n\"\n \"mov x22, %x[nb]\\n\"\n \"mov z12.b, #0x0\\n\"\n \"mov z0.b, #0x0\\n\"\n \"7:\" // Row tail: Block loop\n \"ld1b { z3.b }, p1/Z, [x25]\\n\"\n \"ld1b { z6.b }, p1/Z, [x25, #1, MUL VL]\\n\"\n \"mov z2.s, #0x0\\n\"\n \"mov z25.s, #0x0\\n\"\n \"ld1rqb { z26.b }, p1/Z, [x28]\\n\"\n \"ld1rqb { z21.b }, p1/Z, [x28, #16]\\n\"\n \"mov z27.s, #0x0\\n\"\n \"mov z19.s, #0x0\\n\"\n \"ld1b { z29.b }, p1/Z, [x25, #2, MUL VL]\\n\"\n \"ld1b { z16.b }, p1/Z, [x25, #3, MUL VL]\\n\"\n \"sub x21, x25, #0x10\\n\"\n \"sub x20, x28, #0x8\\n\"\n \"lsl z20.b, z3.b, #0x4\\n\"\n \"lsl z4.b, z6.b, #0x4\\n\"\n \"ld1rqb { z10.b }, p1/Z, [x28, #32]\\n\"\n \"ld1rqb { z23.b }, p1/Z, [x28, #48]\\n\"\n \"and z3.b, z3.b, #0xf0\\n\"\n \"and z6.b, z6.b, #0xf0\\n\"\n \"ld1rqb { z11.b }, p1/Z, [x28, #64]\\n\"\n \"ld1rqb { z7.b }, p1/Z, [x28, #80]\\n\"\n \"lsl z8.b, z29.b, #0x4\\n\"\n \"lsl z14.b, z16.b, #0x4\\n\"\n \"ld1rqb { z18.b }, p1/Z, [x28, #96]\\n\"\n \"ld1rqb { z30.b }, p1/Z, [x28, #112]\\n\"\n \".inst 0x45149b42 // smmla z2.s, z26.b, z20.b\\n\"\n \".inst 0x45049b59 // smmla z25.s, z26.b, z4.b\\n\"\n \"and z29.b, z29.b, #0xf0\\n\"\n \"ld1h { z17.s }, p1/Z, [x21]\\n\"\n \".inst 0x45149abb // smmla z27.s, z21.b, z20.b\\n\"\n \".inst 0x45049ab3 // smmla z19.s, z21.b, z4.b\\n\"\n \"and z16.b, z16.b, #0xf0\\n\"\n \"ld1h { z4.s }, p0/Z, [x20]\\n\"\n \"subs x22, x22, #0x1\\n\"\n \"add x28, x28, #0x88\\n\"\n \"fcvt z17.s, p1/m, z17.h\\n\"\n \"add x25, x25, #0x90\\n\"\n \".inst 0x45089942 // smmla z2.s, z10.b, z8.b\\n\"\n \".inst 0x450e9959 // smmla z25.s, z10.b, z14.b\\n\"\n \"fcvt z4.s, p1/m, z4.h\\n\"\n \".inst 0x45089afb // smmla z27.s, z23.b, z8.b\\n\"\n \".inst 0x450e9af3 // smmla z19.s, z23.b, z14.b\\n\"\n \"fscale z17.s, p1/m, z17.s, z28.s\\n\"\n \"mov z4.q, z4.q[0]\\n\"\n \".inst 0x45039962 // smmla z2.s, z11.b, z3.b\\n\"\n \".inst 0x45069979 // smmla z25.s, z11.b, z6.b\\n\"\n \"fmul z23.s, z17.s, z4.s[0]\\n\"\n \"fmul z9.s, z17.s, z4.s[1]\\n\"\n \"fmul z21.s, z17.s, z4.s[2]\\n\"\n \"fmul z4.s, z17.s, z4.s[3]\\n\"\n \".inst 0x450398fb // smmla z27.s, z7.b, z3.b\\n\"\n \".inst 0x450698f3 // smmla z19.s, z7.b, z6.b\\n\"\n \".inst 0x451d9a42 // smmla z2.s, z18.b, z29.b\\n\"\n \".inst 0x45109a59 // smmla z25.s, z18.b, z16.b\\n\"\n \".inst 0x451d9bdb // smmla z27.s, z30.b, z29.b\\n\"\n \".inst 0x45109bd3 // smmla z19.s, z30.b, z16.b\\n\"\n \"uzp1 z31.d, z2.d, z25.d\\n\"\n \"uzp2 z13.d, z2.d, z25.d\\n\"\n \"scvtf z31.s, p1/m, z31.s\\n\"\n \"uzp1 z17.d, z27.d, z19.d\\n\"\n \"uzp2 z18.d, z27.d, z19.d\\n\"\n \"scvtf z13.s, p1/m, z13.s\\n\"\n \"fmla z24.s, p1/M, z31.s, z23.s\\n\"\n \"scvtf z17.s, p1/m, z17.s\\n\"\n \"scvtf z18.s, p1/m, z18.s\\n\"\n \"fmla z15.s, p1/M, z13.s, z9.s\\n\"\n \"fmla z12.s, p1/M, z17.s, z21.s\\n\"\n \"fmla z0.s, p1/M, z18.s, z4.s\\n\"\n \"bgt 7b\\n\"\n \"mov x20, %x[res_ptr]\\n\"\n \"cmp x13, #0x1\\n\"\n \"st1w { z24.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"ble 8f\\n\"\n \"cmp x13, #0x2\\n\"\n \"st1w { z15.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"ble 8f\\n\"\n \"cmp x13, #0x3\\n\"\n \"st1w { z12.s }, p1, [x20]\\n\"\n \"add x20, x20, %x[res_stride]\\n\"\n \"ble 8f\\n\"\n \"st1w { z0.s }, p1, [x20]\\n\"\n \"8:\" // Row tail: Accumulator store skip\n \"subs x24, x24, #0x8\\n\"\n \"add %x[res_ptr], %x[res_ptr], #0x20\\n\"\n \"bne 6b\\n\"\n \"subs x13, x13, #0x4\\n\"\n \"add %x[a_ptr], %x[a_ptr], x12\\n\"\n \"mov %x[res_ptr], x23\\n\"\n \"bgt 5b\\n\"\n \"9:\" // Row tail: Row loop skip\n : [a_ptr] \"+&r\" (a_ptr), [res_ptr] \"+&r\" (res_ptr)\n : [b_ptr] \"r\" (b_ptr), [nr] \"r\" (nr), [nb] \"r\" (nb), [res_stride] \"r\" (res_stride), [nc] \"r\" (nc)\n : \"cc\", \"memory\", \"p0\", \"p1\", \"x9\", \"x10\", \"x11\", \"x12\", \"x13\", \"x20\", \"x21\", \"x22\", \"x23\", \"x24\", \"x25\", \"x26\", \"x27\", \"x28\", \"z0\", \"z1\", \"z2\", \"z3\", \"z4\", \"z5\", \"z6\", \"z7\", \"z8\", \"z9\", \"z10\", \"z11\", \"z12\", \"z13\", \"z14\", \"z15\", \"z16\", \"z17\", \"z18\", \"z19\", \"z20\", \"z21\", \"z22\", \"z23\", \"z24\", \"z25\", \"z26\", \"z27\", \"z28\", \"z29\", \"z30\", \"z31\"\n );\n return;\n }\n#endif // #if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)\n ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n const int8x16_t kvalues = vld1q_s8(kvalues_iq4nl);\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx4 * b_ptr = (const block_iq4_nlx4 *) vx + (x * nb);\n\n float32x4_t sumf[4];\n for (int m = 0; m < 4; m++) {\n sumf[m] = vdupq_n_f32(0);\n }\n\n for (int l = 0; l < nb; l++) {\n float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *)a_ptr[l].d));\n float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *)b_ptr[l].d));\n\n int32x4_t sumi_0 = vdupq_n_s32(0);\n int32x4_t sumi_1 = vdupq_n_s32(0);\n int32x4_t sumi_2 = vdupq_n_s32(0);\n int32x4_t sumi_3 = vdupq_n_s32(0);\n\n for (int k = 0; k < 4; k++) {\n int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 16 * k + 0);\n int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16 * k + 64);\n\n uint8x16_t b = vld1q_u8(b_ptr[l].qs + 16 * k);\n int8x16_t b_hi = vqtbl1q_s8(kvalues, b >> 4);\n int8x16_t b_lo = vqtbl1q_s8(kvalues, b & 0xF);\n\n sumi_0 = vdotq_laneq_s32(sumi_0, b_lo, a_0, 0);\n sumi_1 = vdotq_laneq_s32(sumi_1, b_lo, a_0, 1);\n sumi_2 = vdotq_laneq_s32(sumi_2, b_lo, a_0, 2);\n sumi_3 = vdotq_laneq_s32(sumi_3, b_lo, a_0, 3);\n sumi_0 = vdotq_laneq_s32(sumi_0, b_hi, a_1, 0);\n sumi_1 = vdotq_laneq_s32(sumi_1, b_hi, a_1, 1);\n sumi_2 = vdotq_laneq_s32(sumi_2, b_hi, a_1, 2);\n sumi_3 = vdotq_laneq_s32(sumi_3, b_hi, a_1, 3);\n }\n\n sumf[0] = vmlaq_f32(sumf[0], vmulq_laneq_f32(b_d, a_d, 0), vcvtq_f32_s32(sumi_0));\n sumf[1] = vmlaq_f32(sumf[1], vmulq_laneq_f32(b_d, a_d, 1), vcvtq_f32_s32(sumi_1));\n sumf[2] = vmlaq_f32(sumf[2], vmulq_laneq_f32(b_d, a_d, 2), vcvtq_f32_s32(sumi_2));\n sumf[3] = vmlaq_f32(sumf[3], vmulq_laneq_f32(b_d, a_d, 3), vcvtq_f32_s32(sumi_3));\n }\n\n for (int m = 0; m < 4; m++) {\n vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]);\n }\n }\n }\n return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n ggml_gemm_iq4_nl_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n const int8x16_t kvalues = vld1q_s8(kvalues_mxfp4);\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);\n\n float32x4_t sumf[4];\n for (int m = 0; m < 4; m++) {\n sumf[m] = vdupq_n_f32(0);\n }\n\n for (int l = 0; l < nb; l++) {\n float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *)a_ptr[l].d));\n float32x4_t b_d = {\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[0]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[1]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[2]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[3]),\n };\n\n int32x4_t sumi_0 = vdupq_n_s32(0);\n int32x4_t sumi_1 = vdupq_n_s32(0);\n int32x4_t sumi_2 = vdupq_n_s32(0);\n int32x4_t sumi_3 = vdupq_n_s32(0);\n\n for (int k = 0; k < 4; k++) {\n int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 16 * k + 0);\n int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16 * k + 64);\n\n uint8x16_t b = vld1q_u8(b_ptr[l].qs + 16 * k);\n int8x16_t b_hi = vqtbl1q_s8(kvalues, b >> 4);\n int8x16_t b_lo = vqtbl1q_s8(kvalues, b & 0xF);\n\n sumi_0 = vdotq_laneq_s32(sumi_0, b_lo, a_0, 0);\n sumi_1 = vdotq_laneq_s32(sumi_1, b_lo, a_0, 1);\n sumi_2 = vdotq_laneq_s32(sumi_2, b_lo, a_0, 2);\n sumi_3 = vdotq_laneq_s32(sumi_3, b_lo, a_0, 3);\n sumi_0 = vdotq_laneq_s32(sumi_0, b_hi, a_1, 0);\n sumi_1 = vdotq_laneq_s32(sumi_1, b_hi, a_1, 1);\n sumi_2 = vdotq_laneq_s32(sumi_2, b_hi, a_1, 2);\n sumi_3 = vdotq_laneq_s32(sumi_3, b_hi, a_1, 3);\n }\n\n sumf[0] = vmlaq_f32(sumf[0], vmulq_laneq_f32(b_d, a_d, 0), vcvtq_f32_s32(sumi_0));\n sumf[1] = vmlaq_f32(sumf[1], vmulq_laneq_f32(b_d, a_d, 1), vcvtq_f32_s32(sumi_1));\n sumf[2] = vmlaq_f32(sumf[2], vmulq_laneq_f32(b_d, a_d, 2), vcvtq_f32_s32(sumi_2));\n sumf[3] = vmlaq_f32(sumf[3], vmulq_laneq_f32(b_d, a_d, 3), vcvtq_f32_s32(sumi_3));\n }\n\n for (int m = 0; m < 4; m++) {\n vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]);\n }\n }\n }\n return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n ggml_gemm_mxfp4_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 4;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n constexpr int q8_k_blocklen = 4;\n constexpr int acc_size = 2 * 4; // 2 row pairs × 4 col pairs\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n\n // 8 accumulators: 2 row pairs × 4 col pairs\n float32x4_t acc_f32[acc_size];\n\n for (int y = 0; y < nr / q8_k_blocklen; y++) {\n const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < acc_size; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n // d4 0 1 2 3, 4 5 6 7\n float32x4_t q4_d_0123 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d));\n float32x4_t q4_d_4567 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d + 4));\n // d8 0 1 2 3\n float32x4_t q8_d_0123 = vld1q_f32(q8_ptr[b].d);\n // mins\n float32x4_t q4_dmin_0123 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin));\n float32x4_t q4_dmin_4567 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin + 4));\n\n // Precomputation of scales and mins\n float32x4_t sbd_scale_0123[q8_k_blocklen];\n float32x4_t sbd_scale_4567[q8_k_blocklen];\n float32x4_t sbd_min_0123[q8_k_blocklen];\n float32x4_t sbd_min_4567[q8_k_blocklen];\n\n sbd_scale_0123[0] = vmulq_laneq_f32(q4_d_0123, q8_d_0123, 0);\n sbd_scale_4567[0] = vmulq_laneq_f32(q4_d_4567, q8_d_0123, 0);\n sbd_min_0123[0] = vmulq_laneq_f32(q4_dmin_0123, q8_d_0123, 0);\n sbd_min_4567[0] = vmulq_laneq_f32(q4_dmin_4567, q8_d_0123, 0);\n\n sbd_scale_0123[1] = vmulq_laneq_f32(q4_d_0123, q8_d_0123, 1);\n sbd_scale_4567[1] = vmulq_laneq_f32(q4_d_4567, q8_d_0123, 1);\n sbd_min_0123[1] = vmulq_laneq_f32(q4_dmin_0123, q8_d_0123, 1);\n sbd_min_4567[1] = vmulq_laneq_f32(q4_dmin_4567, q8_d_0123, 1);\n\n sbd_scale_0123[2] = vmulq_laneq_f32(q4_d_0123, q8_d_0123, 2);\n sbd_scale_4567[2] = vmulq_laneq_f32(q4_d_4567, q8_d_0123, 2);\n sbd_min_0123[2] = vmulq_laneq_f32(q4_dmin_0123, q8_d_0123, 2);\n sbd_min_4567[2] = vmulq_laneq_f32(q4_dmin_4567, q8_d_0123, 2);\n\n sbd_scale_0123[3] = vmulq_laneq_f32(q4_d_0123, q8_d_0123, 3);\n sbd_scale_4567[3] = vmulq_laneq_f32(q4_d_4567, q8_d_0123, 3);\n sbd_min_0123[3] = vmulq_laneq_f32(q4_dmin_0123, q8_d_0123, 3);\n sbd_min_4567[3] = vmulq_laneq_f32(q4_dmin_4567, q8_d_0123, 3);\n\n // Precomputation of bsums, each vpaddq calcs all the bsums for each row\n const int16x8_t bsums[q8_k_blocklen] = {\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n };\n int16_t bsums_arr[QK_K / 64][8];\n for (int q8_row = 0; q8_row < 4; q8_row++) {\n vst1q_s16(bsums_arr[q8_row], bsums[q8_row]);\n }\n\n // interleaved bias_acc: [0]->r0 0123, [1]->r1 0123, .., [4]->r0 4567, [5]->r1 4567 ..\n int32x4_t bias_acc[acc_size];\n for (int i = 0; i < acc_size; i++) {\n bias_acc[i] = vdupq_n_s32(0);\n }\n\n for (int sb = 0; sb < QK_K / 64; sb++) {\n // Int accumulators for qs vecdot (4 row x 2 col quartets)\n int32x4_t acc_lo[acc_size];\n int32x4_t acc_hi[acc_size];\n for (int i = 0; i < acc_size; i++) {\n acc_lo[i] = vdupq_n_s32(0);\n acc_hi[i] = vdupq_n_s32(0);\n }\n // Need scales for the low and high nibbles\n // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n int16x8_t q4sb_scales[2];\n int16x8_t q4sb_mins[2];\n for (int i = 0; i < 2; i++) {\n int8_t aux_q4sb[8];\n const int offset = sb * 24 + i * 12;\n decode_q_Kx8_6bit_scales(&q4_ptr[b].scales[offset], &q4sb_mins[i], aux_q4sb);\n q4sb_scales[i] = vmovl_s8(vld1_s8(aux_q4sb));\n }\n\n constexpr int reads_per_sb = 8; // 8 * 16 bytes each => 32 qs * 4 rows\n for (int k = 0; k < reads_per_sb; k++) {\n const int8x16_t q8_blk0 = vld1q_s8(q8_ptr[b].qs + sb * 256 + 16 * k);\n const int8x16_t q8_blk1 = vld1q_s8(q8_ptr[b].qs + sb * 256 + 16 * k + 128);\n\n // 0..3 & 32..35\n const uint8x16_t q4_0123 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 32 * k);\n const uint8x16_t q4_4567 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 32 * k + 16);\n\n const int8x16_t q4_0123_lo = vreinterpretq_s8_u8(vandq_u8(q4_0123, m4b));\n const int8x16_t q4_0123_hi = vreinterpretq_s8_u8(vshrq_n_u8(q4_0123, 4));\n\n acc_lo[0] = vdotq_laneq_s32(acc_lo[0], q4_0123_lo, q8_blk0, 0); // 0..3 r0 c0123\n acc_lo[1] = vdotq_laneq_s32(acc_lo[1], q4_0123_lo, q8_blk0, 1); // 0..3 r1 c0123\n acc_lo[2] = vdotq_laneq_s32(acc_lo[2], q4_0123_lo, q8_blk0, 2); // 0..3 r2 c0123\n acc_lo[3] = vdotq_laneq_s32(acc_lo[3], q4_0123_lo, q8_blk0, 3); // 0..3 r3 c0123\n\n acc_hi[0] = vdotq_laneq_s32(acc_hi[0], q4_0123_hi, q8_blk1, 0); // 32..35 r0 c0123\n acc_hi[1] = vdotq_laneq_s32(acc_hi[1], q4_0123_hi, q8_blk1, 1); // 32..35 r1 c0123\n acc_hi[2] = vdotq_laneq_s32(acc_hi[2], q4_0123_hi, q8_blk1, 2); // 32..35 r2 c0123\n acc_hi[3] = vdotq_laneq_s32(acc_hi[3], q4_0123_hi, q8_blk1, 3); // 32..35 r3 c0123\n\n const int8x16_t q4_4567_lo = vreinterpretq_s8_u8(vandq_u8(q4_4567, m4b));\n const int8x16_t q4_4567_hi = vreinterpretq_s8_u8(vshrq_n_u8(q4_4567, 4));\n\n acc_lo[4] = vdotq_laneq_s32(acc_lo[4], q4_4567_lo, q8_blk0, 0); // 0..3 r0 c4567\n acc_lo[5] = vdotq_laneq_s32(acc_lo[5], q4_4567_lo, q8_blk0, 1); // 0..3 r1 c4567\n acc_lo[6] = vdotq_laneq_s32(acc_lo[6], q4_4567_lo, q8_blk0, 2); // 0..3 r2 c4567\n acc_lo[7] = vdotq_laneq_s32(acc_lo[7], q4_4567_lo, q8_blk0, 3); // 0..3 r3 c4567\n\n acc_hi[4] = vdotq_laneq_s32(acc_hi[4], q4_4567_hi, q8_blk1, 0); // 32..35 r0 c4567\n acc_hi[5] = vdotq_laneq_s32(acc_hi[5], q4_4567_hi, q8_blk1, 1); // 32..35 r1 c4567\n acc_hi[6] = vdotq_laneq_s32(acc_hi[6], q4_4567_hi, q8_blk1, 2); // 32..35 r2 c4567\n acc_hi[7] = vdotq_laneq_s32(acc_hi[7], q4_4567_hi, q8_blk1, 3); // 32..35 r3 c4567\n }\n\n // Scale and bias application\n // acc is stored interleaved to match output layout\n const int16x4_t sc_0123_lo = vget_low_s16(q4sb_scales[0]);\n const int16x4_t sc_4567_lo = vget_high_s16(q4sb_scales[0]);\n const int16x4_t sc_0123_hi = vget_low_s16(q4sb_scales[1]);\n const int16x4_t sc_4567_hi = vget_high_s16(q4sb_scales[1]);\n for (int row = 0; row < q8_k_blocklen; row++) {\n // Bias correction\n // row c0123 blk0 and blk1\n const float32x4_t sumf_0123 =\n vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_0123_lo), acc_lo[row]),\n vmulq_s32(vmovl_s16(sc_0123_hi), acc_hi[row])));\n acc_f32[2 * row] = vfmaq_f32(acc_f32[2 * row], sbd_scale_0123[row], sumf_0123);\n\n // row c4567 blk0 and blk1\n const float32x4_t sumf_4567 =\n vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_4567_lo), acc_lo[row + 4]),\n vmulq_s32(vmovl_s16(sc_4567_hi), acc_hi[row + 4])));\n acc_f32[2 * row + 1] = vfmaq_f32(acc_f32[2 * row + 1], sbd_scale_4567[row], sumf_4567);\n\n // Bias\n const int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][row * 2]);\n const int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][row * 2 + 1]);\n\n // row c0123 blk0 and blk1\n bias_acc[2 * row] = vmlal_s16(bias_acc[2 * row], bsums_vec_lo, vget_low_s16(q4sb_mins[0]));\n bias_acc[2 * row] = vmlal_s16(bias_acc[2 * row], bsums_vec_hi, vget_low_s16(q4sb_mins[1]));\n\n // row c4567 blk0 and blk1\n bias_acc[2 * row + 1] =\n vmlal_s16(bias_acc[2 * row + 1], bsums_vec_lo, vget_high_s16(q4sb_mins[0]));\n bias_acc[2 * row + 1] =\n vmlal_s16(bias_acc[2 * row + 1], bsums_vec_hi, vget_high_s16(q4sb_mins[1]));\n }\n } // for sb\n\n for (int row = 0; row < q8_k_blocklen; row++) {\n acc_f32[2 * row] = vmlsq_f32(acc_f32[2 * row], vcvtq_f32_s32(bias_acc[2 * row]), sbd_min_0123[row]);\n acc_f32[2 * row + 1] =\n vmlsq_f32(acc_f32[2 * row + 1], vcvtq_f32_s32(bias_acc[2 * row + 1]), sbd_min_4567[row]);\n }\n } // for b\n\n for (int i = 0; i < q8_k_blocklen; i++) {\n int row = y * q8_k_blocklen + i;\n for (int j = 0; j < 2; j++) {\n int col = x * ncols_interleaved + j * 4;\n int offset = row * bs + col;\n vst1q_f32(s + offset, acc_f32[2 * i + j]);\n }\n }\n } // for x\n } // for y\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemm_q4_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q5_K_8x4_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 4;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n constexpr int q8_k_blocklen = 4;\n constexpr int acc_size = 2 * 4; // 2 row pairs, 4 col pairs\n constexpr int col_groups = ncols_interleaved / 4;\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n const uint8x16_t mone = vdupq_n_u8(1);\n const uint8x16_t mtwo = vdupq_n_u8(2);\n\n // 8 accumulators: 2 row pairs, 4 col pairs\n float32x4_t acc_f32[acc_size];\n\n for (int y = 0; y < nr / q8_k_blocklen; y++) {\n const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q5_Kx8 * GGML_RESTRICT q5_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < acc_size; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n // d5 0 1 2 3, 4 5 6 7\n float32x4_t q5_d_0123 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d));\n float32x4_t q5_d_4567 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d + 4));\n // d8 0 1 2 3\n float32x4_t q8_d_0123 = vld1q_f32(q8_ptr[b].d);\n // mins\n float32x4_t q5_dmin_0123 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin));\n float32x4_t q5_dmin_4567 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin + 4));\n\n // Precomputation of scales and mins\n float32x4_t sbd_scale_0123[q8_k_blocklen];\n float32x4_t sbd_scale_4567[q8_k_blocklen];\n float32x4_t sbd_min_0123[q8_k_blocklen];\n float32x4_t sbd_min_4567[q8_k_blocklen];\n\n sbd_scale_0123[0] = vmulq_laneq_f32(q5_d_0123, q8_d_0123, 0);\n sbd_scale_4567[0] = vmulq_laneq_f32(q5_d_4567, q8_d_0123, 0);\n sbd_min_0123[0] = vmulq_laneq_f32(q5_dmin_0123, q8_d_0123, 0);\n sbd_min_4567[0] = vmulq_laneq_f32(q5_dmin_4567, q8_d_0123, 0);\n\n sbd_scale_0123[1] = vmulq_laneq_f32(q5_d_0123, q8_d_0123, 1);\n sbd_scale_4567[1] = vmulq_laneq_f32(q5_d_4567, q8_d_0123, 1);\n sbd_min_0123[1] = vmulq_laneq_f32(q5_dmin_0123, q8_d_0123, 1);\n sbd_min_4567[1] = vmulq_laneq_f32(q5_dmin_4567, q8_d_0123, 1);\n\n sbd_scale_0123[2] = vmulq_laneq_f32(q5_d_0123, q8_d_0123, 2);\n sbd_scale_4567[2] = vmulq_laneq_f32(q5_d_4567, q8_d_0123, 2);\n sbd_min_0123[2] = vmulq_laneq_f32(q5_dmin_0123, q8_d_0123, 2);\n sbd_min_4567[2] = vmulq_laneq_f32(q5_dmin_4567, q8_d_0123, 2);\n\n sbd_scale_0123[3] = vmulq_laneq_f32(q5_d_0123, q8_d_0123, 3);\n sbd_scale_4567[3] = vmulq_laneq_f32(q5_d_4567, q8_d_0123, 3);\n sbd_min_0123[3] = vmulq_laneq_f32(q5_dmin_0123, q8_d_0123, 3);\n sbd_min_4567[3] = vmulq_laneq_f32(q5_dmin_4567, q8_d_0123, 3);\n\n // Precomputation of bsums, each vpaddq calcs all the bsums for each row\n const int16x8_t bsums[q8_k_blocklen] = {\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n };\n int16_t bsums_arr[QK_K / 64][8];\n for (int q8_row = 0; q8_row < 4; q8_row++) {\n vst1q_s16(bsums_arr[q8_row], bsums[q8_row]);\n }\n\n // interleaved bias_acc: [0]->r0 0123, [1]->r1 0123, .., [4]->r0 4567, [5]->r1 4567 ..\n int32x4_t bias_acc[acc_size];\n for (int i = 0; i < acc_size; i++) {\n bias_acc[i] = vdupq_n_s32(0);\n }\n\n uint8x16_t qh[col_groups][8];\n for (int c = 0; c < col_groups; c++) {\n for (int i = 0; i < 8; i++) {\n qh[c][i] = vld1q_u8(q5_ptr[b].qh + i * 32 + 16 * c);\n }\n }\n\n for (int sb = 0; sb < QK_K / 64; sb++) {\n // Int accumulators for qs vecdot (4 row * 2 col quartets)\n int32x4_t acc_lo[acc_size];\n int32x4_t acc_hi[acc_size];\n for (int i = 0; i < acc_size; i++) {\n acc_lo[i] = vdupq_n_s32(0);\n acc_hi[i] = vdupq_n_s32(0);\n }\n // Need scales for the low and high nibbles\n // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n int16x8_t q5sb_scales[2];\n int16x8_t q5sb_mins[2];\n for (int i = 0; i < 2; i++) {\n int8_t aux_q5sb[8];\n const int offset = sb * 24 + i * 12;\n decode_q_Kx8_6bit_scales(&q5_ptr[b].scales[offset], &q5sb_mins[i], aux_q5sb);\n q5sb_scales[i] = vmovl_s8(vld1_s8(aux_q5sb));\n }\n\n constexpr int reads_per_sb = 8; // 8 * 16 bytes each => 32 qs * 4 rows\n for (int k = 0; k < reads_per_sb; k++) {\n const int8x16_t q8_blk0 = vld1q_s8(q8_ptr[b].qs + sb * 256 + 16 * k);\n const int8x16_t q8_blk1 = vld1q_s8(q8_ptr[b].qs + sb * 256 + 16 * k + 128);\n\n // 0..3 & 32..35\n const uint8x16_t q5_0123 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 32 * k);\n const uint8x16_t q5_4567 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 32 * k + 16);\n\n // NOTE: This is the only difference with q4_K\n const uint8x16_t hbit_lo_0123 = vandq_u8(qh[0][k], mone);\n const uint8x16_t hbit_hi_0123 = vshlq_n_u8(vandq_u8(qh[0][k], mtwo), 3);\n qh[0][k] = vshrq_n_u8(qh[0][k], 2);\n const uint8x16_t hbit_lo_4567 = vandq_u8(qh[1][k], mone);\n const uint8x16_t hbit_hi_4567 = vshlq_n_u8(vandq_u8(qh[1][k], mtwo), 3);\n qh[1][k] = vshrq_n_u8(qh[1][k], 2);\n // From here, same as q4_K\n\n const int8x16_t q5_0123_lo =\n vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q5_0123, m4b), hbit_lo_0123, 4));\n const int8x16_t q5_0123_hi =\n vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5_0123, 4), hbit_hi_0123));\n\n acc_lo[0] = vdotq_laneq_s32(acc_lo[0], q5_0123_lo, q8_blk0, 0); // 0..3 r0 c0123\n acc_lo[1] = vdotq_laneq_s32(acc_lo[1], q5_0123_lo, q8_blk0, 1); // 0..3 r1 c0123\n acc_lo[2] = vdotq_laneq_s32(acc_lo[2], q5_0123_lo, q8_blk0, 2); // 0..3 r2 c0123\n acc_lo[3] = vdotq_laneq_s32(acc_lo[3], q5_0123_lo, q8_blk0, 3); // 0..3 r3 c0123\n\n acc_hi[0] = vdotq_laneq_s32(acc_hi[0], q5_0123_hi, q8_blk1, 0); // 32..35 r0 c0123\n acc_hi[1] = vdotq_laneq_s32(acc_hi[1], q5_0123_hi, q8_blk1, 1); // 32..35 r1 c0123\n acc_hi[2] = vdotq_laneq_s32(acc_hi[2], q5_0123_hi, q8_blk1, 2); // 32..35 r2 c0123\n acc_hi[3] = vdotq_laneq_s32(acc_hi[3], q5_0123_hi, q8_blk1, 3); // 32..35 r3 c0123\n\n const int8x16_t q5_4567_lo =\n vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q5_4567, m4b), hbit_lo_4567, 4));\n const int8x16_t q5_4567_hi =\n vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5_4567, 4), hbit_hi_4567));\n\n acc_lo[4] = vdotq_laneq_s32(acc_lo[4], q5_4567_lo, q8_blk0, 0); // 0..3 r0 c4567\n acc_lo[5] = vdotq_laneq_s32(acc_lo[5], q5_4567_lo, q8_blk0, 1); // 0..3 r1 c4567\n acc_lo[6] = vdotq_laneq_s32(acc_lo[6], q5_4567_lo, q8_blk0, 2); // 0..3 r2 c4567\n acc_lo[7] = vdotq_laneq_s32(acc_lo[7], q5_4567_lo, q8_blk0, 3); // 0..3 r3 c4567\n\n acc_hi[4] = vdotq_laneq_s32(acc_hi[4], q5_4567_hi, q8_blk1, 0); // 32..35 r0 c4567\n acc_hi[5] = vdotq_laneq_s32(acc_hi[5], q5_4567_hi, q8_blk1, 1); // 32..35 r1 c4567\n acc_hi[6] = vdotq_laneq_s32(acc_hi[6], q5_4567_hi, q8_blk1, 2); // 32..35 r2 c4567\n acc_hi[7] = vdotq_laneq_s32(acc_hi[7], q5_4567_hi, q8_blk1, 3); // 32..35 r3 c4567\n }\n\n // Scale and bias application\n // acc is stored interleaved to match output layout\n const int16x4_t sc_0123_lo = vget_low_s16(q5sb_scales[0]);\n const int16x4_t sc_4567_lo = vget_high_s16(q5sb_scales[0]);\n const int16x4_t sc_0123_hi = vget_low_s16(q5sb_scales[1]);\n const int16x4_t sc_4567_hi = vget_high_s16(q5sb_scales[1]);\n for (int row = 0; row < q8_k_blocklen; row++) {\n // Bias correction\n // row c0123 blk0 and blk1\n const float32x4_t sumf_0123 =\n vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_0123_lo), acc_lo[row]),\n vmulq_s32(vmovl_s16(sc_0123_hi), acc_hi[row])));\n acc_f32[2 * row] = vfmaq_f32(acc_f32[2 * row], sbd_scale_0123[row], sumf_0123);\n\n // row c4567 blk0 and blk1\n const float32x4_t sumf_4567 =\n vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_4567_lo), acc_lo[row + 4]),\n vmulq_s32(vmovl_s16(sc_4567_hi), acc_hi[row + 4])));\n acc_f32[2 * row + 1] = vfmaq_f32(acc_f32[2 * row + 1], sbd_scale_4567[row], sumf_4567);\n\n // Bias\n const int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][row * 2]);\n const int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][row * 2 + 1]);\n\n // row c0123 blk0 and blk1\n bias_acc[2 * row] = vmlal_s16(bias_acc[2 * row], bsums_vec_lo, vget_low_s16(q5sb_mins[0]));\n bias_acc[2 * row] = vmlal_s16(bias_acc[2 * row], bsums_vec_hi, vget_low_s16(q5sb_mins[1]));\n\n // row c4567 blk0 and blk1\n bias_acc[2 * row + 1] =\n vmlal_s16(bias_acc[2 * row + 1], bsums_vec_lo, vget_high_s16(q5sb_mins[0]));\n bias_acc[2 * row + 1] =\n vmlal_s16(bias_acc[2 * row + 1], bsums_vec_hi, vget_high_s16(q5sb_mins[1]));\n }\n } // for sb\n\n for (int row = 0; row < q8_k_blocklen; row++) {\n acc_f32[2 * row] = vmlsq_f32(acc_f32[2 * row], vcvtq_f32_s32(bias_acc[2 * row]), sbd_min_0123[row]);\n acc_f32[2 * row + 1] =\n vmlsq_f32(acc_f32[2 * row + 1], vcvtq_f32_s32(bias_acc[2 * row + 1]), sbd_min_4567[row]);\n }\n } // for b\n\n for (int i = 0; i < q8_k_blocklen; i++) {\n int row = y * q8_k_blocklen + i;\n for (int j = 0; j < 2; j++) {\n int col = x * ncols_interleaved + j * 4;\n int offset = row * bs + col;\n vst1q_f32(s + offset, acc_f32[2 * i + j]);\n }\n }\n } // for x\n } // for y\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemm_q5_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_K_8x8_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n if (svcntb() * 8 == 256) {\n constexpr int q8_k_blocklen = 4;\n const svuint8_t m4b_1 = svdup_n_u8(0x0f);\n // 8 accumulators: 2 row pairs × 4 col pairs\n svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67;\n uint32_t idx_arr[8] = { 0, 2, 4, 6, 1, 3, 5, 7 };\n svbool_t pg = svptrue_pat_b32(SV_VL8);\n svuint32_t idx = svld1(pg, idx_arr);\n\n static const uint32_t idx_data[8] = {0, 4, 2, 6, 1, 5, 3, 7};\n svuint32_t idx1 = svld1_u32(svptrue_b32(), idx_data);\n\n for (int y = 0; y < nr / q8_k_blocklen; y++) {\n const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n acc_f32_01 = svdup_n_f32(0);\n acc_f32_23 = svdup_n_f32(0);\n acc_f32_45 = svdup_n_f32(0);\n acc_f32_67 = svdup_n_f32(0);\n\n for (int b = 0; b < nb; b++) {\n // bsums pairs belongs to the same q8_k subblock\n // 64 elements loaded and made sum of 0-7 and 8-15 sum || 16-23 and 24 - 31 sum\n const int16x8_t bsums[4]{\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n };\n\n int32_t bsums_arr32[4][8];\n\n for (int q8_row = 0; q8_row < 4; q8_row++) {\n int16x8_t v16 = bsums[q8_row];\n\n // low 4\n int32x4_t v32_lo = vmovl_s16(vget_low_s16(v16));\n vst1q_s32(&bsums_arr32[q8_row][0], v32_lo);\n\n // high 4\n int32x4_t v32_hi = vmovl_s16(vget_high_s16(v16));\n vst1q_s32(&bsums_arr32[q8_row][4], v32_hi);\n }\n\n svint32_t sb_acc_0 = svdup_n_s32(0);\n svint32_t sb_acc_2 = svdup_n_s32(0);\n\n svint32_t acc_00 = svdup_n_s32(0);\n svint32_t acc_11 = svdup_n_s32(0);\n svint32_t acc_22 = svdup_n_s32(0);\n svint32_t acc_33 = svdup_n_s32(0);\n svint32_t acc_44 = svdup_n_s32(0);\n svint32_t acc_55 = svdup_n_s32(0);\n svint32_t acc_66 = svdup_n_s32(0);\n svint32_t acc_77 = svdup_n_s32(0);\n\n svint32_t bias_acc_00 = svdup_n_s32(0);\n svint32_t bias_acc_22 = svdup_n_s32(0);\n svint32_t bias_acc_44 = svdup_n_s32(0);\n svint32_t bias_acc_66 = svdup_n_s32(0);\n\n for (int sb = 0; sb < QK_K / 64; sb++) {\n // Need scales for the low and high nibbles\n // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n svint32_t block_scale_0, block_scale_1, block_scale_2, block_scale_3;\n svint32_t q4sb_mins_0, q4sb_mins_1;\n {\n // 2-superblock I am working on\n const int offset = sb * 24 + 0 * 12;\n const uint8_t * scales_in = &q4_ptr[b].scales[offset];\n\n const int offset1 = sb * 24 + 12;\n const uint8_t * scales_in1 = &q4_ptr[b].scales[offset1];\n\n constexpr uint32_t kmask1 = 0x3f3f3f3f;\n constexpr uint32_t kmask2 = 0x0f0f0f0f;\n constexpr uint32_t kmask3 = 0x03030303;\n constexpr uint8_t scales_size = 12;\n\n uint32_t sm[3];\n memcpy(sm, scales_in, scales_size);\n\n uint32_t sm1[3];\n memcpy(sm1, scales_in1, scales_size);\n\n const uint32_t mins_0_3 = sm[1] & kmask1;\n const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4);\n\n const uint32_t mins_0_3_1 = sm1[1] & kmask1;\n const uint32_t mins_4_7_1 = ((sm1[2] >> 4) & kmask2) | (((sm1[1] >> 6) & kmask3) << 4);\n\n svuint32_t mins_u32_temp = svzip1_u32(svdup_n_u32(mins_0_3), svdup_n_u32(mins_4_7));\n svuint32_t mins_u32_temp_1 = svzip1_u32(svdup_n_u32(mins_0_3_1), svdup_n_u32(mins_4_7_1));\n\n /* reinterpret u32 → u8 */\n svuint8_t mins_u8 = svreinterpret_u8_u32(mins_u32_temp);\n svuint8_t mins_u8_1 = svreinterpret_u8_u32(mins_u32_temp_1);\n\n /* widen u8 → u16->u32 (lower half only) */\n svuint32_t mins_u16 = svunpklo_u32(svunpklo_u16(mins_u8));\n svuint32_t mins_u16_1 = svunpklo_u32(svunpklo_u16(mins_u8_1));\n\n q4sb_mins_0 = svreinterpret_s32_u32(mins_u16);\n q4sb_mins_1 = svreinterpret_s32_u32(mins_u16_1);\n\n uint32_t scales_u32_0 = sm[0] & kmask1;\n uint32_t scales_u32_1 = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4);\n uint32_t scales_u32_2 = sm1[0] & kmask1;\n uint32_t scales_u32_3 = (sm1[2] & kmask2) | (((sm1[0] >> 6) & kmask3) << 4);\n\n svuint32_t S01 = svdup_n_u32(scales_u32_0);\n svuint32_t S23 = svdup_n_u32(scales_u32_1);\n svuint32_t R01 = svdup_n_u32(scales_u32_2);\n svuint32_t R23 = svdup_n_u32(scales_u32_3);\n\n svint8_t S01_b = svreinterpret_s8_u32(S01);\n svint8_t S23_b = svreinterpret_s8_u32(S23);\n svint8_t R01_b = svreinterpret_s8_u32(R01);\n svint8_t R23_b = svreinterpret_s8_u32(R23);\n\n svint32_t S01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S01_b, S01_b)));\n svint32_t R01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R01_b, R01_b)));\n svint32_t S23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S23_b, S23_b)));\n svint32_t R23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R23_b, R23_b)));\n\n block_scale_0 = svtbl_s32(svzip1_s32(S01_d, R01_d), idx);\n block_scale_1 = svtbl_s32(svzip2_s32(S01_d, R01_d), idx);\n block_scale_2 = svtbl_s32(svzip1_s32(S23_d, R23_d), idx);\n block_scale_3 = svtbl_s32(svzip2_s32(S23_d, R23_d), idx);\n }\n\n const int8_t * q8_base_1 = q8_ptr[b].qs + sb * 256;\n\n // Load 32-byte per row pair, 1 subblock each time\n // predicate for activating higher lanes for 16 int8 elements\n const svbool_t ph16 = svptrue_pat_b8(SV_VL16);\n // predicate for activating lower lanes for 16 int8 elements\n const svbool_t pl16 = svnot_b_z(svptrue_b8(), ph16);\n\n svint8_t q8_qs_0 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 0), svld1_s8(pl16, q8_base_1 + 112));\n svint8_t q8_qs_2 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 32), svld1_s8(pl16, q8_base_1 + 144));\n svint8_t q8_qs_4 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 64), svld1_s8(pl16, q8_base_1 + 176));\n svint8_t q8_qs_6 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 96), svld1_s8(pl16, q8_base_1 + 208));\n\n svint8_t q8_qs_1 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 16), svld1_s8(pl16, q8_base_1 + 128));\n svint8_t q8_qs_3 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 48), svld1_s8(pl16, q8_base_1 + 160));\n svint8_t q8_qs_5 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 80), svld1_s8(pl16, q8_base_1 + 192));\n svint8_t q8_qs_7 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 112), svld1_s8(pl16, q8_base_1 + 224));\n\n // Q4s columns iterated in pairs (01, 23, 45, 67)\n for (int cp = 0; cp < ncols_interleaved / 2; cp++) {\n\n sb_acc_0 = svdup_n_s32(0);\n sb_acc_2 = svdup_n_s32(0);\n\n svuint8_t q4_qs_cp_00 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 0);\n svuint8_t q4_qs_cp_01 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 64);\n svuint8_t q4_qs_cp_02 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 128);\n svuint8_t q4_qs_cp_03 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 192);\n\n svint8_t q4_nibbles_00 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_00, m4b_1), 4));\n svint8_t q4_nibbles_01 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_01, m4b_1), 4));\n svint8_t q4_nibbles_02 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_02, m4b_1), 4));\n svint8_t q4_nibbles_03 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_03, m4b_1), 4));\n\n sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_00, q8_qs_0);\n sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_01, q8_qs_2);\n\n sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_02, q8_qs_4);\n sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_03, q8_qs_6);\n\n sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_00, q8_qs_1);\n sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_01, q8_qs_3);\n\n sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_02, q8_qs_5);\n sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_03, q8_qs_7);\n\n if(cp == 0) {\n acc_00 = svmla_s32_m(svptrue_b32(), acc_00, sb_acc_0, block_scale_0);\n acc_44 = svmla_s32_m(svptrue_b32(), acc_44, sb_acc_2, block_scale_0);\n }\n if(cp == 1) {\n acc_11 = svmla_s32_m(svptrue_b32(), acc_11, sb_acc_0, block_scale_1);\n acc_55 = svmla_s32_m(svptrue_b32(), acc_55, sb_acc_2, block_scale_1);\n }\n if(cp == 2) {\n acc_22 = svmla_s32_m(svptrue_b32(), acc_22, sb_acc_0, block_scale_2);\n acc_66 = svmla_s32_m(svptrue_b32(), acc_66, sb_acc_2, block_scale_2);\n }\n if(cp == 3) {\n acc_33 = svmla_s32_m(svptrue_b32(), acc_33, sb_acc_0, block_scale_3);\n acc_77 = svmla_s32_m(svptrue_b32(), acc_77, sb_acc_2, block_scale_3);\n }\n }\n\n bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][0]), q4sb_mins_0);\n bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][1]), q4sb_mins_1);\n\n bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][2]), q4sb_mins_0);\n bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][3]), q4sb_mins_1);\n\n bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][4]), q4sb_mins_0);\n bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][5]), q4sb_mins_1);\n\n bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][6]), q4sb_mins_0);\n bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][7]), q4sb_mins_1);\n } // for sb\n\n\n acc_00 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_00, svext_s32(acc_00, acc_00, 4));\n acc_11 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_11, svext_s32(acc_11, acc_11, 4));\n acc_22 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_22, svext_s32(acc_22, acc_22, 4));\n acc_33 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_33, svext_s32(acc_33, acc_33, 4));\n acc_44 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_44, svext_s32(acc_44, acc_44, 4));\n acc_55 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_55, svext_s32(acc_55, acc_55, 4));\n acc_66 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_66, svext_s32(acc_66, acc_66, 4));\n acc_77 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_77, svext_s32(acc_77, acc_77, 4));\n\n svint32_t reorder_acc_01 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_00, acc_11), svtrn1_s32(acc_22, acc_33)), idx1);\n svint32_t reorder_acc_23 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_00, acc_11), svtrn2_s32(acc_22, acc_33)), idx1);\n\n svint32_t reorder_acc_45 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_44, acc_55), svtrn1_s32(acc_66, acc_77)), idx1);\n svint32_t reorder_acc_67 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_44, acc_55), svtrn2_s32(acc_66, acc_77)), idx1);\n\n // Broadcast q8 scalar\n svfloat32_t q8_d = svdup_f32(q8_ptr[b].d[0]);\n\n svfloat32_t q4_dmin_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].dmin), svdup_f16(0)));\n\n svfloat32_t q4_d_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].d), svdup_f16(0)));\n\n svfloat32_t scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);\n svfloat32_t dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);\n\n acc_f32_01 = svmls_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_00), dmins1);\n acc_f32_01 = svmla_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_01), scale1);\n\n q8_d = svdup_f32(q8_ptr[b].d[1]);\n\n scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);\n dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);\n\n acc_f32_23 = svmls_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_22), dmins1);\n acc_f32_23 = svmla_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_23), scale1);\n\n q8_d = svdup_f32(q8_ptr[b].d[2]);\n\n\n scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);\n dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);\n\n acc_f32_45 = svmls_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_44), dmins1);\n acc_f32_45 = svmla_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_45), scale1);\n\n q8_d = svdup_f32(q8_ptr[b].d[3]);\n\n scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);\n dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);\n\n acc_f32_67 = svmls_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_66), dmins1);\n acc_f32_67 = svmla_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_67), scale1);\n\n } // for b\n\n // With the previous reorder, the tile is already in the correct memory layout.\n // Predicate for exactly 4 lanes\n svbool_t pg4 = svptrue_pat_b32(SV_VL4);\n for (int i = 0; i < q8_k_blocklen; i++) {\n int row = y * q8_k_blocklen + i;\n for (int j = 0; j < 2; j++) {\n int col = x * ncols_interleaved + j * 4;\n int offset = row * bs + col;\n\n if (i == 0 && j == 0) {\n // acc_f32_0 → lower half of acc_f32_01\n svst1_f32(pg4, s + offset, acc_f32_01);\n } else if (i == 0 && j == 1) {\n // acc_f32_1 → upper half of acc_f32_01\n svst1_f32(pg4, s + offset, svext_f32(acc_f32_01, acc_f32_01, 4));\n } else if (i == 1 && j == 0) {\n // acc_f32_2\n svst1_f32(pg4, s + offset, acc_f32_23);\n } else if (i == 1 && j == 1) {\n // acc_f32_3\n svst1_f32(pg4, s + offset, svext_f32(acc_f32_23, acc_f32_23, 4));\n } else if (i == 2 && j == 0) {\n // acc_f32_4\n svst1_f32(pg4, s + offset, acc_f32_45);\n } else if (i == 2 && j == 1) {\n // acc_f32_5\n svst1_f32(pg4, s + offset, svext_f32(acc_f32_45, acc_f32_45, 4));\n } else if (i == 3 && j == 0) {\n // acc_f32_6\n svst1_f32(pg4, s + offset, acc_f32_67);\n } else if (i == 3 && j == 1) {\n // acc_f32_7\n svst1_f32(pg4, s + offset, svext_f32(acc_f32_67, acc_f32_67, 4));\n }\n }\n }\n } // for x\n } // for y\n return;\n }\n#endif // SVE compile-time end\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n constexpr int q8_k_blocklen = 4;\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n\n // 8 accumulators: 2 row pairs × 4 col pairs\n float32x4_t acc_f32[blocklen];\n\n for (int y = 0; y < nr / q8_k_blocklen; y++) {\n const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < blocklen; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n // bsums pairs belongs to the same q8_k subblock\n const int16x8_t bsums[4]{\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n };\n int16_t bsums_arr[4][8];\n for (int q8_row = 0; q8_row < 4; q8_row++) {\n vst1q_s16(bsums_arr[q8_row], bsums[q8_row]);\n }\n\n int32x4_t sb_acc[4]; // Aux accumulators to store subblock (partial) results\n int32x4_t acc[8]; // rows 01 stored in [0][1][2][3] rows 23 stored in [4][5][6][7]\n int32x4_t bias_acc[8]; // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567, [2]->r1 0123 ...\n for (int i = 0; i < 8; i++) {\n acc[i] = vdupq_n_s32(0);\n bias_acc[i] = vdupq_n_s32(0);\n }\n\n for (int sb = 0; sb < QK_K / 64; sb++) {\n // Need scales for the low and high nibbles\n // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n int8_t q4sb_scales[2][8];\n int16x8_t q4sb_mins[2]; // int16 as its needed for bias_acc later\n for (int i = 0; i < 2; i++) {\n const int offset = sb * 24 + i * 12;\n decode_q_Kx8_6bit_scales(&q4_ptr[b].scales[offset], &q4sb_mins[i], q4sb_scales[i]);\n }\n\n // q8_ptr[b].qs has interleaved Q8 rows (01, 23)\n const int8_t * q8_base = q8_ptr[b].qs + sb * 256;\n\n int8x16_t q8_qs_01[8];\n int8x16_t q8_qs_23[8];\n\n // Load 32-byte per row pair, 1 subblock each time\n for (int i = 0; i < 8; i++) {\n const int offset = i * 32; // 16 for row 01, 16 for row 23\n q8_qs_01[i] = vld1q_s8(q8_base + offset);\n q8_qs_23[i] = vld1q_s8(q8_base + offset + 16);\n }\n\n const int8x16_t q8s[2][8] = {\n { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3],\n q8_qs_01[4], q8_qs_01[5], q8_qs_01[6], q8_qs_01[7] },\n { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3],\n q8_qs_23[4], q8_qs_23[5], q8_qs_23[6], q8_qs_23[7] },\n };\n\n // Q4s columns iterated in pairs (01, 23, 45, 67)\n for (int cp = 0; cp < ncols_interleaved / 2; cp++) {\n for (int i = 0; i < 4; i++) {\n sb_acc[i] = vdupq_n_s32(0);\n }\n\n uint8x16_t q4_qs_cp_0 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 16 * cp + 0); // 0 .. 7 & 32..39\n uint8x16_t q4_qs_cp_1 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 16 * cp + 64); // 8 ..15 & 40..47\n uint8x16_t q4_qs_cp_2 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 16 * cp + 128); // 16..23 & 48..55\n uint8x16_t q4_qs_cp_3 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 16 * cp + 192); // 24..31 & 56..63\n const int8x16_t q4_nibbles[2][4] = {\n {\n vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_0, m4b)),\n vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_1, m4b)),\n vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_2, m4b)),\n vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_3, m4b)),\n },\n {\n vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_0, 4)),\n vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_1, 4)),\n vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_2, 4)),\n vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_3, 4)),\n }\n };\n\n // Calculates the Qs muladd of every row pair (rp) rows 01 and 23 of q8\n // for each of the internal 32 qs subblock (blk)\n for (int rp = 0; rp < 2; rp++) {\n for (int blk = 0; blk < 2; blk++) {\n const int8x16_t * q8 = &q8s[rp][4 * blk];\n const int8x16_t * q4 = q4_nibbles[blk];\n int32x4_t acc = sb_acc[2 * rp + blk];\n // mul add for each qs in the same subblock\n for (int qs_offset = 0; qs_offset < 4; qs_offset++) {\n acc = vmmlaq_s32(acc, q4[qs_offset], q8[qs_offset]);\n }\n sb_acc[2 * rp + blk] = acc;\n }\n }\n\n // Scales[i] corresponds to column i\n const int scale_offset = cp * 2;\n const int32_t scale_00 = q4sb_scales[0][scale_offset];\n const int32_t scale_01 = q4sb_scales[0][scale_offset + 1];\n const int32_t scale_10 = q4sb_scales[1][scale_offset];\n const int32_t scale_11 = q4sb_scales[1][scale_offset + 1];\n const int32x4_t block_scale_0 = vcombine_s32(vdup_n_s32(scale_00), vdup_n_s32(scale_01));\n const int32x4_t block_scale_1 = vcombine_s32(vdup_n_s32(scale_10), vdup_n_s32(scale_11));\n\n acc[cp] = vmlaq_s32(acc[cp], sb_acc[0], block_scale_0);\n acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[2], block_scale_0);\n acc[cp] = vmlaq_s32(acc[cp], sb_acc[1], block_scale_1);\n acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[3], block_scale_1);\n }\n\n // Multiply Acc bsum + mins\n for (int q8_row = 0; q8_row < 4; q8_row++) {\n // Each pair of subblocks share the same bsums\n // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)).\n int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][q8_row * 2]);\n int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][q8_row * 2 + 1]);\n\n bias_acc[2 * q8_row] =\n vmlal_s16(bias_acc[2 * q8_row], bsums_vec_lo, vget_low_s16(q4sb_mins[0]));\n bias_acc[2 * q8_row] =\n vmlal_s16(bias_acc[2 * q8_row], bsums_vec_hi, vget_low_s16(q4sb_mins[1]));\n bias_acc[2 * q8_row + 1] =\n vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_lo, vget_high_s16(q4sb_mins[0]));\n bias_acc[2 * q8_row + 1] =\n vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_hi, vget_high_s16(q4sb_mins[1]));\n }\n } // for sb\n\n // Reorder of i8mm output with bias and output layout\n for (int i = 0; i < 8; i++) {\n int32x2x2_t aux = vzip_s32(vget_low_s32(acc[i]), vget_high_s32(acc[i]));\n acc[i] = vcombine_s32(aux.val[0], aux.val[1]);\n }\n int32x4_t reorder_acc[8] = {\n vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])),\n vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])),\n vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])),\n vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])),\n vcombine_s32(vget_low_s32(acc[4]), vget_low_s32(acc[5])),\n vcombine_s32(vget_low_s32(acc[6]), vget_low_s32(acc[7])),\n vcombine_s32(vget_high_s32(acc[4]), vget_high_s32(acc[5])),\n vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])),\n };\n\n for (int i = 0; i < q8_k_blocklen; i++) {\n for (int j = 0; j < 2; j++) {\n float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d[i]);\n float32x4_t q4_dmin = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].dmin + j * 4)));\n const float32x4_t dmins = vmulq_f32(q4_dmin, q8_d);\n\n float32x4_t q4_d = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].d + j * 4)));\n const float32x4_t scale = vmulq_f32(q4_d, q8_d);\n\n acc_f32[2 * i + j] = vmlsq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(bias_acc[2 * i + j]), dmins);\n acc_f32[2 * i + j] =\n vmlaq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(reorder_acc[2 * i + j]), scale);\n }\n }\n } // for b\n\n // With the previous reorder, the tile is already in the correct memory layout.\n for (int i = 0; i < q8_k_blocklen; i++) {\n int row = y * q8_k_blocklen + i;\n for (int j = 0; j < 2; j++) {\n int col = x * ncols_interleaved + j * 4;\n int offset = row * bs + col;\n vst1q_f32(s + offset, acc_f32[2 * i + j]);\n }\n }\n } // for x\n } // for y\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n ggml_gemm_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q5_K_8x8_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n constexpr int q8_k_blocklen = 4;\n constexpr int col_pairs = ncols_interleaved / 2;\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n const uint8x16_t mone = vdupq_n_u8(1);\n const uint8x16_t mtwo = vdupq_n_u8(2);\n\n // 8 accumulators: 2 row pairs × 4 col pairs\n float32x4_t acc_f32[blocklen];\n\n for (int y = 0; y < nr / q8_k_blocklen; y++) {\n const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q5_Kx8 * GGML_RESTRICT q5_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < blocklen; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n // bsums pairs belongs to the same q8_k subblock\n const int16x8_t bsums[4]{\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n };\n int16_t bsums_arr[4][8];\n for (int q8_row = 0; q8_row < 4; q8_row++) {\n vst1q_s16(bsums_arr[q8_row], bsums[q8_row]);\n }\n\n int32x4_t sb_acc[4]; // Aux accumulators to store subblock (partial) results\n int32x4_t acc[8]; // rows 01 stored in [0][1][2][3] rows 23 stored in [4][5][6][7]\n int32x4_t bias_acc[8]; // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567, [2]->r1 0123 ...\n for (int i = 0; i < 8; i++) {\n acc[i] = vdupq_n_s32(0);\n bias_acc[i] = vdupq_n_s32(0);\n }\n\n // Load qh once per block and shift after each subblock\n const uint8_t * qh_base = q5_ptr[b].qh;\n uint8x16_t qh[col_pairs][4];\n for (int cp = 0; cp < col_pairs; cp++) {\n qh[cp][0] = vld1q_u8(qh_base + 16 * cp);\n qh[cp][1] = vld1q_u8(qh_base + 16 * cp + 64);\n qh[cp][2] = vld1q_u8(qh_base + 16 * cp + 128);\n qh[cp][3] = vld1q_u8(qh_base + 16 * cp + 192);\n }\n\n for (int sb = 0; sb < QK_K / 64; sb++) {\n // Need scales for the low and high nibbles\n // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n int8_t q5sb_scales[2][8];\n int16x8_t q5sb_mins[2]; // int16 as its needed for bias_acc later\n for (int i = 0; i < 2; i++) {\n const int offset = sb * 24 + i * 12;\n decode_q_Kx8_6bit_scales(&q5_ptr[b].scales[offset], &q5sb_mins[i], q5sb_scales[i]);\n }\n\n // q8_ptr[b].qs has interleaved Q8 rows (01, 23)\n const int8_t * q8_base = q8_ptr[b].qs + sb * 256;\n\n int8x16_t q8_qs_01[8];\n int8x16_t q8_qs_23[8];\n\n // Load 32-byte per row pair, 1 subblock each time\n for (int i = 0; i < 8; i++) {\n const int offset = i * 32; // 16 for row 01, 16 for row 23\n q8_qs_01[i] = vld1q_s8(q8_base + offset);\n q8_qs_23[i] = vld1q_s8(q8_base + offset + 16);\n }\n\n const int8x16_t q8s[2][8] = {\n { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3], q8_qs_01[4], q8_qs_01[5], q8_qs_01[6],\n q8_qs_01[7] },\n { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3], q8_qs_23[4], q8_qs_23[5], q8_qs_23[6],\n q8_qs_23[7] },\n };\n\n // Q5s columns iterated in pairs (01, 23, 45, 67)\n for (int cp = 0; cp < col_pairs; cp++) {\n for (int i = 0; i < 4; i++) {\n sb_acc[i] = vdupq_n_s32(0);\n }\n\n uint8x16_t qs_cp_0 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 16 * cp + 0); // 0 .. 7 & 32..39\n uint8x16_t qs_cp_1 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 16 * cp + 64); // 8 ..15 & 40..47\n uint8x16_t qs_cp_2 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 16 * cp + 128); // 16..23 & 48..55\n uint8x16_t qs_cp_3 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 16 * cp + 192); // 24..31 & 56..63\n\n // This is the only part of the algorithm that differs with Q4_K\n // Extract High bits and pack into 5 bit weights\n uint8x16_t hbit_lo_0 = vandq_u8(qh[cp][0], mone);\n uint8x16_t hbit_hi_0 = vshlq_n_u8(vandq_u8(qh[cp][0], mtwo), 3);\n qh[cp][0] = vshrq_n_u8(qh[cp][0], 2);\n // Same as Q4_K, i8mm to dequantize the weights.\n const int8x16_t qs_lo_0 = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_cp_0, m4b), hbit_lo_0, 4));\n int32x4_t acc_0 = sb_acc[0];\n acc_0 = vmmlaq_s32(acc_0, qs_lo_0, q8s[0][0]);\n int32x4_t acc_2 = sb_acc[2];\n acc_2 = vmmlaq_s32(acc_2, qs_lo_0, q8s[1][0]);\n const int8x16_t qs_hi_0 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_cp_0, 4), hbit_hi_0));\n int32x4_t acc_1 = sb_acc[1];\n acc_1 = vmmlaq_s32(acc_1, qs_hi_0, q8s[0][4]);\n int32x4_t acc_3 = sb_acc[3];\n acc_3 = vmmlaq_s32(acc_3, qs_hi_0, q8s[1][4]);\n\n // Repeat for the other 3 columns (8..15, 16..23, 24..31)\n uint8x16_t hbit_hi_1 = vshlq_n_u8(vandq_u8(qh[cp][1], mtwo), 3);\n uint8x16_t hbit_lo_1 = vandq_u8(qh[cp][1], mone);\n qh[cp][1] = vshrq_n_u8(qh[cp][1], 2);\n const int8x16_t qs_lo_1 = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_cp_1, m4b), hbit_lo_1, 4));\n acc_0 = vmmlaq_s32(acc_0, qs_lo_1, q8s[0][1]);\n acc_2 = vmmlaq_s32(acc_2, qs_lo_1, q8s[1][1]);\n const int8x16_t qs_hi_1 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_cp_1, 4), hbit_hi_1));\n acc_1 = vmmlaq_s32(acc_1, qs_hi_1, q8s[0][5]);\n acc_3 = vmmlaq_s32(acc_3, qs_hi_1, q8s[1][5]);\n\n uint8x16_t hbit_hi_2 = vshlq_n_u8(vandq_u8(qh[cp][2], mtwo), 3);\n uint8x16_t hbit_lo_2 = vandq_u8(qh[cp][2], mone);\n qh[cp][2] = vshrq_n_u8(qh[cp][2], 2);\n const int8x16_t qs_lo_2 = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_cp_2, m4b), hbit_lo_2, 4));\n acc_0 = vmmlaq_s32(acc_0, qs_lo_2, q8s[0][2]);\n acc_2 = vmmlaq_s32(acc_2, qs_lo_2, q8s[1][2]);\n const int8x16_t qs_hi_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_cp_2, 4), hbit_hi_2));\n acc_1 = vmmlaq_s32(acc_1, qs_hi_2, q8s[0][6]);\n acc_3 = vmmlaq_s32(acc_3, qs_hi_2, q8s[1][6]);\n\n uint8x16_t hbit_lo_3 = vandq_u8(qh[cp][3], mone);\n uint8x16_t hbit_hi_3 = vshlq_n_u8(vandq_u8(qh[cp][3], mtwo), 3);\n qh[cp][3] = vshrq_n_u8(qh[cp][3], 2);\n const int8x16_t qs_lo_3 = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_cp_3, m4b), hbit_lo_3, 4));\n acc_0 = vmmlaq_s32(acc_0, qs_lo_3, q8s[0][3]);\n sb_acc[0] = acc_0;\n acc_2 = vmmlaq_s32(acc_2, qs_lo_3, q8s[1][3]);\n sb_acc[2] = acc_2;\n\n // Scales[i] corresponds to column i\n const int scale_offset = cp * 2;\n const int32_t s0 = q5sb_scales[0][scale_offset];\n const int32_t s1 = q5sb_scales[0][scale_offset + 1];\n const int32x4_t block_scale = vcombine_s32(vdup_n_s32(s0), vdup_n_s32(s1));\n acc[cp] = vmlaq_s32(acc[cp], sb_acc[0], block_scale);\n acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[2], block_scale);\n\n const int8x16_t qs_hi_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_cp_3, 4), hbit_hi_3));\n acc_1 = vmmlaq_s32(acc_1, qs_hi_3, q8s[0][7]);\n sb_acc[1] = acc_1;\n acc_3 = vmmlaq_s32(acc_3, qs_hi_3, q8s[1][7]);\n sb_acc[3] = acc_3;\n\n const int32_t s2 = q5sb_scales[1][scale_offset];\n const int32_t s3 = q5sb_scales[1][scale_offset + 1];\n const int32x4_t block_scale2 = vcombine_s32(vdup_n_s32(s2), vdup_n_s32(s3));\n acc[cp] = vmlaq_s32(acc[cp], sb_acc[1], block_scale2);\n acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[3], block_scale2);\n }\n\n // Multiply Acc bsum + mins\n for (int q8_row = 0; q8_row < 4; q8_row++) {\n // Each pair of subblocks share the same bsums\n // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)).\n int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][q8_row * 2]);\n int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][q8_row * 2 + 1]);\n\n bias_acc[2 * q8_row] =\n vmlal_s16(bias_acc[2 * q8_row], bsums_vec_lo, vget_low_s16(q5sb_mins[0]));\n bias_acc[2 * q8_row] =\n vmlal_s16(bias_acc[2 * q8_row], bsums_vec_hi, vget_low_s16(q5sb_mins[1]));\n bias_acc[2 * q8_row + 1] =\n vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_lo, vget_high_s16(q5sb_mins[0]));\n bias_acc[2 * q8_row + 1] =\n vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_hi, vget_high_s16(q5sb_mins[1]));\n }\n } // for sb\n\n // Reorder of i8mm output with bias and output layout\n for (int i = 0; i < 8; i++) {\n int32x2x2_t aux = vzip_s32(vget_low_s32(acc[i]), vget_high_s32(acc[i]));\n acc[i] = vcombine_s32(aux.val[0], aux.val[1]);\n }\n int32x4_t reorder_acc[8] = {\n vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])),\n vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])),\n vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])),\n vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])),\n vcombine_s32(vget_low_s32(acc[4]), vget_low_s32(acc[5])),\n vcombine_s32(vget_low_s32(acc[6]), vget_low_s32(acc[7])),\n vcombine_s32(vget_high_s32(acc[4]), vget_high_s32(acc[5])),\n vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])),\n };\n\n for (int i = 0; i < q8_k_blocklen; i++) {\n for (int j = 0; j < 2; j++) {\n float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d[i]);\n float32x4_t q5_dmin = vcvt_f32_f16(vld1_f16((const __fp16 *) (q5_ptr[b].dmin + j * 4)));\n const float32x4_t dmins = vmulq_f32(q5_dmin, q8_d);\n\n float32x4_t q5_d = vcvt_f32_f16(vld1_f16((const __fp16 *) (q5_ptr[b].d + j * 4)));\n const float32x4_t scale = vmulq_f32(q5_d, q8_d);\n\n acc_f32[2 * i + j] = vmlsq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(bias_acc[2 * i + j]), dmins);\n acc_f32[2 * i + j] =\n vmlaq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(reorder_acc[2 * i + j]), scale);\n }\n }\n } // for b\n\n // With the previous reorder, the tile is already in the correct memory layout.\n for (int i = 0; i < q8_k_blocklen; i++) {\n int row = y * q8_k_blocklen + i;\n for (int j = 0; j < 2; j++) {\n int col = x * ncols_interleaved + j * 4;\n int offset = row * bs + col;\n vst1q_f32(s + offset, acc_f32[2 * i + j]);\n }\n }\n } // for x\n } // for y\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n ggml_gemm_q5_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q6_K_8x4_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 4;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n constexpr int q8_k_blocklen = 4;\n constexpr int col_groups = ncols_interleaved / 4;\n constexpr int acc_size = q8_k_blocklen * col_groups; // 4 rows, 2 column groups\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n const uint8x16_t mask_lo = vdupq_n_u8(0x03);\n const uint8x16_t mask_hi = vdupq_n_u8(0x30);\n const int8x16_t m32s = vdupq_n_s8(32);\n\n float32x4_t acc_f32[acc_size];\n\n for (int y = 0; y < nr / q8_k_blocklen; y++) {\n const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < acc_size; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n float32x4_t q6_d_0123 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d));\n float32x4_t q6_d_4567 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d + 4));\n float32x4_t q8_d_0123 = vld1q_f32(q8_ptr[b].d);\n\n float32x4_t sbd_scale_0123[q8_k_blocklen];\n float32x4_t sbd_scale_4567[q8_k_blocklen];\n\n sbd_scale_0123[0] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 0);\n sbd_scale_4567[0] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 0);\n sbd_scale_0123[1] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 1);\n sbd_scale_4567[1] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 1);\n sbd_scale_0123[2] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 2);\n sbd_scale_4567[2] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 2);\n sbd_scale_0123[3] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 3);\n sbd_scale_4567[3] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 3);\n\n int32x4_t acc_s32[acc_size];\n for (int i = 0; i < acc_size; i++) {\n acc_s32[i] = vdupq_n_s32(0);\n }\n\n int16_t q6_scales[8 * 16];\n for (int i = 0; i < 16; i++) {\n int16x8_t scales = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));\n vst1q_s16(q6_scales + i * 8, scales);\n }\n\n for (int half = 0; half < 2; half++) {\n const uint8_t * ql_base = q6_ptr[b].ql + half * 512;\n const uint8_t * qh_base = q6_ptr[b].qh + half * 256;\n\n for (int sb = 0; sb < QK_K / 64; sb++) {\n int32x4_t acc_lo[acc_size];\n int32x4_t acc_hi[acc_size];\n for (int i = 0; i < acc_size; i++) {\n acc_lo[i] = vdupq_n_s32(0);\n acc_hi[i] = vdupq_n_s32(0);\n }\n\n const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64;\n const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64;\n\n // 4 rows * 16 elements per scale\n // 4 reads of 16 bytes each\n constexpr int reads_per_sb = 4;\n int8x16_t q8_l[reads_per_sb];\n int8x16_t q8_h[reads_per_sb];\n for (int k = 0; k < reads_per_sb; k++) {\n q8_l[k] = vld1q_s8(q8_base_l + 16 * k);\n q8_h[k] = vld1q_s8(q8_base_h + 16 * k);\n }\n\n const int ql_off_base = sb * QK_K / 2;\n const int qh_off_base = ql_off_base & 255;\n\n uint8x16_t q6_ql_0123[reads_per_sb];\n uint8x16_t q6_ql_4567[reads_per_sb];\n uint8x16_t q6_qh_0123[reads_per_sb];\n uint8x16_t q6_qh_4567[reads_per_sb];\n\n for (int k = 0; k < reads_per_sb; k++) {\n q6_ql_0123[k] = vld1q_u8(ql_base + ql_off_base + k * 32);\n q6_ql_4567[k] = vld1q_u8(ql_base + ql_off_base + k * 32 + 16);\n q6_qh_0123[k] = vld1q_u8(qh_base + qh_off_base + k * 32);\n q6_qh_4567[k] = vld1q_u8(qh_base + qh_off_base + k * 32 + 16);\n }\n\n if (sb > 1) {\n for (int k = 0; k < reads_per_sb; k++) {\n q6_qh_0123[k] = vshrq_n_u8(q6_qh_0123[k], 2);\n q6_qh_4567[k] = vshrq_n_u8(q6_qh_4567[k], 2);\n }\n }\n\n for (int k = 0; k < reads_per_sb; k++) {\n // q = (ql | qh) - 32\n const uint8x16_t hbit_lo_0123 = vandq_u8(q6_qh_0123[k], mask_lo);\n const uint8x16_t hbit_hi_0123 = vandq_u8(q6_qh_0123[k], mask_hi);\n const uint8x16_t hbit_lo_4567 = vandq_u8(q6_qh_4567[k], mask_lo);\n const uint8x16_t hbit_hi_4567 = vandq_u8(q6_qh_4567[k], mask_hi);\n\n const int8x16_t q6_0123_lo = vsubq_s8(\n vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_ql_0123[k], m4b), hbit_lo_0123, 4)), m32s);\n const int8x16_t q6_0123_hi = vsubq_s8(\n vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_ql_0123[k], 4), hbit_hi_0123)), m32s);\n\n acc_lo[0] = vdotq_laneq_s32(acc_lo[0], q6_0123_lo, q8_l[k], 0); // 0..3 r0 c0123\n acc_lo[1] = vdotq_laneq_s32(acc_lo[1], q6_0123_lo, q8_l[k], 1); // 0..3 r1 c0123\n acc_lo[2] = vdotq_laneq_s32(acc_lo[2], q6_0123_lo, q8_l[k], 2); // 0..3 r2 c0123\n acc_lo[3] = vdotq_laneq_s32(acc_lo[3], q6_0123_lo, q8_l[k], 3); // 0..3 r3 c0123\n\n acc_hi[0] = vdotq_laneq_s32(acc_hi[0], q6_0123_hi, q8_h[k], 0); // 64..67 r0 c0123\n acc_hi[1] = vdotq_laneq_s32(acc_hi[1], q6_0123_hi, q8_h[k], 1); // 64..67 r1 c0123\n acc_hi[2] = vdotq_laneq_s32(acc_hi[2], q6_0123_hi, q8_h[k], 2); // 64..67 r2 c0123\n acc_hi[3] = vdotq_laneq_s32(acc_hi[3], q6_0123_hi, q8_h[k], 3); // 64..67 r3 c0123\n\n const int8x16_t q6_4567_lo = vsubq_s8(\n vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_ql_4567[k], m4b), hbit_lo_4567, 4)), m32s);\n const int8x16_t q6_4567_hi = vsubq_s8(\n vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_ql_4567[k], 4), hbit_hi_4567)), m32s);\n\n acc_lo[4] = vdotq_laneq_s32(acc_lo[4], q6_4567_lo, q8_l[k], 0); // 0..3 r0 c4567\n acc_lo[5] = vdotq_laneq_s32(acc_lo[5], q6_4567_lo, q8_l[k], 1); // 0..3 r1 c4567\n acc_lo[6] = vdotq_laneq_s32(acc_lo[6], q6_4567_lo, q8_l[k], 2); // 0..3 r2 c4567\n acc_lo[7] = vdotq_laneq_s32(acc_lo[7], q6_4567_lo, q8_l[k], 3); // 0..3 r3 c4567\n\n acc_hi[4] = vdotq_laneq_s32(acc_hi[4], q6_4567_hi, q8_h[k], 0); // 64..67 r0 c4567\n acc_hi[5] = vdotq_laneq_s32(acc_hi[5], q6_4567_hi, q8_h[k], 1); // 64..67 r1 c4567\n acc_hi[6] = vdotq_laneq_s32(acc_hi[6], q6_4567_hi, q8_h[k], 2); // 64..67 r2 c4567\n acc_hi[7] = vdotq_laneq_s32(acc_hi[7], q6_4567_hi, q8_h[k], 3); // 64..67 r3 c4567\n }\n\n // Scale and bias\n const int scale_idx_l = half * 8 + sb;\n const int scale_idx_h = half * 8 + sb + 4;\n\n for (int g = 0; g < col_groups; g++) {\n const int16x4_t scales_l16 = vld1_s16(q6_scales + scale_idx_l * 8 + g * 4);\n const int16x4_t scales_h16 = vld1_s16(q6_scales + scale_idx_h * 8 + g * 4);\n const int32x4_t scale_vec_l = vmovl_s16(scales_l16);\n const int32x4_t scale_vec_h = vmovl_s16(scales_h16);\n const int acc_offset = g * q8_k_blocklen;\n\n for (int row = 0; row < q8_k_blocklen; row++) {\n const int idx = row * 2 + g;\n acc_s32[idx] = vmlaq_s32(acc_s32[idx], acc_lo[acc_offset + row], scale_vec_l);\n acc_s32[idx] = vmlaq_s32(acc_s32[idx], acc_hi[acc_offset + row], scale_vec_h);\n }\n }\n }\n }\n\n // Finally we apply the superblock scales\n for (int row = 0; row < q8_k_blocklen; row++) {\n const int idx0 = 2 * row;\n const int idx1 = 2 * row + 1;\n const int32x4_t acc_0123 = acc_s32[idx0];\n const int32x4_t acc_4567 = acc_s32[idx1];\n\n acc_f32[idx0] = vmlaq_f32(acc_f32[idx0], vcvtq_f32_s32(acc_0123), sbd_scale_0123[row]);\n acc_f32[idx1] = vmlaq_f32(acc_f32[idx1], vcvtq_f32_s32(acc_4567), sbd_scale_4567[row]);\n }\n } // for b\n\n for (int i = 0; i < q8_k_blocklen; i++) {\n int row = y * q8_k_blocklen + i;\n for (int j = 0; j < 2; j++) {\n int col = x * ncols_interleaved + j * 4;\n int offset = row * bs + col;\n vst1q_f32(s + offset, acc_f32[2 * i + j]);\n }\n }\n } // for x\n } // for y\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemm_q6_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q6_K_8x8_q8_K(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int qk = QK_K;\n const int nb = n / qk;\n\n constexpr int ncols_interleaved = 8;\n constexpr int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n constexpr int q8_k_blocklen = 4;\n const uint8x16_t m4b = vdupq_n_u8(0x0f);\n const uint8x16_t mask_lo = vdupq_n_u8(0x03);\n const uint8x16_t mask_hi = vdupq_n_u8(0x30);\n const int8x16_t m32s = vdupq_n_s8(32);\n\n // 8 accumulators: 4 q8 rows × 2 col groups (0-3, 4-7)\n float32x4_t acc_f32[blocklen];\n\n for (int y = 0; y < nr / q8_k_blocklen; y++) {\n const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n for (int i = 0; i < blocklen; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n int32x4_t acc[8]; // rows 01 stored in [0][1][2][3], rows 23 stored in [4][5][6][7]\n for (int i = 0; i < 8; i++) {\n acc[i] = vdupq_n_s32(0);\n }\n\n // Q6_K has simple 8-bit scales, 16 per block (one per 16 values)\n // Reused for bias and dequantization later\n int16_t q6_scales[16 * 8];\n for (int i = 0; i < 16; ++i) {\n int16x8_t s16 = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));\n vst1q_s16(q6_scales + i * 8, s16);\n }\n\n // Process two 128-value halves per superblock\n for (int half = 0; half < 2; half++) {\n\n const uint8_t * ql_base = q6_ptr[b].ql + half * 512;\n const uint8_t * qh_base = q6_ptr[b].qh + half * 256;\n\n // A subblock (sb) is a set of weights that share the scale\n // Since q6_K scales are per 16 elements\n // num sbs -> 256 elements / (16 elements/scale * 2 elements/byte * 2 halves)\n for (int sb = 0; sb < QK_K / 64; sb++) {\n // Q6_K weight index increasing by 64 instead of 32 requires\n // loading various q8 memory regions\n const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64;\n const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64;\n\n int8x16_t q8_l_01[2];\n int8x16_t q8_l_23[2];\n for (int i = 0; i < 2; i++) {\n const int offset = i * 32;\n q8_l_01[i] = vld1q_s8(q8_base_l + offset); // 0..7 & 8..15 (r01)\n q8_l_23[i] = vld1q_s8(q8_base_l + offset + 16); // 0..7 & 8..15 (r23)\n }\n\n int8x16_t q8_h_01[2];\n int8x16_t q8_h_23[2];\n for (int i = 0; i < 2; i++) {\n const int offset = i * 32;\n q8_h_01[i] = vld1q_s8(q8_base_h + offset);\n q8_h_23[i] = vld1q_s8(q8_base_h + offset + 16);\n }\n\n const int ql_off_base = sb * QK_K / 2;\n\n uint8x16_t q6_ql_0[4];\n uint8x16_t q6_ql_1[4];\n for (int k = 0; k < 4; k++) {\n q6_ql_0[k] = vld1q_u8(ql_base + ql_off_base + 16 * k);\n q6_ql_1[k] = vld1q_u8(ql_base + ql_off_base + 64 + 16 * k);\n }\n\n const int qh_off_base = (sb * QK_K / 2) & 255; // wrap after 256 bytes\n uint8x16_t q6_qh_0[4];\n uint8x16_t q6_qh_1[4];\n for (int k = 0; k < 4; k++) {\n q6_qh_0[k] = vld1q_u8(qh_base + qh_off_base + 16 * k);\n q6_qh_1[k] = vld1q_u8(qh_base + qh_off_base + 64 + 16 * k);\n }\n\n // Adjust for the proper high bits (Sb 2 and 3)\n if (sb > 1) {\n for (int k = 0; k < 4; k++) {\n q6_qh_0[k] = vshrq_n_u8(q6_qh_0[k], 2);\n q6_qh_1[k] = vshrq_n_u8(q6_qh_1[k], 2);\n }\n }\n\n // Process column pairs (0-1, 2-3, 4-5, 6-7)\n for (int cp = 0; cp < ncols_interleaved / 2; cp++) {\n const uint8x16_t q6_qs_cp_0_l = q6_ql_0[cp];\n const uint8x16_t q6_qs_cp_1_l = q6_ql_1[cp];\n const uint8x16_t q6_qs_cp_0_h = q6_qh_0[cp];\n const uint8x16_t q6_qs_cp_1_h = q6_qh_1[cp];\n\n // Extract high 2 bits for upper nibble reconstruction\n const uint8x16_t q6_qs_cp_0_hh = vandq_u8(q6_qs_cp_0_h, mask_hi);\n const uint8x16_t q6_qs_cp_1_hh = vandq_u8(q6_qs_cp_1_h, mask_hi);\n\n // q6 = (low4 | high2<<4) - 32\n // Use vsliq_n_u8 to combine shift-left-insert in one instruction (like Q5_K)\n const int8x16_t q6_l0 = vsubq_s8(\n vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_qs_cp_0_l, m4b), vandq_u8(q6_qs_cp_0_h, mask_lo), 4)),\n m32s);\n const int8x16_t q6_l1 = vsubq_s8(\n vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_qs_cp_1_l, m4b), vandq_u8(q6_qs_cp_1_h, mask_lo), 4)),\n m32s);\n const int8x16_t q6_h0 = vsubq_s8(\n vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_cp_0_l, 4), q6_qs_cp_0_hh)), m32s);\n const int8x16_t q6_h1 = vsubq_s8(\n vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_cp_1_l, 4), q6_qs_cp_1_hh)), m32s);\n\n // row pair 0, base_l\n int32x4_t sb_acc_0l = vmmlaq_s32(vdupq_n_s32(0), q6_l0, q8_l_01[0]);\n sb_acc_0l = vmmlaq_s32(sb_acc_0l, q6_l1, q8_l_01[1]);\n // row pair 0, base_h\n int32x4_t sb_acc_0h = vmmlaq_s32(vdupq_n_s32(0), q6_h0, q8_h_01[0]);\n sb_acc_0h = vmmlaq_s32(sb_acc_0h, q6_h1, q8_h_01[1]);\n // row pair 1, base_l\n int32x4_t sb_acc_1l = vmmlaq_s32(vdupq_n_s32(0), q6_l0, q8_l_23[0]);\n sb_acc_1l = vmmlaq_s32(sb_acc_1l, q6_l1, q8_l_23[1]);\n // row pair 1, base_h\n int32x4_t sb_acc_1h = vmmlaq_s32(vdupq_n_s32(0), q6_h0, q8_h_23[0]);\n sb_acc_1h = vmmlaq_s32(sb_acc_1h, q6_h1, q8_h_23[1]);\n\n const int scale_idx_l = half * 8 + sb;\n const int scale_idx_h = half * 8 + sb + 4;\n\n const int32x4_t scale_vec_l = {\n q6_scales[scale_idx_l * 8 + cp * 2 + 0],\n q6_scales[scale_idx_l * 8 + cp * 2 + 0],\n q6_scales[scale_idx_l * 8 + cp * 2 + 1],\n q6_scales[scale_idx_l * 8 + cp * 2 + 1],\n };\n const int32x4_t scale_vec_h = {\n q6_scales[scale_idx_h * 8 + cp * 2 + 0],\n q6_scales[scale_idx_h * 8 + cp * 2 + 0],\n q6_scales[scale_idx_h * 8 + cp * 2 + 1],\n q6_scales[scale_idx_h * 8 + cp * 2 + 1],\n };\n\n acc[cp] = vmlaq_s32(acc[cp], sb_acc_0l, scale_vec_l);\n acc[cp] = vmlaq_s32(acc[cp], sb_acc_0h, scale_vec_h);\n acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc_1l, scale_vec_l);\n acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc_1h, scale_vec_h);\n }\n }\n } // for half\n\n // Reorder i8mm output to match memory layout\n for (int i = 0; i < 8; i++) {\n int32x2x2_t aux = vzip_s32(vget_low_s32(acc[i]), vget_high_s32(acc[i]));\n acc[i] = vcombine_s32(aux.val[0], aux.val[1]);\n }\n int32x4_t reorder_acc[8] = {\n vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])),\n vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])),\n vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])),\n vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])),\n vcombine_s32(vget_low_s32(acc[4]), vget_low_s32(acc[5])),\n vcombine_s32(vget_low_s32(acc[6]), vget_low_s32(acc[7])),\n vcombine_s32(vget_high_s32(acc[4]), vget_high_s32(acc[5])),\n vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])),\n };\n\n // Apply superblock scale (no mins for q6_K)\n for (int i = 0; i < q8_k_blocklen; i++) {\n for (int j = 0; j < 2; j++) {\n float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d[i]);\n float32x4_t q6_d = vcvt_f32_f16(vld1_f16((const __fp16 *) (q6_ptr[b].d + j * 4)));\n const float32x4_t scale = vmulq_f32(q6_d, q8_d);\n\n acc_f32[2 * i + j] =\n vmlaq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(reorder_acc[2 * i + j]), scale);\n }\n }\n } // for b\n\n // Store results\n for (int i = 0; i < q8_k_blocklen; i++) {\n int row = y * q8_k_blocklen + i;\n for (int j = 0; j < 2; j++) {\n int col = x * ncols_interleaved + j * 4;\n int offset = row * bs + col;\n vst1q_f32(s + offset, acc_f32[2 * i + j]);\n }\n }\n } // for x\n } // for y\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n ggml_gemm_q6_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q8_0_4x4_q8_0(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n\n float32x4_t sumf[4];\n for (int m = 0; m < 4; m++) {\n sumf[m] = vdupq_n_f32(0);\n }\n\n for (int l = 0; l < nb; l++) {\n float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *) a_ptr[l].d));\n float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d));\n\n int32x4_t sumi_0 = vdupq_n_s32(0);\n int32x4_t sumi_1 = vdupq_n_s32(0);\n int32x4_t sumi_2 = vdupq_n_s32(0);\n int32x4_t sumi_3 = vdupq_n_s32(0);\n\n for (int k_group = 0; k_group < 8; k_group += 4) {\n int8x16x4_t a = vld1q_s8_x4(a_ptr[l].qs + 16 * k_group);\n int8x16x4_t b = vld1q_s8_x4(b_ptr[l].qs + 16 * k_group);\n\n for (int k = 0; k < 4; k++) {\n sumi_0 = vdotq_laneq_s32(sumi_0, b.val[k], a.val[k], 0);\n sumi_1 = vdotq_laneq_s32(sumi_1, b.val[k], a.val[k], 1);\n sumi_2 = vdotq_laneq_s32(sumi_2, b.val[k], a.val[k], 2);\n sumi_3 = vdotq_laneq_s32(sumi_3, b.val[k], a.val[k], 3);\n }\n }\n\n sumf[0] = vmlaq_f32(sumf[0], vmulq_laneq_f32(b_d, a_d, 0), vcvtq_f32_s32(sumi_0));\n sumf[1] = vmlaq_f32(sumf[1], vmulq_laneq_f32(b_d, a_d, 1), vcvtq_f32_s32(sumi_1));\n sumf[2] = vmlaq_f32(sumf[2], vmulq_laneq_f32(b_d, a_d, 2), vcvtq_f32_s32(sumi_2));\n sumf[3] = vmlaq_f32(sumf[3], vmulq_laneq_f32(b_d, a_d, 3), vcvtq_f32_s32(sumi_3));\n }\n\n for (int m = 0; m < 4; m++) {\n vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]);\n }\n }\n }\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n ggml_gemm_q8_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q8_0_4x8_q8_0(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n const block_q8_0x4 * b_ptr_base = (const block_q8_0x4 *) vx;\n\n for (int y = 0; y < nr; y += 4) {\n const block_q8_0x4 * a_ptr_base = (const block_q8_0x4 *) vy + (y / 4) * nb;\n\n for (int x = 0; x < nc; x += ncols_interleaved) {\n const block_q8_0x4 * b_ptr = b_ptr_base + (x / 4) * nb;\n const block_q8_0x4 * a_ptr = a_ptr_base;\n\n float32x4_t acc_f32[4];\n for (int i = 0; i < 4; i++) {\n acc_f32[i] = vdupq_n_f32(0);\n }\n\n for (int b = 0; b < nb; b++) {\n int32x4_t acc[4];\n for (int i = 0; i < 4; i++) {\n acc[i] = vdupq_n_s32(0);\n }\n\n // Process 4 chunks of 8 positions each\n for (int chunk = 0; chunk < 4; chunk++) {\n int8x16_t a01 = vld1q_s8(a_ptr->qs + chunk * 32);\n int8x16_t a23 = vld1q_s8(a_ptr->qs + chunk * 32 + 16);\n int8x16_t b01 = vld1q_s8(b_ptr->qs + chunk * 32);\n int8x16_t b23 = vld1q_s8(b_ptr->qs + chunk * 32 + 16);\n\n acc[0] = vmmlaq_s32(acc[0], a01, b01);\n acc[1] = vmmlaq_s32(acc[1], a01, b23);\n acc[2] = vmmlaq_s32(acc[2], a23, b01);\n acc[3] = vmmlaq_s32(acc[3], a23, b23);\n }\n\n // Reorder outputs from 2×2 tiles to row-major\n // acc[0] = [r0c0, r0c1, r1c0, r1c1]\n // acc[1] = [r0c2, r0c3, r1c2, r1c3]\n // acc[2] = [r2c0, r2c1, r3c0, r3c1]\n // acc[3] = [r2c2, r2c3, r3c2, r3c3]\n int32x4_t row0 = vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1]));\n int32x4_t row1 = vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1]));\n int32x4_t row2 = vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3]));\n int32x4_t row3 = vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3]));\n\n // Scales\n float32x4_t a_d = vcvt_f32_f16(vld1_f16((const __fp16 *) a_ptr->d));\n float32x4_t b_d = vcvt_f32_f16(vld1_f16((const __fp16 *) b_ptr->d));\n\n acc_f32[0] = vfmaq_f32(acc_f32[0], vcvtq_f32_s32(row0), vmulq_laneq_f32(b_d, a_d, 0));\n acc_f32[1] = vfmaq_f32(acc_f32[1], vcvtq_f32_s32(row1), vmulq_laneq_f32(b_d, a_d, 1));\n acc_f32[2] = vfmaq_f32(acc_f32[2], vcvtq_f32_s32(row2), vmulq_laneq_f32(b_d, a_d, 2));\n acc_f32[3] = vfmaq_f32(acc_f32[3], vcvtq_f32_s32(row3), vmulq_laneq_f32(b_d, a_d, 3));\n\n a_ptr++;\n b_ptr++;\n }\n\n for (int row = 0; row < 4; row++) {\n vst1q_f32(s + (y + row) * bs + x, acc_f32[row]);\n }\n }\n }\n return;\n#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n ggml_gemm_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n" }, { "path": "src/ggml-cpu/arch/loongarch/quants.c", "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include \n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__loongarch_sx)\n\nstatic __m128i lsx_packs_w(__m128i a, __m128i b) {\n __m128i tmp, tmp1;\n tmp = __lsx_vsat_w(a, 15);\n tmp1 = __lsx_vsat_w(b, 15);\n return __lsx_vpickev_h(tmp1, tmp);\n}\n\nstatic __m128i lsx_packs_h(__m128i a, __m128i b) {\n __m128i tmp, tmp1;\n tmp = __lsx_vsat_h(a, 7);\n tmp1 = __lsx_vsat_h(b, 7);\n return __lsx_vpickev_b(tmp1, tmp);\n}\n\nstatic __m128i lsx_packus_h(__m128i a, __m128i b) {\n __m128i tmp, tmp1;\n tmp = __lsx_vsat_hu(a, 7);\n tmp1 = __lsx_vsat_hu(b, 7);\n return __lsx_vpickev_b(tmp1, tmp);\n}\n\nstatic __m128i lsx_maddubs_h(__m128i a, __m128i b) {\n __m128i tmp1, tmp2;\n tmp1 = __lsx_vmulwev_h_b(a, b);\n tmp2 = __lsx_vmulwod_h_b(a, b);\n return __lsx_vsadd_h(tmp1, tmp2);\n}\n\nstatic __m128i lsx_madd_h(__m128i a, __m128i b) {\n __m128i tmp1, tmp2;\n tmp1 = __lsx_vmulwev_w_h(a, b);\n tmp2 = __lsx_vmulwod_w_h(a, b);\n return __lsx_vadd_w(tmp1, tmp2);\n}\n\nstatic __m128i lsx_set_w(int32_t a, int32_t b, int32_t c, int32_t d) {\n v4i32 __ret = {d, c, b, a};\n return (__m128i)__ret;\n}\n\nstatic __m128i lsx_shuffle_b(__m128i a, __m128i b) {\n __m128i mask_f, zero, tmp0, tmp2, mask;\n int f = 0x8f;\n mask_f = __lsx_vreplgr2vr_b(f);\n zero = __lsx_vldi(0);\n tmp0 = __lsx_vand_v(b, mask_f); // get mask with low 4 bit and sign bits\n tmp0 = __lsx_vori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive\n mask = __lsx_vsle_b(zero, tmp0); // if mask >= 0, set mask\n tmp2 = __lsx_vand_v(tmp0, mask); // maskout the in2 < ones\n return __lsx_vshuf_b(a, zero, tmp2);\n}\n\nstatic __m128i lsx_hadd_h(__m128i a, __m128i b) {\n __m128i tmp1 = __lsx_vpickev_h(b, a);\n __m128i tmp2 = __lsx_vpickod_h(b, a);\n return __lsx_vadd_h(tmp1, tmp2);\n}\n\nstatic __m128i lsx_hadd_w(__m128i a, __m128i b) {\n __m128i tmp1 = __lsx_vpickev_w(b, a);\n __m128i tmp2 = __lsx_vpickod_w(b, a);\n return __lsx_vadd_w(tmp1, tmp2);\n}\n\nstatic __m128 lsx_hadd_s(__m128 a, __m128 b) {\n __m128 tmp1 = (__m128)__lsx_vpickev_w((__m128i)b, (__m128i)a);\n __m128 tmp2 = (__m128)__lsx_vpickod_w((__m128i)b, (__m128i)a);\n\n return __lsx_vfadd_s(tmp1, tmp2);\n}\n\nstatic inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) {\n __m128 res_0 =lsx_hadd_s(a, b);\n __m128 res_1 =lsx_hadd_s(c, d);\n __m128 res =lsx_hadd_s(res_0, res_1);\n res =lsx_hadd_s(res, res);\n res =lsx_hadd_s(res, res);\n\n return ((v4f32)res)[0];\n}\n\n// multiply int8_t, add results pairwise twice\nstatic inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {\n // Get absolute values of x vectors\n const __m128i ax = __lsx_vsigncov_b(x, x);\n // Sign the values of the y vectors\n const __m128i sy = __lsx_vsigncov_b(x, y);\n // Perform multiplication and create 16-bit values\n const __m128i dot = lsx_maddubs_h(ax, sy);\n const __m128i ones = __lsx_vreplgr2vr_h(1);\n return lsx_madd_h(ones, dot);\n}\n#endif\n\n#if defined(__loongarch_asx)\n\n#ifdef __clang__\n#define VREGS_PREFIX \"$vr\"\n#define XREGS_PREFIX \"$xr\"\n#else // GCC\n#define VREGS_PREFIX \"$f\"\n#define XREGS_PREFIX \"$f\"\n#endif\n#define __ALL_REGS \"0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\"\n// Convert __m128i to __m256i\nstatic inline __m256i ____m256i(__m128i in) {\n __m256i out = __lasx_xvldi(0);\n __asm__ volatile (\n \".irp i,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[out], \" XREGS_PREFIX\"\\\\i \\n\\t\"\n \" .irp j,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[in], \" VREGS_PREFIX \"\\\\j \\n\\t\"\n \" xvpermi.q $xr\\\\i, $xr\\\\j, 0x20 \\n\\t\"\n \" .endif \\n\\t\"\n \" .endr \\n\\t\"\n \" .endif \\n\\t\"\n \".endr \\n\\t\"\n : [out] \"+f\" (out) : [in] \"f\" (in)\n );\n return out;\n}\n// Convert two __m128i to __m256i\nstatic inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) {\n __m256i out;\n __asm__ volatile (\n \".irp i,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[hi], \" VREGS_PREFIX \"\\\\i \\n\\t\"\n \" .irp j,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[lo], \" VREGS_PREFIX \"\\\\j \\n\\t\"\n \" xvpermi.q $xr\\\\i, $xr\\\\j, 0x20 \\n\\t\"\n \" .endif \\n\\t\"\n \" .endr \\n\\t\"\n \" .endif \\n\\t\"\n \".endr \\n\\t\"\n \".ifnc %[out], %[hi] \\n\\t\"\n \".irp i,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[out], \" XREGS_PREFIX \"\\\\i \\n\\t\"\n \" .irp j,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[hi], \" VREGS_PREFIX \"\\\\j \\n\\t\"\n \" xvori.b $xr\\\\i, $xr\\\\j, 0 \\n\\t\"\n \" .endif \\n\\t\"\n \" .endr \\n\\t\"\n \" .endif \\n\\t\"\n \".endr \\n\\t\"\n \".endif \\n\\t\"\n : [out] \"=f\" (out), [hi] \"+f\" (inhi)\n : [lo] \"f\" (inlo)\n );\n return out;\n}\n// Convert __m256i low part to __m128i\nstatic inline __m128i lasx_extracti128_lo(__m256i in) {\n __m128i out;\n __asm__ volatile (\n \".ifnc %[out], %[in] \\n\\t\"\n \".irp i,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[out], \" VREGS_PREFIX \"\\\\i \\n\\t\"\n \" .irp j,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[in], \" XREGS_PREFIX \"\\\\j \\n\\t\"\n \" vori.b $vr\\\\i, $vr\\\\j, 0 \\n\\t\"\n \" .endif \\n\\t\"\n \" .endr \\n\\t\"\n \" .endif \\n\\t\"\n \".endr \\n\\t\"\n \".endif \\n\\t\"\n : [out] \"=f\" (out) : [in] \"f\" (in)\n );\n return out;\n}\n// Convert __m256i high part to __m128i\nstatic inline __m128i lasx_extracti128_hi(__m256i in) {\n __m128i out;\n __asm__ volatile (\n \".irp i,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[out], \" VREGS_PREFIX \"\\\\i \\n\\t\"\n \" .irp j,\" __ALL_REGS \"\\n\\t\"\n \" .ifc %[in], \" XREGS_PREFIX \"\\\\j \\n\\t\"\n \" xvpermi.q $xr\\\\i, $xr\\\\j, 0x11 \\n\\t\"\n \" .endif \\n\\t\"\n \" .endr \\n\\t\"\n \" .endif \\n\\t\"\n \".endr \\n\\t\"\n : [out] \"=f\" (out) : [in] \"f\" (in)\n );\n return out;\n}\n\nstatic __m256i lasx_set_w(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0) {\n v8i32 __ret = {e0, e1, e2, e3, e4, e5, e6, e7};\n return (__m256i)__ret;\n}\n\nstatic __m256i lasx_set_d(int64_t a, int64_t b, int64_t c, int64_t d) {\n v4i64 __ret = {d, c, b, a};\n return (__m256i)__ret;\n}\n\nstatic __m256i lasx_insertf128( __m128i x, __m128i y) {\n return lasx_set_q(x, y);\n}\n\nstatic __m256i lasx_shuffle_b(__m256i a, __m256i b) {\n __m256i mask_f, zero, tmp0, tmp2, mask;\n int f = 0x8f;\n mask_f = __lasx_xvreplgr2vr_b(f);\n zero = __lasx_xvldi(0);\n tmp0 = __lasx_xvand_v(b, mask_f); // get mask with low 4 bit and sign bits\n tmp0 = __lasx_xvori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive\n mask = __lasx_xvsle_b(zero, tmp0); // if mask >= 0, set mask\n tmp2 = __lasx_xvand_v(tmp0, mask); // maskout the in2 < ones\n return __lasx_xvshuf_b(a, zero, tmp2);\n}\n\nstatic __m256i lasx_extu8_16(__m128i a) {\n return __lasx_vext2xv_hu_bu(____m256i(a));\n}\n\nstatic __m256i lasx_ext8_16(__m128i a) {\n return __lasx_vext2xv_h_b(____m256i(a));\n}\n\nstatic __m256i lasx_ext16_32(__m128i a) {\n return __lasx_vext2xv_w_h(____m256i(a));\n}\n\nstatic __m128i lasx_extracti128( __m256i a, int pos) {\n __m128i ret;\n if( pos == 0)\n {\n ret = lasx_extracti128_lo(a);\n } else {\n ret = lasx_extracti128_hi(a);\n }\n return ret;\n}\n\nstatic __m128 lasx_extractf128( __m256 a, int pos) {\n __m128 ret;\n if( pos == 0)\n {\n ret = (__m128)lasx_extracti128_lo((__m256i)a);\n } else {\n ret = (__m128)lasx_extracti128_hi((__m256i)a);\n }\n return ret;\n}\n\nstatic __m256i lasx_maddubs_h(__m256i a, __m256i b) {\n __m256i tmp1, tmp2;\n tmp1 = __lasx_xvmulwev_h_b(a, b);\n tmp2 = __lasx_xvmulwod_h_b(a, b);\n return __lasx_xvsadd_h(tmp1, tmp2);\n}\n\nstatic __m256i lasx_madd_h(__m256i a, __m256i b) {\n __m256i tmp1, tmp2;\n tmp1 = __lasx_xvmulwev_w_h(a, b);\n tmp2 = __lasx_xvmulwod_w_h(a, b);\n return __lasx_xvadd_w(tmp1, tmp2);\n}\n\nstatic __m256i lasx_packs_w(__m256i a, __m256i b) {\n __m256i tmp, tmp1;\n tmp = __lasx_xvsat_w(a, 15);\n tmp1 = __lasx_xvsat_w(b, 15);\n return __lasx_xvpickev_h(tmp1, tmp);\n}\n\nstatic __m256i lasx_packs_h(__m256i a, __m256i b) {\n __m256i tmp, tmp1;\n tmp = __lasx_xvsat_h(a, 7);\n tmp1 = __lasx_xvsat_h(b, 7);\n return __lasx_xvpickev_b(tmp1, tmp);\n}\n\nstatic inline __m256i lasx_madd_h_b(__m256i a, __m256i b) {\n __m256i tmp1, tmp2;\n tmp1 = __lasx_xvmulwev_h_b(a, b);\n tmp2 = __lasx_xvmulwod_h_b(a, b);\n return __lasx_xvadd_h(tmp1, tmp2);\n}\n\nstatic inline __m256i lasx_xvrepl128vei_h(__m256i a, const unsigned int b) {\n switch (b) {\n case 0: return __lasx_xvrepl128vei_h(a, 0);\n case 1: return __lasx_xvrepl128vei_h(a, 1);\n case 2: return __lasx_xvrepl128vei_h(a, 2);\n case 3: return __lasx_xvrepl128vei_h(a, 3);\n case 4: return __lasx_xvrepl128vei_h(a, 4);\n case 5: return __lasx_xvrepl128vei_h(a, 5);\n case 6: return __lasx_xvrepl128vei_h(a, 6);\n case 7: return __lasx_xvrepl128vei_h(a, 7);\n default: __builtin_unreachable();\n }\n}\n\nstatic inline __m256i lasx_xvandi_b_bit(__m256i a, const unsigned int b) {\n switch (b) {\n case 0: return __lasx_xvandi_b(a, 1 << 0);\n case 1: return __lasx_xvandi_b(a, 1 << 1);\n case 2: return __lasx_xvandi_b(a, 1 << 2);\n case 3: return __lasx_xvandi_b(a, 1 << 3);\n case 4: return __lasx_xvandi_b(a, 1 << 4);\n case 5: return __lasx_xvandi_b(a, 1 << 5);\n case 6: return __lasx_xvandi_b(a, 1 << 6);\n case 7: return __lasx_xvandi_b(a, 1 << 7);\n default: __builtin_unreachable();\n }\n}\n\n// horizontally add 8 floats\nstatic inline float hsum_float_8(const __m256 x) {\n __m128 res = lasx_extractf128(x, 1);\n res = __lsx_vfadd_s(res, lasx_extractf128(x, 0));\n res = __lsx_vfadd_s(res, (__m128)__lsx_vpickod_d((__m128i)res, (__m128i)res));\n res = __lsx_vfadd_s(res, (__m128)__lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w(res, 1), 0));\n return ((v4f32)res)[0];\n}\n\n// horizontally add 8 int32_t\nstatic inline int hsum_i32_8(const __m256i a) {\n\n __m256i tmp1 = __lasx_xvpermi_q(a, a, 0x11);\n __m256i tmp2 = __lasx_xvpermi_q(a, a, 0x00);\n\n __m128i tmp1_128 = lasx_extracti128_lo(tmp1);\n __m128i tmp2_128 = lasx_extracti128_lo(tmp2);\n\n __m128i sum128 = __lsx_vadd_w(tmp1_128, tmp2_128);\n\n __m128i ev = __lsx_vpickev_w(sum128, sum128);\n __m128i od = __lsx_vpickod_w(sum128, sum128);\n __m128i sum64 = __lsx_vadd_w(ev, od);\n\n int sum64_1, sum64_2;\n sum64_1 = __lsx_vpickve2gr_w(sum64, 0);\n sum64_2 = __lsx_vpickve2gr_w(sum64, 1);\n\n return sum64_1 + sum64_2;\n}\n\n// horizontally add 4 int32_t\nstatic inline int hsum_i32_4(const __m128i a) {\n __m128i ev = __lsx_vpickev_w(a, a);\n __m128i od = __lsx_vpickod_w(a, a);\n __m128i sum64 = __lsx_vadd_w(ev, od);\n\n int sum64_1, sum64_2;\n sum64_1 = __lsx_vpickve2gr_w(sum64, 0);\n sum64_2 = __lsx_vpickve2gr_w(sum64, 1);\n\n return sum64_1 + sum64_2;\n}\n\n// spread 32 bits to 32 bytes { 0x00, 0xFF }\nstatic inline __m256i bytes_from_bits_32(const uint8_t * x) {\n\n uint32_t x32;\n memcpy(&x32, x, sizeof(uint32_t));\n const __m256i shuf_mask = lasx_set_d(\n 0x0303030303030303, 0x0202020202020202,\n 0x0101010101010101, 0x0000000000000000);\n\n __m256i bytes = lasx_shuffle_b(__lasx_xvreplgr2vr_w(x32), shuf_mask);\n const __m256i bit_mask = __lasx_xvreplgr2vr_d(0x7fbfdfeff7fbfdfe);\n bytes = __lasx_xvor_v(bytes, bit_mask);\n return __lasx_xvseq_b(bytes, __lasx_xvreplgr2vr_d(-1));\n}\n\n// Unpack 32 4-bit fields into 32 bytes\n// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval\nstatic inline __m256i bytes_from_nibbles_32(const uint8_t * rsi) {\n const __m128i lo = __lsx_vld((const __m128i *)rsi, 0);\n __m128i hi = __lsx_vsrli_h(lo, 4);\n return __lasx_xvandi_b(lasx_insertf128(hi, lo), 0xf);\n}\n\n// add int16_t pairwise and return as float vector\nstatic inline __m256 sum_i16_pairs_float(const __m256i x) {\n __m256i v = __lasx_xvpackod_h(x, x);\n __m256i summed_pairs = __lasx_xvaddwev_w_h(x, v);\n return __lasx_xvffint_s_w(summed_pairs);\n}\n\nstatic inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {\n // Perform multiplication and create 16-bit values\n const __m256i dot = lasx_maddubs_h(ax, sy);\n return sum_i16_pairs_float(dot);\n}\n\n// multiply int8_t, add results pairwise twice and return as float vector\nstatic inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {\n const __m256i dot = lasx_madd_h_b(x, y);\n return sum_i16_pairs_float(dot);\n}\n\nstatic inline __m128i packNibbles( __m256i bytes ) {\n // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh\n const __m256i lowByte = __lasx_xvreplgr2vr_h(0xFF);\n __m256i high = __lasx_xvandn_v(lowByte, bytes);\n __m256i low = __lasx_xvand_v(lowByte, bytes);\n high = __lasx_xvsrli_h(high, 4);\n bytes = __lasx_xvor_v(low, high);\n // Compress uint16_t lanes into bytes\n __m128i *r0 = (__m128i *)&bytes;\n __m256i tmp_h128 = __lasx_xvpermi_q(bytes, bytes, 0x11);\n __m128i *r1 = (__m128i *)&tmp_h128;\n\n __m128i zero = __lsx_vldi(0);\n __m128i tmp, tmp2, tmp3;\n\n tmp = __lsx_vmax_h(zero, *r0);\n tmp2 = __lsx_vsat_hu(tmp, 7);\n\n tmp = __lsx_vmax_h(zero, *r1);\n tmp3 = __lsx_vsat_hu(tmp, 7);\n return __lsx_vpickev_b(tmp3, tmp2);\n}\n#endif //__loongarch_asx\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__loongarch_asx)\n for (int i = 0; i < nb; i++) {\n __m256 v0 = (__m256)__lasx_xvld( x , 0);\n __m256 v1 = (__m256)__lasx_xvld( x , 32);\n __m256 v2 = (__m256)__lasx_xvld( x , 64);\n __m256 v3 = (__m256)__lasx_xvld( x , 96);\n x += 32;\n\n // Compute max(abs(e)) for the block\n const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );\n __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );\n max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );\n max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );\n max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );\n\n __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs , 0) );\n max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );\n __m128 tmp = max4;\n max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vinsgr2vr_w(tmp, __lsx_vpickve2gr_w( max4, 1 ), 0 ));\n const float max_scalar = ((v4f32)max4)[0];\n\n // Quantize these floats\n const float d = max_scalar / 127.f;\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;\n const __m256 mul = (__m256)__lasx_xvreplfr2vr_s( id );\n\n // Apply the multiplier\n v0 = __lasx_xvfmul_s( v0, mul );\n v1 = __lasx_xvfmul_s( v1, mul );\n v2 = __lasx_xvfmul_s( v2, mul );\n v3 = __lasx_xvfmul_s( v3, mul );\n\n // Round to nearest integer\n __m256i i0 = __lasx_xvftintrne_w_s( v0 );\n __m256i i1 = __lasx_xvftintrne_w_s( v1 );\n __m256i i2 = __lasx_xvftintrne_w_s( v2 );\n __m256i i3 = __lasx_xvftintrne_w_s( v3 );\n\n __m128i ni0 = lasx_extracti128( i0, 0 );\n __m128i ni1 = lasx_extracti128( i0, 1);\n __m128i ni2 = lasx_extracti128( i1, 0);\n __m128i ni3 = lasx_extracti128( i1, 1);\n __m128i ni4 = lasx_extracti128( i2, 0);\n __m128i ni5 = lasx_extracti128( i2, 1);\n __m128i ni6 = lasx_extracti128( i3, 0);\n __m128i ni7 = lasx_extracti128( i3, 1);\n\n // Convert int32 to int16\n ni0 = lsx_packs_w( ni0, ni1 );\n ni2 = lsx_packs_w( ni2, ni3 );\n ni4 = lsx_packs_w( ni4, ni5 );\n ni6 = lsx_packs_w( ni6, ni7 );\n // Convert int16 to int8\n ni0 = lsx_packs_h( ni0, ni2 );\n ni4 = lsx_packs_h( ni4, ni6 );\n\n __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0);\n __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);\n\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK8_1 == 0);\n const int nb = k / QK8_1;\n\n block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__loongarch_asx)\n for (int i = 0; i < nb; i++) {\n __m256 v0 = (__m256)__lasx_xvld( x , 0 );\n __m256 v1 = (__m256)__lasx_xvld( x , 32 );\n __m256 v2 = (__m256)__lasx_xvld( x , 64 );\n __m256 v3 = (__m256)__lasx_xvld( x , 96 );\n x += 32;\n\n // Compute max(abs(e)) for the block\n const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );\n __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );\n max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );\n max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );\n max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );\n\n __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs, 0) );\n max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );\n __m128 tmp = max4;\n max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vextrins_w((__m128i)tmp, (__m128i)max4, 0x1 ));\n const float max_scalar = ((v4f32)max4)[0];\n\n // Quantize these floats\n const float d = max_scalar / 127.f;\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;\n const __m256 mul = __lasx_xvreplfr2vr_s( id );\n\n // Apply the multiplier\n v0 = __lasx_xvfmul_s( v0, mul );\n v1 = __lasx_xvfmul_s( v1, mul );\n v2 = __lasx_xvfmul_s( v2, mul );\n v3 = __lasx_xvfmul_s( v3, mul );\n\n // Round to nearest integer\n __m256i i0 = __lasx_xvftintrne_w_s( v0 );\n __m256i i1 = __lasx_xvftintrne_w_s( v1 );\n __m256i i2 = __lasx_xvftintrne_w_s( v2 );\n __m256i i3 = __lasx_xvftintrne_w_s( v3 );\n\n __m128i ni0 = lasx_extracti128(i0, 0);\n __m128i ni1 = lasx_extracti128( i0, 1);\n __m128i ni2 = lasx_extracti128( i1, 0);\n __m128i ni3 = lasx_extracti128( i1, 1);\n __m128i ni4 = lasx_extracti128( i2, 0 );\n __m128i ni5 = lasx_extracti128( i2, 1);\n __m128i ni6 = lasx_extracti128( i3, 0);\n __m128i ni7 = lasx_extracti128( i3, 1);\n\n // Compute the sum of the quants and set y[i].s\n const __m128i s0 = __lsx_vadd_w(__lsx_vadd_w(ni0, ni1), __lsx_vadd_w(ni2, ni3));\n const __m128i s1 = __lsx_vadd_w(__lsx_vadd_w(ni4, ni5), __lsx_vadd_w(ni6, ni7));\n y[i].s = GGML_CPU_FP32_TO_FP16(d * hsum_i32_4(__lsx_vadd_w(s0, s1)));\n\n // Convert int32 to int16\n ni0 = lsx_packs_w( ni0, ni1 );\n ni2 = lsx_packs_w( ni2, ni3 );\n ni4 = lsx_packs_w( ni4, ni5 );\n ni6 = lsx_packs_w( ni6, ni7 );\n // Convert int16 to int8\n ni0 = lsx_packs_h( ni0, ni2 );\n ni4 = lsx_packs_h( ni4, ni6 );\n\n __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0);\n __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n\n//===================================== Dot products =================================\n\n//\n// Helper functions\n//\n\n#if defined(__loongarch_asx)\n// shuffles to pick the required scales in dot products\nstatic inline __m256i get_scale_shuffle_q3k(int i) {\n static const uint8_t k_shuffle[128] = {\n 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,\n 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,\n 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,\n };\n return __lasx_xvld((const __m256i*)k_shuffle + i, 0);\n}\nstatic inline __m256i get_scale_shuffle_k4(int i) {\n static const uint8_t k_shuffle[256] = {\n 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,\n 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,\n 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,\n 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,\n 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,\n 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15\n };\n return __lasx_xvld((const __m256i*)k_shuffle + i, 0);\n}\nstatic inline __m128i get_scale_shuffle(int i) {\n static const uint8_t k_shuffle[128] = {\n 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,\n 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,\n 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,\n 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,\n 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11,\n 12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13,\n 14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15\n };\n return __lsx_vld((const __m128i*)k_shuffle + i, 0);\n}\n#endif\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__loongarch_asx)\n // Initialize accumulator with zeros\n __m256 acc = (__m256)__lasx_xvldi(0);\n\n // Main loop\n for (; ib < nb; ++ib) {\n /* Compute combined scale for the block */\n const __m256 d = __lasx_xvreplfr2vr_s( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );\n\n __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n\n // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.\n const __m256i off = __lasx_xvreplgr2vr_b( 8 );\n qx = __lasx_xvsub_b( qx, off );\n\n __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);\n\n const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n /* Multiply q with scale and accumulate */\n acc = __lasx_xvfmadd_s( d, q, acc );\n }\n\n sumf = hsum_float_8(acc);\n\n#elif defined(__loongarch_sx)\n // set constants\n const __m128i low_mask = __lsx_vreplgr2vr_b(0xF);\n const __m128i off = __lsx_vreplgr2vr_b(8);\n\n // Initialize accumulator with zeros\n __m128 acc_0 = (__m128)__lsx_vldi(0);\n __m128 acc_1 = (__m128)__lsx_vldi(0);\n __m128 acc_2 = (__m128)__lsx_vldi(0);\n __m128 acc_3 = (__m128)__lsx_vldi(0);\n\n for (; ib + 1 < nb; ib += 2) {\n\n // Compute combined scale for the block 0 and 1\n const float ft0 = GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d);\n const __m128 d_0_1 = (__m128)(v4f32){ft0, ft0, ft0, ft0};\n\n const __m128i tmp_0_1 = __lsx_vld((const __m128i *)x[ib].qs, 0);\n\n __m128i bx_0 = __lsx_vand_v(low_mask, tmp_0_1);\n __m128i by_0 = __lsx_vld((const __m128i *)y[ib].qs, 0);\n bx_0 = __lsx_vsub_b(bx_0, off);\n const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);\n\n __m128i bx_1 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_0_1, 4));\n __m128i by_1 = __lsx_vld((const __m128i *)(y[ib].qs + 16), 0);\n bx_1 = __lsx_vsub_b(bx_1, off);\n const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);\n\n // Compute combined scale for the block 2 and 3\n const float ft1 = GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d);\n const __m128 d_2_3 = (__m128)(v4f32){ft1, ft1, ft1, ft1};\n\n const __m128i tmp_2_3 = __lsx_vld((const __m128i *)x[ib + 1].qs, 0);\n\n __m128i bx_2 = __lsx_vand_v(low_mask, tmp_2_3);\n __m128i by_2 = __lsx_vld((const __m128i *)y[ib + 1].qs, 0);\n bx_2 = __lsx_vsub_b(bx_2, off);\n const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);\n\n __m128i bx_3 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_2_3, 4));\n __m128i by_3 = __lsx_vld((const __m128i *)(y[ib + 1].qs + 16), 0);\n bx_3 = __lsx_vsub_b(bx_3, off);\n const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);\n\n // Convert int32_t to float\n __m128 p0 = __lsx_vffint_s_w(i32_0);\n __m128 p1 = __lsx_vffint_s_w(i32_1);\n __m128 p2 = __lsx_vffint_s_w(i32_2);\n __m128 p3 = __lsx_vffint_s_w(i32_3);\n\n // Apply the scale\n __m128 p0_d = __lsx_vfmul_s( d_0_1, p0 );\n __m128 p1_d = __lsx_vfmul_s( d_0_1, p1 );\n __m128 p2_d = __lsx_vfmul_s( d_2_3, p2 );\n __m128 p3_d = __lsx_vfmul_s( d_2_3, p3 );\n\n // Acummulate\n acc_0 = __lsx_vfadd_s(p0_d, acc_0);\n acc_1 = __lsx_vfadd_s(p1_d, acc_1);\n acc_2 = __lsx_vfadd_s(p2_d, acc_2);\n acc_3 = __lsx_vfadd_s(p3_d, acc_3);\n }\n\n sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);\n\n#endif\n for (; ib < nb; ++ib) {\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n const int v1 = (x[ib].qs[j] >> 4) - 8;\n\n sumi0 += (v0 * y[ib].qs[j]);\n sumi1 += (v1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__loongarch_asx)\n // Initialize accumulator with zeros\n __m256 acc = (__m256)__lasx_xvldi(0);\n\n float summs = 0;\n\n // Main loop\n for (; ib < nb; ++ib) {\n const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);\n const float d1 = GGML_CPU_FP16_TO_FP32(y[ib].d);\n\n summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n const __m256 d0v = __lasx_xvreplfr2vr_s( d0 );\n const __m256 d1v = __lasx_xvreplfr2vr_s( d1 );\n\n // Compute combined scales\n const __m256 d0d1 = __lasx_xvfmul_s( d0v, d1v );\n\n // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes\n const __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n const __m256i qy = __lasx_xvld( (const __m256i *)y[ib].qs, 0);\n\n const __m256 xy = mul_sum_us8_pairs_float(qx, qy);\n\n // Accumulate d0*d1*x*y\n acc = __lasx_xvfmadd_s( d0d1, xy, acc );\n }\n\n sumf = hsum_float_8(acc) + summs;\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__loongarch_asx)\n // Initialize accumulator with zeros\n __m256 acc = (__m256)__lasx_xvldi(0);\n\n // Main loop\n for (; ib < nb; ++ib) {\n /* Compute combined scale for the block */\n const __m256 d = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d)); //FIXME\n\n __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n bxhi = __lasx_xvandn_v(bxhi, __lasx_xvreplgr2vr_b((char)0xF0));\n qx = __lasx_xvor_v(qx, bxhi);\n\n __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);\n\n const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n /* Multiply q with scale and accumulate */\n acc = __lasx_xvfmadd_s(d, q, acc);\n }\n\n sumf = hsum_float_8(acc);\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(sumf);\n UNUSED(x);\n UNUSED(y);\n ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_1);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__loongarch_asx)\n // Initialize accumulator with zeros\n __m256 acc = (__m256)__lasx_xvldi(0);\n\n float summs = 0.0f;\n\n // Main loop\n for (; ib < nb; ++ib) {\n const __m256 dx = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ib].d));\n\n summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n bxhi = __lasx_xvand_v(bxhi, __lasx_xvreplgr2vr_b(0x10));\n qx = __lasx_xvor_v(qx, bxhi);\n\n const __m256 dy = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(y[ib].d));\n const __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);\n\n const __m256 q = mul_sum_us8_pairs_float(qx, qy);\n\n acc = __lasx_xvfmadd_s(q, __lasx_xvfmul_s(dx, dy), acc);\n }\n\n sumf = hsum_float_8(acc) + summs;\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(sumf);\n UNUSED(x);\n UNUSED(y);\n ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q8_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__loongarch_asx)\n // Initialize accumulator with zeros\n __m256 acc = (__m256)__lasx_xvldi(0);\n\n // Main loop\n for (; ib < nb; ++ib) {\n // Compute combined scale for the block\n const __m256 d = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n __m256i qx = __lasx_xvld((const __m256i *)x[ib].qs, 0);\n __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);\n\n const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n // Multiply q with scale and accumulate\n acc = __lasx_xvfmadd_s( d, q, acc );\n }\n\n sumf = hsum_float_8(acc);\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(sumf);\n UNUSED(x);\n UNUSED(y);\n ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q2_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __loongarch_asx\n\n __m256 acc = (__m256)__lasx_xvldi(0);\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const __m128i mins_and_scales128 = __lsx_vld((const __m128i*)x[i].scales, 0);\n const __m128i scales128 = __lsx_vandi_b(mins_and_scales128, 0xf);\n const __m256i mins = lasx_ext8_16(__lsx_vsrli_b(mins_and_scales128, 4));\n const __m256i prod = lasx_madd_h(mins, __lasx_xvld((const __m256i*)y[i].bsums, 0));\n\n acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(dmin), __lasx_xvffint_s_w(prod), acc);\n\n const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};\n const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask));\n\n __m256i sumi = __lasx_xvldi(0);\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n const __m256i q2bits = __lasx_xvld((const __m256i*)q2, 0); q2 += 32;\n\n const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n const __m256i q2_0 = __lasx_xvandi_b(q2bits, 3);\n const __m256i q2_1 = __lasx_xvandi_b(__lasx_xvsrli_b(q2bits, 2), 3);\n const __m256i q2_2 = __lasx_xvandi_b(__lasx_xvsrli_b(q2bits, 4), 3);\n const __m256i q2_3 = __lasx_xvsrli_b(q2bits, 6);\n\n __m256i p0 = lasx_madd_h_b(q2_0, q8_0);\n __m256i p1 = lasx_madd_h_b(q2_1, q8_1);\n __m256i p2 = lasx_madd_h_b(q2_2, q8_2);\n __m256i p3 = lasx_madd_h_b(q2_3, q8_3);\n\n p0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p0);\n p1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p1);\n p2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p2);\n p3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p3);\n\n p0 = __lasx_xvadd_w(p0, p1);\n p2 = __lasx_xvadd_w(p2, p3);\n\n sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p0, p2));\n }\n\n acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);\n\n }\n\n *s = hsum_float_8(acc);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const uint32_t kmask1 = 0x03030303;\n const uint32_t kmask2 = 0x0f0f0f0f;\n\n const block_q3_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __loongarch_asx\n\n const __m128i m32 = __lsx_vreplgr2vr_b(32);\n\n __m256 acc = (__m256)__lasx_xvldi(0);\n\n uint32_t aux[3];\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n // Set up scales\n memcpy(aux, x[i].scales, 12);\n __m128i scales128 = lsx_set_w(\n ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),\n ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),\n (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),\n (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));\n scales128 = __lsx_vsub_b(scales128, m32);\n\n const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};\n const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask));\n\n // high bit\n const __m256i hbits = __lasx_xvld((const __m256i*)x[i].hmask, 0);\n\n // integer accumulator\n __m256i sumi = __lasx_xvldi(0);\n\n for (int j = 0; j < QK_K/128; ++j) {\n // load low 2 bits\n const __m256i q3bits = __lasx_xvld((const __m256i*)q3, 0); q3 += 32;\n\n // prepare low and high bits\n const __m256i q3l_0 = __lasx_xvandi_b(q3bits, 3);\n const __m256i q3l_1 = __lasx_xvandi_b(__lasx_xvsrli_b(q3bits, 2), 3);\n const __m256i q3l_2 = __lasx_xvandi_b(__lasx_xvsrli_b(q3bits, 4), 3);\n const __m256i q3l_3 = __lasx_xvsrli_b(q3bits, 6);\n const __m256i q3h_0 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 0), 0), 2);\n const __m256i q3h_1 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 1), 0), 2);\n const __m256i q3h_2 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 2), 0), 2);\n const __m256i q3h_3 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 3), 0), 2);\n const __m256i q3_0 = __lasx_xvor_v(q3h_0, q3l_0);\n const __m256i q3_1 = __lasx_xvor_v(q3h_1, q3l_1);\n const __m256i q3_2 = __lasx_xvor_v(q3h_2, q3l_2);\n const __m256i q3_3 = __lasx_xvor_v(q3h_3, q3l_3);\n\n // load Q8 quants\n const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n __m256i p16_0 = lasx_madd_h_b(q8_0, q3_0);\n __m256i p16_1 = lasx_madd_h_b(q8_1, q3_1);\n __m256i p16_2 = lasx_madd_h_b(q8_2, q3_2);\n __m256i p16_3 = lasx_madd_h_b(q8_3, q3_3);\n\n // multiply with scales\n p16_0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p16_0);\n p16_1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p16_1);\n p16_2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p16_2);\n p16_3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p16_3);\n\n // accumulate\n p16_0 = __lasx_xvadd_w(p16_0, p16_1);\n p16_2 = __lasx_xvadd_w(p16_2, p16_3);\n sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_2));\n }\n // multiply with block scale and accumulate\n acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);\n }\n\n *s = hsum_float_8(acc);\n\n#else\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined __loongarch_asx\n\n __m256 acc = (__m256)__lasx_xvldi(0);\n __m128 acc_m = (__m128)__lsx_vldi(0);\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const __m128i mins_and_scales128 = lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]);\n const __m128i mins128 = __lsx_vexth_h_b(mins_and_scales128);\n const __m128i scales128 = __lsx_vsllwil_h_b(mins_and_scales128, 0);\n\n const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);\n const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));\n const __m128i prod = lsx_madd_h(mins128, q8s);\n acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m);\n\n const __m256i scales = lasx_insertf128(scales128, scales128);\n\n __m256i sumi = __lasx_xvldi(0);\n\n for (int j = 0; j < QK_K/64; ++j) {\n\n const __m256i scale_l = lasx_xvrepl128vei_h(scales, 2 * j + 0);\n const __m256i scale_h = lasx_xvrepl128vei_h(scales, 2 * j + 1);\n\n const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;\n const __m256i q4l = __lasx_xvandi_b(q4bits, 0xf);\n const __m256i q4h = __lasx_xvsrli_b(q4bits, 4);\n\n const __m256i q8l = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n __m256i p16l = lasx_madd_h_b(q4l, q8l);\n p16l = lasx_madd_h(scale_l, p16l);\n\n const __m256i q8h = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n __m256i p16h = lasx_madd_h_b(q4h, q8h);\n p16h = lasx_madd_h(scale_h, p16h);\n const __m256i sumj = __lasx_xvadd_w(p16l, p16h);\n\n sumi = __lasx_xvadd_w(sumi, sumj);\n }\n\n __m256 vd = __lasx_xvreplfr2vr_s(d);\n acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);\n\n }\n\n acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vpermi_w((__m128i)acc_m, (__m128i)acc_m, 0xee));\n __m128i tmp1 = __lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w((__m128i)acc_m, 1), 0);\n acc_m = __lsx_vfadd_s(acc_m, (__m128)tmp1);\n\n\n *s = hsum_float_8(acc) + ((v4f32)acc_m)[0];\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined __loongarch_asx\n\n __m256 acc = (__m256)__lasx_xvldi(0);\n __m128 acc_m = (__m128)__lsx_vldi(0);\n\n for (int i = 0; i < nb; ++i) {\n\n const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n const __m128i mins_and_scales128 = lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]);\n const __m128i mins128 = __lsx_vexth_h_b(mins_and_scales128);\n const __m128i scales128 = __lsx_vsllwil_h_b(mins_and_scales128, 0);\n\n const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);\n const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));\n const __m128i prod = lsx_madd_h(mins128, q8s);\n acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m);\n\n const __m256i scales = lasx_insertf128(scales128, scales128);\n\n const __m256i hbits = __lasx_xvld((const __m256i*)x[i].qh, 0);\n\n __m256i sumi = __lasx_xvldi(0);\n\n for (int j = 0; j < QK_K/64; ++j) {\n\n const __m256i scale_0 = lasx_xvrepl128vei_h(scales, 2 * j + 0);\n const __m256i scale_1 = lasx_xvrepl128vei_h(scales, 2 * j + 1);\n\n const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0); q5 += 32;\n\n const __m256i q5l_0 = __lasx_xvandi_b(q5bits, 0xf);\n const __m256i q5l_1 = __lasx_xvsrli_b(q5bits, 4);\n const __m256i q5h_0 = __lasx_xvnori_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 2 * j + 0), 0), 0xef);\n const __m256i q5h_1 = __lasx_xvnori_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 2 * j + 1), 0), 0xef);\n const __m256i q5_0 = __lasx_xvor_v(q5l_0, q5h_0);\n const __m256i q5_1 = __lasx_xvor_v(q5l_1, q5h_1);\n\n const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n __m256i p16_0 = lasx_madd_h_b(q5_0, q8_0);\n __m256i p16_1 = lasx_madd_h_b(q5_1, q8_1);\n\n p16_0 = lasx_madd_h(scale_0, p16_0);\n p16_1 = lasx_madd_h(scale_1, p16_1);\n\n sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));\n\n }\n\n __m256 vd = __lasx_xvreplfr2vr_s(d);\n acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);\n\n }\n\n acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vbsrl_v(acc_m, 8));\n acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vbsrl_v(acc_m, 4));\n\n *s = hsum_float_8(acc) + ((v4f32)acc_m)[0];\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q6_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __loongarch_asx\n\n const __m256i m32s = __lasx_xvreplgr2vr_b(32);\n\n __m256 acc = (__m256)__lasx_xvldi(0);\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const __m128i scales128 = __lsx_vld((const __m128i*)x[i].scales, 0);\n const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};\n const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask));\n\n __m256i sumi = __lasx_xvldi(0);\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;\n const __m256i q4bits2 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;\n const __m256i q4bitsH = __lasx_xvld((const __m256i*)qh, 0); qh += 32;\n\n const __m256i q4h_0 = __lasx_xvslli_b(__lasx_xvandi_b(q4bitsH, 3), 4);\n const __m256i q4h_1 = __lasx_xvslli_b(__lasx_xvandi_b(q4bitsH, 3 << 2), 2);\n const __m256i q4h_2 = __lasx_xvandi_b(q4bitsH, 3 << 4);\n const __m256i q4h_3 = __lasx_xvsrli_b(__lasx_xvandi_b(q4bitsH, 3 << 6), 2);\n\n const __m256i q4_0 = __lasx_xvor_v(__lasx_xvandi_b(q4bits1, 0xf), q4h_0);\n const __m256i q4_1 = __lasx_xvor_v(__lasx_xvandi_b(q4bits2, 0xf), q4h_1);\n const __m256i q4_2 = __lasx_xvor_v(__lasx_xvsrli_b(q4bits1, 4), q4h_2);\n const __m256i q4_3 = __lasx_xvor_v(__lasx_xvsrli_b(q4bits2, 4), q4h_3);\n\n const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n __m256i p16_0 = lasx_madd_h_b(__lasx_xvsub_b(q4_0, m32s), q8_0);\n __m256i p16_1 = lasx_madd_h_b(__lasx_xvsub_b(q4_1, m32s), q8_1);\n __m256i p16_2 = lasx_madd_h_b(__lasx_xvsub_b(q4_2, m32s), q8_2);\n __m256i p16_3 = lasx_madd_h_b(__lasx_xvsub_b(q4_3, m32s), q8_3);\n\n p16_0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p16_0);\n p16_1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p16_1);\n p16_2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p16_2);\n p16_3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p16_3);\n\n sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));\n sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_2, p16_3));\n }\n\n acc = __lasx_xvfmadd_s((__m256)__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);\n }\n\n *s = hsum_float_8(acc);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined(__loongarch_asx)\nstatic const int8_t keven_signs_q2xs[1024] = {\n 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,\n 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,\n 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,\n 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,\n 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,\n 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,\n 1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, 1,\n 1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, -1,\n 1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, -1,\n 1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, 1,\n 1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, 1,\n 1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1,\n 1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, 1,\n 1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, -1,\n 1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, -1,\n 1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1,\n 1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, -1,\n 1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1,\n 1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, 1,\n 1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, -1,\n 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1,\n 1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, -1,\n 1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, -1,\n 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1,\n 1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, 1,\n 1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, -1,\n 1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1,\n 1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, 1,\n 1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1,\n 1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, 1,\n 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, 1,\n 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n uint32_t aux32[4];\n const uint8_t * aux8 = (const uint8_t *)aux32;\n\n __m256 accumf = (__m256)__lasx_xvldi(0);\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n __m256i sumi1 = __lasx_xvldi(0);\n __m256i sumi2 = __lasx_xvldi(0);\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;\n\n const __m256i q2_1 = lasx_set_d(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);\n const __m256i q2_2 = lasx_set_d(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);\n const __m256i s2_1 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],\n signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);\n const __m256i s2_2 = lasx_set_d(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],\n signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);\n const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);\n const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);\n const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);\n const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);\n const uint16_t ls1 = aux32[1] >> 28;\n const uint16_t ls2 = aux32[3] >> 28;\n const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));\n const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));\n sumi1 = __lasx_xvadd_w(sumi1, p1);\n sumi2 = __lasx_xvadd_w(sumi2, p2);\n }\n\n accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n }\n\n *s = 0.125f * hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n const __m256i mone = __lasx_xvreplgr2vr_b(1);\n static const char block_sign_shuffle_mask_1[32] = {\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,\n 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,\n };\n static const char block_sign_shuffle_mask_2[32] = {\n 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,\n 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,\n };\n static const uint8_t bit_selector_mask_bytes[32] = {\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n };\n\n const __m256i bit_selector_mask = __lasx_xvld((const __m256i*)bit_selector_mask_bytes, 0);\n const __m256i block_sign_shuffle_1 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_1, 0);\n const __m256i block_sign_shuffle_2 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_2, 0);\n\n static const uint8_t k_bit_helper[32] = {\n 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n };\n const __m256i bit_helper = __lasx_xvld((const __m256i*)k_bit_helper, 0);\n const __m256i m511 = __lasx_xvreplgr2vr_h(511);\n const __m128i m4 = __lsx_vreplgr2vr_b(0xf);\n const __m128i m1 = __lsx_vreplgr2vr_b(1);\n\n uint64_t aux64;\n\n // somewhat hacky, but gives a significant boost in performance\n __m256i aux_gindex;\n const uint16_t * gindex = (const uint16_t *)&aux_gindex;\n\n __m256 accumf = (__m256)__lasx_xvldi(0);\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n memcpy(&aux64, x[i].scales, 8);\n __m128i stmp = __lsx_vreplgr2vr_d(aux64);\n stmp = __lsx_vilvl_b( __lsx_vand_v(__lsx_vsrli_h(stmp, 4), m4), __lsx_vand_v(stmp, m4));\n const __m128i scales = __lsx_vadd_b(__lsx_vslli_h(stmp, 1), m1);\n\n __m256i sumi1 = __lasx_xvldi(0);\n __m256i sumi2 = __lasx_xvldi(0);\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {\n\n const __m256i q2_data = __lasx_xvld((const __m256i*)q2, 0); q2 += 16;\n aux_gindex = __lasx_xvand_v(q2_data, m511);\n\n const __m256i partial_sign_bits = __lasx_xvsrli_h(q2_data, 9);\n const __m256i partial_sign_bits_upper = __lasx_xvsrli_h(q2_data, 13);\n const __m256i partial_sign_bits_for_counting = __lasx_xvxor_v(partial_sign_bits, partial_sign_bits_upper);\n\n const __m256i odd_bits = lasx_shuffle_b(bit_helper, partial_sign_bits_for_counting);\n const __m256i full_sign_bits = __lasx_xvor_v(partial_sign_bits, odd_bits);\n\n const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q8_3 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q8_4 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n\n const __m256i q2_1 = lasx_set_d(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],\n iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);\n const __m256i q2_2 = lasx_set_d(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],\n iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);\n const __m256i q2_3 = lasx_set_d(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],\n iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);\n const __m256i q2_4 = lasx_set_d(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],\n iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);\n\n const __m128i full_signs_l = lasx_extracti128(full_sign_bits, 0);\n const __m128i full_signs_h = lasx_extracti128(full_sign_bits, 1);\n const __m256i full_signs_1 = lasx_insertf128(full_signs_l, full_signs_l);\n const __m256i full_signs_2 = lasx_insertf128(full_signs_h, full_signs_h);\n\n __m256i signs;\n signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_1);\n signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);\n const __m256i q8s_1 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_1);\n\n signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_2);\n signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);\n const __m256i q8s_2 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_2);\n\n signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_1);\n signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);\n const __m256i q8s_3 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_3);\n\n signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_2);\n signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);\n const __m256i q8s_4 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_4);\n\n const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);\n const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);\n const __m256i dot3 = lasx_maddubs_h(q2_3, q8s_3);\n const __m256i dot4 = lasx_maddubs_h(q2_4, q8s_4);\n\n const __m256i sc1 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+0)));\n const __m256i sc2 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+1)));\n const __m256i sc3 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+2)));\n const __m256i sc4 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+3)));\n\n sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot1, sc1));\n sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot2, sc2));\n sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot3, sc3));\n sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot4, sc4));\n }\n\n accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n\n }\n\n *s = 0.125f * hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n };\n\n\n const __m128i m4 = __lsx_vreplgr2vr_b(0xf);\n const __m128i m1 = __lsx_vreplgr2vr_b(1);\n\n const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);\n const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);\n uint64_t aux64;\n\n __m256 accumf = (__m256)__lasx_xvldi(0);\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n __m128i tmp1;\n memcpy(&aux64, x[i].scales, 8);\n tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64, 0);\n tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64 >> 4, 1);\n const __m128i scales8 = __lsx_vadd_b(__lsx_vslli_h(__lsx_vand_v(tmp1, m4), 1), m1);\n const __m256i scales16 = lasx_ext8_16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15\n\n __m256i sumi1 = __lasx_xvldi(0);\n __m256i sumi2 = __lasx_xvldi(0);\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q2_1 = lasx_set_d(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],\n iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)],\n iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],\n iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);\n const __m256i q2_2 = lasx_set_d(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],\n iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)],\n iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],\n iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);\n qs += 8;\n\n __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | ((uint32_t) signs[1] << 16));\n aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);\n const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);\n const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);\n\n aux256 = __lasx_xvreplgr2vr_w(signs[2] | ((uint32_t) signs[3] << 16));\n aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);\n const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);\n const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);\n\n signs += 4;\n\n const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1\n const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3\n\n const __m256i p1 = lasx_madd_h(dot1, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+0)));\n const __m256i p2 = lasx_madd_h(dot2, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+1)));\n sumi1 = __lasx_xvadd_w(sumi1, p1);\n sumi2 = __lasx_xvadd_w(sumi2, p2);\n }\n\n accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n }\n\n *s = 0.125f * hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n uint32_t aux32[2];\n\n __m256 accumf = (__m256)__lasx_xvldi(0);\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n __m256i sumi1 = __lasx_xvldi(0);\n __m256i sumi2 = __lasx_xvldi(0);\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q2_1 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],\n iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n q3 += 8;\n const __m256i q2_2 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],\n iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n q3 += 8;\n memcpy(aux32, gas, 8); gas += 8;\n\n const __m256i s2_1 = lasx_set_d(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],\n signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);\n const __m256i s2_2 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],\n signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);\n const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);\n const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);\n const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);\n const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);\n const uint16_t ls1 = aux32[0] >> 28;\n const uint16_t ls2 = aux32[1] >> 28;\n\n const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));\n const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));\n sumi1 = __lasx_xvadd_w(sumi1, p1);\n sumi2 = __lasx_xvadd_w(sumi2, p2);\n }\n\n accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n }\n\n *s = 0.25f * hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n };\n\n const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);\n const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);\n\n __m256i idx_shift = lasx_set_w(1, 2, 3, 4, 5, 6, 7, 8);\n const __m256i idx_mask = __lasx_xvreplgr2vr_w(256);\n\n typedef union {\n __m256i vec[2];\n uint32_t index[16];\n } index_t;\n\n index_t idx;\n\n __m256 accumf = (__m256)__lasx_xvldi(0);\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n __m256i sumi1 = __lasx_xvldi(0);\n __m256i sumi2 = __lasx_xvldi(0);\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i idx_l = lasx_extu8_16(__lsx_vld(qs, 0)); qs += 16;\n idx.vec[0] = __lasx_xvreplgr2vr_w(qh[ib32+0]);\n idx.vec[1] = __lasx_xvreplgr2vr_w(qh[ib32+1]);\n idx.vec[0] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[0], idx_shift), idx_mask);\n idx.vec[1] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[1], idx_shift), idx_mask);\n idx.vec[0] = __lasx_xvor_v(idx.vec[0], lasx_ext16_32(lasx_extracti128(idx_l, 0)));\n idx.vec[1] = __lasx_xvor_v(idx.vec[1], lasx_ext16_32(lasx_extracti128(idx_l, 1)));\n\n // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.\n //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);\n //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);\n const __m256i q2_1 = lasx_set_w(\n iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],\n iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]\n );\n const __m256i q2_2 = lasx_set_w(\n iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],\n iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]\n );\n\n __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | (signs[1] << 16));\n aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);\n const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);\n const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);\n\n aux256 = __lasx_xvreplgr2vr_w(signs[2] | (signs[3] << 16));\n aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);\n const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);\n const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);\n\n signs += 4;\n\n const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);\n const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);\n const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;\n const uint16_t ls2 = x[i].scales[ib32/2] >> 4;\n const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));\n const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));\n sumi1 = __lasx_xvadd_w(sumi1, p1);\n sumi2 = __lasx_xvadd_w(sumi2, p2);\n }\n\n accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n }\n\n *s = hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined(__loongarch_asx)\nstatic inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {\n const __m256i a = __lasx_xvmulwev_h_b(x, y);\n const __m256i b = __lasx_xvmulwod_h_b(x, y);\n return __lasx_xvadd_h(a, b);\n}\n#endif\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n __m256 accum = (__m256)__lasx_xvldi(0);\n float accum1 = 0;\n for (int i = 0; i < nb; ++i) {\n\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint16_t * qh = x[i].qh;\n\n __m256i sumi = __lasx_xvldi(0);\n int sumi1 = 0;\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n __m256i q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)], 0);\n q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], 1);\n q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)], 2);\n q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], 3);\n\n __m256i q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)], 0);\n q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], 1);\n q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)], 2);\n q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], 3);\n\n qs += 8;\n const __m256i q8b_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n const __m256i q8b_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);\n const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);\n const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;\n const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;\n\n __m256i tmp1, tmp5, tmp6;\n tmp1 = __lasx_xvreplgr2vr_h(ls1);\n tmp5 = __lasx_xvmulwev_w_h(dot1, tmp1);\n tmp6 = __lasx_xvmulwod_w_h(dot1, tmp1);\n const __m256i p1 = __lasx_xvadd_w(tmp5, tmp6);\n\n tmp1 = __lasx_xvreplgr2vr_h(ls2);\n tmp5 = __lasx_xvmulwev_w_h(dot2, tmp1);\n tmp6 = __lasx_xvmulwod_w_h(dot2, tmp1);\n const __m256i p2 = __lasx_xvadd_w(tmp5, tmp6);\n\n sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p1, p2));\n sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1\n + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;\n }\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), accum);\n accum1 += d * sumi1;\n }\n\n *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK4_NL == 0);\n static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n const block_iq4_nl * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK4_NL;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined (__loongarch_asx)\n\n const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);\n const __m128i m4b = __lsx_vreplgr2vr_b(0x0f);\n const __m256i mone = __lasx_xvreplgr2vr_h(1);\n\n __m256 accum1 = (__m256)__lasx_xvldi(0);\n __m256 accum2 = (__m256)__lasx_xvldi(0);\n for (; ib + 1 < nb; ib += 2) {\n const __m128i q4bits_1 = __lsx_vld((const __m128i*)x[ib + 0].qs, 0);\n const __m128i q4bits_2 = __lsx_vld((const __m128i*)x[ib + 1].qs, 0);\n const __m256i q8b_1 = __lasx_xvld((const __m256i *)y[ib + 0].qs, 0);\n const __m256i q8b_2 = __lasx_xvld((const __m256i *)y[ib + 1].qs, 0);\n const __m256i q4b_1 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b)),\n lsx_shuffle_b(values128, __lsx_vand_v(q4bits_1, m4b)));\n const __m256i q4b_2 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b)),\n lsx_shuffle_b(values128, __lsx_vand_v(q4bits_2, m4b)));\n const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n const __m256i p_1 = lasx_madd_h(p16_1, mone);\n const __m256i p_2 = lasx_madd_h(p16_2, mone);\n accum1 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_FP16_TO_FP32(x[ib + 0].d)),\n __lasx_xvffint_s_w(p_1), accum1);\n accum2 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_FP16_TO_FP32(x[ib + 1].d)),\n __lasx_xvffint_s_w(p_2), accum2);\n }\n\n sumf = hsum_float_8(__lasx_xvfadd_s(accum1, accum2));\n\n#endif\n for (; ib < nb; ++ib) {\n const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);\n int sumi1 = 0, sumi2 = 0;\n for (int j = 0; j < QK4_NL/2; ++j) {\n sumi1 += y[ib].qs[j+ 0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];\n sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >> 4];\n }\n sumf += d * (sumi1 + sumi2);\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_K == 0);\n\n const block_iq4_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);\n\n __m256 accum = (__m256)__lasx_xvldi(0);\n\n for (int ibl = 0; ibl < nb; ++ibl) {\n const uint8_t * qs = x[ibl].qs;\n const int8_t * q8 = y[ibl].qs;\n uint16_t sh = x[ibl].scales_h;\n __m256i sumi1 = __lasx_xvldi(0);\n __m256i sumi2 = __lasx_xvldi(0);\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n const __m128i q4bits_1 = __lsx_vld((const __m128i*)qs, 0); qs += 16;\n const __m128i q4bits_2 = __lsx_vld((const __m128i*)qs, 0); qs += 16;\n const __m256i q8b_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q8b_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n const __m256i q4b_1 = lasx_insertf128(__lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits_1, 4)),\n __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits_1, 0xf)));\n const __m256i q4b_2 = lasx_insertf128(__lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits_2, 4)),\n __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits_2, 0xf)));\n const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;\n const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32;\n sh >>= 4;\n const __m256i p_1 = lasx_madd_h(p16_1, __lasx_xvreplgr2vr_h(ls1));\n const __m256i p_2 = lasx_madd_h(p16_2, __lasx_xvreplgr2vr_h(ls2));\n sumi1 = __lasx_xvadd_w(p_1, sumi1);\n sumi2 = __lasx_xvadd_w(p_2, sumi2);\n }\n accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ibl].d)*y[ibl].d),\n __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accum);\n }\n\n *s = hsum_float_8(accum);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n" }, { "path": "src/ggml-cpu/arch/powerpc/cpu-feats.cpp", "content": "# include \"ggml-backend-impl.h\"\n\n#if defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)\n\n#if defined(__linux__)\n#include \n#endif\n\n#include \n\nstruct powerpc_features {\n std::string platform = \"\";\n int power_version = -1;\n\n bool has_vsx = false;\n\n powerpc_features() {\n#if defined(__linux__)\n unsigned long auxval = getauxval(AT_PLATFORM);\n if (auxval) {\n platform = std::string(reinterpret_cast(auxval));\n // TBD: Do systems exist that return this in uppercase?\n if (platform.substr(0, 5) == \"power\") {\n // Extractt a numeric suffix, if one exists\n int vpos = -1;\n for (int i = platform.length() - 1; i >= 0; i--) {\n if (std::isdigit(platform[i])) {\n vpos = i;\n } else {\n break;\n }\n }\n if (vpos > -1) {\n power_version = std::stoi(platform.substr(vpos));\n }\n }\n }\n#endif\n if (power_version >= 9) {\n has_vsx = true;\n }\n }\n};\n\nstatic int ggml_backend_cpu_powerpc_score() {\n int score = 1;\n powerpc_features pf;\n\n// Platform scores\n#if defined(GGML_USE_POWER7)\n if (pf.power_version < 7) { return 0; }\n score += 1<<1;\n#endif\n#if defined(GGML_USE_POWER8)\n if (pf.power_version < 8) { return 0; }\n score += 1<<2;\n#endif\n#if defined(GGML_USE_POWER9)\n if (pf.power_version < 9) { return 0; }\n score += 1<<3;\n#endif\n#if defined(GGML_USE_POWER10)\n if (pf.power_version < 10) { return 0; }\n score += 1<<4;\n#endif\n#if defined(GGML_USE_POWER11)\n if (pf.power_version < 11) { return 0; }\n score += 1<<5;\n#endif\n\n// Feature scores\n#if defined(GGML_USE_VSX)\n if (!pf.has_vsx) { return 0; }\n score += 1<<6;\n#endif\n\n return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_powerpc_score)\n\n#endif // defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)\n" }, { "path": "src/ggml-cpu/arch/powerpc/quants.c", "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include \n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__POWER9_VECTOR__)\n#define B1(c,s,n) 0x ## n ## c , 0x ## n ## s\n#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)\n#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)\n#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)\n#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)\n#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)\n#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)\n#define B8(c,s ) B7(c,s, c), B7(c,s, s)\n\n// precomputed tables for expanding 8bits to 8 bytes:\nstatic const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4\nstatic const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4\n#endif\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__POWER9_VECTOR__)\n for (int i = 0; i < nb; i++) {\n vector float srcv [8];\n vector float asrcv[8];\n vector float amaxv[8];\n vector signed int vi[8];\n\n for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);\n for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);\n\n for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);\n for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);\n for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);\n\n const float amax = MAX(MAX(vec_extract(amaxv[0], 0),\n vec_extract(amaxv[0], 1)),\n MAX(vec_extract(amaxv[0], 2),\n vec_extract(amaxv[0], 3)));\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n const vector float vid = vec_splats(id);\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n for (int j = 0; j < 8; j++) {\n const vector float v = vec_round(vec_mul(srcv[j], vid));\n vi[j] = vec_cts(v, 0);\n }\n vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])), 0, &y[i].qs[0]);\n vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]);\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK8_1 == 0);\n const int nb = k / QK8_1;\n\n block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__POWER9_VECTOR__)\n for (int i = 0; i < nb; i++) {\n vector float srcv [8];\n vector float asrcv[8];\n vector float amaxv[8];\n vector signed int vi[8];\n\n for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);\n for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);\n\n for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);\n for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);\n for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);\n\n const float amax = MAX(MAX(vec_extract(amaxv[0], 0),\n vec_extract(amaxv[0], 1)),\n MAX(vec_extract(amaxv[0], 2),\n vec_extract(amaxv[0], 3)));\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n const vector float vid = vec_splats(id);\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n vector int accv = vec_splats(0);\n\n for (int j = 0; j < 8; j++) {\n const vector float v = vec_round(vec_mul(srcv[j], vid));\n vi[j] = vec_cts(v, 0);\n\n accv = vec_add(accv, vi[j]);\n }\n vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])), 0, &y[i].qs[0]);\n vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]);\n\n accv = vec_add(accv, vec_sld(accv, accv, 4));\n accv = vec_add(accv, vec_sld(accv, accv, 8));\n y[i].s = GGML_CPU_FP32_TO_FP16(d * vec_extract(accv, 0));\n }\n\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector signed int v0 = vec_splats((int32_t)0);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n const vector signed char v8 = vec_splats((signed char)0x8);\n\n vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 8\n for (; ib < nb; ++ib) {\n __builtin_prefetch(x[ib].qs, 0, 1);\n __builtin_prefetch(y[ib].qs, 0, 1);\n\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n vector signed char q4x0 = vec_and(qxs, lowMask);\n vector signed char q4x1 = vec_sr(qxs, v4);\n\n q4x0 = vec_sub(q4x0, v8);\n q4x1 = vec_sub(q4x1, v8);\n\n vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));\n\n vector signed int vsumi0 = v0;\n\n vsumi0 = vec_sum4s(qv0, vsumi0);\n vsumi0 = vec_sum4s(qv1, vsumi0);\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n }\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n sumf = vec_extract(vsumf0, 0);\n\n *s = sumf;\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector signed int v0 = vec_splats((int32_t)0);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 4\n for (; ib < nb; ++ib) {\n __builtin_prefetch(x[ib].qs, 0, 1);\n __builtin_prefetch(y[ib].qs, 0, 1);\n\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n vector float vd = vec_mul(vxd, vyd);\n\n vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].m));\n vector float vys = {GGML_CPU_FP16_TO_FP32(y[ib].s), 0.0f, 0.0f, 0.0f};\n vsumf0 = vec_madd(vxmin, vys, vsumf0);\n\n vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n vector unsigned char q4x0 = (vector unsigned char)vec_and(qxs, lowMask);\n vector unsigned char q4x1 = (vector unsigned char)vec_sr(qxs, v4);\n\n vector signed int vsumi0 = v0;\n\n vsumi0 = vec_msum(q8y0, q4x0, vsumi0);\n vsumi0 = vec_msum(q8y1, q4x1, vsumi0);\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n }\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n sumf = vec_extract(vsumf0, 0);\n\n *s = sumf;\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_MXFP4 == 0);\n static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n const block_mxfp4 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_MXFP4;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector unsigned char vshift4 = vec_splats((unsigned char)4);\n vector float vsumf0 = vec_splats(0.0f);\n\n vector signed char kv = vec_xl(0, (const signed char *)kvalues_mxfp4);\n\n#pragma GCC unroll 8\n for (; ib < nb; ++ib) {\n __builtin_prefetch(x[ib].qs, 0, 1);\n __builtin_prefetch(y[ib].qs, 0, 1);\n\n vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d) *\n GGML_E8M0_TO_FP32_HALF(x[ib].e));\n\n vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n vector signed char qxs = (vector signed char)vec_xl(0, x[ib].qs);\n\n vector unsigned char lo_nibbles = (vector unsigned char)vec_and(qxs, lowMask);\n vector unsigned char hi_nibbles = (vector unsigned char)vec_sr(qxs, vshift4);\n\n vector signed char q4x0 = vec_perm(kv, kv, lo_nibbles);\n vector signed char q4x1 = vec_perm(kv, kv, hi_nibbles);\n\n vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));\n\n vector signed int vsumi0 = vec_splats((int32_t)0);\n vsumi0 = vec_sum4s(qv0, vsumi0);\n vsumi0 = vec_sum4s(qv1, vsumi0);\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vyd, vsumf0);\n }\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n sumf = vec_extract(vsumf0, 0);\n *s = sumf;\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector unsigned char v4 = vec_splats((unsigned char)4);\n\n vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 4\n for (; ib < nb; ++ib) {\n __builtin_prefetch(x[ib].qs, 0, 1);\n __builtin_prefetch(y[ib].qs, 0, 1);\n\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed long long aux64x2_0 = {(uint64_t)(table_b2b_1[x[ib].qh[0]]), (uint64_t)(table_b2b_1[x[ib].qh[1]])};\n vector signed long long aux64x2_1 = {(uint64_t)(table_b2b_1[x[ib].qh[2]]), (uint64_t)(table_b2b_1[x[ib].qh[3]])};\n\n vector signed char qh0 = (vector signed char)aux64x2_0;\n vector signed char qh1 = (vector signed char)aux64x2_1;\n\n vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n\n vector signed char q5x0 = vec_sub(vec_and (qxs, lowMask), qh0);\n vector signed char q5x1 = vec_sub(vec_sr(qxs, v4), qh1);\n\n vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n vector signed char q8y1 = vec_xl( 16, y[ib].qs);\n\n vector signed short qv0 = vec_add(vec_mule(q5x0, q8y0), vec_mulo(q5x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q5x1, q8y1), vec_mulo(q5x1, q8y1));\n\n qv0 = vec_add(qv0, qv1);\n\n vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0));\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n }\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n sumf = vec_extract(vsumf0, 0);\n\n *s = sumf;\n#else\n UNUSED(ib);\n UNUSED(sumf);\n UNUSED(x);\n UNUSED(y);\n ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_1);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector signed int v0 = vec_splats((int32_t)0);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 4\n for (; ib < nb; ++ib) {\n __builtin_prefetch(x[ib].qs, 0, 1);\n __builtin_prefetch(y[ib].qs, 0, 1);\n\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n vector float vd = vec_mul(vxd, vyd);\n\n vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].m));\n vector float vys = {GGML_CPU_FP16_TO_FP32(y[ib].s), 0.f, 0.f, 0.f};\n vsumf0 = vec_madd(vxmin, vys, vsumf0);\n\n vector unsigned long long aux64x2_0 = {(uint64_t)(table_b2b_0[x[ib].qh[0]]), (uint64_t)(table_b2b_0[x[ib].qh[1]])};\n vector unsigned long long aux64x2_1 = {(uint64_t)(table_b2b_0[x[ib].qh[2]]), (uint64_t)(table_b2b_0[x[ib].qh[3]])};\n\n vector signed char qh0 = (vector signed char)aux64x2_0;\n vector signed char qh1 = (vector signed char)aux64x2_1;\n\n vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n\n vector unsigned char q5x0 = (vector unsigned char)vec_or(vec_and(qxs, lowMask), qh0);\n vector unsigned char q5x1 = (vector unsigned char)vec_or(vec_sr(qxs, v4), qh1);\n\n vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n vector signed char q8y1 = vec_xl( 16, y[ib].qs);\n\n vector signed int vsumi0 = v0;\n\n vsumi0 = vec_msum(q8y0, q5x0, vsumi0);\n vsumi0 = vec_msum(q8y1, q5x1, vsumi0);\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n }\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n sumf = vec_extract(vsumf0, 0);\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(sumf);\n UNUSED(x);\n UNUSED(y);\n ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q8_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed int v0 = vec_splats((int32_t)0);\n vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 8\n for (; ib < nb; ++ib) {\n __builtin_prefetch(x[ib].qs, 0, 1);\n __builtin_prefetch(y[ib].qs, 0, 1);\n\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed char q8x0 = vec_xl( 0, x[ib].qs);\n vector signed char q8x1 = vec_xl(16, x[ib].qs);\n vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n vector signed short qv0 = vec_mule(q8x0, q8y0);\n vector signed short qv1 = vec_mulo(q8x0, q8y0);\n vector signed short qv2 = vec_mule(q8x1, q8y1);\n vector signed short qv3 = vec_mulo(q8x1, q8y1);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n\n vsumi0 = vec_sum4s(qv0, vsumi0);\n vsumi1 = vec_sum4s(qv1, vsumi1);\n vsumi0 = vec_sum4s(qv2, vsumi0);\n vsumi1 = vec_sum4s(qv3, vsumi1);\n\n vsumi0 = vec_add(vsumi0, vsumi1);\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n }\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n sumf = vec_extract(vsumf0, 0);\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q2_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0x3);\n const vector signed char lowScaleMask = vec_splats((signed char)0xF);\n const vector int v0 = vec_splats((int32_t)0);\n const vector unsigned char v2 = vec_splats((unsigned char)0x2);\n const vector unsigned char v6 = vec_splats((unsigned char)0x6);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].dmin));\n vector float vdmin = vec_mul(vxmin, vyd);\n\n vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);\n vector signed short q8ysums1 = vec_xl(16, y[i].bsums);\n\n vector signed char q2xmins = (vector signed char)vec_xl( 0, x[i].scales);\n vector signed char vscales = vec_and(q2xmins, lowScaleMask);\n\n q2xmins = vec_sr(q2xmins, v4);\n vector signed short q2xmins0 = vec_unpackh(q2xmins);\n vector signed short q2xmins1 = vec_unpackl(q2xmins);\n\n vector signed int prod0 = vec_mule(q2xmins0, q8ysums0);\n vector signed int prod1 = vec_mulo(q2xmins0, q8ysums0);\n vector signed int prod2 = vec_mule(q2xmins1, q8ysums1);\n vector signed int prod3 = vec_mulo(q2xmins1, q8ysums1);\n\n vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);\n vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);\n vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);\n vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n vector signed int vsumi4 = v0;\n vector signed int vsumi5 = v0;\n vector signed int vsumi6 = v0;\n vector signed int vsumi7 = v0;\n\n const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n for (int j = 0; j < QK_K/128; ++j) {\n __builtin_prefetch(q2, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector signed char qxs0 = (vector signed char)vec_xl( 0, q2);\n vector signed char qxs1 = (vector signed char)vec_xl(16, q2);\n q2 += 32;\n\n vector unsigned char q2x00 = (vector unsigned char)vec_and(qxs0, lowMask);\n vector unsigned char q2x01 = (vector unsigned char)vec_and(vec_sr(qxs0, v2), lowMask);\n vector unsigned char q2x02 = (vector unsigned char)vec_and(vec_sr(qxs0, v4), lowMask);\n vector unsigned char q2x03 = (vector unsigned char)vec_and(vec_sr(qxs0, v6), lowMask);\n vector unsigned char q2x10 = (vector unsigned char)vec_and(qxs1, lowMask);\n vector unsigned char q2x11 = (vector unsigned char)vec_and(vec_sr(qxs1, v2), lowMask);\n vector unsigned char q2x12 = (vector unsigned char)vec_and(vec_sr(qxs1, v4), lowMask);\n vector unsigned char q2x13 = (vector unsigned char)vec_and(vec_sr(qxs1, v6), lowMask);\n\n vector signed char q8y00 = vec_xl( 0, q8);\n vector signed char q8y10 = vec_xl( 16, q8);\n vector signed char q8y01 = vec_xl( 32, q8);\n vector signed char q8y11 = vec_xl( 48, q8);\n vector signed char q8y02 = vec_xl( 64, q8);\n vector signed char q8y12 = vec_xl( 80, q8);\n vector signed char q8y03 = vec_xl( 96, q8);\n vector signed char q8y13 = vec_xl(112, q8);\n q8 += 128;\n\n vector signed int qv0 = vec_msum(q8y00, q2x00, v0);\n vector signed int qv1 = vec_msum(q8y01, q2x01, v0);\n vector signed int qv2 = vec_msum(q8y02, q2x02, v0);\n vector signed int qv3 = vec_msum(q8y03, q2x03, v0);\n vector signed int qv4 = vec_msum(q8y10, q2x10, v0);\n vector signed int qv5 = vec_msum(q8y11, q2x11, v0);\n vector signed int qv6 = vec_msum(q8y12, q2x12, v0);\n vector signed int qv7 = vec_msum(q8y13, q2x13, v0);\n\n vector signed short vscales_07 = vec_unpackh(vscales);\n vector signed int vscales_03 = vec_unpackh(vscales_07);\n vector signed int vscales_47 = vec_unpackl(vscales_07);\n vector signed int vs0 = vec_splat(vscales_03, 0);\n vector signed int vs1 = vec_splat(vscales_03, 1);\n vector signed int vs2 = vec_splat(vscales_03, 2);\n vector signed int vs3 = vec_splat(vscales_03, 3);\n vector signed int vs4 = vec_splat(vscales_47, 0);\n vector signed int vs5 = vec_splat(vscales_47, 1);\n vector signed int vs6 = vec_splat(vscales_47, 2);\n vector signed int vs7 = vec_splat(vscales_47, 3);\n vscales = vec_sld(vscales, vscales, 8);\n\n vsumi0 = vec_add(vec_mul(qv0, vs0), vsumi0);\n vsumi1 = vec_add(vec_mul(qv1, vs2), vsumi1);\n vsumi2 = vec_add(vec_mul(qv2, vs4), vsumi2);\n vsumi3 = vec_add(vec_mul(qv3, vs6), vsumi3);\n vsumi4 = vec_add(vec_mul(qv4, vs1), vsumi4);\n vsumi5 = vec_add(vec_mul(qv5, vs3), vsumi5);\n vsumi6 = vec_add(vec_mul(qv6, vs5), vsumi6);\n vsumi7 = vec_add(vec_mul(qv7, vs7), vsumi7);\n }\n\n vsumi0 = vec_add(vsumi0, vsumi4);\n vsumi1 = vec_add(vsumi1, vsumi5);\n vsumi2 = vec_add(vsumi2, vsumi6);\n vsumi3 = vec_add(vsumi3, vsumi7);\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const uint32_t kmask1 = 0x03030303;\n const uint32_t kmask2 = 0x0f0f0f0f;\n\n const block_q3_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0x3);\n const vector signed char lowMask1 = vec_splats((int8_t)0xf);\n const vector signed char lowMask2 = vec_splats((int8_t)0x30);\n const vector int v0 = vec_splats((int32_t)0);\n const vector signed char v1 = vec_splats((signed char)0x1);\n const vector unsigned char v2 = vec_splats((unsigned char)0x2);\n const vector unsigned char v3 = vec_splats((unsigned char)0x3);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n const vector unsigned char v6 = vec_splats((unsigned char)0x6);\n const vector signed char off = vec_splats((signed char)0x20);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n UNUSED(kmask1);\n UNUSED(kmask2);\n\n vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8);\n vector signed char u1 = vec_and(u0, lowMask1);\n vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4);\n vector signed char u3 = (vector signed char)vec_mergeh((vector signed int)u2, (vector signed int)vec_sr(u2, v2));\n vector signed char u30 = vec_sl(vec_and(u3, lowMask), v4);\n vector signed char u31 = vec_and(u3, lowMask2);\n\n u1 = vec_or(u1, u30);\n u2 = vec_or(vec_sr(u0, v4), u31);\n\n vector signed char vscales = (vector signed char)vec_mergeh((vector signed long long)u1, (vector signed long long)u2);\n vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].hmask);\n vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].hmask);\n\n vscales = vec_sub(vscales, off);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n vector signed int vsumi4 = v0;\n vector signed int vsumi5 = v0;\n vector signed int vsumi6 = v0;\n vector signed int vsumi7 = v0;\n\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n for (int j = 0; j < QK_K/128; ++j) {\n __builtin_prefetch(q3, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector signed char qxs0 = (vector signed char)vec_xl( 0, q3);\n vector signed char qxs1 = (vector signed char)vec_xl(16, q3);\n q3 += 32;\n\n //the low 2 bits\n vector signed char qxs00 = vec_and(qxs0, lowMask);\n vector signed char qxs01 = vec_and(vec_sr(qxs0, v2), lowMask);\n vector signed char qxs02 = vec_and(vec_sr(qxs0, v4), lowMask);\n vector signed char qxs03 = vec_and(vec_sr(qxs0, v6), lowMask);\n vector signed char qxs10 = vec_and(qxs1, lowMask);\n vector signed char qxs11 = vec_and(vec_sr(qxs1, v2), lowMask);\n vector signed char qxs12 = vec_and(vec_sr(qxs1, v4), lowMask);\n vector signed char qxs13 = vec_and(vec_sr(qxs1, v6), lowMask);\n\n //the 3rd bit\n vector signed char qxh00 = vec_sl(vec_andc(v1, qxhs0), v2);\n vector signed char qxh01 = vec_sl(vec_andc(v1, vec_sr(qxhs0, (vector unsigned char)v1)), v2);\n vector signed char qxh02 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v2)), v2);\n vector signed char qxh03 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v3)), v2);\n vector signed char qxh10 = vec_sl(vec_andc(v1, qxhs1), v2);\n vector signed char qxh11 = vec_sl(vec_andc(v1, vec_sr(qxhs1, (vector unsigned char)v1)), v2);\n vector signed char qxh12 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v2)), v2);\n vector signed char qxh13 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v3)), v2);\n qxhs0 = vec_sr(qxhs0, v4);\n qxhs1 = vec_sr(qxhs1, v4);\n\n vector signed char q3x00 = vec_sub(qxs00, qxh00);\n vector signed char q3x01 = vec_sub(qxs01, qxh01);\n vector signed char q3x02 = vec_sub(qxs02, qxh02);\n vector signed char q3x03 = vec_sub(qxs03, qxh03);\n vector signed char q3x10 = vec_sub(qxs10, qxh10);\n vector signed char q3x11 = vec_sub(qxs11, qxh11);\n vector signed char q3x12 = vec_sub(qxs12, qxh12);\n vector signed char q3x13 = vec_sub(qxs13, qxh13);\n\n vector signed char q8y00 = vec_xl( 0, q8);\n vector signed char q8y10 = vec_xl( 16, q8);\n vector signed char q8y01 = vec_xl( 32, q8);\n vector signed char q8y11 = vec_xl( 48, q8);\n vector signed char q8y02 = vec_xl( 64, q8);\n vector signed char q8y12 = vec_xl( 80, q8);\n vector signed char q8y03 = vec_xl( 96, q8);\n vector signed char q8y13 = vec_xl(112, q8);\n q8 += 128;\n\n vector signed short vscales_h = vec_unpackh(vscales);\n vector signed short vs0 = vec_splat(vscales_h, 0);\n vector signed short vs1 = vec_splat(vscales_h, 1);\n vector signed short vs2 = vec_splat(vscales_h, 2);\n vector signed short vs3 = vec_splat(vscales_h, 3);\n vector signed short vs4 = vec_splat(vscales_h, 4);\n vector signed short vs5 = vec_splat(vscales_h, 5);\n vector signed short vs6 = vec_splat(vscales_h, 6);\n vector signed short vs7 = vec_splat(vscales_h, 7);\n vscales = vec_sld(vscales, vscales, 8);\n\n vector signed short qv00 = vec_add(vec_mule(q3x00, q8y00), vec_mulo(q3x00, q8y00));\n vector signed short qv01 = vec_add(vec_mule(q3x01, q8y01), vec_mulo(q3x01, q8y01));\n vector signed short qv02 = vec_add(vec_mule(q3x02, q8y02), vec_mulo(q3x02, q8y02));\n vector signed short qv03 = vec_add(vec_mule(q3x03, q8y03), vec_mulo(q3x03, q8y03));\n vector signed short qv10 = vec_add(vec_mule(q3x10, q8y10), vec_mulo(q3x10, q8y10));\n vector signed short qv11 = vec_add(vec_mule(q3x11, q8y11), vec_mulo(q3x11, q8y11));\n vector signed short qv12 = vec_add(vec_mule(q3x12, q8y12), vec_mulo(q3x12, q8y12));\n vector signed short qv13 = vec_add(vec_mule(q3x13, q8y13), vec_mulo(q3x13, q8y13));\n\n vsumi0 = vec_msum(qv00, vs0, vsumi0);\n vsumi1 = vec_msum(qv01, vs2, vsumi1);\n vsumi2 = vec_msum(qv02, vs4, vsumi2);\n vsumi3 = vec_msum(qv03, vs6, vsumi3);\n vsumi4 = vec_msum(qv10, vs1, vsumi4);\n vsumi5 = vec_msum(qv11, vs3, vsumi5);\n vsumi6 = vec_msum(qv12, vs5, vsumi6);\n vsumi7 = vec_msum(qv13, vs7, vsumi7);\n }\n\n vsumi0 = vec_add(vsumi0, vsumi4);\n vsumi1 = vec_add(vsumi1, vsumi5);\n vsumi2 = vec_add(vsumi2, vsumi6);\n vsumi3 = vec_add(vsumi3, vsumi7);\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = vec_extract(vsumf0, 0);\n\n#else\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector signed char lowMask1 = vec_splats((int8_t)0x3f);\n const vector signed char lowMask2 = vec_splats((int8_t)0x30);\n const vector int v0 = vec_splats((int32_t)0);\n const vector unsigned char v2 = vec_splats((uint8_t)2);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].dmin));\n vector float vdmin = vec_mul(vxmin, vyd);\n\n vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);\n vector signed short q8ysums1 = vec_xl(16, y[i].bsums);\n\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n\n vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8);\n vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2);\n vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4);\n vector signed char u3 = vec_sr(u2, v4);\n\n vector signed char u30 = u1;\n vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3);\n\n u1 = vec_and(u0, lowMask1);\n u2 = vec_or(u30, u31);\n\n vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2);\n\n vector signed short vscales = vec_unpackh(utmps);\n vector signed short q4xmins = vec_unpackl(utmps);\n vector signed short q4xmins0 = vec_mergeh(q4xmins, q4xmins);\n vector signed short q4xmins1 = vec_mergel(q4xmins, q4xmins);\n\n vector signed int prod0 = vec_mule(q4xmins0, q8ysums0);\n vector signed int prod1 = vec_mule(q4xmins1, q8ysums1);\n vector signed int prod2 = vec_mulo(q4xmins0, q8ysums0);\n vector signed int prod3 = vec_mulo(q4xmins1, q8ysums1);\n\n vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);\n vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);\n vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);\n vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n for (int j = 0; j < QK_K/64; j+=2) {\n __builtin_prefetch(q4, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector signed char qxs0 = (vector signed char)vec_xl( 0, q4);\n vector signed char qxs1 = (vector signed char)vec_xl(16, q4);\n vector signed char qxs2 = (vector signed char)vec_xl(32, q4);\n vector signed char qxs3 = (vector signed char)vec_xl(48, q4);\n q4 += 64;\n\n vector unsigned char q4x00 = (vector unsigned char)vec_and(qxs0, lowMask);\n vector unsigned char q4x01 = (vector unsigned char)vec_sr(qxs0, v4);\n vector unsigned char q4x10 = (vector unsigned char)vec_and(qxs1, lowMask);\n vector unsigned char q4x11 = (vector unsigned char)vec_sr(qxs1, v4);\n vector unsigned char q4x20 = (vector unsigned char)vec_and(qxs2, lowMask);\n vector unsigned char q4x21 = (vector unsigned char)vec_sr(qxs2, v4);\n vector unsigned char q4x30 = (vector unsigned char)vec_and(qxs3, lowMask);\n vector unsigned char q4x31 = (vector unsigned char)vec_sr(qxs3, v4);\n\n vector signed char q8y00 = vec_xl( 0, q8);\n vector signed char q8y10 = vec_xl( 16, q8);\n vector signed char q8y01 = vec_xl( 32, q8);\n vector signed char q8y11 = vec_xl( 48, q8);\n vector signed char q8y20 = vec_xl( 64, q8);\n vector signed char q8y30 = vec_xl( 80, q8);\n vector signed char q8y21 = vec_xl( 96, q8);\n vector signed char q8y31 = vec_xl(112, q8);\n q8 += 128;\n\n vector signed int qv00 = vec_msum(q8y00, q4x00, v0);\n vector signed int qv01 = vec_msum(q8y01, q4x01, v0);\n vector signed int qv10 = vec_msum(q8y10, q4x10, v0);\n vector signed int qv11 = vec_msum(q8y11, q4x11, v0);\n vector signed int qv20 = vec_msum(q8y20, q4x20, v0);\n vector signed int qv21 = vec_msum(q8y21, q4x21, v0);\n vector signed int qv30 = vec_msum(q8y30, q4x30, v0);\n vector signed int qv31 = vec_msum(q8y31, q4x31, v0);\n\n vector signed int vscales_h = vec_unpackh(vscales);\n vector signed int vs0 = vec_splat(vscales_h, 0);\n vector signed int vs1 = vec_splat(vscales_h, 1);\n vector signed int vs2 = vec_splat(vscales_h, 2);\n vector signed int vs3 = vec_splat(vscales_h, 3);\n vscales = vec_sld(vscales, vscales, 8);\n\n vsumi0 = vec_add(vec_mul(qv00, vs0), vsumi0);\n vsumi1 = vec_add(vec_mul(qv01, vs1), vsumi1);\n vsumi2 = vec_add(vec_mul(qv20, vs2), vsumi2);\n vsumi3 = vec_add(vec_mul(qv21, vs3), vsumi3);\n\n vsumi0 = vec_add(vec_mul(qv10, vs0), vsumi0);\n vsumi1 = vec_add(vec_mul(qv11, vs1), vsumi1);\n vsumi2 = vec_add(vec_mul(qv30, vs2), vsumi2);\n vsumi3 = vec_add(vec_mul(qv31, vs3), vsumi3);\n }\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector signed char lowMask1 = vec_splats((int8_t)0x3f);\n const vector signed char lowMask2 = vec_splats((int8_t)0x30);\n const vector int v0 = vec_splats((int32_t)0);\n const vector unsigned char v1 = vec_splats((unsigned char)0x1);\n const vector unsigned char v2 = vec_splats((unsigned char)0x2);\n const vector unsigned char v3 = vec_splats((unsigned char)0x3);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].dmin));\n vector float vdmin = vec_mul(vxmin, vyd);\n\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n\n vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8);\n vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2);\n vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4);\n vector signed char u3 = vec_sr(u2, v4);\n\n vector signed char u30 = u1;\n vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3);\n\n u1 = vec_and(u0, lowMask1);\n u2 = vec_or(u30, u31);\n\n vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2);\n\n vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);\n vector signed short q8ysums1 = vec_xl(16, y[i].bsums);\n\n vector signed short vscales = vec_unpackh(utmps);\n\n vector signed short q5xmins = vec_unpackl(utmps);\n vector signed short q5xmins0 = vec_mergeh(q5xmins, q5xmins);\n vector signed short q5xmins1 = vec_mergel(q5xmins, q5xmins);\n\n vector signed int prod0 = vec_mule(q5xmins0, q8ysums0);\n vector signed int prod1 = vec_mule(q5xmins1, q8ysums1);\n vector signed int prod2 = vec_mulo(q5xmins0, q8ysums0);\n vector signed int prod3 = vec_mulo(q5xmins1, q8ysums1);\n\n vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);\n vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);\n vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);\n vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);\n\n vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].qh);\n vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].qh);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n\n const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n for (int j = 0; j < QK_K/64; ++j) {\n __builtin_prefetch(q5, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector signed char qxs0 = (vector signed char)vec_xl( 0, q5);\n vector signed char qxs1 = (vector signed char)vec_xl(16, q5);\n q5 += 32;\n\n vector signed char qxs00 = vec_and(qxs0, lowMask);\n vector signed char qxs01 = vec_sr(qxs0, v4);\n vector signed char qxs10 = vec_and(qxs1, lowMask);\n vector signed char qxs11 = vec_sr(qxs1, v4);\n\n vector signed char q5h00 = vec_sl(vec_and((vector signed char)v1, qxhs0), v4);\n vector signed char q5h01 = vec_sl(vec_and((vector signed char)v2, qxhs0), v3);\n vector signed char q5h10 = vec_sl(vec_and((vector signed char)v1, qxhs1), v4);\n vector signed char q5h11 = vec_sl(vec_and((vector signed char)v2, qxhs1), v3);\n qxhs0 = vec_sr(qxhs0, v2);\n qxhs1 = vec_sr(qxhs1, v2);\n\n vector unsigned char q5x00 = (vector unsigned char)vec_or(q5h00, qxs00);\n vector unsigned char q5x01 = (vector unsigned char)vec_or(q5h01, qxs01);\n vector unsigned char q5x10 = (vector unsigned char)vec_or(q5h10, qxs10);\n vector unsigned char q5x11 = (vector unsigned char)vec_or(q5h11, qxs11);\n\n vector signed char q8y00 = vec_xl( 0, q8);\n vector signed char q8y10 = vec_xl(16, q8);\n vector signed char q8y01 = vec_xl(32, q8);\n vector signed char q8y11 = vec_xl(48, q8);\n q8 += 64;\n\n vector signed int qv00 = vec_msum(q8y00, q5x00, v0);\n vector signed int qv01 = vec_msum(q8y01, q5x01, v0);\n vector signed int qv10 = vec_msum(q8y10, q5x10, v0);\n vector signed int qv11 = vec_msum(q8y11, q5x11, v0);\n\n vector signed int vscales_h = vec_unpackh(vscales);\n vector signed int vs0 = vec_splat(vscales_h, 0);\n vector signed int vs1 = vec_splat(vscales_h, 1);\n vscales = vec_sld(vscales, vscales, 12);\n\n vsumi0 = vec_add(vec_mul(qv00, vs0), vsumi0);\n vsumi1 = vec_add(vec_mul(qv10, vs0), vsumi1);\n vsumi2 = vec_add(vec_mul(qv01, vs1), vsumi2);\n vsumi3 = vec_add(vec_mul(qv11, vs1), vsumi3);\n }\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q6_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector int v0 = vec_splats((int32_t)0);\n const vector unsigned char v2 = vec_splats((unsigned char)0x2);\n const vector unsigned char v3 = vec_splats((unsigned char)0x3);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n const vector unsigned char v6 = vec_splats((unsigned char)0x6);\n const vector signed char off = vec_splats((signed char)0x20);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n vector signed int vsumi4 = v0;\n vector signed int vsumi5 = v0;\n vector signed int vsumi6 = v0;\n vector signed int vsumi7 = v0;\n\n const uint8_t * GGML_RESTRICT q6 = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT qs = x[i].scales;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n for (int j = 0; j < QK_K/128; ++j) {\n __builtin_prefetch(q6, 0, 0);\n __builtin_prefetch(qh, 0, 0);\n __builtin_prefetch(q8, 0, 0);\n\n vector signed char qxs0 = (vector signed char)vec_xl( 0, q6);\n vector signed char qxs1 = (vector signed char)vec_xl(16, q6);\n vector signed char qxs2 = (vector signed char)vec_xl(32, q6);\n vector signed char qxs3 = (vector signed char)vec_xl(48, q6);\n q6 += 64;\n\n vector signed char qxs00 = vec_and(qxs0, lowMask);\n vector signed char qxs01 = vec_sr(qxs0, v4);\n vector signed char qxs10 = vec_and(qxs1, lowMask);\n vector signed char qxs11 = vec_sr(qxs1, v4);\n vector signed char qxs20 = vec_and(qxs2, lowMask);\n vector signed char qxs21 = vec_sr(qxs2, v4);\n vector signed char qxs30 = vec_and(qxs3, lowMask);\n vector signed char qxs31 = vec_sr(qxs3, v4);\n\n vector signed char qxhs0 = (vector signed char)vec_xl( 0, qh);\n vector signed char qxhs1 = (vector signed char)vec_xl(16, qh);\n qh += 32;\n\n vector signed char qxh00 = vec_sl(vec_and((vector signed char)v3, qxhs0), v4);\n vector signed char qxh01 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v4)), v4);\n vector signed char qxh10 = vec_sl(vec_and((vector signed char)v3, qxhs1), v4);\n vector signed char qxh11 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v4)), v4);\n vector signed char qxh20 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v2)), v4);\n vector signed char qxh21 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v6)), v4);\n vector signed char qxh30 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v2)), v4);\n vector signed char qxh31 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v6)), v4);\n\n vector signed char q6x00 = vec_sub(vec_or(qxh00, qxs00), off);\n vector signed char q6x01 = vec_sub(vec_or(qxh01, qxs01), off);\n vector signed char q6x10 = vec_sub(vec_or(qxh10, qxs10), off);\n vector signed char q6x11 = vec_sub(vec_or(qxh11, qxs11), off);\n vector signed char q6x20 = vec_sub(vec_or(qxh20, qxs20), off);\n vector signed char q6x21 = vec_sub(vec_or(qxh21, qxs21), off);\n vector signed char q6x30 = vec_sub(vec_or(qxh30, qxs30), off);\n vector signed char q6x31 = vec_sub(vec_or(qxh31, qxs31), off);\n\n vector signed char q8y00 = vec_xl( 0, q8);\n vector signed char q8y10 = vec_xl( 16, q8);\n vector signed char q8y20 = vec_xl( 32, q8);\n vector signed char q8y30 = vec_xl( 48, q8);\n vector signed char q8y01 = vec_xl( 64, q8);\n vector signed char q8y11 = vec_xl( 80, q8);\n vector signed char q8y21 = vec_xl( 96, q8);\n vector signed char q8y31 = vec_xl(112, q8);\n q8 += 128;\n\n vector signed short qv00 = vec_add(vec_mule(q6x00, q8y00), vec_mulo(q6x00, q8y00));\n vector signed short qv10 = vec_add(vec_mule(q6x10, q8y10), vec_mulo(q6x10, q8y10));\n vector signed short qv20 = vec_add(vec_mule(q6x20, q8y20), vec_mulo(q6x20, q8y20));\n vector signed short qv30 = vec_add(vec_mule(q6x30, q8y30), vec_mulo(q6x30, q8y30));\n vector signed short qv01 = vec_add(vec_mule(q6x01, q8y01), vec_mulo(q6x01, q8y01));\n vector signed short qv11 = vec_add(vec_mule(q6x11, q8y11), vec_mulo(q6x11, q8y11));\n vector signed short qv21 = vec_add(vec_mule(q6x21, q8y21), vec_mulo(q6x21, q8y21));\n vector signed short qv31 = vec_add(vec_mule(q6x31, q8y31), vec_mulo(q6x31, q8y31));\n\n vector signed short vscales = vec_unpackh(vec_xl_len(qs, 8));\n qs += 8;\n\n vector signed short vs0 = vec_splat(vscales, 0);\n vector signed short vs1 = vec_splat(vscales, 1);\n vector signed short vs2 = vec_splat(vscales, 2);\n vector signed short vs3 = vec_splat(vscales, 3);\n vector signed short vs4 = vec_splat(vscales, 4);\n vector signed short vs5 = vec_splat(vscales, 5);\n vector signed short vs6 = vec_splat(vscales, 6);\n vector signed short vs7 = vec_splat(vscales, 7);\n\n vsumi0 = vec_msum(qv00, vs0, vsumi0);\n vsumi1 = vec_msum(qv01, vs4, vsumi1);\n vsumi2 = vec_msum(qv10, vs1, vsumi2);\n vsumi3 = vec_msum(qv11, vs5, vsumi3);\n vsumi4 = vec_msum(qv20, vs2, vsumi4);\n vsumi5 = vec_msum(qv21, vs6, vsumi5);\n vsumi6 = vec_msum(qv30, vs3, vsumi6);\n vsumi7 = vec_msum(qv31, vs7, vsumi7);\n }\n\n vsumi0 = vec_add(vsumi0, vsumi4);\n vsumi1 = vec_add(vsumi1, vsumi5);\n vsumi2 = vec_add(vsumi2, vsumi6);\n vsumi3 = vec_add(vsumi3, vsumi7);\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined (__POWER9_VECTOR__)\nstatic const int8_t keven_signs_q2xs[1024] = {\n 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,\n 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,\n 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,\n 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,\n 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,\n 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,\n 1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, 1,\n 1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, -1,\n 1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, -1,\n 1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, 1,\n 1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, 1,\n 1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1,\n 1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, 1,\n 1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, -1,\n 1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, -1,\n 1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1,\n 1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, -1,\n 1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1,\n 1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, 1,\n 1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, -1,\n 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1,\n 1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, -1,\n 1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, -1,\n 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1,\n 1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, 1,\n 1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, -1,\n 1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1,\n 1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, 1,\n 1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1,\n 1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, 1,\n 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, 1,\n 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n const vector int v0 = vec_splats((int32_t)0);\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n for (int j = 0; j < QK_K/32; j += 2) {\n __builtin_prefetch(q2, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n uint32_t aux32[4];\n const uint8_t * aux8 = (const uint8_t *)aux32;\n\n memcpy(aux32, q2, 4*sizeof(uint32_t));\n q2 += 8;\n\n vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1])};\n vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3])};\n vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9])};\n vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11])};\n\n vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((aux32[1] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 7) & 127))};\n vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127))};\n vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((aux32[3] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 7) & 127))};\n vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127))};\n\n vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0);\n vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1);\n vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2);\n vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3);\n\n vector signed char q8y0 = vec_xl( 0, q8);\n vector signed char q8y1 = vec_xl(16, q8);\n vector signed char q8y2 = vec_xl(32, q8);\n vector signed char q8y3 = vec_xl(48, q8);\n q8 += 64;\n\n vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));\n vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));\n vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));\n\n const uint16_t ls0 = aux32[1] >> 28;\n const uint16_t ls1 = aux32[3] >> 28;\n\n vector signed short vscales01 = vec_splats((int16_t)(2*ls0+1));\n vector signed short vscales23 = vec_splats((int16_t)(2*ls1+1));\n\n vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n }\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = 0.125f * vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n const vector int v0 = vec_splats((int32_t)0);\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const uint8_t * GGML_RESTRICT sc = x[i].scales;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n for (int j = 0; j < QK_K/64; ++j) {\n __builtin_prefetch(q2, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xs_grid + (q2[0] & 511)), *(const int64_t *)(iq2xs_grid + (q2[1] & 511))};\n vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xs_grid + (q2[2] & 511)), *(const int64_t *)(iq2xs_grid + (q2[3] & 511))};\n vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xs_grid + (q2[4] & 511)), *(const int64_t *)(iq2xs_grid + (q2[5] & 511))};\n vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xs_grid + (q2[6] & 511)), *(const int64_t *)(iq2xs_grid + (q2[7] & 511))};\n\n vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((q2[0] >> 9))), *(const int64_t *)(signs64 + ((q2[1] >> 9)))};\n vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((q2[2] >> 9))), *(const int64_t *)(signs64 + ((q2[3] >> 9)))};\n vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((q2[4] >> 9))), *(const int64_t *)(signs64 + ((q2[5] >> 9)))};\n vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((q2[6] >> 9))), *(const int64_t *)(signs64 + ((q2[7] >> 9)))};\n q2 += 8;\n\n vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0);\n vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1);\n vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2);\n vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3);\n\n vector signed char q8y0 = vec_xl( 0, q8);\n vector signed char q8y1 = vec_xl(16, q8);\n vector signed char q8y2 = vec_xl(32, q8);\n vector signed char q8y3 = vec_xl(48, q8);\n q8 += 64;\n\n vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));\n vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));\n vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));\n\n const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);\n const uint16_t ls1 = (uint16_t)(sc[0] >> 4);\n const uint16_t ls2 = (uint16_t)(sc[1] & 0xf);\n const uint16_t ls3 = (uint16_t)(sc[1] >> 4);\n sc += 2;\n\n vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1));\n vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1));\n vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1));\n vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1));\n\n vsumi0 = vec_msum(qv0, vscales0, vsumi0);\n vsumi1 = vec_msum(qv1, vscales1, vsumi1);\n vsumi2 = vec_msum(qv2, vscales2, vsumi2);\n vsumi3 = vec_msum(qv3, vscales3, vsumi3);\n }\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = 0.125f * vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};\n\n const vector int v0 = vec_splats((int32_t)0);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n const vector unsigned char mask0 = vec_xl( 0, k_mask1);\n const vector unsigned char mask1 = vec_xl(16, k_mask1);\n const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2);\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n\n const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n const uint8_t * GGML_RESTRICT sc = x[i].scales;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n for (int j = 0; j < QK_K/32; j += 2) {\n __builtin_prefetch(q2, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector signed long long aux64x2_0 = {*(const int64_t *)(iq2s_grid + (q2[0] | ((qh[0] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[1] | ((qh[0] << 6) & 0x300)))};\n vector signed long long aux64x2_1 = {*(const int64_t *)(iq2s_grid + (q2[2] | ((qh[0] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[3] | ((qh[0] << 2) & 0x300)))};\n vector signed long long aux64x2_2 = {*(const int64_t *)(iq2s_grid + (q2[4] | ((qh[1] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[5] | ((qh[1] << 6) & 0x300)))};\n vector signed long long aux64x2_3 = {*(const int64_t *)(iq2s_grid + (q2[6] | ((qh[1] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[7] | ((qh[1] << 2) & 0x300)))};\n q2 += 8;\n qh += 2;\n\n vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]);\n vector signed char vsigns23 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]);\n signs += 4;\n\n vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0);\n vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1);\n vector signed char vsigns2 = vec_perm(vsigns23, vsigns23, mask0);\n vector signed char vsigns3 = vec_perm(vsigns23, vsigns23, mask1);\n\n vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2);\n vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2);\n vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2);\n vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2);\n\n vector signed char q2x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux64x2_0), vsigns0);\n vector signed char q2x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux64x2_1), vsigns1);\n vector signed char q2x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux64x2_2), vsigns2);\n vector signed char q2x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux64x2_3), vsigns3);\n\n vector signed char q8y0 = vec_xl( 0, q8);\n vector signed char q8y1 = vec_xl(16, q8);\n vector signed char q8y2 = vec_xl(32, q8);\n vector signed char q8y3 = vec_xl(48, q8);\n q8 += 64;\n\n vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));\n vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));\n vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));\n\n const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);\n const uint16_t ls1 = (uint16_t)(sc[0] >> 4);\n const uint16_t ls2 = (uint16_t)(sc[1] & 0xf);\n const uint16_t ls3 = (uint16_t)(sc[1] >> 4);\n sc += 2;\n\n vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1));\n vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1));\n vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1));\n vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1));\n\n vsumi0 = vec_msum(qv0, vscales0, vsumi0);\n vsumi1 = vec_msum(qv1, vscales1, vsumi1);\n vsumi2 = vec_msum(qv2, vscales2, vsumi2);\n vsumi3 = vec_msum(qv3, vscales3, vsumi3);\n }\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = 0.125f * vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n const vector int v0 = vec_splats((int32_t)0);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint32_t * GGML_RESTRICT signs = (const uint32_t *)(x[i].qs + QK_K/4);\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n#pragma GCC unroll 1\n for (int j = 0; j < QK_K/32; j += 2) {\n __builtin_prefetch(q3, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector unsigned int aux32x4_0 = {iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]};\n vector unsigned int aux32x4_1 = {iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]};\n vector unsigned int aux32x4_2 = {iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]};\n vector unsigned int aux32x4_3 = {iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]};\n q3 += 16;\n\n vector unsigned long long aux64x2_0 = {(uint64_t)(signs64[(signs[0] >> 0) & 127]), (uint64_t)(signs64[(signs[0] >> 7) & 127])};\n vector unsigned long long aux64x2_1 = {(uint64_t)(signs64[(signs[0] >> 14) & 127]), (uint64_t)(signs64[(signs[0] >> 21) & 127])};\n vector unsigned long long aux64x2_2 = {(uint64_t)(signs64[(signs[1] >> 0) & 127]), (uint64_t)(signs64[(signs[1] >> 7) & 127])};\n vector unsigned long long aux64x2_3 = {(uint64_t)(signs64[(signs[1] >> 14) & 127]), (uint64_t)(signs64[(signs[1] >> 21) & 127])};\n\n vector signed char q3x0 = vec_mul((vector signed char)aux64x2_0, (vector signed char)aux32x4_0);\n vector signed char q3x1 = vec_mul((vector signed char)aux64x2_1, (vector signed char)aux32x4_1);\n vector signed char q3x2 = vec_mul((vector signed char)aux64x2_2, (vector signed char)aux32x4_2);\n vector signed char q3x3 = vec_mul((vector signed char)aux64x2_3, (vector signed char)aux32x4_3);\n\n vector signed char q8y0 = vec_xl( 0, q8);\n vector signed char q8y1 = vec_xl(16, q8);\n vector signed char q8y2 = vec_xl(32, q8);\n vector signed char q8y3 = vec_xl(48, q8);\n q8 += 64;\n\n vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1));\n vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2));\n vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3));\n\n const uint16_t ls0 = (uint16_t)(signs[0] >> 28);\n const uint16_t ls1 = (uint16_t)(signs[1] >> 28);\n signs += 2;\n\n vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));\n vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));\n\n vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n }\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = 0.25f * vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};\n\n const vector int v0 = vec_splats((int32_t)0);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n const vector unsigned char mask0 = vec_xl( 0, k_mask1);\n const vector unsigned char mask1 = vec_xl(16, k_mask1);\n const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2);\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].signs);\n const uint8_t * GGML_RESTRICT sc = x[i].scales;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n\n for (int j = 0; j < QK_K/32; j += 2) {\n __builtin_prefetch(q3, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector unsigned int aux32x4_0 = {iq3s_grid[q3[ 0] | ((qh[0] << 8) & 256)], iq3s_grid[q3[ 1] | ((qh[0] << 7) & 256)],\n iq3s_grid[q3[ 2] | ((qh[0] << 6) & 256)], iq3s_grid[q3[ 3] | ((qh[0] << 5) & 256)]};\n vector unsigned int aux32x4_1 = {iq3s_grid[q3[ 4] | ((qh[0] << 4) & 256)], iq3s_grid[q3[ 5] | ((qh[0] << 3) & 256)],\n iq3s_grid[q3[ 6] | ((qh[0] << 2) & 256)], iq3s_grid[q3[ 7] | ((qh[0] << 1) & 256)]};\n vector unsigned int aux32x4_2 = {iq3s_grid[q3[ 8] | ((qh[1] << 8) & 256)], iq3s_grid[q3[ 9] | ((qh[1] << 7) & 256)],\n iq3s_grid[q3[10] | ((qh[1] << 6) & 256)], iq3s_grid[q3[11] | ((qh[1] << 5) & 256)]};\n vector unsigned int aux32x4_3 = {iq3s_grid[q3[12] | ((qh[1] << 4) & 256)], iq3s_grid[q3[13] | ((qh[1] << 3) & 256)],\n iq3s_grid[q3[14] | ((qh[1] << 2) & 256)], iq3s_grid[q3[15] | ((qh[1] << 1) & 256)]};\n q3 += 16;\n qh += 2;\n\n vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]);\n vector signed char vsigns02 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]);\n signs += 4;\n\n vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0);\n vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1);\n vector signed char vsigns2 = vec_perm(vsigns02, vsigns02, mask0);\n vector signed char vsigns3 = vec_perm(vsigns02, vsigns02, mask1);\n\n vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2);\n vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2);\n vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2);\n vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2);\n\n vector signed char q3x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux32x4_0), vsigns0);\n vector signed char q3x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux32x4_1), vsigns1);\n vector signed char q3x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux32x4_2), vsigns2);\n vector signed char q3x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux32x4_3), vsigns3);\n\n vector signed char q8y0 = vec_xl( 0, q8);\n vector signed char q8y1 = vec_xl(16, q8);\n vector signed char q8y2 = vec_xl(32, q8);\n vector signed char q8y3 = vec_xl(48, q8);\n q8 += 64;\n\n vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1));\n vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2));\n vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3));\n\n const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);\n const uint16_t ls1 = (uint16_t)(sc[0] >> 4);\n sc ++;\n\n vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));\n vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));\n\n vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n }\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n const vector unsigned char v0 = vec_splats((unsigned char)0x0);\n const vector unsigned short vsign = vec_splats((unsigned short)0x8000);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n for (int i = 0; i < nb; ++i) {\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n vector float vyd = vec_splats(y[i].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed int vsumi0 = vec_splats((int32_t)0);\n vector signed int vsumi1 = vec_splats((int32_t)0);\n vector signed int vsumi2 = vec_splats((int32_t)0);\n vector signed int vsumi3 = vec_splats((int32_t)0);\n vector signed int vsumi8 = vec_splats((int32_t)0);\n\n const uint8_t * GGML_RESTRICT q1 = x[i].qs;\n const uint16_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n const int16_t * GGML_RESTRICT qs = y[i].bsums;\n\n for (int j = 0; j < QK_K/32; j += 2) {\n __builtin_prefetch(q1, 0, 1);\n __builtin_prefetch(qh, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector signed long long aux64x2_0 = {*(const int64_t *)(iq1s_grid + (q1[0] | ((qh[0] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[1] | ((qh[0] << 5) & 0x700)))};\n vector signed long long aux64x2_1 = {*(const int64_t *)(iq1s_grid + (q1[2] | ((qh[0] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[3] | ((qh[0] >> 1) & 0x700)))};\n vector signed long long aux64x2_2 = {*(const int64_t *)(iq1s_grid + (q1[4] | ((qh[1] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[5] | ((qh[1] << 5) & 0x700)))};\n vector signed long long aux64x2_3 = {*(const int64_t *)(iq1s_grid + (q1[6] | ((qh[1] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[7] | ((qh[1] >> 1) & 0x700)))};\n q1 += 8;\n\n vector signed char q1x0 = (vector signed char)aux64x2_0;\n vector signed char q1x1 = (vector signed char)aux64x2_1;\n vector signed char q1x2 = (vector signed char)aux64x2_2;\n vector signed char q1x3 = (vector signed char)aux64x2_3;\n\n vector signed char q8y0 = vec_xl( 0, q8);\n vector signed char q8y1 = vec_xl(16, q8);\n vector signed char q8y2 = vec_xl(32, q8);\n vector signed char q8y3 = vec_xl(48, q8);\n q8 += 64;\n\n vector signed short qv0 = vec_add(vec_mule(q1x0, q8y0), vec_mulo(q1x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q1x1, q8y1), vec_mulo(q1x1, q8y1));\n vector signed short qv2 = vec_add(vec_mule(q1x2, q8y2), vec_mulo(q1x2, q8y2));\n vector signed short qv3 = vec_add(vec_mule(q1x3, q8y3), vec_mulo(q1x3, q8y3));\n\n const uint16_t ls0 = (uint16_t)((qh[0] >> 12) & 7);\n const uint16_t ls1 = (uint16_t)((qh[1] >> 12) & 7);\n\n vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));\n vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));\n vector signed short vscales = vec_sld(vscales23, vscales01, 8);\n\n vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n\n vector signed short q8ysums = vec_xl_len(qs, 8);\n qs += 4;\n q8ysums = vec_mergeh(q8ysums, (vector signed short)v0);\n\n vector signed short qxh = (vector signed short)vec_sld(vec_splats(qh[1]), vec_splats(qh[0]), 8);\n qh += 2;\n vector __bool short vsel = vec_cmpge(qxh, (vector signed short)v0);\n\n vector signed short q8ysum = vec_sel((vector signed short)vec_xor((vector unsigned short)q8ysums, vsign), q8ysums, vsel);\n\n vsumi8 = vec_add(vec_mule(q8ysum, vscales), vsumi8);\n }\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n\n vsumf0 = vec_madd(vec_ctf(vsumi8, 0), vec_mul(vd, vec_splats(IQ1S_DELTA)), vsumf0);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK4_NL == 0);\n static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n const block_iq4_nl * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK4_NL;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector signed int v0 = vec_splats((int32_t)0);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n\n const vector signed char values = vec_xl( 0, kvalues_iq4nl);\n\n#pragma GCC unroll 4\n for (; ib < nb; ++ib) {\n __builtin_prefetch(x[ib].qs, 0, 1);\n __builtin_prefetch(y[ib].qs, 0, 1);\n\n\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n vector signed char q4x0 = vec_and(qxs, lowMask);\n vector signed char q4x1 = vec_sr(qxs, v4);\n\n q4x0 = vec_perm(values, values, (vector unsigned char)q4x0);\n q4x1 = vec_perm(values, values, (vector unsigned char)q4x1);\n\n vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n\n vsumi0 = vec_sum4s(qv0, vsumi0);\n vsumi1 = vec_sum4s(qv1, vsumi1);\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n sumf = vec_extract(vsumf0, 0);\n\n *s = sumf;\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_K == 0);\n\n const block_iq4_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector int v0 = vec_splats((int32_t)0);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n vector float vsumf0 = vec_splats(0.0f);\n vector float vsumf1 = vec_splats(0.0f);\n vector float vsumf2 = vec_splats(0.0f);\n vector float vsumf3 = vec_splats(0.0f);\n\n const vector signed char values = vec_xl( 0, kvalues_iq4nl);\n\n for (int ibl = 0; ibl < nb; ++ibl) {\n\n vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ibl].d));\n vector float vyd = vec_splats(y[ibl].d);\n vector float vd = vec_mul(vxd, vyd);\n\n vector signed int vsumi0 = v0;\n vector signed int vsumi1 = v0;\n vector signed int vsumi2 = v0;\n vector signed int vsumi3 = v0;\n\n uint16_t h = x[ibl].scales_h;\n\n const uint8_t * GGML_RESTRICT q4 = x[ibl].qs;\n const uint8_t * GGML_RESTRICT sc = x[ibl].scales_l;\n const int8_t * GGML_RESTRICT q8 = y[ibl].qs;\n\n for (int ib = 0; ib < QK_K/64; ib ++ ) {\n __builtin_prefetch(q4, 0, 1);\n __builtin_prefetch(q8, 0, 1);\n\n vector signed char qxs0 = (vector signed char)vec_xl( 0, q4);\n vector signed char qxs1 = (vector signed char)vec_xl(16, q4);\n q4 += 32;\n\n vector signed char q4x00 = (vector signed char)vec_and(qxs0, lowMask);\n vector signed char q4x01 = (vector signed char)vec_sr(qxs0, v4);\n vector signed char q4x10 = (vector signed char)vec_and(qxs1, lowMask);\n vector signed char q4x11 = (vector signed char)vec_sr(qxs1, v4);\n\n q4x00 = vec_perm(values, values, (vector unsigned char)q4x00);\n q4x01 = vec_perm(values, values, (vector unsigned char)q4x01);\n q4x10 = vec_perm(values, values, (vector unsigned char)q4x10);\n q4x11 = vec_perm(values, values, (vector unsigned char)q4x11);\n\n vector signed char q8y0 = vec_xl( 0, q8);\n vector signed char q8y1 = vec_xl(16, q8);\n vector signed char q8y2 = vec_xl(32, q8);\n vector signed char q8y3 = vec_xl(48, q8);\n q8 += 64;\n\n vector signed short qv0 = vec_add(vec_mule(q4x00, q8y0), vec_mulo(q4x00, q8y0));\n vector signed short qv1 = vec_add(vec_mule(q4x01, q8y1), vec_mulo(q4x01, q8y1));\n vector signed short qv2 = vec_add(vec_mule(q4x10, q8y2), vec_mulo(q4x10, q8y2));\n vector signed short qv3 = vec_add(vec_mule(q4x11, q8y3), vec_mulo(q4x11, q8y3));\n\n const uint16_t ls0 = (uint16_t)(((sc[0] & 0xf) | ((h << 4) & 0x30)) - 32);\n const uint16_t ls1 = (uint16_t)(((sc[0] >> 4) | ((h << 2) & 0x30)) - 32);\n h >>= 4;\n sc ++;\n\n vector signed short vscales01 = vec_splats((int16_t)ls0);\n vector signed short vscales23 = vec_splats((int16_t)ls1);\n\n vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n }\n\n vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n }\n\n vsumf0 = vec_add(vsumf0, vsumf2);\n vsumf1 = vec_add(vsumf1, vsumf3);\n\n vsumf0 = vec_add(vsumf0, vsumf1);\n\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n *s = vec_extract(vsumf0, 0);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n" }, { "path": "src/ggml-cpu/arch/riscv/cpu-feats.cpp", "content": "#include \"ggml-backend-impl.h\"\n\n#if defined(__riscv) && __riscv_xlen == 64\n#include \n#include \n#include \n\nstruct riscv64_features {\n bool has_rvv = false;\n\n riscv64_features() {\n struct riscv_hwprobe probe;\n probe.key = RISCV_HWPROBE_KEY_IMA_EXT_0;\n probe.value = 0;\n\n int ret = syscall(__NR_riscv_hwprobe, &probe, 1, 0, NULL, 0);\n\n if (0 == ret) {\n has_rvv = !!(probe.value & RISCV_HWPROBE_IMA_V);\n }\n }\n};\n\nstatic int ggml_backend_cpu_riscv64_score() {\n int score = 1;\n riscv64_features rf;\n\n#ifdef GGML_USE_RVV\n if (!rf.has_rvv) { return 0; }\n score += 1 << 1;\n#endif\n\n return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_riscv64_score)\n\n#endif // __riscv && __riscv_xlen == 64\n" }, { "path": "src/ggml-cpu/arch/riscv/quants.c", "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include \n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__riscv_v)\n\n size_t vl = QK8_0;\n\n for (int i = 0; i < nb; i++) {\n // load elements\n vfloat32m8_t v_x = __riscv_vle32_v_f32m8(x+i*QK8_0, vl);\n\n vfloat32m8_t vfabs = __riscv_vfabs_v_f32m8(v_x, vl);\n vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl);\n vfloat32m1_t vmax = __riscv_vfredmax_vs_f32m8_f32m1(vfabs, tmp, vl);\n float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n vfloat32m8_t x0 = __riscv_vfmul_vf_f32m8(v_x, id, vl);\n\n // convert to integer\n vint16m4_t vi = __riscv_vfncvt_x_f_w_i16m4(x0, vl);\n vint8m2_t vs = __riscv_vncvt_x_x_w_i8m2(vi, vl);\n\n // store result\n __riscv_vse8_v_i8m2(y[i].qs , vs, vl);\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK8_1 == 0);\n const int nb = k / QK8_1;\n\n block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__riscv_v)\n\n size_t vl = QK8_1;\n\n for (int i = 0; i < nb; i++) {\n // load elements\n vfloat32m8_t v_x = __riscv_vle32_v_f32m8(x+i*QK8_1, vl);\n\n vfloat32m8_t vfabs = __riscv_vfabs_v_f32m8(v_x, vl);\n vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0, vl);\n vfloat32m1_t vmax = __riscv_vfredmax_vs_f32m8_f32m1(vfabs, tmp, vl);\n float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n vfloat32m8_t x0 = __riscv_vfmul_vf_f32m8(v_x, id, vl);\n\n // convert to integer\n vint16m4_t vi = __riscv_vfncvt_x_f_w_i16m4(x0, vl);\n vint8m2_t vs = __riscv_vncvt_x_x_w_i8m2(vi, vl);\n\n // store result\n __riscv_vse8_v_i8m2(y[i].qs , vs, vl);\n\n // compute sum for y[i].s\n vint16m1_t tmp2 = __riscv_vmv_v_x_i16m1(0, vl);\n vint16m1_t vwrs = __riscv_vwredsum_vs_i8m2_i16m1(vs, tmp2, vl);\n\n // set y[i].s\n int sum = __riscv_vmv_x_s_i16m1_i16(vwrs);\n y[i].s = GGML_CPU_FP32_TO_FP16(sum*d);\n }\n\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n block_q8_K * y_blocks = (block_q8_K *)y;\n size_t nb = k / QK_K;\n\n#if defined(__riscv_v_intrinsic)\n const size_t vlmax_f32m8 = __riscv_vsetvlmax_e32m8();\n\n for (size_t i = 0; i < nb; i++) {\n const float* x_block = x + i * QK_K;\n block_q8_K* y_block = &y_blocks[i];\n\n // 1. Calculate Min/Max\n vfloat32m8_t max_v = __riscv_vfmv_v_f_f32m8(-__builtin_inff(), vlmax_f32m8);\n vfloat32m8_t min_v = __riscv_vfmv_v_f_f32m8(__builtin_inff(), vlmax_f32m8);\n\n size_t rem = QK_K;\n size_t offset = 0;\n while (rem > 0) {\n size_t vl = __riscv_vsetvl_e32m8(rem);\n vfloat32m8_t v_curr = __riscv_vle32_v_f32m8(x_block + offset, vl);\n max_v = __riscv_vfmax_vv_f32m8(max_v, v_curr, vl);\n min_v = __riscv_vfmin_vv_f32m8(min_v, v_curr, vl);\n rem -= vl;\n offset += vl;\n }\n\n vfloat32m1_t v_init_max = __riscv_vfmv_s_f_f32m1(-__builtin_inff(), 1);\n vfloat32m1_t v_init_min = __riscv_vfmv_s_f_f32m1(__builtin_inff(), 1);\n\n vfloat32m1_t v_scalar_max = __riscv_vfredmax_vs_f32m8_f32m1(max_v, v_init_max, vlmax_f32m8);\n vfloat32m1_t v_scalar_min = __riscv_vfredmin_vs_f32m8_f32m1(min_v, v_init_min, vlmax_f32m8);\n\n float max_val = __riscv_vfmv_f_s_f32m1_f32(v_scalar_max);\n float min_val = __riscv_vfmv_f_s_f32m1_f32(v_scalar_min);\n\n float amax = fabsf(max_val) > fabsf(min_val) ? fabsf(max_val) : fabsf(min_val);\n\n if (amax == 0.0f) {\n y_block->d = 0.0f;\n memset(y_block->qs, 0, QK_K);\n memset(y_block->bsums, 0, sizeof(y_block->bsums));\n continue;\n }\n\n const float iscale = -127.f / (fabsf(max_val) > fabsf(min_val) ? max_val : min_val);\n y_block->d = 1.0f / iscale;\n\n // 2. Quantize and Calculate Sums\n offset = 0;\n rem = QK_K;\n vint16m1_t v_zero_sum = __riscv_vmv_v_x_i16m1(0, 1);\n\n while (rem > 0) {\n size_t vl = __riscv_vsetvl_e32m8(rem);\n vfloat32m8_t v_f = __riscv_vle32_v_f32m8(x_block + offset, vl);\n\n v_f = __riscv_vfmul_vf_f32m8(v_f, iscale, vl);\n\n vint32m8_t v_i32 = __riscv_vfcvt_x_f_v_i32m8_rm(v_f, __RISCV_FRM_RNE, vl);\n vint16m4_t v_i16 = __riscv_vnclip_wx_i16m4(v_i32, 0, __RISCV_VXRM_RNE, vl);\n vint8m2_t v_q = __riscv_vnclip_wx_i8m2(v_i16, 0, __RISCV_VXRM_RNE, vl);\n\n __riscv_vse8_v_i8m2(y_block->qs + offset, v_q, vl);\n\n // first iteration clear\n\n int sum_idx;\n vint8m1_t chunk_m1;\n vint16m1_t v_sum;\n sum_idx = offset / 16;\n chunk_m1 = __riscv_vget_v_i8m2_i8m1(v_q, 0);\n v_sum = __riscv_vwredsum_vs_i8m1_i16m1(chunk_m1, v_zero_sum, 16);\n y_block->bsums[sum_idx] = (int16_t)__riscv_vmv_x_s_i16m1_i16(v_sum);\n\n // remaining iterations\n vint8m2_t slid_q = v_q;\n for (size_t k = 16; k < vl; k += 16) {\n slid_q = __riscv_vslidedown_vx_i8m2(slid_q, 16, vl);\n\n sum_idx = (offset + k) / 16;\n chunk_m1 = __riscv_vget_v_i8m2_i8m1(slid_q, 0);\n\n v_sum = __riscv_vwredsum_vs_i8m1_i16m1(chunk_m1, v_zero_sum, 16);\n y_block->bsums[sum_idx] =(int16_t)__riscv_vmv_x_s_i16m1_i16(v_sum);\n }\n\n rem -= vl;\n offset += vl;\n }\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_K_ref(x, y, k);\n#endif\n}\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined(__riscv_v)\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n size_t vl = qk / 2;\n\n for (; ib < nb; ++ib) {\n // load elements\n vuint8m1_t tx = __riscv_vle8_v_u8m1(x[ib].qs, vl);\n\n vint8m1_t y0 = __riscv_vle8_v_i8m1(y[ib].qs, vl);\n vint8m1_t y1 = __riscv_vle8_v_i8m1(y[ib].qs+16, vl);\n\n // mask and store lower part of x, and then upper part\n vuint8m1_t x_a = __riscv_vand_vx_u8m1(tx, 0x0F, vl);\n vuint8m1_t x_l = __riscv_vsrl_vx_u8m1(tx, 0x04, vl);\n\n vint8m1_t x_ai = __riscv_vreinterpret_v_u8m1_i8m1(x_a);\n vint8m1_t x_li = __riscv_vreinterpret_v_u8m1_i8m1(x_l);\n\n // subtract offset\n vint8m1_t v0 = __riscv_vsub_vx_i8m1(x_ai, 8, vl);\n vint8m1_t v1 = __riscv_vsub_vx_i8m1(x_li, 8, vl);\n\n vint16m2_t vec_mul1 = __riscv_vwmul_vv_i16m2(v0, y0, vl);\n vint16m2_t vec_mul2 = __riscv_vwmacc_vv_i16m2(vec_mul1, v1, y1, vl);\n\n vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);\n vint32m1_t vs2 = __riscv_vwredsum_vs_i16m2_i32m1(vec_mul2, vec_zero, vl);\n\n int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);\n\n sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n }\n\n *s = sumf;\n#else\n ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined(__riscv_v)\n const int qk = QK8_1;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n size_t vl = qk / 2;\n\n for (; ib < nb; ++ib) {\n // load elements\n vuint8m1_t tx = __riscv_vle8_v_u8m1(x[ib].qs, vl);\n\n vint8m1_t y0 = __riscv_vle8_v_i8m1(y[ib].qs, vl);\n vint8m1_t y1 = __riscv_vle8_v_i8m1(y[ib].qs+16, vl);\n\n // mask and store lower part of x, and then upper part\n vuint8m1_t x_a = __riscv_vand_vx_u8m1(tx, 0x0F, vl);\n vuint8m1_t x_l = __riscv_vsrl_vx_u8m1(tx, 0x04, vl);\n\n vint8m1_t v0 = __riscv_vreinterpret_v_u8m1_i8m1(x_a);\n vint8m1_t v1 = __riscv_vreinterpret_v_u8m1_i8m1(x_l);\n\n vint16m2_t vec_mul1 = __riscv_vwmul_vv_i16m2(v0, y0, vl);\n vint16m2_t vec_mul2 = __riscv_vwmacc_vv_i16m2(vec_mul1, v1, y1, vl);\n\n vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);\n vint32m1_t vs2 = __riscv_vwredsum_vs_i16m2_i32m1(vec_mul2, vec_zero, vl);\n\n int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);\n\n sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n }\n\n *s = sumf;\n#else\n ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined(__riscv_v)\n const int qk = QK8_0;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n size_t vl;\n size_t vlenb = __riscv_vlenb();\n\n for (; ib < nb; ++ib) {\n vl = qk / 2;\n vuint8m1_t v0 = __riscv_vle8_v_u8m1(x[ib].qs, vl);\n vint8m1_t v0l = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(v0, 0x0F, vl));\n vint8m1_t v0h = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(v0, 4, vl));\n vint8m2_t v0c;\n if (vlenb == 16) {\n v0c = __riscv_vcreate_v_i8m1_i8m2(v0l, v0h);\n } else {\n v0l = __riscv_vslideup_vx_i8m1(v0l, v0h, 16, 32);\n v0c = __riscv_vlmul_ext_v_i8m1_i8m2(v0l);\n }\n\n vl = qk;\n vbool4_t qh = __riscv_vlm_v_b4(x[ib].qh, vl);\n qh = __riscv_vmnand_mm_b4(qh, qh, vl);\n vint8m2_t v0f = __riscv_vsub_vx_i8m2_mu(qh, v0c, v0c, 0x10, vl);\n vint8m2_t v1 = __riscv_vle8_v_i8m2(y[ib].qs, vl);\n vint16m4_t mul = __riscv_vwmul_vv_i16m4(v0f, v1, vl);\n vint32m1_t zero = __riscv_vmv_v_x_i32m1(0, vl);\n vint32m1_t sum = __riscv_vwredsum_vs_i16m4_i32m1(mul, zero, vl);\n int32_t sumi = __riscv_vmv_x_s_i32m1_i32(sum);\n\n sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d)) * sumi;\n }\n\n *s = sumf;\n#else\n ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined(__riscv_v)\n const int qk = QK8_1;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_1);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n size_t vl;\n size_t vlenb = __riscv_vlenb();\n\n for (; ib < nb; ++ib) {\n vl = qk / 2;\n vuint8m1_t v0 = __riscv_vle8_v_u8m1(x[ib].qs, vl);\n vint8m1_t v0l = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(v0, 0x0F, vl));\n vint8m1_t v0h = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(v0, 4, vl));\n vint8m2_t v0c;\n if (vlenb == 16) {\n v0c = __riscv_vcreate_v_i8m1_i8m2(v0l, v0h);\n } else {\n v0l = __riscv_vslideup_vx_i8m1(v0l, v0h, 16, 32);\n v0c = __riscv_vlmul_ext_v_i8m1_i8m2(v0l);\n }\n\n vl = qk;\n vbool4_t qh = __riscv_vlm_v_b4(x[ib].qh, vl);\n vint8m2_t v0f = __riscv_vor_vx_i8m2_mu(qh, v0c, v0c, 0x10, vl);\n vint8m2_t v1 = __riscv_vle8_v_i8m2(y[ib].qs, vl);\n vint16m4_t mul = __riscv_vwmul_vv_i16m4(v0f, v1, vl);\n vint32m1_t zero = __riscv_vmv_v_x_i32m1(0, vl);\n vint32m1_t sum = __riscv_vwredsum_vs_i16m4_i32m1(mul, zero, vl);\n int32_t sumi = __riscv_vmv_x_s_i32m1_i32(sum);\n\n sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n }\n\n *s = sumf;\n#else\n ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q8_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__riscv_v)\n size_t vl = qk;\n\n for (; ib < nb; ++ib) {\n // load elements\n vint8m2_t bx_0 = __riscv_vle8_v_i8m2(x[ib].qs, vl);\n vint8m2_t by_0 = __riscv_vle8_v_i8m2(y[ib].qs, vl);\n\n vint16m4_t vw_mul = __riscv_vwmul_vv_i16m4(bx_0, by_0, vl);\n\n vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl);\n vint32m1_t v_sum = __riscv_vwredsum_vs_i16m4_i32m1(vw_mul, v_zero, vl);\n\n int sumi = __riscv_vmv_x_s_i32m1_i32(v_sum);\n\n sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));\n }\n\n *s = sumf;\n#else\n\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n\n ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q2_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __riscv_xtheadvector\n\n float sumf = 0;\n uint8_t atmp[16];\n\n for (int i = 0; i < nb; ++i) {\n const uint8_t * q2 = x[i].qs;\n const int8_t * q8 = y[i].qs;\n const uint8_t * sc = x[i].scales;\n const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n uint8_t *patmp = atmp;\n int vsums;\n int tmp;\n __asm__ __volatile__(\n \"th.vsetvli zero, %[vl16], e8, m1\\n\\t\"\n \"th.vmv.v.x v8, zero\\n\\t\"\n \"th.vlb.v v1, (%[sc])\\n\\t\"\n \"th.vand.vi v0, v1, 0xF\\n\\t\"\n \"th.vsrl.vi v1, v1, 4\\n\\t\"\n \"th.vsb.v v0, (%[scale])\\n\\t\"\n \"th.vwaddu.vx v16, v1, zero\\n\\t\"\n \"th.vsetvli zero, %[vl16], e16, m2\\n\\t\"\n \"th.vlh.v v2, (%[bsums])\\n\\t\"\n \"th.vwmul.vv v4, v16, v2\\n\\t\"\n \"th.vsetvli zero, %[vl16], e32, m4\\n\\t\"\n \"th.vredsum.vs v8, v4, v8\\n\\t\"\n \"th.vmv.x.s %[vsums], v8\"\n : [tmp] \"=&r\" (tmp), [vsums] \"=&r\" (vsums)\n : [sc] \"r\" (sc), [scale] \"r\" (atmp), [bsums] \"r\" (y[i].bsums)\n , [vl16] \"r\" (16)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n sumf += dmin * vsums;\n int isum = 0;\n\n for (int j = 0; j < QK_K/128; ++j) {\n __asm__ __volatile__(\n \"th.vsetvli zero, %[vl32], e8, m2\\n\\t\"\n \"th.vlb.v v0, (%[q2])\\n\\t\"\n \"th.vsrl.vi v2, v0, 2\\n\\t\"\n \"th.vsrl.vi v4, v0, 4\\n\\t\"\n \"th.vsrl.vi v6, v0, 6\\n\\t\"\n \"th.vand.vi v0, v0, 0x3\\n\\t\"\n \"th.vand.vi v2, v2, 0x3\\n\\t\"\n \"th.vand.vi v4, v4, 0x3\\n\\t\"\n \"th.vsetvli zero, %[vl128], e8, m8\\n\\t\"\n \"th.vlb.v v8, (%[q8])\\n\\t\"\n \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\"\n \"th.vwmul.vv v16, v0, v8\\n\\t\"\n \"th.vwmul.vv v24, v4, v12\\n\\t\"\n \"th.vsetvli zero, %[vl16], e16, m2\\n\\t\"\n \"th.vmv.v.x v0, zero\\n\\t\"\n \"th.vwredsum.vs v10, v16, v0\\n\\t\"\n \"th.vwredsum.vs v9, v18, v0\\n\\t\"\n \"th.vwredsum.vs v8, v20, v0\\n\\t\"\n \"th.vwredsum.vs v7, v22, v0\\n\\t\"\n \"th.vwredsum.vs v11, v24, v0\\n\\t\"\n \"th.vwredsum.vs v12, v26, v0\\n\\t\"\n \"th.vwredsum.vs v13, v28, v0\\n\\t\"\n \"th.vwredsum.vs v14, v30, v0\\n\\t\"\n \"li %[tmp], 4\\n\\t\"\n \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n \"th.vslideup.vi v10, v9, 1\\n\\t\"\n \"th.vslideup.vi v8, v7, 1\\n\\t\"\n \"th.vslideup.vi v11, v12, 1\\n\\t\"\n \"th.vslideup.vi v13, v14, 1\\n\\t\"\n \"th.vslideup.vi v10, v8, 2\\n\\t\"\n \"th.vslideup.vi v11, v13, 2\\n\\t\"\n \"li %[tmp], 8\\n\\t\"\n \"th.vsetvli zero, %[tmp], e32, m2\\n\\t\"\n \"th.vlbu.v v12, (%[scale])\\n\\t\"\n \"th.vmul.vv v10, v10, v12\\n\\t\"\n \"th.vredsum.vs v0, v10, v0\\n\\t\"\n \"th.vmv.x.s %[tmp], v0\\n\\t\"\n \"add %[isum], %[isum], %[tmp]\"\n : [tmp] \"=&r\" (tmp), [isum] \"+&r\" (isum)\n : [q2] \"r\" (q2), [scale] \"r\" (patmp), [q8] \"r\" (q8)\n , [vl16] \"r\" (16), [vl32] \"r\" (32), [vl64] \"r\" (64), [vl128] \"r\" (128)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n q2 += 32; q8 += 128; patmp += 8;\n }\n\n sumf += dall * isum;\n }\n\n *s = sumf;\n\n#elif defined __riscv_v\n\n float sumf = 0;\n uint8_t atmp[16];\n\n const int vector_length = __riscv_vlenb() * 8;\n uint8_t temp_01[32] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };\n\n switch (vector_length) {\n case 256:\n for (int i = 0; i < nb; ++i) {\n const uint8_t * q2 = x[i].qs;\n const int8_t * q8 = y[i].qs;\n const uint8_t * sc = x[i].scales;\n\n const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n size_t vl = 16;\n\n vuint8m1_t scales = __riscv_vle8_v_u8m1(sc, vl);\n vuint8m1_t aux = __riscv_vand_vx_u8m1(scales, 0x0F, vl);\n\n vint16m1_t q8sums = __riscv_vle16_v_i16m1(y[i].bsums, vl);\n\n vuint8mf2_t scales_2 = __riscv_vle8_v_u8mf2(sc, vl);\n vuint8mf2_t mins8 = __riscv_vsrl_vx_u8mf2(scales_2, 0x4, vl);\n vint16m1_t mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl));\n vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, mins, vl);\n vint32m1_t vsums = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);\n\n sumf += dmin * __riscv_vmv_x_s_i32m1_i32(vsums);\n\n vl = 32;\n\n vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);\n vuint8m1_t v_b = __riscv_vle8_v_u8m1(temp_01, vl);\n\n uint8_t is = 0;\n int isum = 0;\n\n for (int j = 0; j < QK_K / 128; ++j) {\n // load Q2\n vuint8m1_t q2_x = __riscv_vle8_v_u8m1(q2, vl);\n\n vuint8m1_t q2_0 = __riscv_vand_vx_u8m1(q2_x, 0x03, vl);\n vuint8m1_t q2_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x2, vl), 0x03, vl);\n vuint8m1_t q2_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x4, vl), 0x03, vl);\n vuint8m1_t q2_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x6, vl), 0x03, vl);\n\n // duplicate scale elements for product\n vuint8m1_t sc0 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 0 + is, vl), vl);\n vuint8m1_t sc1 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 2 + is, vl), vl);\n vuint8m1_t sc2 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 4 + is, vl), vl);\n vuint8m1_t sc3 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 6 + is, vl), vl);\n\n vint16m2_t p0 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_0, sc0, vl));\n vint16m2_t p1 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_1, sc1, vl));\n vint16m2_t p2 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_2, sc2, vl));\n vint16m2_t p3 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_3, sc3, vl));\n\n // load Q8\n vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);\n vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8 + 32, vl);\n vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8 + 64, vl);\n vint8m1_t q8_3 = __riscv_vle8_v_i8m1(q8 + 96, vl);\n\n vint32m4_t s0 = __riscv_vwmul_vv_i32m4(p0, __riscv_vwcvt_x_x_v_i16m2(q8_0, vl), vl);\n vint32m4_t s1 = __riscv_vwmul_vv_i32m4(p1, __riscv_vwcvt_x_x_v_i16m2(q8_1, vl), vl);\n vint32m4_t s2 = __riscv_vwmul_vv_i32m4(p2, __riscv_vwcvt_x_x_v_i16m2(q8_2, vl), vl);\n vint32m4_t s3 = __riscv_vwmul_vv_i32m4(p3, __riscv_vwcvt_x_x_v_i16m2(q8_3, vl), vl);\n\n vint32m1_t isum0 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s0, s1, vl), vzero, vl);\n vint32m1_t isum1 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s2, s3, vl), isum0, vl);\n\n isum += __riscv_vmv_x_s_i32m1_i32(isum1);\n\n q2 += 32;\n q8 += 128;\n is = 8;\n }\n\n sumf += dall * isum;\n }\n break;\n case 128:\n for (int i = 0; i < nb; ++i) {\n const uint8_t * q2 = x[i].qs;\n const int8_t * q8 = y[i].qs;\n const uint8_t * sc = x[i].scales;\n const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n uint8_t *patmp = atmp;\n int vsums;\n int tmp, t1, t2, t3, t4, t5, t6, t7;\n __asm__ __volatile__(\n \"vsetivli zero, 16, e8, m1\\n\\t\"\n \"vmv.v.x v8, zero\\n\\t\"\n \"lb zero, 15(%[sc])\\n\\t\"\n \"vle8.v v1, (%[sc])\\n\\t\"\n \"vle8.v v2, (%[bsums])\\n\\t\"\n \"addi %[tmp], %[bsums], 16\\n\\t\"\n \"vand.vi v0, v1, 0xF\\n\\t\"\n \"vsrl.vi v1, v1, 4\\n\\t\"\n \"vle8.v v3, (%[tmp])\\n\\t\"\n \"vse8.v v0, (%[scale])\\n\\t\"\n \"vsetivli zero, 16, e16, m2\\n\\t\"\n \"vzext.vf2 v0, v1\\n\\t\"\n \"vwmul.vv v4, v0, v2\\n\\t\"\n \"vsetivli zero, 16, e32, m4\\n\\t\"\n \"vredsum.vs v8, v4, v8\\n\\t\"\n \"vmv.x.s %[vsums], v8\"\n : [tmp] \"=&r\" (tmp), [vsums] \"=&r\" (vsums)\n : [sc] \"r\" (sc), [scale] \"r\" (atmp), [bsums] \"r\" (y[i].bsums)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n sumf += dmin * vsums;\n int isum = 0;\n\n for (int j = 0; j < QK_K/128; ++j) {\n __asm__ __volatile__(\n \"lb zero, 31(%[q2])\\n\\t\"\n \"addi %[tmp], %[q2], 16\\n\\t\"\n \"addi %[t1], %[q8], 16\\n\\t\"\n \"vsetivli zero, 16, e8, m1\\n\\t\"\n \"vle8.v v0, (%[q2])\\n\\t\"\n \"vle8.v v1, (%[tmp])\\n\\t\"\n \"vsrl.vi v2, v0, 2\\n\\t\"\n \"vsrl.vi v3, v1, 2\\n\\t\"\n \"vsrl.vi v4, v0, 4\\n\\t\"\n \"addi %[tmp], %[q8], 32\\n\\t\"\n \"vle8.v v8, (%[q8])\\n\\t\"\n \"vle8.v v9, (%[t1])\\n\\t\"\n \"addi %[t1], %[t1], 32\\n\\t\"\n \"vsrl.vi v5, v1, 4\\n\\t\"\n \"vsrl.vi v6, v0, 6\\n\\t\"\n \"vsrl.vi v7, v1, 6\\n\\t\"\n \"vle8.v v10, (%[tmp])\\n\\t\"\n \"vle8.v v11, (%[t1])\\n\\t\"\n \"addi %[tmp], %[tmp], 32\\n\\t\"\n \"addi %[t1], %[t1], 32\\n\\t\"\n \"vand.vi v0, v0, 0x3\\n\\t\"\n \"vand.vi v1, v1, 0x3\\n\\t\"\n \"vand.vi v2, v2, 0x3\\n\\t\"\n \"vle8.v v12, (%[tmp])\\n\\t\"\n \"vle8.v v13, (%[t1])\\n\\t\"\n \"addi %[tmp], %[tmp], 32\\n\\t\"\n \"addi %[t1], %[t1], 32\\n\\t\"\n \"vand.vi v3, v3, 0x3\\n\\t\"\n \"vand.vi v4, v4, 0x3\\n\\t\"\n \"vand.vi v5, v5, 0x3\\n\\t\"\n \"vle8.v v14, (%[tmp])\\n\\t\"\n \"vle8.v v15, (%[t1])\\n\\t\"\n \"vwmul.vv v16, v0, v8\\n\\t\"\n \"vwmul.vv v18, v1, v9\\n\\t\"\n \"vwmul.vv v20, v2, v10\\n\\t\"\n \"vwmul.vv v22, v3, v11\\n\\t\"\n \"vwmul.vv v24, v4, v12\\n\\t\"\n \"vwmul.vv v26, v5, v13\\n\\t\"\n \"vwmul.vv v28, v6, v14\\n\\t\"\n \"vwmul.vv v30, v7, v15\\n\\t\"\n \"vsetivli zero, 8, e16, m1\\n\\t\"\n \"vmv.v.x v0, zero\\n\\t\"\n \"lbu %[tmp], 0(%[scale])\\n\\t\"\n \"vwredsum.vs v8, v16, v0\\n\\t\"\n \"vwredsum.vs v9, v18, v0\\n\\t\"\n \"lbu %[t1], 1(%[scale])\\n\\t\"\n \"vwredsum.vs v10, v20, v0\\n\\t\"\n \"vwredsum.vs v11, v22, v0\\n\\t\"\n \"lbu %[t2], 2(%[scale])\\n\\t\"\n \"vwredsum.vs v12, v24, v0\\n\\t\"\n \"vwredsum.vs v13, v26, v0\\n\\t\"\n \"lbu %[t3], 3(%[scale])\\n\\t\"\n \"vwredsum.vs v14, v28, v0\\n\\t\"\n \"vwredsum.vs v15, v30, v0\\n\\t\"\n \"lbu %[t4], 4(%[scale])\\n\\t\"\n \"vwredsum.vs v8, v17, v8\\n\\t\"\n \"vwredsum.vs v9, v19, v9\\n\\t\"\n \"lbu %[t5], 5(%[scale])\\n\\t\"\n \"vwredsum.vs v10, v21, v10\\n\\t\"\n \"vwredsum.vs v11, v23, v11\\n\\t\"\n \"lbu %[t6], 6(%[scale])\\n\\t\"\n \"vwredsum.vs v12, v25, v12\\n\\t\"\n \"vwredsum.vs v13, v27, v13\\n\\t\"\n \"lbu %[t7], 7(%[scale])\\n\\t\"\n \"vwredsum.vs v14, v29, v14\\n\\t\"\n \"vwredsum.vs v15, v31, v15\\n\\t\"\n \"vsetivli zero, 4, e32, m1\\n\\t\"\n \"vmul.vx v0, v8, %[tmp]\\n\\t\"\n \"vmul.vx v1, v9, %[t1]\\n\\t\"\n \"vmacc.vx v0, %[t2], v10\\n\\t\"\n \"vmacc.vx v1, %[t3], v11\\n\\t\"\n \"vmacc.vx v0, %[t4], v12\\n\\t\"\n \"vmacc.vx v1, %[t5], v13\\n\\t\"\n \"vmacc.vx v0, %[t6], v14\\n\\t\"\n \"vmacc.vx v1, %[t7], v15\\n\\t\"\n \"vmv.x.s %[tmp], v0\\n\\t\"\n \"vmv.x.s %[t1], v1\\n\\t\"\n \"add %[isum], %[isum], %[tmp]\\n\\t\"\n \"add %[isum], %[isum], %[t1]\"\n : [tmp] \"=&r\" (tmp), [t1] \"=&r\" (t1), [t2] \"=&r\" (t2), [t3] \"=&r\" (t3)\n , [t4] \"=&r\" (t4), [t5] \"=&r\" (t5), [t6] \"=&r\" (t6), [t7] \"=&r\" (t7)\n , [isum] \"+&r\" (isum)\n : [q2] \"r\" (q2), [scale] \"r\" (patmp), [q8] \"r\" (q8)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n q2 += 32; q8 += 128; patmp += 8;\n }\n\n sumf += dall * isum;\n }\n break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n\n *s = sumf;\n\n#else\n\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n\n ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const uint32_t kmask1 = 0x03030303;\n const uint32_t kmask2 = 0x0f0f0f0f;\n\n const block_q3_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __riscv_xtheadvector\n\n uint32_t utmp[4];\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n const uint8_t * restrict q3 = x[i].qs;\n const uint8_t * restrict qh = x[i].hmask;\n const int8_t * restrict q8 = y[i].qs;\n\n int8_t * scale = (int8_t *)utmp;\n int tmp;\n __asm__ __volatile__(\n \"li %[tmp], 12\\n\\t\"\n \"th.vsetvli zero, %[tmp], e8, m1\\n\\t\"\n \"th.vlb.v v0, (%[s6b])\\n\\t\"\n \"th.vmv.v.v v2, v0\\n\\t\"\n \"li %[tmp], 2\\n\\t\"\n \"th.vsetvli zero, %[tmp], e64, m1\\n\\t\"\n \"th.vmv.v.x v9, %[sh]\\n\\t\"\\\n \"th.vslidedown.vi v1, v0, 1\\n\\t\"\n \"th.vslide1up.vx v8, v9, zero\\n\\t\" // {0, 0, 4, 4}\n \"th.vslideup.vi v0, v2, 1\\n\\t\" // {aux[0], aux[1], aux[0], aux[1]}\n \"li %[tmp], 4\\n\\t\"\n \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n \"th.vid.v v9\\n\\t\"\n \"th.vmv.x.s %[tmp], v1\\n\\t\"\n \"th.vsll.vi v9, v9, 1\\n\\t\" // {0, 2, 4, 6}\n \"th.vmv.v.x v1, %[tmp]\\n\\t\" // {aux[2], aux[2], aux[2], aux[2]}\n \"th.vsrl.vv v4, v1, v9\\n\\t\"\n \"th.vsrl.vv v2, v0, v8\\n\\t\"\n \"th.vand.vx v5, v4, %[kmask1]\\n\\t\"\n \"th.vand.vx v3, v2, %[kmask2]\\n\\t\"\n \"th.vsll.vi v6, v5, 4\\n\\t\"\n \"th.vor.vv v7, v6, v3\\n\\t\"\n \"li %[tmp], 16\\n\\t\"\n \"th.vsetvli zero, %[tmp], e8, m1\\n\\t\"\n \"th.vsub.vx v0, v7, %[c]\\n\\t\"\n \"th.vsb.v v0, (%[scale])\"\n : [tmp] \"=&r\" (tmp)\n : [sh] \"r\" (0x0000000400000004), [s6b] \"r\" (x[i].scales), [c] \"r\" (32)\n , [scale] \"r\" (scale), [kmask1] \"r\" (kmask1), [kmask2] \"r\" (kmask2)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n\n uint8_t m = 1;\n int isum = 0;\n for (int j = 0; j < QK_K; j += 128) {\n __asm__ __volatile__(\n // fixme: use v0p7 mask layout directly\n \"th.vsetvli zero, %[vl32], e8, m2\\n\\t\"\n \"th.vlb.v v8, (%[q3])\\n\\t\"\n \"th.vsrl.vi v10, v8, 2\\n\\t\"\n \"th.vsrl.vi v12, v8, 4\\n\\t\"\n \"th.vsrl.vi v14, v8, 6\\n\\t\"\n \"th.vand.vi v8, v8, 3\\n\\t\"\n \"th.vand.vi v10, v10, 3\\n\\t\"\n \"th.vand.vi v12, v12, 3\\n\\t\"\n \"th.vlb.v v2, (%[qh])\\n\\t\"\n \"th.vand.vx v4, v2, %[m]\\n\\t\"\n \"slli %[m], %[m], 1\\n\\t\"\n \"th.vmseq.vx v0, v4, zero\\n\\t\"\n \"th.vadd.vi v8, v8, -4, v0.t\\n\\t\"\n \"th.vand.vx v4, v2, %[m]\\n\\t\"\n \"slli %[m], %[m], 1\\n\\t\"\n \"th.vmseq.vx v0, v4, zero\\n\\t\"\n \"th.vadd.vi v10, v10, -4, v0.t\\n\\t\"\n \"th.vand.vx v4, v2, %[m]\\n\\t\"\n \"slli %[m], %[m], 1\\n\\t\"\n \"th.vmseq.vx v0, v4, zero\\n\\t\"\n \"th.vadd.vi v12, v12, -4, v0.t\\n\\t\"\n \"th.vand.vx v4, v2, %[m]\\n\\t\"\n \"slli %[m], %[m], 1\\n\\t\"\n \"th.vmseq.vx v0, v4, zero\\n\\t\"\n \"th.vadd.vi v14, v14, -4, v0.t\\n\\t\"\n \"th.vsetvli zero, %[vl128], e8, m8\\n\\t\"\n \"th.vlb.v v0, (%[q8])\\n\\t\"\n \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\"\n \"th.vwmul.vv v16, v0, v8\\n\\t\"\n \"th.vwmul.vv v24, v4, v12\\n\\t\"\n \"li %[tmp], 16\\n\\t\"\n \"th.vsetvli zero, %[tmp], e16, m2\\n\\t\"\n \"th.vmv.v.x v0, zero\\n\\t\"\n \"th.vwredsum.vs v10, v16, v0\\n\\t\"\n \"th.vwredsum.vs v9, v18, v0\\n\\t\"\n \"th.vwredsum.vs v8, v20, v0\\n\\t\"\n \"th.vwredsum.vs v7, v22, v0\\n\\t\"\n \"th.vwredsum.vs v11, v24, v0\\n\\t\"\n \"th.vwredsum.vs v12, v26, v0\\n\\t\"\n \"th.vwredsum.vs v13, v28, v0\\n\\t\"\n \"th.vwredsum.vs v14, v30, v0\\n\\t\"\n \"li %[tmp], 4\\n\\t\"\n \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n \"th.vslideup.vi v10, v9, 1\\n\\t\"\n \"th.vslideup.vi v8, v7, 1\\n\\t\"\n \"th.vslideup.vi v11, v12, 1\\n\\t\"\n \"th.vslideup.vi v13, v14, 1\\n\\t\"\n \"th.vslideup.vi v10, v8, 2\\n\\t\"\n \"th.vslideup.vi v11, v13, 2\\n\\t\"\n \"li %[tmp], 8\\n\\t\"\n \"th.vsetvli zero, %[tmp], e32, m2\\n\\t\"\n \"th.vlb.v v12, (%[scale])\\n\\t\"\n \"th.vmul.vv v10, v10, v12\\n\\t\"\n \"th.vredsum.vs v0, v10, v0\\n\\t\"\n \"th.vmv.x.s %[tmp], v0\\n\\t\"\n \"add %[isum], %[isum], %[tmp]\"\n : [tmp] \"=&r\" (tmp), [m] \"+&r\" (m), [isum] \"+&r\" (isum)\n : [vl128] \"r\" (128), [vl64] \"r\" (64), [vl32] \"r\" (32)\n , [q3] \"r\" (q3), [qh] \"r\" (qh), [scale] \"r\" (scale), [q8] \"r\" (q8)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n q3 += 32; q8 += 128; scale += 8;\n }\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n sumf += d * isum;\n }\n\n *s = sumf;\n\n#elif defined __riscv_v\n\n uint32_t utmp[4];\n float sumf = 0;\n uint32_t aux[3];\n const int vector_length = __riscv_vlenb() * 8;\n\n switch (vector_length) {\n case 256:\n for (int i = 0; i < nb; ++i) {\n\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].hmask;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n memcpy(aux, x[i].scales, 12);\n utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);\n utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);\n utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);\n utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);\n\n int8_t * scale = (int8_t *)utmp;\n for (int j = 0; j < 16; ++j) scale[j] -= 32;\n\n\n size_t vl = 32;\n uint8_t m = 1;\n\n vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);\n vuint8m1_t vqh = __riscv_vle8_v_u8m1(qh, vl);\n\n int sum_t = 0;\n\n for (int j = 0; j < QK_K; j += 128) {\n\n vl = 32;\n\n // load Q3\n vuint8m1_t q3_x = __riscv_vle8_v_u8m1(q3, vl);\n\n vint8m1_t q3_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q3_x, 0x03, vl));\n vint8m1_t q3_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x2, vl), 0x03 , vl));\n vint8m1_t q3_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x4, vl), 0x03 , vl));\n vint8m1_t q3_3 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x6, vl), 0x03 , vl));\n\n // compute mask for subtraction\n vuint8m1_t qh_m0 = __riscv_vand_vx_u8m1(vqh, m, vl);\n vbool8_t vmask_0 = __riscv_vmseq_vx_u8m1_b8(qh_m0, 0, vl);\n vint8m1_t q3_m0 = __riscv_vsub_vx_i8m1_mu(vmask_0, q3_0, q3_0, 0x4, vl);\n m <<= 1;\n\n vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);\n vbool8_t vmask_1 = __riscv_vmseq_vx_u8m1_b8(qh_m1, 0, vl);\n vint8m1_t q3_m1 = __riscv_vsub_vx_i8m1_mu(vmask_1, q3_1, q3_1, 0x4, vl);\n m <<= 1;\n\n vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);\n vbool8_t vmask_2 = __riscv_vmseq_vx_u8m1_b8(qh_m2, 0, vl);\n vint8m1_t q3_m2 = __riscv_vsub_vx_i8m1_mu(vmask_2, q3_2, q3_2, 0x4, vl);\n m <<= 1;\n\n vuint8m1_t qh_m3 = __riscv_vand_vx_u8m1(vqh, m, vl);\n vbool8_t vmask_3 = __riscv_vmseq_vx_u8m1_b8(qh_m3, 0, vl);\n vint8m1_t q3_m3 = __riscv_vsub_vx_i8m1_mu(vmask_3, q3_3, q3_3, 0x4, vl);\n m <<= 1;\n\n // load Q8 and take product with Q3\n vint16m2_t a0 = __riscv_vwmul_vv_i16m2(q3_m0, __riscv_vle8_v_i8m1(q8, vl), vl);\n vint16m2_t a1 = __riscv_vwmul_vv_i16m2(q3_m1, __riscv_vle8_v_i8m1(q8+32, vl), vl);\n vint16m2_t a2 = __riscv_vwmul_vv_i16m2(q3_m2, __riscv_vle8_v_i8m1(q8+64, vl), vl);\n vint16m2_t a3 = __riscv_vwmul_vv_i16m2(q3_m3, __riscv_vle8_v_i8m1(q8+96, vl), vl);\n\n vl = 16;\n\n // retrieve lane to multiply with scale\n vint32m2_t aux0_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 0), (scale[0]), vl);\n vint32m2_t aux0_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 1), (scale[1]), vl);\n vint32m2_t aux1_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 0), (scale[2]), vl);\n vint32m2_t aux1_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 1), (scale[3]), vl);\n vint32m2_t aux2_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 0), (scale[4]), vl);\n vint32m2_t aux2_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 1), (scale[5]), vl);\n vint32m2_t aux3_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 0), (scale[6]), vl);\n vint32m2_t aux3_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 1), (scale[7]), vl);\n\n vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux0_0, aux0_1, vl), vzero, vl);\n vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux1_0, aux1_1, vl), isum0, vl);\n vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux2_0, aux2_1, vl), isum1, vl);\n vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux3_0, aux3_1, vl), isum2, vl);\n\n sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);\n\n q3 += 32; q8 += 128; scale += 8;\n\n }\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n sumf += d*sum_t;\n\n }\n break;\n case 128:\n for (int i = 0; i < nb; ++i) {\n const uint8_t * restrict q3 = x[i].qs;\n const uint8_t * restrict qh = x[i].hmask;\n const int8_t * restrict q8 = y[i].qs;\n\n int8_t * scale = (int8_t *)utmp;\n int tmp, t1, t2, t3, t4, t5, t6, t7;\n __asm__ __volatile__(\n \"vsetivli zero, 12, e8, m1\\n\\t\"\n \"vle8.v v0, (%[s6b])\\n\\t\"\n \"vmv1r.v v2, v0\\n\\t\"\n \"vsetivli zero, 2, e64, m1\\n\\t\"\n \"vmv.v.x v9, %[sh]\\n\\t\"\\\n \"vslidedown.vi v1, v0, 1\\n\\t\"\n \"vslide1up.vx v8, v9, zero\\n\\t\" // {0, 0, 4, 4}\n \"vslideup.vi v0, v2, 1\\n\\t\" // {aux[0], aux[1], aux[0], aux[1]}\n \"vsetivli zero, 4, e32, m1\\n\\t\"\n \"vid.v v9\\n\\t\"\n \"vmv.x.s %[tmp], v1\\n\\t\"\n \"vsll.vi v9, v9, 1\\n\\t\" // {0, 2, 4, 6}\n \"vmv.v.x v1, %[tmp]\\n\\t\" // {aux[2], aux[2], aux[2], aux[2]}\n \"vsrl.vv v4, v1, v9\\n\\t\"\n \"vsrl.vv v2, v0, v8\\n\\t\"\n \"vand.vx v5, v4, %[kmask1]\\n\\t\"\n \"vand.vx v3, v2, %[kmask2]\\n\\t\"\n \"vsll.vi v6, v5, 4\\n\\t\"\n \"vor.vv v7, v6, v3\\n\\t\"\n \"vsetivli zero, 16, e8, m1\\n\\t\"\n \"vsub.vx v0, v7, %[c]\\n\\t\"\n \"vse8.v v0, (%[scale])\"\n : [tmp] \"=&r\" (tmp)\n : [sh] \"r\" (0x0000000400000004), [s6b] \"r\" (x[i].scales), [c] \"r\" (32)\n , [scale] \"r\" (scale), [kmask1] \"r\" (kmask1), [kmask2] \"r\" (kmask2)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n\n uint8_t m = 1;\n int isum = 0;\n for (int j = 0; j < QK_K; j += 128) {\n __asm__ __volatile__(\n \"lb zero, 31(%[q3])\\n\\t\"\n \"vsetvli zero, %[vl32], e8, m2, ta, mu\\n\\t\"\n \"vle8.v v8, (%[q3])\\n\\t\"\n \"vsrl.vi v10, v8, 2\\n\\t\"\n \"vsrl.vi v12, v8, 4\\n\\t\"\n \"vsrl.vi v14, v8, 6\\n\\t\"\n \"lb zero, 64(%[q8])\\n\\t\"\n \"vand.vi v8, v8, 3\\n\\t\"\n \"vand.vi v10, v10, 3\\n\\t\"\n \"vand.vi v12, v12, 3\\n\\t\"\n \"vle8.v v2, (%[qh])\\n\\t\"\n \"lb zero, 127(%[q8])\\n\\t\"\n \"vand.vx v4, v2, %[m]\\n\\t\"\n \"slli %[m], %[m], 1\\n\\t\"\n \"vmseq.vx v0, v4, zero\\n\\t\"\n \"vadd.vi v8, v8, -4, v0.t\\n\\t\"\n \"lb zero, 0(%[q8])\\n\\t\"\n \"vand.vx v4, v2, %[m]\\n\\t\"\n \"slli %[m], %[m], 1\\n\\t\"\n \"vmseq.vx v0, v4, zero\\n\\t\"\n \"vadd.vi v10, v10, -4, v0.t\\n\\t\"\n \"vand.vx v4, v2, %[m]\\n\\t\"\n \"slli %[m], %[m], 1\\n\\t\"\n \"vmseq.vx v0, v4, zero\\n\\t\"\n \"vadd.vi v12, v12, -4, v0.t\\n\\t\"\n \"vand.vx v4, v2, %[m]\\n\\t\"\n \"slli %[m], %[m], 1\\n\\t\"\n \"vmseq.vx v0, v4, zero\\n\\t\"\n \"vadd.vi v14, v14, -4, v0.t\\n\\t\"\n \"vsetvli zero, %[vl128], e8, m8\\n\\t\"\n \"vle8.v v0, (%[q8])\\n\\t\"\n \"lb %[tmp], 0(%[scale])\\n\\t\"\n \"lb %[t1], 1(%[scale])\\n\\t\"\n \"lb %[t2], 2(%[scale])\\n\\t\"\n \"lb %[t3], 3(%[scale])\\n\\t\"\n \"vsetvli zero, %[vl64], e8, m4\\n\\t\"\n \"vwmul.vv v16, v0, v8\\n\\t\"\n \"vwmul.vv v24, v4, v12\\n\\t\"\n \"vsetivli zero, 16, e16, m2\\n\\t\"\n \"vmv.v.x v0, zero\\n\\t\"\n \"vwredsum.vs v8, v16, v0\\n\\t\"\n \"lb %[t4], 4(%[scale])\\n\\t\"\n \"lb %[t5], 5(%[scale])\\n\\t\"\n \"vwredsum.vs v9, v18, v0\\n\\t\"\n \"vwredsum.vs v10, v20, v0\\n\\t\"\n \"vwredsum.vs v11, v22, v0\\n\\t\"\n \"vwredsum.vs v12, v24, v0\\n\\t\"\n \"lb %[t6], 6(%[scale])\\n\\t\"\n \"lb %[t7], 7(%[scale])\\n\\t\"\n \"vwredsum.vs v13, v26, v0\\n\\t\"\n \"vwredsum.vs v14, v28, v0\\n\\t\"\n \"vwredsum.vs v15, v30, v0\\n\\t\"\n \"vsetivli zero, 4, e32, m1\\n\\t\"\n \"vmul.vx v0, v8, %[tmp]\\n\\t\"\n \"vmul.vx v1, v9, %[t1]\\n\\t\"\n \"vmacc.vx v0, %[t2], v10\\n\\t\"\n \"vmacc.vx v1, %[t3], v11\\n\\t\"\n \"vmacc.vx v0, %[t4], v12\\n\\t\"\n \"vmacc.vx v1, %[t5], v13\\n\\t\"\n \"vmacc.vx v0, %[t6], v14\\n\\t\"\n \"vmacc.vx v1, %[t7], v15\\n\\t\"\n \"vmv.x.s %[tmp], v0\\n\\t\"\n \"vmv.x.s %[t1], v1\\n\\t\"\n \"add %[isum], %[isum], %[tmp]\\n\\t\"\n \"add %[isum], %[isum], %[t1]\"\n : [tmp] \"=&r\" (tmp), [t1] \"=&r\" (t1), [t2] \"=&r\" (t2), [t3] \"=&r\" (t3)\n , [t4] \"=&r\" (t4), [t5] \"=&r\" (t5), [t6] \"=&r\" (t6), [t7] \"=&r\" (t7)\n , [m] \"+&r\" (m), [isum] \"+&r\" (isum)\n : [vl128] \"r\" (128), [vl64] \"r\" (64), [vl32] \"r\" (32)\n , [q3] \"r\" (q3), [qh] \"r\" (qh), [scale] \"r\" (scale), [q8] \"r\" (q8)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n q3 += 32; q8 += 128; scale += 8;\n }\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n sumf += d * isum;\n }\n break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n\n *s = sumf;\n\n#else\n\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n\n ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined __riscv_xtheadvector\n\n const uint8_t * scales = (const uint8_t*)&utmp[0];\n const uint8_t * mins = (const uint8_t*)&utmp[2];\n\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n int tmp, tmp2, sumi;\n __asm__ __volatile__(\n \"li %[t1], 12\\n\\t\"\n \"th.vsetvli zero, %[t1], e8, m1\\n\\t\"\n \"th.vlb.v v1, (%[s6b])\\n\\t\" // {aux[0], aux[1], aux[2]}\n \"li %[t1], 4\\n\\t\"\n \"th.vsetvli zero, %[t1], e32, m1\\n\\t\"\n \"th.vslidedown.vi v2, v1, 2\\n\\t\"\n \"th.vmv.v.v v3, v2\\n\\t\"\n \"th.vslideup.vi v2, v3, 1\\n\\t\" // {aux[2], aux[2]}\n \"li %[t1], 2\\n\\t\"\n \"th.vsetvli zero, %[t1], e32, m1\\n\\t\"\n \"th.vmv.v.i v4, 4\\n\\t\"\n \"th.vand.vx v8, v1, %[kmask1]\\n\\t\"\n \"th.vslide1up.vx v5, v4, zero\\n\\t\" // {0, 4}\n \"th.vsrl.vi v6, v1, 6\\n\\t\"\n \"th.vsrl.vv v7, v2, v5\\n\\t\"\n \"th.vand.vx v0, v6, %[kmask3]\\n\\t\"\n \"th.vand.vx v2, v7, %[kmask2]\\n\\t\"\n \"th.vsll.vi v6, v0, 4\\n\\t\"\n \"li %[t2], 8\\n\\t\"\n \"addi %[t1], %[utmp], 4\\n\\t\"\n \"th.vor.vv v1, v6, v2\\n\\t\"\n \"th.vssw.v v8, (%[utmp]), %[t2]\\n\\t\"\n \"th.vssw.v v1, (%[t1]), %[t2]\\n\\t\"\n \"th.vsetvli zero, zero, e32, m2\\n\\t\" // vl == 8\n \"th.vlw.v v2, (%[bsums])\\n\\t\"\n \"th.vsetvli zero, %[t2], e16, m1\\n\\t\"\n \"th.vnsrl.vi v0, v2, 0\\n\\t\"\n \"th.vnsrl.vi v1, v2, 16\\n\\t\"\n \"th.vadd.vv v2, v0, v1\\n\\t\"\n \"th.vlbu.v v4, (%[mins])\\n\\t\"\n \"th.vwmul.vv v6, v4, v2\\n\\t\"\n \"th.vmv.v.x v0, zero\\n\\t\"\n \"th.vsetvli zero, %[t2], e32, m2\\n\\t\"\n \"th.vredsum.vs v0, v6, v0\\n\\t\"\n \"th.vmv.x.s %[sumi], v0\"\n : [t1] \"=&r\" (tmp), [t2] \"=&r\" (tmp2), [sumi] \"=&r\" (sumi)\n : [bsums] \"r\" (y[i].bsums), [mins] \"r\" (mins), [utmp] \"r\" (utmp)\n , [s6b] \"r\" (x[i].scales), [kmask1] \"r\" (kmask1)\n , [kmask2] \"r\" (kmask2), [kmask3] \"r\" (kmask3)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n sumf -= dmin * sumi;\n\n const uint8_t * restrict q4 = x[i].qs;\n const int8_t * restrict q8 = y[i].qs;\n\n sumi = 0;\n const uint8_t * scale = scales;\n\n for (int j = 0; j < QK_K/128; ++j) {\n int vl128 = 128, vl64 = 64, vl32 = 32;\n __asm__ __volatile__(\n \"th.vsetvli zero, %[vl128], e8, m8\\n\\t\"\n \"th.vlb.v v8, (%[q8])\\n\\t\"\n \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\"\n \"th.vlb.v v0, (%[q4])\\n\\t\"\n \"th.vsrl.vi v4, v0, 4\\n\\t\"\n \"th.vand.vi v0, v0, 0xF\\n\\t\"\n \"th.vsetvli zero, %[vl32], e8, m2\\n\\t\"\n \"th.vwmul.vv v28, v6, v14\\n\\t\"\n \"th.vwmul.vv v20, v4, v10\\n\\t\"\n \"th.vwmul.vv v24, v2, v12\\n\\t\"\n \"th.vwmul.vv v16, v0, v8\\n\\t\"\n \"li %[tmp], 4\\n\\t\"\n \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n \"th.vlbu.v v1, (%[scale])\\n\\t\"\n \"th.vmv.v.x v0, zero\\n\\t\"\n \"th.vsetvli zero, %[vl32], e16, m4\\n\\t\"\n \"th.vwredsum.vs v6, v24, v0\\n\\t\"\n \"th.vwredsum.vs v7, v28, v0\\n\\t\"\n \"th.vwredsum.vs v4, v16, v0\\n\\t\"\n \"th.vwredsum.vs v5, v20, v0\\n\\t\"\n \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n \"th.vslideup.vi v6, v7, 1\\n\\t\"\n \"th.vslideup.vi v4, v5, 1\\n\\t\"\n \"th.vslideup.vi v4, v6, 2\\n\\t\"\n \"th.vmul.vv v8, v4, v1\\n\\t\"\n \"th.vredsum.vs v0, v8, v0\\n\\t\"\n \"th.vmv.x.s %[tmp], v0\\n\\t\"\n \"add %[sumi], %[sumi], %[tmp]\"\n : [tmp] \"=&r\" (tmp), [sumi] \"+&r\" (sumi)\n : [vl128] \"r\" (vl128), [vl64] \"r\" (vl64), [vl32] \"r\" (vl32)\n , [q4] \"r\" (q4), [q8] \"r\" (q8), [scale] \"r\" (scale)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n\n q4 += 64; q8 += 128; scale += 4;\n }\n\n sumf += d * sumi;\n\n }\n\n *s = sumf;\n\n#elif defined __riscv_v\n\n const uint8_t * scales = (const uint8_t*)&utmp[0];\n const uint8_t * mins = (const uint8_t*)&utmp[2];\n\n float sumf = 0;\n const int vector_length = __riscv_vlenb() * 8;\n\n switch (vector_length) {\n case 256:\n for (int i = 0; i < nb; ++i) {\n\n size_t vl = 8;\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);\n vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);\n vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);\n vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));\n vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);\n\n vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);\n sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);\n\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n vl = 32;\n\n int32_t sum_1 = 0;\n int32_t sum_2 = 0;\n\n vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);\n\n for (int j = 0; j < QK_K/64; ++j) {\n // load Q4\n vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);\n\n // load Q8 and multiply it with lower Q4 nibble\n vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);\n vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));\n vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl);\n vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl);\n\n sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0];\n\n // load Q8 and multiply it with upper Q4 nibble\n vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);\n vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));\n vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl);\n vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl);\n\n sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1];\n\n q4 += 32; q8 += 64;\n\n }\n\n sumf += d*(sum_1 + sum_2);\n\n }\n break;\n case 128:\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n float ftmp, ft2;\n const uint8_t * restrict q40;\n const uint8_t * restrict q41;\n const uint8_t * restrict q42;\n const uint8_t * restrict q43;\n const int8_t * restrict q80;\n const int8_t * restrict q81;\n const int8_t * restrict q82;\n const int8_t * restrict q83;\n int s0, s1, s2, s3;\n\n __asm__ __volatile__(\n \"li %[s1], 8\\n\\t\"\n \"vsetivli zero, 4, e32, m1, ta, ma\\n\\t\"\n \"vle32.v v1, (%[s6b])\\n\\t\"\n \"vslide1down.vx v1, v1, zero\\n\\t\"\n \"vmv.v.x v16, zero\\n\\t\"\n \"vslidedown.vi v2, v1, 2\\n\\t\"\n \"vmv1r.v v3, v2\\n\\t\"\n \"vslideup.vi v2, v3, 1\\n\\t\" // {aux[2], aux[2]}\n \"vsetivli zero, 2, e32, m1, ta, ma\\n\\t\"\n \"vmv.v.i v4, 4\\n\\t\"\n \"vand.vx v8, v1, %[kmask1]\\n\\t\"\n \"vslide1up.vx v5, v4, zero\\n\\t\" // {0, 4}\n \"vsrl.vi v6, v1, 6\\n\\t\"\n \"vsrl.vv v7, v2, v5\\n\\t\"\n \"vsse32.v v8, (%[utmp]), %[s1]\\n\\t\"\n \"vand.vx v0, v6, %[kmask3]\\n\\t\"\n \"vand.vx v2, v7, %[kmask2]\\n\\t\"\n \"vsll.vi v6, v0, 4\\n\\t\"\n \"addi %[s0], %[utmp], 4\\n\\t\"\n \"vor.vv v1, v6, v2\\n\\t\"\n \"vsse32.v v1, (%[s0]), %[s1]\\n\\t\"\n \"vsetivli zero, 8, e16, m1, ta, ma\\n\\t\"\n \"vle32.v v2, (%[bsums])\\n\\t\"\n \"vnsrl.wi v0, v2, 0\\n\\t\"\n \"vnsrl.wi v1, v2, 16\\n\\t\"\n \"vadd.vv v2, v0, v1\\n\\t\"\n \"vle8.v v3, (%[mins])\\n\\t\"\n \"vzext.vf2 v4, v3\\n\\t\"\n \"vwmul.vv v6, v4, v2\\n\\t\"\n \"vsetivli zero, 4, e32, m1, ta, ma\\n\\t\"\n \"vredsum.vs v0, v6, v16\\n\\t\"\n \"vredsum.vs v0, v7, v0\\n\\t\"\n \"vfcvt.f.x.v v0, v0\\n\\t\"\n \"vfmv.f.s %[ftmp], v0\\n\\t\"\n \"vsetivli zero, 16, e8, m1, ta, ma\\n\\t\"\n \"vle8.v v0, (%[xs])\\n\\t\"\n \"fnmsub.s %[sumf], %[dmin], %[ftmp], %[sumf]\\n\\t\"\n \"addi %[q40], %[xs], 64\\n\\t\"\n \"addi %[q41], %[xs], 16\\n\\t\"\n \"addi %[q42], %[xs], 32\\n\\t\"\n \"addi %[q43], %[xs], 48\\n\\t\"\n \"addi %[q80], %[ys], 64\\n\\t\"\n \"vle8.v v1, (%[q41])\\n\\t\"\n \"vle8.v v2, (%[q42])\\n\\t\"\n \"addi %[q81], %[ys], 16\\n\\t\"\n \"addi %[q41], %[q41], 64\\n\\t\"\n \"addi %[q82], %[ys], 32\\n\\t\"\n \"vle8.v v3, (%[q43])\\n\\t\"\n \"vle8.v v8, (%[ys])\\n\\t\"\n \"addi %[q42], %[q42], 64\\n\\t\"\n \"addi %[q83], %[ys], 48\\n\\t\"\n \"addi %[q43], %[q43], 64\\n\\t\"\n \"vsrl.vi v4, v0, 4\\n\\t\"\n \"vle8.v v9, (%[q81])\\n\\t\"\n \"vle8.v v10, (%[q82])\\n\\t\"\n \"vand.vi v0, v0, 0xF\\n\\t\"\n \"addi %[q81], %[q81], 64\\n\\t\"\n \"vsrl.vi v5, v1, 4\\n\\t\"\n \"addi %[q82], %[q82], 64\\n\\t\"\n \"vle8.v v11, (%[q83])\\n\\t\"\n \"vle8.v v12, (%[q80])\\n\\t\"\n \"vand.vi v1, v1, 0xF\\n\\t\"\n \"addi %[q83], %[q83], 64\\n\\t\"\n \"vsrl.vi v6, v2, 4\\n\\t\"\n \"addi %[q80], %[q80], 64\\n\\t\"\n \"vle8.v v13, (%[q81])\\n\\t\"\n \"vle8.v v14, (%[q82])\\n\\t\"\n \"vand.vi v2, v2, 0xF\\n\\t\"\n \"addi %[q81], %[q81], 64\\n\\t\"\n \"vsrl.vi v7, v3, 4\\n\\t\"\n \"addi %[q82], %[q82], 64\\n\\t\"\n \"vwmul.vv v16, v0, v8\\n\\t\"\n \"vle8.v v15, (%[q83])\\n\\t\"\n \"vle8.v v0, (%[q40])\\n\\t\"\n \"vand.vi v3, v3, 0xF\\n\\t\"\n \"addi %[q83], %[q83], 64\\n\\t\"\n \"vwmul.vv v24, v2, v12\\n\\t\"\n \"vwmul.vv v20, v4, v10\\n\\t\"\n \"vwmul.vv v28, v6, v14\\n\\t\"\n \"vwmacc.vv v16, v1, v9\\n\\t\"\n \"vle8.v v1, (%[q41])\\n\\t\"\n \"vle8.v v2, (%[q42])\\n\\t\"\n \"vwmacc.vv v24, v3, v13\\n\\t\"\n \"vwmacc.vv v20, v5, v11\\n\\t\"\n \"vwmacc.vv v28, v7, v15\\n\\t\"\n \"addi %[q40], %[q80], 64\\n\\t\"\n \"addi %[q41], %[q81], 64\\n\\t\"\n \"vle8.v v3, (%[q43])\\n\\t\"\n \"vle8.v v8, (%[q80])\\n\\t\"\n \"addi %[q42], %[q82], 64\\n\\t\"\n \"addi %[q43], %[q83], 64\\n\\t\"\n \"vsrl.vi v4, v0, 4\\n\\t\"\n \"vle8.v v9, (%[q81])\\n\\t\"\n \"vle8.v v10, (%[q82])\\n\\t\"\n \"vand.vi v0, v0, 0xF\\n\\t\"\n \"vsrl.vi v5, v1, 4\\n\\t\"\n \"vsrl.vi v7, v3, 4\\n\\t\"\n \"vand.vi v3, v3, 0xF\\n\\t\"\n \"vle8.v v11, (%[q83])\\n\\t\"\n \"vle8.v v12, (%[q40])\\n\\t\"\n \"vand.vi v1, v1, 0xF\\n\\t\"\n \"vsrl.vi v6, v2, 4\\n\\t\"\n \"vand.vi v2, v2, 0xF\\n\\t\"\n \"vwmul.vv v18, v0, v8\\n\\t\"\n \"vle8.v v13, (%[q41])\\n\\t\"\n \"vle8.v v14, (%[q42])\\n\\t\"\n \"vwmul.vv v26, v2, v12\\n\\t\"\n \"vwmul.vv v22, v4, v10\\n\\t\"\n \"vwmul.vv v30, v6, v14\\n\\t\"\n \"vwmacc.vv v18, v1, v9\\n\\t\"\n \"vle8.v v15, (%[q43])\\n\\t\"\n \"vwmacc.vv v26, v3, v13\\n\\t\"\n \"vwmacc.vv v22, v5, v11\\n\\t\"\n \"vwmacc.vv v30, v7, v15\\n\\t\"\n \"vmv.v.x v0, zero\\n\\t\"\n \"vsetivli zero, 16, e16, m2, ta, ma\\n\\t\"\n \"vwredsum.vs v4, v16, v0\\n\\t\"\n \"lbu %[s0], 0(%[scale])\\n\\t\"\n \"vwredsum.vs v5, v20, v0\\n\\t\"\n \"lbu %[s1], 1(%[scale])\\n\\t\"\n \"vwredsum.vs v6, v24, v0\\n\\t\"\n \"lbu %[s2], 2(%[scale])\\n\\t\"\n \"vwredsum.vs v7, v28, v0\\n\\t\"\n \"lbu %[s3], 3(%[scale])\\n\\t\"\n \"vwredsum.vs v8, v18, v0\\n\\t\"\n \"lbu %[q40], 4(%[scale])\\n\\t\"\n \"vwredsum.vs v9, v22, v0\\n\\t\"\n \"lbu %[q41], 5(%[scale])\\n\\t\"\n \"vwredsum.vs v10, v26, v0\\n\\t\"\n \"lbu %[q42], 6(%[scale])\\n\\t\"\n \"vwredsum.vs v11, v30, v0\\n\\t\"\n \"lbu %[q43], 7(%[scale])\\n\\t\"\n \"vsetivli zero, 4, e32, m1, ta, ma\\n\\t\"\n \"vmul.vx v0, v4, %[s0]\\n\\t\"\n \"vmul.vx v1, v8, %[q40]\\n\\t\"\n \"vmacc.vx v0, %[s1], v5\\n\\t\"\n \"vmacc.vx v1, %[q41], v9\\n\\t\"\n \"vmacc.vx v0, %[s2], v6\\n\\t\"\n \"vmacc.vx v1, %[q42], v10\\n\\t\"\n \"vmacc.vx v0, %[s3], v7\\n\\t\"\n \"vmacc.vx v1, %[q43], v11\\n\\t\"\n \"vfcvt.f.x.v v0, v0\\n\\t\"\n \"vfcvt.f.x.v v1, v1\\n\\t\"\n \"vfmv.f.s %[ft2], v0\\n\\t\"\n \"vfmv.f.s %[ftmp], v1\\n\\t\"\n \"fadd.s %[ft2], %[ft2], %[ftmp]\\n\\t\"\n \"fmadd.s %[sumf], %[d], %[ft2], %[sumf]\"\n : [ftmp] \"=&f\" (ftmp), [sumf] \"+&f\" (sumf), [ft2] \"=&f\" (ft2)\n , [s0] \"=&r\" (s0), [s1] \"=&r\" (s1), [s2] \"=&r\" (s2), [s3] \"=&r\" (s3)\n , [q40] \"=&r\" (q40), [q41] \"=&r\" (q41), [q42] \"=&r\" (q42), [q43] \"=&r\" (q43)\n , [q80] \"=&r\" (q80), [q81] \"=&r\" (q81), [q82] \"=&r\" (q82), [q83] \"=&r\" (q83)\n : [d] \"f\" (d), [ys] \"r\" (y[i].qs), [xs] \"r\" (x[i].qs), [scale] \"r\" (scales)\n , [bsums] \"r\" (y[i].bsums), [mins] \"r\" (mins), [utmp] \"r\" (utmp)\n , [s6b] \"r\" (&x[i]), [kmask1] \"r\" (kmask1), [dmin] \"f\" (dmin)\n , [kmask2] \"r\" (kmask2), [kmask3] \"r\" (kmask3)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n }\n break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n\n *s = sumf;\n\n#else\n\n UNUSED(x);\n UNUSED(y);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(nb);\n UNUSED(utmp);\n\n ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined __riscv_v\n\n const uint8_t * scales = (const uint8_t*)&utmp[0];\n const uint8_t * mins = (const uint8_t*)&utmp[2];\n\n float sumf = 0;\n float sums = 0.0;\n\n size_t vl;\n\n for (int i = 0; i < nb; ++i) {\n\n vl = 8;\n\n const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n const uint8_t * GGML_RESTRICT hm = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;\n\n vint16m1_t q8sums_0 = __riscv_vlse16_v_i16m1(y[i].bsums, 4, vl);\n vint16m1_t q8sums_1 = __riscv_vlse16_v_i16m1(y[i].bsums+1, 4, vl);\n vint16m1_t q8sums = __riscv_vadd_vv_i16m1(q8sums_0, q8sums_1, vl);\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n vuint8mf2_t mins8 = __riscv_vle8_v_u8mf2(mins, vl);\n vint16m1_t v_mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl));\n vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, v_mins, vl);\n\n vint32m1_t sumi = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);\n sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);\n\n vl = 32;\n int32_t aux32 = 0;\n int is = 0;\n\n uint8_t m = 1;\n vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);\n vuint8m2_t vqh = __riscv_vle8_v_u8m2(hm, vl);\n\n for (int j = 0; j < QK_K/64; ++j) {\n // load Q5 and Q8\n vuint8m2_t q5_x = __riscv_vle8_v_u8m2(q5, vl);\n vint8m2_t q8_y1 = __riscv_vle8_v_i8m2(q8, vl);\n vint8m2_t q8_y2 = __riscv_vle8_v_i8m2(q8+32, vl);\n\n // compute mask for addition\n vint8m2_t q5_a = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vand_vx_u8m2(q5_x, 0x0F, vl));\n vuint8m2_t qh_m1 = __riscv_vand_vx_u8m2(vqh, m, vl);\n vbool4_t vmask_1 = __riscv_vmsne_vx_u8m2_b4(qh_m1, 0, vl);\n vint8m2_t q5_m1 = __riscv_vadd_vx_i8m2_mu(vmask_1, q5_a, q5_a, 16, vl);\n m <<= 1;\n\n vint8m2_t q5_l = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vsrl_vx_u8m2(q5_x, 0x04, vl));\n vuint8m2_t qh_m2 = __riscv_vand_vx_u8m2(vqh, m, vl);\n vbool4_t vmask_2 = __riscv_vmsne_vx_u8m2_b4(qh_m2, 0, vl);\n vint8m2_t q5_m2 = __riscv_vadd_vx_i8m2_mu(vmask_2, q5_l, q5_l, 16, vl);\n m <<= 1;\n\n vint16m4_t v0 = __riscv_vwmul_vv_i16m4(q5_m1, q8_y1, vl);\n vint16m4_t v1 = __riscv_vwmul_vv_i16m4(q5_m2, q8_y2, vl);\n\n vint32m8_t vs1 = __riscv_vwmul_vx_i32m8(v0, scales[is++], vl);\n vint32m8_t vs2 = __riscv_vwmul_vx_i32m8(v1, scales[is++], vl);\n\n vint32m1_t vacc1 = __riscv_vredsum_vs_i32m8_i32m1(vs1, vzero, vl);\n vint32m1_t vacc2 = __riscv_vredsum_vs_i32m8_i32m1(vs2, vacc1, vl);\n\n aux32 += __riscv_vmv_x_s_i32m1_i32(vacc2);\n q5 += 32; q8 += 64;\n\n }\n\n sums += aux32 * d;\n\n }\n\n *s = sumf+sums;\n\n#else\n\n UNUSED(x);\n UNUSED(y);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(nb);\n UNUSED(utmp);\n\n ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q6_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __riscv_xtheadvector\n\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n const uint8_t * restrict q6 = x[i].ql;\n const uint8_t * restrict qh = x[i].qh;\n const int8_t * restrict q8 = y[i].qs;\n\n const int8_t * restrict scale = x[i].scales;\n\n int sum_t = 0;\n int t0;\n\n for (int j = 0; j < QK_K/128; ++j) {\n __asm__ __volatile__(\n \"th.vsetvli zero, %[vl32], e8, m2\\n\\t\" // vl == 32\n \"th.vlb.v v4, (%[qh])\\n\\t\"\n \"th.vsll.vi v0, v4, 4\\n\\t\"\n \"th.vsll.vi v2, v4, 2\\n\\t\"\n \"th.vsrl.vi v6, v4, 2\\n\\t\"\n \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\" // vl == 64\n \"th.vlb.v v8, (%[q6])\\n\\t\"\n \"th.vsrl.vi v12, v8, 4\\n\\t\"\n \"th.vand.vi v8, v8, 0xF\\n\\t\"\n \"th.vsetvli zero, %[vl128], e8, m8\\n\\t\" // vl == 128\n \"th.vand.vx v0, v0, %[mask]\\n\\t\"\n \"th.vor.vv v8, v8, v0\\n\\t\"\n \"th.vlb.v v0, (%[q8])\\n\\t\"\n \"th.vsub.vx v8, v8, %[vl32]\\n\\t\"\n \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\" // vl == 64\n \"th.vwmul.vv v16, v0, v8\\n\\t\"\n \"th.vwmul.vv v24, v4, v12\\n\\t\"\n \"li %[t0], 16\\n\\t\"\n \"th.vsetvli zero, %[t0], e16, m2\\n\\t\" // vl == 16\n \"th.vmv.v.x v0, zero\\n\\t\"\n \"th.vwredsum.vs v10, v16, v0\\n\\t\"\n \"th.vwredsum.vs v9, v18, v0\\n\\t\"\n \"th.vwredsum.vs v8, v20, v0\\n\\t\"\n \"th.vwredsum.vs v7, v22, v0\\n\\t\"\n \"th.vwredsum.vs v11, v24, v0\\n\\t\"\n \"th.vwredsum.vs v12, v26, v0\\n\\t\"\n \"th.vwredsum.vs v13, v28, v0\\n\\t\"\n \"th.vwredsum.vs v14, v30, v0\\n\\t\"\n \"li %[t0], 4\\n\\t\"\n \"th.vsetvli zero, %[t0], e32, m1\\n\\t\" // vl == 4\n \"th.vslideup.vi v10, v9, 1\\n\\t\"\n \"th.vslideup.vi v8, v7, 1\\n\\t\"\n \"th.vslideup.vi v11, v12, 1\\n\\t\"\n \"th.vslideup.vi v13, v14, 1\\n\\t\"\n \"th.vslideup.vi v10, v8, 2\\n\\t\"\n \"th.vslideup.vi v11, v13, 2\\n\\t\"\n \"li %[t0], 8\\n\\t\"\n \"th.vsetvli zero, %[t0], e32, m2\\n\\t\" // vl == 8\n \"th.vlb.v v4, (%[scale])\\n\\t\"\n \"th.vmul.vv v2, v4, v10\\n\\t\"\n \"th.vredsum.vs v0, v2, v0\\n\\t\"\n \"th.vmv.x.s %[t0], v0\\n\\t\"\n \"add %[sumi], %[sumi], %[t0]\"\n : [sumi] \"+&r\" (sum_t), [t0] \"=&r\" (t0)\n : [qh] \"r\" (qh), [q6] \"r\" (q6), [q8] \"r\" (q8), [scale] \"r\" (scale)\n , [vl32] \"r\" (32), [vl64] \"r\" (64), [vl128] \"r\" (128)\n , [mask] \"r\" (0x30)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n );\n q6 += 64; qh += 32; q8 += 128; scale += 8;\n }\n\n sumf += d * sum_t;\n\n }\n\n *s = sumf;\n\n#elif defined __riscv_v\n\n float sumf = 0;\n const int vector_length = __riscv_vlenb() * 8;\n\n switch (vector_length) {\n case 256:\n for (int i = 0; i < nb; ++i) {\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n const uint8_t * GGML_RESTRICT q6 = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const int8_t * GGML_RESTRICT scale = x[i].scales;\n\n size_t vl;\n\n vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);\n\n int sum_t = 0;\n int is = 0;\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n vl = 32;\n\n // load qh\n vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl);\n\n // load Q6\n vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl);\n vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl);\n\n vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl);\n vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl);\n vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl);\n vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl);\n\n vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl);\n vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl);\n vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl);\n vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl);\n\n vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl);\n vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl);\n vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl);\n vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl);\n\n vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl);\n vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl);\n vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl);\n vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl);\n\n // load Q8 and take product\n vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl);\n vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl);\n vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl);\n vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl);\n\n vl = 16;\n\n vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl);\n vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl);\n vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl);\n vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl);\n vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl);\n vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl);\n vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl);\n vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl);\n\n vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl);\n vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl);\n vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl);\n vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl);\n\n sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);\n\n q6 += 64; qh += 32; q8 += 128; is=8;\n\n }\n\n sumf += d * sum_t;\n\n }\n break;\n case 128:\n for (int i = 0; i < nb; ++i) {\n\n __builtin_prefetch(&x[i + 1].d, 0, 1);\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n const uint8_t * restrict q6 = x[i].ql;\n const uint8_t * restrict qh = x[i].qh;\n const int8_t * restrict q8 = y[i].qs;\n\n const int8_t * restrict scale = x[i].scales;\n\n int q6h;\n float ftmp;\n\n for (int j = 0; j < QK_K/128; ++j) {\n __asm__ __volatile__(\n \"addi %[q6h], %[q6], 32\\n\\t\"\n \"ld t0, 0(%[scale])\\n\\t\"\n \"addi %[scale], %[scale], 8\\n\\t\"\n \"slli t6, t0, 1 * 8\\n\\t\"\n \"lb zero, 0(%[q6])\\n\\t\"\n \"slli t5, t0, 2 * 8\\n\\t\"\n \"slli t4, t0, 3 * 8\\n\\t\"\n \"lb zero, 0(%[q6h])\\n\\t\"\n \"slli t3, t0, 4 * 8\\n\\t\"\n \"slli t2, t0, 5 * 8\\n\\t\"\n \"lb zero, 0(%[qh])\\n\\t\"\n \"lb zero, 31(%[q6h])\\n\\t\"\n \"slli t1, t0, 6 * 8\\n\\t\"\n \"srai a7, t0, 56\\n\\t\"\n \"vsetvli zero, %[vl32], e8, m2\\n\\t\"\n \"vle8.v v8, (%[q6])\\n\\t\"\n \"srai t6, t6, 56\\n\\t\"\n \"srai t5, t5, 56\\n\\t\"\n \"srai t4, t4, 56\\n\\t\"\n \"srai t3, t3, 56\\n\\t\"\n \"vle8.v v10, (%[q6h])\\n\\t\"\n \"addi %[q6], %[q6], 64\\n\\t\"\n \"slli t0, t0, 7 * 8\\n\\t\"\n \"srai t2, t2, 56\\n\\t\"\n \"srai t1, t1, 56\\n\\t\"\n \"srai t0, t0, 56\\n\\t\"\n \"vle8.v v4, (%[qh])\\n\\t\"\n \"vsrl.vi v12, v8, 4\\n\\t\"\n \"vsrl.vi v14, v10, 4\\n\\t\"\n \"lb zero, 0(%[q8])\\n\\t\"\n \"vand.vi v8, v8, 0xF\\n\\t\"\n \"vand.vi v10, v10, 0xF\\n\\t\"\n \"lb zero, 32(%[q8])\\n\\t\"\n \"vsll.vi v0, v4, 4\\n\\t\"\n \"vsll.vi v2, v4, 2\\n\\t\"\n \"lb zero, 64(%[q8])\\n\\t\"\n \"vsrl.vi v6, v4, 2\\n\\t\"\n \"vand.vx v0, v0, %[mask]\\n\\t\"\n \"lb zero, 96(%[q8])\\n\\t\"\n \"vand.vx v2, v2, %[mask]\\n\\t\"\n \"vand.vx v4, v4, %[mask]\\n\\t\"\n \"vand.vx v6, v6, %[mask]\\n\\t\"\n \"vor.vv v8, v8, v0\\n\\t\"\n \"lb zero, 127(%[q8])\\n\\t\"\n \"vor.vv v10, v10, v2\\n\\t\"\n \"vor.vv v12, v12, v4\\n\\t\"\n \"vor.vv v14, v14, v6\\n\\t\"\n \"vsetvli zero, %[vl128], e8, m8\\n\\t\"\n \"vle8.v v0, (%[q8])\\n\\t\"\n \"vsub.vx v8, v8, %[vl32]\\n\\t\"\n \"vsetvli zero, %[vl64], e8, m4\\n\\t\"\n \"vwmul.vv v16, v0, v8\\n\\t\"\n \"vwmul.vv v24, v4, v12\\n\\t\"\n \"vsetivli zero, 16, e16, m2\\n\\t\"\n \"vmv.v.x v0, zero\\n\\t\"\n \"vwredsum.vs v10, v16, v0\\n\\t\"\n \"vwredsum.vs v9, v18, v0\\n\\t\"\n \"vwredsum.vs v8, v20, v0\\n\\t\"\n \"vwredsum.vs v7, v22, v0\\n\\t\"\n \"vwredsum.vs v11, v24, v0\\n\\t\"\n \"vwredsum.vs v12, v26, v0\\n\\t\"\n \"vwredsum.vs v13, v28, v0\\n\\t\"\n \"vwredsum.vs v14, v30, v0\\n\\t\"\n \"vsetivli zero, 4, e32, m1\\n\\t\"\n \"vmul.vx v0, v10, t0\\n\\t\"\n \"vmul.vx v1, v9, t1\\n\\t\"\n \"vmacc.vx v0, t2, v8\\n\\t\"\n \"vmacc.vx v1, t3, v7\\n\\t\"\n \"vmacc.vx v0, t4, v11\\n\\t\"\n \"vmacc.vx v1, t5, v12\\n\\t\"\n \"vmacc.vx v0, t6, v13\\n\\t\"\n \"vmacc.vx v1, a7, v14\\n\\t\"\n \"vadd.vv v0, v0, v1\\n\\t\"\n \"vfcvt.f.x.v v0, v0\\n\\t\"\n \"vfmv.f.s %[ftmp], v0\\n\\t\"\n \"fmadd.s %[sumf], %[d], %[ftmp], %[sumf]\"\n : [q6] \"+&r\" (q6), [q6h] \"=&r\" (q6h)\n , [scale] \"+&r\" (scale)\n , [sumf] \"+&f\" (sumf), [ftmp] \"=&f\" (ftmp)\n : [qh] \"r\" (qh), [q8] \"r\" (q8)\n , [vl32] \"r\" (32), [vl64] \"r\" (64), [vl128] \"r\" (128)\n , [mask] \"r\" (0x30), [d] \"f\" (d)\n : \"memory\"\n , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n , \"t0\", \"t1\", \"t2\", \"t3\", \"t4\", \"t5\", \"t6\", \"a7\"\n , \"a6\", \"a5\", \"a4\", \"a3\"\n );\n qh += 32; q8 += 128;\n }\n }\n break;\n default:\n assert(false && \"Unsupported vector length\");\n break;\n }\n\n *s = sumf;\n\n#else\n\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n\n ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq1_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n // Load qh once for the entire superblock.\n vuint16mf2_t qh = __riscv_vle16_v_u16mf2(x[i].qh, 8);\n\n // Calculate ls.\n vuint16mf2_t temp = __riscv_vsrl_vx_u16mf2(qh, 12, 8);\n temp = __riscv_vand_vx_u16mf2(temp, 7, 8);\n vint32m1_t ls = __riscv_vreinterpret_v_u32m1_i32m1(__riscv_vwmulu_vx_u32m1(temp, 2, 8));\n ls = __riscv_vadd_vx_i32m1(ls, 1, 8);\n\n // Calculate delta.\n vbool32_t mask = __riscv_vmseq_vx_u16mf2_b32(__riscv_vand_vx_u16mf2(qh, 0x8000, 8), 0, 8);\n vint32m1_t delta_neg = __riscv_vmv_v_x_i32m1(-1, 8);\n vint32m1_t delta_pos = __riscv_vmv_v_x_i32m1(1, 8);\n vint32m1_t delta = __riscv_vmerge_vvm_i32m1(delta_neg, delta_pos, mask, 8);\n\n // Load qs.\n vuint8m1_t qs = __riscv_vle8_v_u8m1(x[i].qs, 32);\n\n // Prepare the indices.\n const uint64_t shift = 0x0009000600030000;\n vuint16m2_t qh_shift = __riscv_vreinterpret_v_u64m2_u16m2(__riscv_vmv_v_x_u64m2(shift, 8));\n vuint16m2_t qh_gather_index = __riscv_vreinterpret_v_i16m2_u16m2(\n __riscv_vdiv_vx_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vid_v_u16m2(32)), 4, 32));\n vuint16m2_t qh_ext = __riscv_vlmul_ext_v_u16m1_u16m2(__riscv_vlmul_ext_v_u16mf2_u16m1(qh));\n vuint16m2_t qh_index = __riscv_vrgather_vv_u16m2(qh_ext, qh_gather_index, 32);\n qh_index = __riscv_vsrl_vv_u16m2(qh_index, qh_shift, 32);\n qh_index = __riscv_vand_vx_u16m2(qh_index, 7, 32);\n qh_index = __riscv_vsll_vx_u16m2(qh_index, 8, 32);\n qh_index = __riscv_vor_vv_u16m2(qh_index, __riscv_vzext_vf2_u16m2(qs, 32), 32);\n vuint16m2_t index = __riscv_vsll_vx_u16m2(qh_index, 3, 32);\n\n // Final lsums.\n int32_t lsums_s[8];\n vint32m1_t one_scalar = __riscv_vmv_v_x_i32m1(0, 1);\n\n // Sub-blocks 1-4\n {\n vuint16m1_t grid_index0 = __riscv_vget_v_u16m2_u16m1(index, 0);\n vint8m4_t grid0 = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vluxei16_v_i64m4((const int64_t*)iq1s_grid, grid_index0, 16));\n vint8m4_t q80 = __riscv_vle8_v_i8m4(y[i].qs, 128);\n vint16m8_t lsum0 = __riscv_vwmul_vv_i16m8(grid0, q80, 128);\n lsums_s[0] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 0), one_scalar, 32));\n lsums_s[1] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 1), one_scalar, 32));\n lsums_s[2] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 2), one_scalar, 32));\n lsums_s[3] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 3), one_scalar, 32));\n }\n __asm__ __volatile__(\"\" ::: \"memory\");\n // Sub-blocks 5-8\n {\n vuint16m1_t grid_index1 = __riscv_vget_v_u16m2_u16m1(index, 1);\n vint8m4_t grid1 = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vluxei16_v_i64m4((const int64_t*)iq1s_grid, grid_index1, 16));\n vint8m4_t q81 = __riscv_vle8_v_i8m4(&y[i].qs[128], 128);\n vint16m8_t lsum1 = __riscv_vwmul_vv_i16m8(grid1, q81, 128);\n lsums_s[4] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 0), one_scalar, 32));\n lsums_s[5] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 1), one_scalar, 32));\n lsums_s[6] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 2), one_scalar, 32));\n lsums_s[7] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 3), one_scalar, 32));\n }\n __asm__ __volatile__(\"\" ::: \"memory\");\n vint32m1_t lsums = __riscv_vle32_v_i32m1(&lsums_s[0], 8);\n\n // Calculate the bsums.\n vint16m1_t bsums_0 = __riscv_vle16_v_i16m1(y[i].bsums, 16);\n const vuint32m1_t bsums_i32 = __riscv_vreinterpret_v_u16m1_u32m1(__riscv_vreinterpret_v_i16m1_u16m1(bsums_0));\n const vint16mf2_t bsums_i32_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(bsums_i32, 0, 8));\n const vint16mf2_t bsums_i32_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(bsums_i32, 16, 8));\n const vint32m1_t bsums = __riscv_vwadd_vv_i32m1(bsums_i32_0, bsums_i32_1, 8);\n\n // Accumulation.\n vint32m1_t sumi_v = __riscv_vmul_vv_i32m1(ls, lsums, 8);\n vint32m1_t sumi1_v = __riscv_vmul_vv_i32m1(__riscv_vmul_vv_i32m1(ls, delta, 8), bsums, 8);\n\n // Update sumf.\n int sumi = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m1_i32m1(sumi_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8));\n int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m1_i32m1(sumi1_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8));\n sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 256:\n ggml_vec_dot_iq1_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq1_m_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_m * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n iq1m_scale_t scale;\n float sumf = 0.0f;\n for (int i = 0; i < nb; ++i) {\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint8_t * qh = x[i].qh;\n const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n\n // Accumulators.\n vint32m2_t acc1 = __riscv_vmv_v_x_i32m2(0, 16);\n vint32m2_t acc2 = __riscv_vmv_v_x_i32m2(0, 16);\n\n // We process 4 sub-blocks together.\n for (int ib = 0; ib < QK_K/128; ib++) {\n // Load qh for 4 sub-blocks.\n const vuint8mf4_t qh_8 = __riscv_vle8_v_u8mf4(qh, 8);\n const vuint16mf2_t qh_16_lo = __riscv_vzext_vf2_u16mf2(qh_8, 8);\n const vuint16mf2_t qh_16_hi = __riscv_vsll_vx_u16mf2(qh_16_lo, 8, 8);\n const vuint16m1_t qhb = __riscv_vzext_vf2_u16m1(\n __riscv_vreinterpret_v_u16mf2_u8mf2(__riscv_vor_vv_u16mf2(qh_16_lo, qh_16_hi, 8)), 16);\n qh += 8;\n\n // Prepare grid indices.\n const vuint16m1_t qsb = __riscv_vzext_vf2_u16m1(__riscv_vle8_v_u8mf2(&qs[0], 16), 16);\n const vuint16m1_t shift = __riscv_vreinterpret_v_u32m1_u16m1(__riscv_vmv_v_x_u32m1(0x00040008, 8));\n vuint16m1_t index = __riscv_vor_vv_u16m1(qsb, __riscv_vand_vx_u16m1(__riscv_vsll_vv_u16m1(qhb, shift, 16), 0x700, 16), 16);\n index = __riscv_vsll_vx_u16m1(index, 3, 16);\n qs += 16;\n\n // Load the grid.\n const vint8m4_t iq1b = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vreinterpret_v_u64m4_i64m4(\n __riscv_vluxei16_v_u64m4(iq1s_grid, index, 16)));\n\n // Prepare the deltas.\n const vbool16_t mask = __riscv_vmsgtu_vx_u16m1_b16(\n __riscv_vand_vv_u16m1(qhb, __riscv_vreinterpret_v_u32m1_u16m1(__riscv_vmv_v_x_u32m1(0x00800008, 8)), 16), 0, 16);\n const vint64m4_t delta_pos = __riscv_vmv_v_x_i64m4(0x0101010101010101, 16);\n const vint64m4_t delta_neg = __riscv_vmv_v_x_i64m4(0xffffffffffffffff, 16);\n const vint8m4_t delta = __riscv_vreinterpret_v_i64m4_i8m4(\n __riscv_vmerge_vvm_i64m4(delta_pos, delta_neg, mask, 16));\n\n // Load q8 for sub-blocks.\n const vint8m4_t q8b = __riscv_vle8_v_i8m4(q8, 128);\n q8 += 128;\n\n // Calculate the lsums.\n const vint16m8_t lsum1 = __riscv_vwmul_vv_i16m8(iq1b, q8b, 128);\n const vint16m8_t lsum2 = __riscv_vwmul_vv_i16m8(delta, q8b, 128);\n\n // Prepare the scales.\n const int16_t ls_0_0 = 2*((sc[0] >> 0) & 0x7) + 1;\n const int16_t ls_0_1 = 2*((sc[0] >> 3) & 0x7) + 1;\n const int16_t ls_1_0 = 2*((sc[0] >> 6) & 0x7) + 1;\n const int16_t ls_1_1 = 2*((sc[0] >> 9) & 0x7) + 1;\n const int16_t ls_2_0 = 2*((sc[1] >> 0) & 0x7) + 1;\n const int16_t ls_2_1 = 2*((sc[1] >> 3) & 0x7) + 1;\n const int16_t ls_3_0 = 2*((sc[1] >> 6) & 0x7) + 1;\n const int16_t ls_3_1 = 2*((sc[1] >> 9) & 0x7) + 1;\n sc += 2;\n\n // Accumulate in acc0 and acc1 for each sub-block.\n acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_0_0, __riscv_vget_v_i16m8_i16m1(lsum1, 0), 16);\n acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_0_1, __riscv_vget_v_i16m8_i16m1(lsum1, 1), 16);\n acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_0_0, __riscv_vget_v_i16m8_i16m1(lsum2, 0), 16);\n acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_0_1, __riscv_vget_v_i16m8_i16m1(lsum2, 1), 16);\n //\n acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_1_0, __riscv_vget_v_i16m8_i16m1(lsum1, 2), 16);\n acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_1_1, __riscv_vget_v_i16m8_i16m1(lsum1, 3), 16);\n acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_1_0, __riscv_vget_v_i16m8_i16m1(lsum2, 2), 16);\n acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_1_1, __riscv_vget_v_i16m8_i16m1(lsum2, 3), 16);\n //\n acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_2_0, __riscv_vget_v_i16m8_i16m1(lsum1, 4), 16);\n acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_2_1, __riscv_vget_v_i16m8_i16m1(lsum1, 5), 16);\n acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_2_0, __riscv_vget_v_i16m8_i16m1(lsum2, 4), 16);\n acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_2_1, __riscv_vget_v_i16m8_i16m1(lsum2, 5), 16);\n //\n acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_3_0, __riscv_vget_v_i16m8_i16m1(lsum1, 6), 16);\n acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_3_1, __riscv_vget_v_i16m8_i16m1(lsum1, 7), 16);\n acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_3_0, __riscv_vget_v_i16m8_i16m1(lsum2, 6), 16);\n acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_3_1, __riscv_vget_v_i16m8_i16m1(lsum2, 7), 16);\n }\n\n // Reduce and accumulate in `sumf`.\n vint32m1_t one = __riscv_vmv_v_x_i32m1(0, 1);\n int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc1, one, 16));\n int sumi2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc2, one, 16));\n sumf += y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16) * (sumi1 + IQ1M_DELTA * sumi2);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 256:\n ggml_vec_dot_iq1_m_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic const uint8_t sign_gather_indices_arr[64] = {\n 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,\n 4,4,4,4,4,4,4,4, 5,5,5,5,5,5,5,5, 6,6,6,6,6,6,6,6, 7,7,7,7,7,7,7,7\n};\n\nstatic const uint8_t sign_bit_masks_arr[64] = {\n 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128,\n 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128\n};\n\n\nstatic void ggml_vec_dot_iq2_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs);\n\n const block_iq2_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n const uint64_t * grid64 = (const uint64_t *)iq2s_grid;\n\n // Pre-load Constants\n vuint8m2_t v_ids = __riscv_vid_v_u8m2(32);\n vuint8m2_t v_sign_gather_indices = __riscv_vsrl_vx_u8m2(v_ids, 3, 32);\n vuint8m2_t v_ones = __riscv_vmv_v_x_u8m2(1, 32);\n vuint8m2_t v_shift_amts = __riscv_vand_vx_u8m2(v_ids, 7, 32);\n vuint8m2_t v_sign_masks = __riscv_vsll_vv_u8m2(v_ones, v_shift_amts, 32);\n uint16_t shift_qh_arr[4] = {11, 9, 7, 5};\n vuint16mf2_t v_shift_qh = __riscv_vle16_v_u16mf2(shift_qh_arr, 4);\n\n float sumf = 0.0f;\n\n for (int i = 0; i < nb; ++i) {\n const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint8_t * GGML_RESTRICT scales = x[i].scales;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const uint8_t * signs_ptr = qs + 32;\n float sum_block = 0.0f;\n\n for (int ib = 0; ib < 8; ++ib) {\n\n // Load Low Bits [4 bytes]\n vuint8mf4_t v_qs_u8 = __riscv_vle8_v_u8mf4(qs, 4);\n qs += 4;\n\n // Load 1 byte. It contains bits for 4 mini-blocks.\n uint8_t qh_val = *qh++;\n\n // Combine Low + High bits of 10bit indices\n vuint8mf4_t v_qh_raw = __riscv_vmv_v_x_u8mf4(qh_val, 4);\n vuint16mf2_t v_qh_u16 = __riscv_vwcvtu_x_x_v_u16mf2(v_qh_raw, 4);\n vuint16mf2_t v_qh_mf2 = __riscv_vsll_vv_u16mf2(v_qh_u16, v_shift_qh, 4);\n v_qh_mf2 = __riscv_vand_vx_u16mf2(v_qh_mf2, 0x1800, 4);\n vuint16mf2_t v_qs_u16_mf2 = __riscv_vwcvtu_x_x_v_u16mf2(v_qs_u8, 4);\n vuint16mf2_t v_qs_u16 = __riscv_vsll_vx_u16mf2(v_qs_u16_mf2, 3, 4);\n vuint16mf2_t v_grid_offsets = __riscv_vor_vv_u16mf2(v_qs_u16, v_qh_mf2, 4);\n\n // Lookup Grid\n vint8m2_t v_grid_i8 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(__riscv_vluxei16_v_u64m2(grid64, v_grid_offsets, 4)));\n\n vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs_ptr, 4);\n signs_ptr += 4;\n vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw);\n vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 32);\n\n // generating sign mask\n vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 32);\n vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 32);\n\n vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 32);\n q8 += 32;\n\n // apply signs\n vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative,v_q8, v_q8, 0, 32);\n vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_grid_i8, v_q8_signed, 32);\n\n // Reduction\n vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);\n\n // Reduce 0-15 (First Half)\n int32_t s0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(\n __riscv_vget_v_i16m4_i16m2(v_dot, 0), v_zero, 16));\n\n // Reduce 16-31 (Second Half)\n int32_t s1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(\n __riscv_vget_v_i16m4_i16m2(v_dot, 1), v_zero, 16));\n\n // Apply sub Scales\n uint8_t sc = *scales++;\n\n sum_block += s0 * (2 * (sc & 0xF) + 1);\n sum_block += s1 * (2 * (sc >> 4) + 1);\n }\n sumf += sum_block * combined_scale;\n }\n *s = 0.125f * sumf;\n}\n\nstatic void ggml_vec_dot_iq2_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs);\n\n const block_iq2_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n const uint64_t * grid64 = (const uint64_t *)iq2s_grid;\n\n // --- Pre-load Constants ---\n uint16_t gather_qh_arr[8] = {0, 0, 0, 0, 1, 1, 1, 1};\n vuint16mf2_t v_gather_qh = __riscv_vle16_v_u16mf2(gather_qh_arr, 8);\n uint16_t shift_qh_arr[8] = {11, 9, 7, 5, 11, 9, 7, 5};\n vuint16mf2_t v_shift_qh = __riscv_vle16_v_u16mf2(shift_qh_arr, 8);\n\n // Constants for sign extraction\n vuint8m2_t v_sign_gather_indices = __riscv_vle8_v_u8m2(sign_gather_indices_arr, 64);\n vuint8m2_t v_sign_masks = __riscv_vle8_v_u8m2(sign_bit_masks_arr, 64);\n\n float sumf = 0.0f;\n\n for (int i = 0; i < nb; ++i) {\n const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint8_t * GGML_RESTRICT scales = x[i].scales;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const uint8_t * signs_ptr = qs + 32;\n\n float sum_block = 0.0f;\n\n for (int ib = 0; ib < 4; ++ib) {\n // Combine low + high bits\n vuint8mf4_t v_qs_u8 = __riscv_vle8_v_u8mf4(qs, 8);\n qs += 8;\n uint16_t qh_val;\n memcpy(&qh_val, qh, 2);\n qh += 2;\n vuint8mf8_t v_qh_raw = __riscv_vle8_v_u8mf8((const uint8_t*)&qh_val, 2);\n vuint16mf4_t v_qh_u16 = __riscv_vwcvtu_x_x_v_u16mf4(v_qh_raw, 2);\n vuint16mf2_t v_qh_u16_ext = __riscv_vlmul_ext_v_u16mf4_u16mf2(v_qh_u16);\n vuint16mf2_t v_qh_expanded = __riscv_vrgather_vv_u16mf2(v_qh_u16_ext, v_gather_qh, 8);\n v_qh_expanded = __riscv_vsll_vv_u16mf2(v_qh_expanded, v_shift_qh, 8);\n\n // Mask: We want bits 11-12. 0x1800 = 0001 1000 0000 0000\n v_qh_expanded = __riscv_vand_vx_u16mf2(v_qh_expanded, 0x1800, 8);\n vuint16mf2_t v_qs_u16 = __riscv_vwcvtu_x_x_v_u16mf2(v_qs_u8, 8);\n\n // Multiply by 8 to get byte offset, instead of element offset\n v_qs_u16 = __riscv_vsll_vx_u16mf2(v_qs_u16, 3, 8);\n vuint16mf2_t v_grid_offsets = __riscv_vor_vv_u16mf2(v_qs_u16, v_qh_expanded, 8);\n\n // Lookup Grid using Byte Offsets\n vuint64m2_t v_grid_vals = __riscv_vluxei16_v_u64m2(grid64, v_grid_offsets, 8);\n\n vuint8m2_t v_grid_u8 = __riscv_vreinterpret_v_u64m2_u8m2(v_grid_vals);\n vint8m2_t v_grid_i8 = __riscv_vreinterpret_v_u8m2_i8m2(v_grid_u8);\n\n // Load signs and generate sign mask\n vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs_ptr, 8);\n signs_ptr += 8;\n\n vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw);\n vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 64);\n\n vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 64);\n vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 64);\n\n vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 64);\n q8 += 64;\n\n vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative, v_q8, v_q8, 0, 64);\n vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_grid_i8, v_q8_signed, 64);\n\n vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);\n\n int32_t s0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n __riscv_vget_v_i16m4_i16m1(v_dot, 0), v_zero, 16));\n int32_t s1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n __riscv_vget_v_i16m4_i16m1(v_dot, 1), v_zero, 16));\n int32_t s2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n __riscv_vget_v_i16m4_i16m1(v_dot, 2), v_zero, 16));\n int32_t s3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n __riscv_vget_v_i16m4_i16m1(v_dot, 3), v_zero, 16));\n\n uint8_t sc0 = scales[0];\n uint8_t sc1 = scales[1];\n scales += 2;\n\n sum_block += s0 * (2 * (sc0 & 0xF) + 1);\n sum_block += s1 * (2 * (sc0 >> 4) + 1);\n sum_block += s2 * (2 * (sc1 & 0xF) + 1);\n sum_block += s3 * (2 * (sc1 >> 4) + 1);\n }\n sumf += sum_block * combined_scale;\n }\n *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 128:\n ggml_vec_dot_iq2_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc);\n break;\n case 256:\n ggml_vec_dot_iq2_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined(__riscv_v)\nstatic const int8_t keven_signs_q2xs[1024] = {\n 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,\n 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,\n 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,\n 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,\n 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,\n 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,\n 1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, 1,\n 1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, -1,\n 1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, -1,\n 1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, 1,\n 1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, 1,\n 1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1,\n 1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, 1,\n 1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, -1,\n 1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, -1,\n 1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1,\n 1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, -1,\n 1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1,\n 1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, 1,\n 1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, -1,\n 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1,\n 1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, -1,\n 1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, -1,\n 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1,\n 1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, 1,\n 1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, -1,\n 1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1,\n 1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, 1,\n 1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1,\n 1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, 1,\n 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, 1,\n 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nstatic void ggml_vec_dot_iq2_xs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n const uint64_t * grid64 = (const uint64_t *)iq2xs_grid;\n\n float sumf = 0.0f;\n\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT qs = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n const uint8_t * GGML_RESTRICT scales = x[i].scales;\n\n int32_t sum_int = 0;\n\n // Loop over 4 subblocks of 64 elements (QK_K = 256)\n for (int ib64 = 0; ib64 < QK_K / 64; ++ib64) {\n // Load 8 uint16 indices (controls 64 values)\n vuint16mf2_t v_qs = __riscv_vle16_v_u16mf2(qs, 8);\n qs += 8;\n\n // Extract indices for grid (low 9 bits) and signs (high 7 bits)\n // Multiply by 8 (<< 3) for byte offsets into the uint64 tables\n vuint16mf2_t vidx_grid = __riscv_vsll_vx_u16mf2(__riscv_vand_vx_u16mf2(v_qs, 511, 8), 3, 8);\n vuint16mf2_t vidx_sign = __riscv_vsll_vx_u16mf2(__riscv_vsrl_vx_u16mf2(v_qs, 9, 8), 3, 8);\n\n vuint64m2_t vq2_64 = __riscv_vluxei16_v_u64m2(grid64, vidx_grid, 8);\n vuint64m2_t vs2_64 = __riscv_vluxei16_v_u64m2(signs64, vidx_sign, 8);\n\n vint8m2_t q2u = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vq2_64));\n vint8m2_t q2s = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vs2_64));\n\n vint8m2_t q2_final = __riscv_vmul_vv_i8m2(q2u, q2s, 64);\n\n vint8m2_t q8v = __riscv_vle8_v_i8m2(q8, 64);\n q8 += 64;\n\n vint16m4_t prod = __riscv_vwmul_vv_i16m4(q2_final, q8v, 64);\n\n vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1);\n\n int32_t sum0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n __riscv_vget_v_i16m4_i16m1(prod, 0), zero_vec, 16));\n int32_t sum1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n __riscv_vget_v_i16m4_i16m1(prod, 1), zero_vec, 16));\n int32_t sum2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n __riscv_vget_v_i16m4_i16m1(prod, 2), zero_vec, 16));\n int32_t sum3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n __riscv_vget_v_i16m4_i16m1(prod, 3), zero_vec, 16));\n\n const uint8_t scale_byte_1 = scales[0];\n const uint8_t scale_byte_2 = scales[1];\n scales += 2;\n\n sum_int += sum0 * ((scale_byte_1 & 0x0F) * 2 + 1);\n sum_int += sum1 * ((scale_byte_1 >> 4) * 2 + 1);\n sum_int += sum2 * ((scale_byte_2 & 0x0F) * 2 + 1);\n sum_int += sum3 * ((scale_byte_2 >> 4) * 2 + 1);\n }\n\n sumf += d * sum_int;\n }\n *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 256:\n ggml_vec_dot_iq2_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq2_xxs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n const uint64_t * grid64 = (const uint64_t *)iq2xxs_grid;\n\n uint32_t shift_constants[4] = {0, 7, 14, 21};\n vuint32m1_t v_shifts = __riscv_vle32_v_u32m1(shift_constants, 4);\n\n float sumf = 0.0f;\n for (int i = 0; i < nb; ++i) {\n const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n const uint8_t * GGML_RESTRICT q2_ptr = (const uint8_t *) x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n float sum = 0.0f;\n\n #pragma GCC unroll 1\n for (int ib32 = 0; ib32 < QK_K / 32; ib32 += 2) {\n vint8m2_t q8_1 = __riscv_vle8_v_i8m2(q8, 32); q8 += 32;\n vint8m2_t q8_2 = __riscv_vle8_v_i8m2(q8, 32); q8 += 32;\n\n vuint8mf4_t v_raw_q2_1 = __riscv_vle8_v_u8mf4(q2_ptr, 4);\n vuint8mf4_t v_raw_q2_2 = __riscv_vle8_v_u8mf4(q2_ptr + 8, 4);\n\n vuint16mf2_t vidx_q2_1 = __riscv_vwcvtu_x_x_v_u16mf2(v_raw_q2_1, 4);\n vuint16mf2_t vidx_q2_2 = __riscv_vwcvtu_x_x_v_u16mf2(v_raw_q2_2, 4);\n\n vidx_q2_1 = __riscv_vsll_vx_u16mf2(vidx_q2_1, 3, 4);\n vidx_q2_2 = __riscv_vsll_vx_u16mf2(vidx_q2_2, 3, 4);\n\n uint32_t s_packed_1, s_packed_2;\n memcpy(&s_packed_1, q2_ptr + 4, 4);\n memcpy(&s_packed_2, q2_ptr + 12, 4);\n\n vuint32m1_t v_s_1 = __riscv_vmv_v_x_u32m1(s_packed_1, 4);\n vuint32m1_t v_s_2 = __riscv_vmv_v_x_u32m1(s_packed_2, 4);\n v_s_1 = __riscv_vsrl_vv_u32m1(v_s_1, v_shifts, 4);\n v_s_2 = __riscv_vsrl_vv_u32m1(v_s_2, v_shifts, 4);\n\n v_s_1 = __riscv_vand_vx_u32m1(v_s_1, 127, 4);\n v_s_2 = __riscv_vand_vx_u32m1(v_s_2, 127, 4);\n\n vuint16mf2_t vidx_s2_1 = __riscv_vsll_vx_u16mf2(__riscv_vncvt_x_x_w_u16mf2(v_s_1, 4), 3, 4);\n vuint16mf2_t vidx_s2_2 = __riscv_vsll_vx_u16mf2(__riscv_vncvt_x_x_w_u16mf2(v_s_2, 4), 3, 4);\n\n vuint64m2_t vq2_64_1 = __riscv_vluxei16_v_u64m2(grid64, vidx_q2_1, 4);\n vuint64m2_t vq2_64_2 = __riscv_vluxei16_v_u64m2(grid64, vidx_q2_2, 4);\n\n vint8m2_t q2_1 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vq2_64_1));\n vint8m2_t q2_2 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vq2_64_2));\n\n vuint64m2_t vs2_64_1 = __riscv_vluxei16_v_u64m2(signs64, vidx_s2_1, 4);\n vuint64m2_t vs2_64_2 = __riscv_vluxei16_v_u64m2(signs64, vidx_s2_2, 4);\n vint8m2_t s2_1 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vs2_64_1));\n vint8m2_t s2_2 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vs2_64_2));\n\n vint8m2_t q8s_1 = __riscv_vmul_vv_i8m2(q8_1, s2_1, 32);\n vint8m2_t q8s_2 = __riscv_vmul_vv_i8m2(q8_2, s2_2, 32);\n\n vint16m4_t dot1 = __riscv_vwmul_vv_i16m4(q8s_1, q2_1, 32);\n vint16m4_t dot2 = __riscv_vwmul_vv_i16m4(q8s_2, q2_2, 32);\n\n vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1);\n vint32m1_t sumv1 = __riscv_vwredsum_vs_i16m4_i32m1(dot1, zero_vec, 32);\n vint32m1_t sumv2 = __riscv_vwredsum_vs_i16m4_i32m1(dot2, zero_vec, 32);\n\n int32_t scalar_sum1 = __riscv_vmv_x_s_i32m1_i32(sumv1);\n int32_t scalar_sum2 = __riscv_vmv_x_s_i32m1_i32(sumv2);\n\n int16_t scale1 = 2 * ((s_packed_1 >> 28) & 0xF) + 1;\n int16_t scale2 = 2 * ((s_packed_2 >> 28) & 0xF) + 1;\n\n sum += scalar_sum1 * scale1 + scalar_sum2 * scale2;\n q2_ptr += 16;\n }\n sumf += sum * combined_scale;\n }\n *s = 0.125f * sumf;\n}\n\nstatic void ggml_vec_dot_iq2_xxs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n const uint64_t * grid64 = (const uint64_t *)iq2xxs_grid;\n\n uint32_t shift_constants[4] = {0, 7, 14, 21};\n vuint32mf2_t v_shifts = __riscv_vle32_v_u32mf2(shift_constants, 4);\n\n float sumf = 0.0f;\n\n for (int i = 0; i < nb; ++i) {\n const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n const uint8_t * GGML_RESTRICT q2_ptr = (const uint8_t *) x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n float sum = 0.0f;\n\n for (int ib32 = 0; ib32 < QK_K / 32; ib32 += 2) {\n vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8, 32); q8 += 32;\n vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8, 32); q8 += 32;\n\n vuint8mf8_t v_raw_q2_1 = __riscv_vle8_v_u8mf8(q2_ptr, 4);\n vuint8mf8_t v_raw_q2_2 = __riscv_vle8_v_u8mf8(q2_ptr + 8, 4);\n\n vuint16mf4_t vidx_q2_1 = __riscv_vwcvtu_x_x_v_u16mf4(v_raw_q2_1, 4);\n vuint16mf4_t vidx_q2_2 = __riscv_vwcvtu_x_x_v_u16mf4(v_raw_q2_2, 4);\n\n vidx_q2_1 = __riscv_vsll_vx_u16mf4(vidx_q2_1, 3, 4);\n vidx_q2_2 = __riscv_vsll_vx_u16mf4(vidx_q2_2, 3, 4);\n\n uint32_t s_packed_1, s_packed_2;\n memcpy(&s_packed_1, q2_ptr + 4, 4);\n memcpy(&s_packed_2, q2_ptr + 12, 4);\n\n vuint32mf2_t v_s_1 = __riscv_vmv_v_x_u32mf2(s_packed_1, 4);\n vuint32mf2_t v_s_2 = __riscv_vmv_v_x_u32mf2(s_packed_2, 4);\n\n v_s_1 = __riscv_vsrl_vv_u32mf2(v_s_1, v_shifts, 4);\n v_s_2 = __riscv_vsrl_vv_u32mf2(v_s_2, v_shifts, 4);\n\n v_s_1 = __riscv_vand_vx_u32mf2(v_s_1, 127, 4);\n v_s_2 = __riscv_vand_vx_u32mf2(v_s_2, 127, 4);\n\n // Narrow u32 -> u16 (vncvt) and Scale by 8 to get byte offsets\n vuint16mf4_t vidx_s2_1 = __riscv_vsll_vx_u16mf4(__riscv_vncvt_x_x_w_u16mf4(v_s_1, 4), 3, 4);\n vuint16mf4_t vidx_s2_2 = __riscv_vsll_vx_u16mf4(__riscv_vncvt_x_x_w_u16mf4(v_s_2, 4), 3, 4);\n\n // Load q2 values from lookup grid\n vuint64m1_t vq2_64_1 = __riscv_vluxei16_v_u64m1(grid64, vidx_q2_1, 4);\n vuint64m1_t vq2_64_2 = __riscv_vluxei16_v_u64m1(grid64, vidx_q2_2, 4);\n vint8m1_t q2_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vreinterpret_v_u64m1_u8m1(vq2_64_1));\n vint8m1_t q2_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vreinterpret_v_u64m1_u8m1(vq2_64_2));\n\n // Load sign values\n vuint64m1_t vs2_64_1 = __riscv_vluxei16_v_u64m1(signs64, vidx_s2_1, 4);\n vuint64m1_t vs2_64_2 = __riscv_vluxei16_v_u64m1(signs64, vidx_s2_2, 4);\n vint8m1_t s2_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vreinterpret_v_u64m1_u8m1(vs2_64_1));\n vint8m1_t s2_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vreinterpret_v_u64m1_u8m1(vs2_64_2));\n\n // Apply signs to q8\n vint8m1_t q8s_1 = __riscv_vmul_vv_i8m1(q8_1, s2_1, 32);\n vint8m1_t q8s_2 = __riscv_vmul_vv_i8m1(q8_2, s2_2, 32);\n\n // multiplying q2 with q8\n vint16m2_t dot1 = __riscv_vwmul_vv_i16m2(q8s_1, q2_1, 32);\n vint16m2_t dot2 = __riscv_vwmul_vv_i16m2(q8s_2, q2_2, 32);\n\n vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1);\n vint32m1_t sumv1 = __riscv_vwredsum_vs_i16m2_i32m1(dot1, zero_vec, 32);\n vint32m1_t sumv2 = __riscv_vwredsum_vs_i16m2_i32m1(dot2, zero_vec, 32);\n int32_t scalar_sum1 = __riscv_vmv_x_s_i32m1_i32(sumv1);\n int32_t scalar_sum2 = __riscv_vmv_x_s_i32m1_i32(sumv2);\n int16_t scale1 = 2 * ((s_packed_1 >> 28) & 0xF) + 1;\n int16_t scale2 = 2 * ((s_packed_2 >> 28) & 0xF) + 1;\n\n sum += scalar_sum1 * scale1 + scalar_sum2 * scale2;\n q2_ptr += 16;\n }\n sumf += sum * combined_scale;\n }\n *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 128:\n ggml_vec_dot_iq2_xxs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_iq2_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_iq2_xxs_q8_K(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq3_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n const uint64_t * grid64 = (const uint64_t *)iq3s_grid;\n\n // --- Pre-load Constants ---\n const uint16_t qh_bit_shifts_arr[16] = {\n 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15\n };\n vuint8m2_t v_sign_gather_indices = __riscv_vle8_v_u8m2(sign_gather_indices_arr, 64);\n vuint8m2_t v_sign_masks = __riscv_vle8_v_u8m2(sign_bit_masks_arr, 64);\n vuint16m1_t v_qh_shifts = __riscv_vle16_v_u16m1(qh_bit_shifts_arr, 16);\n\n float sumf = 0.0f;\n\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d);\n const float combined_scale = d * y[i].d;\n\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint8_t * GGML_RESTRICT scales = x[i].scales;\n const uint8_t * GGML_RESTRICT signs = x[i].signs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n float sum_block = 0.0f;\n\n // Loop: Process 64 weights (16 mini-blocks of 4) per iteration\n for (int ib = 0; ib < 4; ++ib) {\n\n vuint8mf2_t v_qs_u8 = __riscv_vle8_v_u8mf2(qs, 16);\n qs += 16;\n\n uint16_t qh_val;\n memcpy(&qh_val, qh, 2);\n qh += 2;\n\n vuint16m1_t v_qh_val = __riscv_vmv_v_x_u16m1(qh_val, 16);\n // Extract bits: (qh >> i) & 1\n v_qh_val = __riscv_vsrl_vv_u16m1(v_qh_val, v_qh_shifts, 16);\n v_qh_val = __riscv_vand_vx_u16m1(v_qh_val, 1, 16);\n\n vuint16m1_t v_qs_u16 = __riscv_vwcvtu_x_x_v_u16m1(v_qs_u8, 16);\n v_qs_u16 = __riscv_vsll_vx_u16m1(v_qs_u16, 2, 16);\n v_qh_val = __riscv_vsll_vx_u16m1(v_qh_val, 10, 16);\n vuint16m1_t v_grid_offsets = __riscv_vor_vv_u16m1(v_qs_u16, v_qh_val, 16);\n\n // Grid value is 4xuint8\n vuint32m2_t v_grid_packed = __riscv_vluxei16_v_u32m2((const uint32_t *)grid64, v_grid_offsets, 16);\n vuint8m2_t v_grid_u8 = __riscv_vreinterpret_v_u32m2_u8m2(v_grid_packed);\n vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs, 8);\n signs += 8;\n\n // Generate sign mask\n vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw);\n vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 64);\n vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 64);\n vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 64);\n\n vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 64);\n q8 += 64;\n\n // Apply Signs\n vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative, v_q8, v_q8, 0, 64);\n vint16m4_t v_dot = __riscv_vwmulsu_vv_i16m4(v_q8_signed, v_grid_u8, 64);\n\n // Reduction\n vint16m2_t v_dot_lo = __riscv_vget_v_i16m4_i16m2(v_dot, 0);\n vint16m2_t v_dot_hi = __riscv_vget_v_i16m4_i16m2(v_dot, 1);\n vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);\n\n int32_t s_lo = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(v_dot_lo, v_zero, 32));\n int32_t s_hi = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(v_dot_hi, v_zero, 32));\n\n // Apply sub-scales\n uint8_t sc_byte = *scales++;\n int sc_lo = (sc_byte & 0xF) * 2 + 1;\n int sc_hi = (sc_byte >> 4) * 2 + 1;\n\n sum_block += s_lo * sc_lo + s_hi * sc_hi;\n }\n sumf += sum_block * combined_scale;\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 256:\n ggml_vec_dot_iq3_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq3_xxs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n const int nb = n / QK_K;\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n const uint32_t * grid32 = (const uint32_t *)iq3xxs_grid;\n\n // constants for unpacking logic\n const uint32_t shifts_val[8] = {0, 7, 14, 21, 0, 7, 14, 21};\n vuint32m1_t v_shifts = __riscv_vle32_v_u32m1(shifts_val, 8);\n\n const uint32_t gather_idx_val[8] = {0, 0, 0, 0, 1, 1, 1, 1};\n vuint32m1_t v_gather_idx = __riscv_vle32_v_u32m1(gather_idx_val, 8);\n\n uint32_t aux32[2];\n float sumf = 0.0f;\n\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n const uint8_t * GGML_RESTRICT q3_indices = x[i].qs;\n const uint8_t * GGML_RESTRICT metadata = x[i].qs + QK_K/4;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n float block_sum = 0.0f;\n\n for (int ib = 0; ib < QK_K / 64; ++ib) {\n // Load q8 (64 bytes)\n vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 64);\n q8 += 64;\n\n // load of metadata via memcpy\n memcpy(aux32, metadata, 2 * sizeof(uint32_t));\n metadata += 2 * sizeof(uint32_t);\n\n // Load q3 indices and gather magnitudes\n vuint8mf2_t v_q3_idx_u8 = __riscv_vle8_v_u8mf2(q3_indices, 16);\n q3_indices += 16;\n\n vuint16m1_t v_q3_idx_u16 = __riscv_vwmulu_vx_u16m1(v_q3_idx_u8, 4, 16);\n vuint32m2_t v_q3_magnitudes_u32 = __riscv_vluxei16_v_u32m2(grid32, v_q3_idx_u16, 16);\n vint8m2_t v_q3_magnitudes = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u32m2_u8m2(v_q3_magnitudes_u32));\n\n // --- Unpacking of Sign Indices ---\n\n // 1. Load the 2 auxiliary 32-bit integers into a vector\n vuint32m1_t v_aux = __riscv_vle32_v_u32m1(aux32, 2);\n\n // 2. Broadcast/Gather: replicate aux[0] to first 4 lanes, aux[1] to next 4 lanes\n vuint32m1_t v_aux_expanded = __riscv_vrgather_vv_u32m1(v_aux, v_gather_idx, 8);\n\n // 3. Apply Shifts and Mask: ((val >> shift) & 127)\n vuint32m1_t v_s_vals_raw = __riscv_vand_vx_u32m1(__riscv_vsrl_vv_u32m1(v_aux_expanded, v_shifts, 8), 127, 8);\n\n // 4. Narrow to u16 (required for vluxei index) and multiply by 8 (byte offset for u64 table)\n vuint16mf2_t sign_indices_byte_offset = __riscv_vsll_vx_u16mf2(__riscv_vncvt_x_x_w_u16mf2(v_s_vals_raw, 8), 3, 8);\n\n // 5. Gather Signs\n vuint64m2_t v_s_vals_u64 = __riscv_vluxei16_v_u64m2(signs64, sign_indices_byte_offset, 8);\n vint8m2_t v_s_vals = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(v_s_vals_u64));\n\n vint8m2_t v_q3_signed = __riscv_vmul_vv_i8m2(v_q3_magnitudes, v_s_vals, 64);\n vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_q8, v_q3_signed, 64);\n\n vint16m2_t v_dot_1 = __riscv_vget_v_i16m4_i16m2(v_dot, 0);\n vint16m2_t v_dot_2 = __riscv_vget_v_i16m4_i16m2(v_dot, 1);\n\n vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);\n vint32m1_t v_sum_1 = __riscv_vwredsum_vs_i16m2_i32m1(v_dot_1, v_zero, 32);\n vint32m1_t v_sum_2 = __riscv_vwredsum_vs_i16m2_i32m1(v_dot_2, v_zero, 32);\n\n int32_t sum1_i = __riscv_vmv_x_s_i32m1_i32(v_sum_1);\n int32_t sum2_i = __riscv_vmv_x_s_i32m1_i32(v_sum_2);\n\n const float scale1_f = (float)(2 * (aux32[0] >> 28) + 1);\n const float scale2_f = (float)(2 * (aux32[1] >> 28) + 1);\n\n block_sum += sum1_i * scale1_f + sum2_i * scale2_f;\n }\n\n sumf += d * block_sum;\n }\n *s = 0.25f * sumf;\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 256:\n ggml_vec_dot_iq3_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq4_nl_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK4_NL == 0);\n static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n const block_iq4_nl * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK4_NL;\n\n int ib = 0;\n float sumf = 0;\n\n // Load the lookup table once.\n const vint8m2_t values = __riscv_vle8_v_i8m2(kvalues_iq4nl, 16);\n int acc1, acc2;\n\n // We process 2 blocks at once.\n for (; ib + 1 < nb; ib += 2) {\n // Weights and activations.\n vuint8m1_t iq4_packed1 = __riscv_vle8_v_u8m1(x[ib + 0].qs, 16);\n vint8m2_t q8b1 = __riscv_vle8_v_i8m2(y[ib + 0].qs, 32);\n vuint8m1_t iq4_packed2 = __riscv_vle8_v_u8m1(x[ib + 1].qs, 16);\n vint8m2_t q8b2 = __riscv_vle8_v_i8m2(y[ib + 1].qs, 32);\n\n // Unpack the weight blocks.\n vuint8m2_t iq4bits1;\n iq4bits1 = __riscv_vset_v_u8m1_u8m2(iq4bits1, 0, __riscv_vand_vx_u8m1(iq4_packed1, 0xf, 16));\n iq4bits1 = __riscv_vset_v_u8m1_u8m2(iq4bits1, 1, __riscv_vsrl_vx_u8m1(iq4_packed1, 4, 16));\n vuint8m2_t iq4bits2;\n iq4bits2 = __riscv_vset_v_u8m1_u8m2(iq4bits2, 0, __riscv_vand_vx_u8m1(iq4_packed2, 0xf, 16));\n iq4bits2 = __riscv_vset_v_u8m1_u8m2(iq4bits2, 1, __riscv_vsrl_vx_u8m1(iq4_packed2, 4, 16));\n\n // Gather values from the lookup table.\n vint8m2_t iq4b1 = __riscv_vrgather_vv_i8m2(values, iq4bits1, 32);\n vint8m2_t iq4b2 = __riscv_vrgather_vv_i8m2(values, iq4bits2, 32);\n\n // Accumulation.\n vint16m4_t sum1 = __riscv_vwmul_vv_i16m4(q8b1, iq4b1, 32);\n vint16m4_t sum2 = __riscv_vwmul_vv_i16m4(q8b2, iq4b2, 32);\n __riscv_vse32_v_i32m1(&acc1,__riscv_vwredsum_vs_i16m4_i32m1(sum1, __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n __riscv_vse32_v_i32m1(&acc2,__riscv_vwredsum_vs_i16m4_i32m1(sum2, __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n sumf += ((GGML_CPU_FP16_TO_FP32(x[ib + 0].d) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * acc1));\n sumf += ((GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * acc2));\n }\n\n *s = sumf;\n}\n\nstatic void ggml_vec_dot_iq4_nl_q8_0_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK4_NL == 0);\n static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n const block_iq4_nl * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK4_NL;\n\n int ib = 0;\n float sumf = 0;\n\n // Load the lookup table once.\n const vint8mf2_t values = __riscv_vle8_v_i8mf2(kvalues_iq4nl, 16);\n int acc1, acc2;\n\n // We process 2 blocks at once.\n for (; ib + 1 < nb; ib += 2) {\n // Weights and activations.\n vuint8mf2_t iq4_packed1 = __riscv_vle8_v_u8mf2(x[ib + 0].qs, 16);\n vint8mf2_t q8b_lo1 = __riscv_vle8_v_i8mf2(y[ib + 0].qs, 16);\n vint8mf2_t q8b_hi1 = __riscv_vle8_v_i8mf2(y[ib + 0].qs + 16, 16);\n vuint8mf2_t iq4_packed2 = __riscv_vle8_v_u8mf2(x[ib + 1].qs, 16);\n vint8mf2_t q8b_lo2 = __riscv_vle8_v_i8mf2(y[ib + 1].qs, 16);\n vint8mf2_t q8b_hi2 = __riscv_vle8_v_i8mf2(y[ib + 1].qs + 16, 16);\n\n // Unpack the weight blocks.\n vuint8mf2_t iq4bits_lo1 = __riscv_vand_vx_u8mf2(iq4_packed1, 0xf, 16);\n vuint8mf2_t iq4bits_hi1 = __riscv_vsrl_vx_u8mf2(iq4_packed1, 4, 16);\n vuint8mf2_t iq4bits_lo2 = __riscv_vand_vx_u8mf2(iq4_packed2, 0xf, 16);\n vuint8mf2_t iq4bits_hi2 = __riscv_vsrl_vx_u8mf2(iq4_packed2, 4, 16);\n\n // Gather values from the lookup table.\n vint8mf2_t iq4b_lo1 = __riscv_vrgather_vv_i8mf2(values, iq4bits_lo1, 16);\n vint8mf2_t iq4b_hi1 = __riscv_vrgather_vv_i8mf2(values, iq4bits_hi1, 16);\n vint8mf2_t iq4b_lo2 = __riscv_vrgather_vv_i8mf2(values, iq4bits_lo2, 16);\n vint8mf2_t iq4b_hi2 = __riscv_vrgather_vv_i8mf2(values, iq4bits_hi2, 16);\n\n // Accumulation.\n vint16m1_t sum1 = __riscv_vwmul_vv_i16m1(q8b_lo1, iq4b_lo1, 16);\n sum1 = __riscv_vwmacc_vv_i16m1(sum1, q8b_hi1, iq4b_hi1, 16);\n vint16m1_t sum2 = __riscv_vwmul_vv_i16m1(q8b_lo2, iq4b_lo2, 16);\n sum2 = __riscv_vwmacc_vv_i16m1(sum2, q8b_hi2, iq4b_hi2, 16);\n __riscv_vse32_v_i32m1(&acc1,__riscv_vwredsum_vs_i16m1_i32m1(sum1, __riscv_vmv_v_x_i32m1(0, 1), 16), 1);\n __riscv_vse32_v_i32m1(&acc2,__riscv_vwredsum_vs_i16m1_i32m1(sum2, __riscv_vmv_v_x_i32m1(0, 1), 16), 1);\n sumf += ((GGML_CPU_FP16_TO_FP32(x[ib + 0].d) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * acc1));\n sumf += ((GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * acc2));\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 128:\n ggml_vec_dot_iq4_nl_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_iq4_nl_q8_0_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq4_xs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_K == 0);\n\n const block_iq4_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __riscv_v_intrinsic\n const vint8m4_t values = __riscv_vle8_v_i8m4(kvalues_iq4nl, 16);\n float sumf = 0;\n int acc[4];\n\n // Indices for re-ordering IQ4 data.\n uint64_t index[16] = {\n 0, 1, 8, 9,\n 2, 3, 10, 11,\n 4, 5,12, 13,\n 6, 7, 14, 15,\n };\n vuint64m4_t i_vec = __riscv_vle64_v_u64m4(index, 16);\n\n for (int ibl = 0; ibl < nb; ++ibl) {\n const int8_t * q8 = y[ibl].qs;\n const uint8_t * iq4 = x[ibl].qs;\n uint16_t h = x[ibl].scales_h;\n\n int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0;\n\n for (int ib = 0; ib < QK_K / 128; ++ib) {\n // Weights and activations.\n vuint8m2_t iq4_packed = __riscv_vle8_v_u8m2(iq4, 64);\n vint8m4_t q8b = __riscv_vle8_v_i8m4(q8, 128);\n iq4 += 64;\n q8 += 128;\n\n // Unpack the weight blocks.\n vuint8m2_t iq4bits_lo = __riscv_vand_vx_u8m2(iq4_packed, 0xf, 64);\n vuint8m2_t iq4bits_hi = __riscv_vsrl_vx_u8m2(iq4_packed, 4, 64);\n vuint8m4_t iq4bits;\n iq4bits = __riscv_vset_v_u8m2_u8m4(iq4bits, 0, iq4bits_lo);\n iq4bits = __riscv_vset_v_u8m2_u8m4(iq4bits, 1, iq4bits_hi);\n vuint8m4_t iq4bits_reorder = __riscv_vreinterpret_v_u64m4_u8m4(__riscv_vrgather_vv_u64m4(__riscv_vreinterpret_v_u8m4_u64m4(iq4bits), i_vec, 16));\n vint8m4_t iq4b = __riscv_vrgather_vv_i8m4(values, iq4bits_reorder, 128);\n\n // Multiply with activations.\n vint16m8_t prod = __riscv_vwmul_vv_i16m8(iq4b, q8b, 128);\n\n // Reduce separately.\n __riscv_vse32_v_i32m1(&acc[0],__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(prod, 0), __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n __riscv_vse32_v_i32m1(&acc[1],__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(prod, 1), __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n __riscv_vse32_v_i32m1(&acc[2],__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(prod, 2), __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n __riscv_vse32_v_i32m1(&acc[3],__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(prod, 3), __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n\n int ls1 = ((x[ibl].scales_l[ib * 2 + 0] & 0xf) | ((h << 4) & 0x30)) - 32;\n int ls2 = ((x[ibl].scales_l[ib * 2 + 0] >> 4) | ((h << 2) & 0x30)) - 32;\n int ls3 = ((x[ibl].scales_l[ib * 2 + 1] & 0xf) | ((h << 0) & 0x30)) - 32;\n int ls4 = ((x[ibl].scales_l[ib * 2 + 1] >> 4) | ((h >> 2) & 0x30)) - 32;\n h >>= 8;\n\n sumi1 += acc[0] * ls1;\n sumi2 += acc[1] * ls2;\n sumi3 += acc[2] * ls3;\n sumi4 += acc[3] * ls4;\n }\n\n sumf += GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2 + sumi3 + sumi4);\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 256:\n ggml_vec_dot_iq4_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_tq1_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_tq1_0 * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n float sumf = 0.0f;\n uint8_t pow[16] = {1, 1, 1, 1, 3, 3, 3, 3, 9, 9, 9, 9, 27, 27, 27, 27};\n\n for (int i = 0; i < nb; i++) {\n // First loop.\n vint32m4_t suml1;\n {\n const int vl = 32;\n vuint8m1_t tq = __riscv_vle8_v_u8m1(x[i].qs, vl);\n\n vuint16m2_t tq0 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(tq, 3, vl), 8, vl);\n vuint16m2_t tq1 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 3, vl), 3, vl), 8, vl);\n vuint16m2_t tq2 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 9, vl), 3, vl), 8, vl);\n vuint16m2_t tq3 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 27, vl), 3, vl), 8, vl);\n vuint16m2_t tq4 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 81, vl), 3, vl), 8, vl);\n\n vint16m2_t q80 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 0, vl), vl);\n vint16m2_t q81 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 32, vl), vl);\n vint16m2_t q82 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 64, vl), vl);\n vint16m2_t q83 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 96, vl), vl);\n vint16m2_t q84 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 128, vl), vl);\n\n vint16m2_t sum0 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq0, 1, vl)), q80, vl);\n vint16m2_t sum1 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq1, 1, vl)), q81, vl);\n vint16m2_t sum2 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq2, 1, vl)), q82, vl);\n vint16m2_t sum3 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq3, 1, vl)), q83, vl);\n vint16m2_t sum4 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq4, 1, vl)), q84, vl);\n\n vint32m4_t sumi0 = __riscv_vwadd_vv_i32m4(sum0, sum1, vl);\n vint32m4_t sumi1 = __riscv_vwadd_vv_i32m4(sum2, sum3, vl);\n suml1 = __riscv_vadd_vv_i32m4(__riscv_vwcvt_x_x_v_i32m4(sum4, vl), __riscv_vadd_vv_i32m4(sumi0, sumi1, vl), vl);\n }\n\n // Second loop.\n vint32m2_t suml2;\n {\n const int vl = 16;\n vuint8mf2_t tq = __riscv_vle8_v_u8mf2(x[i].qs + 32, vl);\n\n vuint16m1_t tq0 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(tq, 3 * 1, vl), 8, vl);\n vuint16m1_t tq1 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 3, vl), 3, vl), 8, vl);\n vuint16m1_t tq2 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 9, vl), 3, vl), 8, vl);\n vuint16m1_t tq3 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 27, vl), 3, vl), 8, vl);\n vuint16m1_t tq4 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 81, vl), 3, vl), 8, vl);\n\n vint16m1_t q80 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 160, vl), vl);\n vint16m1_t q81 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 176, vl), vl);\n vint16m1_t q82 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 192, vl), vl);\n vint16m1_t q83 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 208, vl), vl);\n vint16m1_t q84 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 224, vl), vl);\n\n vint16m1_t sum0 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq0, 1, vl)), q80, vl);\n vint16m1_t sum1 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq1, 1, vl)), q81, vl);\n vint16m1_t sum2 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq2, 1, vl)), q82, vl);\n vint16m1_t sum3 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq3, 1, vl)), q83, vl);\n vint16m1_t sum4 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq4, 1, vl)), q84, vl);\n\n vint32m2_t sumi0 = __riscv_vwadd_vv_i32m2(sum0, sum1, vl);\n vint32m2_t sumi1 = __riscv_vwadd_vv_i32m2(sum2, sum3, vl);\n suml2 = __riscv_vadd_vv_i32m2(__riscv_vwcvt_x_x_v_i32m2(sum4, vl), __riscv_vadd_vv_i32m2(sumi0, sumi1, vl), vl);\n }\n\n // Third loop.\n vint32m2_t suml3;\n {\n const int vl = 16;\n\n uint32_t qh;\n memcpy(&qh, &x[i].qh[0], 4);\n // Prevent fusion with vmv.\n __asm__ __volatile__(\"\" : \"+r\"(qh));\n vuint8mf2_t tq = __riscv_vreinterpret_v_u32mf2_u8mf2(__riscv_vmv_v_x_u32mf2(qh, vl / 4));\n\n vuint8mf2_t p = __riscv_vle8_v_u8mf2(pow, vl);\n\n vuint16m1_t tq0 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vv_u8mf2(tq, p, vl), 3, vl), 8, vl);\n\n vint16m1_t q80 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 240, vl), vl);\n\n vint16m1_t sum0 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq0, 1, vl)), q80, vl);\n suml3 = __riscv_vwcvt_x_x_v_i32m2(sum0, vl);\n }\n\n vint32m2_t sumb = __riscv_vadd_vv_i32m2(__riscv_vget_v_i32m4_i32m2(suml1, 0), __riscv_vget_v_i32m4_i32m2(suml1, 1), 16);\n sumb = __riscv_vadd_vv_i32m2(sumb, suml2, 16);\n sumb = __riscv_vadd_vv_i32m2(sumb, suml3, 16);\n\n vint32m1_t sum = __riscv_vredsum_vs_i32m2_i32m1(sumb, __riscv_vmv_v_x_i32m1(0, 1), 16);\n sumf += __riscv_vmv_x_s_i32m1_i32(sum) * y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 256:\n ggml_vec_dot_tq1_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_tq2_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_tq2_0 * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n float sumf = 0.0f;\n for (int i = 0; i < nb; ++i) {\n int32_t sumi = 0;\n\n for (size_t j = 0; j < sizeof(x[0].qs); j += 32) {\n const int8_t * py0 = &y[i].qs[j * 4 + 0 * 32];\n const int8_t * py1 = &y[i].qs[j * 4 + 1 * 32];\n const int8_t * py2 = &y[i].qs[j * 4 + 2 * 32];\n const int8_t * py3 = &y[i].qs[j * 4 + 3 * 32];\n const uint8_t* px = &x[i].qs[j];\n\n size_t vlmax_16m2 = __riscv_vsetvl_e16m2(32);\n vint16m2_t vacc16 = __riscv_vmv_v_x_i16m2(0, vlmax_16m2);\n\n size_t vl = __riscv_vsetvl_e8m1(32);\n\n vuint8m1_t vx_u8 = __riscv_vle8_v_u8m1(px, vl);\n\n vint8m1_t vy0 = __riscv_vle8_v_i8m1(py0 , vl);\n vint8m1_t vy1 = __riscv_vle8_v_i8m1(py1, vl);\n vint8m1_t vy2 = __riscv_vle8_v_i8m1(py2, vl);\n vint8m1_t vy3 = __riscv_vle8_v_i8m1(py3, vl);\n\n // l=0 (bits 1:0)\n vuint8m1_t t0 = __riscv_vand_vx_u8m1(vx_u8, 0x03, vl);\n vint8m1_t vq0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t0), 1, vl);\n\n // l=1 (bits 3:2)\n vuint8m1_t t1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vx_u8, 2, vl), 0x03, vl);\n vint8m1_t vq1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t1), 1, vl);\n\n // l=2 (bits 5:4)\n vuint8m1_t t2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vx_u8, 4, vl), 0x03, vl);\n vint8m1_t vq2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t2), 1, vl);\n\n // l=3 (bits 7:6)\n vuint8m1_t t3 = __riscv_vsrl_vx_u8m1(vx_u8, 6, vl); // No final AND needed as vsrl shifts in zeros\n vint8m1_t vq3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t3), 1, vl);\n\n // 4. Multiply and accumulate\n vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq0, vy0, vl);\n vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq1, vy1, vl);\n vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq2, vy2, vl);\n vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq3, vy3, vl);\n\n vlmax_16m2 = __riscv_vsetvl_e16m2(32);\n vint32m1_t vzero32 = __riscv_vmv_v_x_i32m1(0, 1);\n vint32m1_t vred32 = __riscv_vwredsum_vs_i16m2_i32m1(vacc16, vzero32, vlmax_16m2);\n\n sumi += __riscv_vmv_x_s_i32m1_i32(vred32);\n }\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n sumf += (float)sumi * d;\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 256:\n ggml_vec_dot_tq2_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_mxfp4_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_MXFP4 == 0);\n static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n const block_mxfp4 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_MXFP4;\n\n int ib = 0;\n float sumf = 0;\n\n // Load the lookup table once.\n const vint8m2_t values = __riscv_vle8_v_i8m2(kvalues_mxfp4, 16);\n int acc1, acc2;\n\n // We process 2 blocks at once.\n for (; ib + 1 < nb; ib += 2) {\n // Weights and activations.\n vuint8m1_t mx_packed1 = __riscv_vle8_v_u8m1(x[ib + 0].qs, 16);\n vint8m2_t q8b1 = __riscv_vle8_v_i8m2(y[ib + 0].qs, 32);\n vuint8m1_t mx_packed2 = __riscv_vle8_v_u8m1(x[ib + 1].qs, 16);\n vint8m2_t q8b2 = __riscv_vle8_v_i8m2(y[ib + 1].qs, 32);\n\n // Unpack the weight blocks.\n vuint8m2_t mxbits1;\n mxbits1 = __riscv_vset_v_u8m1_u8m2(mxbits1, 0, __riscv_vand_vx_u8m1(mx_packed1, 0xf, 16));\n mxbits1 = __riscv_vset_v_u8m1_u8m2(mxbits1, 1, __riscv_vsrl_vx_u8m1(mx_packed1, 4, 16));\n vuint8m2_t mxbits2;\n mxbits2 = __riscv_vset_v_u8m1_u8m2(mxbits2, 0, __riscv_vand_vx_u8m1(mx_packed2, 0xf, 16));\n mxbits2 = __riscv_vset_v_u8m1_u8m2(mxbits2, 1, __riscv_vsrl_vx_u8m1(mx_packed2, 4, 16));\n\n // Gather values from the lookup table.\n vint8m2_t mxb1 = __riscv_vrgather_vv_i8m2(values, mxbits1, 32);\n vint8m2_t mxb2 = __riscv_vrgather_vv_i8m2(values, mxbits2, 32);\n\n // Accumulation.\n vint16m4_t sum1 = __riscv_vwmul_vv_i16m4(q8b1, mxb1, 32);\n vint16m4_t sum2 = __riscv_vwmul_vv_i16m4(q8b2, mxb2, 32);\n __riscv_vse32_v_i32m1(&acc1,__riscv_vwredsum_vs_i16m4_i32m1(sum1, __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n __riscv_vse32_v_i32m1(&acc2,__riscv_vwredsum_vs_i16m4_i32m1(sum2, __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n sumf += ((GGML_E8M0_TO_FP32_HALF(x[ib + 0].e) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * acc1));\n sumf += ((GGML_E8M0_TO_FP32_HALF(x[ib + 1].e) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * acc2));\n }\n\n *s = sumf;\n}\n\nstatic void ggml_vec_dot_mxfp4_q8_0_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_MXFP4 == 0);\n static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n const block_mxfp4 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_MXFP4;\n\n int ib = 0;\n float sumf = 0;\n\n // Load the lookup table once.\n const vint8mf2_t values = __riscv_vle8_v_i8mf2(kvalues_mxfp4, 16);\n int acc1, acc2;\n\n // We process 2 blocks at once.\n for (; ib + 1 < nb; ib+=2) {\n // Weights and activations.\n vuint8mf2_t mx_packed1 = __riscv_vle8_v_u8mf2(x[ib + 0].qs, 16);\n vint8mf2_t q8b_lo1 = __riscv_vle8_v_i8mf2(y[ib + 0].qs, 16);\n vint8mf2_t q8b_hi1 = __riscv_vle8_v_i8mf2(y[ib + 0].qs + 16, 16);\n vuint8mf2_t mx_packed2 = __riscv_vle8_v_u8mf2(x[ib + 1].qs, 16);\n vint8mf2_t q8b_lo2 = __riscv_vle8_v_i8mf2(y[ib + 1].qs, 16);\n vint8mf2_t q8b_hi2 = __riscv_vle8_v_i8mf2(y[ib + 1].qs + 16, 16);\n\n // Unpack the weight blocks.\n vuint8mf2_t mxbits_lo1 = __riscv_vand_vx_u8mf2(mx_packed1, 0xf, 16);\n vuint8mf2_t mxbits_hi1 = __riscv_vsrl_vx_u8mf2(mx_packed1, 4, 16);\n vuint8mf2_t mxbits_lo2 = __riscv_vand_vx_u8mf2(mx_packed2, 0xf, 16);\n vuint8mf2_t mxbits_hi2 = __riscv_vsrl_vx_u8mf2(mx_packed2, 4, 16);\n\n // Gather values from the lookup table.\n vint8mf2_t mxb_lo1 = __riscv_vrgather_vv_i8mf2(values, mxbits_lo1, 16);\n vint8mf2_t mxb_hi1 = __riscv_vrgather_vv_i8mf2(values, mxbits_hi1, 16);\n vint8mf2_t mxb_lo2 = __riscv_vrgather_vv_i8mf2(values, mxbits_lo2, 16);\n vint8mf2_t mxb_hi2 = __riscv_vrgather_vv_i8mf2(values, mxbits_hi2, 16);\n\n // Accumulation.\n vint16m1_t sum1 = __riscv_vwmul_vv_i16m1(q8b_lo1, mxb_lo1, 16);\n sum1 = __riscv_vwmacc_vv_i16m1(sum1, q8b_hi1, mxb_hi1, 16);\n vint16m1_t sum2 = __riscv_vwmul_vv_i16m1(q8b_lo2, mxb_lo2, 16);\n sum2 = __riscv_vwmacc_vv_i16m1(sum2, q8b_hi2, mxb_hi2, 16);\n __riscv_vse32_v_i32m1(&acc1,__riscv_vwredsum_vs_i16m1_i32m1(sum1, __riscv_vmv_v_x_i32m1(0, 1), 16), 1);\n __riscv_vse32_v_i32m1(&acc2,__riscv_vwredsum_vs_i16m1_i32m1(sum2, __riscv_vmv_v_x_i32m1(0, 1), 16), 1);\n sumf += ((GGML_E8M0_TO_FP32_HALF(x[ib + 0].e) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * acc1));\n sumf += ((GGML_E8M0_TO_FP32_HALF(x[ib + 1].e) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * acc2));\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n switch (__riscv_vlenb() * 8) {\n case 128:\n ggml_vec_dot_mxfp4_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc);\n break;\n default:\n ggml_vec_dot_mxfp4_q8_0_vl256(n, s, bs, vx, bx, vy, by, nrc);\n break;\n }\n#else\n return ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n" }, { "path": "src/ggml-cpu/arch/riscv/repack.cpp", "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"traits.h\"\n\n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GGML_CPU_CLANG_WORKAROUND\n#include \"../../repack.h\"\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#endif\n\n#define UNUSED GGML_UNUSED\n\nvoid ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n#if defined(__riscv_v_intrinsic)\n block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n const size_t vl_calc = __riscv_vsetvl_e32m8(QK8_0);\n const size_t vl_save = __riscv_vsetvl_e64m2(4);\n vfloat32m1_t v_scalar_zero = __riscv_vfmv_s_f_f32m1(0.0f, __riscv_vsetvl_e32m1(1));\n\n for (int i = 0; i < nb; i++) {\n const float *x_block_base = x + i * QK8_0;\n vint8m2_t q_r0, q_r1, q_r2, q_r3;\n {\n vfloat32m8_t v_src = __riscv_vle32_v_f32m8(x_block_base + 0 * k, vl_calc);\n vfloat32m8_t v_abs = __riscv_vfabs_v_f32m8(v_src, vl_calc);\n vfloat32m1_t v_max = __riscv_vfredmax_vs_f32m8_f32m1(v_abs, v_scalar_zero, vl_calc);\n float amax = __riscv_vfmv_f_s_f32m1_f32(v_max);\n\n float d = amax / 127.0f;\n y[i].d[0] = GGML_CPU_FP32_TO_FP16(d);\n\n float id = d ? 1.0f / d : 0.0f;\n vfloat32m8_t v_scaled = __riscv_vfmul_vf_f32m8(v_src, id, vl_calc);\n vint16m4_t v_i16 = __riscv_vfncvt_x_f_w_i16m4_rm(v_scaled, 4, vl_calc);\n q_r0 = __riscv_vncvt_x_x_w_i8m2(v_i16, vl_calc);\n }\n asm volatile (\"\" ::: \"memory\");\n\n {\n vfloat32m8_t v_src = __riscv_vle32_v_f32m8(x_block_base + 1 * k, vl_calc);\n vfloat32m8_t v_abs = __riscv_vfabs_v_f32m8(v_src, vl_calc);\n vfloat32m1_t v_max = __riscv_vfredmax_vs_f32m8_f32m1(v_abs, v_scalar_zero, vl_calc);\n float amax = __riscv_vfmv_f_s_f32m1_f32(v_max);\n\n float d = amax / 127.0f;\n y[i].d[1] = GGML_CPU_FP32_TO_FP16(d);\n float id = d ? 1.0f / d : 0.0f;\n\n vfloat32m8_t v_scaled = __riscv_vfmul_vf_f32m8(v_src, id, vl_calc);\n vint16m4_t v_i16 = __riscv_vfncvt_x_f_w_i16m4_rm(v_scaled, 4, vl_calc);\n q_r1 = __riscv_vncvt_x_x_w_i8m2(v_i16, vl_calc);\n }\n asm volatile (\"\" ::: \"memory\");\n {\n vfloat32m8_t v_src = __riscv_vle32_v_f32m8(x_block_base + 2 * k, vl_calc);\n vfloat32m8_t v_abs = __riscv_vfabs_v_f32m8(v_src, vl_calc);\n vfloat32m1_t v_max = __riscv_vfredmax_vs_f32m8_f32m1(v_abs, v_scalar_zero, vl_calc);\n float amax = __riscv_vfmv_f_s_f32m1_f32(v_max);\n\n float d = amax / 127.0f;\n y[i].d[2] = GGML_CPU_FP32_TO_FP16(d);\n float id = d ? 1.0f / d : 0.0f;\n\n vfloat32m8_t v_scaled = __riscv_vfmul_vf_f32m8(v_src, id, vl_calc);\n vint16m4_t v_i16 = __riscv_vfncvt_x_f_w_i16m4_rm(v_scaled, 4, vl_calc);\n q_r2 = __riscv_vncvt_x_x_w_i8m2(v_i16, vl_calc);\n }\n asm volatile (\"\" ::: \"memory\");\n {\n vfloat32m8_t v_src = __riscv_vle32_v_f32m8(x_block_base + 3 * k, vl_calc);\n vfloat32m8_t v_abs = __riscv_vfabs_v_f32m8(v_src, vl_calc);\n vfloat32m1_t v_max = __riscv_vfredmax_vs_f32m8_f32m1(v_abs, v_scalar_zero, vl_calc);\n float amax = __riscv_vfmv_f_s_f32m1_f32(v_max);\n\n float d = amax / 127.0f;\n y[i].d[3] = GGML_CPU_FP32_TO_FP16(d);\n float id = d ? 1.0f / d : 0.0f;\n\n vfloat32m8_t v_scaled = __riscv_vfmul_vf_f32m8(v_src, id, vl_calc);\n vint16m4_t v_i16 = __riscv_vfncvt_x_f_w_i16m4_rm(v_scaled, 4, vl_calc);\n q_r3 = __riscv_vncvt_x_x_w_i8m2(v_i16, vl_calc);\n }\n vint64m2_t v_q64_r0 = __riscv_vreinterpret_v_i8m2_i64m2(q_r0);\n vint64m2_t v_q64_r1 = __riscv_vreinterpret_v_i8m2_i64m2(q_r1);\n vint64m2_t v_q64_r2 = __riscv_vreinterpret_v_i8m2_i64m2(q_r2);\n vint64m2_t v_q64_r3 = __riscv_vreinterpret_v_i8m2_i64m2(q_r3);\n vint64m2x4_t v_quant_tuple = __riscv_vcreate_v_i64m2x4(v_q64_r0, v_q64_r1, v_q64_r2, v_q64_r3);\n __riscv_vsseg4e64_v_i64m2x4((int64_t*)y[i].qs, v_quant_tuple, vl_save);\n }\n#else\n UNUSED(nb);\n UNUSED(y);\n ggml_quantize_mat_q8_0_4x4_generic(x, vy, k);\n#endif\n}\n\nvoid ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined __riscv_v\n if (__riscv_vlenb() >= QK4_0) {\n const size_t vl = QK4_0;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);\n\n vfloat32m1_t sumf = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n for (int l = 0; l < nb; l++) {\n const int64_t a0 = *(const int64_t *)&a_ptr[l].qs[0];\n const int64_t a1 = *(const int64_t *)&a_ptr[l].qs[8];\n const int64_t a2 = *(const int64_t *)&a_ptr[l].qs[16];\n const int64_t a3 = *(const int64_t *)&a_ptr[l].qs[24];\n __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment constraints\n const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(a0, vl / 4));\n const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(a1, vl / 4));\n const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(a2, vl / 4));\n const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(a3, vl / 4));\n\n const vint8m4_t rhs_raw_vec = __riscv_vle8_v_i8m4((const int8_t *)b_ptr[l].qs, vl * 4);\n const vint8m4_t rhs_vec_lo = __riscv_vsra_vx_i8m4(__riscv_vsll_vx_i8m4(rhs_raw_vec, 4, vl * 4), 4, vl * 4);\n const vint8m4_t rhs_vec_hi = __riscv_vsra_vx_i8m4(rhs_raw_vec, 4, vl * 4);\n const vint8m2_t rhs_vec_lo_0 = __riscv_vget_v_i8m4_i8m2(rhs_vec_lo, 0);\n const vint8m2_t rhs_vec_lo_1 = __riscv_vget_v_i8m4_i8m2(rhs_vec_lo, 1);\n const vint8m2_t rhs_vec_hi_0 = __riscv_vget_v_i8m4_i8m2(rhs_vec_hi, 0);\n const vint8m2_t rhs_vec_hi_1 = __riscv_vget_v_i8m4_i8m2(rhs_vec_hi, 1);\n\n const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_hi_m));\n const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n // vector version needs Zvfhmin extension\n const float a_scale = GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n const float b_scales[8] = {\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[4]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[5]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[6]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[7])\n };\n const vfloat32m1_t b_scales_vec = __riscv_vle32_v_f32m1(b_scales, vl / 4);\n const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scale, vl / 4);\n sumf = __riscv_vfmacc_vv_f32m1(sumf, tmp1, b_scales_vec, vl / 4);\n }\n __riscv_vse32_v_f32m1(s + x * ncols_interleaved, sumf, vl / 4);\n }\n return;\n }\n\n#endif\n ggml_gemv_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x16 * b_ptr = (const block_q4_0x16 *) vx + (x * nb);\n\n // 1x16 Accumulator\n vfloat32m2_t sumf = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n for (int l = 0; l < nb; l++) {\n // 1x16 Integer Accumulator\n vint16m1_t sumi_0_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_0_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n // Accumulation loop.\n for (int i = 0; i < QK4_0 / 2; i++) {\n // Load `b_ptr`.\n const vint8mf2_t b_0_packed = __riscv_vle8_v_i8mf2((const int8_t *)&b_ptr[l].qs[i * 16], 16);\n const vint8mf2_t b_0_lo = __riscv_vsra_vx_i8mf2(__riscv_vsll_vx_i8mf2(b_0_packed, 4, 16), 4, 16);\n const vint8mf2_t b_0_hi = __riscv_vsra_vx_i8mf2(b_0_packed, 4, 16);\n\n sumi_0_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_0_lo_16, a_ptr[l].qs[i], b_0_lo, 16);\n sumi_0_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_0_hi_16, a_ptr[l].qs[16 + i], b_0_hi, 16);\n }\n\n const vint32m2_t sumi = __riscv_vwadd_vv_i32m2(sumi_0_lo_16, sumi_0_hi_16, 16);\n\n const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d, 16);\n\n sumf = __riscv_vfmacc_vv_f32m2(sumf, __riscv_vfcvt_f_x_v_f32m2(sumi, 16), d_0, 16);\n }\n\n __riscv_vse32_v_f32m2(s + x * 16, sumf, 16);\n }\n return;\n#endif\n ggml_gemv_q4_0_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n const block_q8_K * a_ptr = (const block_q8_K *) vy;\n\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx16 * b_ptr = (const block_q4_Kx16 *) vx + (x * nb);\n\n // 1x16 Accumulator\n vfloat32m2_t sumf = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n for (int l = 0; l < nb; l++) {\n vint32m2_t sumi = __riscv_vmv_v_x_i32m2(0, 16);\n\n // Load `dmin`.\n const vfloat32m2_t dmins_d = __riscv_vfmul_vf_f32m2(\n __riscv_vfwcvt_f_f_v_f32m2(__riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].dmin, 16), 16), a_ptr[l].d, 16);\n\n // We process 4 sub-blocks at once.\n for (int j = 0; j < QK_K / 128; j++) {\n // Extract the scales and the mins.\n //\n // Low bits.\n vuint8m2_t scales_mins_lo = __riscv_vle8_v_u8m2(&b_ptr[l].scales[j * 64], 64);\n vuint8m2_t scales_lo = __riscv_vand_vx_u8m2(scales_mins_lo, 0x0F, 64);\n vuint8m2_t mins_lo = __riscv_vsrl_vx_u8m2(scales_mins_lo, 4, 64);\n\n // High bits.\n vuint8m2_t scales_mins_hi = __riscv_vle8_v_u8m2(&b_ptr[l].scales[128], 64);\n vuint8m2_t scales_hi;\n vuint8m2_t mins_hi;\n if (!j) {\n scales_hi = __riscv_vsll_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0x03, 64), 4, 64);\n mins_hi = __riscv_vsll_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0x0C, 64), 2, 64);\n } else {\n scales_hi = __riscv_vand_vx_u8m2(scales_mins_hi, 0x30, 64);\n mins_hi = __riscv_vsrl_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0xC0, 64), 2, 64);\n }\n vuint16m4_t scales = __riscv_vzext_vf2_u16m4(__riscv_vor_vv_u8m2(scales_hi, scales_lo, 64), 64);\n vint16m4_t mins = __riscv_vreinterpret_v_u16m4_i16m4(__riscv_vzext_vf2_u16m4(__riscv_vor_vv_u8m2(mins_hi, mins_lo, 64), 64));\n\n // Reduce the mins and multiply with `dmin`.\n //\n // Correct in `sumf`.\n vint32m2_t bsums = __riscv_vmv_v_x_i32m2(0, 16);\n bsums = __riscv_vwmacc_vx_i32m2(bsums, a_ptr[l].bsums[j * 8] + a_ptr[l].bsums[j * 8 + 1], __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n bsums = __riscv_vwmacc_vx_i32m2(bsums, a_ptr[l].bsums[j * 8 + 2] + a_ptr[l].bsums[j * 8 + 3], __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n bsums = __riscv_vwmacc_vx_i32m2(bsums, a_ptr[l].bsums[j * 8 + 4] + a_ptr[l].bsums[j * 8 + 5], __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n bsums = __riscv_vwmacc_vx_i32m2(bsums, a_ptr[l].bsums[j * 8 + 6] + a_ptr[l].bsums[j * 8 + 7], __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n\n sumf = __riscv_vfsub_vv_f32m2(sumf, __riscv_vfmul_vv_f32m2(dmins_d, __riscv_vfcvt_f_x_v_f32m2(bsums, 16), 16), 16);\n\n // Accumulation for 2 sub-blocks.\n //\n // This might overflow, so we accumulate in two steps.\n //\n // Recheck.\n for (int k = 0; k < 2; k++) {\n vint16m1_t sumi_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n for (int i = k * 16; i < k * 16 + QK4_0 / 2; i++) {\n // Load `b_ptr`.\n const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2(&b_ptr[l].qs[j * 1024 + i * 16], 16);\n const vint8mf2_t b_s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(b_0_packed, 0xF, 16));\n const vint8mf2_t b_s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16));\n\n sumi_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_s_0_16, a_ptr[l].qs[j * 128 + i], b_s_0, 16);\n sumi_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_s_1_16, a_ptr[l].qs[j * 128 + 32 + i], b_s_1, 16);\n }\n\n sumi = __riscv_vwmacc_vv_i32m2(sumi,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n sumi_s_0_16, 16);\n sumi = __riscv_vwmacc_vv_i32m2(sumi,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n sumi_s_1_16, 16);\n }\n // Accumulation for 2 sub-blocks.\n //\n // This might overflow, so we accumulate in two steps.\n //\n // Recheck.\n for (int k = 0; k < 2; k++) {\n vint16m1_t sumi_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n for (int i = k * 16; i < k * 16 + QK4_0 / 2; i++) {\n // Load `b_ptr`.\n const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2(&b_ptr[l].qs[j * 1024 + 512 + i * 16], 16);\n const vint8mf2_t b_s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(b_0_packed, 0xF, 16));\n const vint8mf2_t b_s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16));\n\n sumi_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_s_0_16, a_ptr[l].qs[j * 128 + 64 + i], b_s_0, 16);\n sumi_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_s_1_16, a_ptr[l].qs[j * 128 + 96 + i], b_s_1, 16);\n }\n\n sumi = __riscv_vwmacc_vv_i32m2(sumi,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n sumi_s_0_16, 16);\n sumi = __riscv_vwmacc_vv_i32m2(sumi,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n sumi_s_1_16, 16);\n }\n }\n\n const vfloat32m2_t b_d = __riscv_vfwcvt_f_f_v_f32m2(__riscv_vle16_v_f16m1((const _Float16 *)&b_ptr[l].d[0], 16), 16);\n const vfloat32m2_t d_0 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d, 16);\n\n sumf = __riscv_vfmacc_vv_f32m2(sumf, __riscv_vfcvt_f_x_v_f32m2(sumi, 16), d_0, 16);\n }\n\n __riscv_vse32_v_f32m2(s + x * 16, sumf, 16);\n }\n return;\n#endif\n ggml_gemv_q4_K_16x1_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_iq4_nl_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n const vint8mf2_t values = __riscv_vle8_v_i8mf2(kvalues_iq4nl, 16);\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx16 * b_ptr = (const block_iq4_nlx16 *) vx + (x * nb);\n\n // 1x16 Accumulator1\n vfloat32m2_t sumf = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n for (int l = 0; l < nb; l++) {\n // 1x16 integer accumulator\n vint32m2_t sumi = __riscv_vmv_v_x_i32m2(0.0f, 16);\n\n // Accumulation loop.\n for (int i = 0; i < QK4_NL / 2; i++) {\n // Load `b_ptr`.\n const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2((const uint8_t *)&b_ptr[l].qs[i * 16], 16);\n const vint8mf2_t b_0_lo = __riscv_vrgather_vv_i8mf2(values, __riscv_vand_vx_u8mf2(b_0_packed, 0xf, 16), 16);\n const vint8mf2_t b_0_hi = __riscv_vrgather_vv_i8mf2(values, __riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16), 16);\n // const vint16m1_t b_0_lo_16 = __riscv_vwcvt_x_x_v_i16m1(b_0_lo, 16);\n // const vint16m1_t b_0_hi_16 = __riscv_vwcvt_x_x_v_i16m1(b_0_hi, 16);\n\n const vint16m1_t sumi_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i], 16);\n const vint16m1_t sumi_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[16 + i], 16);\n sumi = __riscv_vadd_vv_i32m2(sumi, __riscv_vwadd_vv_i32m2(sumi_lo, sumi_hi, 16), 16);\n }\n\n const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d, 16);\n\n sumf = __riscv_vfmacc_vv_f32m2(sumf, __riscv_vfcvt_f_x_v_f32m2(sumi, 16), d_0, 16);\n }\n\n __riscv_vse32_v_f32m2(s + x * 16, sumf, 16);\n }\n return;\n#endif\n ggml_gemv_iq4_nl_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q8_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n UNUSED(bs);\n\n#if defined __riscv_v_intrinsic\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q8_0x16 * b_ptr = (const block_q8_0x16 *) vx + (x * nb);\n\n // 1x16 Accumulator\n vfloat32m2_t sumf = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n for (int l = 0; l < nb; l++) {\n // 1x16 Integer Accumulator\n vint32m2_t sumi = __riscv_vmv_v_x_i32m2(0.0f, 16);\n\n // Accumulation loop.\n for (int i = 0; i < QK8_0; i++) {\n // Load `b_ptr`.\n const vint8mf2_t b_0 = __riscv_vle8_v_i8mf2((const int8_t *)&b_ptr[l].qs[i * 16], 16);\n // const vint16m1_t b_0_16 = __riscv_vwcvt_x_x_v_i16m1(b_0, 16);\n\n sumi = __riscv_vwadd_wv_i32m2(sumi, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i], 16), 16);\n }\n\n const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d, 16);\n\n sumf = __riscv_vfmacc_vv_f32m2(sumf, __riscv_vfcvt_f_x_v_f32m2(sumi, 16), d_0, 16);\n }\n\n __riscv_vse32_v_f32m2(s + x * 16, sumf, 16);\n }\n return;\n#endif\n ggml_gemv_q8_0_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n assert(n % QK_K == 0);\n assert(nr == 1);\n assert(nc % 16 == 0);\n\n UNUSED(bs);\n\n const int N_COLS_TILE = 16;\n const int num_k_blocks = n / QK_K;\n\n const size_t vl = __riscv_vsetvl_e32m2(N_COLS_TILE);\n for (int col_tile = 0; col_tile < nc; col_tile += N_COLS_TILE) {\n\n const block_q8_K* lhs_base_ptr = (const block_q8_K*)vy;\n const block_q2_Kx16* rhs_base_ptr = (const block_q2_Kx16*)vx + (col_tile / N_COLS_TILE) * num_k_blocks;\n\n vfloat32m2_t v_sumf = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n\n for (int k_block = 0; k_block < num_k_blocks; ++k_block) {\n const block_q8_K* lhs_current = &lhs_base_ptr[k_block];\n const block_q2_Kx16* rhs_current = &rhs_base_ptr[k_block];\n\n // 1. Prepare Global Min Scales\n vfloat16m1_t v_g_min_f16 = __riscv_vle16_v_f16m1((const _Float16*)rhs_current->dmin, vl);\n vfloat32m2_t v_g_min_base = __riscv_vfwcvt_f_f_v_f32m2(v_g_min_f16, vl);\n\n vfloat32m2_t v_g_min_final = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d, vl);\n\n vint32m2_t v_isum = __riscv_vmv_v_x_i32m2(0, vl);\n\n const uint8_t* rhs_qs_ptr = rhs_current->qs;\n const uint8_t* rhs_sc_ptr = rhs_current->scales;\n const int8_t* lhs_qs_ptr = lhs_current->qs;\n\n // --- Phase Loop (4 phases x 64 elements) ---\n for (int phase = 0; phase < 4; ++phase) {\n\n // A. Load Scales/Mins\n vuint16m1_t v_d_sb_0, v_d_sb_1, v_d_sb_2, v_d_sb_3;\n vuint16m1_t v_m_sb_0, v_m_sb_1, v_m_sb_2, v_m_sb_3;\n\n {\n vuint8mf2_t v_raw;\n // Sub-block 0\n v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 0, vl);\n v_d_sb_0 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n v_m_sb_0 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n // Sub-block 1\n v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 16, vl);\n v_d_sb_1 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n v_m_sb_1 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n // Sub-block 2\n v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 32, vl);\n v_d_sb_2 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n v_m_sb_2 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n // Sub-block 3\n v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 48, vl);\n v_d_sb_3 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n v_m_sb_3 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n rhs_sc_ptr += 64;\n }\n\n int base_k_phase = (phase < 2) ? (phase * 16) : (128 + (phase-2)*16);\n int k_offsets[4] = {0, 32, 64, 96};\n\n // B. Inner Dot Product Loop\n for (int l = 0; l < 16; ++l) {\n vuint8mf2_t v_rhs_data = __riscv_vle8_v_u8mf2(rhs_qs_ptr, vl);\n rhs_qs_ptr += 16;\n\n // Sub-block 0\n {\n vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(v_rhs_data, 3, vl);\n vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_0), vl);\n\n int8_t q8 = lhs_qs_ptr[base_k_phase + k_offsets[0] + l];\n v_isum = __riscv_vwmacc_vx_i32m2(v_isum, (int16_t)q8, v_w, vl);\n }\n // Sub-block 1\n {\n vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 2, vl), 3, vl);\n vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_1), vl);\n\n int8_t q8 = lhs_qs_ptr[base_k_phase + k_offsets[1] + l];\n v_isum = __riscv_vwmacc_vx_i32m2(v_isum, (int16_t)q8, v_w, vl);\n }\n // Sub-block 2\n {\n vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 4, vl), 3, vl);\n vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_2), vl);\n\n int8_t q8 = lhs_qs_ptr[base_k_phase + k_offsets[2] + l];\n v_isum = __riscv_vwmacc_vx_i32m2(v_isum, (int16_t)q8, v_w, vl);\n }\n // Sub-block 3\n {\n vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 6, vl), 3, vl);\n vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_3), vl);\n\n int8_t q8 = lhs_qs_ptr[base_k_phase + k_offsets[3] + l];\n v_isum = __riscv_vwmacc_vx_i32m2(v_isum, (int16_t)q8, v_w, vl);\n }\n }\n\n // correction\n int sb_base_abs = base_k_phase / 16;\n\n // Sub-block 0\n {\n int sb_idx = sb_base_abs + (k_offsets[0] / 16);\n int16_t bsum = lhs_current->bsums[sb_idx];\n vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_0);\n vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, bsum, vl);\n vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min_final, vl);\n v_sumf = __riscv_vfsub_vv_f32m2(v_sumf, vf_c, vl);\n }\n // Sub-block 1\n {\n int sb_idx = sb_base_abs + (k_offsets[1] / 16);\n int16_t bsum = lhs_current->bsums[sb_idx];\n vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_1);\n vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, bsum, vl);\n vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min_final, vl);\n v_sumf = __riscv_vfsub_vv_f32m2(v_sumf, vf_c, vl);\n }\n // Sub-block 2\n {\n int sb_idx = sb_base_abs + (k_offsets[2] / 16);\n int16_t bsum = lhs_current->bsums[sb_idx];\n vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_2);\n vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, bsum, vl);\n vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min_final, vl);\n v_sumf = __riscv_vfsub_vv_f32m2(v_sumf, vf_c, vl);\n }\n // Sub-block 3\n {\n int sb_idx = sb_base_abs + (k_offsets[3] / 16);\n int16_t bsum = lhs_current->bsums[sb_idx];\n vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_3);\n vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, bsum, vl);\n vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min_final, vl);\n v_sumf = __riscv_vfsub_vv_f32m2(v_sumf, vf_c, vl);\n }\n\n } // End Phase Loop\n\n // Apply global Scales\n vfloat16m1_t v_g_all_f16 = __riscv_vle16_v_f16m1((const _Float16*)rhs_current->d, vl);\n vfloat32m2_t v_g_all_base = __riscv_vfwcvt_f_f_v_f32m2(v_g_all_f16, vl);\n\n vfloat32m2_t v_g_all_final = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d, vl);\n vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum, vl);\n v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all_final, vl);\n v_sumf = __riscv_vfadd_vv_f32m2(v_sumf, v_sum, vl);\n\n } // End K-Block\n __riscv_vse32_v_f32m2(s + col_tile, v_sumf, vl);\n\n }\n}\n\nvoid ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined __riscv_v\n if (__riscv_vlenb() >= QK4_0) {\n const size_t vl = QK4_0;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);\n vfloat32m1_t sumf0 = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n vfloat32m1_t sumf1 = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n vfloat32m1_t sumf2 = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n vfloat32m1_t sumf3 = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n for (int l = 0; l < nb; l++) {\n const vint8m4_t rhs_raw_vec = __riscv_vle8_v_i8m4((const int8_t *)b_ptr[l].qs, vl * 4);\n const vint8m4_t rhs_vec_lo = __riscv_vsra_vx_i8m4(__riscv_vsll_vx_i8m4(rhs_raw_vec, 4, vl * 4), 4, vl * 4);\n const vint8m4_t rhs_vec_hi = __riscv_vsra_vx_i8m4(rhs_raw_vec, 4, vl * 4);\n const vint8m2_t rhs_vec_lo_0 = __riscv_vget_v_i8m4_i8m2(rhs_vec_lo, 0);\n const vint8m2_t rhs_vec_lo_1 = __riscv_vget_v_i8m4_i8m2(rhs_vec_lo, 1);\n const vint8m2_t rhs_vec_hi_0 = __riscv_vget_v_i8m4_i8m2(rhs_vec_hi, 0);\n const vint8m2_t rhs_vec_hi_1 = __riscv_vget_v_i8m4_i8m2(rhs_vec_hi, 1);\n\n // vector version needs Zvfhmin extension\n const float a_scales[4] = {\n GGML_CPU_FP16_TO_FP32(a_ptr[l].d[0]),\n GGML_CPU_FP16_TO_FP32(a_ptr[l].d[1]),\n GGML_CPU_FP16_TO_FP32(a_ptr[l].d[2]),\n GGML_CPU_FP16_TO_FP32(a_ptr[l].d[3])\n };\n const float b_scales[8] = {\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[4]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[5]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[6]),\n GGML_CPU_FP16_TO_FP32(b_ptr[l].d[7])\n };\n const vfloat32m1_t b_scales_vec = __riscv_vle32_v_f32m1(b_scales, vl / 4);\n\n const int64_t A0 = *(const int64_t *)&a_ptr[l].qs[0];\n const int64_t A4 = *(const int64_t *)&a_ptr[l].qs[32];\n const int64_t A8 = *(const int64_t *)&a_ptr[l].qs[64];\n const int64_t Ac = *(const int64_t *)&a_ptr[l].qs[96];\n __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment\n vint16m4_t sumi_l0;\n {\n const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A0, vl / 4));\n const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A4, vl / 4));\n const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A8, vl / 4));\n const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Ac, vl / 4));\n const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n sumi_l0 = sumi_hi_m;\n }\n\n {\n const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_l0));\n const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scales[0], vl / 4);\n sumf0 = __riscv_vfmacc_vv_f32m1(sumf0, tmp1, b_scales_vec, vl / 4);\n }\n\n const int64_t A1 = *(const int64_t *)&a_ptr[l].qs[8];\n const int64_t A5 = *(const int64_t *)&a_ptr[l].qs[40];\n const int64_t A9 = *(const int64_t *)&a_ptr[l].qs[72];\n const int64_t Ad = *(const int64_t *)&a_ptr[l].qs[104];\n __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment\n vint16m4_t sumi_l1;\n {\n const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A1, vl / 4));\n const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A5, vl / 4));\n const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A9, vl / 4));\n const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Ad, vl / 4));\n const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n sumi_l1 = sumi_hi_m;\n }\n\n {\n const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_l1));\n const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scales[1], vl / 4);\n sumf1 = __riscv_vfmacc_vv_f32m1(sumf1, tmp1, b_scales_vec, vl / 4);\n }\n\n const int64_t A2 = *(const int64_t *)&a_ptr[l].qs[16];\n const int64_t A6 = *(const int64_t *)&a_ptr[l].qs[48];\n const int64_t Aa = *(const int64_t *)&a_ptr[l].qs[80];\n const int64_t Ae = *(const int64_t *)&a_ptr[l].qs[112];\n __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment\n vint16m4_t sumi_l2;\n {\n const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A2, vl / 4));\n const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A6, vl / 4));\n const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Aa, vl / 4));\n const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Ae, vl / 4));\n const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n sumi_l2 = sumi_hi_m;\n }\n\n {\n const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_l2));\n const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scales[2], vl / 4);\n sumf2 = __riscv_vfmacc_vv_f32m1(sumf2, tmp1, b_scales_vec, vl / 4);\n }\n\n const int64_t A3 = *(const int64_t *)&a_ptr[l].qs[24];\n const int64_t A7 = *(const int64_t *)&a_ptr[l].qs[56];\n const int64_t Ab = *(const int64_t *)&a_ptr[l].qs[88];\n const int64_t Af = *(const int64_t *)&a_ptr[l].qs[120];\n __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment\n vint16m4_t sumi_l3;\n {\n const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A3, vl / 4));\n const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A7, vl / 4));\n const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Ab, vl / 4));\n const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Af, vl / 4));\n const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n sumi_l3 = sumi_hi_m;\n }\n\n {\n const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_l3));\n const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scales[3], vl / 4);\n sumf3 = __riscv_vfmacc_vv_f32m1(sumf3, tmp1, b_scales_vec, vl / 4);\n }\n }\n __riscv_vse32_v_f32m1(&s[(y * 4 + 0) * bs + x * ncols_interleaved], sumf0, vl / 4);\n __riscv_vse32_v_f32m1(&s[(y * 4 + 1) * bs + x * ncols_interleaved], sumf1, vl / 4);\n __riscv_vse32_v_f32m1(&s[(y * 4 + 2) * bs + x * ncols_interleaved], sumf2, vl / 4);\n __riscv_vse32_v_f32m1(&s[(y * 4 + 3) * bs + x * ncols_interleaved], sumf3, vl / 4);\n }\n }\n\n return;\n }\n\n#endif\n ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x16 * b_ptr = (const block_q4_0x16 *) vx + (x * nb);\n\n // 4x16 Accumulators\n vfloat32m2_t sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n for (int l = 0; l < nb; l++) {\n // 4x16 integer accumulators\n vint16m1_t sumi_0_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_1_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_2_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_3_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_0_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_1_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_2_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_3_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n // Accumulation loop.\n for (int i = 0; i < QK4_0 / 2; i++) {\n // Load `b_ptr`.\n const vint8mf2_t b_0_packed = __riscv_vle8_v_i8mf2((const int8_t *)&b_ptr[l].qs[i * 16], 16);\n const vint8mf2_t b_0_lo = __riscv_vsra_vx_i8mf2(__riscv_vsll_vx_i8mf2(b_0_packed, 4, 16), 4, 16);\n const vint8mf2_t b_0_hi = __riscv_vsra_vx_i8mf2(b_0_packed, 4, 16);\n\n sumi_0_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_0_lo_16, a_ptr[l].qs[i * 4], b_0_lo, 16);\n sumi_1_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_1_lo_16, a_ptr[l].qs[i * 4 + 1], b_0_lo, 16);\n sumi_2_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_2_lo_16, a_ptr[l].qs[i * 4 + 2], b_0_lo, 16);\n sumi_3_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_3_lo_16, a_ptr[l].qs[i * 4 + 3], b_0_lo, 16);\n\n sumi_0_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_0_hi_16, a_ptr[l].qs[64 + i * 4], b_0_hi, 16);\n sumi_1_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_1_hi_16, a_ptr[l].qs[64 + i * 4 + 1], b_0_hi, 16);\n sumi_2_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_2_hi_16, a_ptr[l].qs[64 + i * 4 + 2], b_0_hi, 16);\n sumi_3_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_3_hi_16, a_ptr[l].qs[64 + i * 4 + 3], b_0_hi, 16);\n }\n\n // Do the final accumulation in i32 to prevent overflow.\n const vint32m2_t sumi_0 = __riscv_vwadd_vv_i32m2(sumi_0_lo_16, sumi_0_hi_16, 16);\n const vint32m2_t sumi_1 = __riscv_vwadd_vv_i32m2(sumi_1_lo_16, sumi_1_hi_16, 16);\n const vint32m2_t sumi_2 = __riscv_vwadd_vv_i32m2(sumi_2_lo_16, sumi_2_hi_16, 16);\n const vint32m2_t sumi_3 = __riscv_vwadd_vv_i32m2(sumi_3_lo_16, sumi_3_hi_16, 16);\n\n const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[0], 16);\n const vfloat32m2_t d_1 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[1], 16);\n const vfloat32m2_t d_2 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[2], 16);\n const vfloat32m2_t d_3 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[3], 16);\n\n sumf_0 = __riscv_vfmacc_vv_f32m2(sumf_0, __riscv_vfcvt_f_x_v_f32m2(sumi_0, 16), d_0, 16);\n sumf_1 = __riscv_vfmacc_vv_f32m2(sumf_1, __riscv_vfcvt_f_x_v_f32m2(sumi_1, 16), d_1, 16);\n sumf_2 = __riscv_vfmacc_vv_f32m2(sumf_2, __riscv_vfcvt_f_x_v_f32m2(sumi_2, 16), d_2, 16);\n sumf_3 = __riscv_vfmacc_vv_f32m2(sumf_3, __riscv_vfcvt_f_x_v_f32m2(sumi_3, 16), d_3, 16);\n }\n\n __riscv_vse32_v_f32m2(s + (y * 4 + 0) * bs + x * 16, sumf_0, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 1) * bs + x * 16, sumf_1, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 2) * bs + x * 16, sumf_2, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 3) * bs + x * 16, sumf_3, 16);\n }\n }\n return;\n#endif\n ggml_gemm_q4_0_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx16 * b_ptr = (const block_q4_Kx16 *) vx + (x * nb);\n\n // 4x16 Accumulators\n vfloat32m2_t sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n for (int l = 0; l < nb; l++) {\n vint32m2_t sumi_0 = __riscv_vmv_v_x_i32m2(0, 16);\n vint32m2_t sumi_1 = __riscv_vmv_v_x_i32m2(0, 16);\n vint32m2_t sumi_2 = __riscv_vmv_v_x_i32m2(0, 16);\n vint32m2_t sumi_3 = __riscv_vmv_v_x_i32m2(0, 16);\n\n // Load `dmin`.\n const vfloat32m2_t dmins = __riscv_vfwcvt_f_f_v_f32m2(__riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].dmin, 16), 16);\n\n // We process 4 sub-blocks at once.\n for (int j = 0; j < QK_K / 128; j++) {\n // Extract the scales and the mins.\n //\n // Low bits.\n vuint8m2_t scales_mins_lo = __riscv_vle8_v_u8m2(&b_ptr[l].scales[j * 64], 64);\n vuint8m2_t scales_lo = __riscv_vand_vx_u8m2(scales_mins_lo, 0x0F, 64);\n vuint8m2_t mins_lo = __riscv_vsrl_vx_u8m2(scales_mins_lo, 4, 64);\n\n // High bits.\n vuint8m2_t scales_mins_hi = __riscv_vle8_v_u8m2(&b_ptr[l].scales[128], 64);\n vuint8m2_t scales_hi;\n vuint8m2_t mins_hi;\n if (!j) {\n scales_hi = __riscv_vsll_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0x03, 64), 4, 64);\n mins_hi = __riscv_vsll_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0x0C, 64), 2, 64);\n } else {\n scales_hi = __riscv_vand_vx_u8m2(scales_mins_hi, 0x30, 64);\n mins_hi = __riscv_vsrl_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0xC0, 64), 2, 64);\n }\n vuint16m4_t scales = __riscv_vzext_vf2_u16m4(__riscv_vor_vv_u8m2(scales_hi, scales_lo, 64), 64);\n vint16m4_t mins = __riscv_vreinterpret_v_u16m4_i16m4(__riscv_vzext_vf2_u16m4(__riscv_vor_vv_u8m2(mins_hi, mins_lo, 64), 64));\n\n // Reduce the mins and multiply with `dmin`.\n //\n // Correct in `sumf`.\n vint32m2_t bsums_0 = __riscv_vmv_v_x_i32m2(0, 16);\n vint32m2_t bsums_1 = __riscv_vmv_v_x_i32m2(0, 16);\n vint32m2_t bsums_2 = __riscv_vmv_v_x_i32m2(0, 16);\n vint32m2_t bsums_3 = __riscv_vmv_v_x_i32m2(0, 16);\n\n bsums_0 = __riscv_vwmacc_vx_i32m2(bsums_0,\n a_ptr[l].bsums[j * 32] + a_ptr[l].bsums[j * 32 + 4],\n __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n bsums_1 = __riscv_vwmacc_vx_i32m2(bsums_1,\n a_ptr[l].bsums[j * 32 + 1] + a_ptr[l].bsums[j * 32 + 5],\n __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n bsums_2 = __riscv_vwmacc_vx_i32m2(bsums_2,\n a_ptr[l].bsums[j * 32 + 2] + a_ptr[l].bsums[j * 32 + 6],\n __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n bsums_3 = __riscv_vwmacc_vx_i32m2(bsums_3,\n a_ptr[l].bsums[j * 32 + 3] + a_ptr[l].bsums[j * 32 + 7],\n __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n bsums_0 = __riscv_vwmacc_vx_i32m2(bsums_0,\n a_ptr[l].bsums[j * 32 + 8] + a_ptr[l].bsums[j * 32 + 8 + 4],\n __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n bsums_1 = __riscv_vwmacc_vx_i32m2(bsums_1,\n a_ptr[l].bsums[j * 32 + 8 + 1] + a_ptr[l].bsums[j * 32 + 8 + 5],\n __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n bsums_2 = __riscv_vwmacc_vx_i32m2(bsums_2,\n a_ptr[l].bsums[j * 32 + 8 + 2] + a_ptr[l].bsums[j * 32 + 8 + 6],\n __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n bsums_3 = __riscv_vwmacc_vx_i32m2(bsums_3,\n a_ptr[l].bsums[j * 32 + 8 + 3] + a_ptr[l].bsums[j * 32 + 8 + 7],\n __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n bsums_0 = __riscv_vwmacc_vx_i32m2(bsums_0,\n a_ptr[l].bsums[j * 32 + 16] + a_ptr[l].bsums[j * 32 + 16 + 4],\n __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n bsums_1 = __riscv_vwmacc_vx_i32m2(bsums_1,\n a_ptr[l].bsums[j * 32 + 16 + 1] + a_ptr[l].bsums[j * 32 + 16 + 5],\n __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n bsums_2 = __riscv_vwmacc_vx_i32m2(bsums_2,\n a_ptr[l].bsums[j * 32 + 16 + 2] + a_ptr[l].bsums[j * 32 + 16 + 6],\n __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n bsums_3 = __riscv_vwmacc_vx_i32m2(bsums_3,\n a_ptr[l].bsums[j * 32 + 16 + 3] + a_ptr[l].bsums[j * 32 + 16 + 7],\n __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n bsums_0 = __riscv_vwmacc_vx_i32m2(bsums_0,\n a_ptr[l].bsums[j * 32 + 24 + 0] + a_ptr[l].bsums[j * 32 + 24 + 4],\n __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n bsums_1 = __riscv_vwmacc_vx_i32m2(bsums_1,\n a_ptr[l].bsums[j * 32 + 24 + 1] + a_ptr[l].bsums[j * 32 + 24 + 5],\n __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n bsums_2 = __riscv_vwmacc_vx_i32m2(bsums_2,\n a_ptr[l].bsums[j * 32 + 24 + 2] + a_ptr[l].bsums[j * 32 + 24 + 6],\n __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n bsums_3 = __riscv_vwmacc_vx_i32m2(bsums_3,\n a_ptr[l].bsums[j * 32 + 24 + 3] + a_ptr[l].bsums[j * 32 + 24 + 7],\n __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n\n const vfloat32m2_t dmins_d_0 = __riscv_vfmul_vf_f32m2(dmins, a_ptr[l].d[0], 16);\n const vfloat32m2_t dmins_d_1 = __riscv_vfmul_vf_f32m2(dmins, a_ptr[l].d[1], 16);\n const vfloat32m2_t dmins_d_2 = __riscv_vfmul_vf_f32m2(dmins, a_ptr[l].d[2], 16);\n const vfloat32m2_t dmins_d_3 = __riscv_vfmul_vf_f32m2(dmins, a_ptr[l].d[3], 16);\n\n sumf_0 = __riscv_vfsub_vv_f32m2(sumf_0, __riscv_vfmul_vv_f32m2(dmins_d_0, __riscv_vfcvt_f_x_v_f32m2(bsums_0, 16), 16), 16);\n sumf_1 = __riscv_vfsub_vv_f32m2(sumf_1, __riscv_vfmul_vv_f32m2(dmins_d_1, __riscv_vfcvt_f_x_v_f32m2(bsums_1, 16), 16), 16);\n sumf_2 = __riscv_vfsub_vv_f32m2(sumf_2, __riscv_vfmul_vv_f32m2(dmins_d_2, __riscv_vfcvt_f_x_v_f32m2(bsums_2, 16), 16), 16);\n sumf_3 = __riscv_vfsub_vv_f32m2(sumf_3, __riscv_vfmul_vv_f32m2(dmins_d_3, __riscv_vfcvt_f_x_v_f32m2(bsums_3, 16), 16), 16);\n\n\n // Accumulation for 2 sub-blocks.\n //\n // This might overflow, so we accumulate in two steps.\n //\n // Recheck.\n for (int k = 0; k < 2; k++) {\n // 4x16 integer accumulators\n vint16m1_t sumi_0_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_1_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_2_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_3_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_0_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_1_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_2_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_3_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n for (int i = k * 16; i < k * 16 + QK4_0 / 2; i++) {\n // Load `b_ptr`.\n const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2(&b_ptr[l].qs[j * 1024 + i * 16], 16);\n const vint8mf2_t b_s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(b_0_packed, 0xF, 16));\n const vint8mf2_t b_s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16));\n\n sumi_0_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_0_s_0_16, a_ptr[l].qs[j * 512 + i * 4], b_s_0, 16);\n sumi_1_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_1_s_0_16, a_ptr[l].qs[j * 512 + i * 4 + 1], b_s_0, 16);\n sumi_2_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_2_s_0_16, a_ptr[l].qs[j * 512 + i * 4 + 2], b_s_0, 16);\n sumi_3_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_3_s_0_16, a_ptr[l].qs[j * 512 + i * 4 + 3], b_s_0, 16);\n\n sumi_0_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_0_s_1_16, a_ptr[l].qs[j * 512 + 128 + i * 4], b_s_1, 16);\n sumi_1_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_1_s_1_16, a_ptr[l].qs[j * 512 + 128 + i * 4 + 1], b_s_1, 16);\n sumi_2_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_2_s_1_16, a_ptr[l].qs[j * 512 + 128 + i * 4 + 2], b_s_1, 16);\n sumi_3_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_3_s_1_16, a_ptr[l].qs[j * 512 + 128 + i * 4 + 3], b_s_1, 16);\n }\n\n sumi_0 = __riscv_vwmacc_vv_i32m2(sumi_0,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n sumi_0_s_0_16, 16);\n sumi_0 = __riscv_vwmacc_vv_i32m2(sumi_0,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n sumi_0_s_1_16, 16);\n sumi_1 = __riscv_vwmacc_vv_i32m2(sumi_1,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n sumi_1_s_0_16, 16);\n sumi_1 = __riscv_vwmacc_vv_i32m2(sumi_1,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n sumi_1_s_1_16, 16);\n sumi_2 = __riscv_vwmacc_vv_i32m2(sumi_2,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n sumi_2_s_0_16, 16);\n sumi_2 = __riscv_vwmacc_vv_i32m2(sumi_2,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n sumi_2_s_1_16, 16);\n sumi_3 = __riscv_vwmacc_vv_i32m2(sumi_3,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n sumi_3_s_0_16, 16);\n sumi_3 = __riscv_vwmacc_vv_i32m2(sumi_3,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n sumi_3_s_1_16, 16);\n }\n // Accumulation for 2 sub-blocks.\n //\n // This might overflow, so we accumulate in two steps.\n //\n // Recheck.\n for (int k = 0; k < 2; k++) {\n // 4x16 integer accumulators\n vint16m1_t sumi_0_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_1_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_2_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_3_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_0_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_1_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_2_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n vint16m1_t sumi_3_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n for (int i = k * 16; i < k * 16 + QK4_0 / 2; i++) {\n // Load `b_ptr`.\n const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2(&b_ptr[l].qs[j * 1024 + 512 + i * 16], 16);\n const vint8mf2_t b_s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(b_0_packed, 0xF, 16));\n const vint8mf2_t b_s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16));\n\n sumi_0_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_0_s_0_16, a_ptr[l].qs[j * 512 + 256 + i * 4], b_s_0, 16);\n sumi_1_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_1_s_0_16, a_ptr[l].qs[j * 512 + 256 + i * 4 + 1], b_s_0, 16);\n sumi_2_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_2_s_0_16, a_ptr[l].qs[j * 512 + 256 + i * 4 + 2], b_s_0, 16);\n sumi_3_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_3_s_0_16, a_ptr[l].qs[j * 512 + 256 + i * 4 + 3], b_s_0, 16);\n\n sumi_0_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_0_s_1_16, a_ptr[l].qs[j * 512 + 384 + i * 4], b_s_1, 16);\n sumi_1_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_1_s_1_16, a_ptr[l].qs[j * 512 + 384 + i * 4 + 1], b_s_1, 16);\n sumi_2_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_2_s_1_16, a_ptr[l].qs[j * 512 + 384 + i * 4 + 2], b_s_1, 16);\n sumi_3_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_3_s_1_16, a_ptr[l].qs[j * 512 + 384 + i * 4 + 3], b_s_1, 16);\n }\n\n sumi_0 = __riscv_vwmacc_vv_i32m2(sumi_0,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n sumi_0_s_0_16, 16);\n sumi_0 = __riscv_vwmacc_vv_i32m2(sumi_0,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n sumi_0_s_1_16, 16);\n sumi_1 = __riscv_vwmacc_vv_i32m2(sumi_1,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n sumi_1_s_0_16, 16);\n sumi_1 = __riscv_vwmacc_vv_i32m2(sumi_1,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n sumi_1_s_1_16, 16);\n sumi_2 = __riscv_vwmacc_vv_i32m2(sumi_2,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n sumi_2_s_0_16, 16);\n sumi_2 = __riscv_vwmacc_vv_i32m2(sumi_2,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n sumi_2_s_1_16, 16);\n sumi_3 = __riscv_vwmacc_vv_i32m2(sumi_3,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n sumi_3_s_0_16, 16);\n sumi_3 = __riscv_vwmacc_vv_i32m2(sumi_3,\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n sumi_3_s_1_16, 16);\n }\n }\n\n const vfloat32m2_t b_d = __riscv_vfwcvt_f_f_v_f32m2(__riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16), 16);\n const vfloat32m2_t d_0 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d[0], 16);\n const vfloat32m2_t d_1 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d[1], 16);\n const vfloat32m2_t d_2 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d[2], 16);\n const vfloat32m2_t d_3 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d[3], 16);\n\n sumf_0 = __riscv_vfmacc_vv_f32m2(sumf_0, __riscv_vfcvt_f_x_v_f32m2(sumi_0, 16), d_0, 16);\n sumf_1 = __riscv_vfmacc_vv_f32m2(sumf_1, __riscv_vfcvt_f_x_v_f32m2(sumi_1, 16), d_1, 16);\n sumf_2 = __riscv_vfmacc_vv_f32m2(sumf_2, __riscv_vfcvt_f_x_v_f32m2(sumi_2, 16), d_2, 16);\n sumf_3 = __riscv_vfmacc_vv_f32m2(sumf_3, __riscv_vfcvt_f_x_v_f32m2(sumi_3, 16), d_3, 16);\n }\n\n __riscv_vse32_v_f32m2(s + (y * 4 + 0) * bs + x * 16, sumf_0, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 1) * bs + x * 16, sumf_1, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 2) * bs + x * 16, sumf_2, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 3) * bs + x * 16, sumf_3, 16);\n }\n }\n return;\n#endif\n ggml_gemm_q4_K_16x1_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_iq4_nl_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n const vint8mf2_t values = __riscv_vle8_v_i8mf2(kvalues_iq4nl, 16);\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx16 * b_ptr = (const block_iq4_nlx16 *) vx + (x * nb);\n\n // 4x16 Accumulators\n vfloat32m2_t sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n for (int l = 0; l < nb; l++) {\n // 4x16 integer accumulators\n vint32m2_t sumi_0 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n vint32m2_t sumi_1 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n vint32m2_t sumi_2 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n vint32m2_t sumi_3 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n\n // Accumulation loop.\n for (int i = 0; i < QK4_NL / 2; i++) {\n // Load `b_ptr`.\n const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2((const uint8_t *)&b_ptr[l].qs[i * 16], 16);\n const vint8mf2_t b_0_lo = __riscv_vrgather_vv_i8mf2(values, __riscv_vand_vx_u8mf2(b_0_packed, 0xf, 16), 16);\n const vint8mf2_t b_0_hi = __riscv_vrgather_vv_i8mf2(values, __riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16), 16);\n // const vint16m1_t b_0_lo_16 = __riscv_vwcvt_x_x_v_i16m1(b_0_lo, 16);\n // const vint16m1_t b_0_hi_16 = __riscv_vwcvt_x_x_v_i16m1(b_0_hi, 16);\n\n const vint16m1_t sumi_0_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i * 4], 16);\n const vint16m1_t sumi_1_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i * 4 + 1], 16);\n const vint16m1_t sumi_2_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i * 4 + 2], 16);\n const vint16m1_t sumi_3_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i * 4 + 3], 16);\n\n const vint16m1_t sumi_0_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[64 + i * 4], 16);\n const vint16m1_t sumi_1_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[64 + i * 4 + 1], 16);\n const vint16m1_t sumi_2_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[64 + i * 4 + 2], 16);\n const vint16m1_t sumi_3_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[64 + i * 4 + 3], 16);\n\n sumi_0 = __riscv_vadd_vv_i32m2(sumi_0, __riscv_vwadd_vv_i32m2(sumi_0_lo, sumi_0_hi, 16), 16);\n sumi_1 = __riscv_vadd_vv_i32m2(sumi_1, __riscv_vwadd_vv_i32m2(sumi_1_lo, sumi_1_hi, 16), 16);\n sumi_2 = __riscv_vadd_vv_i32m2(sumi_2, __riscv_vwadd_vv_i32m2(sumi_2_lo, sumi_2_hi, 16), 16);\n sumi_3 = __riscv_vadd_vv_i32m2(sumi_3, __riscv_vwadd_vv_i32m2(sumi_3_lo, sumi_3_hi, 16), 16);\n }\n\n const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[0], 16);\n const vfloat32m2_t d_1 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[1], 16);\n const vfloat32m2_t d_2 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[2], 16);\n const vfloat32m2_t d_3 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[3], 16);\n\n sumf_0 = __riscv_vfmacc_vv_f32m2(sumf_0, __riscv_vfcvt_f_x_v_f32m2(sumi_0, 16), d_0, 16);\n sumf_1 = __riscv_vfmacc_vv_f32m2(sumf_1, __riscv_vfcvt_f_x_v_f32m2(sumi_1, 16), d_1, 16);\n sumf_2 = __riscv_vfmacc_vv_f32m2(sumf_2, __riscv_vfcvt_f_x_v_f32m2(sumi_2, 16), d_2, 16);\n sumf_3 = __riscv_vfmacc_vv_f32m2(sumf_3, __riscv_vfcvt_f_x_v_f32m2(sumi_3, 16), d_3, 16);\n }\n\n __riscv_vse32_v_f32m2(s + (y * 4 + 0) * bs + x * 16, sumf_0, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 1) * bs + x * 16, sumf_1, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 2) * bs + x * 16, sumf_2, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 3) * bs + x * 16, sumf_3, 16);\n }\n }\n return;\n#endif\n ggml_gemm_iq4_nl_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q8_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q8_0x16 * b_ptr = (const block_q8_0x16 *) vx + (x * nb);\n\n // 4x16 Accumulators\n vfloat32m2_t sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n vfloat32m2_t sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n for (int l = 0; l < nb; l++) {\n // 4x16 Integer Accumulators\n vint32m2_t sumi_0 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n vint32m2_t sumi_1 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n vint32m2_t sumi_2 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n vint32m2_t sumi_3 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n\n // Accumulation loop.\n for (int i = 0; i < QK8_0; i++) {\n // Load `b_ptr`.\n const vint8mf2_t b_0 = __riscv_vle8_v_i8mf2((const int8_t *)&b_ptr[l].qs[i * 16], 16);\n // const vint16m1_t b_0_16 = __riscv_vwcvt_x_x_v_i16m1(b_0, 16);\n\n sumi_0 = __riscv_vwadd_wv_i32m2(sumi_0, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i * 4 + 0], 16), 16);\n sumi_1 = __riscv_vwadd_wv_i32m2(sumi_1, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i * 4 + 1], 16), 16);\n sumi_2 = __riscv_vwadd_wv_i32m2(sumi_2, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i * 4 + 2], 16), 16);\n sumi_3 = __riscv_vwadd_wv_i32m2(sumi_3, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i * 4 + 3], 16), 16);\n }\n\n const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[0], 16);\n const vfloat32m2_t d_1 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[1], 16);\n const vfloat32m2_t d_2 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[2], 16);\n const vfloat32m2_t d_3 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[3], 16);\n\n sumf_0 = __riscv_vfmacc_vv_f32m2(sumf_0, __riscv_vfcvt_f_x_v_f32m2(sumi_0, 16), d_0, 16);\n sumf_1 = __riscv_vfmacc_vv_f32m2(sumf_1, __riscv_vfcvt_f_x_v_f32m2(sumi_1, 16), d_1, 16);\n sumf_2 = __riscv_vfmacc_vv_f32m2(sumf_2, __riscv_vfcvt_f_x_v_f32m2(sumi_2, 16), d_2, 16);\n sumf_3 = __riscv_vfmacc_vv_f32m2(sumf_3, __riscv_vfcvt_f_x_v_f32m2(sumi_3, 16), d_3, 16);\n }\n\n __riscv_vse32_v_f32m2(s + (y * 4 + 0) * bs + x * 16, sumf_0, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 1) * bs + x * 16, sumf_1, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 2) * bs + x * 16, sumf_2, 16);\n __riscv_vse32_v_f32m2(s + (y * 4 + 3) * bs + x * 16, sumf_3, 16);\n }\n }\n return;\n#endif\n ggml_gemm_q8_0_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n assert(n % QK_K == 0);\n const int num_k_blocks = n / QK_K;\n const int N_ROWS_TILE = 4;\n const int N_COLS_TILE = 16;\n assert(nr % N_ROWS_TILE == 0);\n assert(nc % N_COLS_TILE == 0);\n\n const size_t vl = __riscv_vsetvl_e32m2(N_COLS_TILE);\n // --- Tiling Loops ---\n#pragma GCC unroll 1\n for (int row_tile = 0; row_tile < nr; row_tile += N_ROWS_TILE) {\n#pragma GCC unroll 1\n for (int col_tile = 0; col_tile < nc; col_tile += N_COLS_TILE) {\n // Base Pointers\n const block_q8_Kx4* lhs_base_ptr = (const block_q8_Kx4*)vy + (row_tile / N_ROWS_TILE) * num_k_blocks;\n const block_q2_Kx16* rhs_base_ptr = (const block_q2_Kx16*)vx + (col_tile / N_COLS_TILE) * num_k_blocks;\n\n // Persistent Float Accumulators\n vfloat32m2_t v_sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n vfloat32m2_t v_sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n vfloat32m2_t v_sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n vfloat32m2_t v_sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n\n // --- Super-Block Loop (K=0..255) ---\n#pragma GCC unroll 1\n for (int k_block = 0; k_block < num_k_blocks; ++k_block) {\n const block_q8_Kx4* lhs_current = &lhs_base_ptr[k_block];\n const block_q2_Kx16* rhs_current = &rhs_base_ptr[k_block];\n\n // 1. Load Global Min Scales (Keep as F16/LMUL=1 to save registers)\n vfloat16m1_t v_g_min_f16 = __riscv_vle16_v_f16m1((const _Float16*)rhs_current->dmin, vl);\n vfloat32m2_t v_g_min_base = __riscv_vfwcvt_f_f_v_f32m2(v_g_min_f16, vl);\n\n // 2. Initialize Integer Accumulators\n vint32m2_t v_isum_0 = __riscv_vmv_v_x_i32m2(0, vl);\n vint32m2_t v_isum_1 = __riscv_vmv_v_x_i32m2(0, vl);\n vint32m2_t v_isum_2 = __riscv_vmv_v_x_i32m2(0, vl);\n vint32m2_t v_isum_3 = __riscv_vmv_v_x_i32m2(0, vl);\n\n const uint8_t* rhs_qs_ptr = rhs_current->qs;\n const uint8_t* rhs_sc_ptr = rhs_current->scales;\n const int8_t* lhs_qs_ptr = lhs_current->qs;\n\n // --- Phase Loop (4 phases x 64 elements) ---\n#pragma GCC unroll 1\n for (int phase = 0; phase < 4; ++phase) {\n\n // A. Load Scales/Mins for the 4 interleaved sub-blocks\n vuint16m1_t v_d_sb_0, v_d_sb_1, v_d_sb_2, v_d_sb_3;\n vuint16m1_t v_m_sb_0, v_m_sb_1, v_m_sb_2, v_m_sb_3;\n\n // Unrolled Load Logic\n {\n vuint8mf2_t v_raw;\n // Sub-block 0\n v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 0, vl);\n v_d_sb_0 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n v_m_sb_0 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n // Sub-block 1\n v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 16, vl);\n v_d_sb_1 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n v_m_sb_1 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n // Sub-block 2\n v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 32, vl);\n v_d_sb_2 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n v_m_sb_2 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n // Sub-block 3\n v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 48, vl);\n v_d_sb_3 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n v_m_sb_3 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n rhs_sc_ptr += 64;\n }\n\n int base_k_phase = (phase < 2) ? (phase * 16) : (128 + (phase-2)*16);\n int k_offsets[4] = {0, 32, 64, 96};\n\n // B. Inner Dot Product Loop\n#pragma GCC unroll 1\n for (int l = 0; l < 16; ++l) {\n vuint8mf2_t v_rhs_data = __riscv_vle8_v_u8mf2(rhs_qs_ptr, vl);\n rhs_qs_ptr += 16;\n\n // Unroll over 4 sub-blocks (0, 1, 2, 3 relative to phase)\n\n // --- Sub-block 0 ---\n {\n vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(v_rhs_data, 3, vl);\n vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_0), vl);\n\n const int8_t* q8 = &lhs_qs_ptr[(base_k_phase + k_offsets[0] + l) * 4];\n v_isum_0 = __riscv_vwmacc_vx_i32m2(v_isum_0, (int16_t)q8[0], v_w, vl);\n v_isum_1 = __riscv_vwmacc_vx_i32m2(v_isum_1, (int16_t)q8[1], v_w, vl);\n v_isum_2 = __riscv_vwmacc_vx_i32m2(v_isum_2, (int16_t)q8[2], v_w, vl);\n v_isum_3 = __riscv_vwmacc_vx_i32m2(v_isum_3, (int16_t)q8[3], v_w, vl);\n }\n // --- Sub-block 1 ---\n {\n vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 2, vl), 3, vl);\n vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_1), vl);\n\n const int8_t* q8 = &lhs_qs_ptr[(base_k_phase + k_offsets[1] + l) * 4];\n v_isum_0 = __riscv_vwmacc_vx_i32m2(v_isum_0, (int16_t)q8[0], v_w, vl);\n v_isum_1 = __riscv_vwmacc_vx_i32m2(v_isum_1, (int16_t)q8[1], v_w, vl);\n v_isum_2 = __riscv_vwmacc_vx_i32m2(v_isum_2, (int16_t)q8[2], v_w, vl);\n v_isum_3 = __riscv_vwmacc_vx_i32m2(v_isum_3, (int16_t)q8[3], v_w, vl);\n }\n // --- Sub-block 2 ---\n {\n vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 4, vl), 3, vl);\n vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_2), vl);\n\n const int8_t* q8 = &lhs_qs_ptr[(base_k_phase + k_offsets[2] + l) * 4];\n v_isum_0 = __riscv_vwmacc_vx_i32m2(v_isum_0, (int16_t)q8[0], v_w, vl);\n v_isum_1 = __riscv_vwmacc_vx_i32m2(v_isum_1, (int16_t)q8[1], v_w, vl);\n v_isum_2 = __riscv_vwmacc_vx_i32m2(v_isum_2, (int16_t)q8[2], v_w, vl);\n v_isum_3 = __riscv_vwmacc_vx_i32m2(v_isum_3, (int16_t)q8[3], v_w, vl);\n }\n // --- Sub-block 3 ---\n {\n vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 6, vl), 3, vl);\n vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_3), vl);\n\n const int8_t* q8 = &lhs_qs_ptr[(base_k_phase + k_offsets[3] + l) * 4];\n v_isum_0 = __riscv_vwmacc_vx_i32m2(v_isum_0, (int16_t)q8[0], v_w, vl);\n v_isum_1 = __riscv_vwmacc_vx_i32m2(v_isum_1, (int16_t)q8[1], v_w, vl);\n v_isum_2 = __riscv_vwmacc_vx_i32m2(v_isum_2, (int16_t)q8[2], v_w, vl);\n v_isum_3 = __riscv_vwmacc_vx_i32m2(v_isum_3, (int16_t)q8[3], v_w, vl);\n }\n }\n\n // C CORRECTION\n int sb_base_abs = base_k_phase / 16;\n\n // --- Correction Sub-block 0 ---\n {\n int sb_abs = sb_base_abs + (k_offsets[0] / 16);\n vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_0);\n\n // Row 0\n vfloat32m2_t v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[0], vl);\n vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 0], vl);\n vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_0 = __riscv_vfsub_vv_f32m2(v_sumf_0, vf_c, vl);\n\n // Row 1\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[1], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 1], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_1 = __riscv_vfsub_vv_f32m2(v_sumf_1, vf_c, vl);\n\n // Row 2\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[2], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 2], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_2 = __riscv_vfsub_vv_f32m2(v_sumf_2, vf_c, vl);\n\n // Row 3\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[3], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 3], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_3 = __riscv_vfsub_vv_f32m2(v_sumf_3, vf_c, vl);\n }\n\n // --- Correction Sub-block 1 ---\n {\n int sb_abs = sb_base_abs + (k_offsets[1] / 16);\n vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_1);\n\n vfloat32m2_t v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[0], vl);\n vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 0], vl);\n vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_0 = __riscv_vfsub_vv_f32m2(v_sumf_0, vf_c, vl);\n\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[1], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 1], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_1 = __riscv_vfsub_vv_f32m2(v_sumf_1, vf_c, vl);\n\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[2], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 2], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_2 = __riscv_vfsub_vv_f32m2(v_sumf_2, vf_c, vl);\n\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[3], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 3], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_3 = __riscv_vfsub_vv_f32m2(v_sumf_3, vf_c, vl);\n }\n\n // --- Correction Sub-block 2 ---\n {\n int sb_abs = sb_base_abs + (k_offsets[2] / 16);\n vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_2);\n\n vfloat32m2_t v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[0], vl);\n vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 0], vl);\n vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_0 = __riscv_vfsub_vv_f32m2(v_sumf_0, vf_c, vl);\n\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[1], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 1], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_1 = __riscv_vfsub_vv_f32m2(v_sumf_1, vf_c, vl);\n\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[2], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 2], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_2 = __riscv_vfsub_vv_f32m2(v_sumf_2, vf_c, vl);\n\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[3], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 3], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_3 = __riscv_vfsub_vv_f32m2(v_sumf_3, vf_c, vl);\n }\n\n // --- Correction Sub-block 3 ---\n {\n int sb_abs = sb_base_abs + (k_offsets[3] / 16);\n vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_3);\n\n vfloat32m2_t v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[0], vl);\n vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 0], vl);\n vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_0 = __riscv_vfsub_vv_f32m2(v_sumf_0, vf_c, vl);\n\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[1], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 1], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_1 = __riscv_vfsub_vv_f32m2(v_sumf_1, vf_c, vl);\n\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[2], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 2], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_2 = __riscv_vfsub_vv_f32m2(v_sumf_2, vf_c, vl);\n\n v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[3], vl);\n v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 3], vl);\n vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n v_sumf_3 = __riscv_vfsub_vv_f32m2(v_sumf_3, vf_c, vl);\n }\n\n } // End Phase Loop\n\n // --- Apply Main Scales ---\n vfloat16m1_t v_g_all_f16 = __riscv_vle16_v_f16m1((const _Float16*)rhs_current->d, vl);\n vfloat32m2_t v_g_all_base = __riscv_vfwcvt_f_f_v_f32m2(v_g_all_f16, vl);\n\n {\n vfloat32m2_t v_g_all = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d[0], vl);\n vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum_0, vl);\n v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all, vl);\n v_sumf_0 = __riscv_vfadd_vv_f32m2(v_sumf_0, v_sum, vl);\n }\n // Row 1\n {\n vfloat32m2_t v_g_all = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d[1], vl);\n vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum_1, vl);\n v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all, vl);\n v_sumf_1 = __riscv_vfadd_vv_f32m2(v_sumf_1, v_sum, vl);\n }\n // Row 2\n {\n vfloat32m2_t v_g_all = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d[2], vl);\n vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum_2, vl);\n v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all, vl);\n v_sumf_2 = __riscv_vfadd_vv_f32m2(v_sumf_2, v_sum, vl);\n }\n // Row 3\n {\n vfloat32m2_t v_g_all = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d[3], vl);\n vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum_3, vl);\n v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all, vl);\n v_sumf_3 = __riscv_vfadd_vv_f32m2(v_sumf_3, v_sum, vl);\n }\n\n } // End K-Block\n\n __riscv_vse32_v_f32m2(s + (row_tile + 0) * bs + col_tile, v_sumf_0, vl);\n __riscv_vse32_v_f32m2(s + (row_tile + 1) * bs + col_tile, v_sumf_1, vl);\n __riscv_vse32_v_f32m2(s + (row_tile + 2) * bs + col_tile, v_sumf_2, vl);\n __riscv_vse32_v_f32m2(s + (row_tile + 3) * bs + col_tile, v_sumf_3, vl);\n }\n }\n}\n" }, { "path": "src/ggml-cpu/arch/s390/cpu-feats.cpp", "content": "#include \"ggml-backend-impl.h\"\n\n#if defined(__s390x__)\n#include \n\n// find hwcap bits in asm/elf.h\n#ifndef HWCAP_VXRS_EXT2\n#define HWCAP_VXRS_EXT2 (1 << 15)\n#endif\n\n#ifndef HWCAP_NNPA\n#define HWCAP_NNPA (1 << 20)\n#endif\n\nstruct s390x_features {\n bool has_vxe2 = false;\n bool has_nnpa = false;\n\n s390x_features() {\n uint32_t hwcap = getauxval(AT_HWCAP);\n // NOTE: use hwcap2 with DFLT for z17 and later\n // uint32_t hwcap2 = getauxval(AT_HWCAP2);\n\n has_vxe2 = !!(hwcap & HWCAP_VXRS_EXT2);\n has_nnpa = !!(hwcap & HWCAP_NNPA);\n }\n};\n\nstatic int ggml_backend_cpu_s390x_score() {\n int score = 1;\n s390x_features sf;\n\n// IBM z15 / LinuxONE 3\n#ifdef GGML_USE_VXE2\n if (!sf.has_vxe2) { return 0; }\n score += 1 << 1;\n#endif\n\n// IBM z16 / LinuxONE 4 and z17 / LinuxONE 5\n#ifdef GGML_USE_NNPA\n if (!sf.has_nnpa) { return 0; }\n score += 1 << 2;\n#endif\n\n return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_s390x_score)\n\n#endif // __s390x__\n" }, { "path": "src/ggml-cpu/arch/s390/quants.c", "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include \n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__VXE__) || defined(__VXE2__)\n#define B1(c,s,n) 0x ## n ## c , 0x ## n ## s\n#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)\n#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)\n#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)\n#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)\n#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)\n#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)\n#define B8(c,s ) B7(c,s, c), B7(c,s, s)\n\n// precomputed tables for expanding 8bits to 8 bytes:\nstatic const __attribute__((aligned(16))) uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b ) << 4\nstatic const __attribute__((aligned(16))) uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4\n\n// permute mask for byteswapping\nstatic const uint8x16_t v_kperm = (const uint8x16_t){\n 7, 6, 5, 4, 3, 2, 1, 0,\n 15, 14, 13, 12, 11, 10, 9, 8\n};\n#endif\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__VXE__) || defined(__VXE2__)\n for (int i = 0; i < nb; i++) {\n float32x4_t srcv [8];\n float32x4_t asrcv[8];\n float32x4_t amaxv[8];\n\n for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);\n for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);\n for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);\n for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);\n for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);\n\n const float amax = MAX(MAX(vec_extract(amaxv[0], 0),\n vec_extract(amaxv[0], 1)),\n MAX(vec_extract(amaxv[0], 2),\n vec_extract(amaxv[0], 3)));\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f / d : 0.0f;\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n for (int j = 0; j < 8; j++) {\n const float32x4_t v = vec_mul(srcv[j], vec_splats(id));\n /* Uses non-default rounding for vec_signed or vec_round */\n const int32x4_t vi = vec_signed(__builtin_s390_vfisb(v, 4, 1));\n\n y[i].qs[4*j + 0] = vec_extract(vi, 0);\n y[i].qs[4*j + 1] = vec_extract(vi, 1);\n y[i].qs[4*j + 2] = vec_extract(vi, 2);\n y[i].qs[4*j + 3] = vec_extract(vi, 3);\n }\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK8_1 == 0);\n const int nb = k / QK8_1;\n\n block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__VXE__) || defined(__VXE2__)\n for (int i = 0; i < nb; i++) {\n float32x4_t srcv [8];\n float32x4_t asrcv[8];\n float32x4_t amaxv[8];\n\n for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);\n for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);\n for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);\n for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);\n for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);\n\n const float amax = MAX(MAX(vec_extract(amaxv[0], 0),\n vec_extract(amaxv[0], 1)),\n MAX(vec_extract(amaxv[0], 2),\n vec_extract(amaxv[0], 3)));\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f / d : 0.0f;\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n int32x4_t acc = vec_splats(0);\n\n for (int j = 0; j < 8; j++) {\n const float32x4_t v = vec_mul(srcv[j], vec_splats(id));\n /* Uses non-default rounding for vec_signed or vec_round */\n const int32x4_t vi = vec_signed(__builtin_s390_vfisb(v, 4, 1));\n\n y[i].qs[4*j + 0] = vec_extract(vi, 0);\n y[i].qs[4*j + 1] = vec_extract(vi, 1);\n y[i].qs[4*j + 2] = vec_extract(vi, 2);\n y[i].qs[4*j + 3] = vec_extract(vi, 3);\n\n acc = vec_add(acc, vi);\n }\n\n y[i].s = GGML_CPU_FP32_TO_FP16(d * (acc[0] + acc[1] + acc[2] + acc[3]));\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__VXE__) || defined(__VXE2__)\n float32x4_t acc = vec_splats(0.0f);\n\n const uint8x16_t v_m = vec_splats((const uint8_t)0x0F);\n const int8x16_t v_s = vec_splats( (const int8_t)0x08);\n\n for (; ib < nb; ++ib) {\n const uint8x16_t v_x = vec_xl(0, x[ib].qs);\n const int8x16_t v_xl = (const int8x16_t)(v_x & v_m);\n const int8x16_t v_xh = (const int8x16_t)(v_x >> 4);\n\n const int8x16_t v_xls = vec_sub(v_xl, v_s);\n const int8x16_t v_xhs = vec_sub(v_xh, v_s);\n\n const int8x16_t v_yl = vec_xl(0 , y[ib].qs);\n const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs);\n\n const int16x8_t v_xylso = vec_mulo(v_xls, v_yl);\n const int16x8_t v_xyl = vec_meadd(v_xls, v_yl, v_xylso);\n const int16x8_t v_xyhso = vec_mulo(v_xhs, v_yh);\n const int16x8_t v_xyh = vec_meadd(v_xhs, v_yh, v_xyhso);\n\n int16x8_t v_xy_ = v_xyl + v_xyh; v_xy_ += vec_reve(v_xy_);\n\n const float32x4_t v_xy = vec_float(vec_unpackh(v_xy_));\n const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n acc = vec_madd(v_xy, v_d, acc);\n }\n\n sumf = vec_hsum_f32x4(acc);\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__VXE__) || defined(__VXE2__)\n float summs = 0;\n float32x4_t acc = vec_splats(0.0f);\n\n const uint8x16_t v_m = vec_splat_u8(0x0F);\n\n#pragma GCC unroll 4\n for (; ib < nb; ++ib) {\n __builtin_prefetch(x[ib].qs, 0, 1);\n __builtin_prefetch(y[ib].qs, 0, 1);\n\n summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n const uint8x16_t v_x = vec_xl(0, x[ib].qs);\n const int8x16_t v_xl = (const int8x16_t)(v_x & v_m);\n const int8x16_t v_xh = (const int8x16_t)(v_x >> 4);\n\n const int8x16_t v_yl = vec_xl(0 , y[ib].qs);\n const int8x16_t v_yh = vec_xl(QK8_1/2, y[ib].qs);\n\n const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);\n const float32x4_t v_xy = vec_float(v_xy_);\n\n const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n acc = vec_madd(v_xy, v_d, acc);\n }\n\n sumf = vec_hsum_f32x4(acc) + summs;\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_MXFP4 == 0);\n static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n const int qk = QK_MXFP4;\n const int nb = n / qk;\n\n const block_mxfp4 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0.0f;\n\n#if defined(__VXE__) || defined(__VXE2__)\n const int8x16_t v_k = vec_xl(0, kvalues_mxfp4);\n const uint8x16_t v_m = vec_splats((const uint8_t)0x0F);\n\n float32x4_t v_acc = vec_splats(0.0f);\n\n #pragma GCC unroll 8\n for (; ib + 1 < nb; ib += 2) {\n const block_mxfp4 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_mxfp4 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n const uint8x16_t v_x0 = vec_xl(0, x0->qs);\n const uint8x16_t v_x1 = vec_xl(0, x1->qs);\n\n int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);\n int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);\n int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);\n int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);\n\n v_x0l = vec_perm(v_k, v_k, (uchar8x16_t)v_x0l);\n v_x0h = vec_perm(v_k, v_k, (uchar8x16_t)v_x0h);\n v_x1l = vec_perm(v_k, v_k, (uchar8x16_t)v_x1l);\n v_x1h = vec_perm(v_k, v_k, (uchar8x16_t)v_x1h);\n\n const int8x16_t v_y0l = vec_xl(0, y0->qs);\n const int8x16_t v_y0h = vec_xl(QK8_0/2, y0->qs);\n const int8x16_t v_y1l = vec_xl(0, y1->qs);\n const int8x16_t v_y1h = vec_xl(QK8_0/2, y1->qs);\n\n const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h);\n const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h);\n\n const float32x4_t v_xy0f = vec_float(v_xy0);\n const float32x4_t v_xy1f = vec_float(v_xy1);\n\n const float32x4_t v_d0 = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d));\n const float32x4_t v_d1 = vec_splats(GGML_E8M0_TO_FP32_HALF(x1->e) * GGML_CPU_FP16_TO_FP32(y1->d));\n\n v_acc = vec_madd(v_xy0f, v_d0, v_acc);\n v_acc = vec_madd(v_xy1f, v_d1, v_acc);\n }\n\n for (; ib < nb; ++ib) {\n const block_mxfp4 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n\n const uint8x16_t v_x = vec_xl(0, x0->qs);\n\n int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);\n int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);\n\n v_xl = vec_perm(v_k, v_k, (uchar8x16_t)v_xl);\n v_xh = vec_perm(v_k, v_k, (uchar8x16_t)v_xh);\n\n const int8x16_t v_yl = vec_xl(0, y0->qs);\n const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);\n\n const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);\n const float32x4_t v_xyf = vec_float(v_xy);\n\n const float32x4_t v_d = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d));\n v_acc = vec_madd(v_xyf, v_d, v_acc);\n }\n\n sumf = vec_hsum_f32x4(v_acc);\n *s = sumf;\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(qk == QK5_0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0.0f;\n\n#if defined(__VXE__) || defined(__VXE2__)\n float32x4_t v_sum0 = vec_splats(0.0f);\n float32x4_t v_sum1 = vec_splats(0.0f);\n\n uint32_t qh0, qh1;\n uint64_t tmp0[4], tmp1[4];\n\n const uint8x16_t v_m = vec_splats((uint8_t)0x0F);\n\n #pragma GCC unroll 4\n for (; ib + 1 < nb; ib += 2) {\n const block_q5_0 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q5_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n memcpy(&qh0, x0->qh, sizeof(qh0));\n memcpy(&qh1, x1->qh, sizeof(qh1));\n\n tmp0[0] = table_b2b_1[(qh0 >> 0) & 0xFF];\n tmp0[1] = table_b2b_1[(qh0 >> 8) & 0xFF];\n tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];\n tmp0[3] = table_b2b_1[(qh0 >> 24) ];\n\n tmp1[0] = table_b2b_1[(qh1 >> 0) & 0xFF];\n tmp1[1] = table_b2b_1[(qh1 >> 8) & 0xFF];\n tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];\n tmp1[3] = table_b2b_1[(qh1 >> 24) ];\n\n int8x16_t v_qh0l = vec_xl(0, (const int8_t *)(tmp0 + 0));\n int8x16_t v_qh0h = vec_xl(0, (const int8_t *)(tmp0 + 2));\n int8x16_t v_qh1l = vec_xl(0, (const int8_t *)(tmp1 + 0));\n int8x16_t v_qh1h = vec_xl(0, (const int8_t *)(tmp1 + 2));\n\n // required for fixing the byteorder\n v_qh0l = vec_perm(v_qh0l, v_qh0l, v_kperm);\n v_qh0h = vec_perm(v_qh0h, v_qh0h, v_kperm);\n v_qh1l = vec_perm(v_qh1l, v_qh1l, v_kperm);\n v_qh1h = vec_perm(v_qh1h, v_qh1h, v_kperm);\n\n const uint8x16_t v_x0 = vec_xl(0, (const uint8_t *)x0->qs);\n const uint8x16_t v_x1 = vec_xl(0, (const uint8_t *)x1->qs);\n\n int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);\n int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);\n int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);\n int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);\n\n const int8x16_t v_x0lf = vec_sub(v_x0l, v_qh0l);\n const int8x16_t v_x0hf = vec_sub(v_x0h, v_qh0h);\n const int8x16_t v_x1lf = vec_sub(v_x1l, v_qh1l);\n const int8x16_t v_x1hf = vec_sub(v_x1h, v_qh1h);\n\n const int8x16_t v_y0l = vec_xl(0, (const int8_t *)y0->qs);\n const int8x16_t v_y0h = vec_xl(QK8_0/2, (const int8_t *)y0->qs);\n const int8x16_t v_y1l = vec_xl(0, (const int8_t *)y1->qs);\n const int8x16_t v_y1h = vec_xl(QK8_0/2, (const int8_t *)y1->qs);\n\n const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);\n const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);\n\n const float32x4_t v_xy0f = vec_float(v_xy0);\n const float32x4_t v_xy1f = vec_float(v_xy1);\n\n const float32x4_t v_d0 = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));\n const float32x4_t v_d1 = vec_splats(GGML_CPU_FP16_TO_FP32(x1->d) * GGML_CPU_FP16_TO_FP32(y1->d));\n\n v_sum0 = vec_madd(v_xy0f, v_d0, v_sum0);\n v_sum1 = vec_madd(v_xy1f, v_d1, v_sum1);\n }\n\n sumf += vec_hsum_f32x4(v_sum0) + vec_hsum_f32x4(v_sum1);\n\n #pragma GCC unroll 4\n for (; ib < nb; ++ib) {\n const block_q5_0 * GGML_RESTRICT x0 = &x[ib];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n\n uint32_t qh;\n memcpy(&qh, x0->qh, sizeof(qh));\n\n uint64_t tmp[4];\n tmp[0] = table_b2b_1[(qh >> 0) & 0xFF];\n tmp[1] = table_b2b_1[(qh >> 8) & 0xFF];\n tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];\n tmp[3] = table_b2b_1[(qh >> 24) ];\n\n int8x16_t v_qhl = vec_xl(0, (const int8_t *)(tmp + 0));\n int8x16_t v_qhh = vec_xl(0, (const int8_t *)(tmp + 2));\n\n // required for fixing the byteorder\n v_qhl = vec_perm(v_qhl, v_qhl, v_kperm);\n v_qhh = vec_perm(v_qhh, v_qhh, v_kperm);\n\n const uint8x16_t v_x = vec_xl(0, (const uint8_t *)x0->qs);\n int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);\n int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);\n\n const int8x16_t v_xlf = vec_sub(v_xl, v_qhl);\n const int8x16_t v_xhf = vec_sub(v_xh, v_qhh);\n\n const int8x16_t v_yl = vec_xl(0, (const int8_t *)y0->qs);\n const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)y0->qs);\n\n const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);\n const float32x4_t v_xyf = vec_float(v_xy);\n\n const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));\n const float32x4_t v_acc = vec_madd(v_xyf, v_d, vec_splats(0.0f));\n\n sumf += vec_hsum_f32x4(v_acc);\n }\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(qk == QK5_1);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0.0f;\n\n#if defined(__VXE__) || defined(__VXE2__)\n float32x4_t v_sum0 = vec_splats(0.0f);\n float32x4_t v_sum1 = vec_splats(0.0f);\n\n float summs0 = 0.0f;\n float summs1 = 0.0f;\n\n uint32_t qh0;\n uint32_t qh1;\n\n uint64_t tmp0[4];\n uint64_t tmp1[4];\n\n const uint8x16_t v_m = vec_splats((uint8_t)0x0F);\n\n #pragma GCC unroll 4\n for (; ib + 1 < nb; ib += 2) {\n const block_q5_1 * GGML_RESTRICT x0 = &x[ib + 0];\n const block_q5_1 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_1 * GGML_RESTRICT y0 = &y[ib + 0];\n const block_q8_1 * GGML_RESTRICT y1 = &y[ib + 1];\n\n summs0 += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);\n summs1 += GGML_CPU_FP16_TO_FP32(x1->m) * GGML_CPU_FP16_TO_FP32(y1->s);\n\n memcpy(&qh0, x0->qh, sizeof(qh0));\n memcpy(&qh1, x1->qh, sizeof(qh1));\n\n tmp0[0] = table_b2b_0[(qh0 >> 0) & 0xFF];\n tmp0[1] = table_b2b_0[(qh0 >> 8) & 0xFF];\n tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];\n tmp0[3] = table_b2b_0[(qh0 >> 24) ];\n\n tmp1[0] = table_b2b_0[(qh1 >> 0) & 0xFF];\n tmp1[1] = table_b2b_0[(qh1 >> 8) & 0xFF];\n tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];\n tmp1[3] = table_b2b_0[(qh1 >> 24) ];\n\n int8x16_t v_qh0l = vec_xl(0, (const int8_t *)(tmp0 + 0));\n int8x16_t v_qh0h = vec_xl(0, (const int8_t *)(tmp0 + 2));\n int8x16_t v_qh1l = vec_xl(0, (const int8_t *)(tmp1 + 0));\n int8x16_t v_qh1h = vec_xl(0, (const int8_t *)(tmp1 + 2));\n\n // required for fixing the byteorder\n v_qh0l = vec_perm(v_qh0l, v_qh0l, v_kperm);\n v_qh0h = vec_perm(v_qh0h, v_qh0h, v_kperm);\n v_qh1l = vec_perm(v_qh1l, v_qh1l, v_kperm);\n v_qh1h = vec_perm(v_qh1h, v_qh1h, v_kperm);\n\n const uint8x16_t v_x0 = vec_xl(0, x0->qs);\n const uint8x16_t v_x1 = vec_xl(0, x1->qs);\n\n const int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);\n const int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);\n const int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);\n const int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);\n\n const int8x16_t v_x0lf = vec_or(v_x0l, v_qh0l);\n const int8x16_t v_x0hf = vec_or(v_x0h, v_qh0h);\n const int8x16_t v_x1lf = vec_or(v_x1l, v_qh1l);\n const int8x16_t v_x1hf = vec_or(v_x1h, v_qh1h);\n\n const int8x16_t v_y0l = vec_xl(0 , y0->qs);\n const int8x16_t v_y0h = vec_xl(QK8_1/2, y0->qs);\n const int8x16_t v_y1l = vec_xl(0 , y1->qs);\n const int8x16_t v_y1h = vec_xl(QK8_1/2, y1->qs);\n\n const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);\n const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);\n\n const float32x4_t v_xy0f = vec_float(v_xy0);\n const float32x4_t v_xy1f = vec_float(v_xy1);\n\n const float32x4_t v_d0 = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));\n const float32x4_t v_d1 = vec_splats(GGML_CPU_FP16_TO_FP32(x1->d) * GGML_CPU_FP16_TO_FP32(y1->d));\n\n v_sum0 = vec_madd(v_xy0f, v_d0, v_sum0);\n v_sum1 = vec_madd(v_xy1f, v_d1, v_sum1);\n }\n\n sumf += vec_hsum_f32x4(v_sum0) + vec_hsum_f32x4(v_sum1) + summs0 + summs1;\n\n #pragma GCC unroll 4\n for (; ib < nb; ++ib) {\n const block_q5_1 * GGML_RESTRICT x0 = &x[ib];\n const block_q8_1 * GGML_RESTRICT y0 = &y[ib];\n\n float summs = GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);\n\n uint32_t qh;\n memcpy(&qh, x0->qh, sizeof(qh));\n\n uint64_t tmp[4];\n tmp[0] = table_b2b_0[(qh >> 0) & 0xFF];\n tmp[1] = table_b2b_0[(qh >> 8) & 0xFF];\n tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];\n tmp[3] = table_b2b_0[(qh >> 24) ];\n\n int8x16_t v_qhl = vec_xl(0, (const int8_t *)(tmp + 0));\n int8x16_t v_qhh = vec_xl(0, (const int8_t *)(tmp + 2));\n\n // required for fixing the byteorder\n v_qhl = vec_perm(v_qhl, v_qhl, v_kperm);\n v_qhh = vec_perm(v_qhh, v_qhh, v_kperm);\n\n const uint8x16_t v_x = vec_xl(0, x0->qs);\n const int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);\n const int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);\n\n const int8x16_t v_xlf = vec_or(v_xl, v_qhl);\n const int8x16_t v_xhf = vec_or(v_xh, v_qhh);\n\n const int8x16_t v_yl = vec_xl(0 , y0->qs);\n const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs);\n\n const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);\n const float32x4_t v_xyf = vec_float(v_xy);\n\n const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));\n const float32x4_t v_acc = vec_madd(v_xyf, v_d, v_acc);\n\n sumf += vec_hsum_f32x4(v_acc) + summs;\n }\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q8_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__VXE__) || defined(__VXE2__)\n float32x4_t acc = vec_splats(0.0f);\n\n#pragma GCC unroll 8\n for (; ib < nb; ++ib) {\n __builtin_prefetch(x[ib].qs, 0, 1);\n __builtin_prefetch(y[ib].qs, 0, 1);\n\n const int8x16_t v_xl = vec_xl(0 , x[ib].qs);\n const int8x16_t v_xh = vec_xl(QK8_0/2, x[ib].qs);\n const int8x16_t v_yl = vec_xl(0 , y[ib].qs);\n const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs);\n\n const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);\n const float32x4_t v_xy = vec_float(v_xy_);\n const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n acc = vec_madd(v_xy, v_d, acc);\n }\n\n sumf = vec_hsum_f32x4(acc);\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const uint32_t kmask1 = 0x03030303;\n const uint32_t kmask2 = 0x0f0f0f0f;\n\n const block_q3_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__VXE__) || defined(__VXE2__)\n uint32_t aux[3];\n uint32_t utmp[4];\n\n const int32x4_t v_z = vec_splat_s32(0);\n const uint8x16_t v_3m = vec_splat_u8(0x03);\n\n const uint8x16_t v_0c = vec_splat_u8(1);\n const uint8x16_t v_1c = vec_sl(v_0c, 1);\n const uint8x16_t v_2c = vec_sl(v_0c, 2);\n const uint8x16_t v_3c = vec_sl(v_0c, 3);\n\n uint8x16_t q3h[4];\n uint8x16_t q3b[2];\n int8x16_t q3bytes[4];\n int8x16_t q8bytes[8];\n uint8x16_t qhbits[2];\n\n float sum = 0;\n\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * restrict x0l = x[i].qs;\n const uint8_t * restrict x0h = x[i].hmask;\n const int8_t * restrict y0 = y[i].qs;\n\n qhbits[0] = vec_xl(0 , x0h);\n qhbits[1] = vec_xl(16, x0h);\n\n int32_t isum = 0;\n\n memcpy(aux, x[i].scales, 12);\n utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);\n utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);\n utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);\n utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);\n\n int8_t * scale = (int8_t *)utmp;\n for (int j = 0; j < 16; ++j) scale[j] -= 32;\n\n for (int j = 0; j < QK_K/128; ++j) {\n int32x4_t isum0, isum1, isum2, isum3;\n\n q3b[0] = vec_xl(0 , x0l);\n q3b[1] = vec_xl(16, x0l);\n x0l += 32;\n\n q8bytes[0] = vec_xl(0 , y0);\n q8bytes[1] = vec_xl(16 , y0);\n q8bytes[2] = vec_xl(32 , y0);\n q8bytes[3] = vec_xl(48 , y0);\n q8bytes[4] = vec_xl(64 , y0);\n q8bytes[5] = vec_xl(80 , y0);\n q8bytes[6] = vec_xl(96 , y0);\n q8bytes[7] = vec_xl(112, y0);\n y0 += 128;\n\n q3h[0] = vec_sl(vec_andc(v_0c, qhbits[0]), 2);\n q3h[1] = vec_sl(vec_andc(v_0c, qhbits[1]), 2);\n q3h[2] = vec_sl(vec_andc(v_1c, qhbits[0]), 1);\n q3h[3] = vec_sl(vec_andc(v_1c, qhbits[1]), 1);\n\n q3bytes[0] = vec_sub((int8x16_t)vec_and(q3b[0], v_3m), (int8x16_t)q3h[0]);\n q3bytes[1] = vec_sub((int8x16_t)vec_and(q3b[1], v_3m), (int8x16_t)q3h[1]);\n q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 2), v_3m), (int8x16_t)q3h[2]);\n q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 2), v_3m), (int8x16_t)q3h[3]);\n\n isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[0]);\n isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[1]);\n isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[2]);\n isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[3]);\n\n isum += (isum0[0] + isum0[1] + isum0[2] + isum0[3]) * scale[0];\n isum += (isum1[0] + isum1[1] + isum1[2] + isum1[3]) * scale[1];\n isum += (isum2[0] + isum2[1] + isum2[2] + isum2[3]) * scale[2];\n isum += (isum3[0] + isum3[1] + isum3[2] + isum3[3]) * scale[3];\n\n scale += 4;\n\n q3h[0] = vec_andc(v_2c, qhbits[0]);\n q3h[1] = vec_andc(v_2c, qhbits[1]);\n q3h[2] = vec_sr(vec_andc(v_3c, qhbits[0]), 1);\n q3h[3] = vec_sr(vec_andc(v_3c, qhbits[1]), 1);\n\n q3bytes[0] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 4), v_3m), (int8x16_t)q3h[0]);\n q3bytes[1] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 4), v_3m), (int8x16_t)q3h[1]);\n q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 6), v_3m), (int8x16_t)q3h[2]);\n q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 6), v_3m), (int8x16_t)q3h[3]);\n\n isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[4]);\n isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[5]);\n isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[6]);\n isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[7]);\n\n isum += vec_hsum_i32x4(isum0) * scale[0];\n isum += vec_hsum_i32x4(isum1) * scale[1];\n isum += vec_hsum_i32x4(isum2) * scale[2];\n isum += vec_hsum_i32x4(isum3) * scale[3];\n\n scale += 4;\n\n if (j == 0) {\n qhbits[0] = vec_sr(qhbits[0], 4);\n qhbits[1] = vec_sr(qhbits[1], 4);\n }\n }\n\n sum += d * isum;\n }\n\n *s = sum;\n\n#else\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined(__VXE__) || defined(__VXE2__)\n const uint8x16_t v_lm = vec_splat_u8(0x0F);\n const int32x4_t v_z = vec_splat_s32(0);\n\n uint8x16_t v_x[2];\n int8x16_t v_xl[2];\n int8x16_t v_y[2];\n\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums);\n const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums);\n const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh);\n\n memcpy(utmp, x[i].scales, 12);\n\n uint32x4_t v_mins8 = { 0 };\n v_mins8 = vec_insert(utmp[1] & kmask1, v_mins8, 0);\n v_mins8 = vec_insert(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), v_mins8, 1);\n\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[0] &= kmask1;\n\n const int16x8_t v_minsh = (int16x8_t)vec_unpackh((uint8x16_t)v_mins8);\n\n const int32x4_t v_minso = vec_mulo(v_ysums, v_minsh);\n const int32x4_t v_mins = vec_meadd(v_ysums, v_minsh, v_minso);\n sumf -= dmin * (v_mins[0] + v_mins[1] + v_mins[2] + v_mins[3]);\n\n const uint8_t * scales = (const uint8_t *)utmp;\n const uint8_t * GGML_RESTRICT x0 = x[i].qs;\n const int8_t * GGML_RESTRICT y0 = y[i].qs;\n\n int32_t sumi1 = 0;\n int32_t sumi2 = 0;\n\n for (int j = 0; j < QK_K/64; ++j) {\n v_x[0] = vec_xl(0 , x0);\n v_x[1] = vec_xl(16, x0);\n x0 += 32;\n\n v_y[0] = vec_xl(0 , y0);\n v_y[1] = vec_xl(16, y0);\n y0 += 32;\n\n v_xl[0] = (int8x16_t)vec_and(v_x[0], v_lm);\n v_xl[1] = (int8x16_t)vec_and(v_x[1], v_lm);\n\n const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);\n sumi1 += vec_hsum_i32x4(p1) * scales[2*j+0];\n\n v_y[0] = vec_xl(0 , y0);\n v_y[1] = vec_xl(16, y0);\n y0 += 32;\n\n v_xl[0] = (int8x16_t)vec_sr(v_x[0], 4);\n v_xl[1] = (int8x16_t)vec_sr(v_x[1], 4);\n\n const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);\n sumi2 += vec_hsum_i32x4(p2) * scales[2*j+1];\n }\n\n sumf += d * (sumi1 + sumi2);\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined(__VXE__) || defined(__VXE2__)\n const uint8x16_t v_lm = vec_splat_u8(0x0F);\n const uint8x16_t v_1m = vec_splat_u8(0x01);\n const uint8x16_t v_2m = vec_splat_u8(0x02);\n\n const int32x4_t v_z = vec_splat_s32(0);\n\n const uchar8x16_t v_minsm = {\n 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,\n 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF\n };\n\n int8x16_t q5b[4];\n uint8x16_t q5h[4];\n\n uint8x16_t v_xl[2];\n uint8x16_t v_xh[2];\n int8x16_t v_y[4];\n\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums);\n const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums);\n const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh);\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n const uint8x16_t v_mins16 = vec_xl(0, (const uint8_t *)utmp);\n const uint8x16_t v_mins8 = vec_perm(v_mins16, v_mins16, v_minsm);\n const int16x8_t v_minsh = (int16x8_t)vec_unpackh(v_mins8);\n\n const int32x4_t v_minsho = vec_mulo(v_ysums, v_minsh);\n const int32x4_t v_mins = vec_meadd(v_ysums, v_minsh, v_minsho);\n const int32_t mins = vec_hsum_i32x4(v_mins);\n\n const uint8_t * scales = (const uint8_t *)utmp;\n const uint8_t * GGML_RESTRICT x0l = x[i].qs;\n const uint8_t * GGML_RESTRICT x0h = x[i].qh;\n const int8_t * GGML_RESTRICT y0 = y[i].qs;\n\n v_xh[0] = vec_xl(0 , x0h);\n v_xh[1] = vec_xl(16, x0h);\n\n int32_t sumi = 0;\n for (int j = 0; j < QK_K/64; ++j) {\n v_xl[0] = vec_xl(0 , x0l);\n v_xl[1] = vec_xl(16, x0l);\n x0l += 32;\n\n v_y[0] = vec_xl(0 , y0);\n v_y[1] = vec_xl(16, y0);\n v_y[2] = vec_xl(32, y0);\n v_y[3] = vec_xl(48, y0);\n y0 += 64;\n\n q5h[0] = vec_sl(vec_and(v_1m, v_xh[0]), 4);\n q5h[1] = vec_sl(vec_and(v_1m, v_xh[1]), 4);\n q5h[2] = vec_sl(vec_and(v_2m, v_xh[0]), 3);\n q5h[3] = vec_sl(vec_and(v_2m, v_xh[1]), 3);\n v_xh[0] = vec_sr(v_xh[0], 2);\n v_xh[1] = vec_sr(v_xh[1], 2);\n\n q5b[0] = (int8x16_t)vec_or(vec_and(v_xl[0], v_lm), q5h[0]);\n q5b[1] = (int8x16_t)vec_or(vec_and(v_xl[1], v_lm), q5h[1]);\n q5b[2] = (int8x16_t)vec_or(vec_sr(v_xl[0], 4), q5h[2]);\n q5b[3] = (int8x16_t)vec_or(vec_sr(v_xl[1], 4), q5h[3]);\n\n int32x4_t sumi0 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]);\n int32x4_t sumi1 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]);\n\n sumi += vec_hsum_i32x4(sumi0) * *scales++;\n sumi += vec_hsum_i32x4(sumi1) * *scales++;\n }\n\n sumf += d * sumi - dmin * mins;\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q6_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__VXE__) || defined(__VXE2__)\n float sum = 0;\n\n // Lower 4-bit and upper 2-bit masks\n const uint8x16_t v_lm = vec_splat_u8(0x0F);\n const uint8x16_t v_um = vec_splat_u8(0x03);\n\n const int32x4_t v_z = vec_splat_s32(0);\n\n int8x16_t q6b[4];\n uint8x16_t q6h[4];\n\n uint8x16_t v_xl[4];\n uint8x16_t v_xh[2];\n int8x16_t v_y[4];\n\n for (int i = 0; i < nb; ++i) {\n const float d_all = GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT x0l = x[i].ql;\n const uint8_t * GGML_RESTRICT x0h = x[i].qh;\n const int8_t * GGML_RESTRICT y0 = y[i].qs;\n\n const int8_t * GGML_RESTRICT scale = x[i].scales;\n\n const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums);\n const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums);\n\n const int8x16_t v_scale = vec_xl(0, scale);\n const int16x8_t v_scalel = vec_unpackh(v_scale);\n const int16x8_t v_scaleh = vec_unpackl(v_scale);\n\n const int32x4_t v_minslo = vec_mulo(v_ysumsl, v_scalel);\n const int32x4_t v_minsl = vec_meadd(v_ysumsl, v_scalel, v_minslo);\n const int32x4_t v_minsho = vec_mulo(v_ysumsh, v_scaleh);\n const int32x4_t v_minsh = vec_meadd(v_ysumsh, v_scaleh, v_minsho);\n const int32x4_t v_mins = vec_add(v_minsl, v_minsh);\n\n const int32_t mins = vec_hsum_i32x4(v_mins);\n\n int32_t isum = 0;\n for (int j = 0; j < QK_K/128; ++j) {\n // Load model upper 2 bits\n v_xh[0] = vec_xl(0 , x0h);\n v_xh[1] = vec_xl(16, x0h);\n x0h += 32;\n\n // Load model lower 4 bits\n v_xl[0] = vec_xl(0 , x0l);\n v_xl[1] = vec_xl(16, x0l);\n v_xl[2] = vec_xl(32, x0l);\n v_xl[3] = vec_xl(48, x0l);\n x0l += 64;\n\n // Load activation quants\n v_y[0] = vec_xl(0 , y0);\n v_y[1] = vec_xl(16, y0);\n v_y[2] = vec_xl(32, y0);\n v_y[3] = vec_xl(48, y0);\n y0 += 64;\n\n q6h[0] = vec_sl(vec_and(v_um, v_xh[0]), 4);\n q6h[1] = vec_sl(vec_and(v_um, v_xh[1]), 4);\n uint8x16_t shifted = vec_sr(v_xh[0], 2);\n q6h[2] = vec_sl(vec_and(v_um, shifted), 4);\n shifted = vec_sr(v_xh[1], 2);\n q6h[3] = vec_sl(vec_and(v_um, shifted), 4);\n\n q6b[0] = (int8x16_t)(vec_or(vec_and(v_xl[0], v_lm), q6h[0]));\n q6b[1] = (int8x16_t)(vec_or(vec_and(v_xl[1], v_lm), q6h[1]));\n q6b[2] = (int8x16_t)(vec_or(vec_and(v_xl[2], v_lm), q6h[2]));\n q6b[3] = (int8x16_t)(vec_or(vec_and(v_xl[3], v_lm), q6h[3]));\n\n int32x4_t summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);\n int32x4_t summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);\n int32x4_t summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);\n int32x4_t summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);\n\n isum += vec_hsum_i32x4(summs0) * scale[0] +\n vec_hsum_i32x4(summs1) * scale[1] +\n vec_hsum_i32x4(summs2) * scale[2] +\n vec_hsum_i32x4(summs3) * scale[3];\n\n scale += 4;\n\n\n // Load activation quants\n v_y[0] = vec_xl(0 , y0);\n v_y[1] = vec_xl(16, y0);\n v_y[2] = vec_xl(32, y0);\n v_y[3] = vec_xl(48, y0);\n y0 += 64;\n\n shifted = vec_sr(v_xh[0], 4);\n q6h[0] = vec_sl(vec_and(v_um, shifted), 4);\n shifted = vec_sr(v_xh[1], 4);\n q6h[1] = vec_sl(vec_and(v_um, shifted), 4);\n shifted = vec_sr(v_xh[0], 6);\n q6h[2] = vec_sl(vec_and(v_um, shifted), 4);\n shifted = vec_sr(v_xh[1], 6);\n q6h[3] = vec_sl(vec_and(v_um, shifted), 4);\n\n q6b[0] = (int8x16_t)(vec_or(vec_sr(v_xl[0], 4), q6h[0]));\n q6b[1] = (int8x16_t)(vec_or(vec_sr(v_xl[1], 4), q6h[1]));\n q6b[2] = (int8x16_t)(vec_or(vec_sr(v_xl[2], 4), q6h[2]));\n q6b[3] = (int8x16_t)(vec_or(vec_sr(v_xl[3], 4), q6h[3]));\n\n summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);\n summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);\n summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);\n summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);\n\n isum += vec_hsum_i32x4(summs0) * scale[0] +\n vec_hsum_i32x4(summs1) * scale[1] +\n vec_hsum_i32x4(summs2) * scale[2] +\n vec_hsum_i32x4(summs3) * scale[3];\n\n scale += 4;\n }\n\n sum += d_all * y[i].d * (isum - 32 * mins);\n }\n\n *s = sum;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n// #if defined(__VXE__) || defined(__VXE2__)\n// static const int8_t keven_signs_q2xs[1024] = {\n// 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,\n// 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,\n// 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,\n// 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,\n// 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,\n// 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,\n// 1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, 1,\n// 1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, -1,\n// 1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, -1,\n// 1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, 1,\n// 1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, 1,\n// 1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1,\n// 1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, 1,\n// 1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, -1,\n// 1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, -1,\n// 1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1,\n// 1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, -1,\n// 1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1,\n// 1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, 1,\n// 1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, -1,\n// 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1,\n// 1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, -1,\n// 1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, -1,\n// 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1,\n// 1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, 1,\n// 1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, -1,\n// 1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1,\n// 1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, 1,\n// 1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1,\n// 1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, 1,\n// 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, 1,\n// 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,\n// };\n// #endif\n\n// void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n// assert(n % QK_K == 0);\n// assert(nrc == 1);\n// UNUSED(nrc);\n// UNUSED(bx);\n// UNUSED(by);\n// UNUSED(bs);\n\n// const block_iq2_xxs * GGML_RESTRICT x = vx;\n// const block_q8_K * GGML_RESTRICT y = vy;\n\n// const int nb = n / QK_K;\n\n// #if defined(__VXE__) || defined(__VXE2__)\n// const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n// uint32_t aux32[4];\n// const uint8_t * aux8 = (const uint8_t *)aux32;\n\n// float sumf = 0;\n\n// for (int i = 0; i < nb; ++i) {\n// const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n// const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n// const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n// float sumf1 = 0, sumf2 = 0;\n\n// for (int ib32 = 0; ib32 < QK_K/32; ib += 2) {\n// int8x16_t q8b0 = vec_xl( 0, q8);\n// int8x16_t qb81 = vec_xl(16, q8);\n// int8x16_t q8b2 = vec_xl(32, q8);\n// int8x16_t q8b3 = vec_xl(48, q8);\n// q8 += 64;\n\n// memcpy(aux32, q2, 4 * sizeof(uint32_t));\n// q2 += 8;\n\n// int8x16_t q2u0 = { *(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1]) };\n// int8x16_t q2u1 = { *(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3]) };\n// int8x16_t q2u2 = { *(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9]) };\n// int8x16_t q2u3 = { *(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11]) };\n\n// int8x16_t q2s0 = { *(const int64_t *)(signs64 + ((aux32[1] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 7) & 127)) };\n// int8x16_t q2s1 = { *(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127)) };\n// int8x16_t q2s2 = { *(const int64_t *)(signs64 + ((aux32[3] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 7) & 127)) };\n// int8x16_t q2s3 = { *(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127)) };\n\n// q2u0 = vec_mul(q2u0, q2s0);\n// q2u1 = vec_mul(q2u1, q2s1);\n// q2u2 = vec_mul(q2u2, q2s2);\n// q2u3 = vec_mul(q2u3, q2s3);\n\n// const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1);\n// const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3);\n\n// sumf1 += (p1[0] + p1[1] + p1[2] + p1[3]) * (0.5f + (aux32[1] >> 28));\n// sumf2 += (p2[0] + p2[1] + p2[2] + p2[3]) * (0.5f + (aux32[3] >> 28));\n// }\n\n// sumf += d * (sumf1 + sumf2);\n// }\n\n// *s = 0.25f * sumf;\n\n// #else\n\n// uint32_t aux32[2];\n// const uint8_t * aux8 = (const uint8_t *)aux32;\n\n// float sumf = 0.f;\n// for (int i = 0; i < nb; ++i) {\n// const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n// const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n// const int8_t * GGML_RESTRICT q8 = y[i].qs;\n// int32_t bsum = 0;\n// for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n// memcpy(aux32, q2, 2*sizeof(uint32_t));\n// q2 += 4;\n// const uint32_t ls = 2*(aux32[1] >> 28) + 1;\n// int32_t sumi = 0;\n// for (int l = 0; l < 4; ++l) {\n// const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);\n// const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];\n// for (int j = 0; j < 8; ++j) {\n// sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);\n// }\n// q8 += 8;\n// }\n// bsum += sumi * ls;\n// }\n// sumf += d * bsum;\n// }\n// *s = 0.125f * sumf;\n// #endif\n// }\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK4_NL == 0);\n static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n const block_iq4_nl * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK4_NL;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__VXE__) || defined(__VXE2__)\n const int8x16_t v_k = vec_xl(0, kvalues_iq4nl);\n const uint8x16_t v_m = vec_splat_u8(0x0F);\n\n for (; ib < nb; ++ib) {\n const block_iq4_nl * GGML_RESTRICT x0 = &x[ib];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n\n const uint8x16_t v_x = vec_xl(0, x0->qs);\n int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);\n int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);\n\n v_xl = vec_perm(v_k, v_k, (uchar8x16_t)v_xl);\n v_xh = vec_perm(v_k, v_k, (uchar8x16_t)v_xh);\n\n const int8x16_t v_yl = vec_xl(0 , y0->qs);\n const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);\n const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);\n\n sumf += GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d) * vec_hsum_i32x4(v_xy);\n }\n\n *s = sumf;\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_K == 0);\n\n const block_iq4_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__VXE__) || defined(__VXE2__)\n const int8x16_t v_k = vec_xl(0, kvalues_iq4nl);\n const uint8x16_t v_m = vec_splat_u8(0x0F);\n\n float sumf = 0;\n\n for (int ibl = 0; ibl < nb; ++ibl) {\n const uint8_t * GGML_RESTRICT q4 = x[ibl].qs;\n const int8_t * GGML_RESTRICT q8 = y[ibl].qs;\n\n uint16_t h = x[ibl].scales_h;\n\n int sumi1 = 0, sumi2 = 0;\n for (int ib = 0; ib < QK_K/64; ++ib) {\n const uint8x16_t v_x0 = vec_xl(0 , q4);\n const uint8x16_t v_x1 = vec_xl(QK4_NL/2, q4);\n q4 += 32;\n\n int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);\n int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);\n int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);\n int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);\n\n v_x0l = vec_perm(v_k, v_k, (uchar8x16_t)v_x0l);\n v_x0h = vec_perm(v_k, v_k, (uchar8x16_t)v_x0h);\n v_x1l = vec_perm(v_k, v_k, (uchar8x16_t)v_x1l);\n v_x1h = vec_perm(v_k, v_k, (uchar8x16_t)v_x1h);\n\n const int8x16_t v_y0 = vec_xl( 0, q8);\n const int8x16_t v_y1 = vec_xl(16, q8);\n const int8x16_t v_y2 = vec_xl(32, q8);\n const int8x16_t v_y3 = vec_xl(48, q8);\n q8 += 64;\n\n int32x4_t vsumi0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1);\n int32x4_t vsumi1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3);\n\n int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32;\n int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32;\n\n h >>= 4;\n\n sumi1 += vec_hsum_i32x4(vsumi0) * ls1;\n sumi2 += vec_hsum_i32x4(vsumi1) * ls2;\n }\n\n sumf += GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2);\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n" }, { "path": "src/ggml-cpu/arch/wasm/quants.c", "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include \n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__wasm_simd128__)\n#define B1(c,s,n) 0x ## n ## c , 0x ## n ## s\n#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)\n#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)\n#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)\n#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)\n#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)\n#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)\n#define B8(c,s ) B7(c,s, c), B7(c,s, s)\n\n// precomputed tables for expanding 8bits to 8 bytes:\nstatic const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4\nstatic const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4\n#endif\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined __wasm_simd128__\n for (int i = 0; i < nb; i++) {\n v128_t srcv [8];\n v128_t asrcv[8];\n v128_t amaxv[8];\n\n for (int j = 0; j < 8; j++) srcv[j] = wasm_v128_load(x + i*32 + 4*j);\n for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);\n\n for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);\n for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);\n for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);\n\n const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),\n wasm_f32x4_extract_lane(amaxv[0], 1)),\n MAX(wasm_f32x4_extract_lane(amaxv[0], 2),\n wasm_f32x4_extract_lane(amaxv[0], 3)));\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n for (int j = 0; j < 8; j++) {\n const v128_t v = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));\n const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);\n\n y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);\n y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);\n y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);\n y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);\n }\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK8_1 == 0);\n const int nb = k / QK8_1;\n\n block_q8_1 * GGML_RESTRICT y = vy;\n#if defined __wasm_simd128__\n for (int i = 0; i < nb; i++) {\n v128_t srcv [8];\n v128_t asrcv[8];\n v128_t amaxv[8];\n\n for (int j = 0; j < 8; j++) srcv[j] = wasm_v128_load(x + i*32 + 4*j);\n for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);\n\n for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);\n for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);\n for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);\n\n const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),\n wasm_f32x4_extract_lane(amaxv[0], 1)),\n MAX(wasm_f32x4_extract_lane(amaxv[0], 2),\n wasm_f32x4_extract_lane(amaxv[0], 3)));\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n v128_t accv = wasm_i32x4_splat(0);\n\n for (int j = 0; j < 8; j++) {\n const v128_t v = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));\n const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);\n\n y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);\n y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);\n y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);\n y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);\n\n accv = wasm_i32x4_add(accv, vi);\n }\n\n y[i].s = GGML_CPU_FP32_TO_FP16(\n d * (wasm_i32x4_extract_lane(accv, 0) +\n wasm_i32x4_extract_lane(accv, 1) +\n wasm_i32x4_extract_lane(accv, 2) +\n wasm_i32x4_extract_lane(accv, 3)));\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n//===================================== Q8_K ==============================================\n\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n#ifdef __wasm_simd128__\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n block_q8_K * GGML_RESTRICT yc = y; // Cast to proper type\n\n for (int i = 0; i < nb; i++) {\n const float * x_block = x + i * QK_K;\n\n v128_t min_vec = wasm_v128_load(x_block);\n v128_t max_vec = min_vec;\n\n for (int j = 4; j < QK_K; j += 4) {\n v128_t x_vec = wasm_v128_load(x_block + j);\n max_vec = wasm_f32x4_pmax(max_vec, x_vec);\n min_vec = wasm_f32x4_pmin(min_vec, x_vec);\n }\n max_vec = wasm_f32x4_pmax(max_vec, wasm_i32x4_shuffle(max_vec, max_vec, 2, 3, 0, 1));\n max_vec = wasm_f32x4_pmax(max_vec, wasm_i32x4_shuffle(max_vec, max_vec, 1, 0, 3, 2));\n min_vec = wasm_f32x4_pmin(min_vec, wasm_i32x4_shuffle(min_vec, min_vec, 2, 3, 0, 1));\n min_vec = wasm_f32x4_pmin(min_vec, wasm_i32x4_shuffle(min_vec, min_vec, 1, 0, 3, 2));\n float max = wasm_f32x4_extract_lane(max_vec, 0);\n float min = wasm_f32x4_extract_lane(min_vec, 0);\n float amax = -min > max ? min : max;\n\n if (amax == 0.0f) {\n yc[i].d = 0.0f;\n const v128_t zero = wasm_i8x16_splat(0);\n for (int j = 0; j < QK_K; j += 16) {\n wasm_v128_store(yc[i].qs + j, zero);\n }\n continue;\n }\n\n const float iscale = -127.0f / amax;\n const v128_t scale_vec = wasm_f32x4_splat(iscale);\n\n // Process 16 elements per iteration\n for (int j = 0, jb = 0; j < QK_K; j += 16, jb++) {\n // Load and quantize 16 floats\n v128_t x0 = wasm_v128_load(x_block + j);\n v128_t x1 = wasm_v128_load(x_block + j + 4);\n v128_t x2 = wasm_v128_load(x_block + j + 8);\n v128_t x3 = wasm_v128_load(x_block + j + 12);\n\n v128_t q0 = wasm_f32x4_nearest(wasm_f32x4_mul(x0, scale_vec));\n v128_t q1 = wasm_f32x4_nearest(wasm_f32x4_mul(x1, scale_vec));\n v128_t q2 = wasm_f32x4_nearest(wasm_f32x4_mul(x2, scale_vec));\n v128_t q3 = wasm_f32x4_nearest(wasm_f32x4_mul(x3, scale_vec));\n\n // Convert to i32 with saturation\n v128_t i0 = wasm_i32x4_trunc_sat_f32x4(q0);\n v128_t i1 = wasm_i32x4_trunc_sat_f32x4(q1);\n v128_t i2 = wasm_i32x4_trunc_sat_f32x4(q2);\n v128_t i3 = wasm_i32x4_trunc_sat_f32x4(q3);\n\n // Pack into 16 i8 values\n v128_t i8 = wasm_i8x16_narrow_i16x8(\n wasm_i16x8_narrow_i32x4(i0, i1),\n wasm_i16x8_narrow_i32x4(i2, i3)\n );\n wasm_v128_store(yc[i].qs + j, i8);\n\n // Calculate bsums using SIMD\n v128_t sum16 = wasm_i16x8_add(\n wasm_i16x8_extend_low_i8x16(i8),\n wasm_i16x8_extend_high_i8x16(i8)\n );\n v128_t sum32 = wasm_i32x4_add(\n wasm_i32x4_extend_low_i16x8(sum16),\n wasm_i32x4_extend_high_i16x8(sum16)\n );\n sum32 = wasm_i32x4_add(sum32, wasm_i32x4_shuffle(sum32, sum32, 2, 3, 0, 1));\n sum32 = wasm_i32x4_add(sum32, wasm_i32x4_shuffle(sum32, sum32, 1, 0, 3, 2));\n yc[i].bsums[jb] = wasm_i32x4_extract_lane(sum32, 0);\n }\n\n yc[i].d = 1.0f / iscale;\n }\n#else\n quantize_row_q8_K_ref(x, y, k);\n#endif\n}\n\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined __wasm_simd128__\n v128_t sumv = wasm_f32x4_splat(0.0f);\n\n const v128_t m4b = wasm_i8x16_splat(0x0F);\n const v128_t s8b = wasm_i8x16_splat(0x8);\n\n for (; ib + 1 < nb; ib += 2) {\n const block_q4_0 * GGML_RESTRICT x0 = &x[ib];\n const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n // Load and process x0\n v128_t v0_0 = wasm_v128_load(x0->qs);\n v128_t v0_0l = wasm_v128_and(v0_0, m4b);\n v128_t v0_0h = wasm_u8x16_shr(v0_0, 4);\n v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);\n v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);\n\n // Load y0 vectors\n v128_t y0_l = wasm_v128_load(y0->qs);\n v128_t y0_h = wasm_v128_load(y0->qs + 16);\n\n // Extend to i16x8 and compute dot products\n v128_t dx0l = wasm_i16x8_extend_low_i8x16(v0_0ls);\n v128_t dx0h = wasm_i16x8_extend_high_i8x16(v0_0ls);\n v128_t dx0hl = wasm_i16x8_extend_low_i8x16(v0_0hs);\n v128_t dx0hh = wasm_i16x8_extend_high_i8x16(v0_0hs);\n\n v128_t dy0ll = wasm_i16x8_extend_low_i8x16(y0_l);\n v128_t dy0lh = wasm_i16x8_extend_high_i8x16(y0_l);\n v128_t dy0hl = wasm_i16x8_extend_low_i8x16(y0_h);\n v128_t dy0hh = wasm_i16x8_extend_high_i8x16(y0_h);\n\n v128_t dp0 = wasm_i32x4_add(\n wasm_i32x4_add(\n wasm_i32x4_dot_i16x8(dx0l, dy0ll),\n wasm_i32x4_dot_i16x8(dx0h, dy0lh)\n ),\n wasm_i32x4_add(\n wasm_i32x4_dot_i16x8(dx0hl, dy0hl),\n wasm_i32x4_dot_i16x8(dx0hh, dy0hh)\n )\n );\n\n // Load and process x1\n v128_t v0_1 = wasm_v128_load(x1->qs);\n v128_t v0_1l = wasm_v128_and(v0_1, m4b);\n v128_t v0_1h = wasm_u8x16_shr(v0_1, 4);\n v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);\n v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);\n\n // Load y1 vectors\n v128_t y1_l = wasm_v128_load(y1->qs);\n v128_t y1_h = wasm_v128_load(y1->qs + 16);\n\n // Extend to i16x8 and compute dot products\n v128_t dx1l = wasm_i16x8_extend_low_i8x16(v0_1ls);\n v128_t dx1h = wasm_i16x8_extend_high_i8x16(v0_1ls);\n v128_t dx1hl = wasm_i16x8_extend_low_i8x16(v0_1hs);\n v128_t dx1hh = wasm_i16x8_extend_high_i8x16(v0_1hs);\n\n v128_t dy1ll = wasm_i16x8_extend_low_i8x16(y1_l);\n v128_t dy1lh = wasm_i16x8_extend_high_i8x16(y1_l);\n v128_t dy1hl = wasm_i16x8_extend_low_i8x16(y1_h);\n v128_t dy1hh = wasm_i16x8_extend_high_i8x16(y1_h);\n\n v128_t dp1 = wasm_i32x4_add(\n wasm_i32x4_add(\n wasm_i32x4_dot_i16x8(dx1l, dy1ll),\n wasm_i32x4_dot_i16x8(dx1h, dy1lh)\n ),\n wasm_i32x4_add(\n wasm_i32x4_dot_i16x8(dx1hl, dy1hl),\n wasm_i32x4_dot_i16x8(dx1hh, dy1hh)\n )\n );\n\n // Accumulate results with scaling\n float scale0 = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);\n float scale1 = GGML_CPU_FP16_TO_FP32(x1->d) * GGML_CPU_FP16_TO_FP32(y1->d);\n\n sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp0), wasm_f32x4_splat(scale0)));\n sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp1), wasm_f32x4_splat(scale1)));\n }\n\n sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +\n wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);\n\n#endif\n for (; ib < nb; ++ib) {\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n const int v1 = (x[ib].qs[j] >> 4) - 8;\n\n sumi0 += (v0 * y[ib].qs[j]);\n sumi1 += (v1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined __wasm_simd128__\n v128_t sumv = wasm_f32x4_splat(0.0f);\n\n uint32_t qh_;\n uint64_t tmp[4];\n\n // TODO: check if unrolling this is better\n for (; ib < nb; ++ib) {\n const block_q5_0 * GGML_RESTRICT x0 = &x[ib];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n\n const v128_t m4b = wasm_i8x16_splat(0x0F);\n\n // extract the 5th bit\n memcpy(&qh_, x0->qh, sizeof(qh_));\n\n tmp[0] = table_b2b_1[(qh_ >> 0) & 0xFF];\n tmp[1] = table_b2b_1[(qh_ >> 8) & 0xFF];\n tmp[2] = table_b2b_1[(qh_ >> 16) & 0xFF];\n tmp[3] = table_b2b_1[(qh_ >> 24) ];\n\n const v128_t qhl = wasm_v128_load(tmp + 0);\n const v128_t qhh = wasm_v128_load(tmp + 2);\n\n const v128_t v0 = wasm_v128_load(x0->qs);\n\n // 4-bit -> 8-bit\n const v128_t v0l = wasm_v128_and (v0, m4b);\n const v128_t v0h = wasm_u8x16_shr(v0, 4);\n\n // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)\n const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);\n const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);\n\n // load y\n const v128_t v1l = wasm_v128_load(y0->qs);\n const v128_t v1h = wasm_v128_load(y0->qs + 16);\n\n // int8x16 -> int16x8\n const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);\n const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);\n const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);\n const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);\n\n const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);\n const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);\n const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);\n const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);\n\n // dot product\n sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(\n wasm_i32x4_add(\n wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),\n wasm_i32x4_dot_i16x8(v0lfh, v1lh)),\n wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),\n wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),\n wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));\n }\n\n sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +\n wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(sumf);\n UNUSED(x);\n UNUSED(y);\n ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_1);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined __wasm_simd128__\n v128_t sumv = wasm_f32x4_splat(0.0f);\n\n float summs = 0.0f;\n\n uint32_t qh_;\n uint64_t tmp[4];\n\n // TODO: check if unrolling this is better\n for (; ib < nb; ++ib) {\n const block_q5_1 * GGML_RESTRICT x0 = &x[ib];\n const block_q8_1 * GGML_RESTRICT y0 = &y[ib];\n\n summs += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);\n\n const v128_t m4b = wasm_i8x16_splat(0x0F);\n\n // extract the 5th bit\n memcpy(&qh_, x0->qh, sizeof(qh_));\n\n tmp[0] = table_b2b_0[(qh_ >> 0) & 0xFF];\n tmp[1] = table_b2b_0[(qh_ >> 8) & 0xFF];\n tmp[2] = table_b2b_0[(qh_ >> 16) & 0xFF];\n tmp[3] = table_b2b_0[(qh_ >> 24) ];\n\n const v128_t qhl = wasm_v128_load(tmp + 0);\n const v128_t qhh = wasm_v128_load(tmp + 2);\n\n const v128_t v0 = wasm_v128_load(x0->qs);\n\n // 4-bit -> 8-bit\n const v128_t v0l = wasm_v128_and (v0, m4b);\n const v128_t v0h = wasm_u8x16_shr(v0, 4);\n\n // add high bit\n const v128_t v0lf = wasm_v128_or(v0l, qhl);\n const v128_t v0hf = wasm_v128_or(v0h, qhh);\n\n // load y\n const v128_t v1l = wasm_v128_load(y0->qs);\n const v128_t v1h = wasm_v128_load(y0->qs + 16);\n\n // int8x16 -> int16x8\n const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);\n const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);\n const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);\n const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);\n\n const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);\n const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);\n const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);\n const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);\n\n // dot product\n sumv = wasm_f32x4_add(sumv,\n wasm_f32x4_mul(wasm_f32x4_convert_i32x4(wasm_i32x4_add(\n wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),\n wasm_i32x4_dot_i16x8(v0lfh, v1lh)),\n wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),\n wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),\n wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));\n }\n\n sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +\n wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(sumf);\n UNUSED(x);\n UNUSED(y);\n ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q8_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined __wasm_simd128__\n v128_t sumv = wasm_f32x4_splat(0.0f);\n\n for (; ib < nb; ++ib) {\n const block_q8_0 * GGML_RESTRICT x0 = &x[ib];\n const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n\n const v128_t x0_0 = wasm_v128_load(x0->qs);\n const v128_t x0_1 = wasm_v128_load(x0->qs + 16);\n const v128_t y0_0 = wasm_v128_load(y0->qs);\n const v128_t y0_1 = wasm_v128_load(y0->qs + 16);\n\n // Extend 8-bit to 16-bit\n const v128_t x0_0l = wasm_i16x8_extend_low_i8x16(x0_0);\n const v128_t x0_0h = wasm_i16x8_extend_high_i8x16(x0_0);\n const v128_t x0_1l = wasm_i16x8_extend_low_i8x16(x0_1);\n const v128_t x0_1h = wasm_i16x8_extend_high_i8x16(x0_1);\n\n const v128_t y0_0l = wasm_i16x8_extend_low_i8x16(y0_0);\n const v128_t y0_0h = wasm_i16x8_extend_high_i8x16(y0_0);\n const v128_t y0_1l = wasm_i16x8_extend_low_i8x16(y0_1);\n const v128_t y0_1h = wasm_i16x8_extend_high_i8x16(y0_1);\n\n // Compute dot products\n const v128_t dx0_0 = wasm_i32x4_dot_i16x8(x0_0l, y0_0l);\n const v128_t dx0_1 = wasm_i32x4_dot_i16x8(x0_0h, y0_0h);\n const v128_t dx1_0 = wasm_i32x4_dot_i16x8(x0_1l, y0_1l);\n const v128_t dx1_1 = wasm_i32x4_dot_i16x8(x0_1h, y0_1h);\n\n // Sum all dot products\n const v128_t sum_dots = wasm_i32x4_add(wasm_i32x4_add(dx0_0, dx0_1), wasm_i32x4_add(dx1_0, dx1_1));\n\n // Convert to float and accumulate\n const float scale = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);\n sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(sum_dots), wasm_f32x4_splat(scale)));\n }\n\n sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +\n wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);\n\n *s = sumf;\n#else\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n UNUSED(sumf);\n ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q2_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __wasm_simd128__\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n const uint8_t * q2 = x[i].qs;\n const int8_t * q8 = y[i].qs;\n const uint8_t * sc = x[i].scales;\n\n // Vectorized summs calculation\n v128_t summs_vec = wasm_i32x4_splat(0);\n {\n v128_t sc_vec = wasm_v128_load(sc);\n v128_t sc_upper = wasm_u8x16_shr(sc_vec, 4);\n\n v128_t sc_low = wasm_u16x8_extend_low_u8x16(sc_upper);\n v128_t sc_high = wasm_u16x8_extend_high_u8x16(sc_upper);\n\n v128_t bsums1 = wasm_v128_load(&y[i].bsums[0]);\n v128_t bsums2 = wasm_v128_load(&y[i].bsums[8]);\n\n summs_vec = wasm_i32x4_add(\n wasm_i32x4_add(wasm_i32x4_dot_i16x8(sc_low, bsums1),\n wasm_i32x4_dot_i16x8(sc_high, bsums2)),\n summs_vec\n );\n\n summs_vec = wasm_i32x4_add(summs_vec, wasm_i32x4_shuffle(summs_vec, summs_vec, 2, 3, 0, 1));\n summs_vec = wasm_i32x4_add(summs_vec, wasm_i32x4_shuffle(summs_vec, summs_vec, 1, 0, 3, 2));\n }\n int32_t summs = wasm_i32x4_extract_lane(summs_vec, 0);\n\n // Vectorized isum calculation\n int32_t isum = 0;\n const uint8_t * sc_ptr = sc;\n const int k_iters = QK_K/128;\n\n for (int k = 0; k < k_iters; ++k) {\n v128_t isum_vec = wasm_i32x4_splat(0);\n int shift = 0;\n\n for (int j = 0; j < 4; ++j) {\n const int d0 = (sc_ptr[0] & 0xF);\n const int d1 = (sc_ptr[1] & 0xF);\n sc_ptr += 2;\n\n // Process first 16 elements\n v128_t q2_0 = wasm_v128_load(q2);\n v128_t q8_0 = wasm_v128_load(q8);\n v128_t q2_shift_0 = wasm_u8x16_shr(q2_0, shift);\n v128_t q2_bits_0 = wasm_v128_and(q2_shift_0, wasm_i8x16_splat(0x03));\n\n // Process next 16 elements\n v128_t q2_1 = wasm_v128_load(q2 + 16);\n v128_t q8_1 = wasm_v128_load(q8 + 16);\n v128_t q2_shift_1 = wasm_u8x16_shr(q2_1, shift);\n v128_t q2_bits_1 = wasm_v128_and(q2_shift_1, wasm_i8x16_splat(0x03));\n\n // Calculate dot products\n v128_t p0 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q8_0),\n wasm_i16x8_extend_low_i8x16(q2_bits_0)\n );\n v128_t p1 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q8_0),\n wasm_i16x8_extend_high_i8x16(q2_bits_0)\n );\n v128_t p2 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q8_1),\n wasm_i16x8_extend_low_i8x16(q2_bits_1)\n );\n v128_t p3 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q8_1),\n wasm_i16x8_extend_high_i8x16(q2_bits_1)\n );\n\n // Accumulate scaled results\n v128_t scaled = wasm_i32x4_add(\n wasm_i32x4_mul(wasm_i32x4_add(p0, p1), wasm_i32x4_splat(d0)),\n wasm_i32x4_mul(wasm_i32x4_add(p2, p3), wasm_i32x4_splat(d1))\n );\n\n isum_vec = wasm_i32x4_add(isum_vec, scaled);\n q8 += 32;\n shift += 2;\n }\n q2 += 32;\n\n // Horizontal sum of isum_vec\n isum_vec = wasm_i32x4_add(isum_vec, wasm_i32x4_shuffle(isum_vec, isum_vec, 2, 3, 0, 1));\n isum_vec = wasm_i32x4_add(isum_vec, wasm_i32x4_shuffle(isum_vec, isum_vec, 1, 0, 3, 2));\n isum += wasm_i32x4_extract_lane(isum_vec, 0);\n }\n\n const float dall = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;\n sumf += dall * isum - dmin * summs;\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const uint32_t kmask1 = 0x03030303;\n const uint32_t kmask2 = 0x0f0f0f0f;\n\n const block_q3_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __wasm_simd128__\n int8_t aux8[QK_K];\n float sums[8] = {0};\n uint32_t auxs[4];\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT hm = x[i].hmask;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n // Process blocks with SIMD\n int8_t * a = aux8;\n uint8_t m = 1;\n for (int j = 0; j < QK_K; j += 128) {\n for (int shift = 0; shift <= 6; shift += 2) {\n v128_t v_m = wasm_i8x16_splat(m);\n for (int l = 0; l < 32; l += 16) {\n v128_t v_q3 = wasm_v128_load(q3 + l);\n v128_t v_shift = wasm_i8x16_shr(v_q3, shift);\n v128_t v_low2 = wasm_v128_and(v_shift, wasm_i8x16_splat(0x03));\n\n v128_t v_hm = wasm_v128_load(hm + l);\n v128_t v_mask = wasm_v128_and(v_hm, v_m);\n v_mask = wasm_i8x16_ne(v_mask, wasm_i8x16_splat(0));\n\n v_low2 = wasm_i8x16_sub(v_low2, wasm_v128_and(wasm_i8x16_splat(4), wasm_v128_not(v_mask)));\n wasm_v128_store(a + l, v_low2);\n }\n a += 32;\n m <<= 1;\n }\n q3 += 32;\n }\n\n // Extract scales\n memcpy(auxs, x[i].scales, 12);\n uint32_t tmp = auxs[2];\n auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);\n auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);\n auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);\n auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);\n const int8_t * scales = (const int8_t *)auxs;\n\n // SIMD dot product with register accumulators\n v128_t v_acc0 = wasm_i32x4_splat(0);\n v128_t v_acc1 = wasm_i32x4_splat(0);\n a = aux8;\n for (int j = 0; j < QK_K/16; ++j) {\n const v128_t v_scale = wasm_i16x8_splat(scales[j] - 32);\n\n // Process 16 elements per iteration\n for (int k = 0; k < 2; ++k) {\n const v128_t v_q8 = wasm_i16x8_load8x8(q8);\n const v128_t v_a = wasm_i16x8_load8x8(a);\n\n v128_t v_prod = wasm_i16x8_mul(v_q8, v_a);\n v_prod = wasm_i16x8_mul(v_prod, v_scale);\n\n v_acc0 = wasm_i32x4_add(v_acc0, wasm_i32x4_extend_low_i16x8(v_prod));\n v_acc1 = wasm_i32x4_add(v_acc1, wasm_i32x4_extend_high_i16x8(v_prod));\n\n q8 += 8;\n a += 8;\n }\n }\n\n // Accumulate results\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const v128_t v_d = wasm_f32x4_splat(d);\n v128_t v_sum = wasm_f32x4_add(\n wasm_f32x4_mul(wasm_f32x4_convert_i32x4(v_acc0), v_d),\n wasm_f32x4_mul(wasm_f32x4_convert_i32x4(v_acc1), v_d)\n );\n\n // Accumulate into sums vector\n wasm_v128_store(sums, wasm_f32x4_add(wasm_v128_load(sums), v_sum));\n }\n\n // Horizontal sum\n v128_t v_sum = wasm_f32x4_add(wasm_v128_load(sums), wasm_v128_load(sums + 4));\n sumf = wasm_f32x4_extract_lane(v_sum, 0) +\n wasm_f32x4_extract_lane(v_sum, 1) +\n wasm_f32x4_extract_lane(v_sum, 2) +\n wasm_f32x4_extract_lane(v_sum, 3);\n\n *s = sumf;\n\n#else\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined __wasm_simd128__\n const uint8_t * scales = (const uint8_t*)&utmp[0];\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); // Corrected sign\n\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n // Process scales and mins\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n // Sum mins * q8sums\n int32_t sumi = 0;\n const int16_t * GGML_RESTRICT q8sums = y[i].bsums;\n const uint8_t * m = (const uint8_t *)&utmp[2];\n for (int j = 0; j < 16; j += 2) {\n sumi += (q8sums[j] + q8sums[j+1]) * m[j/2];\n }\n sumf -= dmin * sumi;\n\n int32_t sumi1 = 0;\n int32_t sumi2 = 0;\n\n for (int j = 0; j < QK_K/64; ++j) {\n // Load 64 4-bit weights (32 bytes)\n const v128_t q4x0 = wasm_v128_load(q4);\n const v128_t q4x1 = wasm_v128_load(q4 + 16);\n q4 += 32;\n\n // Split into low/high nibbles\n const v128_t q4l0 = wasm_v128_and(q4x0, wasm_i8x16_splat(0x0F));\n const v128_t q4h0 = wasm_u8x16_shr(q4x0, 4);\n const v128_t q4l1 = wasm_v128_and(q4x1, wasm_i8x16_splat(0x0F));\n const v128_t q4h1 = wasm_u8x16_shr(q4x1, 4);\n\n // Load 64 8-bit values (64 bytes)\n const v128_t q8x0 = wasm_v128_load(q8);\n const v128_t q8x1 = wasm_v128_load(q8 + 16);\n const v128_t q8x2 = wasm_v128_load(q8 + 32);\n const v128_t q8x3 = wasm_v128_load(q8 + 48);\n q8 += 64;\n\n // Low nibble products\n v128_t vacc1 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q4l0),\n wasm_i16x8_extend_low_i8x16(q8x0)\n );\n vacc1 = wasm_i32x4_add(vacc1, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q4l0),\n wasm_i16x8_extend_high_i8x16(q8x0)\n ));\n vacc1 = wasm_i32x4_add(vacc1, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q4l1),\n wasm_i16x8_extend_low_i8x16(q8x1)\n ));\n vacc1 = wasm_i32x4_add(vacc1, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q4l1),\n wasm_i16x8_extend_high_i8x16(q8x1)\n ));\n\n // High nibble products\n v128_t vacc2 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q4h0),\n wasm_i16x8_extend_low_i8x16(q8x2)\n );\n vacc2 = wasm_i32x4_add(vacc2, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q4h0),\n wasm_i16x8_extend_high_i8x16(q8x2)\n ));\n vacc2 = wasm_i32x4_add(vacc2, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q4h1),\n wasm_i16x8_extend_low_i8x16(q8x3)\n ));\n vacc2 = wasm_i32x4_add(vacc2, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q4h1),\n wasm_i16x8_extend_high_i8x16(q8x3)\n ));\n\n // Accumulate scaled results\n int32_t vacc1_sum = wasm_i32x4_extract_lane(vacc1, 0) + wasm_i32x4_extract_lane(vacc1, 1) +\n wasm_i32x4_extract_lane(vacc1, 2) + wasm_i32x4_extract_lane(vacc1, 3);\n sumi1 += vacc1_sum * scales[2*j];\n\n int32_t vacc2_sum = wasm_i32x4_extract_lane(vacc2, 0) + wasm_i32x4_extract_lane(vacc2, 1) +\n wasm_i32x4_extract_lane(vacc2, 2) + wasm_i32x4_extract_lane(vacc2, 3);\n sumi2 += vacc2_sum * scales[2*j+1];\n }\n\n sumf += d * (sumi1 + sumi2);\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined __wasm_simd128__\n //const uint8_t * scales = (const uint8_t*)&utmp[0];\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); // Fixed sign\n\n const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n // Process scales and mins\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n // Sum mins * q8sums\n int32_t sumi_mins = 0;\n const int16_t * GGML_RESTRICT q8sums = y[i].bsums;\n const uint8_t * m = (const uint8_t *)&utmp[2];\n for (int j = 0; j < 16; j += 2) {\n sumi_mins += (q8sums[j] + q8sums[j+1]) * m[j/2];\n }\n sumf -= dmin * sumi_mins; // Correct subtraction\n\n v128_t qh0 = wasm_v128_load(qh);\n v128_t qh1 = wasm_v128_load(qh + 16);\n const uint8_t * sc = (const uint8_t *)utmp;\n\n int32_t sumi = 0;\n\n for (int j = 0; j < QK_K/64; ++j) {\n const int shift = j * 2;\n v128_t qh_shift0 = wasm_u8x16_shr(qh0, shift);\n v128_t qh_shift1 = wasm_u8x16_shr(qh1, shift);\n\n v128_t qh_low0 = wasm_i8x16_shl(wasm_v128_and(qh_shift0, wasm_i8x16_splat(0x01)), 4);\n v128_t qh_high0 = wasm_i8x16_shl(wasm_v128_and(qh_shift0, wasm_i8x16_splat(0x02)), 3);\n v128_t qh_low1 = wasm_i8x16_shl(wasm_v128_and(qh_shift1, wasm_i8x16_splat(0x01)), 4);\n v128_t qh_high1 = wasm_i8x16_shl(wasm_v128_and(qh_shift1, wasm_i8x16_splat(0x02)), 3);\n\n v128_t q5_0 = wasm_v128_load(q5);\n v128_t q5_1 = wasm_v128_load(q5 + 16);\n q5 += 32;\n\n v128_t q5l_0 = wasm_v128_or(wasm_v128_and(q5_0, wasm_i8x16_splat(0x0F)), qh_low0);\n v128_t q5h_0 = wasm_v128_or(wasm_u8x16_shr(q5_0, 4), qh_high0);\n v128_t q5l_1 = wasm_v128_or(wasm_v128_and(q5_1, wasm_i8x16_splat(0x0F)), qh_low1);\n v128_t q5h_1 = wasm_v128_or(wasm_u8x16_shr(q5_1, 4), qh_high1);\n\n v128_t q8_0 = wasm_v128_load(q8);\n v128_t q8_1 = wasm_v128_load(q8 + 16);\n v128_t q8_2 = wasm_v128_load(q8 + 32);\n v128_t q8_3 = wasm_v128_load(q8 + 48);\n q8 += 64;\n\n // Process low quants\n v128_t pl0 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q5l_0),\n wasm_i16x8_extend_low_i8x16(q8_0)\n );\n pl0 = wasm_i32x4_add(pl0, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q5l_0),\n wasm_i16x8_extend_high_i8x16(q8_0)\n ));\n v128_t pl1 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q5l_1),\n wasm_i16x8_extend_low_i8x16(q8_1)\n );\n pl1 = wasm_i32x4_add(pl1, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q5l_1),\n wasm_i16x8_extend_high_i8x16(q8_1)\n ));\n v128_t sum_low = wasm_i32x4_add(pl0, pl1);\n\n // Process high quants\n v128_t ph0 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q5h_0),\n wasm_i16x8_extend_low_i8x16(q8_2)\n );\n ph0 = wasm_i32x4_add(ph0, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q5h_0),\n wasm_i16x8_extend_high_i8x16(q8_2)\n ));\n v128_t ph1 = wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_low_i8x16(q5h_1),\n wasm_i16x8_extend_low_i8x16(q8_3)\n );\n ph1 = wasm_i32x4_add(ph1, wasm_i32x4_dot_i16x8(\n wasm_i16x8_extend_high_i8x16(q5h_1),\n wasm_i16x8_extend_high_i8x16(q8_3)\n ));\n v128_t sum_high = wasm_i32x4_add(ph0, ph1);\n\n // Accumulate with scale factors\n int32_t sl = wasm_i32x4_extract_lane(sum_low, 0) + wasm_i32x4_extract_lane(sum_low, 1) +\n wasm_i32x4_extract_lane(sum_low, 2) + wasm_i32x4_extract_lane(sum_low, 3);\n int32_t sh = wasm_i32x4_extract_lane(sum_high, 0) + wasm_i32x4_extract_lane(sum_high, 1) +\n wasm_i32x4_extract_lane(sum_high, 2) + wasm_i32x4_extract_lane(sum_high, 3);\n\n sumi += sl * sc[2*j] + sh * sc[2*j+1];\n }\n\n sumf += d * sumi;\n }\n\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q6_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __wasm_simd128__\n int8_t aux8[QK_K] __attribute__((aligned(16)));\n int32_t aux32[8] __attribute__((aligned(16))) = {0};\n float sums[8] __attribute__((aligned(16))) = {0};\n\n for (int i = 0; i < nb; ++i) {\n // Unpack 6-bit quantized data into aux8 (unchanged)\n const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n int8_t * a = aux8;\n for (int j = 0; j < QK_K; j += 128) {\n for (int l = 0; l < 32; ++l) {\n a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;\n a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;\n a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;\n a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;\n }\n a += 128;\n q4 += 64;\n qh += 32;\n }\n\n const int8_t * GGML_RESTRICT a_ptr = aux8;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n v128_t acc0 = wasm_i32x4_splat(0);\n v128_t acc1 = wasm_i32x4_splat(0);\n\n for (int j = 0; j < QK_K/16; ++j) {\n const int scale = x[i].scales[j];\n const v128_t vscale = wasm_i32x4_splat(scale);\n\n // Load 16 elements from a and q8\n const v128_t a_vec = wasm_v128_load(a_ptr);\n const v128_t q8_vec = wasm_v128_load(q8);\n\n // Process low 8 elements\n v128_t a_low = wasm_i16x8_extend_low_i8x16(a_vec);\n v128_t q8_low = wasm_i16x8_extend_low_i8x16(q8_vec);\n v128_t prod_low = wasm_i16x8_mul(a_low, q8_low);\n v128_t prod_lo_lo = wasm_i32x4_extend_low_i16x8(prod_low);\n v128_t prod_lo_hi = wasm_i32x4_extend_high_i16x8(prod_low);\n\n // Process high 8 elements\n v128_t a_high = wasm_i16x8_extend_high_i8x16(a_vec);\n v128_t q8_high = wasm_i16x8_extend_high_i8x16(q8_vec);\n v128_t prod_high = wasm_i16x8_mul(a_high, q8_high);\n v128_t prod_hi_lo = wasm_i32x4_extend_low_i16x8(prod_high);\n v128_t prod_hi_hi = wasm_i32x4_extend_high_i16x8(prod_high);\n\n // Scale and accumulate\n prod_lo_lo = wasm_i32x4_mul(prod_lo_lo, vscale);\n prod_lo_hi = wasm_i32x4_mul(prod_lo_hi, vscale);\n prod_hi_lo = wasm_i32x4_mul(prod_hi_lo, vscale);\n prod_hi_hi = wasm_i32x4_mul(prod_hi_hi, vscale);\n\n acc0 = wasm_i32x4_add(acc0, wasm_i32x4_add(prod_lo_lo, prod_hi_lo));\n acc1 = wasm_i32x4_add(acc1, wasm_i32x4_add(prod_lo_hi, prod_hi_hi));\n\n a_ptr += 16;\n q8 += 16;\n }\n\n // Store accumulated results\n wasm_v128_store(&aux32[0], acc0);\n wasm_v128_store(&aux32[4], acc1);\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n for (int l = 0; l < 8; ++l) {\n sums[l] += d * aux32[l];\n }\n }\n\n // Sum final results\n float sumf = 0;\n for (int l = 0; l < 8; ++l) {\n sumf += sums[l];\n }\n *s = sumf;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n" }, { "path": "src/ggml-cpu/arch/x86/cpu-feats.cpp", "content": "#include \"ggml-backend-impl.h\"\n\n#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))\n\n#ifdef _MSC_VER\n#include \n#endif\n\n#include \n#include \n#include \n#include \n#include \n\n// ref: https://cdrdv2-public.intel.com/782156/325383-sdm-vol-2abcd.pdf\nstruct cpuid_x86 {\n bool SSE3(void) { return f_1_ecx[0]; }\n bool PCLMULQDQ(void) { return f_1_ecx[1]; }\n bool MONITOR(void) { return f_1_ecx[3]; }\n bool SSSE3(void) { return f_1_ecx[9]; }\n bool FMA(void) { return f_1_ecx[12]; }\n bool CMPXCHG16B(void) { return f_1_ecx[13]; }\n bool SSE41(void) { return f_1_ecx[19]; }\n bool SSE42(void) { return f_1_ecx[20]; }\n bool MOVBE(void) { return f_1_ecx[22]; }\n bool POPCNT(void) { return f_1_ecx[23]; }\n bool AES(void) { return f_1_ecx[25]; }\n bool XSAVE(void) { return f_1_ecx[26]; }\n bool OSXSAVE(void) { return f_1_ecx[27]; }\n bool AVX(void) { return f_1_ecx[28]; }\n bool F16C(void) { return f_1_ecx[29]; }\n bool RDRAND(void) { return f_1_ecx[30]; }\n\n bool MSR(void) { return f_1_edx[5]; }\n bool CX8(void) { return f_1_edx[8]; }\n bool SEP(void) { return f_1_edx[11]; }\n bool CMOV(void) { return f_1_edx[15]; }\n bool CLFSH(void) { return f_1_edx[19]; }\n bool MMX(void) { return f_1_edx[23]; }\n bool FXSR(void) { return f_1_edx[24]; }\n bool SSE(void) { return f_1_edx[25]; }\n bool SSE2(void) { return f_1_edx[26]; }\n\n bool FSGSBASE(void) { return f_7_ebx[0]; }\n bool BMI1(void) { return f_7_ebx[3]; }\n bool HLE(void) { return is_intel && f_7_ebx[4]; }\n bool AVX2(void) { return f_7_ebx[5]; }\n bool BMI2(void) { return f_7_ebx[8]; }\n bool ERMS(void) { return f_7_ebx[9]; }\n bool INVPCID(void) { return f_7_ebx[10]; }\n bool RTM(void) { return is_intel && f_7_ebx[11]; }\n bool AVX512F(void) { return f_7_ebx[16]; }\n bool AVX512DQ(void) { return f_7_ebx[17]; }\n bool RDSEED(void) { return f_7_ebx[18]; }\n bool ADX(void) { return f_7_ebx[19]; }\n bool AVX512PF(void) { return f_7_ebx[26]; }\n bool AVX512ER(void) { return f_7_ebx[27]; }\n bool AVX512CD(void) { return f_7_ebx[28]; }\n bool AVX512BW(void) { return f_7_ebx[30]; }\n bool AVX512VL(void) { return f_7_ebx[31]; }\n\n bool SHA(void) { return f_7_ebx[29]; }\n\n bool PREFETCHWT1(void) { return f_7_ecx[0]; }\n\n bool LAHF(void) { return f_81_ecx[0]; }\n bool LZCNT(void) { return is_intel && f_81_ecx[5]; }\n bool ABM(void) { return is_amd && f_81_ecx[5]; }\n bool SSE4a(void) { return is_amd && f_81_ecx[6]; }\n bool XOP(void) { return is_amd && f_81_ecx[11]; }\n bool TBM(void) { return is_amd && f_81_ecx[21]; }\n\n bool SYSCALL(void) { return is_intel && f_81_edx[11]; }\n bool MMXEXT(void) { return is_amd && f_81_edx[22]; }\n bool RDTSCP(void) { return is_intel && f_81_edx[27]; }\n bool _3DNOWEXT(void) { return is_amd && f_81_edx[30]; }\n bool _3DNOW(void) { return is_amd && f_81_edx[31]; }\n\n bool AVX512_VBMI(void) { return f_7_ecx[1]; }\n bool AVX512_VNNI(void) { return f_7_ecx[11]; }\n bool AVX512_FP16(void) { return f_7_edx[23]; }\n bool AVX512_BF16(void) { return f_7_1_eax[5]; }\n bool AVX_VNNI(void) { return f_7_1_eax[4]; }\n\n bool AMX_TILE(void) { return f_7_edx[24]; }\n bool AMX_INT8(void) { return f_7_edx[25]; }\n bool AMX_FP16(void) { return f_7_1_eax[21]; }\n bool AMX_BF16(void) { return f_7_edx[22]; }\n\n#ifdef _MSC_VER\n static void cpuid(int cpu_info[4], int eax) {\n __cpuid(cpu_info, eax);\n }\n static void cpuidex(int cpu_info[4], int eax, int ecx) {\n __cpuidex(cpu_info, eax, ecx);\n }\n#else\n static void cpuid(int cpu_info[4], int eax) {\n __asm__ __volatile__(\n \"cpuid\"\n : \"=a\"(cpu_info[0]), \"=b\"(cpu_info[1]), \"=c\"(cpu_info[2]), \"=d\"(cpu_info[3])\n : \"a\"(eax), \"c\"(0));\n }\n static void cpuidex(int cpu_info[4], int eax, int ecx) {\n __asm__ __volatile__(\n \"cpuid\"\n : \"=a\"(cpu_info[0]), \"=b\"(cpu_info[1]), \"=c\"(cpu_info[2]), \"=d\"(cpu_info[3])\n : \"a\"(eax), \"c\"(ecx));\n }\n#endif\n\n cpuid_x86() {\n std::array cpui;\n std::vector> data;\n\n // calling __cpuid with 0x0 as the function_id argument\n // gets the number of the highest valid function ID.\n cpuid(cpui.data(), 0);\n int n_ids = cpui[0];\n\n for (int i = 0; i <= n_ids; ++i) {\n cpuidex(cpui.data(), i, 0);\n data.push_back(cpui);\n }\n\n // capture vendor string\n char vendor[0x20] = {};\n *reinterpret_cast(vendor) = data[0][1];\n *reinterpret_cast(vendor + 4) = data[0][3];\n *reinterpret_cast(vendor + 8) = data[0][2];\n this->vendor = vendor;\n if (this->vendor == \"GenuineIntel\") {\n is_intel = true;\n } else if (this->vendor == \"AuthenticAMD\") {\n is_amd = true;\n }\n\n // load bitset with flags for function 0x00000001\n if (n_ids >= 1) {\n f_1_ecx = data[1][2];\n f_1_edx = data[1][3];\n }\n\n // load bitset with flags for function 0x00000007\n if (n_ids >= 7) {\n f_7_ebx = data[7][1];\n f_7_ecx = data[7][2];\n f_7_edx = data[7][3];\n cpuidex(cpui.data(), 7, 1);\n f_7_1_eax = cpui[0];\n }\n\n // calling __cpuid with 0x80000000 as the function_id argument\n // gets the number of the highest valid extended ID.\n cpuid(cpui.data(), 0x80000000);\n unsigned int n_ex_ids = cpui[0];\n\n std::vector> ext_data;\n for (unsigned int i = 0x80000000; i <= n_ex_ids; ++i) {\n cpuidex(cpui.data(), i, 0);\n ext_data.push_back(cpui);\n }\n\n // load bitset with flags for function 0x80000001\n if (n_ex_ids >= 0x80000001) {\n f_81_ecx = ext_data[1][2];\n f_81_edx = ext_data[1][3];\n }\n\n // interpret CPU brand string if reported\n char brand[0x40] = {};\n if (n_ex_ids >= 0x80000004) {\n std::memcpy(brand, ext_data[2].data(), sizeof(cpui));\n std::memcpy(brand + 16, ext_data[3].data(), sizeof(cpui));\n std::memcpy(brand + 32, ext_data[4].data(), sizeof(cpui));\n this->brand = brand;\n }\n }\n\n bool is_intel = false;\n bool is_amd = false;\n std::string vendor;\n std::string brand;\n std::bitset<32> f_1_ecx;\n std::bitset<32> f_1_edx;\n std::bitset<32> f_7_ebx;\n std::bitset<32> f_7_ecx;\n std::bitset<32> f_7_edx;\n std::bitset<32> f_7_1_eax;\n std::bitset<32> f_81_ecx;\n std::bitset<32> f_81_edx;\n};\n\n#if 0\nvoid test_x86_is() {\n cpuid_x86 is;\n printf(\"CPU Vendor: %s\\n\", is.vendor.c_str());\n printf(\"Brand: %s\\n\", is.brand.c_str());\n printf(\"is_intel: %d\\n\", is.is_intel);\n printf(\"is_amd: %d\\n\", is.is_amd);\n printf(\"sse3: %d\\n\", is.SSE3());\n printf(\"pclmulqdq: %d\\n\", is.PCLMULQDQ());\n printf(\"ssse3: %d\\n\", is.SSSE3());\n printf(\"fma: %d\\n\", is.FMA());\n printf(\"cmpxchg16b: %d\\n\", is.CMPXCHG16B());\n printf(\"sse41: %d\\n\", is.SSE41());\n printf(\"sse42: %d\\n\", is.SSE42());\n printf(\"movbe: %d\\n\", is.MOVBE());\n printf(\"popcnt: %d\\n\", is.POPCNT());\n printf(\"aes: %d\\n\", is.AES());\n printf(\"xsave: %d\\n\", is.XSAVE());\n printf(\"osxsave: %d\\n\", is.OSXSAVE());\n printf(\"avx: %d\\n\", is.AVX());\n printf(\"f16c: %d\\n\", is.F16C());\n printf(\"rdrand: %d\\n\", is.RDRAND());\n printf(\"msr: %d\\n\", is.MSR());\n printf(\"cx8: %d\\n\", is.CX8());\n printf(\"sep: %d\\n\", is.SEP());\n printf(\"cmov: %d\\n\", is.CMOV());\n printf(\"clflush: %d\\n\", is.CLFSH());\n printf(\"mmx: %d\\n\", is.MMX());\n printf(\"fxsr: %d\\n\", is.FXSR());\n printf(\"sse: %d\\n\", is.SSE());\n printf(\"sse2: %d\\n\", is.SSE2());\n printf(\"fsgsbase: %d\\n\", is.FSGSBASE());\n printf(\"bmi1: %d\\n\", is.BMI1());\n printf(\"hle: %d\\n\", is.HLE());\n printf(\"avx2: %d\\n\", is.AVX2());\n printf(\"bmi2: %d\\n\", is.BMI2());\n printf(\"erms: %d\\n\", is.ERMS());\n printf(\"invpcid: %d\\n\", is.INVPCID());\n printf(\"rtm: %d\\n\", is.RTM());\n printf(\"avx512f: %d\\n\", is.AVX512F());\n printf(\"rdseed: %d\\n\", is.RDSEED());\n printf(\"adx: %d\\n\", is.ADX());\n printf(\"avx512pf: %d\\n\", is.AVX512PF());\n printf(\"avx512er: %d\\n\", is.AVX512ER());\n printf(\"avx512cd: %d\\n\", is.AVX512CD());\n printf(\"sha: %d\\n\", is.SHA());\n printf(\"prefetchwt1: %d\\n\", is.PREFETCHWT1());\n printf(\"lahf: %d\\n\", is.LAHF());\n printf(\"lzcnt: %d\\n\", is.LZCNT());\n printf(\"abm: %d\\n\", is.ABM());\n printf(\"sse4a: %d\\n\", is.SSE4a());\n printf(\"xop: %d\\n\", is.XOP());\n printf(\"tbm: %d\\n\", is.TBM());\n printf(\"syscall: %d\\n\", is.SYSCALL());\n printf(\"mmxext: %d\\n\", is.MMXEXT());\n printf(\"rdtscp: %d\\n\", is.RDTSCP());\n printf(\"3dnowext: %d\\n\", is._3DNOWEXT());\n printf(\"3dnow: %d\\n\", is._3DNOW());\n printf(\"avx512_vbmi: %d\\n\", is.AVX512_VBMI());\n printf(\"avx512_vnni: %d\\n\", is.AVX512_VNNI());\n printf(\"avx512_fp16: %d\\n\", is.AVX512_FP16());\n printf(\"avx512_bf16: %d\\n\", is.AVX512_BF16());\n printf(\"amx_tile: %d\\n\", is.AMX_TILE());\n printf(\"amx_int8: %d\\n\", is.AMX_INT8());\n printf(\"amx_fp16: %d\\n\", is.AMX_FP16());\n printf(\"amx_bf16: %d\\n\", is.AMX_BF16());\n}\n#endif\n\nstatic int ggml_backend_cpu_x86_score() {\n // FIXME: this does not check for OS support\n\n int score = 1;\n cpuid_x86 is;\n\n#ifdef GGML_FMA\n if (!is.FMA()) { return 0; }\n score += 1;\n#endif\n#ifdef GGML_F16C\n if (!is.F16C()) { return 0; }\n score += 1<<1;\n#endif\n#ifdef GGML_SSE42\n if (!is.SSE42()) { return 0; }\n score += 1<<2;\n#endif\n#ifdef GGML_BMI2\n if (!is.BMI2()) { return 0; }\n score += 1<<3;\n#endif\n#ifdef GGML_AVX\n if (!is.AVX()) { return 0; }\n score += 1<<4;\n#endif\n#ifdef GGML_AVX2\n if (!is.AVX2()) { return 0; }\n score += 1<<5;\n#endif\n#ifdef GGML_AVX_VNNI\n if (!is.AVX_VNNI()) { return 0; }\n score += 1<<6;\n#endif\n#ifdef GGML_AVX512\n if (!is.AVX512F()) { return 0; }\n if (!is.AVX512CD()) { return 0; }\n if (!is.AVX512VL()) { return 0; }\n if (!is.AVX512DQ()) { return 0; }\n if (!is.AVX512BW()) { return 0; }\n score += 1<<7;\n#endif\n#ifdef GGML_AVX512_VBMI\n if (!is.AVX512_VBMI()) { return 0; }\n score += 1<<8;\n#endif\n#ifdef GGML_AVX512_BF16\n if (!is.AVX512_BF16()) { return 0; }\n score += 1<<9;\n#endif\n#ifdef GGML_AVX512_VNNI\n if (!is.AVX512_VNNI()) { return 0; }\n score += 1<<10;\n#endif\n#ifdef GGML_AMX_INT8\n if (!is.AMX_INT8()) { return 0; }\n score += 1<<11;\n#endif\n\n return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_x86_score)\n\n#endif // defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))\n" }, { "path": "src/ggml-cpu/arch/x86/quants.c", "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n// some compilers don't provide _mm256_set_m128i, e.g. gcc 7\n#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)\n\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)\n// multiply int8_t, add results pairwise twice\nstatic inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {\n // Get absolute values of x vectors\n const __m128i ax = _mm_sign_epi8(x, x);\n // Sign the values of the y vectors\n const __m128i sy = _mm_sign_epi8(y, x);\n // Perform multiplication and create 16-bit values\n const __m128i dot = _mm_maddubs_epi16(ax, sy);\n const __m128i ones = _mm_set1_epi16(1);\n return _mm_madd_epi16(ones, dot);\n}\n\n#if __AVX__ || __AVX2__ || __AVX512F__\n// horizontally add 8 floats\nstatic inline float hsum_float_8(const __m256 x) {\n __m128 res = _mm256_extractf128_ps(x, 1);\n res = _mm_add_ps(res, _mm256_castps256_ps128(x));\n res = _mm_add_ps(res, _mm_movehl_ps(res, res));\n res = _mm_add_ss(res, _mm_movehdup_ps(res));\n return _mm_cvtss_f32(res);\n}\n\n// horizontally add 8 int32_t\nstatic inline int hsum_i32_8(const __m256i a) {\n const __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));\n const __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);\n const __m128i sum64 = _mm_add_epi32(hi64, sum128);\n const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));\n return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));\n}\n\n// horizontally add 4 int32_t\nstatic inline int hsum_i32_4(const __m128i a) {\n const __m128i hi64 = _mm_unpackhi_epi64(a, a);\n const __m128i sum64 = _mm_add_epi32(hi64, a);\n const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));\n return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));\n}\n\n#if defined(__AVX2__) || defined(__AVX512F__)\nstatic inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {\n const __m256i ax = _mm256_sign_epi8(x, x);\n const __m256i sy = _mm256_sign_epi8(y, x);\n return _mm256_maddubs_epi16(ax, sy);\n}\n\n// spread 32 bits to 32 bytes { 0x00, 0xFF }\nstatic inline __m256i bytes_from_bits_32(const uint8_t * x) {\n uint32_t x32;\n memcpy(&x32, x, sizeof(uint32_t));\n const __m256i shuf_mask = _mm256_set_epi64x(\n 0x0303030303030303, 0x0202020202020202,\n 0x0101010101010101, 0x0000000000000000);\n __m256i bytes = _mm256_shuffle_epi8(_mm256_set1_epi32(x32), shuf_mask);\n const __m256i bit_mask = _mm256_set1_epi64x(0x7fbfdfeff7fbfdfe);\n bytes = _mm256_or_si256(bytes, bit_mask);\n return _mm256_cmpeq_epi8(bytes, _mm256_set1_epi64x(-1));\n}\n\n// Unpack 32 4-bit fields into 32 bytes\n// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval\nstatic inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)\n{\n const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);\n const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);\n const __m256i lowMask = _mm256_set1_epi8( 0xF );\n return _mm256_and_si256(lowMask, bytes);\n}\n\n// add int16_t pairwise and return as float vector\nstatic inline __m256 sum_i16_pairs_float(const __m256i x) {\n const __m256i ones = _mm256_set1_epi16(1);\n const __m256i summed_pairs = _mm256_madd_epi16(ones, x);\n return _mm256_cvtepi32_ps(summed_pairs);\n}\n\nstatic inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {\n#if defined(__AVX512VNNI__) && defined(__AVX512VL__)\n const __m256i zero = _mm256_setzero_si256();\n const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);\n return _mm256_cvtepi32_ps(summed_pairs);\n#elif defined(__AVXVNNI__)\n const __m256i zero = _mm256_setzero_si256();\n const __m256i summed_pairs = _mm256_dpbusd_avx_epi32(zero, ax, sy);\n return _mm256_cvtepi32_ps(summed_pairs);\n#else\n // Perform multiplication and create 16-bit values\n const __m256i dot = _mm256_maddubs_epi16(ax, sy);\n return sum_i16_pairs_float(dot);\n#endif\n}\n\n// multiply int8_t, add results pairwise twice and return as float vector\nstatic inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {\n#if __AVXVNNIINT8__\n const __m256i zero = _mm256_setzero_si256();\n const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);\n return _mm256_cvtepi32_ps(summed_pairs);\n#else\n // Get absolute values of x vectors\n const __m256i ax = _mm256_sign_epi8(x, x);\n // Sign the values of the y vectors\n const __m256i sy = _mm256_sign_epi8(y, x);\n return mul_sum_us8_pairs_float(ax, sy);\n#endif\n}\n\nstatic inline __m128i packNibbles( __m256i bytes )\n{\n // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh\n#if __AVX512F__\n const __m256i bytes_srli_4 = _mm256_srli_epi16(bytes, 4); // 0000_0000_abcd_0000\n bytes = _mm256_or_si256(bytes, bytes_srli_4); // 0000_abcd_abcd_efgh\n return _mm256_cvtepi16_epi8(bytes); // abcd_efgh\n#else\n const __m256i lowByte = _mm256_set1_epi16( 0xFF );\n __m256i high = _mm256_andnot_si256( lowByte, bytes );\n __m256i low = _mm256_and_si256( lowByte, bytes );\n high = _mm256_srli_epi16( high, 4 );\n bytes = _mm256_or_si256( low, high );\n\n // Compress uint16_t lanes into bytes\n __m128i r0 = _mm256_castsi256_si128( bytes );\n __m128i r1 = _mm256_extracti128_si256( bytes, 1 );\n return _mm_packus_epi16( r0, r1 );\n#endif\n}\n#elif defined(__AVX__)\nstatic inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 )\n{\n // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh\n const __m128i lowByte = _mm_set1_epi16( 0xFF );\n __m128i high = _mm_andnot_si128( lowByte, bytes1 );\n __m128i low = _mm_and_si128( lowByte, bytes1 );\n high = _mm_srli_epi16( high, 4 );\n bytes1 = _mm_or_si128( low, high );\n high = _mm_andnot_si128( lowByte, bytes2 );\n low = _mm_and_si128( lowByte, bytes2 );\n high = _mm_srli_epi16( high, 4 );\n bytes2 = _mm_or_si128( low, high );\n\n return _mm_packus_epi16( bytes1, bytes2);\n}\n\nstatic inline __m128i mul_add_epi8_sse(const __m128i x, const __m128i y) {\n const __m128i ax = _mm_sign_epi8(x, x);\n const __m128i sy = _mm_sign_epi8(y, x);\n return _mm_maddubs_epi16(ax, sy);\n}\n\n// spread 32 bits to 32 bytes { 0x00, 0xFF }\nstatic inline __m256i bytes_from_bits_32(const uint8_t * x) {\n uint32_t x32;\n memcpy(&x32, x, sizeof(uint32_t));\n const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);\n const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);\n __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);\n __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);\n const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);\n bytesl = _mm_or_si128(bytesl, bit_mask);\n bytesh = _mm_or_si128(bytesh, bit_mask);\n bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));\n bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));\n return MM256_SET_M128I(bytesh, bytesl);\n}\n\n// Unpack 32 4-bit fields into 32 bytes\n// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval\nstatic inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)\n{\n // Load 16 bytes from memory\n __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);\n __m128i tmph = _mm_srli_epi16(tmpl, 4);\n const __m128i lowMask = _mm_set1_epi8(0xF);\n tmpl = _mm_and_si128(lowMask, tmpl);\n tmph = _mm_and_si128(lowMask, tmph);\n return MM256_SET_M128I(tmph, tmpl);\n}\n\n// add int16_t pairwise and return as float vector\nstatic inline __m256 sum_i16_pairs_float(const __m128i xh, const __m128i xl) {\n const __m128i ones = _mm_set1_epi16(1);\n const __m128i summed_pairsl = _mm_madd_epi16(ones, xl);\n const __m128i summed_pairsh = _mm_madd_epi16(ones, xh);\n const __m256i summed_pairs = MM256_SET_M128I(summed_pairsh, summed_pairsl);\n return _mm256_cvtepi32_ps(summed_pairs);\n}\n\nstatic inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {\n const __m128i axl = _mm256_castsi256_si128(ax);\n const __m128i axh = _mm256_extractf128_si256(ax, 1);\n const __m128i syl = _mm256_castsi256_si128(sy);\n const __m128i syh = _mm256_extractf128_si256(sy, 1);\n // Perform multiplication and create 16-bit values\n const __m128i dotl = _mm_maddubs_epi16(axl, syl);\n const __m128i doth = _mm_maddubs_epi16(axh, syh);\n return sum_i16_pairs_float(doth, dotl);\n}\n\n// multiply int8_t, add results pairwise twice and return as float vector\nstatic inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {\n const __m128i xl = _mm256_castsi256_si128(x);\n const __m128i xh = _mm256_extractf128_si256(x, 1);\n const __m128i yl = _mm256_castsi256_si128(y);\n const __m128i yh = _mm256_extractf128_si256(y, 1);\n // Get absolute values of x vectors\n const __m128i axl = _mm_sign_epi8(xl, xl);\n const __m128i axh = _mm_sign_epi8(xh, xh);\n // Sign the values of the y vectors\n const __m128i syl = _mm_sign_epi8(yl, xl);\n const __m128i syh = _mm_sign_epi8(yh, xh);\n // Perform multiplication and create 16-bit values\n const __m128i dotl = _mm_maddubs_epi16(axl, syl);\n const __m128i doth = _mm_maddubs_epi16(axh, syh);\n return sum_i16_pairs_float(doth, dotl);\n}\n\n// larger version of mul_sum_i8_pairs_float where x and y are each represented by four 128-bit vectors\nstatic inline __m256 mul_sum_i8_quad_float(const __m128i x_1_0, const __m128i x_1_1, const __m128i x_2_0, const __m128i x_2_1,\n const __m128i y_1_0, const __m128i y_1_1, const __m128i y_2_0, const __m128i y_2_1) {\n const __m128i mone = _mm_set1_epi16(1);\n\n const __m128i p16_1_0 = mul_add_epi8_sse(x_1_0, y_1_0);\n const __m128i p16_1_1 = mul_add_epi8_sse(x_1_1, y_1_1);\n const __m128i p16_2_0 = mul_add_epi8_sse(x_2_0, y_2_0);\n const __m128i p16_2_1 = mul_add_epi8_sse(x_2_1, y_2_1);\n const __m128i p_1_0 = _mm_madd_epi16(p16_1_0, mone);\n const __m128i p_1_1 = _mm_madd_epi16(p16_1_1, mone);\n const __m128i p_2_0 = _mm_madd_epi16(p16_2_0, mone);\n const __m128i p_2_1 = _mm_madd_epi16(p16_2_1, mone);\n const __m128i p_1 = _mm_add_epi32(p_1_0, p_1_1);\n const __m128i p_2 = _mm_add_epi32(p_2_0, p_2_1);\n return _mm256_cvtepi32_ps(MM256_SET_M128I(p_2, p_1));\n}\n\n// quad fp16 delta calculation\nstatic inline __m256 quad_fp16_delta_float(const float x0, const float y0, const float x1, const float y1) {\n // GGML_CPU_FP16_TO_FP32 is faster than Intel F16C\n return _mm256_set_m128(_mm_set1_ps(GGML_CPU_FP16_TO_FP32(x1) * GGML_CPU_FP16_TO_FP32(y1)),\n _mm_set1_ps(GGML_CPU_FP16_TO_FP32(x0) * GGML_CPU_FP16_TO_FP32(y0)));\n}\n\nstatic inline __m256 quad_mx_delta_float(const uint8_t x0, const float y0, const uint8_t x1, const float y1) {\n return _mm256_set_m128(_mm_set1_ps(GGML_CPU_E8M0_TO_FP32_HALF(x1) * GGML_CPU_FP16_TO_FP32(y1)),\n _mm_set1_ps(GGML_CPU_E8M0_TO_FP32_HALF(x0) * GGML_CPU_FP16_TO_FP32(y0)));\n}\n#endif\n#elif defined(__SSSE3__)\n// horizontally add 4x4 floats\nstatic inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) {\n __m128 res_0 =_mm_hadd_ps(a, b);\n __m128 res_1 =_mm_hadd_ps(c, d);\n __m128 res =_mm_hadd_ps(res_0, res_1);\n res =_mm_hadd_ps(res, res);\n res =_mm_hadd_ps(res, res);\n\n return _mm_cvtss_f32(res);\n}\n#endif // __AVX__ || __AVX2__ || __AVX512F__\n#endif // defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__AVX2__) || defined(__AVX__)\n for (int i = 0; i < nb; i++) {\n // Load elements into 4 AVX vectors\n __m256 v0 = _mm256_loadu_ps( x );\n __m256 v1 = _mm256_loadu_ps( x + 8 );\n __m256 v2 = _mm256_loadu_ps( x + 16 );\n __m256 v3 = _mm256_loadu_ps( x + 24 );\n x += 32;\n\n // Compute max(abs(e)) for the block\n const __m256 signBit = _mm256_set1_ps( -0.0f );\n __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );\n maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );\n maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );\n maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );\n\n __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );\n max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );\n max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );\n const float maxScalar = _mm_cvtss_f32( max4 );\n\n // Quantize these floats\n const float d = maxScalar / 127.f;\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;\n const __m256 mul = _mm256_set1_ps( id );\n\n // Apply the multiplier\n v0 = _mm256_mul_ps( v0, mul );\n v1 = _mm256_mul_ps( v1, mul );\n v2 = _mm256_mul_ps( v2, mul );\n v3 = _mm256_mul_ps( v3, mul );\n\n // Round to nearest integer\n v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );\n v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );\n v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );\n v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );\n\n // Convert floats to integers\n __m256i i0 = _mm256_cvtps_epi32( v0 );\n __m256i i1 = _mm256_cvtps_epi32( v1 );\n __m256i i2 = _mm256_cvtps_epi32( v2 );\n __m256i i3 = _mm256_cvtps_epi32( v3 );\n\n#if defined(__AVX2__)\n // Convert int32 to int16\n i0 = _mm256_packs_epi32( i0, i1 );\t// 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15\n i2 = _mm256_packs_epi32( i2, i3 );\t// 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31\n // Convert int16 to int8\n i0 = _mm256_packs_epi16( i0, i2 );\t// 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31\n\n // We got our precious signed bytes, but the order is now wrong\n // These AVX2 pack instructions process 16-byte pieces independently\n // The following instruction is fixing the order\n const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );\n i0 = _mm256_permutevar8x32_epi32( i0, perm );\n\n _mm256_storeu_si256((__m256i *)y[i].qs, i0);\n#else\n // Since we don't have in AVX some necessary functions,\n // we split the registers in half and call AVX2 analogs from SSE\n __m128i ni0 = _mm256_castsi256_si128( i0 );\n __m128i ni1 = _mm256_extractf128_si256( i0, 1);\n __m128i ni2 = _mm256_castsi256_si128( i1 );\n __m128i ni3 = _mm256_extractf128_si256( i1, 1);\n __m128i ni4 = _mm256_castsi256_si128( i2 );\n __m128i ni5 = _mm256_extractf128_si256( i2, 1);\n __m128i ni6 = _mm256_castsi256_si128( i3 );\n __m128i ni7 = _mm256_extractf128_si256( i3, 1);\n\n // Convert int32 to int16\n ni0 = _mm_packs_epi32( ni0, ni1 );\n ni2 = _mm_packs_epi32( ni2, ni3 );\n ni4 = _mm_packs_epi32( ni4, ni5 );\n ni6 = _mm_packs_epi32( ni6, ni7 );\n // Convert int16 to int8\n ni0 = _mm_packs_epi16( ni0, ni2 );\n ni4 = _mm_packs_epi16( ni4, ni6 );\n\n _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);\n _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);\n#endif\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK8_1 == 0);\n const int nb = k / QK8_1;\n\n block_q8_1 * GGML_RESTRICT y = vy;\n#if defined(__AVX2__) || defined(__AVX__)\n for (int i = 0; i < nb; i++) {\n // Load elements into 4 AVX vectors\n __m256 v0 = _mm256_loadu_ps( x );\n __m256 v1 = _mm256_loadu_ps( x + 8 );\n __m256 v2 = _mm256_loadu_ps( x + 16 );\n __m256 v3 = _mm256_loadu_ps( x + 24 );\n x += 32;\n\n // Compute max(abs(e)) for the block\n const __m256 signBit = _mm256_set1_ps( -0.0f );\n __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );\n maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );\n maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );\n maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );\n\n __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );\n max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );\n max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );\n const float max_scalar = _mm_cvtss_f32( max4 );\n\n // Quantize these floats\n const float d = max_scalar / 127.f;\n y[i].d = GGML_CPU_FP32_TO_FP16(d);\n const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;\n const __m256 mul = _mm256_set1_ps( id );\n\n // Apply the multiplier\n v0 = _mm256_mul_ps( v0, mul );\n v1 = _mm256_mul_ps( v1, mul );\n v2 = _mm256_mul_ps( v2, mul );\n v3 = _mm256_mul_ps( v3, mul );\n\n // Round to nearest integer\n v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );\n v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );\n v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );\n v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );\n\n // Convert floats to integers\n __m256i i0 = _mm256_cvtps_epi32( v0 );\n __m256i i1 = _mm256_cvtps_epi32( v1 );\n __m256i i2 = _mm256_cvtps_epi32( v2 );\n __m256i i3 = _mm256_cvtps_epi32( v3 );\n\n#if defined(__AVX2__)\n // Compute the sum of the quants and set y[i].s\n y[i].s = GGML_CPU_FP32_TO_FP16(d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))));\n\n // Convert int32 to int16\n i0 = _mm256_packs_epi32( i0, i1 );\t// 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15\n i2 = _mm256_packs_epi32( i2, i3 );\t// 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31\n // Convert int16 to int8\n i0 = _mm256_packs_epi16( i0, i2 );\t// 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31\n\n // We got our precious signed bytes, but the order is now wrong\n // These AVX2 pack instructions process 16-byte pieces independently\n // The following instruction is fixing the order\n const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );\n i0 = _mm256_permutevar8x32_epi32( i0, perm );\n\n _mm256_storeu_si256((__m256i *)y[i].qs, i0);\n#else\n // Since we don't have in AVX some necessary functions,\n // we split the registers in half and call AVX2 analogs from SSE\n __m128i ni0 = _mm256_castsi256_si128( i0 );\n __m128i ni1 = _mm256_extractf128_si256( i0, 1);\n __m128i ni2 = _mm256_castsi256_si128( i1 );\n __m128i ni3 = _mm256_extractf128_si256( i1, 1);\n __m128i ni4 = _mm256_castsi256_si128( i2 );\n __m128i ni5 = _mm256_extractf128_si256( i2, 1);\n __m128i ni6 = _mm256_castsi256_si128( i3 );\n __m128i ni7 = _mm256_extractf128_si256( i3, 1);\n\n // Compute the sum of the quants and set y[i].s\n const __m128i s0 = _mm_add_epi32(_mm_add_epi32(ni0, ni1), _mm_add_epi32(ni2, ni3));\n const __m128i s1 = _mm_add_epi32(_mm_add_epi32(ni4, ni5), _mm_add_epi32(ni6, ni7));\n y[i].s = GGML_CPU_FP32_TO_FP16(d * hsum_i32_4(_mm_add_epi32(s0, s1)));\n\n // Convert int32 to int16\n ni0 = _mm_packs_epi32( ni0, ni1 );\n ni2 = _mm_packs_epi32( ni2, ni3 );\n ni4 = _mm_packs_epi32( ni4, ni5 );\n ni6 = _mm_packs_epi32( ni6, ni7 );\n // Convert int16 to int8\n ni0 = _mm_packs_epi16( ni0, ni2 );\n ni4 = _mm_packs_epi16( ni4, ni6 );\n\n _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);\n _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);\n#endif\n }\n#else\n GGML_UNUSED(nb);\n // scalar\n quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n// placeholder implementation for Apple targets\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_q8_K_ref(x, y, k);\n}\n\n//===================================== Dot products =================================\n\n//\n// Helper functions\n//\n\n#if __AVX__ || __AVX2__ || __AVX512F__\n\n// shuffles to pick the required scales in dot products\nstatic inline __m256i get_scale_shuffle_q3k(int i) {\n static const uint8_t k_shuffle[128] = {\n 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,\n 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,\n 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,\n };\n return _mm256_loadu_si256((const __m256i*)k_shuffle + i);\n}\nstatic inline __m256i get_scale_shuffle_k4(int i) {\n static const uint8_t k_shuffle[256] = {\n 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,\n 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,\n 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,\n 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,\n 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,\n 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15\n };\n return _mm256_loadu_si256((const __m256i*)k_shuffle + i);\n}\nstatic inline __m128i get_scale_shuffle(int i) {\n static const uint8_t k_shuffle[128] = {\n 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,\n 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,\n 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,\n 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,\n 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11,\n 12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13,\n 14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15\n };\n return _mm_loadu_si128((const __m128i*)k_shuffle + i);\n}\n#endif\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__AVX2__)\n // Initialize accumulator with zeros\n __m256 acc = _mm256_setzero_ps();\n\n // Main loop\n for (; ib < nb; ++ib) {\n /* Compute combined scale for the block */\n const __m256 d = _mm256_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );\n\n __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n\n // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.\n const __m256i off = _mm256_set1_epi8( 8 );\n qx = _mm256_sub_epi8( qx, off );\n\n __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n /* Multiply q with scale and accumulate */\n acc = _mm256_fmadd_ps( d, q, acc );\n }\n\n sumf = hsum_float_8(acc);\n#elif defined(__AVX__)\n __m256 accum = _mm256_setzero_ps();\n for (; ib + 1 < nb; ib += 2) {\n const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);\n const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);\n const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);\n const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);\n\n const __m128i q4b_1_0 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), q4bits_1), _mm_set1_epi8(8));\n const __m128i q4b_1_1 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(q4bits_1, 4)), _mm_set1_epi8(8));\n const __m128i q4b_2_0 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), q4bits_2), _mm_set1_epi8(8));\n const __m128i q4b_2_1 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(q4bits_2, 4)), _mm_set1_epi8(8));\n\n const __m128i p16_1_0 = mul_add_epi8_sse(q4b_1_0, q8b_1_0);\n const __m128i p16_1_1 = mul_add_epi8_sse(q4b_1_1, q8b_1_1);\n const __m128i p16_2_0 = mul_add_epi8_sse(q4b_2_0, q8b_2_0);\n const __m128i p16_2_1 = mul_add_epi8_sse(q4b_2_1, q8b_2_1);\n const __m128i p_1 = _mm_add_epi16(p16_1_0, p16_1_1);\n const __m128i p_2 = _mm_add_epi16(p16_2_0, p16_2_1);\n const __m256 p = sum_i16_pairs_float(p_2, p_1);\n\n const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d);\n accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);\n }\n\n sumf = hsum_float_8(accum);\n#elif defined(__SSSE3__)\n // set constants\n const __m128i lowMask = _mm_set1_epi8(0xF);\n const __m128i off = _mm_set1_epi8(8);\n\n // Initialize accumulator with zeros\n __m128 acc_0 = _mm_setzero_ps();\n __m128 acc_1 = _mm_setzero_ps();\n __m128 acc_2 = _mm_setzero_ps();\n __m128 acc_3 = _mm_setzero_ps();\n\n for (; ib + 1 < nb; ib += 2) {\n _mm_prefetch(&x[ib] + sizeof(block_q4_0), _MM_HINT_T0);\n _mm_prefetch(&y[ib] + sizeof(block_q8_0), _MM_HINT_T0);\n\n // Compute combined scale for the block 0 and 1\n const __m128 d_0_1 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );\n\n const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[ib].qs);\n\n __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);\n __m128i by_0 = _mm_loadu_si128((const __m128i *)y[ib].qs);\n bx_0 = _mm_sub_epi8(bx_0, off);\n const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);\n\n __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));\n __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[ib].qs + 16));\n bx_1 = _mm_sub_epi8(bx_1, off);\n const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);\n\n _mm_prefetch(&x[ib] + 2 * sizeof(block_q4_0), _MM_HINT_T0);\n _mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0);\n\n // Compute combined scale for the block 2 and 3\n const __m128 d_2_3 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) );\n\n const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n\n __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);\n __m128i by_2 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n bx_2 = _mm_sub_epi8(bx_2, off);\n const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);\n\n __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));\n __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[ib + 1].qs + 16));\n bx_3 = _mm_sub_epi8(bx_3, off);\n const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);\n\n // Convert int32_t to float\n __m128 p0 = _mm_cvtepi32_ps(i32_0);\n __m128 p1 = _mm_cvtepi32_ps(i32_1);\n __m128 p2 = _mm_cvtepi32_ps(i32_2);\n __m128 p3 = _mm_cvtepi32_ps(i32_3);\n\n // Apply the scale\n __m128 p0_d = _mm_mul_ps( d_0_1, p0 );\n __m128 p1_d = _mm_mul_ps( d_0_1, p1 );\n __m128 p2_d = _mm_mul_ps( d_2_3, p2 );\n __m128 p3_d = _mm_mul_ps( d_2_3, p3 );\n\n // Acummulate\n acc_0 = _mm_add_ps(p0_d, acc_0);\n acc_1 = _mm_add_ps(p1_d, acc_1);\n acc_2 = _mm_add_ps(p2_d, acc_2);\n acc_3 = _mm_add_ps(p3_d, acc_3);\n }\n\n sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);\n\n#endif\n for (; ib < nb; ++ib) {\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n const int v1 = (x[ib].qs[j] >> 4) - 8;\n\n sumi0 += (v0 * y[ib].qs[j]);\n sumi1 += (v1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n\n#if defined(__AVX2__) || defined(__AVX__)\n // Initialize accumulator with zeros\n __m256 acc = _mm256_setzero_ps();\n\n float summs = 0;\n\n // Main loop\n for (; ib < nb; ++ib) {\n const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);\n const float d1 = GGML_CPU_FP16_TO_FP32(y[ib].d);\n\n summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n const __m256 d0v = _mm256_set1_ps( d0 );\n const __m256 d1v = _mm256_set1_ps( d1 );\n\n // Compute combined scales\n const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );\n\n // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes\n const __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[ib].qs );\n\n const __m256 xy = mul_sum_us8_pairs_float(qx, qy);\n\n // Accumulate d0*d1*x*y\n#if defined(__AVX2__)\n acc = _mm256_fmadd_ps( d0d1, xy, acc );\n#else\n acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );\n#endif\n }\n\n *s = hsum_float_8(acc) + summs;\n#else\n UNUSED(nb);\n UNUSED(x);\n UNUSED(y);\n UNUSED(ib);\n ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_MXFP4 == 0);\n static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n const block_mxfp4 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_MXFP4;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined __AVX2__\n\n const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4);\n const __m128i m4b = _mm_set1_epi8(0x0f);\n const __m256i mone = _mm256_set1_epi16(1);\n\n __m256 accum1 = _mm256_setzero_ps();\n __m256 accum2 = _mm256_setzero_ps();\n\n for (; ib + 1 < nb; ib += 2) {\n const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[ib + 0].qs);\n const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs);\n const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[ib + 0].qs);\n const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[ib + 1].qs);\n const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),\n _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));\n const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),\n _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));\n const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n const __m256i p_1 = _mm256_madd_epi16(p16_1, mone);\n const __m256i p_2 = _mm256_madd_epi16(p16_2, mone);\n const __m256 scale0 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_E8M0_TO_FP32_HALF(x[ib + 0].e));\n const __m256 scale1 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_E8M0_TO_FP32_HALF(x[ib + 1].e));\n accum1 = _mm256_fmadd_ps(scale0, _mm256_cvtepi32_ps(p_1), accum1);\n accum2 = _mm256_fmadd_ps(scale1, _mm256_cvtepi32_ps(p_2), accum2);\n }\n\n sumf = hsum_float_8(_mm256_add_ps(accum1, accum2));\n\n#elif defined __AVX__\n const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4);\n const __m128i m4b = _mm_set1_epi8(0x0f);\n\n __m256 accum = _mm256_setzero_ps();\n for (; ib + 1 < nb; ib += 2) {\n const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);\n const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);\n const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);\n const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);\n\n const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));\n const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));\n const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));\n const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));\n\n const __m256 p = mul_sum_i8_quad_float(q4b_1_0, q4b_1_1, q4b_2_0, q4b_2_1, q8b_1_0, q8b_1_1, q8b_2_0, q8b_2_1);\n const __m256 deltas = quad_mx_delta_float(x[ib].e, y[ib].d, x[ib + 1].e, y[ib + 1].d);\n accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);\n }\n\n sumf = hsum_float_8(accum);\n\n#endif\n for (; ib < nb; ++ib) {\n const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_E8M0_TO_FP32_HALF(x[ib].e);\n int sumi1 = 0;\n int sumi2 = 0;\n for (int j = 0; j < QK_MXFP4/2; ++j) {\n sumi1 += y[ib].qs[j + 0] * kvalues_mxfp4[x[ib].qs[j] & 0xf];\n sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >> 4];\n }\n sumf += d * (sumi1 + sumi2);\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n int ib = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__AVX2__)\n // Initialize accumulator with zeros\n __m256 acc = _mm256_setzero_ps();\n\n // Main loop\n for (; ib < nb; ++ib) {\n /* Compute combined scale for the block */\n const __m256 d = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));\n qx = _mm256_or_si256(qx, bxhi);\n\n __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n /* Multiply q with scale and accumulate */\n acc = _mm256_fmadd_ps(d, q, acc);\n }\n\n *s = hsum_float_8(acc);\n#elif defined(__AVX__)\n // Initialize accumulator with zeros\n __m256 acc = _mm256_setzero_ps();\n __m128i mask = _mm_set1_epi8((char)0xF0);\n\n // Main loop\n for (; ib < nb; ++ib) {\n /* Compute combined scale for the block */\n const __m256 d = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n __m256i bx_0 = bytes_from_nibbles_32(x[ib].qs);\n const __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n __m128i bxhil = _mm256_castsi256_si128(bxhi);\n __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);\n bxhil = _mm_andnot_si128(bxhil, mask);\n bxhih = _mm_andnot_si128(bxhih, mask);\n __m128i bxl = _mm256_castsi256_si128(bx_0);\n __m128i bxh = _mm256_extractf128_si256(bx_0, 1);\n bxl = _mm_or_si128(bxl, bxhil);\n bxh = _mm_or_si128(bxh, bxhih);\n bx_0 = MM256_SET_M128I(bxh, bxl);\n\n const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n const __m256 q = mul_sum_i8_pairs_float(bx_0, by_0);\n\n /* Multiply q with scale and accumulate */\n acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);\n }\n\n *s = hsum_float_8(acc);\n#else\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(x);\n UNUSED(y);\n ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n int ib = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_1);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__AVX2__)\n // Initialize accumulator with zeros\n __m256 acc = _mm256_setzero_ps();\n\n float summs = 0.0f;\n\n // Main loop\n for (; ib < nb; ++ib) {\n const __m256 dx = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d));\n\n summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));\n qx = _mm256_or_si256(qx, bxhi);\n\n const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib].d));\n const __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n const __m256 q = mul_sum_us8_pairs_float(qx, qy);\n\n acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);\n }\n\n *s = hsum_float_8(acc) + summs;\n#elif defined(__AVX__)\n // Initialize accumulator with zeros\n __m256 acc = _mm256_setzero_ps();\n __m128i mask = _mm_set1_epi8(0x10);\n\n float summs = 0.0f;\n\n // Main loop\n for (; ib < nb; ++ib) {\n const __m256 dx = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d));\n\n summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n __m256i bx_0 = bytes_from_nibbles_32(x[ib].qs);\n const __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n __m128i bxhil = _mm256_castsi256_si128(bxhi);\n __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);\n bxhil = _mm_and_si128(bxhil, mask);\n bxhih = _mm_and_si128(bxhih, mask);\n __m128i bxl = _mm256_castsi256_si128(bx_0);\n __m128i bxh = _mm256_extractf128_si256(bx_0, 1);\n bxl = _mm_or_si128(bxl, bxhil);\n bxh = _mm_or_si128(bxh, bxhih);\n bx_0 = MM256_SET_M128I(bxh, bxl);\n\n const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib].d));\n const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n const __m256 q = mul_sum_us8_pairs_float(bx_0, by_0);\n\n acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);\n }\n\n *s = hsum_float_8(acc) + summs;\n#else\n UNUSED(nb);\n UNUSED(ib);\n UNUSED(x);\n UNUSED(y);\n ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q8_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined(__AVX2__)\n // Initialize accumulator with zeros\n __m256 acc = _mm256_setzero_ps();\n\n // Main loop\n for (; ib < nb; ++ib) {\n // Compute combined scale for the block\n const __m256 d = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n __m256i qx = _mm256_loadu_si256((const __m256i *)x[ib].qs);\n __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n // Multiply q with scale and accumulate\n acc = _mm256_fmadd_ps( d, q, acc );\n }\n\n sumf = hsum_float_8(acc);\n#elif defined(__AVX__)\n __m256 accum = _mm256_setzero_ps();\n\n for (; ib + 1 < nb; ib += 2) {\n const __m128i qx_1_0 = _mm_loadu_si128((const __m128i *)x[ib].qs);\n const __m128i qx_1_1 = _mm_loadu_si128((const __m128i *)x[ib].qs + 1);\n const __m128i qx_2_0 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n const __m128i qx_2_1 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs + 1);\n const __m128i qy_1_0 = _mm_loadu_si128((const __m128i *)y[ib].qs);\n const __m128i qy_1_1 = _mm_loadu_si128((const __m128i *)y[ib].qs + 1);\n const __m128i qy_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n const __m128i qy_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);\n\n const __m256 p = mul_sum_i8_quad_float(qx_1_0, qx_1_1, qx_2_0, qx_2_1, qy_1_0, qy_1_1, qy_2_0, qy_2_1);\n const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d);\n accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);\n }\n\n sumf = hsum_float_8(accum);\n#endif\n for (; ib < nb; ++ib) {\n int sumi = 0;\n\n for (int j = 0; j < qk; j++) {\n sumi += x[ib].qs[j]*y[ib].qs[j];\n }\n\n sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_tq1_0 * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n __m256 sumf = _mm256_setzero_ps();\n\n for (int i = 0; i < nb; ++i) {\n // 16-bit sums\n __m256i sumi0 = _mm256_setzero_si256();\n __m256i sumi1 = _mm256_setzero_si256();\n __m256i sumi2 = _mm256_setzero_si256();\n\n // first 32 bytes of 5 elements\n {\n __m256i qx0 = _mm256_loadu_si256((const __m256i *) (x[i].qs));\n // 8-bit multiplies with shifts, masks and adds\n __m256i qx1 = _mm256_add_epi8(qx0, _mm256_add_epi8(qx0, qx0)); // 1 * 3\n __m256i qx2 = _mm256_add_epi8(_mm256_and_si256(_mm256_slli_epi16(qx0, 3), _mm256_set1_epi8(-8)), qx0); // 1 * 9\n __m256i qx3 = _mm256_add_epi8(_mm256_and_si256(_mm256_slli_epi16(qx1, 3), _mm256_set1_epi8(-8)), qx1); // 3 * 9\n __m256i qx4 = _mm256_add_epi8(_mm256_and_si256(_mm256_slli_epi16(qx2, 3), _mm256_set1_epi8(-8)), qx2); // 9 * 9\n\n // TODO: can _mm256_mulhi_epu16 be faster even if 16-bits?\n\n // Cancel the +1 from avg so that it behaves like a halving add\n qx0 = _mm256_subs_epu8(qx0, _mm256_set1_epi8(1));\n qx1 = _mm256_subs_epu8(qx1, _mm256_set1_epi8(1));\n qx2 = _mm256_subs_epu8(qx2, _mm256_set1_epi8(1));\n qx3 = _mm256_subs_epu8(qx3, _mm256_set1_epi8(1));\n qx4 = _mm256_subs_epu8(qx4, _mm256_set1_epi8(1));\n // Multiply by 3 and get the top 2 bits\n qx0 = _mm256_avg_epu8(qx0, _mm256_avg_epu8(qx0, _mm256_setzero_si256()));\n qx1 = _mm256_avg_epu8(qx1, _mm256_avg_epu8(qx1, _mm256_setzero_si256()));\n qx2 = _mm256_avg_epu8(qx2, _mm256_avg_epu8(qx2, _mm256_setzero_si256()));\n qx3 = _mm256_avg_epu8(qx3, _mm256_avg_epu8(qx3, _mm256_setzero_si256()));\n qx4 = _mm256_avg_epu8(qx4, _mm256_avg_epu8(qx4, _mm256_setzero_si256()));\n qx0 = _mm256_and_si256(_mm256_srli_epi16(qx0, 6), _mm256_set1_epi8(3));\n qx1 = _mm256_and_si256(_mm256_srli_epi16(qx1, 6), _mm256_set1_epi8(3));\n qx2 = _mm256_and_si256(_mm256_srli_epi16(qx2, 6), _mm256_set1_epi8(3));\n qx3 = _mm256_and_si256(_mm256_srli_epi16(qx3, 6), _mm256_set1_epi8(3));\n qx4 = _mm256_and_si256(_mm256_srli_epi16(qx4, 6), _mm256_set1_epi8(3));\n\n const __m256i qy0 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 0));\n const __m256i qy1 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 32));\n const __m256i qy2 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 64));\n const __m256i qy3 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 96));\n const __m256i qy4 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 128));\n\n qx0 = _mm256_maddubs_epi16(qx0, qy0);\n qx1 = _mm256_maddubs_epi16(qx1, qy1);\n qx2 = _mm256_maddubs_epi16(qx2, qy2);\n qx3 = _mm256_maddubs_epi16(qx3, qy3);\n qx4 = _mm256_maddubs_epi16(qx4, qy4);\n\n sumi0 = _mm256_add_epi16(sumi0, _mm256_add_epi16(qx0, qx1));\n sumi1 = _mm256_add_epi16(sumi1, _mm256_add_epi16(qx2, qx3));\n sumi2 = _mm256_add_epi16(sumi2, qx4);\n }\n\n // last 16 bytes of 5-element, along with the 4 bytes of 4 elements\n {\n __m128i qx0 = _mm_loadu_si128((const __m128i *) (x[i].qs + 32));\n uint32_t qh;\n memcpy(&qh, x[i].qh, sizeof(qh)); // potentially unaligned\n __m256i qx5_l = _mm256_cvtepu8_epi16(_mm_set1_epi32(qh));\n __m128i qx1 = _mm_add_epi8(qx0, _mm_add_epi8(qx0, qx0)); // 1 * 3\n __m128i qx2 = _mm_add_epi8(_mm_and_si128(_mm_slli_epi16(qx0, 3), _mm_set1_epi8(-8)), qx0); // 1 * 9\n __m128i qx3 = _mm_add_epi8(_mm_and_si128(_mm_slli_epi16(qx1, 3), _mm_set1_epi8(-8)), qx1); // 3 * 9\n __m128i qx4 = _mm_add_epi8(_mm_and_si128(_mm_slli_epi16(qx2, 3), _mm_set1_epi8(-8)), qx2); // 9 * 9\n __m256i qx01 = MM256_SET_M128I(qx1, qx0);\n __m256i qx23 = MM256_SET_M128I(qx3, qx2);\n\n // avx2 does not have 8-bit multiplies, so 16-bit it is.\n qx5_l = _mm256_mullo_epi16(qx5_l, _mm256_set_epi16(27, 27, 27, 27, 9, 9, 9, 9, 3, 3, 3, 3, 1, 1, 1, 1));\n qx5_l = _mm256_and_si256(qx5_l, _mm256_set1_epi16(0xFF));\n __m128i qx5 = _mm_packus_epi16(_mm256_castsi256_si128(qx5_l), _mm256_extracti128_si256(qx5_l, 1));\n\n __m256i qx45 = MM256_SET_M128I(qx5, qx4);\n\n // Cancel the +1 from avg so that it behaves like a halving add\n qx01 = _mm256_subs_epu8(qx01, _mm256_set1_epi8(1));\n qx23 = _mm256_subs_epu8(qx23, _mm256_set1_epi8(1));\n qx45 = _mm256_subs_epu8(qx45, _mm256_set1_epi8(1));\n // Multiply by 3 and get the top 2 bits\n qx01 = _mm256_avg_epu8(qx01, _mm256_avg_epu8(qx01, _mm256_setzero_si256()));\n qx23 = _mm256_avg_epu8(qx23, _mm256_avg_epu8(qx23, _mm256_setzero_si256()));\n qx45 = _mm256_avg_epu8(qx45, _mm256_avg_epu8(qx45, _mm256_setzero_si256()));\n qx01 = _mm256_and_si256(_mm256_srli_epi16(qx01, 6), _mm256_set1_epi8(3));\n qx23 = _mm256_and_si256(_mm256_srli_epi16(qx23, 6), _mm256_set1_epi8(3));\n qx45 = _mm256_and_si256(_mm256_srli_epi16(qx45, 6), _mm256_set1_epi8(3));\n\n const __m256i qy01 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 160));\n const __m256i qy23 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 192));\n const __m256i qy45 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 224));\n\n qx01 = _mm256_maddubs_epi16(qx01, qy01);\n qx23 = _mm256_maddubs_epi16(qx23, qy23);\n qx45 = _mm256_maddubs_epi16(qx45, qy45);\n\n sumi0 = _mm256_add_epi16(sumi0, qx01);\n sumi1 = _mm256_add_epi16(sumi1, qx23);\n sumi2 = _mm256_add_epi16(sumi2, qx45);\n }\n\n const __m256i ysum = _mm256_loadu_si256((const __m256i *) y[i].bsums);\n const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d));\n\n sumi0 = _mm256_sub_epi16(sumi0, ysum);\n sumi0 = _mm256_add_epi16(sumi0, _mm256_add_epi16(sumi1, sumi2));\n sumi0 = _mm256_madd_epi16(sumi0, _mm256_set1_epi16(1));\n\n sumf = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(sumi0), d), sumf);\n }\n\n *s = hsum_float_8(sumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_tq2_0 * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n __m256 sumf = _mm256_setzero_ps();\n\n for (int i = 0; i < nb; ++i) {\n // 16-bit sums, because 256*127 still fits\n __m256i sumi0 = _mm256_setzero_si256();\n __m256i sumi1 = _mm256_setzero_si256();\n\n for (size_t j = 0; j < sizeof(x->qs); j += 32) {\n __m256i qx0 = _mm256_loadu_si256((const __m256i *) (x[i].qs + j));\n __m256i qx1 = _mm256_srli_epi16(qx0, 2);\n __m256i qx2 = _mm256_srli_epi16(qx0, 4);\n __m256i qx3 = _mm256_srli_epi16(qx0, 6);\n\n // 0, 1, 2 (should not be 3)\n qx0 = _mm256_and_si256(qx0, _mm256_set1_epi8(3));\n qx1 = _mm256_and_si256(qx1, _mm256_set1_epi8(3));\n qx2 = _mm256_and_si256(qx2, _mm256_set1_epi8(3));\n qx3 = _mm256_and_si256(qx3, _mm256_set1_epi8(3));\n\n const __m256i qy0 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 0));\n const __m256i qy1 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 32));\n const __m256i qy2 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 64));\n const __m256i qy3 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 96));\n\n qx0 = _mm256_maddubs_epi16(qx0, qy0);\n qx1 = _mm256_maddubs_epi16(qx1, qy1);\n qx2 = _mm256_maddubs_epi16(qx2, qy2);\n qx3 = _mm256_maddubs_epi16(qx3, qy3);\n\n sumi0 = _mm256_add_epi16(sumi0, _mm256_add_epi16(qx0, qx1));\n sumi1 = _mm256_add_epi16(sumi1, _mm256_add_epi16(qx2, qx3));\n }\n\n const __m256i ysum = _mm256_loadu_si256((const __m256i *) y[i].bsums);\n const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d));\n\n sumi0 = _mm256_add_epi16(sumi0, sumi1);\n sumi0 = _mm256_sub_epi16(sumi0, ysum);\n sumi0 = _mm256_madd_epi16(sumi0, _mm256_set1_epi16(1));\n\n sumf = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(sumi0), d), sumf);\n }\n\n *s = hsum_float_8(sumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q2_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n const __m256i m3 = _mm256_set1_epi8(3);\n const __m128i m4 = _mm_set1_epi8(0xF);\n\n __m256 acc = _mm256_setzero_ps();\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);\n const __m128i scales8 = _mm_and_si128(mins_and_scales, m4);\n const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);\n const __m256i mins = _mm256_cvtepi8_epi16(mins8);\n const __m256i prod = _mm256_madd_epi16(mins, _mm256_loadu_si256((const __m256i*)y[i].bsums));\n\n acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc);\n\n const __m256i all_scales = _mm256_cvtepi8_epi16(scales8);\n const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);\n const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);\n const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};\n\n __m256i sumi = _mm256_setzero_si256();\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n const __m256i q2bits = _mm256_loadu_si256((const __m256i*)q2); q2 += 32;\n\n const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n const __m256i q2_0 = _mm256_and_si256(q2bits, m3);\n const __m256i q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3);\n const __m256i q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3);\n const __m256i q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3);\n\n __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);\n __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);\n __m256i p2 = _mm256_maddubs_epi16(q2_2, q8_2);\n __m256i p3 = _mm256_maddubs_epi16(q2_3, q8_3);\n\n p0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(0)), p0);\n p1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(1)), p1);\n p2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(2)), p2);\n p3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(3)), p3);\n\n p0 = _mm256_add_epi32(p0, p1);\n p2 = _mm256_add_epi32(p2, p3);\n\n sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2));\n }\n\n acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);\n\n }\n\n *s = hsum_float_8(acc);\n\n#elif defined __AVX__\n\n const __m128i m3 = _mm_set1_epi8(0x3);\n const __m128i m4 = _mm_set1_epi8(0xF);\n const __m128i m2 = _mm_set1_epi8(0x2);\n\n __m256 acc = _mm256_setzero_ps();\n\n for (int i = 0; i < nb; ++i) {\n\n const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n // load mins and scales from block_q2_K.scales[QK_K/16]\n const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);\n const __m128i scales16 = _mm_and_si128(mins_and_scales, m4);\n const __m128i mins16 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);\n const __m128i mins_0 = _mm_cvtepi8_epi16(mins16);\n const __m128i mins_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(mins16, mins16));\n\n // summs = y[i].bsums * (x[i].scales >> 4) in 16bits*8*2 to 32bits*4*2\n const __m128i summs_0 = _mm_madd_epi16(mins_0, _mm_loadu_si128((const __m128i*)&y[i].bsums[0]));\n const __m128i summs_1 = _mm_madd_epi16(mins_1, _mm_loadu_si128((const __m128i*)&y[i].bsums[8]));\n\n // sumf += -dmin * summs in 32bits*8\n acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(MM256_SET_M128I(summs_1, summs_0))), acc);\n\n const __m128i scales_0 = _mm_cvtepi8_epi16(scales16);\n const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales16, scales16));\n const __m128i scales[2] = { scales_0, scales_1 };\n\n __m128i sumi_0 = _mm_setzero_si128();\n __m128i sumi_1 = _mm_setzero_si128();\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n // load Q8 quants int8*16*8 from block_q8_K.qs[QK_K]\n const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n\n // load 2bits*16*8 from block_q2_K.qs[QK_K/4]\n __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;\n const __m128i q2_0 = _mm_and_si128(q2bits, m3);\n const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);\n const __m128i q2_4 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);\n const __m128i q2_6 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);\n q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;\n const __m128i q2_1 = _mm_and_si128(q2bits, m3);\n const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);\n const __m128i q2_5 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);\n const __m128i q2_7 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);\n\n // isuml = q8[l] * ((q2[l] >> shift) & 3) in 8bits*16*8 to 16bits*8*8\n __m128i p0 = _mm_maddubs_epi16(q2_0, q8_0);\n __m128i p1 = _mm_maddubs_epi16(q2_1, q8_1);\n __m128i p2 = _mm_maddubs_epi16(q2_2, q8_2);\n __m128i p3 = _mm_maddubs_epi16(q2_3, q8_3);\n __m128i p4 = _mm_maddubs_epi16(q2_4, q8_4);\n __m128i p5 = _mm_maddubs_epi16(q2_5, q8_5);\n __m128i p6 = _mm_maddubs_epi16(q2_6, q8_6);\n __m128i p7 = _mm_maddubs_epi16(q2_7, q8_7);\n\n // isum += (x[i].scales[is++] & 0xF) * isuml in 16bits*8*8 to 32bits*4*8\n __m128i shuffle = _mm_set1_epi16(0x0100);\n p0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p0);\n shuffle = _mm_add_epi16(shuffle, m2);\n p1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p1);\n shuffle = _mm_add_epi16(shuffle, m2);\n p2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p2);\n shuffle = _mm_add_epi16(shuffle, m2);\n p3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p3);\n shuffle = _mm_add_epi16(shuffle, m2);\n p4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p4);\n shuffle = _mm_add_epi16(shuffle, m2);\n p5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p5);\n shuffle = _mm_add_epi16(shuffle, m2);\n p6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p6);\n shuffle = _mm_add_epi16(shuffle, m2);\n p7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p7);\n\n p0 = _mm_add_epi32(p0, p1);\n p2 = _mm_add_epi32(p2, p3);\n p4 = _mm_add_epi32(p4, p5);\n p6 = _mm_add_epi32(p6, p7);\n\n // isum in 32bits*4*2\n sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p0, p2));\n sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p4, p6));\n }\n\n // sumf += dall * isum - dmin * summs in 32bits\n __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dall), _mm256_cvtepi32_ps(sumi)), acc);\n }\n\n *s = hsum_float_8(acc);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const uint32_t kmask1 = 0x03030303;\n const uint32_t kmask2 = 0x0f0f0f0f;\n\n const block_q3_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n const __m256i m3 = _mm256_set1_epi8(3);\n const __m256i mone = _mm256_set1_epi8(1);\n const __m128i m32 = _mm_set1_epi8(32);\n\n __m256 acc = _mm256_setzero_ps();\n\n uint32_t aux[3];\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n // Set up scales\n memcpy(aux, x[i].scales, 12);\n __m128i scales128 = _mm_set_epi32(\n ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),\n ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),\n (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),\n (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));\n scales128 = _mm_sub_epi8(scales128, m32);\n const __m256i all_scales = _mm256_cvtepi8_epi16(scales128);\n const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);\n const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);\n const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};\n\n // high bit\n const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].hmask);\n\n // integer accumulator\n __m256i sumi = _mm256_setzero_si256();\n\n int bit = 0;\n int is = 0;\n\n for (int j = 0; j < QK_K/128; ++j) {\n // load low 2 bits\n const __m256i q3bits = _mm256_loadu_si256((const __m256i*)q3); q3 += 32;\n\n // prepare low and high bits\n const __m256i q3l_0 = _mm256_and_si256(q3bits, m3);\n const __m256i q3h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);\n ++bit;\n\n const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3);\n const __m256i q3h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);\n ++bit;\n\n const __m256i q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3);\n const __m256i q3h_2 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);\n ++bit;\n\n const __m256i q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3);\n const __m256i q3h_3 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);\n ++bit;\n\n // load Q8 quants\n const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,\n // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,\n // and 2 if the high bit was set)\n __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);\n __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);\n __m256i q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2);\n __m256i q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3);\n\n __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);\n __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);\n __m256i p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2);\n __m256i p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3);\n\n p16_0 = _mm256_sub_epi16(p16_0, q8s_0);\n p16_1 = _mm256_sub_epi16(p16_1, q8s_1);\n p16_2 = _mm256_sub_epi16(p16_2, q8s_2);\n p16_3 = _mm256_sub_epi16(p16_3, q8s_3);\n\n // multiply with scales\n p16_0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);\n p16_1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);\n p16_2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);\n p16_3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);\n\n // accumulate\n p16_0 = _mm256_add_epi32(p16_0, p16_1);\n p16_2 = _mm256_add_epi32(p16_2, p16_3);\n sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2));\n\n }\n\n // multiply with block scale and accumulate\n acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);\n\n }\n\n *s = hsum_float_8(acc);\n\n#elif defined __AVX__\n\n const __m128i m3 = _mm_set1_epi8(3);\n const __m128i mone = _mm_set1_epi8(1);\n const __m128i m32 = _mm_set1_epi8(32);\n const __m128i m2 = _mm_set1_epi8(2);\n\n __m256 acc = _mm256_setzero_ps();\n\n const uint32_t *aux;\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n // Set up scales\n aux = (const uint32_t *)x[i].scales;\n __m128i scales128 = _mm_set_epi32(\n ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),\n ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),\n (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),\n (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));\n scales128 = _mm_sub_epi8(scales128, m32);\n const __m128i scales_0 = _mm_cvtepi8_epi16(scales128);\n const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales128, scales128));\n const __m128i scales[2] = { scales_0, scales_1 };\n\n // high bit *128*2 from block_q3_K.hmask[QK_K/8]\n const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].hmask[0]);\n const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].hmask[16]);\n\n // integer accumulator\n __m128i sumi_0 = _mm_setzero_si128();\n __m128i sumi_1 = _mm_setzero_si128();\n\n for (int j = 0; j < QK_K/128; ++j) {\n // load low 2 bits *64*2 from block_q3_K.qs[QK_K/4]\n const __m128i q3bits_0 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;\n const __m128i q3bits_1 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;\n\n // prepare low and high bits\n const int bit = j << 2;\n\n const __m128i q3l_0 = _mm_and_si128(q3bits_0, m3);\n const __m128i q3l_1 = _mm_and_si128(q3bits_1, m3);\n const __m128i q3h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit)), bit), 2);\n const __m128i q3h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit)), bit), 2);\n\n const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 2), m3);\n const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 2), m3);\n const __m128i q3h_2 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+1)), bit+1), 2);\n const __m128i q3h_3 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+1)), bit+1), 2);\n\n const __m128i q3l_4 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 4), m3);\n const __m128i q3l_5 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 4), m3);\n const __m128i q3h_4 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+2)), bit+2), 2);\n const __m128i q3h_5 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+2)), bit+2), 2);\n\n const __m128i q3l_6 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 6), m3);\n const __m128i q3l_7 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 6), m3);\n const __m128i q3h_6 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+3)), bit+3), 2);\n const __m128i q3h_7 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+3)), bit+3), 2);\n\n // load Q8 quants from block_q8_K.qs[QK_K]\n const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n\n // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,\n // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,\n // and 2 if the high bit was set)\n __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, q8_0);\n __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, q8_1);\n __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, q8_2);\n __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, q8_3);\n __m128i q8s_4 = _mm_maddubs_epi16(q3h_4, q8_4);\n __m128i q8s_5 = _mm_maddubs_epi16(q3h_5, q8_5);\n __m128i q8s_6 = _mm_maddubs_epi16(q3h_6, q8_6);\n __m128i q8s_7 = _mm_maddubs_epi16(q3h_7, q8_7);\n\n __m128i p16_0 = _mm_maddubs_epi16(q3l_0, q8_0);\n __m128i p16_1 = _mm_maddubs_epi16(q3l_1, q8_1);\n __m128i p16_2 = _mm_maddubs_epi16(q3l_2, q8_2);\n __m128i p16_3 = _mm_maddubs_epi16(q3l_3, q8_3);\n __m128i p16_4 = _mm_maddubs_epi16(q3l_4, q8_4);\n __m128i p16_5 = _mm_maddubs_epi16(q3l_5, q8_5);\n __m128i p16_6 = _mm_maddubs_epi16(q3l_6, q8_6);\n __m128i p16_7 = _mm_maddubs_epi16(q3l_7, q8_7);\n\n p16_0 = _mm_sub_epi16(p16_0, q8s_0);\n p16_1 = _mm_sub_epi16(p16_1, q8s_1);\n p16_2 = _mm_sub_epi16(p16_2, q8s_2);\n p16_3 = _mm_sub_epi16(p16_3, q8s_3);\n p16_4 = _mm_sub_epi16(p16_4, q8s_4);\n p16_5 = _mm_sub_epi16(p16_5, q8s_5);\n p16_6 = _mm_sub_epi16(p16_6, q8s_6);\n p16_7 = _mm_sub_epi16(p16_7, q8s_7);\n\n // multiply with scales\n __m128i shuffle = _mm_set1_epi16(0x0100);\n p16_0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_0);\n shuffle = _mm_add_epi16(shuffle, m2);\n p16_1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_1);\n shuffle = _mm_add_epi16(shuffle, m2);\n p16_2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_2);\n shuffle = _mm_add_epi16(shuffle, m2);\n p16_3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_3);\n shuffle = _mm_add_epi16(shuffle, m2);\n p16_4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_4);\n shuffle = _mm_add_epi16(shuffle, m2);\n p16_5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_5);\n shuffle = _mm_add_epi16(shuffle, m2);\n p16_6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_6);\n shuffle = _mm_add_epi16(shuffle, m2);\n p16_7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_7);\n\n // accumulate\n p16_0 = _mm_add_epi32(p16_0, p16_1);\n p16_2 = _mm_add_epi32(p16_2, p16_3);\n p16_4 = _mm_add_epi32(p16_4, p16_5);\n p16_6 = _mm_add_epi32(p16_6, p16_7);\n sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));\n sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_4, p16_6));\n\n }\n\n // multiply with block scale and accumulate\n __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);\n\n }\n\n *s = hsum_float_8(acc);\n\n#else\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined __AVX2__\n\n const __m256i m4 = _mm256_set1_epi8(0xF);\n\n __m256 acc = _mm256_setzero_ps();\n __m128 acc_m = _mm_setzero_ps();\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));\n\n const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);\n const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);\n acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);\n\n const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);\n const __m256i scales = MM256_SET_M128I(sc128, sc128);\n\n __m256i sumi = _mm256_setzero_si256();\n\n for (int j = 0; j < QK_K/64; ++j) {\n\n const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));\n const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));\n\n const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;\n const __m256i q4l = _mm256_and_si256(q4bits, m4);\n const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);\n\n const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);\n p16l = _mm256_madd_epi16(scale_l, p16l);\n\n const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);\n p16h = _mm256_madd_epi16(scale_h, p16h);\n const __m256i sumj = _mm256_add_epi32(p16l, p16h);\n\n sumi = _mm256_add_epi32(sumi, sumj);\n }\n\n __m256 vd = _mm256_set1_ps(d);\n acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);\n\n }\n\n acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));\n acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));\n\n *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);\n\n#elif defined __AVX__\n\n const __m128i m4 = _mm_set1_epi8(0xF);\n const __m128i m2 = _mm_set1_epi8(0x2);\n\n __m256 acc = _mm256_setzero_ps();\n __m128 acc_m = _mm_setzero_ps();\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);\n const __m128i scales = _mm_cvtepu8_epi16(utmps);\n const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));\n\n const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);\n const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);\n const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);\n const __m128i prod = _mm_madd_epi16(mins, q8s);\n acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);\n\n __m128i sumi_0 = _mm_setzero_si128();\n __m128i sumi_1 = _mm_setzero_si128();\n\n __m128i shuffle = _mm_set1_epi16(0x0100);\n for (int j = 0; j < QK_K/64; ++j) {\n\n const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);\n shuffle = _mm_add_epi16(shuffle, m2);\n const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);\n shuffle = _mm_add_epi16(shuffle, m2);\n\n __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n const __m128i q4l_0 = _mm_and_si128(q4bits, m4);\n const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);\n q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n const __m128i q4l_1 = _mm_and_si128(q4bits, m4);\n const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);\n\n const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);\n p16l = _mm_madd_epi16(scale_l, p16l);\n sumi_0 = _mm_add_epi32(sumi_0, p16l);\n const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n p16l = _mm_maddubs_epi16(q4l_1, q8l_1);\n p16l = _mm_madd_epi16(scale_l, p16l);\n sumi_1 = _mm_add_epi32(sumi_1, p16l);\n\n const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);\n p16h = _mm_madd_epi16(scale_h, p16h);\n sumi_0 = _mm_add_epi32(sumi_0, p16h);\n const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n p16h = _mm_maddubs_epi16(q4h_1, q8h_1);\n p16h = _mm_madd_epi16(scale_h, p16h);\n sumi_1 = _mm_add_epi32(sumi_1, p16h);\n\n }\n\n __m256 vd = _mm256_set1_ps(d);\n __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);\n\n }\n\n acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));\n acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));\n\n *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n#if defined __AVX2__\n\n const __m256i m4 = _mm256_set1_epi8(0xF);\n const __m128i mzero = _mm_setzero_si128();\n const __m256i mone = _mm256_set1_epi8(1);\n\n __m256 acc = _mm256_setzero_ps();\n\n float summs = 0.f;\n\n for (int i = 0; i < nb; ++i) {\n const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));\n\n const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);\n const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);\n const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);\n summs += dmin * _mm_extract_epi32(hsum, 0);\n\n const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);\n const __m256i scales = MM256_SET_M128I(sc128, sc128);\n\n const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].qh);\n __m256i hmask = mone;\n\n __m256i sumi = _mm256_setzero_si256();\n\n int bit = 0;\n\n for (int j = 0; j < QK_K/64; ++j) {\n\n const __m256i scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));\n const __m256i scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));\n\n const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); q5 += 32;\n\n const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);\n const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);\n const __m256i q5_0 = _mm256_add_epi8(q5l_0, q5h_0);\n hmask = _mm256_slli_epi16(hmask, 1);\n\n const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);\n const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);\n const __m256i q5_1 = _mm256_add_epi8(q5l_1, q5h_1);\n hmask = _mm256_slli_epi16(hmask, 1);\n\n const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n __m256i p16_0 = _mm256_maddubs_epi16(q5_0, q8_0);\n __m256i p16_1 = _mm256_maddubs_epi16(q5_1, q8_1);\n\n p16_0 = _mm256_madd_epi16(scale_0, p16_0);\n p16_1 = _mm256_madd_epi16(scale_1, p16_1);\n\n sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));\n\n }\n\n __m256 vd = _mm256_set1_ps(d);\n acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);\n\n }\n\n *s = hsum_float_8(acc) + summs;\n\n#elif defined __AVX__\n\n const __m128i m4 = _mm_set1_epi8(0xF);\n const __m128i mzero = _mm_setzero_si128();\n const __m128i mone = _mm_set1_epi8(1);\n const __m128i m2 = _mm_set1_epi8(2);\n\n __m256 acc = _mm256_setzero_ps();\n\n float summs = 0.f;\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);\n const __m128i scales = _mm_cvtepu8_epi16(utmps);\n const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));\n\n const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);\n const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);\n const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);\n const __m128i prod = _mm_madd_epi16(mins, q8s);\n const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);\n summs += dmin * _mm_extract_epi32(hsum, 0);\n\n const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].qh[0]);\n const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].qh[16]);\n __m128i hmask = mone;\n\n __m128i sumi_0 = _mm_setzero_si128();\n __m128i sumi_1 = _mm_setzero_si128();\n\n int bit = 0;\n\n __m128i shuffle = _mm_set1_epi16(0x0100);\n for (int j = 0; j < QK_K/64; ++j) {\n\n const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);\n shuffle = _mm_add_epi16(shuffle, m2);\n const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);\n shuffle = _mm_add_epi16(shuffle, m2);\n\n const __m128i q5bits_0 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;\n const __m128i q5bits_1 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;\n\n __m128i q5l_0 = _mm_and_si128(q5bits_0, m4);\n __m128i q5l_1 = _mm_and_si128(q5bits_1, m4);\n __m128i q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);\n __m128i q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);\n __m128i q5_0 = _mm_add_epi8(q5l_0, q5h_0);\n __m128i q5_1 = _mm_add_epi8(q5l_1, q5h_1);\n hmask = _mm_slli_epi16(hmask, 1);\n\n __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n __m128i p16_0 = _mm_maddubs_epi16(q5_0, q8_0);\n __m128i p16_1 = _mm_maddubs_epi16(q5_1, q8_1);\n p16_0 = _mm_madd_epi16(scale_0, p16_0);\n p16_1 = _mm_madd_epi16(scale_0, p16_1);\n\n q5l_0 = _mm_and_si128(_mm_srli_epi16(q5bits_0, 4), m4);\n q5l_1 = _mm_and_si128(_mm_srli_epi16(q5bits_1, 4), m4);\n q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);\n q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);\n q5_0 = _mm_add_epi8(q5l_0, q5h_0);\n q5_1 = _mm_add_epi8(q5l_1, q5h_1);\n hmask = _mm_slli_epi16(hmask, 1);\n\n q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n __m128i p16_2 = _mm_maddubs_epi16(q5_0, q8_0);\n __m128i p16_3 = _mm_maddubs_epi16(q5_1, q8_1);\n p16_2 = _mm_madd_epi16(scale_1, p16_2);\n p16_3 = _mm_madd_epi16(scale_1, p16_3);\n\n sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));\n sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));\n\n }\n\n __m256 vd = _mm256_set1_ps(d);\n __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);\n\n }\n\n *s = hsum_float_8(acc) + summs;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n UNUSED(utmp);\n ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q6_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n const __m256i m4 = _mm256_set1_epi8(0xF);\n const __m256i m2 = _mm256_set1_epi8(3);\n const __m256i m32s = _mm256_set1_epi8(32);\n\n __m256 acc = _mm256_setzero_ps();\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);\n\n __m256i sumi = _mm256_setzero_si256();\n\n int is = 0;\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));\n const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));\n const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));\n const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));\n is += 4;\n\n const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;\n const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;\n const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;\n\n const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4);\n const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4);\n const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4);\n const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4);\n\n const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);\n const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);\n const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2);\n const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3);\n\n const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0);\n __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1);\n __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2);\n __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3);\n\n __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);\n __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);\n __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);\n __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);\n\n p16_0 = _mm256_sub_epi16(p16_0, q8s_0);\n p16_1 = _mm256_sub_epi16(p16_1, q8s_1);\n p16_2 = _mm256_sub_epi16(p16_2, q8s_2);\n p16_3 = _mm256_sub_epi16(p16_3, q8s_3);\n\n p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);\n p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);\n p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);\n p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);\n\n sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));\n sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));\n\n }\n\n acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);\n }\n\n *s = hsum_float_8(acc);\n\n#elif defined __AVX__\n\n const __m128i m3 = _mm_set1_epi8(3);\n const __m128i m15 = _mm_set1_epi8(15);\n\n __m256 acc = _mm256_setzero_ps();\n\n for (int i = 0; i < nb; ++i) {\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n // handle the q6_k -32 offset separately using bsums\n const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)y[i].bsums);\n const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)y[i].bsums + 1);\n const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);\n const __m128i scales_16_0 = _mm_cvtepi8_epi16(scales);\n const __m128i scales_16_1 = _mm_cvtepi8_epi16(_mm_bsrli_si128(scales, 8));\n const __m128i q8sclsub_0 = _mm_slli_epi32(_mm_madd_epi16(q8sums_0, scales_16_0), 5);\n const __m128i q8sclsub_1 = _mm_slli_epi32(_mm_madd_epi16(q8sums_1, scales_16_1), 5);\n\n __m128i sumi_0 = _mm_setzero_si128();\n __m128i sumi_1 = _mm_setzero_si128();\n\n int is = 0;\n\n for (int j = 0; j < QK_K/128; ++j) {\n\n const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;\n const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;\n\n const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);\n const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);\n const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, _mm_set1_epi8(12)), 2);\n const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, _mm_set1_epi8(12)), 2);\n const __m128i q4h_4 = _mm_and_si128(q4bitsH_0, _mm_set1_epi8(48));\n const __m128i q4h_5 = _mm_and_si128(q4bitsH_1, _mm_set1_epi8(48));\n const __m128i q4h_6 = _mm_srli_epi16(_mm_and_si128(q4bitsH_0, _mm_set1_epi8(-64)), 2);\n const __m128i q4h_7 = _mm_srli_epi16(_mm_and_si128(q4bitsH_1, _mm_set1_epi8(-64)), 2);\n\n const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n\n const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m15), q4h_0);\n const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m15), q4h_1);\n const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m15), q4h_2);\n const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m15), q4h_3);\n const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m15), q4h_4);\n const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m15), q4h_5);\n const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m15), q4h_6);\n const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m15), q4h_7);\n\n const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n\n __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);\n __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);\n __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);\n __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);\n __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);\n __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);\n __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);\n __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);\n\n const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));\n const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));\n const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));\n const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));\n is += 4;\n\n p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);\n p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_0, 8)), p16_1);\n p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);\n p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_1, 8)), p16_3);\n p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);\n p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_2, 8)), p16_5);\n p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);\n p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_3, 8)), p16_7);\n\n sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));\n sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));\n sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));\n sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));\n\n }\n\n sumi_0 = _mm_sub_epi32(sumi_0, q8sclsub_0);\n sumi_1 = _mm_sub_epi32(sumi_1, q8sclsub_1);\n const __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi)), acc);\n }\n\n *s = hsum_float_8(acc);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined (__AVX__) || defined (__AVX2__)\nstatic const int8_t keven_signs_q2xs[1024] = {\n 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,\n 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,\n 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,\n 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,\n 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,\n 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,\n 1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, 1,\n 1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, -1,\n 1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, -1,\n 1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, 1,\n 1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, 1,\n 1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1,\n 1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, 1,\n 1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, -1,\n 1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, -1,\n 1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, 1,\n 1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, -1,\n 1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1,\n 1, 1, 1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, 1,\n 1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, -1,\n 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1,\n 1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, -1,\n 1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, -1,\n 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1,\n 1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, 1,\n 1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, -1, -1, -1,\n 1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1,\n 1, 1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, 1,\n 1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, -1, -1, -1,\n 1, 1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, 1,\n 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, 1,\n 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n uint32_t aux32[4];\n const uint8_t * aux8 = (const uint8_t *)aux32;\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n __m256i sumi1 = _mm256_setzero_si256();\n __m256i sumi2 = _mm256_setzero_si256();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;\n const __m256i q2_1 = _mm256_set_epi64x(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);\n const __m256i q2_2 = _mm256_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);\n const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],\n signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);\n const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],\n signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);\n const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);\n const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);\n const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);\n const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);\n const uint16_t ls1 = aux32[1] >> 28;\n const uint16_t ls2 = aux32[3] >> 28;\n const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));\n const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));\n sumi1 = _mm256_add_epi32(sumi1, p1);\n sumi2 = _mm256_add_epi32(sumi2, p2);\n }\n\n accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n }\n\n *s = 0.125f * hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n uint32_t aux32[4];\n const uint8_t * aux8 = (const uint8_t *)aux32;\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n __m128i sumi1_0 = _mm_setzero_si128();\n __m128i sumi1_1 = _mm_setzero_si128();\n __m128i sumi2_0 = _mm_setzero_si128();\n __m128i sumi2_1 = _mm_setzero_si128();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;\n const __m128i q2_1_0 = _mm_set_epi64x(iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);\n const __m128i q2_1_1 = _mm_set_epi64x(iq2xxs_grid[aux8[3]], iq2xxs_grid[aux8[2]]);\n const __m128i q2_2_0 = _mm_set_epi64x(iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);\n const __m128i q2_2_1 = _mm_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]]);\n const __m128i s2_1_0 = _mm_set_epi64x(signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);\n const __m128i s2_1_1 = _mm_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127]);\n const __m128i s2_2_0 = _mm_set_epi64x(signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);\n const __m128i s2_2_1 = _mm_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127]);\n const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, s2_1_0);\n const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, s2_1_1);\n const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, s2_2_0);\n const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, s2_2_1);\n const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n const uint16_t ls1 = aux32[1] >> 28;\n const uint16_t ls2 = aux32[3] >> 28;\n const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));\n const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));\n const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));\n const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));\n sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);\n sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);\n sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);\n sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);\n }\n\n accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n }\n\n *s = 0.125f * hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n const __m256i mone = _mm256_set1_epi8(1);\n static const char block_sign_shuffle_mask_1[32] = {\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,\n 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,\n };\n static const char block_sign_shuffle_mask_2[32] = {\n 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,\n 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,\n };\n static const uint8_t bit_selector_mask_bytes[32] = {\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n };\n\n const __m256i bit_selector_mask = _mm256_loadu_si256((const __m256i*)bit_selector_mask_bytes);\n const __m256i block_sign_shuffle_1 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_1);\n const __m256i block_sign_shuffle_2 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_2);\n\n static const uint8_t k_bit_helper[32] = {\n 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n };\n const __m256i bit_helper = _mm256_loadu_si256((const __m256i*)k_bit_helper);\n const __m256i m511 = _mm256_set1_epi16(511);\n const __m128i m4 = _mm_set1_epi8(0xf);\n const __m128i m1 = _mm_set1_epi8(1);\n\n uint64_t aux64;\n\n // somewhat hacky, but gives a significant boost in performance\n __m256i aux_gindex;\n const uint16_t * gindex = (const uint16_t *)&aux_gindex;\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n memcpy(&aux64, x[i].scales, 8);\n __m128i stmp = _mm_set1_epi64x(aux64);\n stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));\n const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);\n\n __m256i sumi1 = _mm256_setzero_si256();\n __m256i sumi2 = _mm256_setzero_si256();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {\n\n const __m256i q2_data = _mm256_loadu_si256((const __m256i*)q2); q2 += 16;\n aux_gindex = _mm256_and_si256(q2_data, m511);\n\n const __m256i partial_sign_bits = _mm256_srli_epi16(q2_data, 9);\n const __m256i partial_sign_bits_upper = _mm256_srli_epi16(q2_data, 13);\n const __m256i partial_sign_bits_for_counting = _mm256_xor_si256(partial_sign_bits, partial_sign_bits_upper);\n\n const __m256i odd_bits = _mm256_shuffle_epi8(bit_helper, partial_sign_bits_for_counting);\n const __m256i full_sign_bits = _mm256_or_si256(partial_sign_bits, odd_bits);\n\n const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q8_3 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q8_4 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n\n const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],\n iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);\n const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],\n iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);\n const __m256i q2_3 = _mm256_set_epi64x(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],\n iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);\n const __m256i q2_4 = _mm256_set_epi64x(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],\n iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);\n\n const __m128i full_signs_l = _mm256_castsi256_si128(full_sign_bits);\n const __m128i full_signs_h = _mm256_extractf128_si256(full_sign_bits, 1);\n const __m256i full_signs_1 = MM256_SET_M128I(full_signs_l, full_signs_l);\n const __m256i full_signs_2 = MM256_SET_M128I(full_signs_h, full_signs_h);\n\n __m256i signs;\n signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_1);\n signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);\n const __m256i q8s_1 = _mm256_sign_epi8(q8_1, _mm256_or_si256(signs, mone));\n\n signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_2);\n signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);\n const __m256i q8s_2 = _mm256_sign_epi8(q8_2, _mm256_or_si256(signs, mone));\n\n signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_1);\n signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);\n const __m256i q8s_3 = _mm256_sign_epi8(q8_3, _mm256_or_si256(signs, mone));\n\n signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_2);\n signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);\n const __m256i q8s_4 = _mm256_sign_epi8(q8_4, _mm256_or_si256(signs, mone));\n\n const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);\n const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);\n const __m256i dot3 = _mm256_maddubs_epi16(q2_3, q8s_3);\n const __m256i dot4 = _mm256_maddubs_epi16(q2_4, q8s_4);\n\n const __m256i sc1 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0)));\n const __m256i sc2 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1)));\n const __m256i sc3 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+2)));\n const __m256i sc4 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+3)));\n\n sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot1, sc1));\n sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot2, sc2));\n sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot3, sc3));\n sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot4, sc4));\n }\n\n accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n }\n\n *s = 0.125f * hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n const __m128i mone = _mm_set1_epi8(1);\n static const char block_sign_shuffle_mask_1[32] = {\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,\n 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,\n };\n static const char block_sign_shuffle_mask_2[32] = {\n 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,\n 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,\n };\n static const uint8_t bit_selector_mask_bytes[32] = {\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n };\n\n const __m128i bit_selector_mask_0 = _mm_loadu_si128((const __m128i*)bit_selector_mask_bytes);\n const __m128i bit_selector_mask_1 = _mm_loadu_si128((const __m128i*)bit_selector_mask_bytes + 1);\n const __m128i block_sign_shuffle_1_0 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_1);\n const __m128i block_sign_shuffle_1_1 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_1 + 1);\n const __m128i block_sign_shuffle_2_0 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_2);\n const __m128i block_sign_shuffle_2_1 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_2 + 1);\n\n static const uint8_t k_bit_helper[32] = {\n 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n };\n const __m128i bit_helper_0 = _mm_loadu_si128((const __m128i*)k_bit_helper);\n const __m128i bit_helper_1 = _mm_loadu_si128((const __m128i*)k_bit_helper + 1);\n const __m128i m511 = _mm_set1_epi16(511);\n const __m128i m4 = _mm_set1_epi8(0xf);\n const __m128i m1 = _mm_set1_epi8(1);\n\n uint64_t aux64;\n\n // somewhat hacky, but gives a significant boost in performance\n __m256i aux_gindex;\n const uint16_t * gindex = (const uint16_t *)&aux_gindex;\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n memcpy(&aux64, x[i].scales, 8);\n __m128i stmp = _mm_set1_epi64x(aux64);\n stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));\n const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);\n\n __m128i sumi1_0 = _mm_setzero_si128();\n __m128i sumi1_1 = _mm_setzero_si128();\n __m128i sumi2_0 = _mm_setzero_si128();\n __m128i sumi2_1 = _mm_setzero_si128();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {\n\n const __m128i q2_data_0 = _mm_loadu_si128((const __m128i*)q2);\n const __m128i q2_data_1 = _mm_loadu_si128((const __m128i*)q2 + 1); q2 += 16;\n aux_gindex = MM256_SET_M128I(_mm_and_si128(q2_data_1, m511), _mm_and_si128(q2_data_0, m511));\n\n const __m128i partial_sign_bits_0 = _mm_srli_epi16(q2_data_0, 9);\n const __m128i partial_sign_bits_1 = _mm_srli_epi16(q2_data_1, 9);\n const __m128i partial_sign_bits_upper_0 = _mm_srli_epi16(q2_data_0, 13);\n const __m128i partial_sign_bits_upper_1 = _mm_srli_epi16(q2_data_1, 13);\n const __m128i partial_sign_bits_for_counting_0 = _mm_xor_si128(partial_sign_bits_0, partial_sign_bits_upper_0);\n const __m128i partial_sign_bits_for_counting_1 = _mm_xor_si128(partial_sign_bits_1, partial_sign_bits_upper_1);\n\n const __m128i odd_bits_0 = _mm_shuffle_epi8(bit_helper_0, partial_sign_bits_for_counting_0);\n const __m128i odd_bits_1 = _mm_shuffle_epi8(bit_helper_1, partial_sign_bits_for_counting_1);\n const __m128i full_sign_bits_0 = _mm_or_si128(partial_sign_bits_0, odd_bits_0);\n const __m128i full_sign_bits_1 = _mm_or_si128(partial_sign_bits_1, odd_bits_1);\n\n const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_3_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_3_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_4_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_4_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n\n const __m128i q2_1_0 = _mm_set_epi64x(iq2xs_grid[gindex[1]], iq2xs_grid[gindex[0]]);\n const __m128i q2_1_1 = _mm_set_epi64x(iq2xs_grid[gindex[3]], iq2xs_grid[gindex[2]]);\n const __m128i q2_2_0 = _mm_set_epi64x(iq2xs_grid[gindex[5]], iq2xs_grid[gindex[4]]);\n const __m128i q2_2_1 = _mm_set_epi64x(iq2xs_grid[gindex[7]], iq2xs_grid[gindex[6]]);\n const __m128i q2_3_0 = _mm_set_epi64x(iq2xs_grid[gindex[9]], iq2xs_grid[gindex[8]]);\n const __m128i q2_3_1 = _mm_set_epi64x(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]]);\n const __m128i q2_4_0 = _mm_set_epi64x(iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);\n const __m128i q2_4_1 = _mm_set_epi64x(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]]);\n\n // AVX2 full_signs_1 is full_sign_bits_0 here\n // AVX2 full_signs_2 is full_sign_bits_1 here\n __m128i signs_0, signs_1;\n signs_0 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_1_0);\n signs_1 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_1_1);\n signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);\n signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);\n const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, _mm_or_si128(signs_0, mone));\n const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, _mm_or_si128(signs_1, mone));\n\n signs_0 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_2_0);\n signs_1 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_2_1);\n signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);\n signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);\n const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, _mm_or_si128(signs_0, mone));\n const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, _mm_or_si128(signs_1, mone));\n\n signs_0 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_1_0);\n signs_1 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_1_1);\n signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);\n signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);\n const __m128i q8s_3_0 = _mm_sign_epi8(q8_3_0, _mm_or_si128(signs_0, mone));\n const __m128i q8s_3_1 = _mm_sign_epi8(q8_3_1, _mm_or_si128(signs_1, mone));\n\n signs_0 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_2_0);\n signs_1 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_2_1);\n signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);\n signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);\n const __m128i q8s_4_0 = _mm_sign_epi8(q8_4_0, _mm_or_si128(signs_0, mone));\n const __m128i q8s_4_1 = _mm_sign_epi8(q8_4_1, _mm_or_si128(signs_1, mone));\n\n const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n const __m128i dot3_0 = _mm_maddubs_epi16(q2_3_0, q8s_3_0);\n const __m128i dot3_1 = _mm_maddubs_epi16(q2_3_1, q8s_3_1);\n const __m128i dot4_0 = _mm_maddubs_epi16(q2_4_0, q8s_4_0);\n const __m128i dot4_1 = _mm_maddubs_epi16(q2_4_1, q8s_4_1);\n\n __m128i sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0));\n const __m128i sc1_0 = _mm_cvtepi8_epi16(sc_tmp);\n const __m128i sc1_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));\n sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1));\n const __m128i sc2_0 = _mm_cvtepi8_epi16(sc_tmp);\n const __m128i sc2_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));\n sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+2));\n const __m128i sc3_0 = _mm_cvtepi8_epi16(sc_tmp);\n const __m128i sc3_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));\n sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+3));\n const __m128i sc4_0 = _mm_cvtepi8_epi16(sc_tmp);\n const __m128i sc4_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));\n\n sumi1_0 = _mm_add_epi32(sumi1_0, _mm_madd_epi16(dot1_0, sc1_0));\n sumi1_1 = _mm_add_epi32(sumi1_1, _mm_madd_epi16(dot1_1, sc1_1));\n sumi2_0 = _mm_add_epi32(sumi2_0, _mm_madd_epi16(dot2_0, sc2_0));\n sumi2_1 = _mm_add_epi32(sumi2_1, _mm_madd_epi16(dot2_1, sc2_1));\n sumi1_0 = _mm_add_epi32(sumi1_0, _mm_madd_epi16(dot3_0, sc3_0));\n sumi1_1 = _mm_add_epi32(sumi1_1, _mm_madd_epi16(dot3_1, sc3_1));\n sumi2_0 = _mm_add_epi32(sumi2_0, _mm_madd_epi16(dot4_0, sc4_0));\n sumi2_1 = _mm_add_epi32(sumi2_1, _mm_madd_epi16(dot4_1, sc4_1));\n }\n\n accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n }\n\n *s = 0.125f * hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n };\n\n const __m128i m4 = _mm_set1_epi8(0xf);\n const __m128i m1 = _mm_set1_epi8(1);\n\n const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1);\n const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2);\n\n uint64_t aux64;\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n memcpy(&aux64, x[i].scales, 8);\n const __m128i scales8 = _mm_add_epi8(_mm_slli_epi16(_mm_and_si128(_mm_set_epi64x(aux64 >> 4, aux64), m4), 1), m1);\n const __m256i scales16 = _mm256_cvtepi8_epi16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15\n\n __m256i sumi1 = _mm256_setzero_si256();\n __m256i sumi2 = _mm256_setzero_si256();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q2_1 = _mm256_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],\n iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)],\n iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],\n iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);\n const __m256i q2_2 = _mm256_set_epi64x(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],\n iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)],\n iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],\n iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);\n qs += 8;\n\n __m256i aux256 = _mm256_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16));\n aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);\n const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2);\n const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1);\n\n aux256 = _mm256_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16));\n aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);\n const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2);\n const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2);\n\n signs += 4;\n\n const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1\n const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3\n\n const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+0)));\n const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+1)));\n sumi1 = _mm256_add_epi32(sumi1, p1);\n sumi2 = _mm256_add_epi32(sumi2, p2);\n }\n\n accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n }\n\n *s = 0.125f * hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n };\n\n const __m128i m4 = _mm_set1_epi8(0xf);\n const __m128i m1 = _mm_set1_epi8(1);\n\n const __m128i mask1_0 = _mm_loadu_si128((const __m128i*)k_mask1);\n const __m128i mask1_1 = _mm_loadu_si128((const __m128i*)k_mask1 + 1);\n const __m128i mask2_0 = _mm_loadu_si128((const __m128i*)k_mask2);\n const __m128i mask2_1 = _mm_loadu_si128((const __m128i*)k_mask2 + 1);\n\n uint64_t aux64;\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n memcpy(&aux64, x[i].scales, 8);\n const __m128i scales8 = _mm_add_epi8(_mm_slli_epi16(_mm_and_si128(_mm_set_epi64x(aux64 >> 4, aux64), m4), 1), m1);\n const __m128i scales16_0 = _mm_cvtepi8_epi16(scales8);\n const __m128i scales16_1 = _mm_cvtepi8_epi16(_mm_srli_si128(scales8, 8));\n\n __m128i sumi1_0 = _mm_setzero_si128();\n __m128i sumi1_1 = _mm_setzero_si128();\n __m128i sumi2_0 = _mm_setzero_si128();\n __m128i sumi2_1 = _mm_setzero_si128();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q2_1_0 = _mm_set_epi64x(iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],\n iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);\n const __m128i q2_1_1 = _mm_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],\n iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)]);\n const __m128i q2_2_0 = _mm_set_epi64x(iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],\n iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);\n const __m128i q2_2_1 = _mm_set_epi64x(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],\n iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)]);\n qs += 8;\n\n __m128i aux128_0 = _mm_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16));\n __m128i aux128_1 = aux128_0;\n aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);\n aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);\n const __m128i s2_1_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);\n const __m128i s2_1_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);\n const __m128i q8s_1_0 = _mm_sub_epi8(_mm_xor_si128(s2_1_0, q8_1_0), s2_1_0);\n const __m128i q8s_1_1 = _mm_sub_epi8(_mm_xor_si128(s2_1_1, q8_1_1), s2_1_1);\n\n aux128_0 = _mm_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16));\n aux128_1 = aux128_0;\n aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);\n aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);\n const __m128i s2_2_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);\n const __m128i s2_2_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);\n const __m128i q8s_2_0 = _mm_sub_epi8(_mm_xor_si128(s2_2_0, q8_2_0), s2_2_0);\n const __m128i q8s_2_1 = _mm_sub_epi8(_mm_xor_si128(s2_2_1, q8_2_1), s2_2_1);\n\n signs += 4;\n\n const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n\n const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_shuffle_epi8(scales16_0, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+0), 0)));\n const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_shuffle_epi8(scales16_1, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+0), 1)));\n const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_shuffle_epi8(scales16_0, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+1), 0)));\n const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_shuffle_epi8(scales16_1, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+1), 1)));\n sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);\n sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);\n sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);\n sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);\n }\n\n accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n }\n\n *s = 0.125f * hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n uint32_t aux32[2];\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n __m256i sumi1 = _mm256_setzero_si256();\n __m256i sumi2 = _mm256_setzero_si256();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q2_1 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],\n iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n q3 += 8;\n const __m256i q2_2 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],\n iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n q3 += 8;\n memcpy(aux32, gas, 8); gas += 8;\n const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],\n signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);\n const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],\n signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);\n const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);\n const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);\n const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);\n const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);\n const uint16_t ls1 = aux32[0] >> 28;\n const uint16_t ls2 = aux32[1] >> 28;\n const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));\n const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));\n sumi1 = _mm256_add_epi32(sumi1, p1);\n sumi2 = _mm256_add_epi32(sumi2, p2);\n }\n\n accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n }\n\n *s = 0.25f * hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n uint32_t aux32[2];\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n __m128i sumi1_0 = _mm_setzero_si128();\n __m128i sumi1_1 = _mm_setzero_si128();\n __m128i sumi2_0 = _mm_setzero_si128();\n __m128i sumi2_1 = _mm_setzero_si128();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q2_1_0 = _mm_set_epi32(iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n const __m128i q2_1_1 = _mm_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]]);\n q3 += 8;\n const __m128i q2_2_0 = _mm_set_epi32(iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n const __m128i q2_2_1 = _mm_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]]);\n q3 += 8;\n memcpy(aux32, gas, 8); gas += 8;\n const __m128i s2_1_0 = _mm_set_epi64x(signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);\n const __m128i s2_1_1 = _mm_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127]);\n const __m128i s2_2_0 = _mm_set_epi64x(signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);\n const __m128i s2_2_1 = _mm_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127]);\n const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, s2_1_0);\n const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, s2_1_1);\n const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, s2_2_0);\n const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, s2_2_1);\n const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n const uint16_t ls1 = aux32[0] >> 28;\n const uint16_t ls2 = aux32[1] >> 28;\n const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));\n const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));\n const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));\n const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));\n sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);\n sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);\n sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);\n sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);\n }\n\n accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n }\n\n *s = 0.25f * hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n };\n\n const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1);\n const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2);\n\n const __m256i idx_shift = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);\n const __m256i idx_mask = _mm256_set1_epi32(256);\n\n typedef union {\n __m256i vec[2];\n uint32_t index[16];\n } index_t;\n\n index_t idx;\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n __m256i sumi1 = _mm256_setzero_si256();\n __m256i sumi2 = _mm256_setzero_si256();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i idx_l = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)qs)); qs += 16;\n idx.vec[0] = _mm256_set1_epi32(qh[ib32+0]);\n idx.vec[1] = _mm256_set1_epi32(qh[ib32+1]);\n idx.vec[0] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[0], idx_shift), idx_mask);\n idx.vec[1] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[1], idx_shift), idx_mask);\n idx.vec[0] = _mm256_or_si256(idx.vec[0], _mm256_cvtepi16_epi32(_mm256_castsi256_si128(idx_l)));\n idx.vec[1] = _mm256_or_si256(idx.vec[1], _mm256_cvtepi16_epi32(_mm256_extractf128_si256(idx_l, 1)));\n\n // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.\n //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);\n //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);\n const __m256i q2_1 = _mm256_set_epi32(\n iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],\n iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]\n );\n const __m256i q2_2 = _mm256_set_epi32(\n iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],\n iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]\n );\n\n __m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16));\n aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);\n const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2);\n const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1);\n\n aux256 = _mm256_set1_epi32(signs[2] | (signs[3] << 16));\n aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);\n const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2);\n const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2);\n\n signs += 4;\n\n const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);\n const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);\n const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;\n const uint16_t ls2 = x[i].scales[ib32/2] >> 4;\n const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));\n const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));\n sumi1 = _mm256_add_epi32(sumi1, p1);\n sumi2 = _mm256_add_epi32(sumi2, p2);\n }\n\n accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n }\n\n *s = hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n };\n\n static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n };\n\n const __m128i mask1_0 = _mm_loadu_si128((const __m128i*)k_mask1);\n const __m128i mask1_1 = _mm_loadu_si128((const __m128i*)k_mask1 + 1);\n const __m128i mask2_0 = _mm_loadu_si128((const __m128i*)k_mask2);\n const __m128i mask2_1 = _mm_loadu_si128((const __m128i*)k_mask2 + 1);\n\n const __m128i idx_mul_0 = _mm_set_epi32(32, 64, 128, 256);\n const __m128i idx_mul_1 = _mm_set_epi32(2, 4, 8, 16);\n const __m128i idx_mask = _mm_set1_epi32(256);\n\n typedef union {\n __m128i vec[4];\n uint32_t index[16];\n } index_t;\n\n index_t idx;\n\n __m256 accumf = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n __m128i sumi1_0 = _mm_setzero_si128();\n __m128i sumi1_1 = _mm_setzero_si128();\n __m128i sumi2_0 = _mm_setzero_si128();\n __m128i sumi2_1 = _mm_setzero_si128();\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i qs_tmp = _mm_loadu_si128((const __m128i *)qs);\n const __m128i idx_l_0 = _mm_cvtepu8_epi16(qs_tmp);\n const __m128i idx_l_1 = _mm_cvtepu8_epi16(_mm_srli_si128(qs_tmp, 8)); qs += 16;\n idx.vec[0] = _mm_set1_epi32(qh[ib32+0]);\n idx.vec[1] = idx.vec[0];\n idx.vec[2] = _mm_set1_epi32(qh[ib32+1]);\n idx.vec[3] = idx.vec[2];\n\n idx.vec[0] = _mm_and_si128(_mm_mullo_epi32(idx.vec[0], idx_mul_0), idx_mask);\n idx.vec[1] = _mm_and_si128(_mm_mullo_epi32(idx.vec[1], idx_mul_1), idx_mask);\n idx.vec[2] = _mm_and_si128(_mm_mullo_epi32(idx.vec[2], idx_mul_0), idx_mask);\n idx.vec[3] = _mm_and_si128(_mm_mullo_epi32(idx.vec[3], idx_mul_1), idx_mask);\n\n idx.vec[0] = _mm_or_si128(idx.vec[0], _mm_cvtepi16_epi32(idx_l_0));\n idx.vec[1] = _mm_or_si128(idx.vec[1], _mm_cvtepi16_epi32(_mm_srli_si128(idx_l_0, 8)));\n idx.vec[2] = _mm_or_si128(idx.vec[2], _mm_cvtepi16_epi32(idx_l_1));\n idx.vec[3] = _mm_or_si128(idx.vec[3], _mm_cvtepi16_epi32(_mm_srli_si128(idx_l_1, 8)));\n\n const __m128i q2_1_0 = _mm_set_epi32(iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]);\n const __m128i q2_1_1 = _mm_set_epi32(iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]]);\n const __m128i q2_2_0 = _mm_set_epi32(iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[9]], iq3s_grid[idx.index[8]]);\n const __m128i q2_2_1 = _mm_set_epi32(iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]]);\n\n __m128i aux128_0 = _mm_set1_epi32(signs[0] | (signs[1] << 16));\n __m128i aux128_1 = aux128_0;\n aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);\n aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);\n const __m128i s2_1_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);\n const __m128i s2_1_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);\n const __m128i q8s_1_0 = _mm_sub_epi8(_mm_xor_si128(s2_1_0, q8_1_0), s2_1_0);\n const __m128i q8s_1_1 = _mm_sub_epi8(_mm_xor_si128(s2_1_1, q8_1_1), s2_1_1);\n\n aux128_0 = _mm_set1_epi32(signs[2] | (signs[3] << 16));\n aux128_1 = aux128_0;\n aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);\n aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);\n const __m128i s2_2_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);\n const __m128i s2_2_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);\n const __m128i q8s_2_0 = _mm_sub_epi8(_mm_xor_si128(s2_2_0, q8_2_0), s2_2_0);\n const __m128i q8s_2_1 = _mm_sub_epi8(_mm_xor_si128(s2_2_1, q8_2_1), s2_2_1);\n\n signs += 4;\n\n const __m128i dot1_0 = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n const __m128i dot1_1 = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n const __m128i dot2_0 = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n const __m128i dot2_1 = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;\n const uint16_t ls2 = x[i].scales[ib32/2] >> 4;\n const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));\n const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));\n const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));\n const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));\n sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);\n sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);\n sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);\n sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);\n }\n\n accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n }\n\n *s = hsum_float_8(accumf);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n __m256 accum = _mm256_setzero_ps();\n float accum1 = 0;\n for (int i = 0; i < nb; ++i) {\n\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint16_t * qh = x[i].qh;\n\n __m256i sumi = _mm256_setzero_si256();\n int sumi1 = 0;\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n#ifdef __BMI2__\n const uint64_t packed_idx1 = _pdep_u64(*(const uint32_t *)qs, 0x00ff00ff00ff00ffULL) | _pdep_u64(qh[ib], 0x700070007000700ULL);\n const uint64_t packed_idx2 = _pdep_u64(*(const uint32_t *)(qs + 4), 0x00ff00ff00ff00ffULL) | _pdep_u64(qh[ib + 1], 0x700070007000700ULL);\n const uint16_t *idx1 = (const uint16_t *)(&packed_idx1);\n const uint16_t *idx2 = (const uint16_t *)(&packed_idx2);\n const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[idx1[3]], iq1s_grid[idx1[2]], iq1s_grid[idx1[1]], iq1s_grid[idx1[0]]);\n const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[idx2[3]], iq1s_grid[idx2[2]], iq1s_grid[idx2[1]], iq1s_grid[idx2[0]]);\n#else\n const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)],\n iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]);\n const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)],\n iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]);\n#endif\n qs += 8;\n const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);\n const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);\n const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;\n const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;\n const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(ls1));\n const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(ls2));\n\n sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p1, p2));\n sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1\n + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;\n }\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n accum = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), accum);\n accum1 += d * sumi1;\n\n }\n\n *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;\n\n#elif defined __AVX__\n __m256 accum = _mm256_setzero_ps();\n float accum1 = 0;\n for (int i = 0; i < nb; ++i) {\n\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint16_t * qh = x[i].qh;\n\n __m128i sumi1_0 = _mm_setzero_si128();\n __m128i sumi1_1 = _mm_setzero_si128();\n int sumi1 = 0;\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n const __m128i q1b_1_0 = _mm_set_epi64x(iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]);\n const __m128i q1b_1_1 = _mm_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)]);\n const __m128i q1b_2_0 = _mm_set_epi64x(iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]);\n const __m128i q1b_2_1 = _mm_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)]);\n qs += 8;\n const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n\n const __m128i dot1_0 = mul_add_epi8_sse(q1b_1_0, q8b_1_0);\n const __m128i dot1_1 = mul_add_epi8_sse(q1b_1_1, q8b_1_1);\n const __m128i dot2_0 = mul_add_epi8_sse(q1b_2_0, q8b_2_0);\n const __m128i dot2_1 = mul_add_epi8_sse(q1b_2_1, q8b_2_1);\n const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;\n const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;\n const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(ls1));\n const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(ls1));\n const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(ls2));\n const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(ls2));\n\n sumi1_0 = _mm_add_epi32(sumi1_0, _mm_add_epi32(p1_0, p2_0));\n sumi1_1 = _mm_add_epi32(sumi1_1, _mm_add_epi32(p1_1, p2_1));\n sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1\n + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;\n }\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n accum = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(sumi1_1, sumi1_0))), accum);\n accum1 += d * sumi1;\n\n }\n\n *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_m * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n iq1m_scale_t scale;\n\n#if defined __AVX2__\n\n const __m256i mask = _mm256_set1_epi16(0x7);\n const __m256i mone = _mm256_set1_epi16(1);\n const __m256i mone8 = _mm256_set1_epi8(1);\n const __m256i mtwo8 = _mm256_set1_epi8(2);\n // VPSHUFB cannot cross 128-bit lanes so odd shifts go to upper half.\n const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);\n\n __m256 accum1 = _mm256_setzero_ps();\n __m256 accum2 = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint8_t * qh = x[i].qh;\n const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n // Extract 3-bit scales (16 values)\n __m256i scales = _mm256_set1_epi64x(*(const uint64_t*)sc);\n scales = _mm256_srlv_epi64(scales, scales_shift);\n scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);\n\n // Indices to repeat each scale 8 times.\n __m256i scales_idx1 = _mm256_set1_epi16(0x0100);\n __m256i scales_idx2 = _mm256_add_epi8(scales_idx1, _mm256_set1_epi8(8));\n\n __m256i sumi1 = _mm256_setzero_si256();\n __m256i sumi2 = _mm256_setzero_si256();\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n#ifdef __BMI2__\n const uint64_t packed_idx1 = _pdep_u64(*(const uint32_t *)qs, 0x00ff00ff00ff00ffULL)\n | _pdep_u64(*(const uint16_t*)(qh) & 0x7777, 0xf000f000f000f00ULL);\n const uint64_t packed_idx2 = _pdep_u64(*(const uint32_t *)(qs + 4), 0x00ff00ff00ff00ffULL)\n | _pdep_u64(*(const uint16_t*)(qh + 2) & 0x7777, 0xf000f000f000f00ULL);\n const uint16_t *idx1 = (const uint16_t *)(&packed_idx1);\n const uint16_t *idx2 = (const uint16_t *)(&packed_idx2);\n const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[idx1[3]], iq1s_grid[idx1[2]], iq1s_grid[idx1[1]], iq1s_grid[idx1[0]]);\n const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[idx2[3]], iq1s_grid[idx2[2]], iq1s_grid[idx2[1]], iq1s_grid[idx2[0]]);\n\n // Convert signs to bytes 0x81 (negative) or 0x01 (positive)\n const uint64_t delta_sign = _pdep_u64(*(const uint32_t*)(qh) & 0x88888888, 0xf0f0f0f0f0f0f0f0ULL);\n const __m256i delta1 = _mm256_or_si256(mone8, _mm256_cvtepi8_epi64(_mm_set1_epi32(delta_sign)));\n const __m256i delta2 = _mm256_or_si256(mone8, _mm256_cvtepi8_epi64(_mm_set1_epi32(delta_sign >> 32)));\n#else\n const __m256i q1b_1 = _mm256_set_epi64x(\n iq1s_grid[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)], iq1s_grid[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)],\n iq1s_grid[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)], iq1s_grid[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)]\n );\n const __m256i q1b_2 = _mm256_set_epi64x(\n iq1s_grid[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)], iq1s_grid[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)],\n iq1s_grid[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)], iq1s_grid[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)]\n );\n\n const __m256i delta1 = _mm256_set_epi64x(qh[1] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[1] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[0] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[0] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n const __m256i delta2 = _mm256_set_epi64x(qh[3] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[3] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[2] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[2] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n#endif\n const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);\n const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);\n const __m256i dot3 = _mm256_maddubs_epi16(mone8, _mm256_sign_epi8(q8b_1, delta1));\n const __m256i dot4 = _mm256_maddubs_epi16(mone8, _mm256_sign_epi8(q8b_2, delta2));\n\n __m256i scale1 = _mm256_shuffle_epi8(scales, scales_idx1);\n __m256i scale2 = _mm256_shuffle_epi8(scales, scales_idx2);\n\n scales_idx1 = _mm256_add_epi8(scales_idx1, mtwo8);\n scales_idx2 = _mm256_add_epi8(scales_idx2, mtwo8);\n\n const __m256i p1 = _mm256_madd_epi16(dot1, scale1);\n const __m256i p2 = _mm256_madd_epi16(dot2, scale2);\n const __m256i p3 = _mm256_madd_epi16(dot3, scale1);\n const __m256i p4 = _mm256_madd_epi16(dot4, scale2);\n\n sumi1 = _mm256_add_epi32(sumi1, _mm256_add_epi32(p1, p2));\n sumi2 = _mm256_add_epi32(sumi2, _mm256_add_epi32(p3, p4));\n\n qs += 8; qh += 4;\n }\n\n const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16));\n\n accum1 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi1), accum1);\n accum2 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi2), accum2);\n }\n\n *s = hsum_float_8(accum1) + IQ1M_DELTA * hsum_float_8(accum2);\n\n#elif defined __AVX__\n const __m128i mask = _mm_set1_epi16(0x7);\n const __m128i mone = _mm_set1_epi16(1);\n\n __m256 accum1 = _mm256_setzero_ps();\n __m256 accum2 = _mm256_setzero_ps();\n for (int i = 0; i < nb; ++i) {\n\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint8_t * qh = x[i].qh;\n const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n\n __m128i sumi1_0 = _mm_setzero_si128();\n __m128i sumi1_1 = _mm_setzero_si128();\n __m128i sumi2_0 = _mm_setzero_si128();\n __m128i sumi2_1 = _mm_setzero_si128();\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n const __m128i q1b_1_0 = _mm_set_epi64x(\n iq1s_grid[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)], iq1s_grid[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)]);\n const __m128i q1b_1_1 = _mm_set_epi64x(\n iq1s_grid[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)], iq1s_grid[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)]);\n const __m128i q1b_2_0 = _mm_set_epi64x(\n iq1s_grid[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)], iq1s_grid[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)]);\n const __m128i q1b_2_1 = _mm_set_epi64x(\n iq1s_grid[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)], iq1s_grid[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)]);\n const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n\n const __m128i dot1_0 = mul_add_epi8_sse(q1b_1_0, q8b_1_0);\n const __m128i dot1_1 = mul_add_epi8_sse(q1b_1_1, q8b_1_1);\n const __m128i dot2_0 = mul_add_epi8_sse(q1b_2_0, q8b_2_0);\n const __m128i dot2_1 = mul_add_epi8_sse(q1b_2_1, q8b_2_1);\n\n const __m128i delta1_0 = _mm_set_epi64x(qh[0] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[0] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n const __m128i delta1_1 = _mm_set_epi64x(qh[1] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[1] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n const __m128i delta2_0 = _mm_set_epi64x(qh[2] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[2] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n const __m128i delta2_1 = _mm_set_epi64x(qh[3] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n qh[3] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n\n const __m128i dot3_0 = mul_add_epi8_sse(delta1_0, q8b_1_0);\n const __m128i dot3_1 = mul_add_epi8_sse(delta1_1, q8b_1_1);\n const __m128i dot4_0 = mul_add_epi8_sse(delta2_0, q8b_2_0);\n const __m128i dot4_1 = mul_add_epi8_sse(delta2_1, q8b_2_1);\n\n __m128i scale1_0 = _mm_set1_epi16(sc[ib/2] >> 0);\n __m128i scale1_1 = _mm_set1_epi16(sc[ib/2] >> 3);\n __m128i scale2_0 = _mm_set1_epi16(sc[ib/2] >> 6);\n __m128i scale2_1 = _mm_set1_epi16(sc[ib/2] >> 9);\n\n scale1_0 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale1_0, mask), 1), mone);\n scale1_1 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale1_1, mask), 1), mone);\n scale2_0 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale2_0, mask), 1), mone);\n scale2_1 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale2_1, mask), 1), mone);\n const __m128i p1_0 = _mm_madd_epi16(dot1_0, scale1_0);\n const __m128i p1_1 = _mm_madd_epi16(dot1_1, scale1_1);\n const __m128i p2_0 = _mm_madd_epi16(dot2_0, scale2_0);\n const __m128i p2_1 = _mm_madd_epi16(dot2_1, scale2_1);\n const __m128i p3_0 = _mm_madd_epi16(dot3_0, scale1_0);\n const __m128i p3_1 = _mm_madd_epi16(dot3_1, scale1_1);\n const __m128i p4_0 = _mm_madd_epi16(dot4_0, scale2_0);\n const __m128i p4_1 = _mm_madd_epi16(dot4_1, scale2_1);\n\n sumi1_0 = _mm_add_epi32(sumi1_0, _mm_add_epi32(p1_0, p2_0));\n sumi1_1 = _mm_add_epi32(sumi1_1, _mm_add_epi32(p1_1, p2_1));\n sumi2_0 = _mm_add_epi32(sumi2_0, _mm_add_epi32(p3_0, p4_0));\n sumi2_1 = _mm_add_epi32(sumi2_1, _mm_add_epi32(p3_1, p4_1));\n\n qs += 8; qh += 4;\n }\n\n const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16));\n\n accum1 = _mm256_add_ps(_mm256_mul_ps(d, _mm256_cvtepi32_ps(MM256_SET_M128I(sumi1_1, sumi1_0))), accum1);\n accum2 = _mm256_add_ps(_mm256_mul_ps(d, _mm256_cvtepi32_ps(MM256_SET_M128I(sumi2_1, sumi2_0))), accum2);\n }\n\n *s = hsum_float_8(accum1) + IQ1M_DELTA * hsum_float_8(accum2);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n UNUSED(scale);\n ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK4_NL == 0);\n static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n const block_iq4_nl * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK4_NL;\n\n int ib = 0;\n float sumf = 0;\n\n#if defined __AVX2__\n\n const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);\n const __m128i m4b = _mm_set1_epi8(0x0f);\n const __m256i mone = _mm256_set1_epi16(1);\n\n __m256 accum1 = _mm256_setzero_ps();\n __m256 accum2 = _mm256_setzero_ps();\n for (; ib + 1 < nb; ib += 2) {\n const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[ib + 0].qs);\n const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs);\n const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[ib + 0].qs);\n const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[ib + 1].qs);\n const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),\n _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));\n const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),\n _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));\n const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n const __m256i p_1 = _mm256_madd_epi16(p16_1, mone);\n const __m256i p_2 = _mm256_madd_epi16(p16_2, mone);\n accum1 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_FP16_TO_FP32(x[ib + 0].d)),\n _mm256_cvtepi32_ps(p_1), accum1);\n accum2 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_FP16_TO_FP32(x[ib + 1].d)),\n _mm256_cvtepi32_ps(p_2), accum2);\n }\n\n sumf = hsum_float_8(_mm256_add_ps(accum1, accum2));\n\n#elif defined __AVX__\n const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);\n const __m128i m4b = _mm_set1_epi8(0x0f);\n\n __m256 accum = _mm256_setzero_ps();\n for (; ib + 1 < nb; ib += 2) {\n const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);\n const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);\n const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);\n const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);\n\n const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));\n const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));\n const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));\n const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));\n\n const __m256 p = mul_sum_i8_quad_float(q4b_1_0, q4b_1_1, q4b_2_0, q4b_2_1, q8b_1_0, q8b_1_1, q8b_2_0, q8b_2_1);\n const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d);\n accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);\n }\n\n sumf = hsum_float_8(accum);\n\n#endif\n for (; ib < nb; ++ib) {\n const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);\n int sumi1 = 0, sumi2 = 0;\n for (int j = 0; j < QK4_NL/2; ++j) {\n sumi1 += y[ib].qs[j+ 0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];\n sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >> 4];\n }\n sumf += d * (sumi1 + sumi2);\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_K == 0);\n\n const block_iq4_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);\n const __m128i m4b = _mm_set1_epi8(0x0f);\n\n __m256 accum = _mm256_setzero_ps();\n for (int ibl = 0; ibl < nb; ++ibl) {\n const uint8_t * qs = x[ibl].qs;\n const int8_t * q8 = y[ibl].qs;\n uint16_t sh = x[ibl].scales_h;\n __m256i sumi1 = _mm256_setzero_si256();\n __m256i sumi2 = _mm256_setzero_si256();\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)qs); qs += 16;\n const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)qs); qs += 16;\n const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),\n _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));\n const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),\n _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));\n const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;\n const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32;\n sh >>= 4;\n const __m256i p_1 = _mm256_madd_epi16(p16_1, _mm256_set1_epi16(ls1));\n const __m256i p_2 = _mm256_madd_epi16(p16_2, _mm256_set1_epi16(ls2));\n sumi1 = _mm256_add_epi32(p_1, sumi1);\n sumi2 = _mm256_add_epi32(p_2, sumi2);\n }\n accum = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ibl].d)*y[ibl].d),\n _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accum);\n }\n\n *s = hsum_float_8(accum);\n\n#elif defined __AVX__\n const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);\n const __m128i m4b = _mm_set1_epi8(0x0f);\n\n __m256 accum = _mm256_setzero_ps();\n for (int ibl = 0; ibl < nb; ++ibl) {\n const uint8_t * qs = x[ibl].qs;\n const int8_t * q8 = y[ibl].qs;\n uint16_t sh = x[ibl].scales_h;\n __m128i sumi1_0 = _mm_setzero_si128();\n __m128i sumi1_1 = _mm_setzero_si128();\n __m128i sumi2_0 = _mm_setzero_si128();\n __m128i sumi2_1 = _mm_setzero_si128();\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)qs); qs += 16;\n const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)qs); qs += 16;\n const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));\n const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));\n const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));\n const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));\n const __m128i p16_1_0 = mul_add_epi8_sse(q4b_1_0, q8b_1_0);\n const __m128i p16_1_1 = mul_add_epi8_sse(q4b_1_1, q8b_1_1);\n const __m128i p16_2_0 = mul_add_epi8_sse(q4b_2_0, q8b_2_0);\n const __m128i p16_2_1 = mul_add_epi8_sse(q4b_2_1, q8b_2_1);\n const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;\n const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32;\n sh >>= 4;\n const __m128i p_1_0 = _mm_madd_epi16(p16_1_0, _mm_set1_epi16(ls1));\n const __m128i p_1_1 = _mm_madd_epi16(p16_1_1, _mm_set1_epi16(ls1));\n const __m128i p_2_0 = _mm_madd_epi16(p16_2_0, _mm_set1_epi16(ls2));\n const __m128i p_2_1 = _mm_madd_epi16(p16_2_1, _mm_set1_epi16(ls2));\n sumi1_0 = _mm_add_epi32(p_1_0, sumi1_0);\n sumi1_1 = _mm_add_epi32(p_1_1, sumi1_1);\n sumi2_0 = _mm_add_epi32(p_2_0, sumi2_0);\n sumi2_1 = _mm_add_epi32(p_2_1, sumi2_1);\n }\n __m128i sumi12_0 = _mm_add_epi32(sumi1_0, sumi2_0);\n __m128i sumi12_1 = _mm_add_epi32(sumi1_1, sumi2_1);\n accum = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ibl].d)*y[ibl].d),\n _mm256_cvtepi32_ps(MM256_SET_M128I(sumi12_1, sumi12_0))), accum);\n }\n\n *s = hsum_float_8(accum);\n\n#else\n UNUSED(x);\n UNUSED(y);\n UNUSED(nb);\n ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n" }, { "path": "src/ggml-cpu/arch/x86/repack.cpp", "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"traits.h\"\n\n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GGML_CPU_CLANG_WORKAROUND\n#include \"../../repack.h\"\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#endif\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__AVX__)\n#if defined(__F16C__)\n#if defined(__AVX512F__)\n#define GGML_F32Cx8x2_LOAD(x, y) _mm512_cvtph_ps(_mm256_set_m128i(_mm_loadu_si128((const __m128i *)(y)), _mm_loadu_si128((const __m128i *)(x))))\n#define GGML_F32Cx16_REPEAT_LOAD(x) _mm512_cvtph_ps(_mm256_set_m128i(x, x))\n#endif\n// the _mm256_cvt intrinsics require F16C\n#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))\n#define GGML_F32Cx8_REPEAT_LOAD(x, loadMask) _mm256_cvtph_ps(_mm_shuffle_epi32(_mm_maskload_epi32((int const*)(x), loadMask), 68))\n#define GGML_F32Cx8_REARRANGE_LOAD(x, arrangeMask) _mm256_cvtph_ps(_mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) x), arrangeMask))\n#else\n#if defined(__AVX512F__)\nstatic inline __m512 __avx512_f32cx8x2_load(ggml_fp16_t *x, ggml_fp16_t *y) {\n float tmp[16];\n\n for (int i = 0; i < 8; i++) {\n tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n }\n\n for (int i = 0; i < 8; i++) {\n tmp[i + 8] = GGML_CPU_FP16_TO_FP32(y[i]);\n }\n\n return _mm512_loadu_ps(tmp);\n}\nstatic inline __m512 __avx512_repeat_f32cx16_load(__m128i x) {\n float tmp[16];\n uint16_t tmphalf[8];\n _mm_storeu_si128((__m128i*)tmphalf, x);\n\n for (int i = 0; i < 4; i++) {\n tmp[i] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n tmp[i + 4] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n tmp[i + 8] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n tmp[i + 12] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n }\n\n return _mm512_loadu_ps(tmp);\n}\n#endif\nstatic inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) {\n float tmp[8];\n\n for (int i = 0; i < 8; i++) {\n tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n }\n\n return _mm256_loadu_ps(tmp);\n}\nstatic inline __m256 __avx_repeat_f32cx8_load(ggml_fp16_t *x) {\n float tmp[8];\n\n for (int i = 0; i < 4; i++) {\n tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n tmp[i + 4] = GGML_CPU_FP16_TO_FP32(x[i]);\n }\n\n return _mm256_loadu_ps(tmp);\n}\nstatic inline __m256 __avx_rearranged_f32cx8_load(ggml_fp16_t *x, __m128i arrangeMask) {\n uint16_t tmphalf[8];\n float tmp[8];\n\n _mm_storeu_si128((__m128i*)tmphalf, _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) x), arrangeMask));\n for (int i = 0; i < 8; i++) {\n tmp[i] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n }\n\n return _mm256_loadu_ps(tmp);\n}\n\n#define GGML_F32Cx8_LOAD(x) __avx_f32cx8_load(x)\n#define GGML_F32Cx8_REPEAT_LOAD(x, loadMask) __avx_repeat_f32cx8_load(x)\n#define GGML_F32Cx8_REARRANGE_LOAD(x, arrangeMask) __avx_rearranged_f32cx8_load(x, arrangeMask)\n#if defined(__AVX512F__)\n#define GGML_F32Cx8x2_LOAD(x, y) __avx512_f32cx8x2_load(x, y)\n#define GGML_F32Cx16_REPEAT_LOAD(x) __avx512_repeat_f32cx16_load(x)\n#endif\n#endif\n#endif\n\nstatic inline int nearest_int(float fval) {\n assert(fabsf(fval) <= 4194303.f);\n float val = fval + 12582912.f;\n int i; memcpy(&i, &val, sizeof(int));\n return (i & 0x007fffff) - 0x00400000;\n}\n\n#if defined(__AVX2__) || defined(__AVX512F__)\n#if defined(__AVX512F__)\n// add int16_t pairwise and return as 512 bit int vector, then add the accumulator\nstatic inline __m512i sum_i16_pairs_acc_int32x16(const __m512i acc, const __m512i x) {\n const __m512i ones = _mm512_set1_epi16(1);\n return _mm512_add_epi32(acc, _mm512_madd_epi16(ones, x));\n}\n\nstatic inline __m512i mul_sum_us8_pairs_acc_int32x16(const __m512i acc, const __m512i ax, const __m512i sy) {\n#if defined(__AVX512VNNI__)\n return _mm512_dpbusd_epi32(acc, ax, sy);\n#else\n // Perform multiplication and create 16-bit values\n const __m512i dot = _mm512_maddubs_epi16(ax, sy);\n return sum_i16_pairs_acc_int32x16(acc, dot);\n#endif\n}\n\n// multiply int8_t, add results pairwise twice and return as 512 bit int vector，then add the accumulator\nstatic inline __m512i mul_sum_i8_pairs_acc_int32x16(const __m512i acc, const __m512i x, const __m512i y) {\n const __m512i zero = _mm512_setzero_si512();\n // Get absolute values of x vectors\n const __m512i ax = _mm512_abs_epi8(x);\n // Sign the values of the y vectors\n __mmask64 blt0 = _mm512_movepi8_mask(x);\n const __m512i sy = _mm512_mask_sub_epi8(y, blt0, zero, y);\n return mul_sum_us8_pairs_acc_int32x16(acc, ax, sy);\n}\n#endif\n\n// add int16_t pairwise and return as 256 bit int vector, then add the accumulator\nstatic inline __m256i sum_i16_pairs_acc_int32x8(const __m256i acc, const __m256i x) {\n const __m256i ones = _mm256_set1_epi16(1);\n return _mm256_add_epi32(acc, _mm256_madd_epi16(ones, x));\n}\n\nstatic inline __m256i mul_sum_us8_pairs_acc_int32x8(const __m256i acc, const __m256i ax, const __m256i sy) {\n#if defined(__AVX512VNNI__) && defined(__AVX512VL__)\n return _mm256_dpbusd_epi32(acc, ax, sy);\n#elif defined(__AVXVNNI__)\n return _mm256_dpbusd_avx_epi32(acc, ax, sy);\n#else\n // Perform multiplication and create 16-bit values\n const __m256i dot = _mm256_maddubs_epi16(ax, sy);\n return sum_i16_pairs_acc_int32x8(acc, dot);\n#endif\n}\n\n// Integer variant of the function defined in ggml-quants.c\n// multiply int8_t, add results pairwise twice and return as 256 bit int vector, then add the accumulator\nstatic inline __m256i mul_sum_i8_pairs_acc_int32x8(const __m256i acc, const __m256i x, const __m256i y) {\n#if defined(__AVXVNNIINT8__)\n return _mm256_dpbssd_epi32(acc, x, y);\n#else\n // Get absolute values of x vectors\n const __m256i ax = _mm256_sign_epi8(x, x);\n // Sign the values of the y vectors\n const __m256i sy = _mm256_sign_epi8(y, x);\n return mul_sum_us8_pairs_acc_int32x8(acc, ax, sy);\n#endif\n}\n#endif\n\nvoid ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n#if defined(__AVX2__) || defined(__AVX__)\n float id[4];\n __m256 srcv[4][4];\n __m256 idvec[4];\n\n for (int i = 0; i < nb; i++) {\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n // Load elements into 4 AVX vectors\n __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 32 );\n __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 32 + 8 );\n __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 32 + 16 );\n __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 32 + 24 );\n\n // Compute max(abs(e)) for the block\n const __m256 signBit = _mm256_set1_ps( -0.0f );\n __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );\n maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );\n maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );\n maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );\n\n __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );\n max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );\n max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );\n const float maxScalar = _mm_cvtss_f32( max4 );\n\n // Divided by 127.f to mirror results in quantize_row_q8_0\n const float d = maxScalar / 127.f;\n id[row_iter] = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; //d ? 1.0f / d : 0.0f;\n\n // Store the scale for the individual block\n y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n\n // Store the values in blocks of eight values - Aim is to use these later for block interleaving\n srcv[row_iter][0] = v0;\n srcv[row_iter][1] = v1;\n srcv[row_iter][2] = v2;\n srcv[row_iter][3] = v3;\n idvec[row_iter] = _mm256_set1_ps(id[row_iter]);\n }\n\n // The loop iterates four times - The aim is to get 4 corresponding chunks of eight bytes from the original weight blocks that are interleaved\n for (int j = 0; j < 4; j++) {\n // Apply the multiplier\n __m256 v0 = _mm256_mul_ps(srcv[0][j], idvec[0]);\n __m256 v1 = _mm256_mul_ps(srcv[1][j], idvec[1]);\n __m256 v2 = _mm256_mul_ps(srcv[2][j], idvec[2]);\n __m256 v3 = _mm256_mul_ps(srcv[3][j], idvec[3]);\n\n // Round to nearest integer\n v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );\n v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );\n v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );\n v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );\n\n // Convert floats to integers\n __m256i i0 = _mm256_cvtps_epi32( v0 );\n __m256i i1 = _mm256_cvtps_epi32( v1 );\n __m256i i2 = _mm256_cvtps_epi32( v2 );\n __m256i i3 = _mm256_cvtps_epi32( v3 );\n\n#if defined(__AVX2__)\n // Convert int32 to int16\n i0 = _mm256_packs_epi32( i0, i1 );\n i2 = _mm256_packs_epi32( i2, i3 );\n // Convert int16 to int8\n i0 = _mm256_packs_epi16( i0, i2 );\n\n // Permute and store the quantized weights in the required order after the pack instruction\n const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );\n i0 = _mm256_permutevar8x32_epi32( i0, perm );\n\n _mm256_storeu_si256((__m256i *)(y[i].qs + 32 * j), i0);\n#else\n // Since we don't have in AVX some necessary functions,\n // we split the registers in half and call AVX2 analogs from SSE\n __m128i ni0 = _mm256_castsi256_si128( i0 );\n __m128i ni1 = _mm256_extractf128_si256( i0, 1);\n __m128i ni2 = _mm256_castsi256_si128( i1 );\n __m128i ni3 = _mm256_extractf128_si256( i1, 1);\n __m128i ni4 = _mm256_castsi256_si128( i2 );\n __m128i ni5 = _mm256_extractf128_si256( i2, 1);\n __m128i ni6 = _mm256_castsi256_si128( i3 );\n __m128i ni7 = _mm256_extractf128_si256( i3, 1);\n\n // Convert int32 to int16\n ni0 = _mm_packs_epi32( ni0, ni1 );\n ni2 = _mm_packs_epi32( ni2, ni3 );\n ni4 = _mm_packs_epi32( ni4, ni5 );\n ni6 = _mm_packs_epi32( ni6, ni7 );\n // Convert int16 to int8\n ni0 = _mm_packs_epi16( ni0, ni2 );\n ni4 = _mm_packs_epi16( ni4, ni6 );\n _mm_storeu_si128((__m128i *)(y[i].qs + 32 * j), ni0);\n _mm_storeu_si128((__m128i *)(y[i].qs + 32 * j + 16), ni4);\n#endif\n }\n }\n\n#else\n UNUSED(nb);\n UNUSED(y);\n ggml_quantize_mat_q8_0_4x8_generic(x, vy, k);\n#endif\n}\n\nvoid ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK_K == 256);\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy;\n\n#if defined(__AVX2__)\n float iscale[4];\n __m256 srcv[4][32];\n __m256 iscale_vec[4];\n\n for (int i = 0; i < nb; i++) {\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n // Load elements into 4 AVX vectors\n __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 );\n __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 8 );\n __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 16 );\n __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 24 );\n\n // Compute max(abs(e)) for the block\n const __m256 signBit = _mm256_set1_ps( -0.0f );\n __m256 abs0 = _mm256_andnot_ps( signBit, v0 );\n __m256 abs1 = _mm256_andnot_ps( signBit, v1 );\n __m256 abs2 = _mm256_andnot_ps( signBit, v2 );\n __m256 abs3 = _mm256_andnot_ps( signBit, v3 );\n\n __m256 maxAbs = _mm256_max_ps( abs0, abs1 );\n maxAbs = _mm256_max_ps( maxAbs, abs2 );\n maxAbs = _mm256_max_ps( maxAbs, abs3 );\n\n __m256 mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ );\n __m256 mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ );\n __m256 mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ );\n __m256 mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ );\n\n __m256 maskAbs = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3));\n\n srcv[row_iter][0] = v0;\n srcv[row_iter][1] = v1;\n srcv[row_iter][2] = v2;\n srcv[row_iter][3] = v3;\n\n for (int sb = 1; sb < 8; sb++) {\n // Temporarily stores absolute quant values\n __m256 tempAbs = maxAbs;\n\n // Load elements into 4 AVX vectors\n __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32);\n __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 8 );\n __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 16 );\n __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 24 );\n\n // Compute max(abs(e)) for the block\n __m256 abs0 = _mm256_andnot_ps( signBit, v0 );\n __m256 abs1 = _mm256_andnot_ps( signBit, v1 );\n __m256 abs2 = _mm256_andnot_ps( signBit, v2 );\n __m256 abs3 = _mm256_andnot_ps( signBit, v3 );\n\n maxAbs = _mm256_max_ps( maxAbs, abs0 );\n maxAbs = _mm256_max_ps( maxAbs, abs1 );\n maxAbs = _mm256_max_ps( maxAbs, abs2 );\n maxAbs = _mm256_max_ps( maxAbs, abs3 );\n\n __m256 mask_prev = _mm256_cmp_ps( tempAbs, maxAbs, _CMP_EQ_OQ );\n maskAbs = _mm256_and_ps( maskAbs, mask_prev );\n\n mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ );\n mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ );\n mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ );\n mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ );\n\n __m256 mask_curr = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3));\n maskAbs = _mm256_or_ps(maskAbs, mask_curr);\n\n srcv[row_iter][sb * 4] = v0;\n srcv[row_iter][sb * 4 + 1] = v1;\n srcv[row_iter][sb * 4 + 2] = v2;\n srcv[row_iter][sb * 4 + 3] = v3;\n }\n\n __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );\n max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );\n max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );\n const float maxScalar = _mm_cvtss_f32( max4 );\n\n __m256 maxScalarVec = _mm256_set1_ps(maxScalar);\n\n __m256 mask_next = _mm256_cmp_ps( maxScalarVec, maxAbs, _CMP_EQ_OQ );\n __m256 finalMask = _mm256_and_ps(maskAbs, mask_next);\n\n const int mask = _mm256_movemask_ps(finalMask);\n iscale[row_iter] = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;\n\n if(mask) {\n iscale[row_iter] = ( maxScalar != 0.0f ) ? -127.f / maxScalar: 0.0f;\n }\n\n y[i].d[row_iter] = maxScalar ? 1/iscale[row_iter] : 0;\n iscale_vec[row_iter] = _mm256_set1_ps(iscale[row_iter]);\n }\n\n __m256i quants_interleaved[32];\n for (int j = 0; j < 32; j++) {\n // Apply the multiplier\n __m256 v0 = _mm256_mul_ps(srcv[0][j], iscale_vec[0]);\n __m256 v1 = _mm256_mul_ps(srcv[1][j], iscale_vec[1]);\n __m256 v2 = _mm256_mul_ps(srcv[2][j], iscale_vec[2]);\n __m256 v3 = _mm256_mul_ps(srcv[3][j], iscale_vec[3]);\n\n // Round to nearest integer\n v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );\n v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );\n v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );\n v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );\n\n // Convert floats to integers\n __m256i i0 = _mm256_cvtps_epi32( v0 );\n __m256i i1 = _mm256_cvtps_epi32( v1 );\n __m256i i2 = _mm256_cvtps_epi32( v2 );\n __m256i i3 = _mm256_cvtps_epi32( v3 );\n\n // Convert int32 to int16\n i0 = _mm256_packs_epi32( i0, i1 );\n i2 = _mm256_packs_epi32( i2, i3 );\n // Convert int16 to int8\n i0 = _mm256_packs_epi16( i0, i2 );\n\n // Permute and store the quantized weights in the required order after the pack instruction\n const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );\n i0 = _mm256_permutevar8x32_epi32( i0, perm );\n\n _mm256_storeu_si256((__m256i *)(y[i].qs + 32 * j), i0);\n quants_interleaved[j] = i0;\n }\n\n // Masks to shuffle the quants of corresponding sub blocks for rearranging quants for vectorized bsums computation\n __m256i shuffle_mask_sb2 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 0, 1, 4, 5, 6, 7, 8, 9, 8, 9, 12, 13, 14, 15));\n shuffle_mask_sb2 = _mm256_permute2f128_si256(shuffle_mask_sb2, shuffle_mask_sb2, 0);\n __m256i shuffle_mask_sb3 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 0, 1, 6, 7, 8, 9, 10, 11, 8, 9, 14, 15));\n shuffle_mask_sb3 = _mm256_permute2f128_si256(shuffle_mask_sb3, shuffle_mask_sb3, 0);\n __m256i shuffle_mask_sb4 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 4, 5, 0, 1, 8, 9, 10, 11, 12, 13, 8, 9));\n shuffle_mask_sb4 = _mm256_permute2f128_si256(shuffle_mask_sb4, shuffle_mask_sb4, 0);\n\n for (int k = 0; k < 4; k++) {\n // Quants from four different sub blocks are taken\n __m256i q0 = quants_interleaved[k * 8 + 0];\n __m256i q1 = quants_interleaved[k * 8 + 1];\n __m256i q2 = quants_interleaved[k * 8 + 2];\n __m256i q3 = quants_interleaved[k * 8 + 3];\n __m256i q4 = quants_interleaved[k * 8 + 4];\n __m256i q5 = quants_interleaved[k * 8 + 5];\n __m256i q6 = quants_interleaved[k * 8 + 6];\n __m256i q7 = quants_interleaved[k * 8 + 7];\n\n\n // The below code block has the first half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time\n __m256i sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2);\n __m256i sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34);\n __m256i sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3);\n sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68);\n __m256i sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4);\n sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136);\n\n __m256i one = _mm256_set1_epi8(1);\n __m256i bsums_r1 = _mm256_maddubs_epi16(one, sb_h1_interleaved);\n\n for (int l = 0; l < 3; l++) {\n // Quants value shifted to process next two values from each sub block\n q0 = _mm256_srli_epi64(q0, 16);\n q2 = _mm256_srli_epi64(q2, 16);\n q4 = _mm256_srli_epi64(q4, 16);\n q6 = _mm256_srli_epi64(q6, 16);\n\n sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2);\n sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34);\n sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3);\n sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68);\n sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4);\n sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136);\n\n bsums_r1 = _mm256_add_epi16(bsums_r1, _mm256_maddubs_epi16(one, sb_h1_interleaved));\n }\n\n // The below code block has the second half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time\n __m256i sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2);\n __m256i sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34);\n __m256i sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3);\n sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68);\n __m256i sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4);\n sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136);\n\n __m256i bsums_r2 = _mm256_maddubs_epi16(one, sb_h2_interleaved);\n\n for (int l = 0; l < 3; l++) {\n // Quants value shifted to process next two values from each sub block\n q1 = _mm256_srli_epi64(q1, 16);\n q3 = _mm256_srli_epi64(q3, 16);\n q5 = _mm256_srli_epi64(q5, 16);\n q7 = _mm256_srli_epi64(q7, 16);\n\n sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2);\n sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34);\n sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3);\n sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68);\n sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4);\n sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136);\n\n bsums_r2 = _mm256_add_epi16(bsums_r2, _mm256_maddubs_epi16(one, sb_h2_interleaved));\n }\n\n // Overall bsums in interleaved fashion computed by adding results of both halves\n __m256i bsums_r = _mm256_add_epi16(bsums_r1, bsums_r2);\n _mm256_storeu_si256((__m256i *)(y[i].bsums + 16 * k), bsums_r);\n }\n }\n\n#else\n UNUSED(nb);\n UNUSED(y);\n ggml_quantize_mat_q8_K_4x8_generic(x, vy, k);\n#endif\n}\n\n//\n// GEMV/GEMM templates\n//\n\n#if defined(__AVX2__) || defined(__AVX512F__)\n\n// GEMV for 8x blocks of 32 4-bit quants with a single scale factor per block\ntemplate\nstatic void gemv_q4_b32_8x8_q8_0_lut_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc, __m256i signextendlut) {\n static_assert(\n std::is_same_v ||\n std::is_same_v ||\n std::is_same_v,\n \"Unsupported block type\");\n\n const int qk = QK8_0;\n const int nb = n / qk;\n\n UNUSED(bs);\n\n __m128i changemask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0);\n __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);\n\n // Permute mask used for easier vector processing at later stages\n const __m256i m4b = _mm256_set1_epi8(0x0F);\n\n int64_t b_nb = n / 32;\n\n const block_tx8 * b_ptr_start = (const block_tx8 *)vx;\n const block_q8_0 * a_ptr_start = (const block_q8_0 *)vy;\n\n // Process Q8_0 blocks one by one\n for (int64_t y = 0; y < nr; y++) {\n\n // Pointers to LHS blocks of block_q8_0 format\n const block_q8_0 * a_ptr = a_ptr_start + (y * nb);\n\n // Take group of eight blocks at each pass of the loop and perform dot product operation\n for (int64_t x = 0; x < nc / 8; x++) {\n\n // Pointers to RHS blocks\n const block_tx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n // Master FP accumulator\n __m256 acc_row = _mm256_setzero_ps();\n\n for (int64_t b = 0; b < nb; b++) {\n // Load 8 blocks of 32 interleaved as 8 bytes (B0 - B7)\n const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs));\n const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs) + 1);\n const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs) + 2);\n const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs) + 3);\n\n // 4-bit -> 8-bit - Sign is maintained\n const __m256i rhs_vec_0123_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_0123_0, m4b)); // B0(0-7) B1(0-7) B2(0-7) B3(0-7)\n const __m256i rhs_vec_4567_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_4567_0, m4b)); // B4(0-7) B5(0-7) B6(0-7) B7(0-7)\n const __m256i rhs_vec_0123_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_0123_1, m4b)); // B0(8-15) B1(8-15) B2(8-15) B3(8-15)\n const __m256i rhs_vec_4567_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_4567_1, m4b)); // B0(8-15) B1(8-15) B2(8-15) B3(8-15)\n\n const __m256i rhs_vec_0123_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m4b)); // B0(16-23) B1(16-23) B2(16-23) B3(16-23)\n const __m256i rhs_vec_4567_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m4b)); // B4(16-23) B5(16-23) B6(16-23) B7(16-23)\n const __m256i rhs_vec_0123_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m4b)); // B0(24-31) B1(24-31) B2(24-31) B3(24-31)\n const __m256i rhs_vec_4567_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m4b)); // B4(24-31) B5(24-31) B6(24-31) B7(24-31)\n\n // Load the scale values for the 8 blocks interleaved in block_tx8\n __m256 col_scale_f32;\n if constexpr (\n std::is_same_v ||\n std::is_same_v) {\n col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, changemask);\n } else if constexpr (std::is_same_v) {\n // Load 8 E8M0 exponents and convert to float via LUT\n // Rearranged to match changemask order: 0,4,1,5,2,6,3,7\n col_scale_f32 = _mm256_set_ps(\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[7]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[3]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[6]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[2]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[5]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[1]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[4]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[0]));\n }\n\n // Load and convert to FP32 scale from block_q8_0\n const __m256 row_scale_f32 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(a_ptr[b].d));\n\n // Load the block values in block_q8_0 in batches of 16 bytes and replicate the same across 256 bit vector\n __m256i lhs_vec_0 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)a_ptr[b].qs));\n __m256i lhs_vec_1 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16)));\n\n lhs_vec_0 = _mm256_permute2f128_si256(lhs_vec_0, lhs_vec_0, 0); // A0 (0-15) A0(0-15)\n lhs_vec_1 = _mm256_permute2f128_si256(lhs_vec_1, lhs_vec_1, 0); // A0 (16-31) A0(16-31))\n\n __m256i iacc = _mm256_setzero_si256();\n\n // Dot product done within 32 bit lanes and accumulated in the same vector\n // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3)\n // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7)\n // ...........................................................................\n // B0(28-31) B4(28-31) B1(28-31) B5(28-31) B2(28-31) B6(28-31) B3(28-31) B7(28-31) with A0(28-31)\n\n iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_0 ,_mm256_shuffle_epi32(rhs_vec_4567_0, 177), 170), _mm256_shuffle_epi32(lhs_vec_0, 0));\n iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_0, 177) ,rhs_vec_4567_0, 170), _mm256_shuffle_epi32(lhs_vec_0, 85));\n\n iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_1 ,_mm256_shuffle_epi32(rhs_vec_4567_1, 177), 170), _mm256_shuffle_epi32(lhs_vec_0, 170));\n iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_1, 177) ,rhs_vec_4567_1, 170), _mm256_shuffle_epi32(lhs_vec_0, 255));\n\n iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_2 ,_mm256_shuffle_epi32(rhs_vec_4567_2, 177), 170), _mm256_shuffle_epi32(lhs_vec_1, 0));\n iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_2, 177) ,rhs_vec_4567_2, 170), _mm256_shuffle_epi32(lhs_vec_1, 85));\n\n iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_3 ,_mm256_shuffle_epi32(rhs_vec_4567_3, 177), 170), _mm256_shuffle_epi32(lhs_vec_1, 170));\n iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_3, 177) ,rhs_vec_4567_3, 170), _mm256_shuffle_epi32(lhs_vec_1, 255));\n\n // Accumulated values multiplied with appropriate scales\n acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row);\n }\n\n // Accumulated output values permuted so as to be stored in appropriate order post accumulation\n acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask);\n _mm256_storeu_ps(s + (y * nr + x * 8), acc_row);\n }\n }\n}\n\n// GEMM for 8x blocks of 32 4-bit quants with a single scale factor per block\ntemplate\nstatic void gemm_q4_b32_8x8_q8_0_lut_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc, __m256i signextendlut) {\n static_assert(\n std::is_same_v ||\n std::is_same_v ||\n std::is_same_v,\n \"Unsupported block type\");\n\n const int qk = QK8_0;\n const int nb = n / qk;\n\n const block_tx8 * b_ptr_start = (const block_tx8 *)vx;\n const block_q8_0x4 * a_ptr_start = (const block_q8_0x4 *)vy;\n\n int64_t b_nb = n / 32;\n int64_t y = 0;\n // Mask to mask out nibbles from packed bytes\n const __m256i m4b = _mm256_set1_epi8(0x0F);\n const __m128i loadMask = _mm_blend_epi32(_mm_setzero_si128(), _mm_set1_epi32(0xFFFFFFFF), 3);\n // Permute mask used for easier vector processing at later stages\n __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4);\n int64_t xstart = 0;\n int anr = nr - nr%16; // Used to align nr with boundary of 16\n#if defined(__AVX512BW__) && defined(__AVX512DQ__)\n int anc = nc - nc%16; // Used to align nc with boundary of 16\n // Mask to mask out nibbles from packed bytes expanded to 512 bit length\n const __m512i m4bexpanded = _mm512_set1_epi8(0x0F);\n // Lookup table to convert signed nibbles to signed bytes expanded to 512 bit length\n __m512i signextendlutexpanded = _mm512_inserti32x8(_mm512_castsi256_si512(signextendlut), signextendlut, 1);\n\n // Take group of four block_q8_0x4 structures at each pass of the loop and perform dot product operation\n for (; y < anr / 4; y += 4) {\n\n const block_q8_0x4 * a_ptrs[4];\n\n a_ptrs[0] = a_ptr_start + (y * nb);\n for (int i = 0; i < 3; ++i) {\n a_ptrs[i + 1] = a_ptrs[i] + nb;\n }\n\n // Take group of two block_tx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = 0; x < anc / 8; x += 2) {\n\n const block_tx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n const block_tx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n // Master FP accumulators\n __m512 acc_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_rows[i] = _mm512_setzero_ps();\n }\n\n for (int64_t b = 0; b < nb; b++) {\n // Load the sixteen blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....BE,BF\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 32));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 64));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 96));\n\n const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs));\n const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 32));\n const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 64));\n const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 96));\n\n // Save the values in the following vectors in the formats B0B1B4B5B8B9BCBD, B2B3B6B7BABBBEBF for further processing and storing of values\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n\n const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n // 4-bit -> 8-bit - Sign is maintained\n const __m512i rhs_mat_014589CD_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7) B8(0-7) B9(0-7) BC(0-7) BD(0-7)\n const __m512i rhs_mat_2367ABEF_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7) BA(0-7) BB(0-7) BE(0-7) BF(0-7)\n\n const __m512i rhs_mat_014589CD_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15) B8(8-15) B9(8-15) BC(8-15) BD(8-15)\n const __m512i rhs_mat_2367ABEF_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15) BA(8-15) BB(8-15) BE(8-15) BF(8-15)\n\n const __m512i rhs_mat_014589CD_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23) B8(16-23) B9(16-23) BC(16-23) BD(16-23)\n const __m512i rhs_mat_2367ABEF_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23) BA(16-23) BB(16-23) BE(16-23) BF(16-23)\n\n const __m512i rhs_mat_014589CD_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31) B8(24-31) B9(24-31) BC(24-31) BD(24-31)\n const __m512i rhs_mat_2367ABEF_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31) BA(24-31) BB(24-31) BE(24-31) BF(24-31)\n\n // Shuffle pattern one - right side input\n const __m512i rhs_mat_014589CD_0_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)136); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3) B8(0-3) B9(0-3) B8(0-3) B9(0-3) BC(0-3) BD(0-3) BC(0-3) BD(0-3)\n const __m512i rhs_mat_2367ABEF_0_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)136); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3) BA(0-3) BB(0-3) BA(0-3) BB(0-3) BE(0-3) BF(0-3) BE(0-3) BF(0-3)\n\n const __m512i rhs_mat_014589CD_1_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)136); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11) B8(8-11) B9(8-11) B8(8-11) B9(8-11) BC(8-11) BD(8-11) BC(8-11) BD(8-11)\n const __m512i rhs_mat_2367ABEF_1_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)136); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11) BA(8-11) BB(8-11) BA(8-11) BB(8-11) BE(8-11) BF(8-11) BE(8-11) BF(8-11)\n\n const __m512i rhs_mat_014589CD_2_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)136); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19) B8(16-19) B9(16-19) B8(16-19) B9(16-19) BC(16-19) BD(16-19) BC(16-19) BD(16-19)\n const __m512i rhs_mat_2367ABEF_2_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)136); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19) BA(16-19) BB(16-19) BA(16-19) BB(16-19) BE(16-19) BF(16-19) BE(16-19) BF(16-19)\n\n const __m512i rhs_mat_014589CD_3_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)136); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27) B8(24-27) B9(24-27) B8(24-27) B9(24-27) BC(24-27) BD(24-27) BC(24-27) BD(24-27)\n const __m512i rhs_mat_2367ABEF_3_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)136); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27) BA(24-27) BB(24-27) BA(24-27) BB(24-27) BE(24-27) BF(24-27) BE(24-27) BF(24-27)\n\n // Shuffle pattern two - right side input\n\n const __m512i rhs_mat_014589CD_0_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)221); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7) B8(4-7) B9(4-7) B8(4-7) B9(4-7) BC(4-7) BD(4-7) BC(4-7) BD(4-7)\n const __m512i rhs_mat_2367ABEF_0_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)221); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7) BA(4-7) BB(4-7) BA(4-7) BB(4-7) BE(4-7) BF(4-7) BE(4-7) BF(4-7)\n\n const __m512i rhs_mat_014589CD_1_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)221); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15) B8(12-15) B9(12-15) B8(12-15) B9(12-15) BC(12-15) BD(12-15) BC(12-15) BD(12-15)\n const __m512i rhs_mat_2367ABEF_1_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)221); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15) BA(12-15) BB(12-15) BA(12-15) BB(12-15) BE(12-15) BF(12-15) BE(12-15) BF(12-15)\n\n const __m512i rhs_mat_014589CD_2_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)221); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23) B8(20-23) B9(20-23) B8(20-23) B9(20-23) BC(20-23) BD(20-23) BC(20-23) BD(20-23)\n const __m512i rhs_mat_2367ABEF_2_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)221); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23) BA(20-23) BB(20-23) BA(20-23) BB(20-23) BE(20-23) BF(20-23) BE(20-23) BF(20-23)\n\n const __m512i rhs_mat_014589CD_3_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)221); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31) B8(28-31) B9(28-31) B8(28-31) B9(28-31) BC(28-31) BD(28-31) BC(28-31) BD(28-31)\n const __m512i rhs_mat_2367ABEF_3_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)221); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31) BA(28-31) BB(28-31) BA(28-31) BB(28-31) BE(28-31) BF(28-31) BE(28-31) BF(28-31)\n\n // Scale values - Load the weight scale values of two block_tx8\n __m512 col_scale_f32;\n if constexpr (\n std::is_same_v ||\n std::is_same_v) {\n col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n } else if constexpr (std::is_same_v) {\n //TODO: simd-ify\n col_scale_f32 = _mm512_set_ps(\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[7]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[6]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[5]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[4]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[3]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[2]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[1]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[0]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[7]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[6]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[5]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[4]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[3]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[2]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[1]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[0]));\n }\n\n // Process LHS in pairs of rows\n for (int rp = 0; rp < 4; rp++) {\n\n // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector\n __m256i lhs_mat_ymm_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs)));\n __m256i lhs_mat_ymm_01_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 0);\n __m256i lhs_mat_ymm_23_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 17);\n __m256i lhs_mat_ymm_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 32)));\n __m256i lhs_mat_ymm_01_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 0);\n __m256i lhs_mat_ymm_23_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 17);\n __m256i lhs_mat_ymm_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 64)));\n __m256i lhs_mat_ymm_01_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 0);\n __m256i lhs_mat_ymm_23_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 17);\n __m256i lhs_mat_ymm_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 96)));\n __m256i lhs_mat_ymm_01_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 0);\n __m256i lhs_mat_ymm_23_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 17);\n\n __m512i lhs_mat_01_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_0), lhs_mat_ymm_01_0, 1);\n __m512i lhs_mat_23_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_0), lhs_mat_ymm_23_0, 1);\n __m512i lhs_mat_01_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_1), lhs_mat_ymm_01_1, 1);\n __m512i lhs_mat_23_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_1), lhs_mat_ymm_23_1, 1);\n __m512i lhs_mat_01_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_2), lhs_mat_ymm_01_2, 1);\n __m512i lhs_mat_23_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_2), lhs_mat_ymm_23_2, 1);\n __m512i lhs_mat_01_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_3), lhs_mat_ymm_01_3, 1);\n __m512i lhs_mat_23_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_3), lhs_mat_ymm_23_3, 1);\n\n // Shuffle pattern one - left side input\n\n const __m512i lhs_mat_01_0_sp1 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)160); //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3)\n const __m512i lhs_mat_23_0_sp1 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)160); //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3)\n\n const __m512i lhs_mat_01_1_sp1 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)160); //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11)\n const __m512i lhs_mat_23_1_sp1 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)160); //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11)\n\n const __m512i lhs_mat_01_2_sp1 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)160); //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19)\n const __m512i lhs_mat_23_2_sp1 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)160); //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19)\n\n const __m512i lhs_mat_01_3_sp1 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)160); //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27)\n const __m512i lhs_mat_23_3_sp1 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)160); //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27)\n\n // Shuffle pattern two - left side input\n\n const __m512i lhs_mat_01_0_sp2 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)245); //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7)\n const __m512i lhs_mat_23_0_sp2 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)245); //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7)\n\n const __m512i lhs_mat_01_1_sp2 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)245); //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15)\n const __m512i lhs_mat_23_1_sp2 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)245); //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15)\n\n const __m512i lhs_mat_01_2_sp2 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)245); //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23)\n const __m512i lhs_mat_23_2_sp2 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)245); //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23)\n\n const __m512i lhs_mat_01_3_sp2 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)245); //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31)\n const __m512i lhs_mat_23_3_sp2 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)245); //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n // Resembles MMLAs into 2x2 matrices in ARM Version\n const __m512i zero = _mm512_setzero_epi32();\n __m512i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_01_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_01_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_01_0_sp1, rhs_mat_014589CD_0_sp1);\n __m512i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367ABEF_0_sp1);\n __m512i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_23_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_23_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_23_0_sp1, rhs_mat_014589CD_0_sp1);\n __m512i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367ABEF_0_sp1);\n __m512i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_01_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_01_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_01_0_sp2, rhs_mat_014589CD_0_sp2);\n __m512i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367ABEF_0_sp2);\n __m512i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_23_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_23_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_23_0_sp2, rhs_mat_014589CD_0_sp2);\n __m512i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367ABEF_0_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n __m512i iacc_mat_00 = _mm512_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2);\n __m512i iacc_mat_01 = _mm512_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2);\n __m512i iacc_mat_10 = _mm512_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2);\n __m512i iacc_mat_11 = _mm512_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2);\n\n\n // Straighten out to make 4 row vectors\n __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78));\n __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78), iacc_mat_01);\n __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78));\n __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78), iacc_mat_11);\n\n // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes\n const __m128i row_scale_f16 = _mm_shuffle_epi32(_mm_maskload_epi32((int const*)(a_ptrs[rp][b].d), loadMask), 68);\n const __m512 row_scale_f32 = GGML_F32Cx16_REPEAT_LOAD(row_scale_f16);\n\n // Multiply with appropriate scales and accumulate\n acc_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n acc_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n acc_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n acc_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n }\n }\n\n // Store the accumulated values\n for (int i = 0; i < 16; i++) {\n _mm512_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]);\n }\n }\n }\n\n // Take a block_q8_0x4 structures at each pass of the loop and perform dot product operation\n for (; y < nr / 4; y ++) {\n const block_q8_0x4 * a_ptr = a_ptr_start + (y * nb);\n\n // Take group of two block_tx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = 0; x < anc / 8; x += 2) {\n\n const block_tx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n const block_tx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n // Master FP accumulators\n __m512 acc_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_rows[i] = _mm512_setzero_ps();\n }\n\n for (int64_t b = 0; b < nb; b++) {\n // Load the sixteen blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....BE,BF\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 32));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 64));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 96));\n\n const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs));\n const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 32));\n const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 64));\n const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 96));\n\n // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n\n const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n // 4-bit -> 8-bit - Sign is maintained\n const __m512i rhs_mat_014589CD_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7) B8(0-7) B9(0-7) BC(0-7) BD(0-7)\n const __m512i rhs_mat_2367ABEF_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7) BA(0-7) BB(0-7) BE(0-7) BF(0-7)\n\n const __m512i rhs_mat_014589CD_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15) B8(8-15) B9(8-15) BC(8-15) BD(8-15)\n const __m512i rhs_mat_2367ABEF_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15) BA(8-15) BB(8-15) BE(8-15) BF(8-15)\n\n const __m512i rhs_mat_014589CD_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23) B8(16-23) B9(16-23) BC(16-23) BD(16-23)\n const __m512i rhs_mat_2367ABEF_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23) BA(16-23) BB(16-23) BE(16-23) BF(16-23)\n\n const __m512i rhs_mat_014589CD_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31) B8(24-31) B9(24-31) BC(24-31) BD(24-31)\n const __m512i rhs_mat_2367ABEF_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31) BA(24-31) BB(24-31) BE(24-31) BF(24-31)\n\n // Shuffle pattern one - right side input\n const __m512i rhs_mat_014589CD_0_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)136); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3) B8(0-3) B9(0-3) B8(0-3) B9(0-3) BC(0-3) BD(0-3) BC(0-3) BD(0-3)\n const __m512i rhs_mat_2367ABEF_0_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)136); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3) BA(0-3) BB(0-3) BA(0-3) BB(0-3) BE(0-3) BF(0-3) BE(0-3) BF(0-3)\n\n const __m512i rhs_mat_014589CD_1_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)136); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11) B8(8-11) B9(8-11) B8(8-11) B9(8-11) BC(8-11) BD(8-11) BC(8-11) BD(8-11)\n const __m512i rhs_mat_2367ABEF_1_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)136); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11) BA(8-11) BB(8-11) BA(8-11) BB(8-11) BE(8-11) BF(8-11) BE(8-11) BF(8-11)\n\n const __m512i rhs_mat_014589CD_2_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)136); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19) B8(16-19) B9(16-19) B8(16-19) B9(16-19) BC(16-19) BD(16-19) BC(16-19) BD(16-19)\n const __m512i rhs_mat_2367ABEF_2_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)136); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19) BA(16-19) BB(16-19) BA(16-19) BB(16-19) BE(16-19) BF(16-19) BE(16-19) BF(16-19)\n\n const __m512i rhs_mat_014589CD_3_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)136); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27) B8(24-27) B9(24-27) B8(24-27) B9(24-27) BC(24-27) BD(24-27) BC(24-27) BD(24-27)\n const __m512i rhs_mat_2367ABEF_3_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)136); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27) BA(24-27) BB(24-27) BA(24-27) BB(24-27) BE(24-27) BF(24-27) BE(24-27) BF(24-27)\n\n // Shuffle pattern two - right side input\n\n const __m512i rhs_mat_014589CD_0_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)221); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7) B8(4-7) B9(4-7) B8(4-7) B9(4-7) BC(4-7) BD(4-7) BC(4-7) BD(4-7)\n const __m512i rhs_mat_2367ABEF_0_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)221); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7) BA(4-7) BB(4-7) BA(4-7) BB(4-7) BE(4-7) BF(4-7) BE(4-7) BF(4-7)\n\n const __m512i rhs_mat_014589CD_1_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)221); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15) B8(12-15) B9(12-15) B8(12-15) B9(12-15) BC(12-15) BD(12-15) BC(12-15) BD(12-15)\n const __m512i rhs_mat_2367ABEF_1_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)221); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15) BA(12-15) BB(12-15) BA(12-15) BB(12-15) BE(12-15) BF(12-15) BE(12-15) BF(12-15)\n\n const __m512i rhs_mat_014589CD_2_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)221); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23) B8(20-23) B9(20-23) B8(20-23) B9(20-23) BC(20-23) BD(20-23) BC(20-23) BD(20-23)\n const __m512i rhs_mat_2367ABEF_2_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)221); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23) BA(20-23) BB(20-23) BA(20-23) BB(20-23) BE(20-23) BF(20-23) BE(20-23) BF(20-23)\n\n const __m512i rhs_mat_014589CD_3_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)221); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31) B8(28-31) B9(28-31) B8(28-31) B9(28-31) BC(28-31) BD(28-31) BC(28-31) BD(28-31)\n const __m512i rhs_mat_2367ABEF_3_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)221); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31) BA(28-31) BB(28-31) BA(28-31) BB(28-31) BE(28-31) BF(28-31) BE(28-31) BF(28-31)\n\n\n // Scale values - Load the weight scale values of two block_tx8\n __m512 col_scale_f32;\n if constexpr (\n std::is_same_v ||\n std::is_same_v) {\n col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n } else if constexpr (std::is_same_v) {\n //TODO: simd-ify\n col_scale_f32 = _mm512_set_ps(\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[7]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[6]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[5]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[4]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[3]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[2]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[1]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[0]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[7]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[6]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[5]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[4]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[3]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[2]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[1]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[0]));\n }\n\n // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector\n __m256i lhs_mat_ymm_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs)));\n __m256i lhs_mat_ymm_01_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 0);\n __m256i lhs_mat_ymm_23_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 17);\n __m256i lhs_mat_ymm_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 32)));\n __m256i lhs_mat_ymm_01_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 0);\n __m256i lhs_mat_ymm_23_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 17);\n __m256i lhs_mat_ymm_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 64)));\n __m256i lhs_mat_ymm_01_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 0);\n __m256i lhs_mat_ymm_23_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 17);\n __m256i lhs_mat_ymm_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 96)));\n __m256i lhs_mat_ymm_01_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 0);\n __m256i lhs_mat_ymm_23_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 17);\n\n __m512i lhs_mat_01_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_0), lhs_mat_ymm_01_0, 1);\n __m512i lhs_mat_23_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_0), lhs_mat_ymm_23_0, 1);\n __m512i lhs_mat_01_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_1), lhs_mat_ymm_01_1, 1);\n __m512i lhs_mat_23_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_1), lhs_mat_ymm_23_1, 1);\n __m512i lhs_mat_01_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_2), lhs_mat_ymm_01_2, 1);\n __m512i lhs_mat_23_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_2), lhs_mat_ymm_23_2, 1);\n __m512i lhs_mat_01_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_3), lhs_mat_ymm_01_3, 1);\n __m512i lhs_mat_23_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_3), lhs_mat_ymm_23_3, 1);\n\n // Shuffle pattern one - left side input\n\n const __m512i lhs_mat_01_0_sp1 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)160); //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3)\n const __m512i lhs_mat_23_0_sp1 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)160); //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3)\n\n const __m512i lhs_mat_01_1_sp1 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)160); //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11)\n const __m512i lhs_mat_23_1_sp1 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)160); //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11)\n\n const __m512i lhs_mat_01_2_sp1 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)160); //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19)\n const __m512i lhs_mat_23_2_sp1 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)160); //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19)\n\n const __m512i lhs_mat_01_3_sp1 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)160); //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27)\n const __m512i lhs_mat_23_3_sp1 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)160); //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27)\n\n // Shuffle pattern two - left side input\n\n const __m512i lhs_mat_01_0_sp2 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)245); //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7)\n const __m512i lhs_mat_23_0_sp2 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)245); //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7)\n\n const __m512i lhs_mat_01_1_sp2 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)245); //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15)\n const __m512i lhs_mat_23_1_sp2 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)245); //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15)\n\n const __m512i lhs_mat_01_2_sp2 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)245); //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23)\n const __m512i lhs_mat_23_2_sp2 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)245); //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23)\n\n const __m512i lhs_mat_01_3_sp2 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)245); //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31)\n const __m512i lhs_mat_23_3_sp2 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)245); //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n // Resembles MMLAs into 2x2 matrices in ARM Version\n const __m512i zero = _mm512_setzero_epi32();\n __m512i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_01_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_01_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_01_0_sp1, rhs_mat_014589CD_0_sp1);\n __m512i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367ABEF_0_sp1);\n __m512i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_23_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_23_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_23_0_sp1, rhs_mat_014589CD_0_sp1);\n __m512i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367ABEF_0_sp1);\n __m512i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_01_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_01_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_01_0_sp2, rhs_mat_014589CD_0_sp2);\n __m512i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367ABEF_0_sp2);\n __m512i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_23_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_23_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_23_0_sp2, rhs_mat_014589CD_0_sp2);\n __m512i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367ABEF_0_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n __m512i iacc_mat_00 = _mm512_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2);\n __m512i iacc_mat_01 = _mm512_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2);\n __m512i iacc_mat_10 = _mm512_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2);\n __m512i iacc_mat_11 = _mm512_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2);\n\n\n // Straighten out to make 4 row vectors\n __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78));\n __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78), iacc_mat_01);\n __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78));\n __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78), iacc_mat_11);\n\n // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes\n const __m128i row_scale_f16 = _mm_shuffle_epi32(_mm_maskload_epi32((int const*)(a_ptr[b].d), loadMask), 68);\n const __m512 row_scale_f32 = GGML_F32Cx16_REPEAT_LOAD(row_scale_f16);\n\n // Multiply with appropriate scales and accumulate\n acc_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n acc_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n acc_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n acc_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n }\n\n // Store the accumulated values\n for (int i = 0; i < 4; i++) {\n _mm512_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]);\n }\n }\n }\n if (anc != nc) {\n xstart = anc/8;\n y = 0;\n }\n#endif // __AVX512BW__ && __AVX512DQ__\n\n // Take group of four block_q8_0x4 structures at each pass of the loop and perform dot product operation\n\n for (; y < anr / 4; y += 4) {\n const block_q8_0x4 * a_ptrs[4];\n\n a_ptrs[0] = a_ptr_start + (y * nb);\n for (int i = 0; i < 3; ++i) {\n a_ptrs[i + 1] = a_ptrs[i] + nb;\n }\n\n // Take group of eight block_tx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = xstart; x < nc / 8; x++) {\n\n const block_tx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n // Master FP accumulators\n __m256 acc_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_rows[i] = _mm256_setzero_ps();\n }\n\n for (int64_t b = 0; b < nb; b++) {\n // Load the eight blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96));\n\n // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n // 4-bit -> 8-bit - Sign is maintained\n const __m256i rhs_mat_0145_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_0, m4b)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7)\n const __m256i rhs_mat_2367_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_0, m4b)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7)\n\n const __m256i rhs_mat_0145_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_1, m4b)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15)\n const __m256i rhs_mat_2367_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_1, m4b)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15)\n\n const __m256i rhs_mat_0145_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23)\n const __m256i rhs_mat_2367_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23)\n\n const __m256i rhs_mat_0145_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31)\n const __m256i rhs_mat_2367_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31)\n\n // Shuffle pattern one - right side input\n const __m256i rhs_mat_0145_0_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_0, 136); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3)\n const __m256i rhs_mat_2367_0_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_0, 136); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3)\n\n const __m256i rhs_mat_0145_1_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_1, 136); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11)\n const __m256i rhs_mat_2367_1_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_1, 136); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11)\n\n const __m256i rhs_mat_0145_2_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_2, 136); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19)\n const __m256i rhs_mat_2367_2_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_2, 136); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19)\n\n const __m256i rhs_mat_0145_3_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_3, 136); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27)\n const __m256i rhs_mat_2367_3_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_3, 136); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27)\n\n // Shuffle pattern two - right side input\n\n const __m256i rhs_mat_0145_0_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_0, 221); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7)\n const __m256i rhs_mat_2367_0_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_0, 221); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7)\n\n const __m256i rhs_mat_0145_1_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_1, 221); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15)\n const __m256i rhs_mat_2367_1_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_1, 221); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15)\n\n const __m256i rhs_mat_0145_2_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_2, 221); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23)\n const __m256i rhs_mat_2367_2_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_2, 221); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23)\n\n const __m256i rhs_mat_0145_3_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_3, 221); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31)\n const __m256i rhs_mat_2367_3_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_3, 221); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31)\n\n // Scale values - Load the wight scale values of block_tx8\n __m256 col_scale_f32;\n if constexpr (\n std::is_same_v ||\n std::is_same_v) {\n col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n } else if constexpr (std::is_same_v) {\n col_scale_f32 = _mm256_set_ps(\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[7]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[6]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[5]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[4]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[3]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[2]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[1]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[0]));\n }\n\n // Process LHS in groups of four\n for (int rp = 0; rp < 4; rp++) {\n // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n __m256i lhs_mat_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs)));\n __m256i lhs_mat_01_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 0);\n __m256i lhs_mat_23_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 17);\n __m256i lhs_mat_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 32)));\n __m256i lhs_mat_01_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 0);\n __m256i lhs_mat_23_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 17);\n __m256i lhs_mat_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 64)));\n __m256i lhs_mat_01_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 0);\n __m256i lhs_mat_23_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 17);\n __m256i lhs_mat_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 96)));\n __m256i lhs_mat_01_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 0);\n __m256i lhs_mat_23_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 17);\n\n // Shuffle pattern one - left side input\n const __m256i lhs_mat_01_0_sp1 = _mm256_shuffle_epi32(lhs_mat_01_0, 160); //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3)\n const __m256i lhs_mat_23_0_sp1 = _mm256_shuffle_epi32(lhs_mat_23_0, 160); //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3)\n\n const __m256i lhs_mat_01_1_sp1 = _mm256_shuffle_epi32(lhs_mat_01_1, 160); //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11)\n const __m256i lhs_mat_23_1_sp1 = _mm256_shuffle_epi32(lhs_mat_23_1, 160); //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11)\n\n const __m256i lhs_mat_01_2_sp1 = _mm256_shuffle_epi32(lhs_mat_01_2, 160); //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19)\n const __m256i lhs_mat_23_2_sp1 = _mm256_shuffle_epi32(lhs_mat_23_2, 160); //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19)\n\n const __m256i lhs_mat_01_3_sp1 = _mm256_shuffle_epi32(lhs_mat_01_3, 160); //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27)\n const __m256i lhs_mat_23_3_sp1 = _mm256_shuffle_epi32(lhs_mat_23_3, 160); //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27)\n\n // Shuffle pattern two - left side input\n const __m256i lhs_mat_01_0_sp2 = _mm256_shuffle_epi32(lhs_mat_01_0, 245); //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7)\n const __m256i lhs_mat_23_0_sp2 = _mm256_shuffle_epi32(lhs_mat_23_0, 245); //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7)\n\n const __m256i lhs_mat_01_1_sp2 = _mm256_shuffle_epi32(lhs_mat_01_1, 245); //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15)\n const __m256i lhs_mat_23_1_sp2 = _mm256_shuffle_epi32(lhs_mat_23_1, 245); //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15)\n\n const __m256i lhs_mat_01_2_sp2 = _mm256_shuffle_epi32(lhs_mat_01_2, 245); //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23)\n const __m256i lhs_mat_23_2_sp2 = _mm256_shuffle_epi32(lhs_mat_23_2, 245); //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23)\n\n const __m256i lhs_mat_01_3_sp2 = _mm256_shuffle_epi32(lhs_mat_01_3, 245); //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31)\n const __m256i lhs_mat_23_3_sp2 = _mm256_shuffle_epi32(lhs_mat_23_3, 245); //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n // Resembles MMLAs into 2x2 matrices in ARM Version\n const __m256i zero = _mm256_setzero_si256();\n __m256i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_01_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_01_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_01_0_sp1, rhs_mat_0145_0_sp1);\n __m256i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367_0_sp1);\n __m256i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_23_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_23_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_23_0_sp1, rhs_mat_0145_0_sp1);\n __m256i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367_0_sp1);\n __m256i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_01_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_01_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_01_0_sp2, rhs_mat_0145_0_sp2);\n __m256i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367_0_sp2);\n __m256i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_23_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_23_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_23_0_sp2, rhs_mat_0145_0_sp2);\n __m256i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367_0_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n __m256i iacc_mat_00 = _mm256_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2);\n __m256i iacc_mat_01 = _mm256_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2);\n __m256i iacc_mat_10 = _mm256_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2);\n __m256i iacc_mat_11 = _mm256_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2);\n\n // Straighten out to make 4 row vectors\n __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204);\n __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204);\n __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204);\n __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204);\n\n // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes\n const __m256 row_scale_f32 = GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask);\n\n // Multiply with appropriate scales and accumulate\n acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n }\n }\n\n // Store the accumulated values\n for (int i = 0; i < 16; i++) {\n _mm256_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]);\n }\n }\n }\n\n // Take a block_q8_0x4 structures at each pass of the loop and perform dot product operation\n for (; y < nr / 4; y ++) {\n const block_q8_0x4 * a_ptr = a_ptr_start + (y * nb);\n\n // Load the eight blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n for (int64_t x = xstart; x < nc / 8; x++) {\n const block_tx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n // Master FP accumulators\n __m256 acc_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_rows[i] = _mm256_setzero_ps();\n }\n\n for (int64_t b = 0; b < nb; b++) {\n // Load the eight block_q8_0 quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96));\n\n // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n // 4-bit -> 8-bit - Sign is maintained\n const __m256i rhs_mat_0145_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_0, m4b)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7)\n const __m256i rhs_mat_2367_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_0, m4b)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7)\n\n const __m256i rhs_mat_0145_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_1, m4b)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15)\n const __m256i rhs_mat_2367_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_1, m4b)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15)\n\n const __m256i rhs_mat_0145_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23)\n const __m256i rhs_mat_2367_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23)\n\n const __m256i rhs_mat_0145_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31)\n const __m256i rhs_mat_2367_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31)\n\n // Shuffle pattern one - right side input\n const __m256i rhs_mat_0145_0_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_0, 136); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3)\n const __m256i rhs_mat_2367_0_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_0, 136); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3)\n\n const __m256i rhs_mat_0145_1_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_1, 136); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11)\n const __m256i rhs_mat_2367_1_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_1, 136); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11)\n\n const __m256i rhs_mat_0145_2_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_2, 136); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19)\n const __m256i rhs_mat_2367_2_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_2, 136); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19)\n\n const __m256i rhs_mat_0145_3_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_3, 136); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27)\n const __m256i rhs_mat_2367_3_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_3, 136); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27)\n\n // Shuffle pattern two - right side input\n\n const __m256i rhs_mat_0145_0_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_0, 221); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7)\n const __m256i rhs_mat_2367_0_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_0, 221); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7)\n\n const __m256i rhs_mat_0145_1_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_1, 221); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15)\n const __m256i rhs_mat_2367_1_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_1, 221); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15)\n\n const __m256i rhs_mat_0145_2_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_2, 221); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23)\n const __m256i rhs_mat_2367_2_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_2, 221); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23)\n\n const __m256i rhs_mat_0145_3_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_3, 221); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31)\n const __m256i rhs_mat_2367_3_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_3, 221); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31)\n\n // Scale values - Load the wight scale values of block_tx8\n __m256 col_scale_f32;\n if constexpr (\n std::is_same_v ||\n std::is_same_v) {\n col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n } else if constexpr (std::is_same_v) {\n col_scale_f32 = _mm256_set_ps(\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[7]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[6]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[5]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[4]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[3]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[2]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[1]),\n GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[0]));\n }\n\n // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n __m256i lhs_mat_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs)));\n __m256i lhs_mat_01_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 0);\n __m256i lhs_mat_23_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 17);\n __m256i lhs_mat_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 32)));\n __m256i lhs_mat_01_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 0);\n __m256i lhs_mat_23_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 17);\n __m256i lhs_mat_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 64)));\n __m256i lhs_mat_01_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 0);\n __m256i lhs_mat_23_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 17);\n __m256i lhs_mat_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 96)));\n __m256i lhs_mat_01_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 0);\n __m256i lhs_mat_23_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 17);\n\n // Shuffle pattern one - left side input\n\n const __m256i lhs_mat_01_0_sp1 = _mm256_shuffle_epi32(lhs_mat_01_0, 160); //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3)\n const __m256i lhs_mat_23_0_sp1 = _mm256_shuffle_epi32(lhs_mat_23_0, 160); //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3)\n\n const __m256i lhs_mat_01_1_sp1 = _mm256_shuffle_epi32(lhs_mat_01_1, 160); //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11)\n const __m256i lhs_mat_23_1_sp1 = _mm256_shuffle_epi32(lhs_mat_23_1, 160); //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11)\n\n const __m256i lhs_mat_01_2_sp1 = _mm256_shuffle_epi32(lhs_mat_01_2, 160); //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19)\n const __m256i lhs_mat_23_2_sp1 = _mm256_shuffle_epi32(lhs_mat_23_2, 160); //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19)\n\n const __m256i lhs_mat_01_3_sp1 = _mm256_shuffle_epi32(lhs_mat_01_3, 160); //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27)\n const __m256i lhs_mat_23_3_sp1 = _mm256_shuffle_epi32(lhs_mat_23_3, 160); //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27)\n\n // Shuffle pattern two - left side input\n\n const __m256i lhs_mat_01_0_sp2 = _mm256_shuffle_epi32(lhs_mat_01_0, 245); //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7)\n const __m256i lhs_mat_23_0_sp2 = _mm256_shuffle_epi32(lhs_mat_23_0, 245); //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7)\n\n const __m256i lhs_mat_01_1_sp2 = _mm256_shuffle_epi32(lhs_mat_01_1, 245); //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15)\n const __m256i lhs_mat_23_1_sp2 = _mm256_shuffle_epi32(lhs_mat_23_1, 245); //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15)\n\n const __m256i lhs_mat_01_2_sp2 = _mm256_shuffle_epi32(lhs_mat_01_2, 245); //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23)\n const __m256i lhs_mat_23_2_sp2 = _mm256_shuffle_epi32(lhs_mat_23_2, 245); //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23)\n\n const __m256i lhs_mat_01_3_sp2 = _mm256_shuffle_epi32(lhs_mat_01_3, 245); //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31)\n const __m256i lhs_mat_23_3_sp2 = _mm256_shuffle_epi32(lhs_mat_23_3, 245); //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n // Resembles MMLAs into 2x2 matrices in ARM Version\n const __m256i zero = _mm256_setzero_si256();\n __m256i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_01_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_01_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_01_0_sp1, rhs_mat_0145_0_sp1);\n __m256i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367_0_sp1);\n __m256i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_23_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_23_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_23_0_sp1, rhs_mat_0145_0_sp1);\n __m256i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367_0_sp1);\n __m256i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_01_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_01_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_01_0_sp2, rhs_mat_0145_0_sp2);\n __m256i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367_0_sp2);\n __m256i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_23_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_23_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_23_0_sp2, rhs_mat_0145_0_sp2);\n __m256i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367_0_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n __m256i iacc_mat_00 = _mm256_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2);\n __m256i iacc_mat_01 = _mm256_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2);\n __m256i iacc_mat_10 = _mm256_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2);\n __m256i iacc_mat_11 = _mm256_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2);\n\n\n // Straighten out to make 4 row vectors\n __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204);\n __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204);\n __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204);\n __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204);\n\n // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes\n const __m256 row_scale_f32 = GGML_F32Cx8_REPEAT_LOAD(a_ptr[b].d, loadMask);\n\n // Multiply with appropriate scales and accumulate\n acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n }\n\n // Store the accumulated values\n for (int i = 0; i < 4; i++) {\n _mm256_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]);\n }\n }\n }\n}\n\n#endif // defined(__AVX2__) || defined(__AVX512F__)\n\nvoid ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__) || defined(__AVX512F__)\n {\n // Lookup table to convert signed nibbles to signed bytes\n __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0));\n signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n gemv_q4_b32_8x8_q8_0_lut_avx(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n return;\n }\n#endif\n\n ggml_gemv_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__AVX2__)\n // Lookup table to convert signed nibbles to signed bytes\n __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0));\n signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n // Shuffle masks to rearrange delta and scale values to multiply with appropriate scales\n __m128i deltamask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0);\n __m128i scalemask = _mm_set_epi8(7, 7, 3, 3, 6, 6, 2, 2, 5, 5, 1, 1, 4, 4, 0, 0);\n // Permute mask used for easier vector processing at later stages\n __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);\n\n // Mask to extract nibbles from bytes\n const __m256i m4b = _mm256_set1_epi8(0x0F);\n\n int64_t b_nb = n / QK_K;\n\n const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 *)vx;\n const block_q8_K * a_ptr_start = (const block_q8_K *)vy;\n\n // Process Q8_K blocks one by one\n for (int64_t y = 0; y < nr; y++) {\n\n // Pointers to LHS blocks of block_q8_K format\n const block_q8_K * a_ptr = a_ptr_start + (y * nb);\n\n // Take group of eight interleaved block_q4_K structures at each pass of the loop and perform dot product operation\n for (int64_t x = 0; x < nc / 8; x++) {\n\n // Pointers to RHS blocks\n const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n // Master FP accumulators\n __m256 acc_row = _mm256_setzero_ps();\n __m256 acc_min_rows = _mm256_setzero_ps();\n\n for (int64_t b = 0; b < nb; b++) {\n\n // Load and convert to FP32 scale from block_q8_K\n const __m256 row_scale_f32 = _mm256_set1_ps((a_ptr[b].d));\n\n // Load the scale values for the 8 blocks interleaved in block_q4_Kx8\n // col_scale_f32 rearranged so as to multiply with appropriate quants\n const __m256 col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, deltamask);\n const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n __m256i iacc_b = _mm256_setzero_si256();\n __m256i iacc_min_b = _mm256_setzero_si256();\n\n const __m256i q8sums = _mm256_loadu_si256((const __m256i * )(a_ptr[b].bsums));\n __m256i q8s = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(q8sums), _mm256_extracti128_si256(q8sums, 1)));\n q8s = _mm256_permute2f128_si256(q8s, q8s, 0);\n\n // Processes two sub blocks from each Q4_K in each iteration\n for (int sb = 0; sb < QK_K / 64; sb++) {\n\n // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256));\n const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_vec_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_vec_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_vec_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_vec_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256));\n\n // 4-bit -> 8-bit\n // Values of the first sub block of eight block_q4_K structures for the sb loop\n const __m256i rhs_vec_0123_00 = _mm256_and_si256(rhs_raw_vec_0123_0, m4b);\n const __m256i rhs_vec_4567_00 = _mm256_and_si256(rhs_raw_vec_4567_0, m4b);\n const __m256i rhs_vec_0123_01 = _mm256_and_si256(rhs_raw_vec_0123_1, m4b);\n const __m256i rhs_vec_4567_01 = _mm256_and_si256(rhs_raw_vec_4567_1, m4b);\n const __m256i rhs_vec_0123_02 = _mm256_and_si256(rhs_raw_vec_0123_2, m4b);\n const __m256i rhs_vec_4567_02 = _mm256_and_si256(rhs_raw_vec_4567_2, m4b);\n const __m256i rhs_vec_0123_03 = _mm256_and_si256(rhs_raw_vec_0123_3, m4b);\n const __m256i rhs_vec_4567_03 = _mm256_and_si256(rhs_raw_vec_4567_3, m4b);\n\n // Values of the second sub block of eight block_q4_K structures when sb = 1\n const __m256i rhs_vec_0123_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m4b);\n const __m256i rhs_vec_4567_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m4b);\n const __m256i rhs_vec_0123_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m4b);\n const __m256i rhs_vec_4567_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m4b);\n const __m256i rhs_vec_0123_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 4), m4b);\n const __m256i rhs_vec_4567_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 4), m4b);\n const __m256i rhs_vec_0123_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 4), m4b);\n const __m256i rhs_vec_4567_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 4), m4b);\n\n uint32_t utmp_0[4], utmp_1[4];\n\n // Scales and Mins of corresponding sub blocks from different Q8_K structures are stored together\n // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12);\n utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_0 = utmp_0[1] & kmask1;\n utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4);\n utmp_0[2] = uaux_0;\n utmp_0[0] &= kmask1;\n\n // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12);\n utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_1 = utmp_1[1] & kmask1;\n utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4);\n utmp_1[2] = uaux_1;\n utmp_1[0] &= kmask1;\n\n // Scales of first sub block in the sb loop\n const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]);\n __m128i scales_rearrange_0 = _mm_shuffle_epi8(mins_and_scales_0, scalemask);\n __m256i scales_0 = _mm256_cvtepu8_epi16(scales_rearrange_0);\n\n // Scales of second sub block in the sb loop\n __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]);\n __m128i scales_rearrange_1 = _mm_shuffle_epi8(mins_and_scales_1, scalemask);\n __m256i scales_1 = _mm256_cvtepu8_epi16(scales_rearrange_1);\n\n // Mins of first and second sub block of Q4_K block are arranged side by side\n __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78)));\n\n // Load the two sub block values corresponding to sb in block_q8_K in batches of 16 bytes and replicate the same across 256 bit vector\n __m256i lhs_vec_00 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + sb * 64)));\n __m256i lhs_vec_01 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16 + sb * 64)));\n __m256i lhs_vec_10 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 32 + sb * 64)));\n __m256i lhs_vec_11 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 48 + sb * 64)));\n\n lhs_vec_00 = _mm256_permute2f128_si256(lhs_vec_00, lhs_vec_00, 0);\n lhs_vec_01 = _mm256_permute2f128_si256(lhs_vec_01, lhs_vec_01, 0);\n lhs_vec_10 = _mm256_permute2f128_si256(lhs_vec_10, lhs_vec_10, 0);\n lhs_vec_11 = _mm256_permute2f128_si256(lhs_vec_11, lhs_vec_11, 0);\n\n // Dot product done within 32 bit lanes and accumulated in the same vector\n // First done for first sub block and then for second sub block in each sb\n // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3)\n // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7)\n // ...........................................................................\n // B0(28-31) B4(28-31) B1(28-31) B5(28-31) B2(28-31) B6(28-31) B3(28-31) B7(28-31) with A0(28-31)\n\n\n __m256i iacc_0 = _mm256_setzero_si256();\n __m256i iacc_1 = _mm256_setzero_si256();\n\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_00 ,_mm256_shuffle_epi32(rhs_vec_4567_00, 177), 170), _mm256_shuffle_epi32(lhs_vec_00, 0)));\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_00, 177) ,rhs_vec_4567_00, 170), _mm256_shuffle_epi32(lhs_vec_00, 85)));\n\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_01 ,_mm256_shuffle_epi32(rhs_vec_4567_01, 177), 170), _mm256_shuffle_epi32(lhs_vec_00, 170)));\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_01, 177) ,rhs_vec_4567_01, 170), _mm256_shuffle_epi32(lhs_vec_00, 255)));\n\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_02 ,_mm256_shuffle_epi32(rhs_vec_4567_02, 177), 170), _mm256_shuffle_epi32(lhs_vec_01, 0)));\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_02, 177) ,rhs_vec_4567_02, 170), _mm256_shuffle_epi32(lhs_vec_01, 85)));\n\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_03 ,_mm256_shuffle_epi32(rhs_vec_4567_03, 177), 170), _mm256_shuffle_epi32(lhs_vec_01, 170)));\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_03, 177) ,rhs_vec_4567_03, 170), _mm256_shuffle_epi32(lhs_vec_01, 255)));\n\n iacc_0 = _mm256_madd_epi16(iacc_0, scales_0);\n\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_10 ,_mm256_shuffle_epi32(rhs_vec_4567_10, 177), 170), _mm256_shuffle_epi32(lhs_vec_10, 0)));\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_10, 177) ,rhs_vec_4567_10, 170), _mm256_shuffle_epi32(lhs_vec_10, 85)));\n\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_11 ,_mm256_shuffle_epi32(rhs_vec_4567_11, 177), 170), _mm256_shuffle_epi32(lhs_vec_10, 170)));\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_11, 177) ,rhs_vec_4567_11, 170), _mm256_shuffle_epi32(lhs_vec_10, 255)));\n\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_12 ,_mm256_shuffle_epi32(rhs_vec_4567_12, 177), 170), _mm256_shuffle_epi32(lhs_vec_11, 0)));\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_12, 177) ,rhs_vec_4567_12, 170), _mm256_shuffle_epi32(lhs_vec_11, 85)));\n\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_13 ,_mm256_shuffle_epi32(rhs_vec_4567_13, 177), 170), _mm256_shuffle_epi32(lhs_vec_11, 170)));\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_13, 177) ,rhs_vec_4567_13, 170), _mm256_shuffle_epi32(lhs_vec_11, 255)));\n\n iacc_1 = _mm256_madd_epi16(iacc_1, scales_1);\n\n // Accumulate the iacc value for one sb\n __m256i iacc_sb = _mm256_add_epi32(iacc_0, iacc_1);\n\n // Broadcast the bsums of the two sub blocks of the iteration of Q8_K across the vector\n // Multiply-Add with corresponding mins of Q4_Kx8 with bsums\n __m256i q8s_sb = _mm256_shuffle_epi32(q8s, 0);\n __m256i iacc_min_sb = _mm256_madd_epi16(q8s_sb, mins_01);\n q8s = _mm256_bsrli_epi128(q8s, 4);\n\n // Accumulate for the complete block\n iacc_b = _mm256_add_epi32(iacc_b, iacc_sb);\n iacc_min_b = _mm256_add_epi32(iacc_min_b, iacc_min_sb);\n }\n\n // Multiply-Add with scale values for the complete super block\n acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_b), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row);\n acc_min_rows = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_min_b), _mm256_mul_ps(col_dmin_f32, row_scale_f32), acc_min_rows);\n\n }\n\n // Accumulated output values permuted so as to be stored in appropriate order post accumulation\n acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask);\n _mm256_storeu_ps(s + (y * nr + x * 8), _mm256_sub_ps(acc_row, acc_min_rows));\n }\n }\n\n#else\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n ggml_gemv_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n#endif\n}\n\nvoid ggml_gemv_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__)\n __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_iq4nl));\n signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n gemv_q4_b32_8x8_q8_0_lut_avx(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n return;\n#endif\n\n ggml_gemv_iq4_nl_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__)\n __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_mxfp4));\n signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n gemv_q4_b32_8x8_q8_0_lut_avx(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n return;\n#endif\n\n ggml_gemv_mxfp4_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__AVX2__)\n // Lookup table to convert signed nibbles to signed bytes\n __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0));\n signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n // Shuffle masks to rearrange delta values to multiply with appropriate scales\n __m128i deltamask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0);\n // Permute mask used for easier vector processing at later stages\n __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);\n\n const __m256i m3b = _mm256_set1_epi8(3);\n const __m128i m4b_sse = _mm_set1_epi8(0xF);\n\n //Mask to get appropriate scales\n __m128i scalemask1 = _mm_set_epi8(14,14,6,6,12,12,4,4,10,10,2,2,8,8,0,0);\n __m128i scalemask2 = _mm_set_epi8(15,15,7,7,13,13,5,5,11,11,3,3,9,9,1,1);\n\n int64_t b_nb = n / QK_K;\n\n const block_q2_Kx8 * b_ptr_start = (const block_q2_Kx8 *)vx;\n const block_q8_K * a_ptr_start = (const block_q8_K *)vy;\n\n // Process Q8_K blocks one by one\n for (int64_t y = 0; y < nr; y++) {\n\n // Pointers to LHS blocks of block_q8_K format\n const block_q8_K * a_ptr = a_ptr_start + (y * nb);\n\n // Take group of eight interleaved block_q2_K structures at each pass of the loop and perform dot product operation\n for(int64_t x = 0; x < nc / 8; x++) {\n\n // Pointers to RHS blocks\n const block_q2_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n // Master FP accumulators\n __m256 acc_row = _mm256_setzero_ps();\n __m256 acc_min_rows = _mm256_setzero_ps();\n\n for (int64_t b = 0; b < nb; b++) {\n\n // Load and convert to FP32 delta from block_q8_K\n const __m256 row_scale_f32 = _mm256_set1_ps((a_ptr[b].d));\n\n // Load the delta values for the 8 blocks interleaved in block_q2_Kx8\n // col_scale_f32 rearranged so as to multiply with appropriate quants\n const __m256 col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, deltamask);\n const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n __m256i iacc_b = _mm256_setzero_si256();\n __m256i iacc_min_b = _mm256_setzero_si256();\n\n // Processes eight sub blocks from each Q2_K in each iteration\n for(int sb = 0; sb < QK_K / 128; sb++) {\n\n // Load the eight block_q2_K for eight sub blocks quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256));\n const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_vec_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_vec_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_vec_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_vec_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256));\n\n // 2-bit -> 8-bit\n // Values of the 0th,2nd,4th,6th sub blocks of eight block_q2_K structures for the sb loop\n const __m256i rhs_vec_0123_00 = _mm256_and_si256(rhs_raw_vec_0123_0, m3b); //B00(0-7) B01(0-7) B02(0-7) B03(0-7)\n const __m256i rhs_vec_0123_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 2), m3b); //B20(0-7) B21(0-7) B22(0-7) B23(0-7)\n const __m256i rhs_vec_0123_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m3b); //B40(0-7) B41(0-7) B42(0-7) B43(0-7)\n const __m256i rhs_vec_0123_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 6), m3b); //B60(0-7) B61(0-7) B62(0-7) B63(0-7)\n\n const __m256i rhs_vec_4567_00 = _mm256_and_si256(rhs_raw_vec_4567_0, m3b); //B04(0-7) B05(0-7) B06(0-7) B07(0-7)\n const __m256i rhs_vec_4567_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 2), m3b); //B24(0-7) B25(0-7) B26(0-7) B27(0-7)\n const __m256i rhs_vec_4567_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m3b); //B44(0-7) B45(0-7) B46(0-7) B47(0-7)\n const __m256i rhs_vec_4567_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 6), m3b); //B64(0-7) B65(0-7) B66(0-7) B67(0-7)\n\n const __m256i rhs_vec_0123_01 = _mm256_and_si256(rhs_raw_vec_0123_1, m3b); //B00(8-15) B01(8-15) B02(8-15) B03(8-15)\n const __m256i rhs_vec_0123_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 2), m3b); //B20(8-15) B21(8-15) B22(8-15) B23(8-15)\n const __m256i rhs_vec_0123_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m3b); //B40(8-15) B41(8-15) B42(8-15) B43(8-15)\n const __m256i rhs_vec_0123_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 6), m3b); //B60(8-15) B61(8-15) B62(8-15) B63(8-15)\n\n const __m256i rhs_vec_4567_01 = _mm256_and_si256(rhs_raw_vec_4567_1, m3b); //B04(8-15) B05(8-15) B06(8-15) B07(8-15)\n const __m256i rhs_vec_4567_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 2), m3b); //B24(8-15) B25(8-15) B26(8-15) B27(8-15)\n const __m256i rhs_vec_4567_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m3b); //B44(8-15) B45(8-15) B46(8-15) B47(8-15)\n const __m256i rhs_vec_4567_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 6), m3b); //B64(8-15) B65(8-15) B66(8-15) B67(8-15)\n\n // Values of the 1st,3rd,5th,7th sub blocks of eight block_q2_K structures for the sb loop\n const __m256i rhs_vec_0123_10 = _mm256_and_si256(rhs_raw_vec_0123_2, m3b); //B10(0-7) B11(0-7) B12(0-7) B13(0-7)\n const __m256i rhs_vec_0123_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 2), m3b); //B30(0-7) B31(0-7) B32(0-7) B33(0-7)\n const __m256i rhs_vec_0123_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 4), m3b); //B50(0-7) B51(0-7) B52(0-7) B53(0-7)\n const __m256i rhs_vec_0123_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 6), m3b); //B70(0-7) B71(0-7) B72(0-7) B73(0-7)\n\n const __m256i rhs_vec_4567_10 = _mm256_and_si256(rhs_raw_vec_4567_2, m3b); //B14(0-7) B15(0-7) B16(0-7) B17(0-7)\n const __m256i rhs_vec_4567_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 2), m3b); //B34(0-7) B35(0-7) B36(0-7) B37(0-7)\n const __m256i rhs_vec_4567_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 4), m3b); //B54(0-7) B55(0-7) B56(0-7) B57(0-7)\n const __m256i rhs_vec_4567_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 6), m3b); //B74(0-7) B75(0-7) B76(0-7) B77(0-7)\n\n const __m256i rhs_vec_0123_11 = _mm256_and_si256(rhs_raw_vec_0123_3, m3b); //B10(8-15) B11(8-15) B12(8-15) B13(8-15)\n const __m256i rhs_vec_0123_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 2), m3b); //B30(8-15) B31(8-15) B32(8-15) B33(8-15)\n const __m256i rhs_vec_0123_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 4), m3b); //B50(8-15) B51(8-15) B52(8-15) B53(8-15)\n const __m256i rhs_vec_0123_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 6), m3b); //B70(8-15) B71(8-15) B72(8-15) B73(8-15)\n\n const __m256i rhs_vec_4567_11 = _mm256_and_si256(rhs_raw_vec_4567_3, m3b); //B14(8-15) B15(8-15) B16(8-15) B17(8-15)\n const __m256i rhs_vec_4567_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 2), m3b); //B34(8-15) B35(8-15) B36(8-15) B37(8-15)\n const __m256i rhs_vec_4567_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 4), m3b); //B54(8-15) B55(8-15) B56(8-15) B57(8-15)\n const __m256i rhs_vec_4567_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 6), m3b); //B74(8-15) B75(8-15) B76(8-15) B77(8-15)\n\n //Scales and Mins of corresponding sub blocks from different Q2_K structures are stored together\n //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71\n\n const __m128i mins_and_scales_01 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + sb * 64));\n const __m128i mins_and_scales_23 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 16 + sb * 64));\n const __m128i mins_and_scales_45 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 32 + sb * 64));\n const __m128i mins_and_scales_67 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 48 + sb * 64));\n\n // Extract scales which is lower half from mins_and_scales\n const __m128i scales_01 = _mm_and_si128(mins_and_scales_01, m4b_sse);\n const __m128i scales_23 = _mm_and_si128(mins_and_scales_23, m4b_sse);\n const __m128i scales_45 = _mm_and_si128(mins_and_scales_45, m4b_sse);\n const __m128i scales_67 = _mm_and_si128(mins_and_scales_67, m4b_sse);\n\n // Extract mins which is upper half from mins_and_scales\n const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_01, 4), m4b_sse));\n const __m256i mins_23 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_23, 4), m4b_sse));\n const __m256i mins_45 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_45, 4), m4b_sse));\n const __m256i mins_67 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_67, 4), m4b_sse));\n\n // Scales of sub blocks in the sb loop\n // Scales of the 0th sub block from each super block\n __m128i scales_rearrange_0 = _mm_shuffle_epi8(scales_01, scalemask1);\n __m256i scales_0 = _mm256_cvtepu8_epi16(scales_rearrange_0);\n\n // Scales of the 1st sub block from each super block\n __m128i scales_rearrange_1 = _mm_shuffle_epi8(scales_01, scalemask2);\n __m256i scales_1 = _mm256_cvtepu8_epi16(scales_rearrange_1);\n\n // Scales of the 2nd sub block from each super block\n __m128i scales_rearrange_2 = _mm_shuffle_epi8(scales_23, scalemask1);\n __m256i scales_2 = _mm256_cvtepu8_epi16(scales_rearrange_2);\n\n // Scales of the 3rd sub block from each super block\n __m128i scales_rearrange_3 = _mm_shuffle_epi8(scales_23, scalemask2);\n __m256i scales_3 = _mm256_cvtepu8_epi16(scales_rearrange_3);\n\n // Scales of the 4th sub block from each super block\n __m128i scales_rearrange_4 = _mm_shuffle_epi8(scales_45, scalemask1);\n __m256i scales_4 = _mm256_cvtepu8_epi16(scales_rearrange_4);\n\n // Scales of the 5th sub block from each super block\n __m128i scales_rearrange_5 = _mm_shuffle_epi8(scales_45, scalemask2);\n __m256i scales_5 = _mm256_cvtepu8_epi16(scales_rearrange_5);\n\n // Scales of the 6th sub block from each super block\n __m128i scales_rearrange_6 = _mm_shuffle_epi8(scales_67, scalemask1);\n __m256i scales_6 = _mm256_cvtepu8_epi16(scales_rearrange_6);\n\n // Scales of the 7th sub block from each super block\n __m128i scales_rearrange_7 = _mm_shuffle_epi8(scales_67, scalemask2);\n __m256i scales_7 = _mm256_cvtepu8_epi16(scales_rearrange_7);\n\n // Load the sub block values corresponding to sb in block_q8_K in batches of 16 bytes and replicate the same across 256 bit vector\n __m256i lhs_vec_0 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + sb * 128)));\n __m256i lhs_vec_1 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16 + sb * 128)));\n __m256i lhs_vec_2 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 32 + sb * 128)));\n __m256i lhs_vec_3 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 48 + sb * 128)));\n __m256i lhs_vec_4 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 64 + sb * 128)));\n __m256i lhs_vec_5 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 80 + sb * 128)));\n __m256i lhs_vec_6 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 96 + sb * 128)));\n __m256i lhs_vec_7 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 112 + sb * 128)));\n\n lhs_vec_0 = _mm256_permute2f128_si256(lhs_vec_0, lhs_vec_0, 0);\n lhs_vec_1 = _mm256_permute2f128_si256(lhs_vec_1, lhs_vec_1, 0);\n lhs_vec_2 = _mm256_permute2f128_si256(lhs_vec_2, lhs_vec_2, 0);\n lhs_vec_3 = _mm256_permute2f128_si256(lhs_vec_3, lhs_vec_3, 0);\n lhs_vec_4 = _mm256_permute2f128_si256(lhs_vec_4, lhs_vec_4, 0);\n lhs_vec_5 = _mm256_permute2f128_si256(lhs_vec_5, lhs_vec_5, 0);\n lhs_vec_6 = _mm256_permute2f128_si256(lhs_vec_6, lhs_vec_6, 0);\n lhs_vec_7 = _mm256_permute2f128_si256(lhs_vec_7, lhs_vec_7, 0);\n\n __m256i iacc_0 = _mm256_setzero_si256();\n __m256i iacc_1 = _mm256_setzero_si256();\n __m256i iacc_2 = _mm256_setzero_si256();\n __m256i iacc_3 = _mm256_setzero_si256();\n __m256i iacc_4 = _mm256_setzero_si256();\n __m256i iacc_5 = _mm256_setzero_si256();\n __m256i iacc_6 = _mm256_setzero_si256();\n __m256i iacc_7 = _mm256_setzero_si256();\n\n // Dot product done within 32 bit lanes and accumulated in the same vector\n // First done for 0th sub block and then for seven (1st - 7th) other sub blocks processed for each sb (sb < QK_K/128 loop) // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3)\n // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7)\n // B0(8-11) B4(8-11) B1(8-11) B5(8-11) B2(8-11) B6(8-11) B3(8-11) B7(8-11) with A0(8-11)\n // B0(12-15) B4(12-15) B1(12-15) B5(12-15) B2(12-15) B6(12-15) B3(12-15) B7(12-15) with A0(12-15)\n\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_00 ,_mm256_shuffle_epi32(rhs_vec_4567_00, 177), 170), _mm256_shuffle_epi32(lhs_vec_0, 0)));\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_00, 177) ,rhs_vec_4567_00, 170), _mm256_shuffle_epi32(lhs_vec_0, 85)));\n\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_01 ,_mm256_shuffle_epi32(rhs_vec_4567_01, 177), 170), _mm256_shuffle_epi32(lhs_vec_0, 170)));\n iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_01, 177) ,rhs_vec_4567_01, 170), _mm256_shuffle_epi32(lhs_vec_0, 255)));\n\n iacc_0 = _mm256_madd_epi16(iacc_0, scales_0);\n\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_10 ,_mm256_shuffle_epi32(rhs_vec_4567_10, 177), 170), _mm256_shuffle_epi32(lhs_vec_1, 0)));\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_10, 177) ,rhs_vec_4567_10, 170), _mm256_shuffle_epi32(lhs_vec_1, 85)));\n\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_11 ,_mm256_shuffle_epi32(rhs_vec_4567_11, 177), 170), _mm256_shuffle_epi32(lhs_vec_1, 170)));\n iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_11, 177) ,rhs_vec_4567_11, 170), _mm256_shuffle_epi32(lhs_vec_1, 255)));\n\n iacc_1 = _mm256_madd_epi16(iacc_1, scales_1);\n\n iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_20 ,_mm256_shuffle_epi32(rhs_vec_4567_20, 177), 170), _mm256_shuffle_epi32(lhs_vec_2, 0)));\n iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_20, 177) ,rhs_vec_4567_20, 170), _mm256_shuffle_epi32(lhs_vec_2, 85)));\n\n iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_21 ,_mm256_shuffle_epi32(rhs_vec_4567_21, 177), 170), _mm256_shuffle_epi32(lhs_vec_2, 170)));\n iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_21, 177) ,rhs_vec_4567_21, 170), _mm256_shuffle_epi32(lhs_vec_2, 255)));\n\n iacc_2 = _mm256_madd_epi16(iacc_2, scales_2);\n\n iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_30 ,_mm256_shuffle_epi32(rhs_vec_4567_30, 177), 170), _mm256_shuffle_epi32(lhs_vec_3, 0)));\n iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_30, 177) ,rhs_vec_4567_30, 170), _mm256_shuffle_epi32(lhs_vec_3, 85)));\n\n iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_31 ,_mm256_shuffle_epi32(rhs_vec_4567_31, 177), 170), _mm256_shuffle_epi32(lhs_vec_3, 170)));\n iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_31, 177) ,rhs_vec_4567_31, 170), _mm256_shuffle_epi32(lhs_vec_3, 255)));\n\n iacc_3 = _mm256_madd_epi16(iacc_3, scales_3);\n\n iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_40 ,_mm256_shuffle_epi32(rhs_vec_4567_40, 177), 170), _mm256_shuffle_epi32(lhs_vec_4, 0)));\n iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_40, 177) ,rhs_vec_4567_40, 170), _mm256_shuffle_epi32(lhs_vec_4, 85)));\n\n iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_41 ,_mm256_shuffle_epi32(rhs_vec_4567_41, 177), 170), _mm256_shuffle_epi32(lhs_vec_4, 170)));\n iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_41, 177) ,rhs_vec_4567_41, 170), _mm256_shuffle_epi32(lhs_vec_4, 255)));\n\n iacc_4 = _mm256_madd_epi16(iacc_4, scales_4);\n\n iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_50 ,_mm256_shuffle_epi32(rhs_vec_4567_50, 177), 170), _mm256_shuffle_epi32(lhs_vec_5, 0)));\n iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_50, 177) ,rhs_vec_4567_50, 170), _mm256_shuffle_epi32(lhs_vec_5, 85)));\n\n iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_51 ,_mm256_shuffle_epi32(rhs_vec_4567_51, 177), 170), _mm256_shuffle_epi32(lhs_vec_5, 170)));\n iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_51, 177) ,rhs_vec_4567_51, 170), _mm256_shuffle_epi32(lhs_vec_5, 255)));\n\n iacc_5 = _mm256_madd_epi16(iacc_5, scales_5);\n\n iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_60 ,_mm256_shuffle_epi32(rhs_vec_4567_60, 177), 170), _mm256_shuffle_epi32(lhs_vec_6, 0)));\n iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_60, 177) ,rhs_vec_4567_60, 170), _mm256_shuffle_epi32(lhs_vec_6, 85)));\n\n iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_61 ,_mm256_shuffle_epi32(rhs_vec_4567_61, 177), 170), _mm256_shuffle_epi32(lhs_vec_6, 170)));\n iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_61, 177) ,rhs_vec_4567_61, 170), _mm256_shuffle_epi32(lhs_vec_6, 255)));\n\n iacc_6 = _mm256_madd_epi16(iacc_6, scales_6);\n\n iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_70 ,_mm256_shuffle_epi32(rhs_vec_4567_70, 177), 170), _mm256_shuffle_epi32(lhs_vec_7, 0)));\n iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_70, 177) ,rhs_vec_4567_70, 170), _mm256_shuffle_epi32(lhs_vec_7, 85)));\n\n iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_71 ,_mm256_shuffle_epi32(rhs_vec_4567_71, 177), 170), _mm256_shuffle_epi32(lhs_vec_7, 170)));\n iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_71, 177) ,rhs_vec_4567_71, 170), _mm256_shuffle_epi32(lhs_vec_7, 255)));\n\n iacc_7 = _mm256_madd_epi16(iacc_7, scales_7);\n\n // Accumulate the iacc value for one sb\n __m256i iacc_sb = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_0, iacc_1), _mm256_add_epi32(iacc_2, iacc_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_4, iacc_5), _mm256_add_epi32(iacc_6, iacc_7)));\n\n __m128i q8sums = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + sb * 8));\n __m256i q8s = _mm256_castsi128_si256(q8sums);\n q8s= _mm256_permute2f128_si256(q8s, q8s, 0);\n\n // Broadcast the bsums of the two corresponding subblocks of q8_k\n // Multiply-Add with corresponding mins of Q2_Kx8 with bsums\n __m256i iacc_min_sb_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 0), mins_01);\n __m256i iacc_min_sb_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 85), mins_23);\n __m256i iacc_min_sb_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 170), mins_45);\n __m256i iacc_min_sb_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 255), mins_67);\n\n __m256i iacc_min_sb = _mm256_add_epi32(_mm256_add_epi32(iacc_min_sb_01, iacc_min_sb_23), _mm256_add_epi32(iacc_min_sb_45,iacc_min_sb_67));\n\n // Accumulate for the complete block\n iacc_b = _mm256_add_epi32(iacc_b, iacc_sb);\n iacc_min_b = _mm256_add_epi32(iacc_min_b, iacc_min_sb);\n }\n\n //Multiply-Add with scale values for complete super block\n acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_b), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row);\n acc_min_rows = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_min_b), _mm256_mul_ps(col_dmin_f32, row_scale_f32), acc_min_rows);\n }\n // Accumulated output values permuted so as to be stored in appropriate order post accumulation\n acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask);\n _mm256_storeu_ps(s + (y * nr + x * 8), _mm256_sub_ps(acc_row, acc_min_rows));\n }\n }\n#else\n\n ggml_gemv_q2_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n\n#endif\n}\n\nvoid ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__) || defined(__AVX512F__)\n {\n // Lookup table to convert signed nibbles to signed bytes\n __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0));\n signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n gemm_q4_b32_8x8_q8_0_lut_avx(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n return;\n }\n#endif // defined(__AVX2__) || defined(__AVX512F__)\n\n ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__AVX2__) || defined(__AVX512F__)\n const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 * ) vx;\n const block_q8_Kx4 * a_ptr_start = (const block_q8_Kx4 * ) vy;\n int64_t b_nb = n / QK_K;\n int64_t y = 0;\n\n // Mask to mask out nibbles from packed bytes\n const __m256i m4b = _mm256_set1_epi8(0x0F);\n // Permute mask used for easier vector processing at later stages\n __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4);\n int64_t xstart = 0;\n int anr = nr - nr % 16;; // Used to align nr with boundary of 16\n#if defined(__AVX512BW__) && defined(__AVX512DQ__)\n int anc = nc - nc % 16; // Used to align nc with boundary of 16\n // Mask to mask out nibbles from packed bytes expanded to 512 bit length\n const __m512i m4bexpanded = _mm512_set1_epi8(0x0F);\n //Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation\n for (; y < anr / 4; y += 4) {\n\n const block_q8_Kx4 * a_ptrs[4];\n\n a_ptrs[0] = a_ptr_start + (y * nb);\n for (int i = 0; i < 3; ++i) {\n a_ptrs[i + 1] = a_ptrs[i] + nb;\n }\n\n // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = 0; x < anc / 8; x += 2) {\n\n const block_q4_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n const block_q4_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n // Master FP accumulators\n __m512 acc_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_rows[i] = _mm512_setzero_ps();\n }\n\n __m512 acc_min_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_min_rows[i] = _mm512_setzero_ps();\n }\n\n // For super block\n for (int64_t b = 0; b < nb; b++) {\n // Scale values - Load the sixteen scale values from two block_q4_kx8 structures\n const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n\n // dmin values - Load the sixteen dmin values from two block_q4_kx8 structures\n const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin);\n\n // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration\n for (int sb = 0; sb < QK_K / 64; sb++) {\n\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256));\n\n const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256));\n const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256));\n\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240);\n const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240);\n\n const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1);\n const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1);\n const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1);\n const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1);\n\n //4-bit -> 8-bit\n const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7)\n const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7)\n const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15)\n const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15)\n\n const __m512i rhs_mat_014589CD_02 = _mm512_and_si512(rhs_raw_mat_014589CD_2, m4bexpanded); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) B08(16-23) B09(16-23) B0C(16-23) B0D(16-23)\n const __m512i rhs_mat_2367ABEF_02 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2, m4bexpanded); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) B0A(16-23) B0B(16-23) B0E(16-23) B0F(16-23)\n const __m512i rhs_mat_014589CD_03 = _mm512_and_si512(rhs_raw_mat_014589CD_3, m4bexpanded); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) B08(24-31) B09(24-31) B0C(24-31) B0D(24-31)\n const __m512i rhs_mat_2367ABEF_03 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3, m4bexpanded); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) B0A(24-31) B0B(24-31) B0E(24-31) B0F(24-31)\n\n const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7)\n const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7)\n const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15)\n const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15)\n\n const __m512i rhs_mat_014589CD_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m4bexpanded); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) B18(16-23) B19(16-23) B1C(16-23) B1D(16-23)\n const __m512i rhs_mat_2367ABEF_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m4bexpanded); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) B1A(16-23) B1B(16-23) B1E(16-23) B1F(16-23)\n const __m512i rhs_mat_014589CD_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m4bexpanded); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) B18(24-31) B19(24-31) B1C(24-31) B1D(24-31)\n const __m512i rhs_mat_2367ABEF_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m4bexpanded); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) B1A(24-31) B1B(24-31) B1E(24-31) B1F(24-31)\n\n // Shuffle pattern one - right side input\n const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3)\n const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3)\n const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11)\n const __m512i rhs_mat_014589CD_02_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) B08(16-19) B09(16-19) B08(16-19) B09(16-19) B0C(16-19) B0D(16-19) B0C(16-19) B0D(16-19)\n const __m512i rhs_mat_2367ABEF_02_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) B0A(16-19) B0B(16-19) B0A(16-19) B0B(16-19) B0E(16-19) B0F(16-19) B0E(16-19) B0F(16-19)\n const __m512i rhs_mat_014589CD_03_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) B08(24-27) B09(24-27) B08(24-27) B09(24-27) B0C(24-27) B0D(24-27) B0C(24-27) B0D(24-27)\n const __m512i rhs_mat_2367ABEF_03_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) B0A(24-27) B0B(24-27) B0A(24-27) B0B(24-27) B0E(24-27) B0F(24-27) B0E(24-27) B0F(24-27)\n\n const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3)\n const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3)\n const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11)\n const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11)\n const __m512i rhs_mat_014589CD_12_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) B18(16-19) B19(16-19) B18(16-19) B19(16-19) B1C(16-19) B1D(16-19) B1C(16-19) B1D(16-19)\n const __m512i rhs_mat_2367ABEF_12_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) B1A(16-19) B1B(16-19) B1A(16-19) B1B(16-19) B1E(16-19) B1F(16-19) B1E(16-19) B1F(16-19)\n const __m512i rhs_mat_014589CD_13_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) B18(24-27) B19(24-27) B18(24-27) B19(24-27) B1C(24-27) B1D(24-27) B1C(24-27) B1D(24-27)\n const __m512i rhs_mat_2367ABEF_13_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) B1A(24-27) B1B(24-27) B1A(24-27) B1B(24-27) B1E(24-27) B1F(24-27) B1E(24-27) B1F(24-27)\n\n // Shuffle pattern two - right side input\n const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7)\n const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7)\n const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15)\n const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15)\n const __m512i rhs_mat_014589CD_02_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) B08(20-23) B09(20-23) B08(20-23) B09(20-23) B0C(20-23) B0D(20-23) B0C(20-23) B0D(20-23)\n const __m512i rhs_mat_2367ABEF_02_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) B0A(20-23) B0B(20-23) B0A(20-23) B0B(20-23) B0E(20-23) B0F(20-23) B0E(20-23) B0F(20-23)\n const __m512i rhs_mat_014589CD_03_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) B08(28-31) B09(28-31) B08(28-31) B09(28-31) B0C(28-31) B0D(28-31) B0C(28-31) 0BD(28-31)\n const __m512i rhs_mat_2367ABEF_03_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) B0A(28-31) B0B(28-31) B0A(28-31) B0B(28-31) B0E(28-31) B0F(28-31) B0E(28-31) B0F(28-31)\n\n const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7)\n const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7)\n const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15)\n const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15)\n const __m512i rhs_mat_014589CD_12_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) B18(20-23) B19(20-23) B18(20-23) B19(20-23) B1C(20-23) B1D(20-23) B1C(20-23) B1D(20-23)\n const __m512i rhs_mat_2367ABEF_12_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) B1A(20-23) B1B(20-23) B1A(20-23) B1B(20-23) B1E(20-23) B1F(20-23) B1E(20-23) B1F(20-23)\n const __m512i rhs_mat_014589CD_13_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) B18(28-31) B19(28-31) B18(28-31) B19(28-31) B1C(28-31) B1D(28-31) B1C(28-31) B1D(28-31)\n const __m512i rhs_mat_2367ABEF_13_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) B1A(28-31) B1B(28-31) B1A(28-31) B1B(28-31) B1E(28-31) B1F(28-31) B1E(28-31) B1F(28-31)\n\n uint32_t utmp_00[4], utmp_01[4], utmp_10[4], utmp_11[4];\n\n // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together\n // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_00, b_ptr_0[b].scales + 24 * sb, 12);\n utmp_00[3] = ((utmp_00[2] >> 4) & kmask2) | (((utmp_00[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_00 = utmp_00[1] & kmask1;\n utmp_00[1] = (utmp_00[2] & kmask2) | (((utmp_00[0] >> 6) & kmask3) << 4);\n utmp_00[2] = uaux_00;\n utmp_00[0] &= kmask1;\n\n // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_01, b_ptr_0[b].scales + 12 + sb * 24, 12);\n utmp_01[3] = ((utmp_01[2] >> 4) & kmask2) | (((utmp_01[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_01 = utmp_01[1] & kmask1;\n utmp_01[1] = (utmp_01[2] & kmask2) | (((utmp_01[0] >> 6) & kmask3) << 4);\n utmp_01[2] = uaux_01;\n utmp_01[0] &= kmask1;\n\n memcpy(utmp_10, b_ptr_1[b].scales + sb * 24, 12);\n utmp_10[3] = ((utmp_10[2] >> 4) & kmask2) | (((utmp_10[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_10 = utmp_10[1] & kmask1;\n utmp_10[1] = (utmp_10[2] & kmask2) | (((utmp_10[0] >> 6) & kmask3) << 4);\n utmp_10[2] = uaux_10;\n utmp_10[0] &= kmask1;\n\n // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_11, b_ptr_1[b].scales + 12 + sb * 24, 12);\n utmp_11[3] = ((utmp_11[2] >> 4) & kmask2) | (((utmp_11[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_11 = utmp_11[1] & kmask1;\n utmp_11[1] = (utmp_11[2] & kmask2) | (((utmp_11[0] >> 6) & kmask3) << 4);\n utmp_11[2] = uaux_11;\n utmp_11[0] &= kmask1;\n\n // Scales of first sub block in the sb loop\n const __m256i mins_and_scales_0 = _mm256_set_epi32(utmp_10[3], utmp_10[2], utmp_10[1], utmp_10[0], utmp_00[3], utmp_00[2], utmp_00[1], utmp_00[0]);\n const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0));\n\n // Scales of second sub block in the sb loop\n const __m256i mins_and_scales_1 = _mm256_set_epi32(utmp_11[3], utmp_11[2], utmp_11[1], utmp_11[0], utmp_01[3], utmp_01[2], utmp_01[1], utmp_01[0]);\n const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1));\n\n // Mins of first and second sub block of Q4_K block are arranged side by side\n const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(_mm256_shuffle_epi32(mins_and_scales_0, 78), _mm256_shuffle_epi32(mins_and_scales_1, 78)));\n\n const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238);\n\n for (int rp = 0; rp < 4; rp++) {\n\n // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector\n __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 * sb)));\n __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0);\n __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17);\n __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 256 * sb)));\n __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0);\n __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17);\n __m256i lhs_mat_ymm_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 256 * sb)));\n __m256i lhs_mat_ymm_01_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 0);\n __m256i lhs_mat_ymm_23_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 17);\n __m256i lhs_mat_ymm_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 256 * sb)));\n __m256i lhs_mat_ymm_01_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 0);\n __m256i lhs_mat_ymm_23_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 17);\n __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 256 * sb)));\n __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0);\n __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17);\n __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 256 * sb)));\n __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0);\n __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17);\n __m256i lhs_mat_ymm_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 256 * sb)));\n __m256i lhs_mat_ymm_01_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 0);\n __m256i lhs_mat_ymm_23_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 17);\n __m256i lhs_mat_ymm_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 256 * sb)));\n __m256i lhs_mat_ymm_01_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 0);\n __m256i lhs_mat_ymm_23_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 17);\n\n __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1);\n __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1);\n __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1);\n __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1);\n __m512i lhs_mat_01_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_02), lhs_mat_ymm_01_02, 1);\n __m512i lhs_mat_23_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_02), lhs_mat_ymm_23_02, 1);\n __m512i lhs_mat_01_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_03), lhs_mat_ymm_01_03, 1);\n __m512i lhs_mat_23_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_03), lhs_mat_ymm_23_03, 1);\n\n __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1);\n __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1);\n __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1);\n __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1);\n __m512i lhs_mat_01_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_12), lhs_mat_ymm_01_12, 1);\n __m512i lhs_mat_23_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_12), lhs_mat_ymm_23_12, 1);\n __m512i lhs_mat_01_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_13), lhs_mat_ymm_01_13, 1);\n __m512i lhs_mat_23_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_13), lhs_mat_ymm_23_13, 1);\n\n // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks\n __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].bsums + 16 * sb)));\n __m256i lhs_bsums_hsum_ymm_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)));\n lhs_bsums_hsum_ymm_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_ymm_0123_01, lhs_bsums_hsum_ymm_0123_01, 0);\n __m512i lhs_bsums_hsum_0123_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_hsum_ymm_0123_01), lhs_bsums_hsum_ymm_0123_01, 1);\n\n // Shuffle pattern one - left side input\n const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3)\n const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11)\n const __m512i lhs_mat_01_02_sp1 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19)\n const __m512i lhs_mat_23_02_sp1 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)160); //A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19)\n const __m512i lhs_mat_01_03_sp1 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27)\n const __m512i lhs_mat_23_03_sp1 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)160); //A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27)\n\n const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3)\n const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11)\n const __m512i lhs_mat_01_12_sp1 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19)\n const __m512i lhs_mat_23_12_sp1 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)160); //A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19)\n const __m512i lhs_mat_01_13_sp1 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27)\n const __m512i lhs_mat_23_13_sp1 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)160); //A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27)\n\n const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7)\n const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15)\n const __m512i lhs_mat_01_02_sp2 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23)\n const __m512i lhs_mat_23_02_sp2 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)245); //A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23)\n const __m512i lhs_mat_01_03_sp2 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31)\n const __m512i lhs_mat_23_03_sp2 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)245); //A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31)\n\n const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7)\n const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15)\n const __m512i lhs_mat_01_12_sp2 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23)\n const __m512i lhs_mat_23_12_sp2 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)245); //A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23)\n const __m512i lhs_mat_01_13_sp2 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31)\n const __m512i lhs_mat_23_13_sp2 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)245); //A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1));\n __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1));\n __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1));\n __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1));\n __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1));\n __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1));\n __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1));\n __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1));\n\n __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2));\n __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2));\n __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2));\n __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2));\n __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2));\n __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2));\n __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2));\n __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2));\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0);\n iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0);\n iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0);\n iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0);\n\n iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1);\n iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1);\n iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1);\n iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1);\n\n // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step)\n __m512i iacc_row_0_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_0, _mm512_shuffle_epi32(iacc_mat_01_0, (_MM_PERM_ENUM)78));\n __m512i iacc_row_1_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_0, (_MM_PERM_ENUM)78), iacc_mat_01_0);\n __m512i iacc_row_2_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_0, _mm512_shuffle_epi32(iacc_mat_11_0, (_MM_PERM_ENUM)78));\n __m512i iacc_row_3_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10_0, (_MM_PERM_ENUM)78), iacc_mat_11_0);\n __m512i iacc_row_0_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_1, _mm512_shuffle_epi32(iacc_mat_01_1, (_MM_PERM_ENUM)78));\n __m512i iacc_row_1_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_1, (_MM_PERM_ENUM)78), iacc_mat_01_1);\n __m512i iacc_row_2_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_1, _mm512_shuffle_epi32(iacc_mat_11_1, (_MM_PERM_ENUM)78));\n __m512i iacc_row_3_1 = _mm512_mask_blend_epi32(0xCCCC,_mm512_shuffle_epi32(iacc_mat_10_1, (_MM_PERM_ENUM)78), iacc_mat_11_1);\n\n __m512i iacc_row_0 = _mm512_add_epi32(iacc_row_0_0, iacc_row_0_1);\n __m512i iacc_row_1 = _mm512_add_epi32(iacc_row_1_0, iacc_row_1_1);\n __m512i iacc_row_2 = _mm512_add_epi32(iacc_row_2_0, iacc_row_2_1);\n __m512i iacc_row_3 = _mm512_add_epi32(iacc_row_3_0, iacc_row_3_1);\n\n // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d);\n const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1);\n\n // Multiply with appropriate scales and accumulate (for both d and dmin) below\n acc_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n acc_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n acc_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n acc_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n\n __m512i iacc_row_min_0 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)0), mins_01);\n __m512i iacc_row_min_1 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)85), mins_01);\n __m512i iacc_row_min_2 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)170), mins_01);\n __m512i iacc_row_min_3 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)255), mins_01);\n\n acc_min_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]);\n acc_min_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]);\n acc_min_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]);\n acc_min_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]);\n }\n }\n }\n // Store the accumulated values\n for (int i = 0; i < 16; i++) {\n _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i]));\n }\n }\n }\n\n for (; y < nr / 4; y++) {\n\n const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb);\n\n // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = 0; x < anc / 8; x += 2) {\n\n const block_q4_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n const block_q4_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n // Master FP accumulators\n __m512 acc_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_rows[i] = _mm512_setzero_ps();\n }\n\n __m512 acc_min_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_min_rows[i] = _mm512_setzero_ps();\n }\n\n // For super block\n for (int64_t b = 0; b < nb; b++) {\n // Scale values - Load the sixteen scale values from two block_q4_kx8 structures\n const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n\n // dmin values - Load the sixteen dmin values from two block_q4_kx8 structures\n const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin);\n\n // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration\n for (int sb = 0; sb < QK_K / 64; sb++) {\n\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256));\n\n const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256));\n const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256));\n\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240);\n const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240);\n\n const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1);\n const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1);\n const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1);\n const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1);\n\n //4-bit -> 8-bit\n const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7)\n const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7)\n const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15)\n const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15)\n\n const __m512i rhs_mat_014589CD_02 = _mm512_and_si512(rhs_raw_mat_014589CD_2, m4bexpanded); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) B08(16-23) B09(16-23) B0C(16-23) B0D(16-23)\n const __m512i rhs_mat_2367ABEF_02 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2, m4bexpanded); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) B0A(16-23) B0B(16-23) B0E(16-23) B0F(16-23)\n const __m512i rhs_mat_014589CD_03 = _mm512_and_si512(rhs_raw_mat_014589CD_3, m4bexpanded); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) B08(24-31) B09(24-31) B0C(24-31) B0D(24-31)\n const __m512i rhs_mat_2367ABEF_03 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3, m4bexpanded); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) B0A(24-31) B0B(24-31) B0E(24-31) B0F(24-31)\n\n const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7)\n const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7)\n const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15)\n const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15)\n\n const __m512i rhs_mat_014589CD_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m4bexpanded); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) B18(16-23) B19(16-23) B1C(16-23) B1D(16-23)\n const __m512i rhs_mat_2367ABEF_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m4bexpanded); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) B1A(16-23) B1B(16-23) B1E(16-23) B1F(16-23)\n const __m512i rhs_mat_014589CD_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m4bexpanded); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) B18(24-31) B19(24-31) B1C(24-31) B1D(24-31)\n const __m512i rhs_mat_2367ABEF_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m4bexpanded); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) B1A(24-31) B1B(24-31) B1E(24-31) B1F(24-31)\n\n // Shuffle pattern one - right side input\n const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3)\n const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3)\n const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11)\n const __m512i rhs_mat_014589CD_02_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) B08(16-19) B09(16-19) B08(16-19) B09(16-19) B0C(16-19) B0D(16-19) B0C(16-19) B0D(16-19)\n const __m512i rhs_mat_2367ABEF_02_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) B0A(16-19) B0B(16-19) B0A(16-19) B0B(16-19) B0E(16-19) B0F(16-19) B0E(16-19) B0F(16-19)\n const __m512i rhs_mat_014589CD_03_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) B08(24-27) B09(24-27) B08(24-27) B09(24-27) B0C(24-27) B0D(24-27) B0C(24-27) B0D(24-27)\n const __m512i rhs_mat_2367ABEF_03_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) B0A(24-27) B0B(24-27) B0A(24-27) B0B(24-27) B0E(24-27) B0F(24-27) B0E(24-27) B0F(24-27)\n\n const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3)\n const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3)\n const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11)\n const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11)\n const __m512i rhs_mat_014589CD_12_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) B18(16-19) B19(16-19) B18(16-19) B19(16-19) B1C(16-19) B1D(16-19) B1C(16-19) B1D(16-19)\n const __m512i rhs_mat_2367ABEF_12_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) B1A(16-19) B1B(16-19) B1A(16-19) B1B(16-19) B1E(16-19) B1F(16-19) B1E(16-19) B1F(16-19)\n const __m512i rhs_mat_014589CD_13_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) B18(24-27) B19(24-27) B18(24-27) B19(24-27) B1C(24-27) B1D(24-27) B1C(24-27) B1D(24-27)\n const __m512i rhs_mat_2367ABEF_13_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) B1A(24-27) B1B(24-27) B1A(24-27) B1B(24-27) B1E(24-27) B1F(24-27) B1E(24-27) B1F(24-27)\n\n // Shuffle pattern two - right side input\n const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7)\n const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7)\n const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15)\n const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15)\n const __m512i rhs_mat_014589CD_02_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) B08(20-23) B09(20-23) B08(20-23) B09(20-23) B0C(20-23) B0D(20-23) B0C(20-23) B0D(20-23)\n const __m512i rhs_mat_2367ABEF_02_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) B0A(20-23) B0B(20-23) B0A(20-23) B0B(20-23) B0E(20-23) B0F(20-23) B0E(20-23) B0F(20-23)\n const __m512i rhs_mat_014589CD_03_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) B08(28-31) B09(28-31) B08(28-31) B09(28-31) B0C(28-31) B0D(28-31) B0C(28-31) 0BD(28-31)\n const __m512i rhs_mat_2367ABEF_03_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) B0A(28-31) B0B(28-31) B0A(28-31) B0B(28-31) B0E(28-31) B0F(28-31) B0E(28-31) B0F(28-31)\n\n const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7)\n const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7)\n const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15)\n const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15)\n const __m512i rhs_mat_014589CD_12_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) B18(20-23) B19(20-23) B18(20-23) B19(20-23) B1C(20-23) B1D(20-23) B1C(20-23) B1D(20-23)\n const __m512i rhs_mat_2367ABEF_12_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) B1A(20-23) B1B(20-23) B1A(20-23) B1B(20-23) B1E(20-23) B1F(20-23) B1E(20-23) B1F(20-23)\n const __m512i rhs_mat_014589CD_13_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) B18(28-31) B19(28-31) B18(28-31) B19(28-31) B1C(28-31) B1D(28-31) B1C(28-31) B1D(28-31)\n const __m512i rhs_mat_2367ABEF_13_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) B1A(28-31) B1B(28-31) B1A(28-31) B1B(28-31) B1E(28-31) B1F(28-31) B1E(28-31) B1F(28-31)\n\n uint32_t utmp_00[4], utmp_01[4], utmp_10[4], utmp_11[4];\n\n // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together\n // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_00, b_ptr_0[b].scales + 24 * sb, 12);\n utmp_00[3] = ((utmp_00[2] >> 4) & kmask2) | (((utmp_00[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_00 = utmp_00[1] & kmask1;\n utmp_00[1] = (utmp_00[2] & kmask2) | (((utmp_00[0] >> 6) & kmask3) << 4);\n utmp_00[2] = uaux_00;\n utmp_00[0] &= kmask1;\n\n // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_01, b_ptr_0[b].scales + 12 + sb * 24, 12);\n utmp_01[3] = ((utmp_01[2] >> 4) & kmask2) | (((utmp_01[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_01 = utmp_01[1] & kmask1;\n utmp_01[1] = (utmp_01[2] & kmask2) | (((utmp_01[0] >> 6) & kmask3) << 4);\n utmp_01[2] = uaux_01;\n utmp_01[0] &= kmask1;\n\n // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_10, b_ptr_1[b].scales + sb * 24, 12);\n utmp_10[3] = ((utmp_10[2] >> 4) & kmask2) | (((utmp_10[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_10 = utmp_10[1] & kmask1;\n utmp_10[1] = (utmp_10[2] & kmask2) | (((utmp_10[0] >> 6) & kmask3) << 4);\n utmp_10[2] = uaux_10;\n utmp_10[0] &= kmask1;\n\n // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_11, b_ptr_1[b].scales + 12 + sb * 24, 12);\n utmp_11[3] = ((utmp_11[2] >> 4) & kmask2) | (((utmp_11[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_11 = utmp_11[1] & kmask1;\n utmp_11[1] = (utmp_11[2] & kmask2) | (((utmp_11[0] >> 6) & kmask3) << 4);\n utmp_11[2] = uaux_11;\n utmp_11[0] &= kmask1;\n\n // Scales of first sub block in the sb loop\n const __m256i mins_and_scales_0 = _mm256_set_epi32(utmp_10[3], utmp_10[2], utmp_10[1], utmp_10[0], utmp_00[3], utmp_00[2], utmp_00[1], utmp_00[0]);\n const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0));\n\n // Scales of second sub block in the sb loop\n const __m256i mins_and_scales_1 = _mm256_set_epi32(utmp_11[3], utmp_11[2], utmp_11[1], utmp_11[0], utmp_01[3], utmp_01[2], utmp_01[1], utmp_01[0]);\n const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1));\n\n // Mins of first and second sub block of Q4_K block are arranged side by side\n const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(_mm256_shuffle_epi32(mins_and_scales_0, 78), _mm256_shuffle_epi32(mins_and_scales_1, 78)));\n\n const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238);\n\n // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 * sb)));\n __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0);\n __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17);\n __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 256 * sb)));\n __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0);\n __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17);\n __m256i lhs_mat_ymm_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 256 * sb)));\n __m256i lhs_mat_ymm_01_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 0);\n __m256i lhs_mat_ymm_23_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 17);\n __m256i lhs_mat_ymm_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 256 * sb)));\n __m256i lhs_mat_ymm_01_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 0);\n __m256i lhs_mat_ymm_23_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 17);\n __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 256 * sb)));\n __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0);\n __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17);\n __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 256 * sb)));\n __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0);\n __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17);\n __m256i lhs_mat_ymm_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 256 * sb)));\n __m256i lhs_mat_ymm_01_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 0);\n __m256i lhs_mat_ymm_23_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 17);\n __m256i lhs_mat_ymm_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 256 * sb)));\n __m256i lhs_mat_ymm_01_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 0);\n __m256i lhs_mat_ymm_23_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 17);\n\n //Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into a 512 bit vector\n __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1);\n __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1);\n __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1);\n __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1);\n __m512i lhs_mat_01_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_02), lhs_mat_ymm_01_02, 1);\n __m512i lhs_mat_23_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_02), lhs_mat_ymm_23_02, 1);\n __m512i lhs_mat_01_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_03), lhs_mat_ymm_01_03, 1);\n __m512i lhs_mat_23_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_03), lhs_mat_ymm_23_03, 1);\n\n __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1);\n __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1);\n __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1);\n __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1);\n __m512i lhs_mat_01_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_12), lhs_mat_ymm_01_12, 1);\n __m512i lhs_mat_23_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_12), lhs_mat_ymm_23_12, 1);\n __m512i lhs_mat_01_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_13), lhs_mat_ymm_01_13, 1);\n __m512i lhs_mat_23_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_13), lhs_mat_ymm_23_13, 1);\n\n // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks\n __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].bsums + 16 * sb)));\n __m256i lhs_bsums_hsum_ymm_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)));\n lhs_bsums_hsum_ymm_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_ymm_0123_01, lhs_bsums_hsum_ymm_0123_01, 0);\n __m512i lhs_bsums_hsum_0123_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_hsum_ymm_0123_01), lhs_bsums_hsum_ymm_0123_01, 1);\n\n // Shuffle pattern one - left side input\n const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3)\n const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11)\n const __m512i lhs_mat_01_02_sp1 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19)\n const __m512i lhs_mat_23_02_sp1 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)160); //A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19)\n const __m512i lhs_mat_01_03_sp1 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27)\n const __m512i lhs_mat_23_03_sp1 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)160); //A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27)\n\n const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3)\n const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11)\n const __m512i lhs_mat_01_12_sp1 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19)\n const __m512i lhs_mat_23_12_sp1 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)160); //A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19)\n const __m512i lhs_mat_01_13_sp1 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27)\n const __m512i lhs_mat_23_13_sp1 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)160); //A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27)\n\n const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7)\n const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15)\n const __m512i lhs_mat_01_02_sp2 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23)\n const __m512i lhs_mat_23_02_sp2 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)245); //A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23)\n const __m512i lhs_mat_01_03_sp2 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31)\n const __m512i lhs_mat_23_03_sp2 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)245); //A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31)\n\n const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7)\n const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15)\n const __m512i lhs_mat_01_12_sp2 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23)\n const __m512i lhs_mat_23_12_sp2 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)245); //A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23)\n const __m512i lhs_mat_01_13_sp2 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31)\n const __m512i lhs_mat_23_13_sp2 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)245); //A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1));\n __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1));\n __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1));\n __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1));\n __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1));\n __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1));\n __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1));\n __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1));\n\n __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2));\n __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2));\n __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2));\n __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2));\n __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2));\n __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2));\n __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2));\n __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2));\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0);\n iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0);\n iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0);\n iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0);\n\n iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1);\n iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1);\n iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1);\n iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1);\n\n // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step)\n __m512i iacc_row_0_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_0, _mm512_shuffle_epi32(iacc_mat_01_0, (_MM_PERM_ENUM)78));\n __m512i iacc_row_1_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_0, (_MM_PERM_ENUM)78), iacc_mat_01_0);\n __m512i iacc_row_2_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_0, _mm512_shuffle_epi32(iacc_mat_11_0, (_MM_PERM_ENUM)78));\n __m512i iacc_row_3_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10_0, (_MM_PERM_ENUM)78), iacc_mat_11_0);\n __m512i iacc_row_0_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_1, _mm512_shuffle_epi32(iacc_mat_01_1, (_MM_PERM_ENUM)78));\n __m512i iacc_row_1_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_1, (_MM_PERM_ENUM)78), iacc_mat_01_1);\n __m512i iacc_row_2_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_1, _mm512_shuffle_epi32(iacc_mat_11_1, (_MM_PERM_ENUM)78));\n __m512i iacc_row_3_1 = _mm512_mask_blend_epi32(0xCCCC,_mm512_shuffle_epi32(iacc_mat_10_1, (_MM_PERM_ENUM)78), iacc_mat_11_1);\n\n __m512i iacc_row_0 = _mm512_add_epi32(iacc_row_0_0, iacc_row_0_1);\n __m512i iacc_row_1 = _mm512_add_epi32(iacc_row_1_0, iacc_row_1_1);\n __m512i iacc_row_2 = _mm512_add_epi32(iacc_row_2_0, iacc_row_2_1);\n __m512i iacc_row_3 = _mm512_add_epi32(iacc_row_3_0, iacc_row_3_1);\n\n // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d);\n const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1);\n\n // Multiply with appropriate scales and accumulate (for both d and dmin) below\n acc_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n acc_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n acc_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n acc_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n\n __m512i iacc_row_min_0 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)0), mins_01);\n __m512i iacc_row_min_1 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)85), mins_01);\n __m512i iacc_row_min_2 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)170), mins_01);\n __m512i iacc_row_min_3 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)255), mins_01);\n\n acc_min_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]);\n acc_min_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]);\n acc_min_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]);\n acc_min_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]);\n }\n }\n // Store accumulated values\n for (int i = 0; i < 4; i++) {\n _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i]));\n }\n }\n }\n if (anc != nc) {\n xstart = anc/8;\n y = 0;\n }\n#endif // __AVX512BW__ && __AVX512DQ__\n\n // Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation\n for (; y < anr / 4; y += 4) {\n\n const block_q8_Kx4 * a_ptrs[4];\n\n a_ptrs[0] = a_ptr_start + (y * nb);\n for (int i = 0; i < 3; ++i) {\n a_ptrs[i + 1] = a_ptrs[i] + nb;\n }\n\n // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = xstart; x < nc / 8; x++) {\n\n const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n // Master FP accumulators\n __m256 acc_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_rows[i] = _mm256_setzero_ps();\n }\n\n __m256 acc_min_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_min_rows[i] = _mm256_setzero_ps();\n }\n\n // For super block\n for (int64_t b = 0; b < nb; b++) {\n\n // Scale values - Load the eight scale values of block_q4_kx8\n const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n\n // dmin values - Load the eight dmin values of block_q4_kx8\n const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration\n for (int sb = 0; sb < QK_K / 64; sb++) {\n\n // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256));\n\n // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n // 4-bit -> 8-bit\n // First sub block of the two sub blocks processed in the iteration\n const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7)\n const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7)\n\n const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15)\n const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15)\n\n const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23)\n const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23)\n\n const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31)\n const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31)\n\n // Second sub block of the two sub blocks processed in the iteration\n const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7)\n const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7)\n\n const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15)\n const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15)\n\n const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23)\n const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23)\n\n const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31)\n const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31)\n\n // Shuffle pattern one - right side input\n const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3)\n const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3)\n\n const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11)\n const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11)\n\n const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19)\n const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19)\n\n const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27)\n const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27)\n\n const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3)\n const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3)\n\n const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11)\n const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11)\n\n const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19)\n const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19)\n\n const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27)\n const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27)\n\n\n // Shuffle pattern two - right side input\n const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7)\n const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7)\n\n const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15)\n const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15)\n\n const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23)\n const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23)\n\n const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31)\n const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31)\n\n const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7)\n const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7)\n\n const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15)\n const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15)\n\n const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23)\n const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23)\n\n const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31)\n const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31)\n\n uint32_t utmp_0[4], utmp_1[4];\n\n // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together\n // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12);\n utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_0 = utmp_0[1] & kmask1;\n utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4);\n utmp_0[2] = uaux_0;\n utmp_0[0] &= kmask1;\n\n // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12);\n utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_1 = utmp_1[1] & kmask1;\n utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4);\n utmp_1[2] = uaux_1;\n utmp_1[0] &= kmask1;\n\n // Scales of first sub block in the sb loop\n const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]);\n const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0));\n\n // Scales of second sub block in the sb loop\n const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]);\n const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1));\n\n // Mins of first and second sub block of Q4_K block are arranged side by side\n const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78)));\n\n const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68);\n const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238);\n\n const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68);\n const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238);\n\n for (int rp = 0; rp < 4; rp++) {\n\n // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 * sb)));\n __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0);\n __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17);\n __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 256 * sb)));\n __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0);\n __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17);\n __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 256 * sb)));\n __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0);\n __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17);\n __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 256 * sb)));\n __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0);\n __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17);\n __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 256 * sb)));\n __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0);\n __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17);\n __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 256 * sb)));\n __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0);\n __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17);\n __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 256 * sb)));\n __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0);\n __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17);\n __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 256 * sb)));\n __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0);\n __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17);\n\n // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks\n __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].bsums + 16 * sb)));\n __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)));\n lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0);\n\n // Shuffle pattern one - left side input\n const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3)\n\n const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11)\n\n const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19)\n const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160); //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19)\n\n const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27)\n const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27)\n\n const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3)\n\n const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11)\n\n const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19)\n const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160); //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19)\n\n const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27)\n const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27)\n\n // Shuffle pattern two- left side input\n const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7)\n\n const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15)\n\n const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23)\n const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23)\n\n const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31)\n const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31)\n\n const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7)\n\n const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15)\n\n const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23)\n const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23)\n\n const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31)\n const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1));\n __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1));\n __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1));\n __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1));\n __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1));\n __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1));\n __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1));\n __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1));\n\n __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2));\n __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2));\n __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2));\n __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2));\n __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2));\n __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2));\n __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2));\n __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2));\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0);\n iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0);\n iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0);\n iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0);\n\n iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1);\n iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1);\n iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1);\n iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1);\n\n // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step)\n __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204);\n __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204);\n __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204);\n __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204);\n __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204);\n __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204);\n __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204);\n __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204);\n\n __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1);\n __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1);\n __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1);\n __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1);\n\n // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d);\n const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);//GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask);\n\n // Multiply with appropriate scales and accumulate (for both d and dmin) below\n acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n\n __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0), mins_01);\n __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85), mins_01);\n __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170), mins_01);\n __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255), mins_01);\n\n acc_min_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]);\n acc_min_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]);\n acc_min_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]);\n acc_min_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]);\n\n }\n }\n }\n // Store the accumulated values\n for (int i = 0; i < 16; i++) {\n _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i]));\n }\n }\n }\n for (; y < nr / 4; y++) {\n\n const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb);\n\n for (int64_t x = xstart; x < nc / 8; x++) {\n\n const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n // Master FP accumulators\n __m256 acc_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_rows[i] = _mm256_setzero_ps();\n }\n\n __m256 acc_min_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_min_rows[i] = _mm256_setzero_ps();\n }\n\n for (int64_t b = 0; b < nb; b++) {\n\n // Scale values - Load the eight scale values of block_q4_Kx8\n const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n\n // dmin values - Load the eight dmin values of block_q4_Kx8\n const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration\n for (int sb = 0; sb < QK_K / 64; sb++) {\n\n // Load the eight block_q4_k for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256));\n\n // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n // 4-bit -> 8-bit\n // First sub block of the two sub blocks processed in the iteration\n const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7)\n const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7)\n\n const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15)\n const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15)\n\n const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23)\n const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23)\n\n const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31)\n const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31)\n\n // Second sub block of the two sub blocks processed in the iteration\n const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7)\n const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7)\n\n const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15)\n const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15)\n\n const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23)\n const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23)\n\n const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31)\n const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31)\n\n // Shuffle pattern one - right side input\n const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3)\n const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3)\n\n const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11)\n const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11)\n\n const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19)\n const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19)\n\n const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27)\n const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27)\n\n const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3)\n const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3)\n\n const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11)\n const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11)\n\n const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19)\n const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19)\n\n const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27)\n const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27)\n\n // Shuffle pattern two - right side input\n const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7)\n const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7)\n\n const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15)\n const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15)\n\n const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23)\n const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23)\n\n const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31)\n const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31)\n\n const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7)\n const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7)\n\n const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15)\n const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15)\n\n const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23)\n const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23)\n\n const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31)\n const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31)\n\n uint32_t utmp_0[4], utmp_1[4];\n\n // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together\n // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12);\n utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_0 = utmp_0[1] & kmask1;\n utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4);\n utmp_0[2] = uaux_0;\n utmp_0[0] &= kmask1;\n\n // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures when sb = 1\n memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12);\n utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4);\n const uint32_t uaux_1 = utmp_1[1] & kmask1;\n utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4);\n utmp_1[2] = uaux_1;\n utmp_1[0] &= kmask1;\n\n // Scales of first sub block in the sb loop\n const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]);\n const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0));\n\n // Scales of second sub block in the sb loop\n const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]);\n const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1));\n\n // Mins of first and second sub block of Q4_K block are arranged side by side\n const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78)));\n\n const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68);\n const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238);\n\n const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68);\n const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238);\n\n // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 * sb)));\n __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0);\n __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17);\n __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 256 * sb)));\n __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0);\n __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17);\n __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 256 * sb)));\n __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0);\n __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17);\n __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 256 * sb)));\n __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0);\n __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17);\n __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 256 * sb)));\n __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0);\n __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17);\n __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 256 * sb)));\n __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0);\n __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17);\n __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 256 * sb)));\n __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0);\n __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17);\n __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 256 * sb)));\n __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0);\n __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17);\n\n // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks\n __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].bsums + 16 * sb)));\n __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)));\n lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0);\n\n // Shuffle pattern one - left side input\n const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3)\n\n const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11)\n\n const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19)\n const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160); //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19)\n\n const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27)\n const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27)\n\n const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3)\n\n const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11)\n\n const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19)\n const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160); //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19)\n\n const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27)\n const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27)\n\n // Shuffle pattern two- left side input\n const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7)\n\n const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15)\n\n const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23)\n const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23)\n\n const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31)\n const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31)\n\n const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7)\n\n const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15)\n\n const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23)\n const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23)\n\n const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31)\n const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1));\n __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1));\n __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1));\n __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1));\n __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1));\n __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1));\n __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1));\n __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1));\n\n __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2));\n __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2));\n __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2));\n __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2));\n __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2));\n __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2));\n __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2));\n __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2));\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0);\n iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0);\n iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0);\n iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0);\n\n iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1);\n iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1);\n iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1);\n iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1);\n\n // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step)\n __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204);\n __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204);\n __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204);\n __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204);\n __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204);\n __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204);\n __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204);\n __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204);\n\n __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1);\n __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1);\n __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1);\n __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1);\n\n // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d);\n const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); //GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask);\n\n // Multiply with appropriate scales and accumulate (for both d and dmin) below\n acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n\n __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0), mins_01);\n __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85), mins_01);\n __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170), mins_01);\n __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255), mins_01);\n\n acc_min_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]);\n acc_min_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]);\n acc_min_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]);\n acc_min_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]);\n }\n }\n\n // Store the accumulated values\n for (int i = 0; i < 4; i++) {\n _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i]));\n }\n }\n }\n\n#else\n UNUSED(kmask1);\n UNUSED(kmask2);\n UNUSED(kmask3);\n ggml_gemm_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n#endif\n}\n\nvoid ggml_gemm_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__) || defined(__AVX512F__)\n {\n __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_iq4nl));\n signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n gemm_q4_b32_8x8_q8_0_lut_avx(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n return;\n }\n#endif // defined(__AVX2__) || defined(__AVX512F__)\n\n ggml_gemm_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__) || defined(__AVX512F__)\n {\n __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_mxfp4));\n signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n gemm_q4_b32_8x8_q8_0_lut_avx(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n return;\n }\n#endif // defined(__AVX2__) || defined(__AVX512F__)\n\n ggml_gemm_mxfp4_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n#if defined(__AVX2__) || defined(__AVX512F__)\n const block_q2_Kx8 * b_ptr_start = (const block_q2_Kx8 * ) vx;\n const block_q8_Kx4 * a_ptr_start = (const block_q8_Kx4 * ) vy;\n int64_t b_nb = n / QK_K;\n int64_t y = 0;\n\n // Permute mask used for easier vector processing at later stages\n __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4);\n int64_t xstart = 0;\n int anr = nr - nr % 16; // Used to align nr with boundary of 16\n\n // Mask to convert 2 bit and 4 bit values into a bytes\n const __m256i m3b = _mm256_set1_epi8(3);\n const __m128i m4b_sse = _mm_set1_epi8(0xF);\n\n //Mask to get appropriate scales\n __m128i scalesmask1_sse = _mm_set_epi8(14,14,12,12,10,10,8,8,6,6,4,4,2,2,0,0);\n __m128i scalesmask2_sse = _mm_set_epi8(15,15,13,13,11,11,9,9,7,7,5,5,3,3,1,1);\n\n __m256i scalesmask1 = _mm256_castsi128_si256(scalesmask1_sse);\n scalesmask1 = _mm256_permute2f128_si256(scalesmask1, scalesmask1, 0);\n __m256i scalesmask2 = _mm256_castsi128_si256(scalesmask2_sse);\n scalesmask2 = _mm256_permute2f128_si256(scalesmask2, scalesmask2, 0);\n\n#if defined(__AVX512BW__) && defined(__AVX512DQ__)\n\n int anc = nc - nc % 16; // Used to align nc with boundary of 16\n\n // Mask to mask out nibbles from packed bytes\n const __m256i m4b = _mm256_set1_epi8(0x0F);\n // Mask to mask out nibbles from packed bytes expanded to 512 bit length\n const __m512i m3bexpanded = _mm512_set1_epi8(3);\n //Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation\n for (; y < anr / 4; y += 4) {\n\n const block_q8_Kx4 * a_ptrs[4];\n\n a_ptrs[0] = a_ptr_start + (y * nb);\n for (int i = 0; i < 3; ++i) {\n a_ptrs[i + 1] = a_ptrs[i] + nb;\n }\n\n // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = 0; x < anc / 8; x += 2) {\n\n const block_q2_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n const block_q2_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n // Master FP accumulators\n __m512 acc_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_rows[i] = _mm512_setzero_ps();\n }\n\n __m512 acc_min_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_min_rows[i] = _mm512_setzero_ps();\n }\n // For super block\n for (int64_t b = 0; b < nb; b++) {\n // Delta values - Load the sixteen scale values from two block_q2_kx8 structures\n const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n\n // dmin values - Load the sixteen dmin values from two block_q2_kx8 structures\n const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin);\n\n // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration\n for (int sb = 0; sb < QK_K / 128; sb++) {\n\n // Load the eight block_q2_k for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256));\n\n const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256));\n const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256));\n\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240);\n const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240);\n\n const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1);\n const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1);\n const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1);\n const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1);\n\n //2-bit -> 8-bit\n const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0,m3bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7)\n const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0,m3bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7)\n const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1,m3bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15)\n const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1,m3bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15)\n const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(rhs_raw_mat_014589CD_2,m3bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7)\n const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2,m3bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7)\n const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(rhs_raw_mat_014589CD_3,m3bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15)\n const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3,m3bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15)\n\n const __m512i rhs_mat_014589CD_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 2), m3bexpanded); //B20(0-7) B21(0-7) B24(0-7) B25(0-7) B28(0-7) B29(0-7) B2C(0-7) B2D(0-7)\n const __m512i rhs_mat_2367ABEF_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 2), m3bexpanded); //B22(0-7) B23(0-7) B26(0-7) B27(0-7) B2A(0-7) B2B(0-7) B2E(0-7) B2F(0-7)\n\n const __m512i rhs_mat_014589CD_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 2), m3bexpanded); //B20(8-15) B21(8-15) B24(8-15) B25(8-15) B28(8-15) B29(8-15) B2C(8-15) B2D(8-15)\n const __m512i rhs_mat_2367ABEF_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 2), m3bexpanded); //B22(8-15) B23(8-15) B26(8-15) B27(8-15) B2A(8-15) B2B(8-15) B2E(8-15) B2F(8-15)\n\n const __m512i rhs_mat_014589CD_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 2), m3bexpanded); //B30(0-7) B31(0-7) B34(0-7) B35(0-7) B38(0-7) B39(0-7) B3C(0-7) B3D(0-7)\n const __m512i rhs_mat_2367ABEF_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 2), m3bexpanded); //B32(0-7) B33(0-7) B36(0-7) B37(0-7) B3A(0-7) B3B(0-7) B3E(0-7) B3F(0-7)\n\n const __m512i rhs_mat_014589CD_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 2), m3bexpanded); //B30(8-15) B31(8-15) B34(8-15) B35(8-15) B38(8-15) B39(8-15) B3C(8-15) B3D(8-15)\n const __m512i rhs_mat_2367ABEF_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 2), m3bexpanded); //B32(8-15) B33(8-15) B36(8-15) B37(8-15) B3A(8-15) B3B(8-15) B3E(8-15) B3F(8-15)\n\n const __m512i rhs_mat_014589CD_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m3bexpanded); //B40(0-7) B41(0-7) B44(0-7) B45(0-7) B48(0-7) B49(0-7) B4C(0-7) B4D(0-7)\n const __m512i rhs_mat_2367ABEF_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m3bexpanded); //B42(0-7) B43(0-7) B46(0-7) B47(0-7) B4A(0-7) B4B(0-7) B4E(0-7) B4F(0-7)\n\n const __m512i rhs_mat_014589CD_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m3bexpanded); //B40(8-15) B41(8-15) B44(8-15) B45(8-15) B48(8-15) B49(8-15) B4C(8-15) B4D(8-15)\n const __m512i rhs_mat_2367ABEF_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m3bexpanded); //B42(8-15) B43(8-15) B46(8-15) B47(8-15) B4A(8-15) B4B(8-15) B4E(8-15) B4F(8-15)\n\n const __m512i rhs_mat_014589CD_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m3bexpanded); //B50(0-7) B51(0-7) B54(0-7) B55(0-7) B58(0-7) B59(0-7) B5C(0-7) B5D(0-7)\n const __m512i rhs_mat_2367ABEF_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m3bexpanded); //B52(0-7) B53(0-7) B56(0-7) B57(0-7) B5A(0-7) B5B(0-7) B5E(0-7) B5F(0-7)\n\n const __m512i rhs_mat_014589CD_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m3bexpanded); //B50(8-15) B51(8-15) B54(8-15) B55(8-15) B58(8-15) B59(8-15) B5C(8-15) B5D(8-15)\n const __m512i rhs_mat_2367ABEF_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m3bexpanded); //B52(8-15) B53(8-15) B56(8-15) B57(8-15) B5A(8-15) B5B(8-15) B5E(8-15) B5F(8-15)\n\n const __m512i rhs_mat_014589CD_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 6), m3bexpanded); //B60(0-7) B61(0-7) B64(0-7) B65(0-7) B68(0-7) B69(0-7) B6C(0-7) B6D(0-7)\n const __m512i rhs_mat_2367ABEF_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 6), m3bexpanded); //B62(0-7) B63(0-7) B66(0-7) B67(0-7) B6A(0-7) B6B(0-7) B6E(0-7) B6F(0-7)\n\n const __m512i rhs_mat_014589CD_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 6), m3bexpanded); //B60(8-15) B61(8-15) B64(8-15) B65(8-15) B68(8-15) B69(8-15) B6C(8-15) B6D(8-15)\n const __m512i rhs_mat_2367ABEF_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 6), m3bexpanded); //B62(8-15) B63(8-15) B66(8-15) B67(8-15) B6A(8-15) B6B(8-15) B6E(8-15) B6F(8-15)\n\n const __m512i rhs_mat_014589CD_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 6), m3bexpanded); //B70(0-7) B71(0-7) B74(0-7) B75(0-7) B78(0-7) B79(0-7) B7C(0-7) B7D(0-7)\n const __m512i rhs_mat_2367ABEF_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 6), m3bexpanded); //B72(0-7) B73(0-7) B76(0-7) B77(0-7) B7A(0-7) B7B(0-7) B7E(0-7) B7F(0-7)\n\n const __m512i rhs_mat_014589CD_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 6), m3bexpanded); //B70(8-15) B71(8-15) B74(8-15) B75(8-15) B78(8-15) B79(8-15) B7C(8-15) B7D(8-15)\n const __m512i rhs_mat_2367ABEF_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 6), m3bexpanded); //B72(8-15) B73(8-15) B76(8-15) B77(8-15) B7A(8-15) B7B(8-15) B7E(8-15) B7F(8-15)\n\n const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3)\n const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3)\n\n const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11)\n\n const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3)\n const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3)\n\n const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11)\n const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11)\n\n const __m512i rhs_mat_014589CD_20_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)136); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3) B28(0-3) B29(0-3) B28(0-3) B29(0-3) B2C(0-3) B2D(0-3) B2C(0-3) B2D(0-3)\n const __m512i rhs_mat_2367ABEF_20_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)136); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3) B2A(0-3) B2B(0-3) B2A(0-3) B2B(0-3) B2E(0-3) B2F(0-3) B2E(0-3) B2F(0-3)\n\n const __m512i rhs_mat_014589CD_21_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)136); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11) B28(8-11) B29(8-11) B28(8-11) B29(8-11) B2C(8-11) B2D(8-11) B2C(8-11) B2D(8-11)\n const __m512i rhs_mat_2367ABEF_21_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)136); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11) B2A(8-11) B2B(8-11) B2A(8-11) B2B(8-11) B2E(8-11) B2F(8-11) B2E(8-11) B2F(8-11)\n\n const __m512i rhs_mat_014589CD_30_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)136); ///B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3) B38(0-3) B39(0-3) B38(0-3) B39(0-3) B3C(0-3) B3D(0-3) B3C(0-3) B3D(0-3)\n const __m512i rhs_mat_2367ABEF_30_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)136); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3) B3A(0-3) B3B(0-3) B3A(0-3) B3B(0-3) B3E(0-3) B3F(0-3) B3E(0-3) B3F(0-3)\n\n const __m512i rhs_mat_014589CD_31_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)136); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11) B38(8-11) B39(8-11) B38(8-11) B39(8-11) B3C(8-11) B3D(8-11) B3C(8-11) B3D(8-11)\n const __m512i rhs_mat_2367ABEF_31_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)136); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11) B3A(8-11) B3B(8-11) B3A(8-11) B3B(8-11) B3E(8-11) B3F(8-11) B3E(8-11) B3F(8-11)\n\n const __m512i rhs_mat_014589CD_40_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)136); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3) B48(0-3) B49(0-3) B48(0-3) B49(0-3) B4C(0-3) B4D(0-3) B4C(0-3) B4D(0-3)\n const __m512i rhs_mat_2367ABEF_40_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)136); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3) B4A(0-3) B4B(0-3) B4A(0-3) B4B(0-3) B4E(0-3) B4F(0-3) B4E(0-3) B4F(0-3)\n\n const __m512i rhs_mat_014589CD_41_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)136); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11) B48(8-11) B49(8-11) B48(8-11) B49(8-11) B4C(8-11) B4D(8-11) B4C(8-11) B4D(8-11)\n const __m512i rhs_mat_2367ABEF_41_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)136); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11) B4A(8-11) B4B(8-11) B4A(8-11) B4B(8-11) B4E(8-11) B4F(8-11) B4E(8-11) B4F(8-11)\n\n const __m512i rhs_mat_014589CD_50_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)136); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3) B58(0-3) B59(0-3) B58(0-3) B59(0-3) B5C(0-3) B5D(0-3) B5C(0-3) B5D(0-3)\n const __m512i rhs_mat_2367ABEF_50_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)136); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3) B5A(0-3) B5B(0-3) B5A(0-3) B5B(0-3) B5E(0-3) B5F(0-3) B5E(0-3) B5F(0-3)\n\n const __m512i rhs_mat_014589CD_51_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)136); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11) B58(8-11) B59(8-11) B58(8-11) B59(8-11) B5C(8-11) B5D(8-11) B5C(8-11) B5D(8-11)\n const __m512i rhs_mat_2367ABEF_51_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)136); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11) B5A(8-11) B5B(8-11) B5A(8-11) B5B(8-11) B5E(8-11) B5F(8-11) B5E(8-11) B5F(8-11)\n\n const __m512i rhs_mat_014589CD_60_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)136); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3) B68(0-3) B69(0-3) B68(0-3) B69(0-3) B6C(0-3) B6D(0-3) B6C(0-3) B6D(0-3)\n const __m512i rhs_mat_2367ABEF_60_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)136); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3) B6A(0-3) B6B(0-3) B6A(0-3) B6B(0-3) B6E(0-3) B6F(0-3) B6E(0-3) B6F(0-3)\n\n const __m512i rhs_mat_014589CD_61_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)136); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11) B68(8-11) B69(8-11) B68(8-11) B69(8-11) B6C(8-11) B6D(8-11) B6C(8-11) B6D(8-11)\n const __m512i rhs_mat_2367ABEF_61_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)136); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11) B6A(8-11) B6B(8-11) B6A(8-11) B6B(8-11) B6E(8-11) B6F(8-11) B6E(8-11) B6F(8-11)\n\n const __m512i rhs_mat_014589CD_70_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)136); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3) B78(0-3) B79(0-3) B78(0-3) B79(0-3) B7C(0-3) B7D(0-3) B7C(0-3) B7D(0-3)\n const __m512i rhs_mat_2367ABEF_70_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)136); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3) B7A(0-3) B7B(0-3) B7A(0-3) B7B(0-3) B7E(0-3) B7F(0-3) B7E(0-3) B7F(0-3)\n\n const __m512i rhs_mat_014589CD_71_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n const __m512i rhs_mat_2367ABEF_71_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)136); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11) B7A(8-11) B7B(8-11) B7A(8-11) B7B(8-11) B7E(8-11) B7F(8-11) B7E(8-11) B7F(8-11)\n\n const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7)\n const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7)\n\n const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15)\n const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15)\n\n const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7)\n const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7)\n\n const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15)\n const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15)\n\n const __m512i rhs_mat_014589CD_20_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)221); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7) B28(4-7) B29(4-7) B28(4-7) B29(4-7) B2C(4-7) B2D(4-7) B2C(4-7) B2D(4-7)\n const __m512i rhs_mat_2367ABEF_20_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)221); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7) B2A(4-7) B2B(4-7) B2A(4-7) B2B(4-7) B2E(4-7) B2F(4-7) B2E(4-7) B2F(4-7)\n\n const __m512i rhs_mat_014589CD_21_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)221); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15) B28(12-15) B29(12-15) B28(12-15) B29(12-15) B2C(12-15) B2D(12-15) B2C(12-15) B2D(12-15)\n const __m512i rhs_mat_2367ABEF_21_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)221); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15) B2A(12-15) B2B(12-15) B2A(12-15) B2B(12-15) B2E(12-15) B2F(12-15) B2E(12-15) B2F(12-15)\n\n const __m512i rhs_mat_014589CD_30_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)221); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7) B38(4-7) B39(4-7) B38(4-7) B39(4-7) B3C(4-7) B3D(4-7) B3C(4-7) B3D(4-7)\n const __m512i rhs_mat_2367ABEF_30_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)221); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7) B3A(4-7) B3B(4-7) B3A(4-7) B3B(4-7) B3E(4-7) B3F(4-7) B3E(4-7) B3F(4-7)\n\n const __m512i rhs_mat_014589CD_31_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)221); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15) B38(12-15) B39(12-15) B38(12-15) B39(12-15) B3C(12-15) B3D(12-15) B3C(12-15) B3D(12-15)\n const __m512i rhs_mat_2367ABEF_31_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)221); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15) B3A(12-15) B3B(12-15) B3A(12-15) B3B(12-15) B3E(12-15) B3F(12-15) B3E(12-15) B3F(12-15)\n\n const __m512i rhs_mat_014589CD_40_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)221); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7) B48(4-7) B49(4-7) B48(4-7) B49(4-7) B4C(4-7) B4D(4-7) B4C(4-7) B4D(4-7)\n const __m512i rhs_mat_2367ABEF_40_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)221); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7) B4A(4-7) B4B(4-7) B4A(4-7) B4B(4-7) B4E(4-7) B4F(4-7) B4E(4-7) B4F(4-7)\n\n const __m512i rhs_mat_014589CD_41_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)221); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15) B48(12-15) B49(12-15) B48(12-15) B49(12-15) B4C(12-15) B4D(12-15) B4C(12-15) B4D(12-15)\n const __m512i rhs_mat_2367ABEF_41_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)221); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15) B4A(12-15) B4B(12-15) B4A(12-15) B4B(12-15) B4E(12-15) B4F(12-15) B4E(12-15) B4F(12-15)\n\n const __m512i rhs_mat_014589CD_50_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)221); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7) B58(4-7) B59(4-7) B58(4-7) B59(4-7) B5C(4-7) B5D(4-7) B5C(4-7) B5D(4-7)\n const __m512i rhs_mat_2367ABEF_50_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)221); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7) B5A(4-7) B5B(4-7) B5A(4-7) B5B(4-7) B5E(4-7) B5F(4-7) B5E(4-7) B5F(4-7)\n\n const __m512i rhs_mat_014589CD_51_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)221); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15) B58(12-15) B59(12-15) B58(12-15) B59(12-15) B5C(12-15) B5D(12-15) B5C(12-15) B5D(12-15)\n const __m512i rhs_mat_2367ABEF_51_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)221); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15) B5A(12-15) B5B(12-15) B5A(12-15) B5B(12-15) B5E(12-15) B5F(12-15) B5E(12-15) B5F(12-15)\n\n const __m512i rhs_mat_014589CD_60_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)221); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7) B68(4-7) B69(4-7) B68(4-7) B69(4-7) B6C(4-7) B6D(4-7) B6C(4-7) B6D(4-7)\n const __m512i rhs_mat_2367ABEF_60_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)221); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7) B6A(4-7) B6B(4-7) B6A(4-7) B6B(4-7) B6E(4-7) B6F(4-7) B6E(4-7) B6F(4-7)\n\n const __m512i rhs_mat_014589CD_61_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)221); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15) B68(12-15) B69(12-15) B68(12-15) B69(12-15) B6C(12-15) B6D(12-15) B6C(12-15) B6D(12-15)\n const __m512i rhs_mat_2367ABEF_61_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)221); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15) B6A(12-15) B6B(12-15) B6A(12-15) B6B(12-15) B6E(12-15) B6F(12-15) B6E(12-15) B6F(12-15)\n\n const __m512i rhs_mat_014589CD_70_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)221); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7) B78(4-7) B79(4-7) B78(4-7) B79(4-7) B7C(4-7) B7D(4-7) B7C(4-7) B7D(4-7)\n const __m512i rhs_mat_2367ABEF_70_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)221); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7) B7A(4-7) B7B(4-7) B7A(4-7) B7B(4-7) B7E(4-7) B7F(4-7) B7E(4-7) B7F(4-7)\n\n const __m512i rhs_mat_014589CD_71_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)221); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15) B78(12-15) B79(12-15) B78(12-15) B79(12-15) B7C(12-15) B7D(12-15) B7C(12-15) B7D(12-15)\n const __m512i rhs_mat_2367ABEF_71_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)221); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15) B7A(12-15) B7B(12-15) B7A(12-15) B7B(12-15) B7E(12-15) B7F(12-15) B7E(12-15) B7F(12-15)\n\n //notation:superblock subblock\n //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71\n\n const __m128i mins_and_scales_01_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + sb * 64));\n const __m128i mins_and_scales_23_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 16 + sb * 64));\n const __m128i mins_and_scales_45_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 32 + sb * 64));\n const __m128i mins_and_scales_67_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 48 + sb * 64));\n\n const __m128i mins_and_scales_01_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + sb * 64));\n const __m128i mins_and_scales_23_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 16 + sb * 64));\n const __m128i mins_and_scales_45_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 32 + sb * 64));\n const __m128i mins_and_scales_67_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 48 + sb * 64));\n\n // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop\n const __m256i mins_and_scales_01 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_01_0), mins_and_scales_01_1, 1);\n const __m256i mins_and_scales_23 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_23_0), mins_and_scales_23_1, 1);\n const __m256i mins_and_scales_45 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_45_0), mins_and_scales_45_1, 1);\n const __m256i mins_and_scales_67 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_67_0), mins_and_scales_67_1, 1);\n\n // Extract scales which is lower half from mins_and_scales\n const __m256i scales_01 = _mm256_and_si256(mins_and_scales_01, m4b);\n const __m256i scales_23 = _mm256_and_si256(mins_and_scales_23, m4b);\n const __m256i scales_45 = _mm256_and_si256(mins_and_scales_45, m4b);\n const __m256i scales_67 = _mm256_and_si256(mins_and_scales_67, m4b);\n\n // Extract mins which is upper half from mins_and_scales\n const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_01, 4), m4b));\n const __m512i mins_23 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_23, 4), m4b));\n const __m512i mins_45 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_45, 4), m4b));\n const __m512i mins_67 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_67, 4), m4b));\n\n const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01,scalesmask1));\n const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01,scalesmask2));\n const __m512i scales_2 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23,scalesmask1));\n const __m512i scales_3 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23,scalesmask2));\n const __m512i scales_4 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45,scalesmask1));\n const __m512i scales_5 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45,scalesmask2));\n const __m512i scales_6 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67,scalesmask1));\n const __m512i scales_7 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67,scalesmask2));\n\n const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)238);\n\n\n for (int rp = 0; rp < 4; rp++) {\n\n // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector\n __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 512 * sb)));\n __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0);\n __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17);\n __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 512 * sb)));\n __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0);\n __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17);\n __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 512 * sb)));\n __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0);\n __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17);\n __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 512 * sb)));\n __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0);\n __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17);\n __m256i lhs_mat_ymm_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 512 * sb)));\n __m256i lhs_mat_ymm_01_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 0);\n __m256i lhs_mat_ymm_23_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 17);\n __m256i lhs_mat_ymm_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 512 * sb)));\n __m256i lhs_mat_ymm_01_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 0);\n __m256i lhs_mat_ymm_23_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 17);\n __m256i lhs_mat_ymm_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 512 * sb)));\n __m256i lhs_mat_ymm_01_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 0);\n __m256i lhs_mat_ymm_23_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 17);\n __m256i lhs_mat_ymm_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 512 * sb)));\n __m256i lhs_mat_ymm_01_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 0);\n __m256i lhs_mat_ymm_23_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 17);\n\n __m256i lhs_mat_ymm_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 + 512 * sb)));\n __m256i lhs_mat_ymm_01_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 0);\n __m256i lhs_mat_ymm_23_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 17);\n __m256i lhs_mat_ymm_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 288 + 512 * sb)));\n __m256i lhs_mat_ymm_01_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 0);\n __m256i lhs_mat_ymm_23_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 17);\n __m256i lhs_mat_ymm_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 320 + 512 * sb)));\n __m256i lhs_mat_ymm_01_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 0);\n __m256i lhs_mat_ymm_23_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 17);\n __m256i lhs_mat_ymm_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 352 + 512 * sb)));\n __m256i lhs_mat_ymm_01_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 0);\n __m256i lhs_mat_ymm_23_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 17);\n __m256i lhs_mat_ymm_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 384 + 512 * sb)));\n __m256i lhs_mat_ymm_01_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 0);\n __m256i lhs_mat_ymm_23_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 17);\n __m256i lhs_mat_ymm_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 416 + 512 * sb)));\n __m256i lhs_mat_ymm_01_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 0);\n __m256i lhs_mat_ymm_23_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 17);\n __m256i lhs_mat_ymm_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 448 + 512 * sb)));\n __m256i lhs_mat_ymm_01_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 0);\n __m256i lhs_mat_ymm_23_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 17);\n __m256i lhs_mat_ymm_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 480 + 512 * sb)));\n __m256i lhs_mat_ymm_01_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 0);\n __m256i lhs_mat_ymm_23_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 17);\n\n\n __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1);\n __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1);\n __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1);\n __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1);\n\n __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1);\n __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1);\n __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1);\n __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1);\n\n __m512i lhs_mat_01_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_20), lhs_mat_ymm_01_20, 1);\n __m512i lhs_mat_23_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_20), lhs_mat_ymm_23_20, 1);\n __m512i lhs_mat_01_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_21), lhs_mat_ymm_01_21, 1);\n __m512i lhs_mat_23_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_21), lhs_mat_ymm_23_21, 1);\n\n __m512i lhs_mat_01_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_30), lhs_mat_ymm_01_30, 1);\n __m512i lhs_mat_23_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_30), lhs_mat_ymm_23_30, 1);\n __m512i lhs_mat_01_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_31), lhs_mat_ymm_01_31, 1);\n __m512i lhs_mat_23_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_31), lhs_mat_ymm_23_31, 1);\n\n __m512i lhs_mat_01_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_40), lhs_mat_ymm_01_40, 1);\n __m512i lhs_mat_23_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_40), lhs_mat_ymm_23_40, 1);\n __m512i lhs_mat_01_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_41), lhs_mat_ymm_01_41, 1);\n __m512i lhs_mat_23_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_41), lhs_mat_ymm_23_41, 1);\n\n __m512i lhs_mat_01_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_50), lhs_mat_ymm_01_50, 1);\n __m512i lhs_mat_23_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_50), lhs_mat_ymm_23_50, 1);\n __m512i lhs_mat_01_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_51), lhs_mat_ymm_01_51, 1);\n __m512i lhs_mat_23_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_51), lhs_mat_ymm_23_51, 1);\n\n __m512i lhs_mat_01_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_60), lhs_mat_ymm_01_60, 1);\n __m512i lhs_mat_23_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_60), lhs_mat_ymm_23_60, 1);\n __m512i lhs_mat_01_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_61), lhs_mat_ymm_01_61, 1);\n __m512i lhs_mat_23_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_61), lhs_mat_ymm_23_61, 1);\n\n __m512i lhs_mat_01_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_70), lhs_mat_ymm_01_70, 1);\n __m512i lhs_mat_23_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_70), lhs_mat_ymm_23_70, 1);\n __m512i lhs_mat_01_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_71), lhs_mat_ymm_01_71, 1);\n __m512i lhs_mat_23_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_71), lhs_mat_ymm_23_71, 1);\n\n // Bsums are loaded for the different Q8_K blocks\n __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 32 * sb)));\n __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 8 + 32 * sb));\n __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 16 + 32 * sb)));\n __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 24 + 32 * sb));\n\n __m256i lhs_bsums_ymm_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1);\n __m512i lhs_bsums_01_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_0123), lhs_bsums_ymm_01_0123, 1);\n __m256i lhs_bsums_ymm_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1);\n __m512i lhs_bsums_23_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_0123), lhs_bsums_ymm_23_0123, 1); __m256i lhs_bsums_ymm_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1);\n __m512i lhs_bsums_01_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_4567), lhs_bsums_ymm_01_4567, 1);\n __m256i lhs_bsums_ymm_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1);\n __m512i lhs_bsums_23_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_4567), lhs_bsums_ymm_23_4567, 1);\n\n // Shuffle pattern one - left side input\n const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3)\n\n const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11)\n\n const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3)\n\n const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11)\n\n const __m512i lhs_mat_01_20_sp1 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)160); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3)\n const __m512i lhs_mat_23_20_sp1 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)160); //A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3)\n\n const __m512i lhs_mat_01_21_sp1 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)160); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11)\n const __m512i lhs_mat_23_21_sp1 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)160); //A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11)\n\n const __m512i lhs_mat_01_30_sp1 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)160); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3)\n const __m512i lhs_mat_23_30_sp1 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)160); //A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3)\n\n const __m512i lhs_mat_01_31_sp1 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)160); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11)\n const __m512i lhs_mat_23_31_sp1 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)160); //A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11)\n\n const __m512i lhs_mat_01_40_sp1 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)160); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3)\n const __m512i lhs_mat_23_40_sp1 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)160); //A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3)\n\n const __m512i lhs_mat_01_41_sp1 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)160); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11)\n const __m512i lhs_mat_23_41_sp1 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)160); //A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11)\n\n const __m512i lhs_mat_01_50_sp1 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)160); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3)\n const __m512i lhs_mat_23_50_sp1 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)160); //A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3)\n\n const __m512i lhs_mat_01_51_sp1 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)160); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11)\n const __m512i lhs_mat_23_51_sp1 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)160); //A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11)\n\n const __m512i lhs_mat_01_60_sp1 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)160); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3)\n const __m512i lhs_mat_23_60_sp1 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)160); //A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3)\n\n const __m512i lhs_mat_01_61_sp1 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)160); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11)\n const __m512i lhs_mat_23_61_sp1 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)160); //A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11)\n\n const __m512i lhs_mat_01_70_sp1 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)160); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3)\n const __m512i lhs_mat_23_70_sp1 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)160); //A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3)\n\n const __m512i lhs_mat_01_71_sp1 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)160); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11)\n const __m512i lhs_mat_23_71_sp1 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)160); //A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11)\n\n const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7)\n\n const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15)\n\n const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7)\n\n const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15)\n\n const __m512i lhs_mat_01_20_sp2 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)245); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7)\n const __m512i lhs_mat_23_20_sp2 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)245); //A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7)\n\n const __m512i lhs_mat_01_21_sp2 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)245); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15)\n const __m512i lhs_mat_23_21_sp2 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)245); //A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15)\n\n const __m512i lhs_mat_01_30_sp2 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)245); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7)\n const __m512i lhs_mat_23_30_sp2 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)245); //A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7)\n\n const __m512i lhs_mat_01_31_sp2 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)245); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15)\n const __m512i lhs_mat_23_31_sp2 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)245); //A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15)\n\n const __m512i lhs_mat_01_40_sp2 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)245); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7)\n const __m512i lhs_mat_23_40_sp2 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)245); //A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7)\n\n const __m512i lhs_mat_01_41_sp2 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)245); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15)\n const __m512i lhs_mat_23_41_sp2 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)245); //A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15)\n\n const __m512i lhs_mat_01_50_sp2 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)245); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7)\n const __m512i lhs_mat_23_50_sp2 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)245); //A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7)\n\n const __m512i lhs_mat_01_51_sp2 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)245); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15)\n const __m512i lhs_mat_23_51_sp2 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)245); //A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15)\n\n const __m512i lhs_mat_01_60_sp2 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)245); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7)\n const __m512i lhs_mat_23_60_sp2 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)245); //A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7)\n\n const __m512i lhs_mat_01_61_sp2 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)245); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15)\n const __m512i lhs_mat_23_61_sp2 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)245); //A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15)\n\n const __m512i lhs_mat_01_70_sp2 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)245); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7)\n const __m512i lhs_mat_23_70_sp2 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)245); //A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7)\n\n const __m512i lhs_mat_01_71_sp2 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)245); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15)\n const __m512i lhs_mat_23_71_sp2 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)245); //A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1));\n __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1));\n\n __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1));\n __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1));\n\n __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1));\n __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1));\n\n __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1));\n __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1));\n\n __m512i iacc_mat_00_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_01_21_sp1));\n __m512i iacc_mat_01_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_01_21_sp1));\n\n __m512i iacc_mat_10_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_23_21_sp1));\n __m512i iacc_mat_11_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_23_21_sp1));\n\n __m512i iacc_mat_00_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_01_31_sp1));\n __m512i iacc_mat_01_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_01_31_sp1));\n\n __m512i iacc_mat_10_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_23_31_sp1));\n __m512i iacc_mat_11_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_23_31_sp1));\n\n __m512i iacc_mat_00_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_01_41_sp1));\n __m512i iacc_mat_01_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_01_41_sp1));\n\n __m512i iacc_mat_10_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_23_41_sp1));\n __m512i iacc_mat_11_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_23_41_sp1));\n\n __m512i iacc_mat_00_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_01_51_sp1));\n __m512i iacc_mat_01_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_01_51_sp1));\n\n __m512i iacc_mat_10_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_23_51_sp1));\n __m512i iacc_mat_11_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_23_51_sp1));\n\n __m512i iacc_mat_00_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_01_61_sp1));\n __m512i iacc_mat_01_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_01_61_sp1));\n\n __m512i iacc_mat_10_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_23_61_sp1));\n __m512i iacc_mat_11_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_23_61_sp1));\n\n __m512i iacc_mat_00_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_01_71_sp1));\n __m512i iacc_mat_01_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_01_71_sp1));\n\n __m512i iacc_mat_10_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_23_71_sp1));\n __m512i iacc_mat_11_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_23_71_sp1));\n\n\n __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2));\n __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2));\n\n __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2));\n __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2));\n\n __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2));\n __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2));\n\n __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2));\n __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2));\n\n __m512i iacc_mat_00_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_01_21_sp2));\n __m512i iacc_mat_01_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_01_21_sp2));\n\n __m512i iacc_mat_10_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_23_21_sp2));\n __m512i iacc_mat_11_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_23_21_sp2));\n\n __m512i iacc_mat_00_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_01_31_sp2));\n __m512i iacc_mat_01_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_01_31_sp2));\n\n __m512i iacc_mat_10_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_23_31_sp2));\n __m512i iacc_mat_11_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_23_31_sp2));\n\n __m512i iacc_mat_00_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_01_41_sp2));\n __m512i iacc_mat_01_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_01_41_sp2));\n\n __m512i iacc_mat_10_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_23_41_sp2));\n __m512i iacc_mat_11_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_23_41_sp2));\n\n __m512i iacc_mat_00_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_01_51_sp2));\n __m512i iacc_mat_01_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_01_51_sp2));\n\n __m512i iacc_mat_10_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_23_51_sp2));\n __m512i iacc_mat_11_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_23_51_sp2));\n\n __m512i iacc_mat_00_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_01_61_sp2));\n __m512i iacc_mat_01_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_01_61_sp2));\n\n __m512i iacc_mat_10_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_23_61_sp2));\n __m512i iacc_mat_11_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_23_61_sp2));\n\n __m512i iacc_mat_00_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_01_71_sp2));\n __m512i iacc_mat_01_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_01_71_sp2));\n\n __m512i iacc_mat_10_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_23_71_sp2));\n __m512i iacc_mat_11_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_23_71_sp2));\n\n // Combine results from both shuffle patterns for each output block\n __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n __m512i iacc_mat_00_2 = _mm512_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2);\n __m512i iacc_mat_01_2 = _mm512_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2);\n __m512i iacc_mat_10_2 = _mm512_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2);\n __m512i iacc_mat_11_2 = _mm512_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2);\n\n __m512i iacc_mat_00_3 = _mm512_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2);\n __m512i iacc_mat_01_3 = _mm512_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2);\n __m512i iacc_mat_10_3 = _mm512_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2);\n __m512i iacc_mat_11_3 = _mm512_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2);\n\n __m512i iacc_mat_00_4 = _mm512_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2);\n __m512i iacc_mat_01_4 = _mm512_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2);\n __m512i iacc_mat_10_4 = _mm512_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2);\n __m512i iacc_mat_11_4 = _mm512_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2);\n\n __m512i iacc_mat_00_5 = _mm512_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2);\n __m512i iacc_mat_01_5 = _mm512_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2);\n __m512i iacc_mat_10_5 = _mm512_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2);\n __m512i iacc_mat_11_5 = _mm512_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2);\n\n __m512i iacc_mat_00_6 = _mm512_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2);\n __m512i iacc_mat_01_6 = _mm512_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2);\n __m512i iacc_mat_10_6 = _mm512_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2);\n __m512i iacc_mat_11_6 = _mm512_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2);\n\n __m512i iacc_mat_00_7 = _mm512_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2);\n __m512i iacc_mat_01_7 = _mm512_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2);\n __m512i iacc_mat_10_7 = _mm512_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2);\n __m512i iacc_mat_11_7 = _mm512_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0);\n iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0);\n iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0);\n iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0);\n\n iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1);\n iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1);\n iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1);\n iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1);\n\n iacc_mat_00_2 = _mm512_madd_epi16(iacc_mat_00_2, scale_014589CD_2);\n iacc_mat_01_2 = _mm512_madd_epi16(iacc_mat_01_2, scale_2367ABEF_2);\n iacc_mat_10_2 = _mm512_madd_epi16(iacc_mat_10_2, scale_014589CD_2);\n iacc_mat_11_2 = _mm512_madd_epi16(iacc_mat_11_2, scale_2367ABEF_2);\n\n iacc_mat_00_3 = _mm512_madd_epi16(iacc_mat_00_3, scale_014589CD_3);\n iacc_mat_01_3 = _mm512_madd_epi16(iacc_mat_01_3, scale_2367ABEF_3);\n iacc_mat_10_3 = _mm512_madd_epi16(iacc_mat_10_3, scale_014589CD_3);\n iacc_mat_11_3 = _mm512_madd_epi16(iacc_mat_11_3, scale_2367ABEF_3);\n\n iacc_mat_00_4 = _mm512_madd_epi16(iacc_mat_00_4, scale_014589CD_4);\n iacc_mat_01_4 = _mm512_madd_epi16(iacc_mat_01_4, scale_2367ABEF_4);\n iacc_mat_10_4 = _mm512_madd_epi16(iacc_mat_10_4, scale_014589CD_4);\n iacc_mat_11_4 = _mm512_madd_epi16(iacc_mat_11_4, scale_2367ABEF_4);\n\n iacc_mat_00_5 = _mm512_madd_epi16(iacc_mat_00_5, scale_014589CD_5);\n iacc_mat_01_5 = _mm512_madd_epi16(iacc_mat_01_5, scale_2367ABEF_5);\n iacc_mat_10_5 = _mm512_madd_epi16(iacc_mat_10_5, scale_014589CD_5);\n iacc_mat_11_5 = _mm512_madd_epi16(iacc_mat_11_5, scale_2367ABEF_5);\n\n iacc_mat_00_6 = _mm512_madd_epi16(iacc_mat_00_6, scale_014589CD_6);\n iacc_mat_01_6 = _mm512_madd_epi16(iacc_mat_01_6, scale_2367ABEF_6);\n iacc_mat_10_6 = _mm512_madd_epi16(iacc_mat_10_6, scale_014589CD_6);\n iacc_mat_11_6 = _mm512_madd_epi16(iacc_mat_11_6, scale_2367ABEF_6);\n\n iacc_mat_00_7 = _mm512_madd_epi16(iacc_mat_00_7, scale_014589CD_7);\n iacc_mat_01_7 = _mm512_madd_epi16(iacc_mat_01_7, scale_2367ABEF_7);\n iacc_mat_10_7 = _mm512_madd_epi16(iacc_mat_10_7, scale_014589CD_7);\n iacc_mat_11_7 = _mm512_madd_epi16(iacc_mat_11_7, scale_2367ABEF_7);\n\n __m512i iacc_mat_00 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm512_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm512_add_epi32(iacc_mat_00_6, iacc_mat_00_7)));\n __m512i iacc_mat_01 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm512_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm512_add_epi32(iacc_mat_01_6, iacc_mat_01_7)));\n __m512i iacc_mat_10 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm512_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm512_add_epi32(iacc_mat_10_6, iacc_mat_10_7)));\n __m512i iacc_mat_11 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm512_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm512_add_epi32(iacc_mat_11_6, iacc_mat_11_7)));\n\n // Straighten out to make 4 row vectors\n __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78));\n __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78), iacc_mat_01);\n __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78));\n __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78), iacc_mat_11);\n\n // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d);\n const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1);\n\n // Multiply with appropriate scales and accumulate (for both d and dmin) below\n acc_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n acc_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n acc_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n acc_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n\n // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K\n __m512i iacc_row_min_0_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)0), mins_01);\n __m512i iacc_row_min_1_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)170), mins_01);\n __m512i iacc_row_min_2_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)0), mins_01);\n __m512i iacc_row_min_3_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)170), mins_01);\n\n __m512i iacc_row_min_0_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)85), mins_23);\n __m512i iacc_row_min_1_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)255), mins_23);\n __m512i iacc_row_min_2_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)85), mins_23);\n __m512i iacc_row_min_3_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)255), mins_23);\n\n __m512i iacc_row_min_0_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)0), mins_45);\n __m512i iacc_row_min_1_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)170), mins_45);\n __m512i iacc_row_min_2_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)0), mins_45);\n __m512i iacc_row_min_3_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)170), mins_45);\n\n __m512i iacc_row_min_0_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)85), mins_67);\n __m512i iacc_row_min_1_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)255), mins_67);\n __m512i iacc_row_min_2_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)85), mins_67);\n __m512i iacc_row_min_3_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)255), mins_67);\n\n __m512i iacc_row_min_0 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm512_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67));\n __m512i iacc_row_min_1 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm512_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67));\n __m512i iacc_row_min_2 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm512_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67));\n __m512i iacc_row_min_3 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm512_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67));\n\n acc_min_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]);\n acc_min_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]);\n acc_min_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]);\n acc_min_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]);\n }\n }\n }\n // Store the accumulated values\n for (int i = 0; i < 16; i++) {\n _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i]));\n }\n }\n }\n\n for (; y < nr / 4; y ++) {\n\n const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb);\n\n // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = 0; x < anc / 8; x += 2) {\n\n const block_q2_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n const block_q2_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n // Master FP accumulators\n __m512 acc_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_rows[i] = _mm512_setzero_ps();\n }\n\n __m512 acc_min_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_min_rows[i] = _mm512_setzero_ps();\n }\n // For super block\n for (int64_t b = 0; b < nb; b++) {\n // Delta values - Load the sixteen scale values from two block_q2_kx8 structures\n const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n\n // dmin values - Load the sixteen dmin values from two block_q2_kx8 structures\n const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin);\n\n // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration\n for (int sb = 0; sb < QK_K / 128; sb++) {\n\n // Load the eight block_q2_k for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256));\n\n const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256));\n const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256));\n\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240);\n const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240);\n\n const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1);\n const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1);\n const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1);\n const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1);\n\n //2-bit -> 8-bit\n const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0,m3bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7)\n const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0,m3bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7)\n const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1,m3bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15)\n const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1,m3bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15)\n const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(rhs_raw_mat_014589CD_2,m3bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7)\n const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2,m3bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7)\n const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(rhs_raw_mat_014589CD_3,m3bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15)\n const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3,m3bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15)\n\n const __m512i rhs_mat_014589CD_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 2), m3bexpanded); //B20(0-7) B21(0-7) B24(0-7) B25(0-7) B28(0-7) B29(0-7) B2C(0-7) B2D(0-7)\n const __m512i rhs_mat_2367ABEF_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 2), m3bexpanded); //B22(0-7) B23(0-7) B26(0-7) B27(0-7) B2A(0-7) B2B(0-7) B2E(0-7) B2F(0-7)\n\n const __m512i rhs_mat_014589CD_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 2), m3bexpanded); //B20(8-15) B21(8-15) B24(8-15) B25(8-15) B28(8-15) B29(8-15) B2C(8-15) B2D(8-15)\n const __m512i rhs_mat_2367ABEF_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 2), m3bexpanded); //B22(8-15) B23(8-15) B26(8-15) B27(8-15) B2A(8-15) B2B(8-15) B2E(8-15) B2F(8-15)\n\n const __m512i rhs_mat_014589CD_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 2), m3bexpanded); //B30(0-7) B31(0-7) B34(0-7) B35(0-7) B38(0-7) B39(0-7) B3C(0-7) B3D(0-7)\n const __m512i rhs_mat_2367ABEF_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 2), m3bexpanded); //B32(0-7) B33(0-7) B36(0-7) B37(0-7) B3A(0-7) B3B(0-7) B3E(0-7) B3F(0-7)\n\n const __m512i rhs_mat_014589CD_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 2), m3bexpanded); //B30(8-15) B31(8-15) B34(8-15) B35(8-15) B38(8-15) B39(8-15) B3C(8-15) B3D(8-15)\n const __m512i rhs_mat_2367ABEF_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 2), m3bexpanded); //B32(8-15) B33(8-15) B36(8-15) B37(8-15) B3A(8-15) B3B(8-15) B3E(8-15) B3F(8-15)\n\n const __m512i rhs_mat_014589CD_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m3bexpanded); //B40(0-7) B41(0-7) B44(0-7) B45(0-7) B48(0-7) B49(0-7) B4C(0-7) B4D(0-7)\n const __m512i rhs_mat_2367ABEF_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m3bexpanded); //B42(0-7) B43(0-7) B46(0-7) B47(0-7) B4A(0-7) B4B(0-7) B4E(0-7) B4F(0-7)\n\n const __m512i rhs_mat_014589CD_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m3bexpanded); //B40(8-15) B41(8-15) B44(8-15) B45(8-15) B48(8-15) B49(8-15) B4C(8-15) B4D(8-15)\n const __m512i rhs_mat_2367ABEF_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m3bexpanded); //B42(8-15) B43(8-15) B46(8-15) B47(8-15) B4A(8-15) B4B(8-15) B4E(8-15) B4F(8-15)\n\n const __m512i rhs_mat_014589CD_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m3bexpanded); //B50(0-7) B51(0-7) B54(0-7) B55(0-7) B58(0-7) B59(0-7) B5C(0-7) B5D(0-7)\n const __m512i rhs_mat_2367ABEF_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m3bexpanded); //B52(0-7) B53(0-7) B56(0-7) B57(0-7) B5A(0-7) B5B(0-7) B5E(0-7) B5F(0-7)\n\n const __m512i rhs_mat_014589CD_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m3bexpanded); //B50(8-15) B51(8-15) B54(8-15) B55(8-15) B58(8-15) B59(8-15) B5C(8-15) B5D(8-15)\n const __m512i rhs_mat_2367ABEF_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m3bexpanded); //B52(8-15) B53(8-15) B56(8-15) B57(8-15) B5A(8-15) B5B(8-15) B5E(8-15) B5F(8-15)\n\n const __m512i rhs_mat_014589CD_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 6), m3bexpanded); //B60(0-7) B61(0-7) B64(0-7) B65(0-7) B68(0-7) B69(0-7) B6C(0-7) B6D(0-7)\n const __m512i rhs_mat_2367ABEF_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 6), m3bexpanded); //B62(0-7) B63(0-7) B66(0-7) B67(0-7) B6A(0-7) B6B(0-7) B6E(0-7) B6F(0-7)\n\n const __m512i rhs_mat_014589CD_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 6), m3bexpanded); //B60(8-15) B61(8-15) B64(8-15) B65(8-15) B68(8-15) B69(8-15) B6C(8-15) B6D(8-15)\n const __m512i rhs_mat_2367ABEF_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 6), m3bexpanded); //B62(8-15) B63(8-15) B66(8-15) B67(8-15) B6A(8-15) B6B(8-15) B6E(8-15) B6F(8-15)\n\n const __m512i rhs_mat_014589CD_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 6), m3bexpanded); //B70(0-7) B71(0-7) B74(0-7) B75(0-7) B78(0-7) B79(0-7) B7C(0-7) B7D(0-7)\n const __m512i rhs_mat_2367ABEF_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 6), m3bexpanded); //B72(0-7) B73(0-7) B76(0-7) B77(0-7) B7A(0-7) B7B(0-7) B7E(0-7) B7F(0-7)\n\n const __m512i rhs_mat_014589CD_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 6), m3bexpanded); //B70(8-15) B71(8-15) B74(8-15) B75(8-15) B78(8-15) B79(8-15) B7C(8-15) B7D(8-15)\n const __m512i rhs_mat_2367ABEF_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 6), m3bexpanded); //B72(8-15) B73(8-15) B76(8-15) B77(8-15) B7A(8-15) B7B(8-15) B7E(8-15) B7F(8-15)\n\n const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3)\n const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3)\n\n const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11)\n\n const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3)\n const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3)\n\n const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11)\n const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11)\n\n const __m512i rhs_mat_014589CD_20_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)136); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3) B28(0-3) B29(0-3) B28(0-3) B29(0-3) B2C(0-3) B2D(0-3) B2C(0-3) B2D(0-3)\n const __m512i rhs_mat_2367ABEF_20_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)136); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3) B2A(0-3) B2B(0-3) B2A(0-3) B2B(0-3) B2E(0-3) B2F(0-3) B2E(0-3) B2F(0-3)\n\n const __m512i rhs_mat_014589CD_21_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)136); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11) B28(8-11) B29(8-11) B28(8-11) B29(8-11) B2C(8-11) B2D(8-11) B2C(8-11) B2D(8-11)\n const __m512i rhs_mat_2367ABEF_21_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)136); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11) B2A(8-11) B2B(8-11) B2A(8-11) B2B(8-11) B2E(8-11) B2F(8-11) B2E(8-11) B2F(8-11)\n const __m512i rhs_mat_014589CD_30_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)136); ///B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3) B38(0-3) B39(0-3) B38(0-3) B39(0-3) B3C(0-3) B3D(0-3) B3C(0-3) B3D(0-3)\n const __m512i rhs_mat_2367ABEF_30_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)136); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3) B3A(0-3) B3B(0-3) B3A(0-3) B3B(0-3) B3E(0-3) B3F(0-3) B3E(0-3) B3F(0-3)\n\n const __m512i rhs_mat_014589CD_31_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)136); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11) B38(8-11) B39(8-11) B38(8-11) B39(8-11) B3C(8-11) B3D(8-11) B3C(8-11) B3D(8-11)\n const __m512i rhs_mat_2367ABEF_31_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)136); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11) B3A(8-11) B3B(8-11) B3A(8-11) B3B(8-11) B3E(8-11) B3F(8-11) B3E(8-11) B3F(8-11)\n\n const __m512i rhs_mat_014589CD_40_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)136); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3) B48(0-3) B49(0-3) B48(0-3) B49(0-3) B4C(0-3) B4D(0-3) B4C(0-3) B4D(0-3)\n const __m512i rhs_mat_2367ABEF_40_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)136); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3) B4A(0-3) B4B(0-3) B4A(0-3) B4B(0-3) B4E(0-3) B4F(0-3) B4E(0-3) B4F(0-3)\n\n const __m512i rhs_mat_014589CD_41_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)136); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11) B48(8-11) B49(8-11) B48(8-11) B49(8-11) B4C(8-11) B4D(8-11) B4C(8-11) B4D(8-11)\n const __m512i rhs_mat_2367ABEF_41_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)136); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11) B4A(8-11) B4B(8-11) B4A(8-11) B4B(8-11) B4E(8-11) B4F(8-11) B4E(8-11) B4F(8-11)\n\n const __m512i rhs_mat_014589CD_50_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)136); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3) B58(0-3) B59(0-3) B58(0-3) B59(0-3) B5C(0-3) B5D(0-3) B5C(0-3) B5D(0-3)\n const __m512i rhs_mat_2367ABEF_50_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)136); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3) B5A(0-3) B5B(0-3) B5A(0-3) B5B(0-3) B5E(0-3) B5F(0-3) B5E(0-3) B5F(0-3)\n\n const __m512i rhs_mat_014589CD_51_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)136); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11) B58(8-11) B59(8-11) B58(8-11) B59(8-11) B5C(8-11) B5D(8-11) B5C(8-11) B5D(8-11)\n const __m512i rhs_mat_2367ABEF_51_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)136); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11) B5A(8-11) B5B(8-11) B5A(8-11) B5B(8-11) B5E(8-11) B5F(8-11) B5E(8-11) B5F(8-11)\n\n const __m512i rhs_mat_014589CD_60_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)136); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3) B68(0-3) B69(0-3) B68(0-3) B69(0-3) B6C(0-3) B6D(0-3) B6C(0-3) B6D(0-3)\n const __m512i rhs_mat_2367ABEF_60_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)136); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3) B6A(0-3) B6B(0-3) B6A(0-3) B6B(0-3) B6E(0-3) B6F(0-3) B6E(0-3) B6F(0-3)\n\n const __m512i rhs_mat_014589CD_61_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)136); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11) B68(8-11) B69(8-11) B68(8-11) B69(8-11) B6C(8-11) B6D(8-11) B6C(8-11) B6D(8-11)\n const __m512i rhs_mat_2367ABEF_61_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)136); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11) B6A(8-11) B6B(8-11) B6A(8-11) B6B(8-11) B6E(8-11) B6F(8-11) B6E(8-11) B6F(8-11)\n\n const __m512i rhs_mat_014589CD_70_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)136); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3) B78(0-3) B79(0-3) B78(0-3) B79(0-3) B7C(0-3) B7D(0-3) B7C(0-3) B7D(0-3)\n const __m512i rhs_mat_2367ABEF_70_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)136); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3) B7A(0-3) B7B(0-3) B7A(0-3) B7B(0-3) B7E(0-3) B7F(0-3) B7E(0-3) B7F(0-3)\n\n const __m512i rhs_mat_014589CD_71_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n const __m512i rhs_mat_2367ABEF_71_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)136); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11) B7A(8-11) B7B(8-11) B7A(8-11) B7B(8-11) B7E(8-11) B7F(8-11) B7E(8-11) B7F(8-11)\n\n const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7)\n const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7)\n\n const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15)\n const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15)\n\n const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7)\n const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7)\n\n const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15)\n const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15)\n\n const __m512i rhs_mat_014589CD_20_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)221); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7) B28(4-7) B29(4-7) B28(4-7) B29(4-7) B2C(4-7) B2D(4-7) B2C(4-7) B2D(4-7)\n const __m512i rhs_mat_2367ABEF_20_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)221); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7) B2A(4-7) B2B(4-7) B2A(4-7) B2B(4-7) B2E(4-7) B2F(4-7) B2E(4-7) B2F(4-7)\n\n const __m512i rhs_mat_014589CD_21_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)221); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15) B28(12-15) B29(12-15) B28(12-15) B29(12-15) B2C(12-15) B2D(12-15) B2C(12-15) B2D(12-15)\n const __m512i rhs_mat_2367ABEF_21_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)221); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15) B2A(12-15) B2B(12-15) B2A(12-15) B2B(12-15) B2E(12-15) B2F(12-15) B2E(12-15) B2F(12-15)\n\n const __m512i rhs_mat_014589CD_30_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)221); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7) B38(4-7) B39(4-7) B38(4-7) B39(4-7) B3C(4-7) B3D(4-7) B3C(4-7) B3D(4-7)\n const __m512i rhs_mat_2367ABEF_30_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)221); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7) B3A(4-7) B3B(4-7) B3A(4-7) B3B(4-7) B3E(4-7) B3F(4-7) B3E(4-7) B3F(4-7)\n\n const __m512i rhs_mat_014589CD_31_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)221); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15) B38(12-15) B39(12-15) B38(12-15) B39(12-15) B3C(12-15) B3D(12-15) B3C(12-15) B3D(12-15)\n const __m512i rhs_mat_2367ABEF_31_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)221); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15) B3A(12-15) B3B(12-15) B3A(12-15) B3B(12-15) B3E(12-15) B3F(12-15) B3E(12-15) B3F(12-15)\n\n const __m512i rhs_mat_014589CD_40_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)221); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7) B48(4-7) B49(4-7) B48(4-7) B49(4-7) B4C(4-7) B4D(4-7) B4C(4-7) B4D(4-7)\n const __m512i rhs_mat_2367ABEF_40_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)221); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7) B4A(4-7) B4B(4-7) B4A(4-7) B4B(4-7) B4E(4-7) B4F(4-7) B4E(4-7) B4F(4-7)\n\n const __m512i rhs_mat_014589CD_41_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)221); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15) B48(12-15) B49(12-15) B48(12-15) B49(12-15) B4C(12-15) B4D(12-15) B4C(12-15) B4D(12-15)\n const __m512i rhs_mat_2367ABEF_41_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)221); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15) B4A(12-15) B4B(12-15) B4A(12-15) B4B(12-15) B4E(12-15) B4F(12-15) B4E(12-15) B4F(12-15)\n\n const __m512i rhs_mat_014589CD_50_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)221); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7) B58(4-7) B59(4-7) B58(4-7) B59(4-7) B5C(4-7) B5D(4-7) B5C(4-7) B5D(4-7)\n const __m512i rhs_mat_2367ABEF_50_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)221); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7) B5A(4-7) B5B(4-7) B5A(4-7) B5B(4-7) B5E(4-7) B5F(4-7) B5E(4-7) B5F(4-7)\n\n const __m512i rhs_mat_014589CD_51_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)221); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15) B58(12-15) B59(12-15) B58(12-15) B59(12-15) B5C(12-15) B5D(12-15) B5C(12-15) B5D(12-15)\n const __m512i rhs_mat_2367ABEF_51_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)221); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15) B5A(12-15) B5B(12-15) B5A(12-15) B5B(12-15) B5E(12-15) B5F(12-15) B5E(12-15) B5F(12-15)\n\n const __m512i rhs_mat_014589CD_60_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)221); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7) B68(4-7) B69(4-7) B68(4-7) B69(4-7) B6C(4-7) B6D(4-7) B6C(4-7) B6D(4-7)\n const __m512i rhs_mat_2367ABEF_60_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)221); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7) B6A(4-7) B6B(4-7) B6A(4-7) B6B(4-7) B6E(4-7) B6F(4-7) B6E(4-7) B6F(4-7)\n\n const __m512i rhs_mat_014589CD_61_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)221); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15) B68(12-15) B69(12-15) B68(12-15) B69(12-15) B6C(12-15) B6D(12-15) B6C(12-15) B6D(12-15)\n const __m512i rhs_mat_2367ABEF_61_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)221); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15) B6A(12-15) B6B(12-15) B6A(12-15) B6B(12-15) B6E(12-15) B6F(12-15) B6E(12-15) B6F(12-15)\n\n const __m512i rhs_mat_014589CD_70_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)221); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7) B78(4-7) B79(4-7) B78(4-7) B79(4-7) B7C(4-7) B7D(4-7) B7C(4-7) B7D(4-7)\n const __m512i rhs_mat_2367ABEF_70_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)221); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7) B7A(4-7) B7B(4-7) B7A(4-7) B7B(4-7) B7E(4-7) B7F(4-7) B7E(4-7) B7F(4-7)\n\n const __m512i rhs_mat_014589CD_71_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)221); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15) B78(12-15) B79(12-15) B78(12-15) B79(12-15) B7C(12-15) B7D(12-15) B7C(12-15) B7D(12-15)\n const __m512i rhs_mat_2367ABEF_71_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)221); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15) B7A(12-15) B7B(12-15) B7A(12-15) B7B(12-15) B7E(12-15) B7F(12-15) B7E(12-15) B7F(12-15)\n\n //notation:superblock subblock\n //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71\n\n const __m128i mins_and_scales_01_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + sb * 64));\n const __m128i mins_and_scales_23_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 16 + sb * 64));\n const __m128i mins_and_scales_45_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 32 + sb * 64));\n const __m128i mins_and_scales_67_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 48 + sb * 64));\n\n const __m128i mins_and_scales_01_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + sb * 64));\n const __m128i mins_and_scales_23_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 16 + sb * 64));\n const __m128i mins_and_scales_45_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 32 + sb * 64));\n const __m128i mins_and_scales_67_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 48 + sb * 64));\n\n // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop\n const __m256i mins_and_scales_01 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_01_0), mins_and_scales_01_1, 1);\n const __m256i mins_and_scales_23 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_23_0), mins_and_scales_23_1, 1);\n const __m256i mins_and_scales_45 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_45_0), mins_and_scales_45_1, 1);\n const __m256i mins_and_scales_67 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_67_0), mins_and_scales_67_1, 1);\n\n // Extract scales which is lower half from mins_and_scales\n const __m256i scales_01 = _mm256_and_si256(mins_and_scales_01, m4b);\n const __m256i scales_23 = _mm256_and_si256(mins_and_scales_23, m4b);\n const __m256i scales_45 = _mm256_and_si256(mins_and_scales_45, m4b);\n const __m256i scales_67 = _mm256_and_si256(mins_and_scales_67, m4b);\n\n // Extract mins which is upper half from mins_and_scales\n const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_01, 4), m4b));\n const __m512i mins_23 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_23, 4), m4b));\n const __m512i mins_45 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_45, 4), m4b));\n const __m512i mins_67 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_67, 4), m4b));\n\n const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01, scalesmask1));\n const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01, scalesmask2));\n const __m512i scales_2 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23, scalesmask1));\n const __m512i scales_3 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23, scalesmask2));\n const __m512i scales_4 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45, scalesmask1));\n const __m512i scales_5 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45, scalesmask2));\n const __m512i scales_6 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67, scalesmask1));\n const __m512i scales_7 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67, scalesmask2));\n\n const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)238);\n\n const __m512i scale_014589CD_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)68);\n const __m512i scale_2367ABEF_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)238);\n\n // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 512 * sb)));\n __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0);\n __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17);\n __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 512 * sb)));\n __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0);\n __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17);\n __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 512 * sb)));\n __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0);\n __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17);\n __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 512 * sb)));\n __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0);\n __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17);\n __m256i lhs_mat_ymm_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 512 * sb)));\n __m256i lhs_mat_ymm_01_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 0);\n __m256i lhs_mat_ymm_23_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 17);\n __m256i lhs_mat_ymm_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 512 * sb)));\n __m256i lhs_mat_ymm_01_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 0);\n __m256i lhs_mat_ymm_23_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 17);\n __m256i lhs_mat_ymm_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 512 * sb)));\n __m256i lhs_mat_ymm_01_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 0);\n __m256i lhs_mat_ymm_23_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 17);\n __m256i lhs_mat_ymm_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 512 * sb)));\n __m256i lhs_mat_ymm_01_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 0);\n __m256i lhs_mat_ymm_23_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 17);\n\n __m256i lhs_mat_ymm_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 + 512 * sb)));\n __m256i lhs_mat_ymm_01_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 0);\n __m256i lhs_mat_ymm_23_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 17);\n __m256i lhs_mat_ymm_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 288 + 512 * sb)));\n __m256i lhs_mat_ymm_01_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 0);\n __m256i lhs_mat_ymm_23_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 17);\n __m256i lhs_mat_ymm_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 320 + 512 * sb)));\n __m256i lhs_mat_ymm_01_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 0);\n __m256i lhs_mat_ymm_23_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 17);\n __m256i lhs_mat_ymm_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 352 + 512 * sb)));\n __m256i lhs_mat_ymm_01_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 0);\n __m256i lhs_mat_ymm_23_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 17);\n __m256i lhs_mat_ymm_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 384 + 512 * sb)));\n __m256i lhs_mat_ymm_01_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 0);\n __m256i lhs_mat_ymm_23_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 17);\n __m256i lhs_mat_ymm_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 416 + 512 * sb)));\n __m256i lhs_mat_ymm_01_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 0);\n __m256i lhs_mat_ymm_23_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 17);\n __m256i lhs_mat_ymm_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 448 + 512 * sb)));\n __m256i lhs_mat_ymm_01_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 0);\n __m256i lhs_mat_ymm_23_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 17);\n __m256i lhs_mat_ymm_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 480 + 512 * sb)));\n __m256i lhs_mat_ymm_01_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 0);\n __m256i lhs_mat_ymm_23_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 17);\n\n __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1);\n __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1);\n __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1);\n __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1);\n\n __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1);\n __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1);\n __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1);\n __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1);\n\n __m512i lhs_mat_01_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_20), lhs_mat_ymm_01_20, 1);\n __m512i lhs_mat_23_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_20), lhs_mat_ymm_23_20, 1);\n __m512i lhs_mat_01_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_21), lhs_mat_ymm_01_21, 1);\n __m512i lhs_mat_23_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_21), lhs_mat_ymm_23_21, 1);\n\n __m512i lhs_mat_01_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_30), lhs_mat_ymm_01_30, 1);\n __m512i lhs_mat_23_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_30), lhs_mat_ymm_23_30, 1);\n __m512i lhs_mat_01_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_31), lhs_mat_ymm_01_31, 1);\n __m512i lhs_mat_23_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_31), lhs_mat_ymm_23_31, 1);\n\n __m512i lhs_mat_01_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_40), lhs_mat_ymm_01_40, 1);\n __m512i lhs_mat_23_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_40), lhs_mat_ymm_23_40, 1);\n __m512i lhs_mat_01_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_41), lhs_mat_ymm_01_41, 1);\n __m512i lhs_mat_23_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_41), lhs_mat_ymm_23_41, 1);\n\n __m512i lhs_mat_01_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_50), lhs_mat_ymm_01_50, 1);\n __m512i lhs_mat_23_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_50), lhs_mat_ymm_23_50, 1);\n __m512i lhs_mat_01_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_51), lhs_mat_ymm_01_51, 1);\n __m512i lhs_mat_23_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_51), lhs_mat_ymm_23_51, 1);\n\n __m512i lhs_mat_01_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_60), lhs_mat_ymm_01_60, 1);\n __m512i lhs_mat_23_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_60), lhs_mat_ymm_23_60, 1);\n __m512i lhs_mat_01_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_61), lhs_mat_ymm_01_61, 1);\n __m512i lhs_mat_23_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_61), lhs_mat_ymm_23_61, 1);\n\n __m512i lhs_mat_01_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_70), lhs_mat_ymm_01_70, 1);\n __m512i lhs_mat_23_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_70), lhs_mat_ymm_23_70, 1);\n __m512i lhs_mat_01_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_71), lhs_mat_ymm_01_71, 1);\n __m512i lhs_mat_23_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_71), lhs_mat_ymm_23_71, 1);\n\n // Bsums are loaded for the different Q8_K blocks\n __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 32 * sb)));\n __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 8 + 32 * sb));\n __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 16 + 32 * sb)));\n __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 24 + 32 * sb));\n\n __m256i lhs_bsums_ymm_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1);\n __m512i lhs_bsums_01_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_0123), lhs_bsums_ymm_01_0123, 1);\n __m256i lhs_bsums_ymm_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1);\n __m512i lhs_bsums_23_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_0123), lhs_bsums_ymm_23_0123, 1);\n __m256i lhs_bsums_ymm_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1);\n __m512i lhs_bsums_01_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_4567), lhs_bsums_ymm_01_4567, 1);\n __m256i lhs_bsums_ymm_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1);\n __m512i lhs_bsums_23_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_4567), lhs_bsums_ymm_23_4567, 1);\n\n // Shuffle pattern one - left side input\n const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3)\n\n const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11)\n\n const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3)\n\n const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11)\n\n const __m512i lhs_mat_01_20_sp1 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)160); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3)\n const __m512i lhs_mat_23_20_sp1 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)160); //A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3)\n\n const __m512i lhs_mat_01_21_sp1 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)160); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11)\n const __m512i lhs_mat_23_21_sp1 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)160); //A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11)\n\n const __m512i lhs_mat_01_30_sp1 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)160); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3)\n const __m512i lhs_mat_23_30_sp1 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)160); //A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3)\n\n const __m512i lhs_mat_01_31_sp1 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)160); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11)\n const __m512i lhs_mat_23_31_sp1 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)160); //A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11)\n\n const __m512i lhs_mat_01_40_sp1 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)160); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3)\n const __m512i lhs_mat_23_40_sp1 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)160); //A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3)\n\n const __m512i lhs_mat_01_41_sp1 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)160); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11)\n const __m512i lhs_mat_23_41_sp1 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)160); //A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11)\n\n const __m512i lhs_mat_01_50_sp1 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)160); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3)\n const __m512i lhs_mat_23_50_sp1 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)160); //A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3)\n\n const __m512i lhs_mat_01_51_sp1 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)160); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11)\n const __m512i lhs_mat_23_51_sp1 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)160); //A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11)\n\n const __m512i lhs_mat_01_60_sp1 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)160); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3)\n const __m512i lhs_mat_23_60_sp1 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)160); //A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3)\n\n const __m512i lhs_mat_01_61_sp1 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)160); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11)\n const __m512i lhs_mat_23_61_sp1 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)160); //A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11)\n\n const __m512i lhs_mat_01_70_sp1 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)160); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3)\n const __m512i lhs_mat_23_70_sp1 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)160); //A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3)\n\n const __m512i lhs_mat_01_71_sp1 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)160); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11)\n const __m512i lhs_mat_23_71_sp1 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)160); //A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11)\n\n const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7)\n\n const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15)\n\n const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7)\n\n const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15)\n\n const __m512i lhs_mat_01_20_sp2 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)245); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7)\n const __m512i lhs_mat_23_20_sp2 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)245); //A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7)\n\n const __m512i lhs_mat_01_21_sp2 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)245); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15)\n const __m512i lhs_mat_23_21_sp2 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)245); //A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15)\n\n const __m512i lhs_mat_01_30_sp2 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)245); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7)\n const __m512i lhs_mat_23_30_sp2 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)245); //A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7)\n\n const __m512i lhs_mat_01_31_sp2 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)245); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15)\n const __m512i lhs_mat_23_31_sp2 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)245); //A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15)\n\n const __m512i lhs_mat_01_40_sp2 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)245); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7)\n const __m512i lhs_mat_23_40_sp2 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)245); //A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7)\n\n const __m512i lhs_mat_01_41_sp2 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)245); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15)\n const __m512i lhs_mat_23_41_sp2 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)245); //A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15)\n\n const __m512i lhs_mat_01_50_sp2 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)245); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7)\n const __m512i lhs_mat_23_50_sp2 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)245); //A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7)\n\n const __m512i lhs_mat_01_51_sp2 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)245); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15)\n const __m512i lhs_mat_23_51_sp2 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)245); //A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15)\n\n const __m512i lhs_mat_01_60_sp2 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)245); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7)\n const __m512i lhs_mat_23_60_sp2 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)245); //A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7)\n\n const __m512i lhs_mat_01_61_sp2 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)245); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15)\n const __m512i lhs_mat_23_61_sp2 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)245); //A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15)\n\n const __m512i lhs_mat_01_70_sp2 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)245); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7)\n const __m512i lhs_mat_23_70_sp2 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)245); //A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7)\n\n const __m512i lhs_mat_01_71_sp2 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)245); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15)\n const __m512i lhs_mat_23_71_sp2 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)245); //A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1));\n __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1));\n\n __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1));\n __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1));\n\n __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1));\n __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1));\n\n __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1));\n __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1));\n\n __m512i iacc_mat_00_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_01_21_sp1));\n __m512i iacc_mat_01_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_01_21_sp1));\n\n __m512i iacc_mat_10_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_23_21_sp1));\n __m512i iacc_mat_11_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_23_21_sp1));\n\n __m512i iacc_mat_00_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_01_31_sp1));\n __m512i iacc_mat_01_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_01_31_sp1));\n\n __m512i iacc_mat_10_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_23_31_sp1));\n __m512i iacc_mat_11_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_23_31_sp1));\n\n __m512i iacc_mat_00_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_01_41_sp1));\n __m512i iacc_mat_01_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_01_41_sp1));\n\n __m512i iacc_mat_10_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_23_41_sp1));\n __m512i iacc_mat_11_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_23_41_sp1));\n\n __m512i iacc_mat_00_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_01_51_sp1));\n __m512i iacc_mat_01_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_01_51_sp1));\n\n __m512i iacc_mat_10_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_23_51_sp1));\n __m512i iacc_mat_11_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_23_51_sp1));\n\n __m512i iacc_mat_00_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_01_61_sp1));\n __m512i iacc_mat_01_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_01_61_sp1));\n\n __m512i iacc_mat_10_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_23_61_sp1));\n __m512i iacc_mat_11_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_23_61_sp1));\n\n __m512i iacc_mat_00_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_01_71_sp1));\n __m512i iacc_mat_01_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_01_71_sp1));\n\n __m512i iacc_mat_10_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_23_71_sp1));\n __m512i iacc_mat_11_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_23_71_sp1));\n\n\n __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2));\n __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2));\n\n __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2));\n __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2));\n\n __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2));\n __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2));\n\n __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2));\n __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2));\n\n __m512i iacc_mat_00_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_01_21_sp2));\n __m512i iacc_mat_01_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_01_21_sp2));\n\n __m512i iacc_mat_10_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_23_21_sp2));\n __m512i iacc_mat_11_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_23_21_sp2));\n\n __m512i iacc_mat_00_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_01_31_sp2));\n __m512i iacc_mat_01_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_01_31_sp2));\n\n __m512i iacc_mat_10_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_23_31_sp2));\n __m512i iacc_mat_11_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_23_31_sp2));\n\n __m512i iacc_mat_00_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_01_41_sp2));\n __m512i iacc_mat_01_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_01_41_sp2));\n\n __m512i iacc_mat_10_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_23_41_sp2));\n __m512i iacc_mat_11_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_23_41_sp2));\n\n __m512i iacc_mat_00_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_01_51_sp2));\n __m512i iacc_mat_01_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_01_51_sp2));\n\n __m512i iacc_mat_10_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_23_51_sp2));\n __m512i iacc_mat_11_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_23_51_sp2));\n\n __m512i iacc_mat_00_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_01_61_sp2));\n __m512i iacc_mat_01_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_01_61_sp2));\n\n __m512i iacc_mat_10_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_23_61_sp2));\n __m512i iacc_mat_11_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_23_61_sp2));\n\n __m512i iacc_mat_00_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_01_71_sp2));\n __m512i iacc_mat_01_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_01_71_sp2));\n\n __m512i iacc_mat_10_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_23_71_sp2));\n __m512i iacc_mat_11_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_23_71_sp2));\n\n // Combine results from both shuffle patterns for each output block\n __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n __m512i iacc_mat_00_2 = _mm512_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2);\n __m512i iacc_mat_01_2 = _mm512_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2);\n __m512i iacc_mat_10_2 = _mm512_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2);\n __m512i iacc_mat_11_2 = _mm512_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2);\n\n __m512i iacc_mat_00_3 = _mm512_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2);\n __m512i iacc_mat_01_3 = _mm512_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2);\n __m512i iacc_mat_10_3 = _mm512_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2);\n __m512i iacc_mat_11_3 = _mm512_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2);\n\n __m512i iacc_mat_00_4 = _mm512_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2);\n __m512i iacc_mat_01_4 = _mm512_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2);\n __m512i iacc_mat_10_4 = _mm512_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2);\n __m512i iacc_mat_11_4 = _mm512_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2);\n\n __m512i iacc_mat_00_5 = _mm512_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2);\n __m512i iacc_mat_01_5 = _mm512_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2);\n __m512i iacc_mat_10_5 = _mm512_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2);\n __m512i iacc_mat_11_5 = _mm512_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2);\n\n __m512i iacc_mat_00_6 = _mm512_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2);\n __m512i iacc_mat_01_6 = _mm512_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2);\n __m512i iacc_mat_10_6 = _mm512_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2);\n __m512i iacc_mat_11_6 = _mm512_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2);\n\n __m512i iacc_mat_00_7 = _mm512_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2);\n __m512i iacc_mat_01_7 = _mm512_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2);\n __m512i iacc_mat_10_7 = _mm512_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2);\n __m512i iacc_mat_11_7 = _mm512_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0);\n iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0);\n iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0);\n iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0);\n\n iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1);\n iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1);\n iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1);\n iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1);\n\n iacc_mat_00_2 = _mm512_madd_epi16(iacc_mat_00_2, scale_014589CD_2);\n iacc_mat_01_2 = _mm512_madd_epi16(iacc_mat_01_2, scale_2367ABEF_2);\n iacc_mat_10_2 = _mm512_madd_epi16(iacc_mat_10_2, scale_014589CD_2);\n iacc_mat_11_2 = _mm512_madd_epi16(iacc_mat_11_2, scale_2367ABEF_2);\n\n iacc_mat_00_3 = _mm512_madd_epi16(iacc_mat_00_3, scale_014589CD_3);\n iacc_mat_01_3 = _mm512_madd_epi16(iacc_mat_01_3, scale_2367ABEF_3);\n iacc_mat_10_3 = _mm512_madd_epi16(iacc_mat_10_3, scale_014589CD_3);\n iacc_mat_11_3 = _mm512_madd_epi16(iacc_mat_11_3, scale_2367ABEF_3);\n\n iacc_mat_00_4 = _mm512_madd_epi16(iacc_mat_00_4, scale_014589CD_4);\n iacc_mat_01_4 = _mm512_madd_epi16(iacc_mat_01_4, scale_2367ABEF_4);\n iacc_mat_10_4 = _mm512_madd_epi16(iacc_mat_10_4, scale_014589CD_4);\n iacc_mat_11_4 = _mm512_madd_epi16(iacc_mat_11_4, scale_2367ABEF_4);\n\n iacc_mat_00_5 = _mm512_madd_epi16(iacc_mat_00_5, scale_014589CD_5);\n iacc_mat_01_5 = _mm512_madd_epi16(iacc_mat_01_5, scale_2367ABEF_5);\n iacc_mat_10_5 = _mm512_madd_epi16(iacc_mat_10_5, scale_014589CD_5);\n iacc_mat_11_5 = _mm512_madd_epi16(iacc_mat_11_5, scale_2367ABEF_5);\n\n iacc_mat_00_6 = _mm512_madd_epi16(iacc_mat_00_6, scale_014589CD_6);\n iacc_mat_01_6 = _mm512_madd_epi16(iacc_mat_01_6, scale_2367ABEF_6);\n iacc_mat_10_6 = _mm512_madd_epi16(iacc_mat_10_6, scale_014589CD_6);\n iacc_mat_11_6 = _mm512_madd_epi16(iacc_mat_11_6, scale_2367ABEF_6);\n\n iacc_mat_00_7 = _mm512_madd_epi16(iacc_mat_00_7, scale_014589CD_7);\n iacc_mat_01_7 = _mm512_madd_epi16(iacc_mat_01_7, scale_2367ABEF_7);\n iacc_mat_10_7 = _mm512_madd_epi16(iacc_mat_10_7, scale_014589CD_7);\n iacc_mat_11_7 = _mm512_madd_epi16(iacc_mat_11_7, scale_2367ABEF_7);\n\n __m512i iacc_mat_00 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm512_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm512_add_epi32(iacc_mat_00_6, iacc_mat_00_7)));\n __m512i iacc_mat_01 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm512_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm512_add_epi32(iacc_mat_01_6, iacc_mat_01_7)));\n __m512i iacc_mat_10 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm512_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm512_add_epi32(iacc_mat_10_6, iacc_mat_10_7)));\n __m512i iacc_mat_11 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm512_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm512_add_epi32(iacc_mat_11_6, iacc_mat_11_7)));\n\n // Straighten out to make 4 row vectors\n __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78));\n __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78), iacc_mat_01);\n __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78));\n __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78), iacc_mat_11);\n\n // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d);\n const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1);\n\n // Multiply with appropiate scales and accumulate (for both d and dmin) below\n acc_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n acc_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n acc_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n acc_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n\n // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K\n __m512i iacc_row_min_0_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)0), mins_01);\n __m512i iacc_row_min_1_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)170), mins_01);\n __m512i iacc_row_min_2_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)0), mins_01);\n __m512i iacc_row_min_3_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)170), mins_01);\n\n __m512i iacc_row_min_0_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)85), mins_23);\n __m512i iacc_row_min_1_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)255), mins_23);\n __m512i iacc_row_min_2_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)85), mins_23);\n __m512i iacc_row_min_3_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)255), mins_23);\n\n __m512i iacc_row_min_0_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)0), mins_45);\n __m512i iacc_row_min_1_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)170), mins_45);\n __m512i iacc_row_min_2_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)0), mins_45);\n __m512i iacc_row_min_3_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)170), mins_45);\n\n __m512i iacc_row_min_0_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)85), mins_67);\n __m512i iacc_row_min_1_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)255), mins_67);\n __m512i iacc_row_min_2_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)85), mins_67);\n __m512i iacc_row_min_3_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)255), mins_67);\n\n __m512i iacc_row_min_0 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm512_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67));\n __m512i iacc_row_min_1 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm512_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67));\n __m512i iacc_row_min_2 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm512_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67));\n __m512i iacc_row_min_3 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm512_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67));\n\n acc_min_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]);\n acc_min_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]);\n acc_min_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]);\n acc_min_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]);\n }\n }\n // Store accumulated values\n for (int i = 0; i < 4; i++) {\n _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i]));\n }\n }\n }\n\n if (anc != nc) {\n xstart = anc/8;\n y = 0;\n }\n\n#endif // __AVX512BW__ && __AVX512DQ__\n\n // Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation\n for (; y < anr / 4; y += 4) {\n\n const block_q8_Kx4 * a_ptrs[4];\n\n a_ptrs[0] = a_ptr_start + (y * nb);\n for (int i = 0; i < 3; ++i) {\n a_ptrs[i + 1] = a_ptrs[i] + nb;\n }\n\n // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = xstart; x < nc / 8; x++) {\n\n const block_q2_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n // Master FP accumulators\n __m256 acc_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_rows[i] = _mm256_setzero_ps();\n }\n\n __m256 acc_min_rows[16];\n for (int i = 0; i < 16; i++) {\n acc_min_rows[i] = _mm256_setzero_ps();\n }\n\n // For super block\n for (int64_t b = 0; b < nb; b++) {\n // Delta values - Load the eight scale values of block_q2_kx8\n const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n\n // dmin values - Load the eight dmin values of block_q2_kx8\n const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration\n for (int sb = 0; sb < QK_K / 128; sb++) {\n\n // Load the eight block_q2_K for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + sb * 256));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 224 + sb * 256));\n\n // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n //superblock sub block which part of sub block\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n\n const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n // 2-bit -> 8-bit\n // First sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m3b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7)\n const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m3b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7)\n\n const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m3b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15)\n const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m3b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15)\n\n // Second sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_10 = _mm256_and_si256(rhs_raw_mat_0145_2, m3b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7)\n const __m256i rhs_mat_2367_10 = _mm256_and_si256(rhs_raw_mat_2367_2, m3b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7)\n\n const __m256i rhs_mat_0145_11 = _mm256_and_si256(rhs_raw_mat_0145_3, m3b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15)\n const __m256i rhs_mat_2367_11 = _mm256_and_si256(rhs_raw_mat_2367_3, m3b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15)\n\n // Third sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 2), m3b); //B20(0-7) B21(0-7) B24(0-7) B25(0-7)\n const __m256i rhs_mat_2367_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 2), m3b); //B22(0-7) B23(0-7) B26(0-7) B27(0-7)\n\n const __m256i rhs_mat_0145_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 2), m3b); //B20(8-15) B21(8-15) B24(8-15) B25(8-15)\n const __m256i rhs_mat_2367_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 2), m3b); //B22(8-15) B23(8-15) B26(8-15) B27(8-15)\n\n // Fourth sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 2), m3b); //B30(0-7) B31(0-7) B34(0-7) B35(0-7)\n const __m256i rhs_mat_2367_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 2), m3b); //B32(0-7) B33(0-7) B36(0-7) B37(0-7)\n\n const __m256i rhs_mat_0145_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 2), m3b); //B30(8-15) B31(8-15) B34(8-15) B35(8-15)\n const __m256i rhs_mat_2367_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 2), m3b); //B32(8-15) B33(8-15) B36(8-15) B37(8-15)\n\n // Fifth sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m3b); //B40(0-7) B41(0-7) B44(0-7) B45(0-7)\n const __m256i rhs_mat_2367_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m3b); //B42(0-7) B43(0-7) B46(0-7) B47(0-7)\n\n const __m256i rhs_mat_0145_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m3b); //B40(8-15) B41(8-15) B44(8-15) B45(8-15)\n const __m256i rhs_mat_2367_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m3b); //B42(8-15) B43(8-15) B46(8-15) B47(8-15)\n\n // Sixth sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m3b); //B50(0-7) B51(0-7) B54(0-7) B55(0-7)\n const __m256i rhs_mat_2367_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m3b); //B52(0-7) B53(0-7) B56(0-7) B57(0-7)\n\n const __m256i rhs_mat_0145_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m3b); //B50(8-15) B51(8-15) B54(8-15) B55(8-15)\n const __m256i rhs_mat_2367_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m3b); //B52(8-15) B53(8-15) B56(8-15) B57(8-15)\n\n // Seventh sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 6), m3b); //B60(0-7) B61(0-7) B64(0-7) B65(0-7)\n const __m256i rhs_mat_2367_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 6), m3b); //B62(0-7) B63(0-7) B66(0-7) B67(0-7)\n\n const __m256i rhs_mat_0145_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 6), m3b); //B60(8-15) B61(8-15) B64(8-15) B65(8-15)\n const __m256i rhs_mat_2367_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 6), m3b); //B62(8-15) B63(8-15) B66(8-15) B67(8-15)\n\n // Eighth sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 6), m3b); //B70(0-7) B71(0-7) B74(0-7) B75(0-7)\n const __m256i rhs_mat_2367_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 6), m3b); //B72(0-7) B73(0-7) B76(0-7) B77(0-7)\n\n const __m256i rhs_mat_0145_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 6), m3b); //B70(8-15) B71(8-15) B74(8-15) B75(8-15)\n const __m256i rhs_mat_2367_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 6), m3b); //B72(8-15) B73(8-15) B76(8-15) B77(8-15)\n\n // Shuffle pattern one - right side input\n const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3)\n const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3)\n\n const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11)\n const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11)\n\n const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3)\n const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3)\n\n const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11)\n const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11)\n\n const __m256i rhs_mat_0145_20_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_20, 136); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3)\n const __m256i rhs_mat_2367_20_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_20, 136); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3)\n\n const __m256i rhs_mat_0145_21_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_21, 136); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11)\n const __m256i rhs_mat_2367_21_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_21, 136); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11)\n\n const __m256i rhs_mat_0145_30_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_30, 136); //B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3)\n const __m256i rhs_mat_2367_30_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_30, 136); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3)\n\n const __m256i rhs_mat_0145_31_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_31, 136); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11\n const __m256i rhs_mat_2367_31_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_31, 136); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11)\n\n const __m256i rhs_mat_0145_40_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_40, 136); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3)\n const __m256i rhs_mat_2367_40_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_40, 136); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3)\n\n const __m256i rhs_mat_0145_41_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_41, 136); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11)\n const __m256i rhs_mat_2367_41_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_41, 136); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11)\n\n const __m256i rhs_mat_0145_50_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_50, 136); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3)\n const __m256i rhs_mat_2367_50_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_50, 136); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3)\n\n const __m256i rhs_mat_0145_51_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_51, 136); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11)\n const __m256i rhs_mat_2367_51_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_51, 136); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11)\n\n const __m256i rhs_mat_0145_60_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_60, 136); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3)\n const __m256i rhs_mat_2367_60_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_60, 136); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3)\n\n const __m256i rhs_mat_0145_61_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_61, 136); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11)\n const __m256i rhs_mat_2367_61_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_61, 136); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11)\n\n const __m256i rhs_mat_0145_70_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_70, 136); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3)\n const __m256i rhs_mat_2367_70_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_70, 136); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3)\n\n const __m256i rhs_mat_0145_71_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_71, 136); //B70(8-11) B71(8-11) B70(8-11) B71(8-11) B74(8-11) B75(8-11) B74(8-11) B75(8-11)\n const __m256i rhs_mat_2367_71_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_71, 136); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11)\n\n\n // Shuffle pattern two - right side input\n const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7)\n const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7)\n\n const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15)\n const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15)\n\n const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7)\n const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7)\n\n const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15)\n const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15)\n\n const __m256i rhs_mat_0145_20_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_20, 221); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7)\n const __m256i rhs_mat_2367_20_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_20, 221); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7)\n\n const __m256i rhs_mat_0145_21_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_21, 221); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15)\n const __m256i rhs_mat_2367_21_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_21, 221); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15)\n\n const __m256i rhs_mat_0145_30_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_30, 221); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7)\n const __m256i rhs_mat_2367_30_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_30, 221); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7)\n\n const __m256i rhs_mat_0145_31_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_31, 221); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15)\n const __m256i rhs_mat_2367_31_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_31, 221); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15)\n\n const __m256i rhs_mat_0145_40_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_40, 221); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7)\n const __m256i rhs_mat_2367_40_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_40, 221); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7)\n\n const __m256i rhs_mat_0145_41_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_41, 221); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15)\n const __m256i rhs_mat_2367_41_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_41, 221); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15)\n\n const __m256i rhs_mat_0145_50_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_50, 221); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7)\n const __m256i rhs_mat_2367_50_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_50, 221); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7)\n\n const __m256i rhs_mat_0145_51_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_51, 221); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15)\n const __m256i rhs_mat_2367_51_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_51, 221); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15)\n\n const __m256i rhs_mat_0145_60_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_60, 221); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7)\n const __m256i rhs_mat_2367_60_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_60, 221); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7)\n\n const __m256i rhs_mat_0145_61_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_61, 221); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15)\n const __m256i rhs_mat_2367_61_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_61, 221); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15)\n\n const __m256i rhs_mat_0145_70_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_70, 221); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7)\n const __m256i rhs_mat_2367_70_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_70, 221); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7)\n\n const __m256i rhs_mat_0145_71_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_71, 221); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15)\n const __m256i rhs_mat_2367_71_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_71, 221); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15)\n\n //Scales and Mins of corresponding sub blocks from different Q2_K structures are stored together\n //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71\n\n // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop\n const __m128i mins_and_scales_01 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + sb * 64));\n const __m128i mins_and_scales_23 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 16 + sb * 64));\n const __m128i mins_and_scales_45 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 32 + sb * 64));\n const __m128i mins_and_scales_67 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 48 + sb * 64));\n\n // Extract scales which is lower half from mins_and_scales\n const __m128i scales_01 = _mm_and_si128(mins_and_scales_01, m4b_sse);\n const __m128i scales_23 = _mm_and_si128(mins_and_scales_23, m4b_sse);\n const __m128i scales_45 = _mm_and_si128(mins_and_scales_45, m4b_sse);\n const __m128i scales_67 = _mm_and_si128(mins_and_scales_67, m4b_sse);\n\n // Extract mins which is upper half from mins_and_scales\n const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_01, 4), m4b_sse));\n const __m256i mins_23 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_23, 4), m4b_sse));\n const __m256i mins_45 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_45, 4), m4b_sse));\n const __m256i mins_67 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_67, 4), m4b_sse));\n\n const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask1_sse));\n const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask2_sse));\n\n const __m256i scales_2 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask1_sse));\n const __m256i scales_3 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask2_sse));\n\n const __m256i scales_4 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask1_sse));\n const __m256i scales_5 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask2_sse));\n\n const __m256i scales_6 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask1_sse));\n const __m256i scales_7 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask2_sse));\n\n const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68);\n const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238);\n\n const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68);\n const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238);\n\n const __m256i scale_0145_2 = _mm256_shuffle_epi32(scales_2, 68);\n const __m256i scale_2367_2 = _mm256_shuffle_epi32(scales_2, 238);\n\n const __m256i scale_0145_3 = _mm256_shuffle_epi32(scales_3, 68);\n const __m256i scale_2367_3 = _mm256_shuffle_epi32(scales_3, 238);\n\n const __m256i scale_0145_4 = _mm256_shuffle_epi32(scales_4, 68);\n const __m256i scale_2367_4 = _mm256_shuffle_epi32(scales_4, 238);\n\n const __m256i scale_0145_5 = _mm256_shuffle_epi32(scales_5, 68);\n const __m256i scale_2367_5 = _mm256_shuffle_epi32(scales_5, 238);\n\n const __m256i scale_0145_6 = _mm256_shuffle_epi32(scales_6, 68);\n const __m256i scale_2367_6 = _mm256_shuffle_epi32(scales_6, 238);\n\n const __m256i scale_0145_7 = _mm256_shuffle_epi32(scales_7, 68);\n const __m256i scale_2367_7 = _mm256_shuffle_epi32(scales_7, 238);\n\n\n for (int rp = 0; rp < 4; rp++) {\n\n // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 512 * sb)));\n __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0);\n __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17);\n __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 512 * sb)));\n __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0);\n __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17);\n __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 512 * sb)));\n __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0);\n __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17);\n __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 512 * sb)));\n __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0);\n __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17);\n __m256i lhs_mat_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 512 * sb)));\n __m256i lhs_mat_01_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 0);\n __m256i lhs_mat_23_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 17);\n __m256i lhs_mat_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 512 * sb)));\n __m256i lhs_mat_01_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 0);\n __m256i lhs_mat_23_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 17);\n __m256i lhs_mat_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 512 * sb)));\n __m256i lhs_mat_01_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 0);\n __m256i lhs_mat_23_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 17);\n __m256i lhs_mat_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 512 * sb)));\n __m256i lhs_mat_01_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 0);\n __m256i lhs_mat_23_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 17);\n\n __m256i lhs_mat_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 + 512 * sb)));\n __m256i lhs_mat_01_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 0);\n __m256i lhs_mat_23_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 17);\n __m256i lhs_mat_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 288 + 512 * sb)));\n __m256i lhs_mat_01_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 0);\n __m256i lhs_mat_23_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 17);\n __m256i lhs_mat_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 320 + 512 * sb)));\n __m256i lhs_mat_01_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 0);\n __m256i lhs_mat_23_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 17);\n __m256i lhs_mat_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 352 + 512 * sb)));\n __m256i lhs_mat_01_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 0);\n __m256i lhs_mat_23_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 17);\n __m256i lhs_mat_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 384 + 512 * sb)));\n __m256i lhs_mat_01_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 0);\n __m256i lhs_mat_23_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 17);\n __m256i lhs_mat_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 416 + 512 * sb)));\n __m256i lhs_mat_01_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 0);\n __m256i lhs_mat_23_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 17);\n __m256i lhs_mat_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 448 + 512 * sb)));\n __m256i lhs_mat_01_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 0);\n __m256i lhs_mat_23_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 17);\n __m256i lhs_mat_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 480 + 512 * sb)));\n __m256i lhs_mat_01_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 0);\n __m256i lhs_mat_23_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 17);\n\n // Bsums are loaded for the different Q8_K blocks\n __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 32 * sb)));\n __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 8 + 32 * sb));\n __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 16 + 32 * sb)));\n __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 24 + 32 * sb));\n\n // Shuffle pattern one - left side input\n const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3)\n\n const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11)\n\n const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3)\n\n const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11)\n\n const __m256i lhs_mat_01_20_sp1 = _mm256_shuffle_epi32(lhs_mat_01_20, 160); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3)\n const __m256i lhs_mat_23_20_sp1 = _mm256_shuffle_epi32(lhs_mat_23_20, 160); //A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3)\n\n const __m256i lhs_mat_01_21_sp1 = _mm256_shuffle_epi32(lhs_mat_01_21, 160); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11)\n const __m256i lhs_mat_23_21_sp1 = _mm256_shuffle_epi32(lhs_mat_23_21, 160); //A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11)\n\n const __m256i lhs_mat_01_30_sp1 = _mm256_shuffle_epi32(lhs_mat_01_30, 160); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3)\n const __m256i lhs_mat_23_30_sp1 = _mm256_shuffle_epi32(lhs_mat_23_30, 160); //A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3)\n\n const __m256i lhs_mat_01_31_sp1 = _mm256_shuffle_epi32(lhs_mat_01_31, 160); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11)\n const __m256i lhs_mat_23_31_sp1 = _mm256_shuffle_epi32(lhs_mat_23_31, 160); //A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11)\n\n const __m256i lhs_mat_01_40_sp1 = _mm256_shuffle_epi32(lhs_mat_01_40, 160); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3)\n const __m256i lhs_mat_23_40_sp1 = _mm256_shuffle_epi32(lhs_mat_23_40, 160); //A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3)\n\n const __m256i lhs_mat_01_41_sp1 = _mm256_shuffle_epi32(lhs_mat_01_41, 160); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11)\n const __m256i lhs_mat_23_41_sp1 = _mm256_shuffle_epi32(lhs_mat_23_41, 160); //A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11)\n\n const __m256i lhs_mat_01_50_sp1 = _mm256_shuffle_epi32(lhs_mat_01_50, 160); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3)\n const __m256i lhs_mat_23_50_sp1 = _mm256_shuffle_epi32(lhs_mat_23_50, 160); //A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3)\n\n const __m256i lhs_mat_01_51_sp1 = _mm256_shuffle_epi32(lhs_mat_01_51, 160); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11)\n const __m256i lhs_mat_23_51_sp1 = _mm256_shuffle_epi32(lhs_mat_23_51, 160); //A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11)\n\n const __m256i lhs_mat_01_60_sp1 = _mm256_shuffle_epi32(lhs_mat_01_60, 160); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3)\n const __m256i lhs_mat_23_60_sp1 = _mm256_shuffle_epi32(lhs_mat_23_60, 160); //A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3)\n\n const __m256i lhs_mat_01_61_sp1 = _mm256_shuffle_epi32(lhs_mat_01_61, 160); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11)\n const __m256i lhs_mat_23_61_sp1 = _mm256_shuffle_epi32(lhs_mat_23_61, 160); //A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11)\n\n const __m256i lhs_mat_01_70_sp1 = _mm256_shuffle_epi32(lhs_mat_01_70, 160); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3)\n const __m256i lhs_mat_23_70_sp1 = _mm256_shuffle_epi32(lhs_mat_23_70, 160); //A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3)\n\n const __m256i lhs_mat_01_71_sp1 = _mm256_shuffle_epi32(lhs_mat_01_71, 160); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11)\n const __m256i lhs_mat_23_71_sp1 = _mm256_shuffle_epi32(lhs_mat_23_71, 160); //A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11)\n\n // Shuffle pattern two- left side input\n const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7)\n\n const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15)\n\n const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7)\n\n const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15)\n\n const __m256i lhs_mat_01_20_sp2 = _mm256_shuffle_epi32(lhs_mat_01_20, 245); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7)\n const __m256i lhs_mat_23_20_sp2 = _mm256_shuffle_epi32(lhs_mat_23_20, 245); //A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7)\n\n const __m256i lhs_mat_01_21_sp2 = _mm256_shuffle_epi32(lhs_mat_01_21, 245); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15)\n const __m256i lhs_mat_23_21_sp2 = _mm256_shuffle_epi32(lhs_mat_23_21, 245); //A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15)\n\n const __m256i lhs_mat_01_30_sp2 = _mm256_shuffle_epi32(lhs_mat_01_30, 245); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7)\n const __m256i lhs_mat_23_30_sp2 = _mm256_shuffle_epi32(lhs_mat_23_30, 245); //A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7)\n\n const __m256i lhs_mat_01_31_sp2 = _mm256_shuffle_epi32(lhs_mat_01_31, 245); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15)\n const __m256i lhs_mat_23_31_sp2 = _mm256_shuffle_epi32(lhs_mat_23_31, 245); //A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15)\n\n const __m256i lhs_mat_01_40_sp2 = _mm256_shuffle_epi32(lhs_mat_01_40, 245); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7)\n const __m256i lhs_mat_23_40_sp2 = _mm256_shuffle_epi32(lhs_mat_23_40, 245); //A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7)\n\n const __m256i lhs_mat_01_41_sp2 = _mm256_shuffle_epi32(lhs_mat_01_41, 245); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15)\n const __m256i lhs_mat_23_41_sp2 = _mm256_shuffle_epi32(lhs_mat_23_41, 245); //A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15)\n\n const __m256i lhs_mat_01_50_sp2 = _mm256_shuffle_epi32(lhs_mat_01_50, 245); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7)\n const __m256i lhs_mat_23_50_sp2 = _mm256_shuffle_epi32(lhs_mat_23_50, 245); //A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7)\n\n const __m256i lhs_mat_01_51_sp2 = _mm256_shuffle_epi32(lhs_mat_01_51, 245); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15)\n const __m256i lhs_mat_23_51_sp2 = _mm256_shuffle_epi32(lhs_mat_23_51, 245); //A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15)\n\n const __m256i lhs_mat_01_60_sp2 = _mm256_shuffle_epi32(lhs_mat_01_60, 245); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7)\n const __m256i lhs_mat_23_60_sp2 = _mm256_shuffle_epi32(lhs_mat_23_60, 245); //A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7)\n\n const __m256i lhs_mat_01_61_sp2 = _mm256_shuffle_epi32(lhs_mat_01_61, 245); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15)\n const __m256i lhs_mat_23_61_sp2 = _mm256_shuffle_epi32(lhs_mat_23_61, 245); //A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15)\n\n const __m256i lhs_mat_01_70_sp2 = _mm256_shuffle_epi32(lhs_mat_01_70, 245); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7)\n const __m256i lhs_mat_23_70_sp2 = _mm256_shuffle_epi32(lhs_mat_23_70, 245); //A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7)\n\n const __m256i lhs_mat_01_71_sp2 = _mm256_shuffle_epi32(lhs_mat_01_71, 245); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15)\n const __m256i lhs_mat_23_71_sp2 = _mm256_shuffle_epi32(lhs_mat_23_71, 245); //A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1));\n __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1));\n\n __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1));\n __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1));\n\n __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1));\n __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1));\n\n __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1));\n __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1));\n\n __m256i iacc_mat_00_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_01_21_sp1));\n __m256i iacc_mat_01_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_01_21_sp1));\n\n __m256i iacc_mat_10_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_23_21_sp1));\n __m256i iacc_mat_11_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_23_21_sp1));\n\n __m256i iacc_mat_00_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_01_31_sp1));\n __m256i iacc_mat_01_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_01_31_sp1));\n\n __m256i iacc_mat_10_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_23_31_sp1));\n __m256i iacc_mat_11_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_23_31_sp1));\n\n __m256i iacc_mat_00_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_01_41_sp1));\n __m256i iacc_mat_01_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_01_41_sp1));\n\n __m256i iacc_mat_10_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_23_41_sp1));\n __m256i iacc_mat_11_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_23_41_sp1));\n\n __m256i iacc_mat_00_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_01_51_sp1));\n __m256i iacc_mat_01_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_01_51_sp1));\n\n __m256i iacc_mat_10_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_23_51_sp1));\n __m256i iacc_mat_11_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_23_51_sp1));\n\n __m256i iacc_mat_00_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_01_61_sp1));\n __m256i iacc_mat_01_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_01_61_sp1));\n\n __m256i iacc_mat_10_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_23_61_sp1));\n __m256i iacc_mat_11_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_23_61_sp1));\n\n __m256i iacc_mat_00_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_01_71_sp1));\n __m256i iacc_mat_01_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_01_71_sp1));\n\n __m256i iacc_mat_10_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_23_71_sp1));\n __m256i iacc_mat_11_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_23_71_sp1));\n\n\n __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2));\n __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2));\n\n __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2));\n __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2));\n\n __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2));\n __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2));\n\n __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2));\n __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2));\n\n __m256i iacc_mat_00_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_01_21_sp2));\n __m256i iacc_mat_01_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_01_21_sp2));\n\n __m256i iacc_mat_10_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_23_21_sp2));\n __m256i iacc_mat_11_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_23_21_sp2));\n\n __m256i iacc_mat_00_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_01_31_sp2));\n __m256i iacc_mat_01_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_01_31_sp2));\n\n __m256i iacc_mat_10_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_23_31_sp2));\n __m256i iacc_mat_11_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_23_31_sp2));\n\n __m256i iacc_mat_00_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_01_41_sp2));\n __m256i iacc_mat_01_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_01_41_sp2));\n\n __m256i iacc_mat_10_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_23_41_sp2));\n __m256i iacc_mat_11_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_23_41_sp2));\n\n __m256i iacc_mat_00_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_01_51_sp2));\n __m256i iacc_mat_01_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_01_51_sp2));\n\n __m256i iacc_mat_10_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_23_51_sp2));\n __m256i iacc_mat_11_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_23_51_sp2));\n\n __m256i iacc_mat_00_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_01_61_sp2));\n __m256i iacc_mat_01_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_01_61_sp2));\n\n __m256i iacc_mat_10_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_23_61_sp2));\n __m256i iacc_mat_11_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_23_61_sp2));\n\n __m256i iacc_mat_00_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_01_71_sp2));\n __m256i iacc_mat_01_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_01_71_sp2));\n\n __m256i iacc_mat_10_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_23_71_sp2));\n __m256i iacc_mat_11_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_23_71_sp2));\n\n // Combine results from both shuffle patterns for each output block\n __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n __m256i iacc_mat_00_2 = _mm256_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2);\n __m256i iacc_mat_01_2 = _mm256_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2);\n __m256i iacc_mat_10_2 = _mm256_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2);\n __m256i iacc_mat_11_2 = _mm256_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2);\n\n __m256i iacc_mat_00_3 = _mm256_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2);\n __m256i iacc_mat_01_3 = _mm256_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2);\n __m256i iacc_mat_10_3 = _mm256_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2);\n __m256i iacc_mat_11_3 = _mm256_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2);\n\n __m256i iacc_mat_00_4 = _mm256_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2);\n __m256i iacc_mat_01_4 = _mm256_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2);\n __m256i iacc_mat_10_4 = _mm256_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2);\n __m256i iacc_mat_11_4 = _mm256_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2);\n\n __m256i iacc_mat_00_5 = _mm256_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2);\n __m256i iacc_mat_01_5 = _mm256_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2);\n __m256i iacc_mat_10_5 = _mm256_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2);\n __m256i iacc_mat_11_5 = _mm256_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2);\n\n __m256i iacc_mat_00_6 = _mm256_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2);\n __m256i iacc_mat_01_6 = _mm256_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2);\n __m256i iacc_mat_10_6 = _mm256_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2);\n __m256i iacc_mat_11_6 = _mm256_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2);\n\n __m256i iacc_mat_00_7 = _mm256_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2);\n __m256i iacc_mat_01_7 = _mm256_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2);\n __m256i iacc_mat_10_7 = _mm256_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2);\n __m256i iacc_mat_11_7 = _mm256_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0);\n iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0);\n iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0);\n iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0);\n\n iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1);\n iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1);\n iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1);\n iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1);\n\n iacc_mat_00_2 = _mm256_madd_epi16(iacc_mat_00_2, scale_0145_2);\n iacc_mat_01_2 = _mm256_madd_epi16(iacc_mat_01_2, scale_2367_2);\n iacc_mat_10_2 = _mm256_madd_epi16(iacc_mat_10_2, scale_0145_2);\n iacc_mat_11_2 = _mm256_madd_epi16(iacc_mat_11_2, scale_2367_2);\n\n iacc_mat_00_3 = _mm256_madd_epi16(iacc_mat_00_3, scale_0145_3);\n iacc_mat_01_3 = _mm256_madd_epi16(iacc_mat_01_3, scale_2367_3);\n iacc_mat_10_3 = _mm256_madd_epi16(iacc_mat_10_3, scale_0145_3);\n iacc_mat_11_3 = _mm256_madd_epi16(iacc_mat_11_3, scale_2367_3);\n\n iacc_mat_00_4 = _mm256_madd_epi16(iacc_mat_00_4, scale_0145_4);\n iacc_mat_01_4 = _mm256_madd_epi16(iacc_mat_01_4, scale_2367_4);\n iacc_mat_10_4 = _mm256_madd_epi16(iacc_mat_10_4, scale_0145_4);\n iacc_mat_11_4 = _mm256_madd_epi16(iacc_mat_11_4, scale_2367_4);\n\n iacc_mat_00_5 = _mm256_madd_epi16(iacc_mat_00_5, scale_0145_5);\n iacc_mat_01_5 = _mm256_madd_epi16(iacc_mat_01_5, scale_2367_5);\n iacc_mat_10_5 = _mm256_madd_epi16(iacc_mat_10_5, scale_0145_5);\n iacc_mat_11_5 = _mm256_madd_epi16(iacc_mat_11_5, scale_2367_5);\n\n iacc_mat_00_6 = _mm256_madd_epi16(iacc_mat_00_6, scale_0145_6);\n iacc_mat_01_6 = _mm256_madd_epi16(iacc_mat_01_6, scale_2367_6);\n iacc_mat_10_6 = _mm256_madd_epi16(iacc_mat_10_6, scale_0145_6);\n iacc_mat_11_6 = _mm256_madd_epi16(iacc_mat_11_6, scale_2367_6);\n\n iacc_mat_00_7 = _mm256_madd_epi16(iacc_mat_00_7, scale_0145_7);\n iacc_mat_01_7 = _mm256_madd_epi16(iacc_mat_01_7, scale_2367_7);\n iacc_mat_10_7 = _mm256_madd_epi16(iacc_mat_10_7, scale_0145_7);\n iacc_mat_11_7 = _mm256_madd_epi16(iacc_mat_11_7, scale_2367_7);\n\n __m256i iacc_mat_00 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm256_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm256_add_epi32(iacc_mat_00_6, iacc_mat_00_7)));\n __m256i iacc_mat_01 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm256_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm256_add_epi32(iacc_mat_01_6, iacc_mat_01_7)));\n __m256i iacc_mat_10 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm256_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm256_add_epi32(iacc_mat_10_6, iacc_mat_10_7)));\n __m256i iacc_mat_11 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm256_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm256_add_epi32(iacc_mat_11_6, iacc_mat_11_7)));\n\n // Straighten out to make 4 row vectors\n __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204);\n __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204);\n __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204);\n __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204);\n\n // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d);\n const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n\n // Multiply with appropriate scales and accumulate (for both d and dmin) below\n acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n\n __m256i lhs_bsums_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1);\n __m256i lhs_bsums_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1);\n __m256i lhs_bsums_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1);\n __m256i lhs_bsums_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1);\n\n // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K\n __m256i iacc_row_min_0_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 0), mins_01);\n __m256i iacc_row_min_1_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 170), mins_01);\n __m256i iacc_row_min_2_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 0), mins_01);\n __m256i iacc_row_min_3_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 170), mins_01);\n\n __m256i iacc_row_min_0_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 85), mins_23);\n __m256i iacc_row_min_1_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 255), mins_23);\n __m256i iacc_row_min_2_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 85), mins_23);\n __m256i iacc_row_min_3_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 255), mins_23);\n\n __m256i iacc_row_min_0_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 0), mins_45);\n __m256i iacc_row_min_1_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 170), mins_45);\n __m256i iacc_row_min_2_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 0), mins_45);\n __m256i iacc_row_min_3_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 170), mins_45);\n\n __m256i iacc_row_min_0_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 85), mins_67);\n __m256i iacc_row_min_1_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 255), mins_67);\n __m256i iacc_row_min_2_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 85), mins_67);\n __m256i iacc_row_min_3_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 255), mins_67);\n\n __m256i iacc_row_min_0 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm256_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67));\n __m256i iacc_row_min_1 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm256_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67));\n __m256i iacc_row_min_2 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm256_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67));\n __m256i iacc_row_min_3 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm256_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67));\n\n acc_min_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]);\n acc_min_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]);\n acc_min_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]);\n acc_min_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]);\n\n }\n }\n }\n // Store the accumulated values\n for (int i = 0; i < 16; i++) {\n _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i]));\n\n }\n }\n }\n\n for (; y < nr / 4; y ++) {\n\n const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb);\n\n // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation\n for (int64_t x = xstart; x < nc / 8; x++) {\n\n const block_q2_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n // Master FP accumulators\n __m256 acc_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_rows[i] = _mm256_setzero_ps();\n }\n\n __m256 acc_min_rows[4];\n for (int i = 0; i < 4; i++) {\n acc_min_rows[i] = _mm256_setzero_ps();\n }\n\n for (int64_t b = 0; b < nb; b++) {\n // Delta values - Load the eight scale values of block_q2_kx8\n const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n\n // dmin values - Load the eight dmin values of block_q2_kx8\n const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration\n for (int sb = 0; sb < QK_K / 128; sb++) {\n\n // Load the eight block_q2_k for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + sb * 256));\n const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32 + sb * 256));\n const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64 + sb * 256));\n const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96 + sb * 256));\n const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 128 + sb * 256));\n const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 160 + sb * 256));\n const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 192 + sb * 256));\n const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 224 + sb * 256));\n\n // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n //superblock sub block which part of sub block\n const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n\n const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n\n const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n // 2-bit -> 8-bit\n // First sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m3b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7)\n const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m3b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7)\n\n const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m3b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15)\n const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m3b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15)\n\n // Second sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_10 = _mm256_and_si256(rhs_raw_mat_0145_2, m3b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7)\n const __m256i rhs_mat_2367_10 = _mm256_and_si256(rhs_raw_mat_2367_2, m3b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7)\n\n const __m256i rhs_mat_0145_11 = _mm256_and_si256(rhs_raw_mat_0145_3, m3b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15)\n const __m256i rhs_mat_2367_11 = _mm256_and_si256(rhs_raw_mat_2367_3, m3b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15)\n\n // Third sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 2), m3b); //B20(0-7) B21(0-7) B24(0-7) B25(0-7)\n const __m256i rhs_mat_2367_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 2), m3b); //B22(0-7) B23(0-7) B26(0-7) B27(0-7)\n\n const __m256i rhs_mat_0145_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 2), m3b); //B20(8-15) B21(8-15) B24(8-15) B25(8-15)\n const __m256i rhs_mat_2367_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 2), m3b); //B22(8-15) B23(8-15) B26(8-15) B27(8-15)\n\n // Fourth sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 2), m3b); //B30(0-7) B31(0-7) B34(0-7) B35(0-7)\n const __m256i rhs_mat_2367_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 2), m3b); //B32(0-7) B33(0-7) B36(0-7) B37(0-7)\n\n const __m256i rhs_mat_0145_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 2), m3b); //B30(8-15) B31(8-15) B34(8-15) B35(8-15)\n const __m256i rhs_mat_2367_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 2), m3b); //B32(8-15) B33(8-15) B36(8-15) B37(8-15)\n\n // Fifth sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m3b); //B40(0-7) B41(0-7) B44(0-7) B45(0-7)\n const __m256i rhs_mat_2367_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m3b); //B42(0-7) B43(0-7) B46(0-7) B47(0-7)\n\n const __m256i rhs_mat_0145_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m3b); //B40(8-15) B41(8-15) B44(8-15) B45(8-15)\n const __m256i rhs_mat_2367_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m3b); //B42(8-15) B43(8-15) B46(8-15) B47(8-15)\n\n // Sixth sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m3b); //B50(0-7) B51(0-7) B54(0-7) B55(0-7)\n const __m256i rhs_mat_2367_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m3b); //B52(0-7) B53(0-7) B56(0-7) B57(0-7)\n\n const __m256i rhs_mat_0145_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m3b); //B50(8-15) B51(8-15) B54(8-15) B55(8-15)\n const __m256i rhs_mat_2367_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m3b); //B52(8-15) B53(8-15) B56(8-15) B57(8-15)\n\n // Seventh sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 6), m3b); //B60(0-7) B61(0-7) B64(0-7) B65(0-7)\n const __m256i rhs_mat_2367_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 6), m3b); //B62(0-7) B63(0-7) B66(0-7) B67(0-7)\n\n const __m256i rhs_mat_0145_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 6), m3b); //B60(8-15) B61(8-15) B64(8-15) B65(8-15)\n const __m256i rhs_mat_2367_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 6), m3b); //B62(8-15) B63(8-15) B66(8-15) B67(8-15)\n\n // Eighth sub block of the eight sub blocks processed in the iteration\n const __m256i rhs_mat_0145_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 6), m3b); //B70(0-7) B71(0-7) B74(0-7) B75(0-7)\n const __m256i rhs_mat_2367_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 6), m3b); //B72(0-7) B73(0-7) B76(0-7) B77(0-7)\n\n const __m256i rhs_mat_0145_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 6), m3b); //B70(8-15) B71(8-15) B74(8-15) B75(8-15)\n const __m256i rhs_mat_2367_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 6), m3b); //B72(8-15) B73(8-15) B76(8-15) B77(8-15)\n\n // Shuffle pattern one - right side input\n const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3)\n const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3)\n\n const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11)\n const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11)\n\n const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3)\n const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3)\n\n const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11)\n const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11)\n\n const __m256i rhs_mat_0145_20_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_20, 136); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3)\n const __m256i rhs_mat_2367_20_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_20, 136); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3)\n\n const __m256i rhs_mat_0145_21_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_21, 136); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11)\n const __m256i rhs_mat_2367_21_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_21, 136); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11)\n\n const __m256i rhs_mat_0145_30_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_30, 136); //B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3)\n const __m256i rhs_mat_2367_30_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_30, 136); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3)\n\n const __m256i rhs_mat_0145_31_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_31, 136); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11\n const __m256i rhs_mat_2367_31_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_31, 136); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11)\n\n const __m256i rhs_mat_0145_40_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_40, 136); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3)\n const __m256i rhs_mat_2367_40_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_40, 136); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3)\n\n const __m256i rhs_mat_0145_41_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_41, 136); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11)\n const __m256i rhs_mat_2367_41_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_41, 136); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11)\n\n const __m256i rhs_mat_0145_50_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_50, 136); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3)\n const __m256i rhs_mat_2367_50_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_50, 136); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3)\n\n const __m256i rhs_mat_0145_51_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_51, 136); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11)\n const __m256i rhs_mat_2367_51_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_51, 136); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11)\n\n const __m256i rhs_mat_0145_60_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_60, 136); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3)\n const __m256i rhs_mat_2367_60_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_60, 136); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3)\n\n const __m256i rhs_mat_0145_61_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_61, 136); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11)\n const __m256i rhs_mat_2367_61_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_61, 136); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11)\n\n const __m256i rhs_mat_0145_70_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_70, 136); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3)\n const __m256i rhs_mat_2367_70_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_70, 136); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3)\n\n const __m256i rhs_mat_0145_71_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_71, 136); //B70(8-11) B71(8-11) B70(8-11) B71(8-11) B74(8-11) B75(8-11) B74(8-11) B75(8-11)\n const __m256i rhs_mat_2367_71_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_71, 136); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11)\n\n\n // Shuffle pattern two - right side input\n const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7)\n const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7)\n\n const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15)\n const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15)\n\n const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7)\n const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7)\n\n const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15)\n const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15)\n\n const __m256i rhs_mat_0145_20_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_20, 221); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7)\n const __m256i rhs_mat_2367_20_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_20, 221); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7)\n\n const __m256i rhs_mat_0145_21_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_21, 221); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15)\n const __m256i rhs_mat_2367_21_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_21, 221); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15)\n\n const __m256i rhs_mat_0145_30_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_30, 221); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7)\n const __m256i rhs_mat_2367_30_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_30, 221); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7)\n\n const __m256i rhs_mat_0145_31_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_31, 221); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15)\n const __m256i rhs_mat_2367_31_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_31, 221); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15)\n\n const __m256i rhs_mat_0145_40_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_40, 221); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7)\n const __m256i rhs_mat_2367_40_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_40, 221); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7)\n\n const __m256i rhs_mat_0145_41_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_41, 221); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15)\n const __m256i rhs_mat_2367_41_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_41, 221); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15)\n\n const __m256i rhs_mat_0145_50_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_50, 221); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7)\n const __m256i rhs_mat_2367_50_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_50, 221); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7)\n\n const __m256i rhs_mat_0145_51_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_51, 221); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15)\n const __m256i rhs_mat_2367_51_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_51, 221); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15)\n\n const __m256i rhs_mat_0145_60_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_60, 221); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7)\n const __m256i rhs_mat_2367_60_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_60, 221); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7)\n\n const __m256i rhs_mat_0145_61_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_61, 221); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15)\n const __m256i rhs_mat_2367_61_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_61, 221); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15)\n\n const __m256i rhs_mat_0145_70_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_70, 221); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7)\n const __m256i rhs_mat_2367_70_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_70, 221); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7)\n\n const __m256i rhs_mat_0145_71_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_71, 221); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15)\n const __m256i rhs_mat_2367_71_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_71, 221); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15)\n\n\n //Scales and Mins of corresponding sub blocks from different Q2_K structures are stored together\n //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71\n\n // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop\n const __m128i mins_and_scales_01 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + sb * 64));\n const __m128i mins_and_scales_23 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 16 + sb * 64));\n const __m128i mins_and_scales_45 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 32 + sb * 64));\n const __m128i mins_and_scales_67 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 48 + sb * 64));\n\n // Extract scales which is lower half from mins_and_scales\n const __m128i scales_01 = _mm_and_si128(mins_and_scales_01, m4b_sse);\n const __m128i scales_23 = _mm_and_si128(mins_and_scales_23, m4b_sse);\n const __m128i scales_45 = _mm_and_si128(mins_and_scales_45, m4b_sse);\n const __m128i scales_67 = _mm_and_si128(mins_and_scales_67, m4b_sse);\n\n // Extract mins which is upper half from mins_and_scales\n const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_01, 4), m4b_sse));\n const __m256i mins_23 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_23, 4), m4b_sse));\n const __m256i mins_45 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_45, 4), m4b_sse));\n const __m256i mins_67 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_67, 4), m4b_sse));\n\n const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask1_sse));\n const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask2_sse));\n\n const __m256i scales_2 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask1_sse));\n const __m256i scales_3 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask2_sse));\n\n const __m256i scales_4 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask1_sse));\n const __m256i scales_5 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask2_sse));\n\n const __m256i scales_6 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask1_sse));\n const __m256i scales_7 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask2_sse));\n\n const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68);\n const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238);\n\n const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68);\n const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238);\n\n const __m256i scale_0145_2 = _mm256_shuffle_epi32(scales_2, 68);\n const __m256i scale_2367_2 = _mm256_shuffle_epi32(scales_2, 238);\n\n const __m256i scale_0145_3 = _mm256_shuffle_epi32(scales_3, 68);\n const __m256i scale_2367_3 = _mm256_shuffle_epi32(scales_3, 238);\n\n const __m256i scale_0145_4 = _mm256_shuffle_epi32(scales_4, 68);\n const __m256i scale_2367_4 = _mm256_shuffle_epi32(scales_4, 238);\n\n const __m256i scale_0145_5 = _mm256_shuffle_epi32(scales_5, 68);\n const __m256i scale_2367_5 = _mm256_shuffle_epi32(scales_5, 238);\n\n const __m256i scale_0145_6 = _mm256_shuffle_epi32(scales_6, 68);\n const __m256i scale_2367_6 = _mm256_shuffle_epi32(scales_6, 238);\n\n const __m256i scale_0145_7 = _mm256_shuffle_epi32(scales_7, 68);\n const __m256i scale_2367_7 = _mm256_shuffle_epi32(scales_7, 238);\n\n // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 512 * sb)));\n __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0);\n __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17);\n __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 512 * sb)));\n __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0);\n __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17);\n __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 512 * sb)));\n __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0);\n __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17);\n __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 512 * sb)));\n __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0);\n __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17);\n __m256i lhs_mat_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 512 * sb)));\n __m256i lhs_mat_01_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 0);\n __m256i lhs_mat_23_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 17);\n __m256i lhs_mat_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 512 * sb)));\n __m256i lhs_mat_01_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 0);\n __m256i lhs_mat_23_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 17);\n __m256i lhs_mat_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 512 * sb)));\n __m256i lhs_mat_01_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 0);\n __m256i lhs_mat_23_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 17);\n __m256i lhs_mat_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 512 * sb)));\n __m256i lhs_mat_01_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 0);\n __m256i lhs_mat_23_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 17);\n\n __m256i lhs_mat_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 + 512 * sb)));\n __m256i lhs_mat_01_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 0);\n __m256i lhs_mat_23_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 17);\n __m256i lhs_mat_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 288 + 512 * sb)));\n __m256i lhs_mat_01_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 0);\n __m256i lhs_mat_23_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 17);\n __m256i lhs_mat_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 320 + 512 * sb)));\n __m256i lhs_mat_01_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 0);\n __m256i lhs_mat_23_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 17);\n __m256i lhs_mat_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 352 + 512 * sb)));\n __m256i lhs_mat_01_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 0);\n __m256i lhs_mat_23_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 17);\n __m256i lhs_mat_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 384 + 512 * sb)));\n __m256i lhs_mat_01_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 0);\n __m256i lhs_mat_23_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 17);\n __m256i lhs_mat_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 416 + 512 * sb)));\n __m256i lhs_mat_01_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 0);\n __m256i lhs_mat_23_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 17);\n __m256i lhs_mat_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 448 + 512 * sb)));\n __m256i lhs_mat_01_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 0);\n __m256i lhs_mat_23_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 17);\n __m256i lhs_mat_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 480 + 512 * sb)));\n __m256i lhs_mat_01_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 0);\n __m256i lhs_mat_23_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 17);\n\n // Bsums are loaded for the different Q8_K blocks\n __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 32 * sb)));\n __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 8 + 32 * sb));\n __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 16 + 32 * sb)));\n __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 24 + 32 * sb));\n\n // Shuffle pattern one - left side input\n const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3)\n\n const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11)\n\n const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3)\n\n const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11)\n\n const __m256i lhs_mat_01_20_sp1 = _mm256_shuffle_epi32(lhs_mat_01_20, 160); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3)\n const __m256i lhs_mat_23_20_sp1 = _mm256_shuffle_epi32(lhs_mat_23_20, 160); //A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3)\n\n const __m256i lhs_mat_01_21_sp1 = _mm256_shuffle_epi32(lhs_mat_01_21, 160); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11)\n const __m256i lhs_mat_23_21_sp1 = _mm256_shuffle_epi32(lhs_mat_23_21, 160); //A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11)\n\n const __m256i lhs_mat_01_30_sp1 = _mm256_shuffle_epi32(lhs_mat_01_30, 160); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3)\n const __m256i lhs_mat_23_30_sp1 = _mm256_shuffle_epi32(lhs_mat_23_30, 160); //A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3)\n\n const __m256i lhs_mat_01_31_sp1 = _mm256_shuffle_epi32(lhs_mat_01_31, 160); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11)\n const __m256i lhs_mat_23_31_sp1 = _mm256_shuffle_epi32(lhs_mat_23_31, 160); //A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11)\n\n const __m256i lhs_mat_01_40_sp1 = _mm256_shuffle_epi32(lhs_mat_01_40, 160); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3)\n const __m256i lhs_mat_23_40_sp1 = _mm256_shuffle_epi32(lhs_mat_23_40, 160); //A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3)\n\n const __m256i lhs_mat_01_41_sp1 = _mm256_shuffle_epi32(lhs_mat_01_41, 160); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11)\n const __m256i lhs_mat_23_41_sp1 = _mm256_shuffle_epi32(lhs_mat_23_41, 160); //A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11)\n\n const __m256i lhs_mat_01_50_sp1 = _mm256_shuffle_epi32(lhs_mat_01_50, 160); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3)\n const __m256i lhs_mat_23_50_sp1 = _mm256_shuffle_epi32(lhs_mat_23_50, 160); //A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3)\n\n const __m256i lhs_mat_01_51_sp1 = _mm256_shuffle_epi32(lhs_mat_01_51, 160); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11)\n const __m256i lhs_mat_23_51_sp1 = _mm256_shuffle_epi32(lhs_mat_23_51, 160); //A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11)\n\n const __m256i lhs_mat_01_60_sp1 = _mm256_shuffle_epi32(lhs_mat_01_60, 160); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3)\n const __m256i lhs_mat_23_60_sp1 = _mm256_shuffle_epi32(lhs_mat_23_60, 160); //A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3)\n\n const __m256i lhs_mat_01_61_sp1 = _mm256_shuffle_epi32(lhs_mat_01_61, 160); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11)\n const __m256i lhs_mat_23_61_sp1 = _mm256_shuffle_epi32(lhs_mat_23_61, 160); //A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11)\n\n const __m256i lhs_mat_01_70_sp1 = _mm256_shuffle_epi32(lhs_mat_01_70, 160); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3)\n const __m256i lhs_mat_23_70_sp1 = _mm256_shuffle_epi32(lhs_mat_23_70, 160); //A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3)\n\n const __m256i lhs_mat_01_71_sp1 = _mm256_shuffle_epi32(lhs_mat_01_71, 160); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11)\n const __m256i lhs_mat_23_71_sp1 = _mm256_shuffle_epi32(lhs_mat_23_71, 160); //A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11)\n\n // Shuffle pattern two- left side input\n const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7)\n\n const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15)\n\n const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7)\n\n const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15)\n\n const __m256i lhs_mat_01_20_sp2 = _mm256_shuffle_epi32(lhs_mat_01_20, 245); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7)\n const __m256i lhs_mat_23_20_sp2 = _mm256_shuffle_epi32(lhs_mat_23_20, 245); //A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7)\n\n const __m256i lhs_mat_01_21_sp2 = _mm256_shuffle_epi32(lhs_mat_01_21, 245); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15)\n const __m256i lhs_mat_23_21_sp2 = _mm256_shuffle_epi32(lhs_mat_23_21, 245); //A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15)\n\n const __m256i lhs_mat_01_30_sp2 = _mm256_shuffle_epi32(lhs_mat_01_30, 245); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7)\n const __m256i lhs_mat_23_30_sp2 = _mm256_shuffle_epi32(lhs_mat_23_30, 245); //A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7)\n\n const __m256i lhs_mat_01_31_sp2 = _mm256_shuffle_epi32(lhs_mat_01_31, 245); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15)\n const __m256i lhs_mat_23_31_sp2 = _mm256_shuffle_epi32(lhs_mat_23_31, 245); //A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15)\n\n const __m256i lhs_mat_01_40_sp2 = _mm256_shuffle_epi32(lhs_mat_01_40, 245); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7)\n const __m256i lhs_mat_23_40_sp2 = _mm256_shuffle_epi32(lhs_mat_23_40, 245); //A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7)\n\n const __m256i lhs_mat_01_41_sp2 = _mm256_shuffle_epi32(lhs_mat_01_41, 245); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15)\n const __m256i lhs_mat_23_41_sp2 = _mm256_shuffle_epi32(lhs_mat_23_41, 245); //A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15)\n\n const __m256i lhs_mat_01_50_sp2 = _mm256_shuffle_epi32(lhs_mat_01_50, 245); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7)\n const __m256i lhs_mat_23_50_sp2 = _mm256_shuffle_epi32(lhs_mat_23_50, 245); //A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7)\n\n const __m256i lhs_mat_01_51_sp2 = _mm256_shuffle_epi32(lhs_mat_01_51, 245); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15)\n const __m256i lhs_mat_23_51_sp2 = _mm256_shuffle_epi32(lhs_mat_23_51, 245); //A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15)\n\n const __m256i lhs_mat_01_60_sp2 = _mm256_shuffle_epi32(lhs_mat_01_60, 245); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7)\n const __m256i lhs_mat_23_60_sp2 = _mm256_shuffle_epi32(lhs_mat_23_60, 245); //A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7)\n\n const __m256i lhs_mat_01_61_sp2 = _mm256_shuffle_epi32(lhs_mat_01_61, 245); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15)\n const __m256i lhs_mat_23_61_sp2 = _mm256_shuffle_epi32(lhs_mat_23_61, 245); //A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15)\n\n const __m256i lhs_mat_01_70_sp2 = _mm256_shuffle_epi32(lhs_mat_01_70, 245); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7)\n const __m256i lhs_mat_23_70_sp2 = _mm256_shuffle_epi32(lhs_mat_23_70, 245); //A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7)\n\n const __m256i lhs_mat_01_71_sp2 = _mm256_shuffle_epi32(lhs_mat_01_71, 245); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15)\n const __m256i lhs_mat_23_71_sp2 = _mm256_shuffle_epi32(lhs_mat_23_71, 245); //A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15)\n\n // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1));\n __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1));\n\n __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1));\n __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1));\n\n __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1));\n __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1));\n\n __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1));\n __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1));\n\n __m256i iacc_mat_00_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_01_21_sp1));\n __m256i iacc_mat_01_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_01_21_sp1));\n\n __m256i iacc_mat_10_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_23_21_sp1));\n __m256i iacc_mat_11_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_23_21_sp1));\n\n __m256i iacc_mat_00_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_01_31_sp1));\n __m256i iacc_mat_01_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_01_31_sp1));\n\n __m256i iacc_mat_10_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_23_31_sp1));\n __m256i iacc_mat_11_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_23_31_sp1));\n\n __m256i iacc_mat_00_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_01_41_sp1));\n __m256i iacc_mat_01_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_01_41_sp1));\n\n __m256i iacc_mat_10_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_23_41_sp1));\n __m256i iacc_mat_11_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_23_41_sp1));\n\n __m256i iacc_mat_00_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_01_51_sp1));\n __m256i iacc_mat_01_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_01_51_sp1));\n\n __m256i iacc_mat_10_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_23_51_sp1));\n __m256i iacc_mat_11_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_23_51_sp1));\n\n __m256i iacc_mat_00_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_01_61_sp1));\n __m256i iacc_mat_01_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_01_61_sp1));\n\n __m256i iacc_mat_10_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_23_61_sp1));\n __m256i iacc_mat_11_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_23_61_sp1));\n\n __m256i iacc_mat_00_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_01_71_sp1));\n __m256i iacc_mat_01_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_01_71_sp1));\n\n __m256i iacc_mat_10_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_23_71_sp1));\n __m256i iacc_mat_11_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_23_71_sp1));\n\n\n __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2));\n __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2));\n\n __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2));\n __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2));\n\n __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2));\n __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2));\n\n __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2));\n __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2));\n\n __m256i iacc_mat_00_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_01_21_sp2));\n __m256i iacc_mat_01_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_01_21_sp2));\n\n __m256i iacc_mat_10_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_23_21_sp2));\n __m256i iacc_mat_11_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_23_21_sp2));\n\n __m256i iacc_mat_00_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_01_31_sp2));\n __m256i iacc_mat_01_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_01_31_sp2));\n\n __m256i iacc_mat_10_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_23_31_sp2));\n __m256i iacc_mat_11_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_23_31_sp2));\n\n __m256i iacc_mat_00_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_01_41_sp2));\n __m256i iacc_mat_01_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_01_41_sp2));\n\n __m256i iacc_mat_10_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_23_41_sp2));\n __m256i iacc_mat_11_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_23_41_sp2));\n\n __m256i iacc_mat_00_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_01_51_sp2));\n __m256i iacc_mat_01_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_01_51_sp2));\n\n __m256i iacc_mat_10_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_23_51_sp2));\n __m256i iacc_mat_11_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_23_51_sp2));\n\n __m256i iacc_mat_00_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_01_61_sp2));\n __m256i iacc_mat_01_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_01_61_sp2));\n\n __m256i iacc_mat_10_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_23_61_sp2));\n __m256i iacc_mat_11_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_23_61_sp2));\n\n __m256i iacc_mat_00_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_01_71_sp2));\n __m256i iacc_mat_01_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_01_71_sp2));\n\n __m256i iacc_mat_10_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_23_71_sp2));\n __m256i iacc_mat_11_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_23_71_sp2));\n\n // Combine results from both shuffle patterns for each output block.\n __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n __m256i iacc_mat_00_2 = _mm256_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2);\n __m256i iacc_mat_01_2 = _mm256_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2);\n __m256i iacc_mat_10_2 = _mm256_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2);\n __m256i iacc_mat_11_2 = _mm256_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2);\n\n __m256i iacc_mat_00_3 = _mm256_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2);\n __m256i iacc_mat_01_3 = _mm256_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2);\n __m256i iacc_mat_10_3 = _mm256_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2);\n __m256i iacc_mat_11_3 = _mm256_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2);\n\n __m256i iacc_mat_00_4 = _mm256_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2);\n __m256i iacc_mat_01_4 = _mm256_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2);\n __m256i iacc_mat_10_4 = _mm256_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2);\n __m256i iacc_mat_11_4 = _mm256_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2);\n\n __m256i iacc_mat_00_5 = _mm256_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2);\n __m256i iacc_mat_01_5 = _mm256_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2);\n __m256i iacc_mat_10_5 = _mm256_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2);\n __m256i iacc_mat_11_5 = _mm256_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2);\n\n __m256i iacc_mat_00_6 = _mm256_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2);\n __m256i iacc_mat_01_6 = _mm256_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2);\n __m256i iacc_mat_10_6 = _mm256_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2);\n __m256i iacc_mat_11_6 = _mm256_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2);\n\n __m256i iacc_mat_00_7 = _mm256_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2);\n __m256i iacc_mat_01_7 = _mm256_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2);\n __m256i iacc_mat_10_7 = _mm256_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2);\n __m256i iacc_mat_11_7 = _mm256_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2);\n\n // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0);\n iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0);\n iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0);\n iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0);\n\n iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1);\n iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1);\n iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1);\n iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1);\n\n iacc_mat_00_2 = _mm256_madd_epi16(iacc_mat_00_2, scale_0145_2);\n iacc_mat_01_2 = _mm256_madd_epi16(iacc_mat_01_2, scale_2367_2);\n iacc_mat_10_2 = _mm256_madd_epi16(iacc_mat_10_2, scale_0145_2);\n iacc_mat_11_2 = _mm256_madd_epi16(iacc_mat_11_2, scale_2367_2);\n\n iacc_mat_00_3 = _mm256_madd_epi16(iacc_mat_00_3, scale_0145_3);\n iacc_mat_01_3 = _mm256_madd_epi16(iacc_mat_01_3, scale_2367_3);\n iacc_mat_10_3 = _mm256_madd_epi16(iacc_mat_10_3, scale_0145_3);\n iacc_mat_11_3 = _mm256_madd_epi16(iacc_mat_11_3, scale_2367_3);\n\n iacc_mat_00_4 = _mm256_madd_epi16(iacc_mat_00_4, scale_0145_4);\n iacc_mat_01_4 = _mm256_madd_epi16(iacc_mat_01_4, scale_2367_4);\n iacc_mat_10_4 = _mm256_madd_epi16(iacc_mat_10_4, scale_0145_4);\n iacc_mat_11_4 = _mm256_madd_epi16(iacc_mat_11_4, scale_2367_4);\n\n iacc_mat_00_5 = _mm256_madd_epi16(iacc_mat_00_5, scale_0145_5);\n iacc_mat_01_5 = _mm256_madd_epi16(iacc_mat_01_5, scale_2367_5);\n iacc_mat_10_5 = _mm256_madd_epi16(iacc_mat_10_5, scale_0145_5);\n iacc_mat_11_5 = _mm256_madd_epi16(iacc_mat_11_5, scale_2367_5);\n\n iacc_mat_00_6 = _mm256_madd_epi16(iacc_mat_00_6, scale_0145_6);\n iacc_mat_01_6 = _mm256_madd_epi16(iacc_mat_01_6, scale_2367_6);\n iacc_mat_10_6 = _mm256_madd_epi16(iacc_mat_10_6, scale_0145_6);\n iacc_mat_11_6 = _mm256_madd_epi16(iacc_mat_11_6, scale_2367_6);\n\n iacc_mat_00_7 = _mm256_madd_epi16(iacc_mat_00_7, scale_0145_7);\n iacc_mat_01_7 = _mm256_madd_epi16(iacc_mat_01_7, scale_2367_7);\n iacc_mat_10_7 = _mm256_madd_epi16(iacc_mat_10_7, scale_0145_7);\n iacc_mat_11_7 = _mm256_madd_epi16(iacc_mat_11_7, scale_2367_7);\n\n __m256i iacc_mat_00 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm256_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm256_add_epi32(iacc_mat_00_6, iacc_mat_00_7)));\n __m256i iacc_mat_01 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm256_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm256_add_epi32(iacc_mat_01_6, iacc_mat_01_7)));\n __m256i iacc_mat_10 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm256_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm256_add_epi32(iacc_mat_10_6, iacc_mat_10_7)));\n __m256i iacc_mat_11 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm256_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm256_add_epi32(iacc_mat_11_6, iacc_mat_11_7)));\n\n // Straighten out to make 4 row vectors\n __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204);\n __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204);\n __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204);\n __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204);\n\n // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d);\n const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n\n // Multiply with appropriate scales and accumulate (for both d and dmin) below\n acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n\n __m256i lhs_bsums_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1);\n __m256i lhs_bsums_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1);\n __m256i lhs_bsums_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1);\n __m256i lhs_bsums_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1);\n\n // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K\n __m256i iacc_row_min_0_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 0), mins_01);\n __m256i iacc_row_min_1_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 170), mins_01);\n __m256i iacc_row_min_2_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 0), mins_01);\n __m256i iacc_row_min_3_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 170), mins_01);\n\n __m256i iacc_row_min_0_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 85), mins_23);\n __m256i iacc_row_min_1_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 255), mins_23);\n __m256i iacc_row_min_2_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 85), mins_23);\n __m256i iacc_row_min_3_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 255), mins_23);\n\n __m256i iacc_row_min_0_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 0), mins_45);\n __m256i iacc_row_min_1_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 170), mins_45);\n __m256i iacc_row_min_2_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 0), mins_45);\n __m256i iacc_row_min_3_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 170), mins_45);\n\n __m256i iacc_row_min_0_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 85), mins_67);\n __m256i iacc_row_min_1_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 255), mins_67);\n __m256i iacc_row_min_2_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 85), mins_67);\n __m256i iacc_row_min_3_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 255), mins_67);\n\n __m256i iacc_row_min_0 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm256_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67));\n __m256i iacc_row_min_1 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm256_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67));\n __m256i iacc_row_min_2 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm256_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67));\n __m256i iacc_row_min_3 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm256_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67));\n\n acc_min_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]);\n acc_min_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]);\n acc_min_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]);\n acc_min_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]);\n }\n }\n // Store the accumulated values\n for (int i = 0; i < 4; i++) {\n _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i]));\n }\n }\n }\n#else\n\n ggml_gemm_q2_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n\n\n#endif\n}\n" }, { "path": "src/ggml-cpu/arch-fallback.h", "content": "\n#pragma once\n\n// Rename `_generic` functions if no native implementation is available.\n// This effectively selects the generic implementation.\n\n#if defined(GGML_CPU_GENERIC)\n// quants.c\n#define quantize_row_q8_0_generic quantize_row_q8_0\n#define quantize_row_q8_1_generic quantize_row_q8_1\n#define quantize_row_q8_K_generic quantize_row_q8_K\n#define ggml_vec_dot_q4_0_q8_0_generic ggml_vec_dot_q4_0_q8_0\n#define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1\n#define ggml_vec_dot_q5_0_q8_0_generic ggml_vec_dot_q5_0_q8_0\n#define ggml_vec_dot_q5_1_q8_1_generic ggml_vec_dot_q5_1_q8_1\n#define ggml_vec_dot_q8_0_q8_0_generic ggml_vec_dot_q8_0_q8_0\n#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K\n#define ggml_vec_dot_q3_K_q8_K_generic ggml_vec_dot_q3_K_q8_K\n#define ggml_vec_dot_q4_K_q8_K_generic ggml_vec_dot_q4_K_q8_K\n#define ggml_vec_dot_q5_K_q8_K_generic ggml_vec_dot_q5_K_q8_K\n#define ggml_vec_dot_q6_K_q8_K_generic ggml_vec_dot_q6_K_q8_K\n#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K\n#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K\n#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K\n#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K\n#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K\n#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0\n#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)\n// repack.cpp\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)\n// quants.c\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__POWERPC__) || defined(__powerpc__)\n// ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679\n// quants.c\n#define quantize_row_q8_K_generic quantize_row_q8_K\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__loongarch64)\n// quants.c\n#define quantize_row_q8_K_generic quantize_row_q8_K\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__riscv)\n// quants.c\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_K_4x1_generic ggml_quantize_mat_q8_K_4x1\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__s390x__)\n// quants.c\n#define quantize_row_q8_K_generic quantize_row_q8_K\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K\n#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K\n#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K\n#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K\n#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K\n#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K\n#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__wasm__)\n// quants.c\n#define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K\n#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K\n#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K\n#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K\n#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K\n#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0\n#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K\n#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#endif\n" }, { "path": "src/ggml-cpu/binary-ops.cpp", "content": "#include \"binary-ops.h\"\n\n#if defined(GGML_USE_ACCELERATE)\n#include \n\nusing vDSP_fn_t = void (*)(const float *, vDSP_Stride, const float *, vDSP_Stride, float *, vDSP_Stride, vDSP_Length);\n#endif\n\nstatic inline float op_add(float a, float b) {\n return a + b;\n}\n\nstatic inline float op_sub(float a, float b) {\n return a - b;\n}\n\nstatic inline float op_mul(float a, float b) {\n return a * b;\n}\n\nstatic inline float op_div(float a, float b) {\n return a / b;\n}\n\ntemplate \nstatic inline void vec_binary_op_contiguous(const int64_t n, dst_t * z, const src0_t * x, const src1_t * y) {\n constexpr auto src0_to_f32 = type_conversion_table::to_f32;\n constexpr auto src1_to_f32 = type_conversion_table::to_f32;\n constexpr auto f32_to_dst = type_conversion_table::from_f32;\n\n for (int i = 0; i < n; i++) {\n z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(y[i])));\n }\n}\n\ntemplate \nstatic inline void vec_binary_op_non_contiguous(const int64_t n, const int64_t ne10, const int64_t nb10, dst_t * z, const src0_t * x, const src1_t * y) {\n constexpr auto src0_to_f32 = type_conversion_table::to_f32;\n constexpr auto src1_to_f32 = type_conversion_table::to_f32;\n constexpr auto f32_to_dst = type_conversion_table::from_f32;\n\n for (int i = 0; i < n; i++) {\n int i10 = i % ne10;\n const src1_t * y_ptr = (const src1_t *)((const char *)y + i10*nb10);\n z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(*y_ptr)));\n }\n}\n\ntemplate \nstatic void apply_binary_op(const ggml_compute_params * params, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n GGML_ASSERT( nb0 == sizeof(dst_t));\n GGML_ASSERT(nb00 == sizeof(src0_t));\n\n const auto [ir0, ir1] = get_thread_range(params, src0);\n const bool is_src1_contiguous_rows = ggml_is_contiguous_rows(src1);\n\n#ifdef GGML_USE_ACCELERATE\n vDSP_fn_t vDSP_op = nullptr;\n // TODO - avoid the f32-only check using type 'trait' lookup tables and row-based src-to-float conversion functions\n if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n if (op == op_add) {\n vDSP_op = vDSP_vadd;\n } else if (op == op_sub) {\n vDSP_op = vDSP_vsub;\n } else if (op == op_mul) {\n vDSP_op = vDSP_vmul;\n } else if (op == op_div) {\n vDSP_op = vDSP_vdiv;\n }\n }\n#endif\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir/(ne02*ne01);\n const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n const int64_t i13 = i03 % ne13;\n const int64_t i12 = i02 % ne12;\n const int64_t i11 = i01 % ne11;\n\n dst_t * dst_ptr = (dst_t *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );\n const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);\n const src1_t * src1_ptr = (const src1_t *) ((const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);\n\n if (is_src1_contiguous_rows) {\n // src1 is broadcastable across src0 and dst in i1, i2, i3\n const int64_t nr0 = ne00 / ne10;\n\n for (int64_t r = 0; r < nr0; ++r) {\n#ifdef GGML_USE_ACCELERATE\n if constexpr (std::is_same_v && std::is_same_v && std::is_same_v) {\n if (vDSP_op != nullptr) {\n vDSP_op(src1_ptr, 1, src0_ptr + r*ne10, 1, dst_ptr + r*ne10, 1, ne10);\n continue;\n }\n }\n#endif\n vec_binary_op_contiguous(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);\n }\n } else {\n vec_binary_op_non_contiguous(ne0, ne10, nb10, dst_ptr, src0_ptr, src1_ptr);\n }\n }\n}\n\n// TODO: Use the 'traits' lookup table (for type conversion fns), instead of a mass of 'if' conditions with long templates\ntemplate \nstatic void binary_op(const ggml_compute_params * params, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n /* */ if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { // all f32\n apply_binary_op(params, dst);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { // all f16\n apply_binary_op(params, dst);\n } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16\n apply_binary_op(params, dst);\n } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_BF16) {\n apply_binary_op(params, dst);\n } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n apply_binary_op(params, dst);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) {\n apply_binary_op(params, dst);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n apply_binary_op(params, dst);\n } else {\n GGML_ABORT(\"%s: unsupported types: dst: %s, src0: %s, src1: %s\\n\", __func__,\n ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));\n }\n}\n\nvoid ggml_compute_forward_add_non_quantized(const ggml_compute_params * params, ggml_tensor * dst) {\n binary_op(params, dst);\n}\n\nvoid ggml_compute_forward_sub(const ggml_compute_params * params, ggml_tensor * dst) {\n binary_op(params, dst);\n}\n\nvoid ggml_compute_forward_mul(const ggml_compute_params * params, ggml_tensor * dst) {\n binary_op(params, dst);\n}\n\nvoid ggml_compute_forward_div(const ggml_compute_params * params, ggml_tensor * dst) {\n binary_op(params, dst);\n}\n" }, { "path": "src/ggml-cpu/binary-ops.h", "content": "#pragma once\n\n#include \"common.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nvoid ggml_compute_forward_add_non_quantized(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sub(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_mul(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_div(const struct ggml_compute_params * params, struct ggml_tensor * dst);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cpu/cmake/FindSIMD.cmake", "content": "include(CheckCSourceRuns)\n\nset(AVX_CODE \"\n #include \n int main()\n {\n __m256 a;\n a = _mm256_set1_ps(0);\n return 0;\n }\n\")\n\nset(AVX512_CODE \"\n #include \n int main()\n {\n __m512i a = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0);\n __m512i b = a;\n __mmask64 equality_mask = _mm512_cmp_epi8_mask(a, b, _MM_CMPINT_EQ);\n return 0;\n }\n\")\n\nset(AVX2_CODE \"\n #include \n int main()\n {\n __m256i a = {0};\n a = _mm256_abs_epi16(a);\n __m256i x;\n _mm256_extract_epi64(x, 0); // we rely on this in our AVX2 code\n return 0;\n }\n\")\n\nset(FMA_CODE \"\n #include \n int main()\n {\n __m256 acc = _mm256_setzero_ps();\n const __m256 d = _mm256_setzero_ps();\n const __m256 p = _mm256_setzero_ps();\n acc = _mm256_fmadd_ps( d, p, acc );\n return 0;\n }\n\")\n\nmacro(check_sse type flags)\n set(__FLAG_I 1)\n set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})\n foreach (__FLAG ${flags})\n if (NOT ${type}_FOUND)\n set(CMAKE_REQUIRED_FLAGS ${__FLAG})\n check_c_source_runs(\"${${type}_CODE}\" HAS_${type}_${__FLAG_I})\n if (HAS_${type}_${__FLAG_I})\n set(${type}_FOUND TRUE CACHE BOOL \"${type} support\")\n set(${type}_FLAGS \"${__FLAG}\" CACHE STRING \"${type} flags\")\n endif()\n math(EXPR __FLAG_I \"${__FLAG_I}+1\")\n endif()\n endforeach()\n set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})\n\n if (NOT ${type}_FOUND)\n set(${type}_FOUND FALSE CACHE BOOL \"${type} support\")\n set(${type}_FLAGS \"\" CACHE STRING \"${type} flags\")\n endif()\n\n mark_as_advanced(${type}_FOUND ${type}_FLAGS)\nendmacro()\n\n# flags are for MSVC only!\ncheck_sse(\"AVX\" \" ;/arch:AVX\")\nif (NOT ${AVX_FOUND})\n set(GGML_AVX OFF)\nelse()\n set(GGML_AVX ON)\nendif()\n\ncheck_sse(\"AVX2\" \" ;/arch:AVX2\")\ncheck_sse(\"FMA\" \" ;/arch:AVX2\")\nif ((NOT ${AVX2_FOUND}) OR (NOT ${FMA_FOUND}))\n set(GGML_AVX2 OFF)\nelse()\n set(GGML_AVX2 ON)\nendif()\n\ncheck_sse(\"AVX512\" \" ;/arch:AVX512\")\nif (NOT ${AVX512_FOUND})\n set(GGML_AVX512 OFF)\nelse()\n set(GGML_AVX512 ON)\nendif()\n" }, { "path": "src/ggml-cpu/common.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"traits.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"ggml-impl.h\"\n#include \"simd-mappings.h\"\n\n#define GGML_FA_TILE_Q 64\n#define GGML_FA_TILE_KV 64\n\n#ifdef __cplusplus\n\n#include \n\n// convenience functions/macros for use in template calls\n// note: these won't be required after the 'traits' lookup table is used.\nstatic inline ggml_fp16_t f32_to_f16(float x) {\n return GGML_CPU_FP32_TO_FP16(x);\n}\n\nstatic inline float f16_to_f32(ggml_fp16_t x) {\n return GGML_CPU_FP16_TO_FP32(x);\n}\n\nstatic inline ggml_bf16_t f32_to_bf16(float x) {\n return GGML_FP32_TO_BF16(x);\n}\n\nstatic inline float bf16_to_f32(ggml_bf16_t x) {\n return GGML_BF16_TO_FP32(x);\n}\n\nstatic inline float i32_to_f32(int32_t x) {\n return x;\n}\n\nstatic inline int32_t f32_to_i32(float x) {\n return x;\n}\n\nstatic inline float f32_to_f32(float x) {\n return x;\n}\n\n// TODO - merge this into the traits table, after using row-based conversions\ntemplate \nstruct type_conversion_table;\n\ntemplate <>\nstruct type_conversion_table {\n static constexpr float (*to_f32)(ggml_fp16_t) = f16_to_f32;\n static constexpr ggml_fp16_t (*from_f32)(float) = f32_to_f16;\n};\n\ntemplate <>\nstruct type_conversion_table {\n static constexpr float (*to_f32)(float) = f32_to_f32;\n static constexpr float (*from_f32)(float) = f32_to_f32;\n};\n\ntemplate <>\nstruct type_conversion_table {\n static constexpr float (*to_f32)(ggml_bf16_t) = bf16_to_f32;\n static constexpr ggml_bf16_t (*from_f32)(float) = f32_to_bf16;\n};\n\ntemplate <>\nstruct type_conversion_table {\n static constexpr float (*to_f32)(int32_t) = i32_to_f32;\n static constexpr int32_t (*from_f32)(float) = f32_to_i32;\n};\n\nstatic std::pair get_thread_range(const struct ggml_compute_params * params, const struct ggml_tensor * src0) {\n const int64_t ith = params->ith;\n const int64_t nth = params->nth;\n\n const int64_t nr = ggml_nrows(src0);\n\n // rows per thread\n const int64_t dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int64_t ir0 = dr*ith;\n const int64_t ir1 = MIN(ir0 + dr, nr);\n\n return {ir0, ir1};\n}\n\nstruct ggml_fa_tile_config {\n static constexpr size_t Q = GGML_FA_TILE_Q;\n static constexpr size_t KV = GGML_FA_TILE_KV;\n};\n\n#endif\n" }, { "path": "src/ggml-cpu/ggml-cpu-impl.h", "content": "#pragma once\n\n// GGML CPU internal header\n\n#include \"ggml.h\"\n#include \"ggml-impl.h\"\n\n#include // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/\n//#include \n#include \n#include // memcpy\n#include // fabsf\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nstruct ggml_compute_params {\n // ith = thread index, nth = number of threads\n int ith, nth;\n\n // work buffer for all threads\n size_t wsize;\n void * wdata;\n\n struct ggml_threadpool * threadpool;\n\n // use reference implementation\n bool use_ref;\n};\n\n\n#if defined(_MSC_VER)\n\n#define m512bh(p) p\n#define m512i(p) p\n\n#else\n\n#define m512bh(p) (__m512bh)(p)\n#define m512i(p) (__m512i)(p)\n\n#endif\n\n// __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512\n#if defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))\n#ifndef __FMA__\n#define __FMA__\n#endif\n#ifndef __F16C__\n#define __F16C__\n#endif\n#endif\n\n// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available\n#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__))\n#ifndef __SSE3__\n#define __SSE3__\n#endif\n#ifndef __SSSE3__\n#define __SSSE3__\n#endif\n#endif\n\n#if defined(__s390x__) && defined(__VEC__)\n#ifndef __VXE__\n#define __VXE__\n#endif // __VXE__\n#ifndef __VXE2__\n#define __VXE2__\n#endif // __VXE2__\n#endif // __s390x__ && __VEC__\n\n#if defined(__ARM_FEATURE_SVE) && defined(__linux__)\n#include \n#endif\n\n#if defined(__ARM_NEON)\n\n// ref: https://github.com/ggml-org/llama.cpp/pull/5404\n#ifdef _MSC_VER\n#define ggml_vld1q_u32(w,x,y,z) { ((w) + ((uint64_t)(x) << 32)), ((y) + ((uint64_t)(z) << 32)) }\n#else\n#define ggml_vld1q_u32(w,x,y,z) { (w), (x), (y), (z) }\n#endif // _MSC_VER\n\n#if !defined(__aarch64__)\n\n// 32-bit ARM compatibility\n\n// vaddlvq_s16\n// vpaddq_s16\n// vpaddq_s32\n// vaddvq_s32\n// vaddvq_f32\n// vmaxvq_f32\n// vcvtnq_s32_f32\n// vzip1_u8\n// vzip2_u8\n\ninline static int32_t vaddlvq_s16(int16x8_t v) {\n int32x4_t v0 = vreinterpretq_s32_s64(vpaddlq_s32(vpaddlq_s16(v)));\n return vgetq_lane_s32(v0, 0) + vgetq_lane_s32(v0, 2);\n}\n\ninline static int16x8_t vpaddq_s16(int16x8_t a, int16x8_t b) {\n int16x4_t a0 = vpadd_s16(vget_low_s16(a), vget_high_s16(a));\n int16x4_t b0 = vpadd_s16(vget_low_s16(b), vget_high_s16(b));\n return vcombine_s16(a0, b0);\n}\n\ninline static int32x4_t vpaddq_s32(int32x4_t a, int32x4_t b) {\n int32x2_t a0 = vpadd_s32(vget_low_s32(a), vget_high_s32(a));\n int32x2_t b0 = vpadd_s32(vget_low_s32(b), vget_high_s32(b));\n return vcombine_s32(a0, b0);\n}\n\ninline static int32_t vaddvq_s32(int32x4_t v) {\n return vgetq_lane_s32(v, 0) + vgetq_lane_s32(v, 1) + vgetq_lane_s32(v, 2) + vgetq_lane_s32(v, 3);\n}\n\ninline static float vaddvq_f32(float32x4_t v) {\n return vgetq_lane_f32(v, 0) + vgetq_lane_f32(v, 1) + vgetq_lane_f32(v, 2) + vgetq_lane_f32(v, 3);\n}\n\ninline static float vmaxvq_f32(float32x4_t v) {\n return\n MAX(MAX(vgetq_lane_f32(v, 0), vgetq_lane_f32(v, 1)),\n MAX(vgetq_lane_f32(v, 2), vgetq_lane_f32(v, 3)));\n}\n\ninline static int32x4_t vcvtnq_s32_f32(float32x4_t v) {\n int32x4_t res;\n\n res[0] = roundf(vgetq_lane_f32(v, 0));\n res[1] = roundf(vgetq_lane_f32(v, 1));\n res[2] = roundf(vgetq_lane_f32(v, 2));\n res[3] = roundf(vgetq_lane_f32(v, 3));\n\n return res;\n}\n\ninline static uint8x8_t vzip1_u8(uint8x8_t a, uint8x8_t b) {\n uint8x8_t res;\n\n res[0] = a[0]; res[1] = b[0];\n res[2] = a[1]; res[3] = b[1];\n res[4] = a[2]; res[5] = b[2];\n res[6] = a[3]; res[7] = b[3];\n\n return res;\n}\n\ninline static uint8x8_t vzip2_u8(uint8x8_t a, uint8x8_t b) {\n uint8x8_t res;\n\n res[0] = a[4]; res[1] = b[4];\n res[2] = a[5]; res[3] = b[5];\n res[4] = a[6]; res[5] = b[6];\n res[6] = a[7]; res[7] = b[7];\n\n return res;\n}\n\n// vld1q_s16_x2\n// vld1q_u8_x2\n// vld1q_u8_x4\n// vld1q_s8_x2\n// vld1q_s8_x4\n// TODO: double-check these work correctly\n\ntypedef struct ggml_int16x8x2_t {\n int16x8_t val[2];\n} ggml_int16x8x2_t;\n\ninline static ggml_int16x8x2_t ggml_vld1q_s16_x2(const int16_t * ptr) {\n ggml_int16x8x2_t res;\n\n res.val[0] = vld1q_s16(ptr + 0);\n res.val[1] = vld1q_s16(ptr + 8);\n\n return res;\n}\n\ntypedef struct ggml_uint8x16x2_t {\n uint8x16_t val[2];\n} ggml_uint8x16x2_t;\n\ninline static ggml_uint8x16x2_t ggml_vld1q_u8_x2(const uint8_t * ptr) {\n ggml_uint8x16x2_t res;\n\n res.val[0] = vld1q_u8(ptr + 0);\n res.val[1] = vld1q_u8(ptr + 16);\n\n return res;\n}\n\ntypedef struct ggml_uint8x16x4_t {\n uint8x16_t val[4];\n} ggml_uint8x16x4_t;\n\ninline static ggml_uint8x16x4_t ggml_vld1q_u8_x4(const uint8_t * ptr) {\n ggml_uint8x16x4_t res;\n\n res.val[0] = vld1q_u8(ptr + 0);\n res.val[1] = vld1q_u8(ptr + 16);\n res.val[2] = vld1q_u8(ptr + 32);\n res.val[3] = vld1q_u8(ptr + 48);\n\n return res;\n}\n\ntypedef struct ggml_int8x16x2_t {\n int8x16_t val[2];\n} ggml_int8x16x2_t;\n\ninline static ggml_int8x16x2_t ggml_vld1q_s8_x2(const int8_t * ptr) {\n ggml_int8x16x2_t res;\n\n res.val[0] = vld1q_s8(ptr + 0);\n res.val[1] = vld1q_s8(ptr + 16);\n\n return res;\n}\n\ntypedef struct ggml_int8x16x4_t {\n int8x16_t val[4];\n} ggml_int8x16x4_t;\n\ninline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) {\n ggml_int8x16x4_t res;\n\n res.val[0] = vld1q_s8(ptr + 0);\n res.val[1] = vld1q_s8(ptr + 16);\n res.val[2] = vld1q_s8(ptr + 32);\n res.val[3] = vld1q_s8(ptr + 48);\n\n return res;\n}\n\n// NOTE: not tested\ninline static int8x16_t ggml_vqtbl1q_s8(int8x16_t a, uint8x16_t b) {\n int8x16_t res;\n\n res[ 0] = a[b[ 0]];\n res[ 1] = a[b[ 1]];\n res[ 2] = a[b[ 2]];\n res[ 3] = a[b[ 3]];\n res[ 4] = a[b[ 4]];\n res[ 5] = a[b[ 5]];\n res[ 6] = a[b[ 6]];\n res[ 7] = a[b[ 7]];\n res[ 8] = a[b[ 8]];\n res[ 9] = a[b[ 9]];\n res[10] = a[b[10]];\n res[11] = a[b[11]];\n res[12] = a[b[12]];\n res[13] = a[b[13]];\n res[14] = a[b[14]];\n res[15] = a[b[15]];\n\n return res;\n}\n\n// NOTE: not tested\ninline static uint8x16_t ggml_vqtbl1q_u8(uint8x16_t a, uint8x16_t b) {\n uint8x16_t res;\n\n res[ 0] = a[b[ 0]];\n res[ 1] = a[b[ 1]];\n res[ 2] = a[b[ 2]];\n res[ 3] = a[b[ 3]];\n res[ 4] = a[b[ 4]];\n res[ 5] = a[b[ 5]];\n res[ 6] = a[b[ 6]];\n res[ 7] = a[b[ 7]];\n res[ 8] = a[b[ 8]];\n res[ 9] = a[b[ 9]];\n res[10] = a[b[10]];\n res[11] = a[b[11]];\n res[12] = a[b[12]];\n res[13] = a[b[13]];\n res[14] = a[b[14]];\n res[15] = a[b[15]];\n\n return res;\n}\n\n#else\n\n#define ggml_int16x8x2_t int16x8x2_t\n#define ggml_uint8x16x2_t uint8x16x2_t\n#define ggml_uint8x16x4_t uint8x16x4_t\n#define ggml_int8x16x2_t int8x16x2_t\n#define ggml_int8x16x4_t int8x16x4_t\n\n#define ggml_vld1q_s16_x2 vld1q_s16_x2\n#define ggml_vld1q_u8_x2 vld1q_u8_x2\n#define ggml_vld1q_u8_x4 vld1q_u8_x4\n#define ggml_vld1q_s8_x2 vld1q_s8_x2\n#define ggml_vld1q_s8_x4 vld1q_s8_x4\n#define ggml_vqtbl1q_s8 vqtbl1q_s8\n#define ggml_vqtbl1q_u8 vqtbl1q_u8\n\n#endif // !defined(__aarch64__)\n\n#if !defined(__ARM_FEATURE_DOTPROD)\n\ninline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) {\n const int16x8_t p0 = vmull_s8(vget_low_s8 (a), vget_low_s8 (b));\n const int16x8_t p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b));\n\n return vaddq_s32(acc, vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1)));\n}\n\n#else\n\n#define ggml_vdotq_s32(a, b, c) vdotq_s32(a, b, c)\n\n#endif // !defined(__ARM_FEATURE_DOTPROD)\n\n#endif // defined(__ARM_NEON)\n\n#ifdef __wasm_simd128__\n#include \n#endif\n\n#ifdef __POWER9_VECTOR__\n#include \n#endif\n\n#if defined(_MSC_VER) || defined(__MINGW32__)\n#include \n#elif defined(__SSE__) || defined(__SSE3__) || defined(__SSSE3__) || defined(__AVX__) || defined(__F16C__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX512BF16__)\n#include \n#endif\n\n#ifdef __riscv_v_intrinsic\n#include \n#endif\n\n#if defined(__loongarch64)\n#if defined(__loongarch_asx)\n#include \n#endif\n#if defined(__loongarch_sx)\n#include \n#endif\n#endif\n\n#if defined(__VXE__) || defined(__VXE2__)\n#include \n\n#define vec_neg(a) (-(a)) // Vector Negate\n#define vec_add(a, b) ((a) + (b)) // Vector Add\n#define vec_sub(a, b) ((a) - (b)) // Vector Subtract\n#define vec_mul(a, b) ((a) * (b)) // Vector Multiply\n#define vec_div(a, b) ((a) / (b)) // Vector Divide\n#define vec_sl(a, b) ((a) << (b)) // Vector Shift Left\n#define vec_sra(a, b) ((a) >> (b)) // Vector Shift Right\n#define vec_sr(a, b) ((a) >> (b)) // Vector Shift Right Algebraic\n#define vec_slo(a, b) vec_slb(a, (b) << 64) // Vector Shift Left by Octet\n#define vec_sro(a, b) vec_srb(a, (b) << 64) // Vector Shift Right by Octet\n\n#ifndef vec_and\n#define vec_and(a, b) ((a) & (b)) // Vector AND\n#endif\n\n#ifndef vec_or\n#define vec_or(a, b) ((a) | (b)) // Vector OR\n#endif\n\n#ifndef vec_xor\n#define vec_xor(a, b) ((a) ^ (b)) // Vector XOR\n#endif\n\ntypedef signed char char8x16_t __attribute__((vector_size(16)));\ntypedef unsigned char uchar8x16_t __attribute__((vector_size(16)));\n\ntypedef int8_t int8x16_t __attribute__((vector_size(16)));\ntypedef int16_t int16x8_t __attribute__((vector_size(16)));\ntypedef int32_t int32x4_t __attribute__((vector_size(16)));\n\ntypedef uint8_t uint8x16_t __attribute__((vector_size(16)));\ntypedef uint16_t uint16x8_t __attribute__((vector_size(16)));\ntypedef uint32_t uint32x4_t __attribute__((vector_size(16)));\n\ntypedef float float32x4_t __attribute__((vector_size(16)));\ntypedef double double64x2_t __attribute__((vector_size(16)));\n\ntypedef signed long long long64x2_t __attribute__((vector_size(16)));\ntypedef unsigned long long ulong64x2_t __attribute__((vector_size(16)));\n\ntypedef struct ggml_uint8x16x2_t {\n uint8x16_t val[2];\n} ggml_uint8x16x2_t;\n\ninline static ggml_uint8x16x2_t ggml_vec_xl_u8x2(const uint8_t * ptr) {\n ggml_uint8x16x2_t res;\n\n res.val[0] = vec_xl( 0, ptr);\n res.val[1] = vec_xl(16, ptr);\n\n return res;\n}\n\ntypedef struct ggml_uint8x16x4_t {\n uint8x16_t val[4];\n} ggml_uint8x16x4_t;\n\ninline static ggml_uint8x16x4_t ggml_vec_xl_u8x4(const uint8_t * ptr) {\n ggml_uint8x16x4_t res;\n\n res.val[0] = vec_xl( 0, ptr);\n res.val[1] = vec_xl(16, ptr);\n res.val[2] = vec_xl(32, ptr);\n res.val[3] = vec_xl(48, ptr);\n\n return res;\n}\n\ntypedef struct ggml_int8x16x4_t {\n int8x16_t val[4];\n} ggml_int8x16x4_t;\n\ninline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) {\n ggml_int8x16x4_t res;\n\n res.val[0] = vec_xl( 0, ptr);\n res.val[1] = vec_xl(16, ptr);\n res.val[2] = vec_xl(32, ptr);\n res.val[3] = vec_xl(48, ptr);\n\n return res;\n}\n\ntypedef struct ggml_int16x8x2_t {\n int16x8_t val[2];\n} ggml_int16x8x2_t;\n\ninline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) {\n ggml_int16x8x2_t res;\n\n res.val[0] = vec_xl( 0, ptr);\n res.val[1] = vec_xl(16, ptr);\n\n return res;\n}\n\n/*\n ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs\n ! or iq4_nl for example implementation.\n*/\ninline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) {\n int8x16_t res;\n\n res[ 0] = a[b[ 0]];\n res[ 1] = a[b[ 1]];\n res[ 2] = a[b[ 2]];\n res[ 3] = a[b[ 3]];\n res[ 4] = a[b[ 4]];\n res[ 5] = a[b[ 5]];\n res[ 6] = a[b[ 6]];\n res[ 7] = a[b[ 7]];\n res[ 8] = a[b[ 8]];\n res[ 9] = a[b[ 9]];\n res[10] = a[b[10]];\n res[11] = a[b[11]];\n res[12] = a[b[12]];\n res[13] = a[b[13]];\n res[14] = a[b[14]];\n res[15] = a[b[15]];\n\n return res;\n}\n\ninline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) {\n const uchar8x16_t v_maske = { 0, 1, 4, 5, 8, 9, 12, 13,\n 16, 17, 20, 21, 24, 25, 28, 29 };\n\n const int16x8_t v_abo = vec_pack((int32x4_t)a, (int32x4_t)b);\n const int16x8_t v_abe = vec_perm(a, b, v_maske);\n return v_abo + v_abe;\n}\n\n/**\n * @see https://github.com/ggml-org/llama.cpp/pull/14037\n */\ninline static float vec_hsum_f32x4(float32x4_t v) {\n float32x4_t v_temp = v + vec_reve(v);\n return v_temp[0] + v_temp[1];\n}\n\ninline static int32_t vec_hsum_i32x4(int32x4_t v) {\n int32x4_t v_temp = v + vec_reve(v);\n return v_temp[0] + v_temp[1];\n}\n\ninline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) {\n const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b);\n return acc + (vec_unpackh(p) + vec_unpackl(p));\n}\n\n#endif\n\n#if defined(__loongarch_sx)\n/* float type data load instructions */\nstatic __m128 __lsx_vreplfr2vr_s(const float val) {\n v4f32 res = {val, val, val, val};\n return (__m128)res;\n}\n#endif\n\n#if defined(__loongarch_asx)\nstatic __m256 __lasx_xvreplfr2vr_s(const float val) {\n v8f32 res = {val, val, val, val, val, val, val, val};\n return (__m256)res;\n}\n#endif\n\n// TODO: move to ggml-threading\nvoid ggml_barrier(struct ggml_threadpool * tp);\n\nvoid ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value);\nint ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cpu/ggml-cpu.c", "content": "#define _CRT_SECURE_NO_DEPRECATE // Disables \"unsafe\" warnings on Windows\n#define _USE_MATH_DEFINES // For M_PI on MSVC\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n#include \"traits.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"ggml-impl.h\"\n#include \"quants.h\"\n#include \"ggml-threading.h\"\n#include \"unary-ops.h\"\n#include \"binary-ops.h\"\n#include \"vec.h\"\n#include \"ops.h\"\n#include \"ggml.h\"\n#include \"common.h\"\n\n#if defined(_MSC_VER) || defined(__MINGW32__)\n#include // using malloc.h with MSC/MINGW\n#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)\n#include \n#endif\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#if defined(__gnu_linux__)\n#include \n#endif\n\n#ifdef GGML_USE_OPENMP\n#include \n#endif\n\n#if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8)\n#undef GGML_USE_LLAMAFILE\n#endif\n\n#ifdef GGML_USE_LLAMAFILE\n#include \"llamafile/sgemm.h\"\n#endif\n\n// Note: once we move threading into a separate C++ file\n// will use std::hardware_destructive_interference_size instead of hardcoding it here\n// and we'll use C++ attribute syntax.\n#define GGML_CACHE_LINE 64\n\n#if defined(__clang__) || defined(__GNUC__)\n#define GGML_CACHE_ALIGN __attribute__((aligned(GGML_CACHE_LINE)))\n#endif\n\n#if defined(__has_feature)\n#if __has_feature(thread_sanitizer)\n#define GGML_TSAN_ENABLED 1\n#endif\n#else // __has_feature\n#if defined(__SANITIZE_THREAD__)\n#define GGML_TSAN_ENABLED 1\n#endif\n#endif // __has_feature\n\n#define UNUSED GGML_UNUSED\n#define SWAP(x, y, T) do { T SWAP = x; (x) = y; (y) = SWAP; } while (0)\n\n// precomputed f32 table for f16 (256 KB) (simd-mappings.h)\nfloat ggml_table_f32_f16[1 << 16];\n\n// precomputed f32 table for e8m0 half (1 KB) (simd-mappings.h)\nfloat ggml_table_f32_e8m0_half[1 << 8];\n\n#if defined(__ARM_ARCH)\nstruct ggml_arm_arch_features_type {\n int sve_cnt;\n} ggml_arm_arch_features = { 0 };\n#endif\n\n#if defined(__riscv)\nstruct ggml_riscv_arch_features_type {\n int rvv_vlen;\n} ggml_riscv_arch_features = { 0 };\n#endif\n\n#if defined(_WIN32)\n\n#define WIN32_LEAN_AND_MEAN\n#ifndef NOMINMAX\n #define NOMINMAX\n#endif\n#include \n\n#if defined(_MSC_VER) && !defined(__clang__)\n#define GGML_CACHE_ALIGN __declspec(align(GGML_CACHE_LINE))\n\ntypedef volatile LONG atomic_int;\ntypedef atomic_int atomic_bool;\ntypedef atomic_int atomic_flag;\n\n#define ATOMIC_FLAG_INIT 0\n\ntypedef enum {\n memory_order_relaxed,\n memory_order_consume,\n memory_order_acquire,\n memory_order_release,\n memory_order_acq_rel,\n memory_order_seq_cst\n} memory_order;\n\nstatic void atomic_store(atomic_int * ptr, LONG val) {\n InterlockedExchange(ptr, val);\n}\nstatic void atomic_store_explicit(atomic_int * ptr, LONG val, memory_order mo) {\n // TODO: add support for explicit memory order\n InterlockedExchange(ptr, val);\n}\nstatic LONG atomic_load(atomic_int * ptr) {\n return InterlockedCompareExchange(ptr, 0, 0);\n}\nstatic LONG atomic_load_explicit(atomic_int * ptr, memory_order mo) {\n // TODO: add support for explicit memory order\n return InterlockedCompareExchange(ptr, 0, 0);\n}\nstatic LONG atomic_fetch_add(atomic_int * ptr, LONG inc) {\n return InterlockedExchangeAdd(ptr, inc);\n}\nstatic LONG atomic_fetch_add_explicit(atomic_int * ptr, LONG inc, memory_order mo) {\n // TODO: add support for explicit memory order\n return InterlockedExchangeAdd(ptr, inc);\n}\nstatic atomic_bool atomic_flag_test_and_set(atomic_flag * ptr) {\n return InterlockedExchange(ptr, 1);\n}\nstatic void atomic_flag_clear(atomic_flag * ptr) {\n InterlockedExchange(ptr, 0);\n}\nstatic void atomic_thread_fence(memory_order mo) {\n MemoryBarrier();\n}\n#else // clang\n#include \n#endif\n\ntypedef HANDLE pthread_t;\n\ntypedef DWORD thread_ret_t;\nstatic int pthread_create(pthread_t * out, void * unused, thread_ret_t(*func)(void *), void * arg) {\n (void) unused;\n HANDLE handle = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL);\n if (handle == NULL)\n {\n return EAGAIN;\n }\n\n *out = handle;\n return 0;\n}\n\nstatic int pthread_join(pthread_t thread, void * unused) {\n (void) unused;\n int ret = (int) WaitForSingleObject(thread, INFINITE);\n CloseHandle(thread);\n return ret;\n}\n\nstatic int sched_yield (void) {\n Sleep (0);\n return 0;\n}\n#else\n\n#include \n#include \n#include \n#if defined(__FreeBSD__)\n#include \n#endif\n\ntypedef void * thread_ret_t;\n\n#include \n#include \n#include \n\n#endif\n\ntypedef pthread_t ggml_thread_t;\n\n#define GGML_THREADPOOL_N_THREADS_MASK (0xffffU)\n#define GGML_THREADPOOL_N_THREADS_BITS (16)\n\n#if defined(__APPLE__)\n#include \n#include \n#include \n#endif\n\nstatic const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {\n [GGML_TYPE_F32] = {\n .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp32,\n .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f32,\n .vec_dot_type = GGML_TYPE_F32,\n .nrows = 1,\n },\n [GGML_TYPE_F16] = {\n .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp16,\n .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16,\n .vec_dot_type = GGML_TYPE_F16,\n .nrows = 1,\n },\n [GGML_TYPE_Q4_0] = {\n .from_float = quantize_row_q4_0,\n .vec_dot = ggml_vec_dot_q4_0_q8_0,\n .vec_dot_type = GGML_TYPE_Q8_0,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n .nrows = 2,\n#else\n .nrows = 1,\n#endif\n },\n [GGML_TYPE_Q4_1] = {\n .from_float = quantize_row_q4_1,\n .vec_dot = ggml_vec_dot_q4_1_q8_1,\n .vec_dot_type = GGML_TYPE_Q8_1,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n .nrows = 2,\n#else\n .nrows = 1,\n#endif\n },\n [GGML_TYPE_Q5_0] = {\n .from_float = quantize_row_q5_0,\n .vec_dot = ggml_vec_dot_q5_0_q8_0,\n .vec_dot_type = GGML_TYPE_Q8_0,\n .nrows = 1,\n },\n [GGML_TYPE_Q5_1] = {\n .from_float = quantize_row_q5_1,\n .vec_dot = ggml_vec_dot_q5_1_q8_1,\n .vec_dot_type = GGML_TYPE_Q8_1,\n .nrows = 1,\n },\n [GGML_TYPE_Q8_0] = {\n .from_float = quantize_row_q8_0,\n .vec_dot = ggml_vec_dot_q8_0_q8_0,\n .vec_dot_type = GGML_TYPE_Q8_0,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n .nrows = 2,\n#else\n .nrows = 1,\n#endif\n },\n [GGML_TYPE_Q8_1] = {\n .from_float = quantize_row_q8_1,\n .vec_dot_type = GGML_TYPE_Q8_1,\n .nrows = 1,\n },\n [GGML_TYPE_MXFP4] = {\n .from_float = quantize_row_mxfp4,\n .vec_dot = ggml_vec_dot_mxfp4_q8_0,\n .vec_dot_type = GGML_TYPE_Q8_0,\n .nrows = 1,\n },\n [GGML_TYPE_NVFP4] = {\n .from_float = quantize_row_nvfp4,\n .vec_dot = ggml_vec_dot_nvfp4_q8_0,\n .vec_dot_type = GGML_TYPE_Q8_0,\n .nrows = 1,\n },\n [GGML_TYPE_Q2_K] = {\n .from_float = quantize_row_q2_K,\n .vec_dot = ggml_vec_dot_q2_K_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_Q3_K] = {\n .from_float = quantize_row_q3_K,\n .vec_dot = ggml_vec_dot_q3_K_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_Q4_K] = {\n .from_float = quantize_row_q4_K,\n .vec_dot = ggml_vec_dot_q4_K_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n .nrows = 2,\n#else\n .nrows = 1,\n#endif\n },\n [GGML_TYPE_Q5_K] = {\n .from_float = quantize_row_q5_K,\n .vec_dot = ggml_vec_dot_q5_K_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_Q6_K] = {\n .from_float = quantize_row_q6_K,\n .vec_dot = ggml_vec_dot_q6_K_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n .nrows = 2,\n#else\n .nrows = 1,\n#endif\n },\n [GGML_TYPE_IQ2_XXS] = {\n .from_float = NULL,\n .vec_dot = ggml_vec_dot_iq2_xxs_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_IQ2_XS] = {\n .from_float = NULL,\n .vec_dot = ggml_vec_dot_iq2_xs_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_IQ3_XXS] = {\n // NOTE: from_float for iq3 and iq2_s was removed because these quants require initialization in ggml_quantize_init\n //.from_float = quantize_row_iq3_xxs,\n .vec_dot = ggml_vec_dot_iq3_xxs_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_IQ3_S] = {\n //.from_float = quantize_row_iq3_s,\n .vec_dot = ggml_vec_dot_iq3_s_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_IQ2_S] = {\n //.from_float = quantize_row_iq2_s,\n .vec_dot = ggml_vec_dot_iq2_s_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_IQ1_S] = {\n .from_float = NULL,\n .vec_dot = ggml_vec_dot_iq1_s_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_IQ1_M] = {\n .from_float = NULL,\n .vec_dot = ggml_vec_dot_iq1_m_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_IQ4_NL] = {\n .from_float = quantize_row_iq4_nl,\n .vec_dot = ggml_vec_dot_iq4_nl_q8_0,\n .vec_dot_type = GGML_TYPE_Q8_0,\n .nrows = 1,\n },\n [GGML_TYPE_IQ4_XS] = {\n .from_float = quantize_row_iq4_xs,\n .vec_dot = ggml_vec_dot_iq4_xs_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_Q8_K] = {\n .from_float = quantize_row_q8_K,\n },\n [GGML_TYPE_BF16] = {\n .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_bf16,\n .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16,\n .vec_dot_type = GGML_TYPE_BF16,\n .nrows = 1,\n },\n [GGML_TYPE_TQ1_0] = {\n .from_float = quantize_row_tq1_0,\n .vec_dot = ggml_vec_dot_tq1_0_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_TQ2_0] = {\n .from_float = quantize_row_tq2_0,\n .vec_dot = ggml_vec_dot_tq2_0_q8_K,\n .vec_dot_type = GGML_TYPE_Q8_K,\n .nrows = 1,\n },\n [GGML_TYPE_I32] = {\n .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32,\n },\n};\n\nconst struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) {\n return &type_traits_cpu[type];\n}\n\n//\n// Threading defs\n//\n\ntypedef pthread_t ggml_thread_t;\n\n#if defined(_WIN32)\n\ntypedef CONDITION_VARIABLE ggml_cond_t;\ntypedef SRWLOCK ggml_mutex_t;\n\n#define ggml_mutex_init(m) InitializeSRWLock(m)\n#define ggml_mutex_destroy(m)\n#define ggml_mutex_lock(m) AcquireSRWLockExclusive(m)\n#define ggml_mutex_unlock(m) ReleaseSRWLockExclusive(m)\n#define ggml_mutex_lock_shared(m) AcquireSRWLockShared(m)\n#define ggml_mutex_unlock_shared(m) ReleaseSRWLockShared(m)\n\n#define ggml_cond_init(c) InitializeConditionVariable(c)\n#define ggml_cond_destroy(c)\n#define ggml_cond_wait(c, m) SleepConditionVariableSRW(c, m, INFINITE, CONDITION_VARIABLE_LOCKMODE_SHARED)\n#define ggml_cond_broadcast(c) WakeAllConditionVariable(c)\n\n#define ggml_thread_create pthread_create\n#define ggml_thread_join pthread_join\n\n#else\n\ntypedef pthread_cond_t ggml_cond_t;\ntypedef pthread_mutex_t ggml_mutex_t;\n\n#define ggml_mutex_init(m) pthread_mutex_init(m, NULL)\n#define ggml_mutex_destroy(m) pthread_mutex_destroy(m)\n#define ggml_mutex_lock(m) pthread_mutex_lock(m)\n#define ggml_mutex_unlock(m) pthread_mutex_unlock(m)\n#define ggml_mutex_lock_shared(m) pthread_mutex_lock(m)\n#define ggml_mutex_unlock_shared(m) pthread_mutex_unlock(m)\n\n#define ggml_lock_init(x) UNUSED(x)\n#define ggml_lock_destroy(x) UNUSED(x)\n#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))\n#define ggml_lock_lock(x) _mm_pause()\n#else\n#define ggml_lock_lock(x) UNUSED(x)\n#endif\n#define ggml_lock_unlock(x) UNUSED(x)\n\n#define GGML_LOCK_INITIALIZER 0\n#define ggml_cond_init(c) pthread_cond_init(c, NULL)\n#define ggml_cond_destroy(c) pthread_cond_destroy(c)\n#define ggml_cond_wait(c, m) pthread_cond_wait(c, m)\n#define ggml_cond_broadcast(c) pthread_cond_broadcast(c)\n\n#define ggml_thread_create pthread_create\n#define ggml_thread_join pthread_join\n\n#endif\n\n// Threadpool def\nstruct ggml_threadpool {\n ggml_mutex_t mutex; // mutex for cond.var\n ggml_cond_t cond; // cond.var for waiting for new work\n\n struct ggml_cgraph * cgraph;\n struct ggml_cplan * cplan;\n\n // synchronization primitives\n atomic_int n_graph; // updated when there is work to be done (i.e each graph) holds graph and active thread counts.\n atomic_int GGML_CACHE_ALIGN n_barrier;\n atomic_int GGML_CACHE_ALIGN n_barrier_passed;\n atomic_int GGML_CACHE_ALIGN current_chunk; // currently processing chunk during Mat_Mul, shared between all the threads.\n\n // these are atomic as an annotation for thread-sanitizer\n atomic_bool stop; // Used for stopping the threadpool altogether\n atomic_bool pause; // Used for pausing the threadpool or individual threads\n atomic_int abort; // Used for aborting processing of a graph\n\n struct ggml_compute_state * workers; // per thread state\n int n_threads; // Number of threads in the pool\n int32_t prio; // Scheduling priority\n uint32_t poll; // Polling level (0 - no polling)\n\n enum ggml_status ec;\n};\n\n// Per-thread state\nstruct ggml_compute_state {\n#ifndef GGML_USE_OPENMP\n ggml_thread_t thrd;\n int last_graph;\n bool pending;\n#endif\n bool cpumask[GGML_MAX_N_THREADS];\n struct ggml_threadpool * threadpool;\n int ith;\n};\n\n// Helpers for polling loops\n#if defined(__aarch64__) && ( defined(__clang__) || defined(__GNUC__) )\nstatic inline void ggml_thread_cpu_relax(void) {\n __asm__ volatile(\"yield\" ::: \"memory\");\n}\n#elif defined(__x86_64__)\nstatic inline void ggml_thread_cpu_relax(void) {\n _mm_pause();\n}\n#elif defined(__riscv)\nstatic inline void ggml_thread_cpu_relax(void) {\n #ifdef __riscv_zihintpause\n __asm__ __volatile__ (\"pause\");\n #else\n /* Encoding of the pause instruction */\n __asm__ __volatile__ (\".4byte 0x100000F\");\n #endif\n}\n#else\nstatic inline void ggml_thread_cpu_relax(void) {;}\n#endif\n\n//\n// NUMA support\n//\n\n#define GGML_NUMA_MAX_NODES 8\n#define GGML_NUMA_MAX_CPUS 512\n\nstruct ggml_numa_node {\n uint32_t cpus[GGML_NUMA_MAX_CPUS]; // hardware threads on this node\n uint32_t n_cpus;\n};\n\nstruct ggml_numa_nodes {\n enum ggml_numa_strategy numa_strategy;\n struct ggml_numa_node nodes[GGML_NUMA_MAX_NODES];\n uint32_t n_nodes;\n uint32_t total_cpus; // hardware threads on system\n uint32_t current_node; // node on which main process is execting\n#if defined(__gnu_linux__)\n cpu_set_t cpuset; // cpuset from numactl\n#else\n uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype\n#endif\n};\n\n//\n// ggml state\n//\n\nstruct ggml_state {\n struct ggml_numa_nodes numa;\n};\n\nstatic struct ggml_state g_state = {0};\n\nvoid ggml_barrier(struct ggml_threadpool * tp) {\n int n_threads = atomic_load_explicit(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK;\n if (n_threads == 1) {\n return;\n }\n\n#ifdef GGML_USE_OPENMP\n #pragma omp barrier\n#else\n int n_passed = atomic_load_explicit(&tp->n_barrier_passed, memory_order_relaxed);\n\n // enter barrier (full seq-cst fence)\n int n_barrier = atomic_fetch_add_explicit(&tp->n_barrier, 1, memory_order_seq_cst);\n\n if (n_barrier == (n_threads - 1)) {\n // last thread\n atomic_store_explicit(&tp->n_barrier, 0, memory_order_relaxed);\n\n // exit barrier (full seq-cst fence)\n atomic_fetch_add_explicit(&tp->n_barrier_passed, 1, memory_order_seq_cst);\n return;\n }\n\n // wait for other threads\n while (atomic_load_explicit(&tp->n_barrier_passed, memory_order_relaxed) == n_passed) {\n ggml_thread_cpu_relax();\n }\n\n // exit barrier (full seq-cst fence)\n // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead\n #ifdef GGML_TSAN_ENABLED\n atomic_fetch_add_explicit(&tp->n_barrier_passed, 0, memory_order_seq_cst);\n #else\n atomic_thread_fence(memory_order_seq_cst);\n #endif\n#endif\n}\n\nvoid ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value) {\n atomic_store_explicit(&tp->current_chunk, value, memory_order_relaxed);\n}\n\nint ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value) {\n return atomic_fetch_add_explicit(&tp->current_chunk, value, memory_order_relaxed);\n}\n\n#if defined(__gnu_linux__)\nstatic cpu_set_t ggml_get_numa_affinity(void) {\n cpu_set_t cpuset;\n pthread_t thread;\n thread = pthread_self();\n CPU_ZERO(&cpuset);\n pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);\n return cpuset;\n}\n#else\nstatic uint32_t ggml_get_numa_affinity(void) {\n return 0; // no NUMA support\n}\n#endif\n\nvoid ggml_numa_init(enum ggml_numa_strategy numa_flag) {\n if (g_state.numa.n_nodes > 0) {\n fprintf(stderr, \"ggml_numa_init: NUMA already initialized\\n\");\n\n return;\n }\n\n#if defined(__gnu_linux__)\n struct stat st;\n char path[256];\n int rv;\n\n // set numa scheme\n g_state.numa.numa_strategy = numa_flag;\n\n GGML_PRINT_DEBUG(\"numa strategy %u\\n\",g_state.numa.numa_strategy);\n\n g_state.numa.cpuset = ggml_get_numa_affinity();\n\n // enumerate nodes\n while (g_state.numa.n_nodes < GGML_NUMA_MAX_NODES) {\n rv = snprintf(path, sizeof(path), \"/sys/devices/system/node/node%u\", g_state.numa.n_nodes);\n GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));\n if (stat(path, &st) != 0) { break; }\n ++g_state.numa.n_nodes;\n }\n\n // enumerate CPUs\n while (g_state.numa.total_cpus < GGML_NUMA_MAX_CPUS) {\n rv = snprintf(path, sizeof(path), \"/sys/devices/system/cpu/cpu%u\", g_state.numa.total_cpus);\n GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));\n if (stat(path, &st) != 0) { break; }\n ++g_state.numa.total_cpus;\n }\n\n GGML_PRINT_DEBUG(\"found %u numa nodes, %u CPUs\\n\", g_state.numa.n_nodes, g_state.numa.total_cpus);\n\n // figure out which node we're on\n uint current_cpu;\n int getcpu_ret = 0;\n#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 33) || defined(__COSMOPOLITAN__)\n getcpu_ret = getcpu(¤t_cpu, &g_state.numa.current_node);\n#else\n // old glibc doesn't have a wrapper for this call. Fall back on direct syscall\n# if !defined(SYS_getcpu) && defined(SYS_get_cpu)\n# define SYS_getcpu SYS_get_cpu // some older glibc versions use this name\n# endif\n getcpu_ret = syscall(SYS_getcpu, ¤t_cpu, &g_state.numa.current_node);\n#endif\n\n if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) {\n g_state.numa.n_nodes = 0;\n return;\n }\n\n GGML_PRINT_DEBUG(\"found our process on numa node %u, CPU %u\\n\", g_state.numa.current_node, current_cpu);\n\n for (uint32_t n = 0; n < g_state.numa.n_nodes; ++n) {\n struct ggml_numa_node * node = &g_state.numa.nodes[n];\n GGML_PRINT_DEBUG(\"CPUs on node %u:\", n);\n node->n_cpus = 0;\n for (uint32_t c = 0; c < g_state.numa.total_cpus; ++c) {\n rv = snprintf(path, sizeof(path), \"/sys/devices/system/node/node%u/cpu%u\", n, c);\n GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));\n if (stat(path, &st) == 0) {\n node->cpus[node->n_cpus++] = c;\n GGML_PRINT_DEBUG(\" %u\", c);\n }\n }\n GGML_PRINT_DEBUG(\"\\n\");\n }\n\n if (ggml_is_numa()) {\n FILE *fptr = fopen(\"/proc/sys/kernel/numa_balancing\", \"r\");\n if (fptr != NULL) {\n char buf[42];\n if (fgets(buf, sizeof(buf), fptr) && strncmp(buf, \"0\\n\", sizeof(buf)) != 0) {\n GGML_LOG_WARN(\"/proc/sys/kernel/numa_balancing is enabled, this has been observed to impair performance\\n\");\n }\n fclose(fptr);\n }\n }\n#else\n UNUSED(numa_flag);\n // TODO\n#endif\n}\n\nbool ggml_is_numa(void) {\n return g_state.numa.n_nodes > 1;\n}\n\n#if defined(__ARM_ARCH)\n#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)\n#include \nstatic void ggml_init_arm_arch_features(void) {\n ggml_arm_arch_features.sve_cnt = svcntb();\n}\n#else\nstatic void ggml_init_arm_arch_features(void) {}\n#endif\n#endif // __ARM_ARCH\n\n#if defined(__riscv) && defined(__riscv_v_intrinsic)\n#include \nstatic void ggml_init_riscv_arch_features(void) {\n ggml_riscv_arch_features.rvv_vlen = __riscv_vlenb();\n}\n#else\nstatic void ggml_init_riscv_arch_features(void) {}\n#endif\n\nstruct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value) {\n GGML_ASSERT(!ggml_get_no_alloc(ctx));\n\n struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);\n\n ggml_set_i32(result, value);\n\n return result;\n}\n\nstruct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value) {\n GGML_ASSERT(!ggml_get_no_alloc(ctx));\n\n struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);\n\n ggml_set_f32(result, value);\n\n return result;\n}\n\nstruct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value) {\n const int n = ggml_nrows(tensor);\n const int nc = tensor->ne[0];\n const size_t n1 = tensor->nb[1];\n\n char * const data = tensor->data;\n\n switch (tensor->type) {\n case GGML_TYPE_I8:\n {\n assert(tensor->nb[0] == sizeof(int8_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);\n }\n } break;\n case GGML_TYPE_I16:\n {\n assert(tensor->nb[0] == sizeof(int16_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);\n }\n } break;\n case GGML_TYPE_I32:\n {\n assert(tensor->nb[0] == sizeof(int32_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);\n }\n } break;\n case GGML_TYPE_F16:\n {\n assert(tensor->nb[0] == sizeof(ggml_fp16_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value));\n }\n } break;\n case GGML_TYPE_BF16:\n {\n assert(tensor->nb[0] == sizeof(ggml_fp16_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));\n }\n } break;\n case GGML_TYPE_F32:\n {\n assert(tensor->nb[0] == sizeof(float));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_f32(nc, (float *)(data + i*n1), value);\n }\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n\n return tensor;\n}\n\nstruct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {\n const int n = ggml_nrows(tensor);\n const int nc = tensor->ne[0];\n const size_t n1 = tensor->nb[1];\n\n char * const data = tensor->data;\n\n switch (tensor->type) {\n case GGML_TYPE_I8:\n {\n assert(tensor->nb[0] == sizeof(int8_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);\n }\n } break;\n case GGML_TYPE_I16:\n {\n assert(tensor->nb[0] == sizeof(int16_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);\n }\n } break;\n case GGML_TYPE_I32:\n {\n assert(tensor->nb[0] == sizeof(int32_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);\n }\n } break;\n case GGML_TYPE_F16:\n {\n assert(tensor->nb[0] == sizeof(ggml_fp16_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value));\n }\n } break;\n case GGML_TYPE_BF16:\n {\n assert(tensor->nb[0] == sizeof(ggml_bf16_t));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));\n }\n } break;\n case GGML_TYPE_F32:\n {\n assert(tensor->nb[0] == sizeof(float));\n for (int i = 0; i < n; i++) {\n ggml_vec_set_f32(nc, (float *)(data + i*n1), value);\n }\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n\n return tensor;\n}\n\nint32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {\n if (!ggml_is_contiguous(tensor)) {\n int64_t id[4] = { 0, 0, 0, 0 };\n ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);\n return ggml_get_i32_nd(tensor, id[0], id[1], id[2], id[3]);\n }\n switch (tensor->type) {\n case GGML_TYPE_I8:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));\n return ((int8_t *)(tensor->data))[i];\n }\n case GGML_TYPE_I16:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));\n return ((int16_t *)(tensor->data))[i];\n }\n case GGML_TYPE_I32:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));\n return ((int32_t *)(tensor->data))[i];\n }\n case GGML_TYPE_F16:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));\n return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);\n }\n case GGML_TYPE_BF16:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));\n return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);\n }\n case GGML_TYPE_F32:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(float));\n return ((float *)(tensor->data))[i];\n }\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nvoid ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {\n if (!ggml_is_contiguous(tensor)) {\n int64_t id[4] = { 0, 0, 0, 0 };\n ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);\n ggml_set_i32_nd(tensor, id[0], id[1], id[2], id[3], value);\n return;\n }\n switch (tensor->type) {\n case GGML_TYPE_I8:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));\n ((int8_t *)(tensor->data))[i] = value;\n } break;\n case GGML_TYPE_I16:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));\n ((int16_t *)(tensor->data))[i] = value;\n } break;\n case GGML_TYPE_I32:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));\n ((int32_t *)(tensor->data))[i] = value;\n } break;\n case GGML_TYPE_F16:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));\n ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value);\n } break;\n case GGML_TYPE_BF16:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));\n ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);\n } break;\n case GGML_TYPE_F32:\n {\n GGML_ASSERT(tensor->nb[0] == sizeof(float));\n ((float *)(tensor->data))[i] = value;\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nint32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {\n void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];\n switch (tensor->type) {\n case GGML_TYPE_I8:\n return ((int8_t *) data)[0];\n case GGML_TYPE_I16:\n return ((int16_t *) data)[0];\n case GGML_TYPE_I32:\n return ((int32_t *) data)[0];\n case GGML_TYPE_F16:\n return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);\n case GGML_TYPE_BF16:\n return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);\n case GGML_TYPE_F32:\n return ((float *) data)[0];\n default:\n GGML_ABORT(\"fatal error\");\n }\n}\n\nvoid ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value) {\n void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];\n switch (tensor->type) {\n case GGML_TYPE_I8:\n {\n ((int8_t *)(data))[0] = value;\n } break;\n case GGML_TYPE_I16:\n {\n ((int16_t *)(data))[0] = value;\n } break;\n case GGML_TYPE_I32:\n {\n ((int32_t *)(data))[0] = value;\n } break;\n case GGML_TYPE_F16:\n {\n ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value);\n } break;\n case GGML_TYPE_BF16:\n {\n ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);\n } break;\n case GGML_TYPE_F32:\n {\n ((float *)(data))[0] = value;\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nfloat ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {\n if (!ggml_is_contiguous(tensor)) {\n int64_t id[4] = { 0, 0, 0, 0 };\n ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);\n return ggml_get_f32_nd(tensor, id[0], id[1], id[2], id[3]);\n }\n switch (tensor->type) {\n case GGML_TYPE_I8:\n {\n return ((int8_t *)(tensor->data))[i];\n }\n case GGML_TYPE_I16:\n {\n return ((int16_t *)(tensor->data))[i];\n }\n case GGML_TYPE_I32:\n {\n return ((int32_t *)(tensor->data))[i];\n }\n case GGML_TYPE_F16:\n {\n return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);\n }\n case GGML_TYPE_BF16:\n {\n return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);\n }\n case GGML_TYPE_F32:\n {\n return ((float *)(tensor->data))[i];\n }\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nvoid ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {\n if (!ggml_is_contiguous(tensor)) {\n int64_t id[4] = { 0, 0, 0, 0 };\n ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);\n ggml_set_f32_nd(tensor, id[0], id[1], id[2], id[3], value);\n return;\n }\n switch (tensor->type) {\n case GGML_TYPE_I8:\n {\n ((int8_t *)(tensor->data))[i] = value;\n } break;\n case GGML_TYPE_I16:\n {\n ((int16_t *)(tensor->data))[i] = value;\n } break;\n case GGML_TYPE_I32:\n {\n ((int32_t *)(tensor->data))[i] = value;\n } break;\n case GGML_TYPE_F16:\n {\n ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value);\n } break;\n case GGML_TYPE_BF16:\n {\n ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);\n } break;\n case GGML_TYPE_F32:\n {\n ((float *)(tensor->data))[i] = value;\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nfloat ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {\n void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];\n switch (tensor->type) {\n case GGML_TYPE_I8:\n return ((int8_t *) data)[0];\n case GGML_TYPE_I16:\n return ((int16_t *) data)[0];\n case GGML_TYPE_I32:\n return ((int32_t *) data)[0];\n case GGML_TYPE_F16:\n return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);\n case GGML_TYPE_BF16:\n return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);\n case GGML_TYPE_F32:\n return ((float *) data)[0];\n default:\n GGML_ABORT(\"fatal error\");\n }\n}\n\nvoid ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value) {\n void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];\n switch (tensor->type) {\n case GGML_TYPE_I8:\n {\n ((int8_t *)(data))[0] = value;\n } break;\n case GGML_TYPE_I16:\n {\n ((int16_t *)(data))[0] = value;\n } break;\n case GGML_TYPE_I32:\n {\n ((int32_t *)(data))[0] = value;\n } break;\n case GGML_TYPE_F16:\n {\n ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value);\n } break;\n case GGML_TYPE_BF16:\n {\n ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);\n } break;\n case GGML_TYPE_F32:\n {\n ((float *)(data))[0] = value;\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n// ggml_compute_forward_mul_mat\n\nstatic void ggml_compute_forward_mul_mat_one_chunk(\n const struct ggml_compute_params * params,\n struct ggml_tensor * dst,\n const enum ggml_type type,\n const int64_t num_rows_per_vec_dot,\n const int64_t ir0_start,\n const int64_t ir0_end,\n const int64_t ir1_start,\n const int64_t ir1_end) {\n\n const struct ggml_tensor * src0 = dst->src[0];\n const struct ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const bool src1_cont = ggml_is_contiguous(src1);\n\n ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;\n enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;\n\n // broadcast factors\n const int64_t r2 = ne12 / ne02;\n const int64_t r3 = ne13 / ne03;\n\n //printf(\"ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\\n\", ir0_start, ir0_end, ir1_start, ir1_end);\n\n // threads with no work simply yield (not sure if it helps)\n if (ir0_start >= ir0_end || ir1_start >= ir1_end) {\n return;\n }\n\n const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;\n const size_t row_size = ggml_row_size(vec_dot_type, ne10);\n\n assert(ne12 % ne02 == 0);\n assert(ne13 % ne03 == 0);\n\n // block-tiling attempt\n const int64_t blck_0 = 16;\n const int64_t blck_1 = 16;\n\n const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11;\n\n // attempt to reduce false-sharing (does not seem to make a difference)\n // 16 * 2, accounting for mmla kernels\n float tmp[32];\n\n for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {\n for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {\n for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) {\n const int64_t i13 = (ir1 / (ne12 * ne1));\n const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1;\n const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1);\n\n // broadcast src0 into src1\n const int64_t i03 = i13 / r3;\n const int64_t i02 = i12 / r2;\n\n const int64_t i1 = i11;\n const int64_t i2 = i12;\n const int64_t i3 = i13;\n\n const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03);\n\n // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides\n // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using\n // the original src1 data pointer, so we should index using the indices directly\n // TODO: this is a bit of a hack, we should probably have a better way to handle this\n const char * src1_col = (const char*)wdata +\n (src1_cont || src1->type != vec_dot_type\n ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size\n : (i11 * nb11 + i12 * nb12 + i13 * nb13));\n float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3));\n\n //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {\n // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);\n //}\n\n for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) {\n vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);\n }\n\n for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) {\n memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float));\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_mul_mat(\n const struct ggml_compute_params * params,\n struct ggml_tensor * dst) {\n\n const struct ggml_tensor * src0 = dst->src[0];\n const struct ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;\n ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;\n int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows;\n\n GGML_ASSERT(ne0 == ne01);\n GGML_ASSERT(ne1 == ne11);\n GGML_ASSERT(ne2 == ne12);\n GGML_ASSERT(ne3 == ne13);\n\n // we don't support permuted src0 or src1\n GGML_ASSERT(nb00 == ggml_type_size(src0->type));\n GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n // nb01 >= nb00 - src0 is not transposed\n // compute by src0 rows\n\n // TODO: extract to \"extra_op\"\n#if GGML_USE_LLAMAFILE\n // broadcast factors\n const int64_t r2 = ne12 / ne02;\n const int64_t r3 = ne13 / ne03;\n\n const bool src1_cont = ggml_is_contiguous(src1);\n\n if (src1_cont) {\n for (int64_t i13 = 0; i13 < ne13; i13++)\n for (int64_t i12 = 0; i12 < ne12; i12++)\n if (!llamafile_sgemm(params,\n ne01, ne11, ne00/ggml_blck_size(src0->type),\n (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,\n nb01/ggml_type_size(src0->type),\n (const char *)src1->data + i12*nb12 + i13*nb13,\n nb11/ggml_type_size(src1->type),\n (char *)dst->data + i12*nb2 + i13*nb3,\n nb1/ggml_type_size(dst->type),\n src0->type,\n src1->type,\n dst->type))\n goto UseGgmlGemm1;\n return;\n }\nUseGgmlGemm1:;\n#endif\n\n if (src1->type != vec_dot_type) {\n char * wdata = params->wdata;\n\n const size_t nbw0 = ggml_type_size(vec_dot_type);\n const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);\n const size_t nbw2 = nbw1*ne11;\n const size_t nbw3 = nbw2*ne12;\n\n assert(params->wsize >= ne13*nbw3);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n #if 0\n for (int64_t i13 = 0; i13 < ne13; ++i13) {\n for (int64_t i12 = 0; i12 < ne12; ++i12) {\n for (int64_t i11 = ith; i11 < ne11; i11 += nth) {\n from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),\n (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),\n ne10);\n }\n }\n }\n #else\n for (int64_t i13 = 0; i13 < ne13; ++i13) {\n for (int64_t i12 = 0; i12 < ne12; ++i12) {\n for (int64_t i11 = 0; i11 < ne11; ++i11) {\n size_t bs = ggml_blck_size(vec_dot_type);\n int64_t ne10_block_start = (ith * ne10/bs) / nth;\n int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth;\n from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),\n (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),\n (ne10_block_end - ne10_block_start) * bs);\n }\n }\n }\n #endif\n }\n\n if (ith == 0) {\n // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.\n atomic_store_explicit(¶ms->threadpool->current_chunk, nth, memory_order_relaxed);\n }\n\n ggml_barrier(params->threadpool);\n\n#if GGML_USE_LLAMAFILE\n if (src1->type != vec_dot_type) {\n const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;\n const size_t row_size = ggml_row_size(vec_dot_type, ne10);\n\n for (int64_t i13 = 0; i13 < ne13; i13++)\n for (int64_t i12 = 0; i12 < ne12; i12++)\n if (!llamafile_sgemm(params,\n ne01, ne11, ne00/ggml_blck_size(src0->type),\n (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,\n nb01/ggml_type_size(src0->type),\n (const char *)wdata + (i12*ne11 + i13*ne12*ne11)*row_size,\n row_size/ggml_type_size(vec_dot_type),\n (char *)dst->data + i12*nb2 + i13*nb3,\n nb1/ggml_type_size(dst->type),\n src0->type,\n vec_dot_type,\n dst->type))\n goto UseGgmlGemm2;\n return;\n }\nUseGgmlGemm2:;\n#endif\n\n // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers)\n const int64_t nr0 = ne0;\n\n // This is the size of the rest of the dimensions of the result\n const int64_t nr1 = ne1 * ne2 * ne3;\n\n // Now select a reasonable chunk size.\n int chunk_size = 16;\n\n // We need to step up the size if it's small\n if (nr0 == 1 || nr1 == 1) {\n chunk_size = 64;\n }\n\n // distribute the work across the inner or outer loop based on which one is larger\n // The number of chunks in the 0/1 dim.\n // CEIL(nr0/chunk_size)\n int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;\n int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;\n\n // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread.\n // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggml-org/llama.cpp/pull/6915\n // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that.\n if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) {\n // distribute the thread work across the inner or outer loop based on which one is larger\n nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows\n nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows\n }\n\n // The number of elements in each chunk\n const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;\n const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;\n\n // The first chunk comes from our thread_id, the rest will get auto-assigned.\n int current_chunk = ith;\n\n while (current_chunk < nchunk0 * nchunk1) {\n const int64_t ith0 = current_chunk % nchunk0;\n const int64_t ith1 = current_chunk / nchunk0;\n\n const int64_t ir0_start = dr0 * ith0;\n const int64_t ir0_end = MIN(ir0_start + dr0, nr0);\n\n const int64_t ir1_start = dr1 * ith1;\n const int64_t ir1_end = MIN(ir1_start + dr1, nr1);\n\n // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols\n int64_t num_rows_per_vec_dot = vec_dot_num_rows;\n\n // these checks are needed to avoid crossing dim1 boundaries\n // can be optimized, but the logic would become more complicated, so keeping it like this for simplicity\n if ((nr0 % 2 != 0) || (ne11 % 2 != 0) || ((ir0_end - ir0_start) % 2 != 0) || ((ir1_end - ir1_start) % 2 != 0)) {\n num_rows_per_vec_dot = 1;\n }\n ggml_compute_forward_mul_mat_one_chunk(params, dst, src0->type, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end);\n\n if (nth >= nchunk0 * nchunk1) {\n break;\n }\n\n current_chunk = atomic_fetch_add_explicit(¶ms->threadpool->current_chunk, 1, memory_order_relaxed);\n }\n}\n\n// ggml_compute_forward_mul_mat_id\n\n#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id)*ids->ne[0]*ids->ne[1] + (i1)]\n\nstruct mmid_row_mapping {\n int32_t i1;\n int32_t i2;\n};\n\nstatic void ggml_compute_forward_mul_mat_id_one_chunk(\n struct ggml_tensor * dst,\n const struct ggml_tensor * src0,\n const struct ggml_tensor * src1,\n const struct ggml_tensor * ids,\n const int64_t cur_a,\n const int64_t ir0_start,\n const int64_t ir0_end,\n const int64_t ir1_start,\n const int64_t ir1_end,\n const char * src0_cur,\n const struct mmid_row_mapping * matrix_rows,\n const size_t row_size,\n const bool src1_cont,\n const void * wdata) {\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const enum ggml_type type = src0->type;\n\n ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;\n enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;\n\n const int64_t blck_0 = 16;\n const int64_t blck_1 = 16;\n\n float tmp[16];\n\n for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {\n for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {\n for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ++ir1) {\n const int64_t _i12 = ir1; // logical row index for this expert\n\n struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, _i12);\n const int id = row_mapping.i1; // selected expert index\n\n const int64_t i11 = id % ne11;\n const int64_t i12 = row_mapping.i2; // row index in src1\n\n const int64_t i1 = id; // selected expert index\n const int64_t i2 = i12; // row\n\n // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides\n // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using\n // the original src1 data pointer, so we should index using the indices directly\n // TODO: this is a bit of a hack, we should probably have a better way to handle this\n const char * src1_col = (const char *) wdata +\n (src1_cont || src1->type != vec_dot_type\n ? (i11 + i12*ne11)*row_size\n : (i11*nb11 + i12*nb12));\n\n float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2));\n\n for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {\n vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_cur + ir0*nb01, 0, src1_col, 0, 1);\n }\n\n memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir0_end) - iir0)*sizeof(float));\n }\n }\n }\n}\n\nstatic void * incr_ptr_aligned(void ** p, size_t size, size_t align) {\n\n void * ptr = *p;\n ptr = (void *) GGML_PAD((uintptr_t) ptr, align);\n *p = (void *) ((char *) ptr + size);\n return ptr;\n}\n\nstatic void ggml_compute_forward_mul_mat_id(\n const struct ggml_compute_params * params,\n struct ggml_tensor * dst) {\n\n const struct ggml_tensor * src0 = dst->src[0];\n const struct ggml_tensor * src1 = dst->src[1];\n const struct ggml_tensor * ids = dst->src[2];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const enum ggml_type type = src0->type;\n\n const bool src1_cont = ggml_is_contiguous(src1);\n\n enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;\n ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;\n\n // we don't support permuted src0 or src1\n GGML_ASSERT(nb00 == ggml_type_size(type));\n GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n // row groups\n const int n_ids = ids->ne[0]; // n_expert_used\n const int n_as = ne02; // n_expert\n\n void * wdata_cur = params->wdata;\n\n if (src1->type != vec_dot_type) {\n incr_ptr_aligned(&wdata_cur, ggml_row_size(vec_dot_type, ggml_nelements(src1)), sizeof(int64_t));\n }\n\n int64_t * matrix_row_counts = // [n_as]\n incr_ptr_aligned(&wdata_cur, n_as*sizeof(int64_t), sizeof(int64_t));\n\n struct mmid_row_mapping * matrix_rows = // [n_as][ids->ne[0]*ids->ne[1]]\n incr_ptr_aligned(&wdata_cur, n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping), sizeof(int64_t));\n\n char (*atomic_current_chunk)[CACHE_LINE_SIZE] = // [n_as]\n incr_ptr_aligned(&wdata_cur, CACHE_LINE_SIZE * n_as, CACHE_LINE_SIZE);\n\n GGML_ASSERT(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata));\n\n if (src1->type != vec_dot_type) {\n char * wdata = params->wdata;\n\n const size_t nbw0 = ggml_type_size(vec_dot_type);\n const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);\n const size_t nbw2 = nbw1*ne11;\n const size_t nbw3 = nbw2*ne12;\n\n assert(params->wsize >= ne13*nbw3);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n#if 0\n for (int64_t i13 = 0; i13 < ne13; ++i13) {\n for (int64_t i12 = ith; i12 < ne12; i12 += nth) {\n for (int64_t i11 = 0; i11 < ne11; ++i11) {\n from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),\n (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),\n ne10);\n }\n }\n }\n#else\n for (int64_t i13 = 0; i13 < ne13; ++i13) {\n for (int64_t i12 = 0; i12 < ne12; ++i12) {\n for (int64_t i11 = 0; i11 < ne11; ++i11) {\n size_t bs = ggml_blck_size(vec_dot_type);\n int64_t ne10_block_start = (ith * ne10/bs) / nth;\n int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth;\n from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),\n (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),\n (ne10_block_end - ne10_block_start) * bs);\n }\n }\n }\n#endif\n }\n\n if (ith == 0) {\n // initialize matrix_row_counts\n memset(matrix_row_counts, 0, n_as*sizeof(int64_t));\n\n // group rows by src0 matrix\n for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {\n for (int id = 0; id < n_ids; ++id) {\n const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]);\n\n assert(i02 >= 0 && i02 < n_as);\n\n MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) {id, iid1};\n matrix_row_counts[i02] += 1;\n }\n }\n }\n\n // reset current_chunk\n for (int cur_a = ith; cur_a < n_as; cur_a += nth) {\n atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);\n *current_chunk_ctr = nth;\n }\n\n ggml_barrier(params->threadpool);\n\n for (int cur_a = 0; cur_a < n_as; ++cur_a) {\n const int64_t cne1 = matrix_row_counts[cur_a];\n\n if (cne1 == 0) {\n continue;\n }\n\n const char * src0_cur = (const char *) src0->data + cur_a * nb02;\n const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;\n const size_t row_size = ggml_row_size(vec_dot_type, ne10);\n\n const int64_t nr0 = ne01;\n const int64_t nr1 = cne1;\n\n int chunk_size = 16;\n if (nr0 == 1 || nr1 == 1) {\n chunk_size = 64;\n }\n\n // disable for NUMA\n const bool disable_chunking = ggml_is_numa();\n\n int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;\n int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;\n\n if (nchunk0 * nchunk1 < nth * 4 || disable_chunking) {\n nchunk0 = nr0 > nr1 ? nth : 1;\n nchunk1 = nr0 > nr1 ? 1 : nth;\n }\n\n const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;\n const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;\n\n int current_chunk = ith;\n\n atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);\n\n while (current_chunk < nchunk0 * nchunk1) {\n const int64_t ith0 = current_chunk % nchunk0;\n const int64_t ith1 = current_chunk / nchunk0;\n\n const int64_t ir0_start = dr0 * ith0;\n const int64_t ir0_end = MIN(ir0_start + dr0, nr0);\n\n const int64_t ir1_start = dr1 * ith1;\n const int64_t ir1_end = MIN(ir1_start + dr1, nr1);\n\n ggml_compute_forward_mul_mat_id_one_chunk(\n dst, src0, src1, ids, cur_a,\n ir0_start, ir0_end, ir1_start, ir1_end,\n src0_cur, matrix_rows, row_size, src1_cont, wdata\n );\n\n if (nth >= nchunk0 * nchunk1) {\n break;\n }\n\n current_chunk = atomic_fetch_add_explicit(current_chunk_ctr, 1, memory_order_relaxed);\n }\n }\n}\n\n/////////////////////////////////\n\nstatic void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {\n GGML_ASSERT(params);\n\n if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) {\n return;\n }\n\n // extra_buffer op?\n if (ggml_cpu_extra_compute_forward(params, tensor)) {\n return;\n }\n\n switch (tensor->op) {\n case GGML_OP_DUP:\n {\n ggml_compute_forward_dup(params, tensor);\n } break;\n case GGML_OP_ADD:\n {\n ggml_compute_forward_add(params, tensor);\n } break;\n case GGML_OP_ADD_ID:\n {\n ggml_compute_forward_add_id(params, tensor);\n } break;\n case GGML_OP_ADD1:\n {\n ggml_compute_forward_add1(params, tensor);\n } break;\n case GGML_OP_ACC:\n {\n ggml_compute_forward_acc(params, tensor);\n } break;\n case GGML_OP_SUB:\n {\n ggml_compute_forward_sub(params, tensor);\n } break;\n case GGML_OP_MUL:\n {\n ggml_compute_forward_mul(params, tensor);\n } break;\n case GGML_OP_DIV:\n {\n ggml_compute_forward_div(params, tensor);\n } break;\n case GGML_OP_SQR:\n {\n ggml_compute_forward_sqr(params, tensor);\n } break;\n case GGML_OP_SQRT:\n {\n ggml_compute_forward_sqrt(params, tensor);\n } break;\n case GGML_OP_LOG:\n {\n ggml_compute_forward_log(params, tensor);\n } break;\n case GGML_OP_SIN:\n {\n ggml_compute_forward_sin(params, tensor);\n } break;\n case GGML_OP_COS:\n {\n ggml_compute_forward_cos(params, tensor);\n } break;\n case GGML_OP_SUM:\n {\n ggml_compute_forward_sum(params, tensor);\n } break;\n case GGML_OP_SUM_ROWS:\n {\n ggml_compute_forward_sum_rows(params, tensor);\n } break;\n case GGML_OP_CUMSUM:\n {\n ggml_compute_forward_cumsum(params, tensor);\n } break;\n case GGML_OP_MEAN:\n {\n ggml_compute_forward_mean(params, tensor);\n } break;\n case GGML_OP_ARGMAX:\n {\n ggml_compute_forward_argmax(params, tensor);\n } break;\n case GGML_OP_COUNT_EQUAL:\n {\n ggml_compute_forward_count_equal(params, tensor);\n } break;\n case GGML_OP_REPEAT:\n {\n ggml_compute_forward_repeat(params, tensor);\n } break;\n case GGML_OP_REPEAT_BACK:\n {\n ggml_compute_forward_repeat_back(params, tensor);\n } break;\n case GGML_OP_CONCAT:\n {\n ggml_compute_forward_concat(params, tensor);\n } break;\n case GGML_OP_SILU_BACK:\n {\n ggml_compute_forward_silu_back(params, tensor);\n } break;\n case GGML_OP_NORM:\n {\n ggml_compute_forward_norm(params, tensor);\n } break;\n case GGML_OP_RMS_NORM:\n {\n ggml_compute_forward_rms_norm(params, tensor);\n } break;\n case GGML_OP_RMS_NORM_BACK:\n {\n ggml_compute_forward_rms_norm_back(params, tensor);\n } break;\n case GGML_OP_GROUP_NORM:\n {\n ggml_compute_forward_group_norm(params, tensor);\n } break;\n case GGML_OP_L2_NORM:\n {\n ggml_compute_forward_l2_norm(params, tensor);\n } break;\n case GGML_OP_MUL_MAT:\n {\n ggml_compute_forward_mul_mat(params, tensor);\n } break;\n case GGML_OP_MUL_MAT_ID:\n {\n ggml_compute_forward_mul_mat_id(params, tensor);\n } break;\n case GGML_OP_OUT_PROD:\n {\n ggml_compute_forward_out_prod(params, tensor);\n } break;\n case GGML_OP_SCALE:\n {\n ggml_compute_forward_scale(params, tensor);\n } break;\n case GGML_OP_SET:\n {\n ggml_compute_forward_set(params, tensor);\n } break;\n case GGML_OP_CPY:\n {\n ggml_compute_forward_cpy(params, tensor);\n } break;\n case GGML_OP_CONT:\n {\n ggml_compute_forward_cont(params, tensor);\n } break;\n case GGML_OP_GET_ROWS:\n {\n ggml_compute_forward_get_rows(params, tensor);\n } break;\n case GGML_OP_GET_ROWS_BACK:\n {\n ggml_compute_forward_get_rows_back(params, tensor);\n } break;\n case GGML_OP_SET_ROWS:\n {\n ggml_compute_forward_set_rows(params, tensor);\n } break;\n case GGML_OP_DIAG:\n {\n ggml_compute_forward_diag(params, tensor);\n } break;\n case GGML_OP_DIAG_MASK_INF:\n {\n ggml_compute_forward_diag_mask_inf(params, tensor);\n } break;\n case GGML_OP_DIAG_MASK_ZERO:\n {\n ggml_compute_forward_diag_mask_zero(params, tensor);\n } break;\n case GGML_OP_SOFT_MAX:\n {\n ggml_compute_forward_soft_max(params, tensor);\n } break;\n case GGML_OP_SOFT_MAX_BACK:\n {\n ggml_compute_forward_soft_max_ext_back(params, tensor);\n } break;\n case GGML_OP_ROPE:\n {\n ggml_compute_forward_rope(params, tensor);\n } break;\n case GGML_OP_ROPE_BACK:\n {\n ggml_compute_forward_rope_back(params, tensor);\n } break;\n case GGML_OP_CLAMP:\n {\n ggml_compute_forward_clamp(params, tensor);\n } break;\n case GGML_OP_CONV_TRANSPOSE_1D:\n {\n ggml_compute_forward_conv_transpose_1d(params, tensor);\n } break;\n case GGML_OP_IM2COL:\n {\n ggml_compute_forward_im2col(params, tensor);\n } break;\n case GGML_OP_IM2COL_BACK:\n {\n ggml_compute_forward_im2col_back_f32(params, tensor);\n } break;\n case GGML_OP_IM2COL_3D:\n {\n ggml_compute_forward_im2col_3d(params, tensor);\n } break;\n case GGML_OP_CONV_2D:\n {\n ggml_compute_forward_conv_2d(params, tensor);\n } break;\n case GGML_OP_CONV_3D:\n {\n ggml_compute_forward_conv_3d(params, tensor);\n } break;\n case GGML_OP_CONV_2D_DW:\n {\n ggml_compute_forward_conv_2d_dw(params, tensor);\n } break;\n case GGML_OP_CONV_TRANSPOSE_2D:\n {\n ggml_compute_forward_conv_transpose_2d(params, tensor);\n } break;\n case GGML_OP_POOL_1D:\n {\n ggml_compute_forward_pool_1d(params, tensor);\n } break;\n case GGML_OP_POOL_2D:\n {\n ggml_compute_forward_pool_2d(params, tensor);\n } break;\n case GGML_OP_POOL_2D_BACK:\n {\n ggml_compute_forward_pool_2d_back(params, tensor);\n } break;\n case GGML_OP_UPSCALE:\n {\n ggml_compute_forward_upscale(params, tensor);\n } break;\n case GGML_OP_PAD:\n {\n ggml_compute_forward_pad(params, tensor);\n } break;\n case GGML_OP_PAD_REFLECT_1D:\n {\n ggml_compute_forward_pad_reflect_1d(params, tensor);\n } break;\n case GGML_OP_ROLL:\n {\n ggml_compute_forward_roll(params, tensor);\n } break;\n case GGML_OP_ARANGE:\n {\n ggml_compute_forward_arange(params, tensor);\n } break;\n case GGML_OP_TIMESTEP_EMBEDDING:\n {\n ggml_compute_forward_timestep_embedding(params, tensor);\n } break;\n case GGML_OP_ARGSORT:\n {\n ggml_compute_forward_argsort(params, tensor);\n } break;\n case GGML_OP_TOP_K:\n {\n ggml_compute_forward_top_k(params, tensor);\n } break;\n case GGML_OP_LEAKY_RELU:\n {\n ggml_compute_forward_leaky_relu(params, tensor);\n } break;\n case GGML_OP_TRI:\n {\n ggml_compute_forward_tri(params, tensor);\n } break;\n case GGML_OP_FILL:\n {\n ggml_compute_forward_fill(params, tensor);\n } break;\n case GGML_OP_FLASH_ATTN_EXT:\n {\n ggml_compute_forward_flash_attn_ext(params, tensor);\n } break;\n case GGML_OP_FLASH_ATTN_BACK:\n {\n int32_t t = ggml_get_op_params_i32(tensor, 0);\n GGML_ASSERT(t == 0 || t == 1);\n bool masked = t != 0;\n ggml_compute_forward_flash_attn_back(params, masked, tensor);\n } break;\n case GGML_OP_SSM_CONV:\n {\n ggml_compute_forward_ssm_conv(params, tensor);\n } break;\n case GGML_OP_SSM_SCAN:\n {\n ggml_compute_forward_ssm_scan(params, tensor);\n } break;\n case GGML_OP_WIN_PART:\n {\n ggml_compute_forward_win_part(params, tensor);\n } break;\n case GGML_OP_WIN_UNPART:\n {\n ggml_compute_forward_win_unpart(params, tensor);\n } break;\n case GGML_OP_UNARY:\n {\n ggml_compute_forward_unary(params, tensor);\n } break;\n case GGML_OP_GLU:\n {\n ggml_compute_forward_glu(params, tensor);\n } break;\n case GGML_OP_GET_REL_POS:\n {\n ggml_compute_forward_get_rel_pos(params, tensor);\n } break;\n case GGML_OP_ADD_REL_POS:\n {\n ggml_compute_forward_add_rel_pos(params, tensor);\n } break;\n case GGML_OP_RWKV_WKV6:\n {\n ggml_compute_forward_rwkv_wkv6(params, tensor);\n } break;\n case GGML_OP_GATED_LINEAR_ATTN:\n {\n ggml_compute_forward_gla(params, tensor);\n } break;\n case GGML_OP_RWKV_WKV7:\n {\n ggml_compute_forward_rwkv_wkv7(params, tensor);\n } break;\n case GGML_OP_SOLVE_TRI:\n {\n ggml_compute_forward_solve_tri(params, tensor);\n } break;\n case GGML_OP_GATED_DELTA_NET:\n {\n ggml_compute_forward_gated_delta_net(params, tensor);\n } break;\n case GGML_OP_MAP_CUSTOM1:\n {\n ggml_compute_forward_map_custom1(params, tensor);\n }\n break;\n case GGML_OP_MAP_CUSTOM2:\n {\n ggml_compute_forward_map_custom2(params, tensor);\n }\n break;\n case GGML_OP_MAP_CUSTOM3:\n {\n ggml_compute_forward_map_custom3(params, tensor);\n }\n break;\n case GGML_OP_CUSTOM:\n {\n ggml_compute_forward_custom(params, tensor);\n }\n break;\n case GGML_OP_CROSS_ENTROPY_LOSS:\n {\n ggml_compute_forward_cross_entropy_loss(params, tensor);\n }\n break;\n case GGML_OP_CROSS_ENTROPY_LOSS_BACK:\n {\n ggml_compute_forward_cross_entropy_loss_back(params, tensor);\n }\n break;\n case GGML_OP_OPT_STEP_ADAMW:\n {\n ggml_compute_forward_opt_step_adamw(params, tensor);\n }\n break;\n case GGML_OP_OPT_STEP_SGD:\n {\n ggml_compute_forward_opt_step_sgd(params, tensor);\n }\n break;\n case GGML_OP_NONE:\n {\n // nop\n } break;\n case GGML_OP_RESHAPE:\n {\n // nop\n } break;\n case GGML_OP_PERMUTE:\n {\n // nop\n } break;\n case GGML_OP_VIEW:\n {\n // nop\n } break;\n case GGML_OP_TRANSPOSE:\n {\n // nop\n } break;\n case GGML_OP_COUNT:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// Android's libc implementation \"bionic\" does not support setting affinity\n#if defined(__gnu_linux__)\nstatic void set_numa_thread_affinity(int thread_n) {\n if (!ggml_is_numa()) {\n return;\n }\n\n int node_num;\n int rv;\n size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);\n\n switch(g_state.numa.numa_strategy) {\n case GGML_NUMA_STRATEGY_DISTRIBUTE:\n // run thread on node_num thread_n / (threads per node)\n node_num = thread_n % g_state.numa.n_nodes;\n break;\n case GGML_NUMA_STRATEGY_ISOLATE:\n // run thread on current_node\n node_num = g_state.numa.current_node;\n break;\n case GGML_NUMA_STRATEGY_NUMACTL:\n // use the cpuset that numactl gave us\n rv = pthread_setaffinity_np(pthread_self(), setsize, &g_state.numa.cpuset);\n if (rv) {\n fprintf(stderr, \"warning: pthread_setaffinity_np() failed: %s\\n\",strerror(rv));\n }\n return;\n default:\n return;\n }\n\n struct ggml_numa_node * node = &g_state.numa.nodes[node_num];\n\n cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);\n CPU_ZERO_S(setsize, cpus);\n for (size_t i = 0; i < node->n_cpus; ++i) {\n CPU_SET_S(node->cpus[i], setsize, cpus);\n }\n\n rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);\n if (rv) {\n fprintf(stderr, \"warning: pthread_setaffinity_np() failed: %s\\n\", strerror(rv));\n }\n\n CPU_FREE(cpus);\n}\n\nstatic void clear_numa_thread_affinity(void) {\n if (!ggml_is_numa()) {\n return;\n }\n\n size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);\n\n cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);\n CPU_ZERO_S(setsize, cpus);\n for (unsigned i = 0; i < g_state.numa.total_cpus; ++i) {\n CPU_SET_S(i, setsize, cpus);\n }\n\n int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);\n if (rv) {\n fprintf(stderr, \"warning: pthread_setaffinity_np() failed: %s\\n\", strerror(rv));\n }\n\n CPU_FREE(cpus);\n}\n#else\n// TODO: Windows etc.\n// (the linux implementation may also work on BSD, someone should test)\nstatic void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n); }\nstatic void clear_numa_thread_affinity(void) {}\n#endif\n\nstatic int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {\n int n_tasks = 0;\n\n if (ggml_is_empty(node)) {\n // no need to multi-thread a no-op\n n_tasks = 1;\n return n_tasks;\n }\n\n switch (node->op) {\n case GGML_OP_CPY:\n case GGML_OP_DUP:\n case GGML_OP_CONT:\n case GGML_OP_ADD:\n case GGML_OP_ADD_ID:\n case GGML_OP_ADD1:\n case GGML_OP_ACC:\n case GGML_OP_CUMSUM:\n case GGML_OP_TRI:\n case GGML_OP_FILL:\n {\n n_tasks = n_threads;\n } break;\n case GGML_OP_SUB:\n case GGML_OP_SQR:\n case GGML_OP_SQRT:\n case GGML_OP_LOG:\n case GGML_OP_SIN:\n case GGML_OP_COS:\n case GGML_OP_SUM:\n case GGML_OP_SUM_ROWS:\n case GGML_OP_MEAN:\n case GGML_OP_ARGMAX:\n {\n n_tasks = 1;\n } break;\n case GGML_OP_COUNT_EQUAL:\n case GGML_OP_SOLVE_TRI:\n case GGML_OP_GATED_DELTA_NET:\n {\n n_tasks = n_threads;\n } break;\n case GGML_OP_REPEAT:\n case GGML_OP_REPEAT_BACK:\n case GGML_OP_LEAKY_RELU:\n {\n n_tasks = 1;\n } break;\n case GGML_OP_UNARY:\n switch (ggml_get_unary_op(node)) {\n case GGML_UNARY_OP_ABS:\n case GGML_UNARY_OP_SGN:\n case GGML_UNARY_OP_NEG:\n case GGML_UNARY_OP_STEP:\n case GGML_UNARY_OP_TANH:\n case GGML_UNARY_OP_ELU:\n case GGML_UNARY_OP_RELU:\n case GGML_UNARY_OP_SIGMOID:\n case GGML_UNARY_OP_HARDSWISH:\n case GGML_UNARY_OP_HARDSIGMOID:\n case GGML_UNARY_OP_EXP:\n case GGML_UNARY_OP_SOFTPLUS:\n case GGML_UNARY_OP_EXPM1:\n case GGML_UNARY_OP_FLOOR:\n case GGML_UNARY_OP_CEIL:\n case GGML_UNARY_OP_ROUND:\n case GGML_UNARY_OP_TRUNC:\n {\n n_tasks = 1;\n } break;\n\n case GGML_UNARY_OP_GELU:\n case GGML_UNARY_OP_GELU_ERF:\n case GGML_UNARY_OP_GELU_QUICK:\n case GGML_UNARY_OP_SILU:\n case GGML_UNARY_OP_XIELU:\n {\n n_tasks = n_threads;\n } break;\n default:\n GGML_ABORT(\"fatal error\");\n }\n break;\n case GGML_OP_GLU:\n switch (ggml_get_glu_op(node)) {\n case GGML_GLU_OP_REGLU:\n case GGML_GLU_OP_GEGLU:\n case GGML_GLU_OP_SWIGLU:\n case GGML_GLU_OP_SWIGLU_OAI:\n case GGML_GLU_OP_GEGLU_ERF:\n case GGML_GLU_OP_GEGLU_QUICK:\n {\n n_tasks = n_threads;\n } break;\n default:\n GGML_ABORT(\"fatal error\");\n }\n break;\n case GGML_OP_SILU_BACK:\n case GGML_OP_MUL:\n case GGML_OP_DIV:\n case GGML_OP_NORM:\n case GGML_OP_RMS_NORM:\n case GGML_OP_RMS_NORM_BACK:\n case GGML_OP_L2_NORM:\n case GGML_OP_GROUP_NORM:\n case GGML_OP_CONCAT:\n case GGML_OP_MUL_MAT:\n case GGML_OP_MUL_MAT_ID:\n case GGML_OP_OUT_PROD:\n {\n n_tasks = n_threads;\n } break;\n case GGML_OP_GET_ROWS:\n case GGML_OP_SET_ROWS:\n {\n // FIXME: get_rows can use additional threads, but the cost of launching additional threads\n // decreases performance with GPU offloading\n //n_tasks = n_threads;\n n_tasks = 1;\n } break;\n case GGML_OP_SCALE:\n case GGML_OP_SET:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n case GGML_OP_GET_ROWS_BACK:\n case GGML_OP_DIAG:\n {\n n_tasks = 1;\n } break;\n case GGML_OP_DIAG_MASK_ZERO:\n case GGML_OP_DIAG_MASK_INF:\n case GGML_OP_SOFT_MAX_BACK:\n case GGML_OP_ROPE:\n case GGML_OP_ROPE_BACK:\n case GGML_OP_ADD_REL_POS:\n {\n n_tasks = n_threads;\n } break;\n case GGML_OP_CLAMP:\n {\n n_tasks = 1; //TODO\n } break;\n case GGML_OP_SOFT_MAX:\n {\n n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));\n } break;\n case GGML_OP_IM2COL:\n case GGML_OP_IM2COL_BACK:\n case GGML_OP_IM2COL_3D:\n case GGML_OP_CONV_2D:\n case GGML_OP_CONV_3D:\n case GGML_OP_CONV_2D_DW:\n case GGML_OP_CONV_TRANSPOSE_1D:\n case GGML_OP_CONV_TRANSPOSE_2D:\n {\n n_tasks = n_threads;\n } break;\n case GGML_OP_POOL_1D:\n case GGML_OP_POOL_2D:\n case GGML_OP_POOL_2D_BACK:\n {\n n_tasks = 1;\n } break;\n case GGML_OP_UPSCALE:\n case GGML_OP_PAD:\n case GGML_OP_PAD_REFLECT_1D:\n case GGML_OP_ROLL:\n case GGML_OP_ARANGE:\n case GGML_OP_TIMESTEP_EMBEDDING:\n case GGML_OP_ARGSORT:\n case GGML_OP_TOP_K:\n case GGML_OP_FLASH_ATTN_EXT:\n case GGML_OP_FLASH_ATTN_BACK:\n case GGML_OP_SSM_CONV:\n case GGML_OP_SSM_SCAN:\n case GGML_OP_RWKV_WKV6:\n case GGML_OP_GATED_LINEAR_ATTN:\n case GGML_OP_RWKV_WKV7:\n {\n n_tasks = n_threads;\n } break;\n case GGML_OP_WIN_PART:\n case GGML_OP_WIN_UNPART:\n case GGML_OP_GET_REL_POS:\n {\n n_tasks = 1;\n } break;\n case GGML_OP_MAP_CUSTOM1:\n {\n struct ggml_map_custom1_op_params p;\n memcpy(&p, node->op_params, sizeof(p));\n if (p.n_tasks == GGML_N_TASKS_MAX) {\n n_tasks = n_threads;\n } else {\n n_tasks = MIN(p.n_tasks, n_threads);\n }\n } break;\n case GGML_OP_MAP_CUSTOM2:\n {\n struct ggml_map_custom2_op_params p;\n memcpy(&p, node->op_params, sizeof(p));\n if (p.n_tasks == GGML_N_TASKS_MAX) {\n n_tasks = n_threads;\n } else {\n n_tasks = MIN(p.n_tasks, n_threads);\n }\n } break;\n case GGML_OP_MAP_CUSTOM3:\n {\n struct ggml_map_custom3_op_params p;\n memcpy(&p, node->op_params, sizeof(p));\n if (p.n_tasks == GGML_N_TASKS_MAX) {\n n_tasks = n_threads;\n } else {\n n_tasks = MIN(p.n_tasks, n_threads);\n }\n } break;\n case GGML_OP_CUSTOM:\n {\n struct ggml_custom_op_params p;\n memcpy(&p, node->op_params, sizeof(p));\n if (p.n_tasks == GGML_N_TASKS_MAX) {\n n_tasks = n_threads;\n } else {\n n_tasks = MIN(p.n_tasks, n_threads);\n }\n } break;\n case GGML_OP_CROSS_ENTROPY_LOSS:\n case GGML_OP_CROSS_ENTROPY_LOSS_BACK:\n case GGML_OP_OPT_STEP_ADAMW:\n case GGML_OP_OPT_STEP_SGD:\n {\n n_tasks = n_threads;\n } break;\n case GGML_OP_NONE:\n {\n n_tasks = 1;\n } break;\n case GGML_OP_COUNT:\n {\n GGML_ABORT(\"fatal error\");\n }\n default:\n {\n fprintf(stderr, \"%s: op not implemented: \", __func__);\n if (node->op < GGML_OP_COUNT) {\n fprintf(stderr, \"%s\\n\", ggml_op_name(node->op));\n } else {\n fprintf(stderr, \"%d\\n\", node->op);\n }\n GGML_ABORT(\"fatal error\");\n }\n }\n\n assert(n_tasks > 0);\n\n return n_tasks;\n}\n\nstatic thread_ret_t ggml_graph_compute_secondary_thread(void* data);\n\n#if defined(_WIN32)\n#include \"windows.h\"\n\n// TODO: support > 64 CPUs\nstatic bool ggml_thread_apply_affinity(bool * mask) {\n HANDLE h = GetCurrentThread();\n uint64_t bitmask = 0ULL;\n\n assert(GGML_MAX_N_THREADS >= 64);\n\n for (int32_t i = 0; i < 8; i++) {\n int32_t idx = i * 8;\n uint8_t val = 0;\n val |= mask[idx + 0] << 0;\n val |= mask[idx + 1] << 1;\n val |= mask[idx + 2] << 2;\n val |= mask[idx + 3] << 3;\n val |= mask[idx + 4] << 4;\n val |= mask[idx + 5] << 5;\n val |= mask[idx + 6] << 6;\n val |= mask[idx + 7] << 7;\n bitmask |= (uint64_t)val << idx;\n }\n\n for (int32_t i = 64; i < GGML_MAX_N_THREADS; i++) {\n if (mask[i]) {\n fprintf(stderr, \"warn: setting thread-affinity for > 64 CPUs isn't supported on windows!\\n\");\n break;\n }\n }\n\n DWORD_PTR m = (DWORD_PTR)bitmask;\n\n m = SetThreadAffinityMask(h, m);\n\n return m != 0;\n}\n\nstatic bool ggml_thread_apply_priority(int32_t prio) {\n // Note that on Windows the Process Priority Class must be updated in order to set Thread priority.\n // This is up to the applications.\n DWORD p = THREAD_PRIORITY_NORMAL;\n switch (prio) {\n case GGML_SCHED_PRIO_LOW: p = THREAD_PRIORITY_BELOW_NORMAL; break;\n case GGML_SCHED_PRIO_NORMAL: p = THREAD_PRIORITY_NORMAL; break;\n case GGML_SCHED_PRIO_MEDIUM: p = THREAD_PRIORITY_ABOVE_NORMAL; break;\n case GGML_SCHED_PRIO_HIGH: p = THREAD_PRIORITY_HIGHEST; break;\n case GGML_SCHED_PRIO_REALTIME: p = THREAD_PRIORITY_TIME_CRITICAL; break;\n }\n\n if (prio != GGML_SCHED_PRIO_LOW) {\n // Tell Windows that this thread should not be throttled (needs its own CPU core).\n // Newer Windows 11 versions aggressively park (offline) CPU cores and often place\n // all our threads onto the first 4 cores which results in terrible performance with\n // n_threads > 4\n #if _WIN32_WINNT >= 0x0602\n THREAD_POWER_THROTTLING_STATE t;\n ZeroMemory(&t, sizeof(t));\n t.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION;\n t.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;\n t.StateMask = 0;\n\n if (!SetThreadInformation(GetCurrentThread(), ThreadPowerThrottling, &t, sizeof(t))) {\n GGML_LOG_DEBUG(\"failed to disable thread power throttling %d : (%d)\\n\", prio, (int) GetLastError());\n return false;\n }\n #endif\n }\n\n if (prio == GGML_SCHED_PRIO_NORMAL) {\n // Keep inherited policy/priority\n return true;\n }\n\n if (!SetThreadPriority(GetCurrentThread(), p)) {\n fprintf(stderr, \"warn: failed to set thread priority %d : (%d)\\n\", prio, (int) GetLastError());\n return false;\n }\n\n return true;\n}\n\n#elif defined(__APPLE__)\n#include \n#include \n\nstatic bool ggml_thread_apply_affinity(const bool * mask) {\n // Not supported on Apple platforms\n UNUSED(mask);\n return true;\n}\n\nstatic bool ggml_thread_apply_priority(int32_t prio) {\n struct sched_param p;\n int32_t policy = SCHED_OTHER;\n switch (prio) {\n // TODO: there seems to be no way to set lower prio on Apple platforms\n case GGML_SCHED_PRIO_LOW: policy = SCHED_OTHER; p.sched_priority = 0; break;\n case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER; p.sched_priority = 0; break;\n case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO; p.sched_priority = 40; break;\n case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO; p.sched_priority = 80; break;\n case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO; p.sched_priority = 90; break;\n }\n\n if (prio == GGML_SCHED_PRIO_NORMAL) {\n // Keep inherited policy/priority\n return true;\n }\n\n int32_t err = pthread_setschedparam(pthread_self(), policy, &p);\n if (err != 0) {\n fprintf(stderr, \"warn: failed to set thread priority %d : %s (%d)\\n\", prio, strerror(err), err);\n return false;\n }\n\n return true;\n}\n\n#elif defined(__gnu_linux__)\n// TODO: this may not work on BSD, to be verified\n\nstatic bool ggml_thread_apply_affinity(const bool * mask) {\n cpu_set_t cpuset;\n int err;\n\n CPU_ZERO(&cpuset);\n\n for (uint32_t i = 0; i < GGML_MAX_N_THREADS; i++) {\n if (mask[i]) {\n GGML_PRINT_DEBUG(\"Thread %lx: adding %d to cpuset\\n\", pthread_self(), i);\n CPU_SET(i, &cpuset);\n }\n }\n\n#ifdef __ANDROID__\n err = sched_setaffinity(0, sizeof(cpuset), &cpuset);\n if (err < 0) {\n err = errno;\n }\n#else\n err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);\n#endif\n if (err != 0) {\n fprintf(stderr, \"warn: failed to set affinity mask 0x%llx : %s (%d)\\n\", (unsigned long long)mask, strerror(err), err);\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_thread_apply_priority(int32_t prio) {\n struct sched_param p;\n int32_t policy = SCHED_OTHER;\n switch (prio) {\n case GGML_SCHED_PRIO_LOW: policy = SCHED_BATCH; p.sched_priority = 0; break;\n case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER; p.sched_priority = 0; break;\n case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO; p.sched_priority = 40; break;\n case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO; p.sched_priority = 80; break;\n case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO; p.sched_priority = 90; break;\n }\n\n if (prio == GGML_SCHED_PRIO_NORMAL) {\n // Keep inherited policy/priority\n return true;\n }\n\n int32_t err = pthread_setschedparam(pthread_self(), policy, &p);\n if (err != 0) {\n fprintf(stderr, \"warn: failed to set thread priority %d : %s (%d)\\n\", prio, strerror(err), err);\n return false;\n }\n\n return true;\n}\n\n#else // unsupported platforms\n\nstatic bool ggml_thread_apply_affinity(const bool * mask) {\n UNUSED(mask);\n return true;\n}\n\nstatic bool ggml_thread_apply_priority(int32_t prio) {\n UNUSED(prio);\n return true;\n}\n\n#endif\n\nstatic bool ggml_thread_cpumask_is_valid(const bool * mask) {\n for (int i = 0; i < GGML_MAX_N_THREADS; i++) {\n if (mask[i]) { return true; }\n }\n return false;\n}\n\nstatic void ggml_thread_cpumask_next(const bool * global_mask, bool * local_mask, bool strict, int32_t* iter) {\n if (!strict) {\n memcpy(local_mask, global_mask, GGML_MAX_N_THREADS);\n return;\n } else {\n memset(local_mask, 0, GGML_MAX_N_THREADS);\n int32_t base_idx = *iter;\n for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {\n int32_t idx = base_idx + i;\n if (idx >= GGML_MAX_N_THREADS) {\n // Just a cheaper modulo\n idx -= GGML_MAX_N_THREADS;\n }\n if (global_mask[idx]) {\n local_mask[idx] = 1;\n *iter = idx + 1;\n return;\n }\n }\n }\n}\n\nvoid ggml_threadpool_free(struct ggml_threadpool* threadpool) {\n if (!threadpool) return;\n\n const int n_threads = threadpool->n_threads;\n\n#ifndef GGML_USE_OPENMP\n struct ggml_compute_state* workers = threadpool->workers;\n\n ggml_mutex_lock(&threadpool->mutex);\n\n threadpool->stop = true;\n threadpool->pause = false;\n\n ggml_cond_broadcast(&threadpool->cond);\n ggml_mutex_unlock(&threadpool->mutex);\n\n for (int j = 1; j < n_threads; j++) {\n int32_t rc = ggml_thread_join(workers[j].thrd, NULL);\n GGML_ASSERT(rc == GGML_EXIT_SUCCESS || rc == GGML_EXIT_ABORTED);\n UNUSED(rc);\n }\n\n ggml_mutex_destroy(&threadpool->mutex);\n ggml_cond_destroy(&threadpool->cond);\n#endif // GGML_USE_OPENMP\n\n const size_t workers_size = sizeof(struct ggml_compute_state) * n_threads;\n ggml_aligned_free(threadpool->workers, workers_size);\n ggml_aligned_free(threadpool, sizeof(struct ggml_threadpool));\n}\n\n#ifndef GGML_USE_OPENMP\n// pause/resume must be called under mutex\nstatic void ggml_threadpool_pause_locked(struct ggml_threadpool * threadpool) {\n GGML_PRINT_DEBUG(\"Pausing threadpool\\n\");\n threadpool->pause = true;\n ggml_cond_broadcast(&threadpool->cond);\n}\n\nstatic void ggml_threadpool_resume_locked(struct ggml_threadpool * threadpool) {\n GGML_PRINT_DEBUG(\"Resuming threadpool\\n\");\n threadpool->pause = false;\n ggml_cond_broadcast(&threadpool->cond);\n}\n#endif\n\nvoid ggml_threadpool_pause(struct ggml_threadpool * threadpool) {\n#ifndef GGML_USE_OPENMP\n ggml_mutex_lock(&threadpool->mutex);\n if (!threadpool->pause) {\n ggml_threadpool_pause_locked(threadpool);\n }\n ggml_mutex_unlock(&threadpool->mutex);\n#else\n UNUSED(threadpool);\n#endif\n}\n\nvoid ggml_threadpool_resume(struct ggml_threadpool * threadpool) {\n#ifndef GGML_USE_OPENMP\n ggml_mutex_lock(&threadpool->mutex);\n if (threadpool->pause) {\n ggml_threadpool_resume_locked(threadpool);\n }\n ggml_mutex_unlock(&threadpool->mutex);\n#else\n UNUSED(threadpool);\n#endif\n}\n\nstruct ggml_cplan ggml_graph_plan(\n const struct ggml_cgraph * cgraph,\n int n_threads,\n struct ggml_threadpool * threadpool) {\n\n if (threadpool == NULL) {\n //GGML_PRINT_DEBUG(\"Threadpool is not specified. Will create a disposable threadpool : n_threads %d\\n\", n_threads);\n }\n if (n_threads <= 0) {\n n_threads = threadpool ? threadpool->n_threads : GGML_DEFAULT_N_THREADS;\n }\n\n#if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__)\n // Emscripten without pthreads support can only use a single thread\n n_threads = 1;\n#endif\n\n size_t work_size = 0;\n\n struct ggml_cplan cplan;\n memset(&cplan, 0, sizeof(struct ggml_cplan));\n\n int max_tasks = 1;\n\n // thread scheduling for the different operations + work buffer size estimation\n for (int i = 0; i < cgraph->n_nodes; i++) {\n struct ggml_tensor * node = cgraph->nodes[i];\n\n const int n_tasks = ggml_get_n_tasks(node, n_threads);\n\n max_tasks = MAX(max_tasks, n_tasks);\n\n size_t cur = 0;\n\n if (!ggml_cpu_extra_work_size(n_threads, node, &cur)) {\n switch (node->op) {\n case GGML_OP_CPY:\n case GGML_OP_DUP:\n {\n if (ggml_is_quantized(node->type) ||\n // F16 -> BF16 and BF16 -> F16 copies go through intermediate F32\n (node->src[0]->type == GGML_TYPE_F16 && node->src[1] && node->src[1]->type == GGML_TYPE_BF16) ||\n (node->src[0]->type == GGML_TYPE_BF16 && node->src[1] && node->src[1]->type == GGML_TYPE_F16) ||\n // conversion between F32 and I32\n (node->src[0]->type == GGML_TYPE_F32 && node->src[1] && node->src[1]->type == GGML_TYPE_I32) ||\n (node->src[0]->type == GGML_TYPE_I32 && node->src[1] && node->src[1]->type == GGML_TYPE_F32)) {\n cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;\n }\n } break;\n case GGML_OP_ADD:\n case GGML_OP_ADD_ID:\n case GGML_OP_ADD1:\n {\n if (ggml_is_quantized(node->src[0]->type)) {\n cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;\n }\n } break;\n case GGML_OP_ACC:\n {\n if (ggml_is_quantized(node->src[0]->type)) {\n cur = ggml_type_size(GGML_TYPE_F32) * node->src[1]->ne[0] * n_tasks;\n }\n } break;\n case GGML_OP_COUNT_EQUAL:\n {\n cur = ggml_type_size(node->type)*n_tasks;\n } break;\n case GGML_OP_MUL_MAT:\n {\n const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type;\n\n if (node->src[1]->type != vec_dot_type) {\n cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1]));\n }\n } break;\n case GGML_OP_MUL_MAT_ID:\n {\n cur = 0;\n const struct ggml_tensor * src0 = node->src[0];\n const struct ggml_tensor * src1 = node->src[1];\n const struct ggml_tensor * ids = node->src[2];\n const enum ggml_type vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;\n const int n_as = src0->ne[2];\n // src1\n if (src1->type != vec_dot_type) {\n cur += ggml_row_size(vec_dot_type, ggml_nelements(src1)) + sizeof(int64_t);\n }\n // matrix_row_counts\n cur += n_as * sizeof(int64_t) + sizeof(int64_t);\n // matrix_rows\n cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t);\n // atomic_current_chunk\n cur += CACHE_LINE_SIZE*n_as + CACHE_LINE_SIZE;\n } break;\n case GGML_OP_OUT_PROD:\n {\n if (ggml_is_quantized(node->src[0]->type)) {\n cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;\n }\n } break;\n case GGML_OP_SOFT_MAX:\n case GGML_OP_ROPE:\n case GGML_OP_ROPE_BACK:\n {\n cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;\n } break;\n case GGML_OP_CONV_TRANSPOSE_1D:\n {\n GGML_ASSERT(node->src[0]->ne[3] == 1);\n GGML_ASSERT(node->src[1]->ne[2] == 1);\n GGML_ASSERT(node->src[1]->ne[3] == 1);\n\n const int64_t ne00 = node->src[0]->ne[0]; // K\n const int64_t ne01 = node->src[0]->ne[1]; // Cout\n const int64_t ne02 = node->src[0]->ne[2]; // Cin\n const int64_t ne10 = node->src[1]->ne[0]; // L\n const int64_t ne11 = node->src[1]->ne[1]; // Cin\n\n if ((node->src[0]->type == GGML_TYPE_F16 ||\n node->src[0]->type == GGML_TYPE_BF16) &&\n node->src[1]->type == GGML_TYPE_F32) {\n cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;\n cur += sizeof(ggml_fp16_t)*ne10*ne11;\n } else if (node->src[0]->type == GGML_TYPE_F32 &&\n node->src[1]->type == GGML_TYPE_F32) {\n cur += sizeof(float)*ne00*ne01*ne02;\n cur += sizeof(float)*ne10*ne11;\n } else {\n GGML_ABORT(\"fatal error\");\n }\n } break;\n case GGML_OP_CONV_2D:\n case GGML_OP_CONV_3D:\n {\n cur = GGML_IM2COL_WORK_SIZE;\n } break;\n case GGML_OP_CONV_TRANSPOSE_2D:\n {\n const int64_t ne00 = node->src[0]->ne[0]; // W\n const int64_t ne01 = node->src[0]->ne[1]; // H\n const int64_t ne02 = node->src[0]->ne[2]; // Channels Out\n const int64_t ne03 = node->src[0]->ne[3]; // Channels In\n\n const int64_t ne10 = node->src[1]->ne[0]; // W\n const int64_t ne11 = node->src[1]->ne[1]; // H\n const int64_t ne12 = node->src[1]->ne[2]; // Channels In\n\n cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02*ne03;\n cur += sizeof(ggml_fp16_t)*ne10*ne11*ne12;\n } break;\n case GGML_OP_TOP_K:\n {\n cur += sizeof(int32_t)*node->src[0]->ne[0]*n_tasks;\n } break;\n case GGML_OP_FLASH_ATTN_EXT:\n {\n const int64_t neq2 = node->src[0]->ne[2]; // number of query heads\n const int64_t DK = node->src[1]->ne[0];\n const int64_t DV = node->src[2]->ne[0];\n\n // Tiled flash attention scratch (tile sizes defined in common.h)\n // Per-thread: Q_q + KQ + mask + VKQ32 + V32 + K_f32 + padding\n size_t prefill = sizeof(float)*(GGML_FA_TILE_Q*DK + 2*GGML_FA_TILE_Q*GGML_FA_TILE_KV + GGML_FA_TILE_Q*DV + GGML_FA_TILE_KV*DV + GGML_FA_TILE_KV*DK)*n_tasks;\n\n // Decode path: n_kv_chunks = n_tasks (one chunk per thread)\n // Per-thread: VKQ accmulator (DV), partial M, partial S + intra-thread scratch for V, Q and VKQ\n size_t n_chunks = n_tasks;\n size_t decode = sizeof(float)*(neq2*n_chunks*(2+DV) + n_tasks*(DK + 2*DV));\n\n cur += MAX(prefill, decode);\n } break;\n case GGML_OP_FLASH_ATTN_BACK:\n {\n const int64_t D = node->src[0]->ne[0];\n const int64_t ne11 = ggml_up(node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL);\n const int64_t mxDn = MAX(D, ne11) * 2; // *2 because of S and SM in ggml_compute_forward_flash_attn_back\n if (node->src[1]->type == GGML_TYPE_F32) {\n cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)\n cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2\n } else if (node->src[1]->type == GGML_TYPE_F16) {\n cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)\n cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2\n } else if (node->src[1]->type == GGML_TYPE_BF16) {\n cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)\n cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2\n }\n } break;\n\n case GGML_OP_CROSS_ENTROPY_LOSS:\n {\n cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);\n } break;\n case GGML_OP_GATED_DELTA_NET:\n {\n const int64_t S_v = node->src[2]->ne[0];\n cur = S_v * sizeof(float) * n_tasks;\n } break;\n case GGML_OP_COUNT:\n {\n GGML_ABORT(\"fatal error\");\n }\n default:\n break;\n }\n }\n\n work_size = MAX(work_size, cur);\n }\n\n if (work_size > 0) {\n work_size += CACHE_LINE_SIZE*(n_threads);\n }\n\n cplan.threadpool = threadpool;\n cplan.n_threads = MIN(max_tasks, n_threads);\n cplan.work_size = work_size;\n cplan.work_data = NULL;\n\n return cplan;\n}\n\nstatic thread_ret_t ggml_graph_compute_thread(void * data) {\n struct ggml_compute_state * state = (struct ggml_compute_state *) data;\n struct ggml_threadpool * tp = state->threadpool;\n\n const struct ggml_cgraph * cgraph = tp->cgraph;\n const struct ggml_cplan * cplan = tp->cplan;\n\n set_numa_thread_affinity(state->ith);\n\n struct ggml_compute_params params = {\n /*.ith =*/ state->ith,\n /*.nth =*/ atomic_load_explicit(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK,\n /*.wsize =*/ cplan->work_size,\n /*.wdata =*/ cplan->work_data,\n /*.threadpool =*/ tp,\n /*.use_ref =*/ cplan->use_ref,\n };\n\n#ifdef GGML_USE_OPENMP\n GGML_PRINT_DEBUG(\"thread #%d compute-start cplan %p\\n\", state->ith, (const void *)cplan);\n#else\n GGML_PRINT_DEBUG(\"thread #%d compute-start cplan %p last-graph %d\\n\", state->ith, (const void *)cplan, state->last_graph);\n#endif\n\n for (int node_n = 0; node_n < cgraph->n_nodes && atomic_load_explicit(&tp->abort, memory_order_relaxed) != node_n; node_n++) {\n struct ggml_tensor * node = cgraph->nodes[node_n];\n\n if (ggml_op_is_empty(node->op)) {\n // skip NOPs\n continue;\n }\n\n if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n continue;\n }\n\n ggml_compute_forward(¶ms, node);\n\n if (state->ith == 0 && cplan->abort_callback &&\n cplan->abort_callback(cplan->abort_callback_data)) {\n atomic_store_explicit(&tp->abort, node_n + 1, memory_order_relaxed);\n tp->ec = GGML_STATUS_ABORTED;\n }\n\n if (node_n + 1 < cgraph->n_nodes) {\n ggml_barrier(state->threadpool);\n }\n }\n\n#ifdef GGML_USE_OPENMP\n GGML_PRINT_DEBUG(\"thread #%d compute-done cplan %p\\n\", state->ith, (const void *)cplan);\n#else\n GGML_PRINT_DEBUG(\"thread #%d compute-done cplan %p last-graph %d\\n\", state->ith, (const void *)cplan, state->last_graph);\n#endif\n\n ggml_barrier(state->threadpool);\n\n return 0;\n}\n\n#ifndef GGML_USE_OPENMP\n\n// check if thread is ready to proceed (exit from polling or sleeping)\n// returns true if loops should exit, sets state->pending to indicate new work\nstatic inline bool ggml_graph_compute_thread_ready(struct ggml_compute_state * state) {\n struct ggml_threadpool * threadpool = state->threadpool;\n\n if (state->pending || threadpool->stop || threadpool->pause) { return true; }\n\n // check for new graph/work\n int n_graph = atomic_load_explicit(&threadpool->n_graph, memory_order_relaxed);\n int n_threads = n_graph & GGML_THREADPOOL_N_THREADS_MASK;\n if (n_graph != state->last_graph) {\n state->pending = (state->ith < n_threads);\n state->last_graph = n_graph;\n return true;\n }\n\n return false;\n}\n\n// sync thread state after polling\nstatic inline void ggml_graph_compute_thread_sync(struct ggml_compute_state * state) {\n // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead\n #ifdef GGML_TSAN_ENABLED\n atomic_fetch_add_explicit(&state->threadpool->n_graph, 0, memory_order_seq_cst);\n #else\n atomic_thread_fence(memory_order_seq_cst);\n #endif\n UNUSED(state);\n}\n\nstatic inline bool ggml_graph_compute_poll_for_work(struct ggml_compute_state * state) {\n struct ggml_threadpool * threadpool = state->threadpool;\n\n // This seems to make 0 ... 100 a decent range for polling level across modern processors.\n // Perhaps, we can adjust it dynamically based on load and things.\n const uint64_t n_rounds = 1024UL * 128 * threadpool->poll;\n\n for (uint64_t i=0; !ggml_graph_compute_thread_ready(state) && i < n_rounds; i++) {\n // No new work. Keep polling.\n ggml_thread_cpu_relax();\n }\n\n return state->pending;\n}\n\nstatic inline bool ggml_graph_compute_check_for_work(struct ggml_compute_state * state) {\n struct ggml_threadpool * threadpool = state->threadpool;\n\n if (ggml_graph_compute_poll_for_work(state)) {\n ggml_graph_compute_thread_sync(state);\n return state->pending;\n }\n\n ggml_mutex_lock_shared(&threadpool->mutex);\n while (!ggml_graph_compute_thread_ready(state)) {\n // No new work. Wait for the signal.\n GGML_PRINT_DEBUG(\"thread #%d waiting for work (sleeping)\\n\", state->ith);\n ggml_cond_wait(&threadpool->cond, &threadpool->mutex);\n }\n ggml_mutex_unlock_shared(&threadpool->mutex);\n\n return state->pending;\n}\n\nstatic thread_ret_t ggml_graph_compute_secondary_thread(void* data) {\n struct ggml_compute_state * state = (struct ggml_compute_state *) data;\n struct ggml_threadpool * threadpool = state->threadpool;\n\n ggml_thread_apply_priority(threadpool->prio);\n if (ggml_thread_cpumask_is_valid(state->cpumask)) {\n ggml_thread_apply_affinity(state->cpumask);\n }\n\n while (true) {\n // Check if we need to sleep\n while (threadpool->pause) {\n GGML_PRINT_DEBUG(\"thread #%d inside pause loop\\n\", state->ith);\n ggml_mutex_lock_shared(&threadpool->mutex);\n if (threadpool->pause) {\n ggml_cond_wait(&threadpool->cond, &threadpool->mutex);\n }\n GGML_PRINT_DEBUG(\"thread #%d resuming after wait\\n\", state->ith);\n ggml_mutex_unlock_shared(&threadpool->mutex);\n }\n\n // This needs to be checked for after the cond_wait\n if (threadpool->stop) break;\n\n // Check if there is new work\n // The main thread is the only one that can dispatch new work\n\n ggml_graph_compute_check_for_work(state);\n if (state->pending) {\n state->pending = false;\n ggml_graph_compute_thread(state);\n }\n }\n\n return (thread_ret_t) 0;\n}\n\n// Start processing new graph\nstatic void ggml_graph_compute_kickoff(struct ggml_threadpool * threadpool, int n_threads)\n{\n // Always take the mutex here because the worker threads are doing hybrid poll/wait\n\n ggml_mutex_lock(&threadpool->mutex);\n\n // Update the number of active threads and the graph count\n int n_graph = atomic_load_explicit(&threadpool->n_graph, memory_order_relaxed) >> GGML_THREADPOOL_N_THREADS_BITS;\n n_graph = ((n_graph + 1) << GGML_THREADPOOL_N_THREADS_BITS) | (n_threads & GGML_THREADPOOL_N_THREADS_MASK);\n\n GGML_PRINT_DEBUG(\"compute-kickoff: n_threads %d n_graph %d\\n\", n_threads, n_graph);\n\n // Indicate the graph is ready to be processed\n // We need the full seq-cst fence here because of the polling threads (used in thread_sync)\n atomic_store_explicit(&threadpool->n_graph, n_graph, memory_order_seq_cst);\n\n if (threadpool->pause) {\n // Update main thread prio and affinity to match the threadpool settings\n ggml_thread_apply_priority(threadpool->prio);\n if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {\n ggml_thread_apply_affinity(threadpool->workers[0].cpumask);\n }\n\n // resume does cond broadcast\n ggml_threadpool_resume_locked(threadpool);\n } else {\n ggml_cond_broadcast(&threadpool->cond);\n }\n\n ggml_mutex_unlock(&threadpool->mutex);\n}\n\n#endif // GGML_USE_OPENMP\n\nstatic struct ggml_threadpool * ggml_threadpool_new_impl(\n struct ggml_threadpool_params * tpp,\n struct ggml_cgraph * cgraph,\n struct ggml_cplan * cplan) {\n\n struct ggml_threadpool * threadpool =\n ggml_aligned_malloc(sizeof(struct ggml_threadpool));\n {\n threadpool->cgraph = cgraph;\n threadpool->cplan = cplan;\n threadpool->n_graph = 0;\n threadpool->n_barrier = 0;\n threadpool->n_barrier_passed = 0;\n threadpool->current_chunk = 0;\n threadpool->stop = false;\n threadpool->pause = tpp->paused;\n threadpool->abort = -1;\n threadpool->workers = NULL;\n threadpool->n_threads = tpp->n_threads;\n threadpool->poll = tpp->poll;\n threadpool->prio = tpp->prio;\n threadpool->ec = GGML_STATUS_SUCCESS;\n }\n\n // Allocate and init workers state\n const size_t workers_size = sizeof(struct ggml_compute_state) * tpp->n_threads;\n struct ggml_compute_state * workers = ggml_aligned_malloc(workers_size);\n\n memset(workers, 0, workers_size);\n for (int j = 0; j < tpp->n_threads; j++) {\n workers[j].threadpool = threadpool;\n workers[j].ith = j;\n }\n\n threadpool->workers = workers;\n\n#ifdef GGML_USE_OPENMP\n int32_t cpumask_iter = 0;\n\n // Compute CPU masks for each thread\n for (int j = 0; j < tpp->n_threads; j++) {\n ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);\n }\n#else // GGML_USE_OPENMP\n ggml_mutex_init(&threadpool->mutex);\n ggml_cond_init(&threadpool->cond);\n\n // Spin the threads for all workers, and update CPU placements.\n // Place the main thread last (towards the higher numbered CPU cores).\n\n int32_t cpumask_iter = 0;\n\n for (int j = 1; j < tpp->n_threads; j++) {\n ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);\n\n int32_t rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_secondary_thread, &workers[j]);\n GGML_ASSERT(rc == 0);\n }\n\n ggml_thread_cpumask_next(tpp->cpumask, workers[0].cpumask, tpp->strict_cpu, &cpumask_iter);\n\n if (!threadpool->pause) {\n // Update main thread prio and affinity at the start, otherwise we'll do it in resume\n ggml_thread_apply_priority(threadpool->prio);\n if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {\n ggml_thread_apply_affinity(threadpool->workers[0].cpumask);\n }\n }\n#endif // GGML_USE_OPENMP\n\n return threadpool;\n}\n\nstruct ggml_threadpool * ggml_threadpool_new(struct ggml_threadpool_params * tpp) {\n return ggml_threadpool_new_impl(tpp, NULL, NULL);\n}\n\nenum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {\n ggml_cpu_init();\n\n GGML_ASSERT(cplan);\n GGML_ASSERT(cplan->n_threads > 0);\n GGML_ASSERT(cplan->work_size == 0 || cplan->work_data != NULL);\n\n int n_threads = cplan->n_threads;\n struct ggml_threadpool * threadpool = cplan->threadpool;\n\n bool disposable_threadpool = false;\n\n if (threadpool == NULL) {\n //GGML_PRINT_DEBUG(\"Threadpool is not specified. Will create a disposable threadpool : n_threads %d\\n\", n_threads);\n disposable_threadpool = true;\n\n struct ggml_threadpool_params ttp = ggml_threadpool_params_default(n_threads);\n threadpool = ggml_threadpool_new_impl(&ttp, cgraph, cplan);\n } else {\n // Reset some of the parameters that need resetting\n // No worker threads should be accessing the parameters below at this stage\n threadpool->cgraph = cgraph;\n threadpool->cplan = cplan;\n threadpool->current_chunk = 0;\n threadpool->abort = -1;\n threadpool->ec = GGML_STATUS_SUCCESS;\n }\n\n#ifdef GGML_USE_OPENMP\n if (n_threads > 1) {\n #pragma omp parallel num_threads(n_threads)\n {\n #pragma omp single\n {\n // update the number of threads from the actual number of threads that we got from OpenMP\n n_threads = omp_get_num_threads();\n atomic_store_explicit(&threadpool->n_graph, n_threads, memory_order_relaxed);\n }\n\n // Apply thread CPU mask and priority\n int ith = omp_get_thread_num();\n\n ggml_thread_apply_priority(threadpool->prio);\n if (ggml_thread_cpumask_is_valid(threadpool->workers[ith].cpumask)) {\n ggml_thread_apply_affinity(threadpool->workers[ith].cpumask);\n }\n ggml_graph_compute_thread(&threadpool->workers[ith]);\n }\n } else {\n atomic_store_explicit(&threadpool->n_graph, 1, memory_order_relaxed);\n ggml_graph_compute_thread(&threadpool->workers[0]);\n }\n#else\n if (n_threads > threadpool->n_threads) {\n GGML_LOG_WARN(\"cplan requested more threads (%d) than available (%d)\\n\", n_threads, threadpool->n_threads);\n n_threads = threadpool->n_threads;\n }\n\n // Kick all threads to start the new graph\n ggml_graph_compute_kickoff(threadpool, n_threads);\n\n // This is a work thread too\n ggml_graph_compute_thread(&threadpool->workers[0]);\n#endif\n\n // don't leave affinity set on the main thread\n clear_numa_thread_affinity();\n\n enum ggml_status ret = threadpool->ec;\n\n if (disposable_threadpool) {\n ggml_threadpool_free(threadpool);\n }\n\n return ret;\n}\n\nenum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) {\n struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads, NULL);\n\n cplan.work_data = (uint8_t *)ggml_new_buffer(ctx, cplan.work_size);\n\n return ggml_graph_compute(cgraph, &cplan);\n}\n\nvoid ggml_cpu_fp32_to_fp32(const float * x, float * y, int64_t n) {\n memcpy(y, x, n * sizeof(float));\n}\n\nvoid ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) {\n int64_t i = 0;\n#if defined(__F16C__)\n#if defined(__AVX512F__)\n for (; i + 15 < n; i += 16) {\n __m512 x_vec = _mm512_loadu_ps(x + i);\n __m256i y_vec = _mm512_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);\n _mm256_storeu_si256((__m256i *)(y + i), y_vec);\n }\n#endif\n for (; i + 7 < n; i += 8) {\n __m256 x_vec = _mm256_loadu_ps(x + i);\n __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);\n _mm_storeu_si128((__m128i *)(y + i), y_vec);\n }\n for (; i + 3 < n; i += 4) {\n __m128 x_vec = _mm_loadu_ps(x + i);\n __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);\n _mm_storel_epi64((__m128i *)(y + i), y_vec);\n }\n#elif defined(__riscv_zvfh)\n for (int vl; i < n; i += vl) {\n vl = __riscv_vsetvl_e32m2(n - i);\n vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);\n vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl);\n __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl);\n }\n#endif\n for (; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(x[i]);\n }\n}\n\nvoid ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) {\n int64_t i = 0;\n#if defined(__F16C__)\n#if defined(__AVX512F__)\n for (; i + 15 < n; i += 16) {\n __m256i x_vec = _mm256_loadu_si256((const __m256i *)(x + i));\n __m512 y_vec = _mm512_cvtph_ps(x_vec);\n _mm512_storeu_ps(y + i, y_vec);\n }\n#endif\n for (; i + 7 < n; i += 8) {\n __m128i x_vec = _mm_loadu_si128((const __m128i *)(x + i));\n __m256 y_vec = _mm256_cvtph_ps(x_vec);\n _mm256_storeu_ps(y + i, y_vec);\n }\n for (; i + 3 < n; i += 4) {\n __m128i x_vec = _mm_loadl_epi64((const __m128i *)(x + i));\n __m128 y_vec = _mm_cvtph_ps(x_vec);\n _mm_storeu_ps(y + i, y_vec);\n }\n\n#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin)\n // calculate step size\n const int epr = __riscv_vsetvlmax_e16m2();\n const int step = epr * 2;\n const int np = (n & ~(step - 1));\n\n // unroll by 2\n for (; i < np; i += step) {\n vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, epr);\n vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, epr);\n __riscv_vse32_v_f32m4(y + i, ay0, epr);\n\n vfloat16m2_t ax1 = __riscv_vle16_v_f16m2((const _Float16*)x + i + epr, epr);\n vfloat32m4_t ay1 = __riscv_vfwcvt_f_f_v_f32m4(ax1, epr);\n __riscv_vse32_v_f32m4(y + i + epr, ay1, epr);\n }\n\n // leftovers\n int vl;\n for (i = np; i < n; i += vl) {\n vl = __riscv_vsetvl_e16m2(n - i);\n vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, vl);\n vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, vl);\n __riscv_vse32_v_f32m4(y + i, ay0, vl);\n }\n\n#endif\n\n for (; i < n; ++i) {\n y[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n }\n}\n\nvoid ggml_cpu_fp32_to_bf16(const float * x, ggml_bf16_t * y, int64_t n) {\n int64_t i = 0;\n for (; i < n; ++i) {\n y[i] = GGML_FP32_TO_BF16(x[i]);\n }\n}\n\nvoid ggml_cpu_fp32_to_i32(const float * x, int32_t * y, int64_t n) {\n int64_t i = 0;\n for (; i < n; ++i) {\n y[i] = x[i];\n }\n}\n\nvoid ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) {\n int64_t i = 0;\n#if defined(__AVX2__)\n#if defined(__AVX512F__)\n for (; i + 15 < n; i += 16) {\n _mm512_storeu_ps(y + i,\n _mm512_castsi512_ps(\n _mm512_slli_epi32(\n _mm512_cvtepu16_epi32(\n _mm256_loadu_si256(\n (const __m256i *)(x + i))),\n 16)));\n }\n#endif\n for (; i + 7 < n; i += 8) {\n _mm256_storeu_ps(y + i,\n _mm256_castsi256_ps(\n _mm256_slli_epi32(\n _mm256_cvtepu16_epi32(\n _mm_loadu_si128(\n (const __m128i *)(x + i))),\n 16)));\n }\n#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfmin)\n // calculate step size\n const int epr = __riscv_vsetvlmax_e16m2();\n const int step = epr * 2;\n const int np = (n & ~(step - 1));\n\n // unroll by 2\n for (; i < np; i += step) {\n vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, epr);\n vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, epr);\n __riscv_vse32_v_f32m4(y + i, ay0, epr);\n\n vbfloat16m2_t ax1 = __riscv_vle16_v_bf16m2((const __bf16*)x + i + epr, epr);\n vfloat32m4_t ay1 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax1, epr);\n __riscv_vse32_v_f32m4(y + i + epr, ay1, epr);\n }\n\n // leftovers\n int vl;\n for (i = np; i < n; i += vl) {\n vl = __riscv_vsetvl_e16m2(n - i);\n vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, vl);\n vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, vl);\n __riscv_vse32_v_f32m4(y + i, ay0, vl);\n }\n#endif\n for (; i < n; i++) {\n y[i] = GGML_BF16_TO_FP32(x[i]);\n }\n}\n\nint ggml_cpu_has_avx(void) {\n#if defined(__AVX__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_avx_vnni(void) {\n#if defined(__AVXVNNI__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_avx2(void) {\n#if defined(__AVX2__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_avx512(void) {\n#if defined(__AVX512F__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_avx512_vbmi(void) {\n#if defined(__AVX512VBMI__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_avx512_vnni(void) {\n#if defined(__AVX512VNNI__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_avx512_bf16(void) {\n#if defined(__AVX512BF16__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_amx_int8(void) {\n#if defined(__AMX_INT8__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_bmi2(void) {\n#if defined(__BMI2__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_fma(void) {\n#if defined(__FMA__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_arm_fma(void) {\n#if defined(__ARM_FEATURE_FMA)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_riscv_v(void) {\n#if defined(__riscv_v_intrinsic)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_get_rvv_vlen(void) {\n#if defined(__riscv) && defined(__riscv_v_intrinsic)\n return ggml_riscv_arch_features.rvv_vlen;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_f16c(void) {\n#if defined(__F16C__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_fp16_va(void) {\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_wasm_simd(void) {\n#if defined(__wasm_simd128__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_llamafile(void) {\n#if defined(GGML_USE_LLAMAFILE)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_sse3(void) {\n#if defined(__SSE3__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_ssse3(void) {\n#if defined(__SSSE3__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_vsx(void) {\n#if defined(__POWER9_VECTOR__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_vxe(void) {\n#if defined(__VXE__) || defined(__VXE2__)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_neon(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_NEON)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_dotprod(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_DOTPROD)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_sve(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_matmul_int8(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_MATMUL_INT8)\n return 1;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_get_sve_cnt(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)\n return ggml_arm_arch_features.sve_cnt;\n#else\n return 0;\n#endif\n}\n\nint ggml_cpu_has_sme(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME)\n return 1;\n#else\n return 0;\n#endif\n}\n\nvoid ggml_cpu_init(void) {\n // needed to initialize ggml_time\n {\n struct ggml_init_params params = { 0, NULL, false };\n struct ggml_context * ctx = ggml_init(params);\n ggml_free(ctx);\n }\n\n ggml_critical_section_start();\n\n static bool is_first_call = true;\n\n if (is_first_call) {\n // initialize GELU, Quick GELU, SILU and EXP F32 tables\n {\n const uint64_t t_start = ggml_time_us(); UNUSED(t_start);\n\n for (int i = 0; i < (1 << 16); ++i) {\n union {\n uint16_t u16;\n ggml_fp16_t fp16;\n } u = {i};\n float f = GGML_COMPUTE_FP16_TO_FP32(u.fp16);\n ggml_table_f32_f16[i] = f;\n ggml_table_gelu_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_f32(f));\n ggml_table_gelu_quick_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_quick_f32(f));\n }\n\n // initialize E8M0 half table (256 entries)\n for (int i = 0; i < (1 << 8); ++i) {\n ggml_table_f32_e8m0_half[i] = GGML_E8M0_TO_FP32_HALF(i);\n }\n\n const uint64_t t_end = ggml_time_us(); UNUSED(t_end);\n\n GGML_PRINT_DEBUG(\"%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\\n\", __func__, (t_end - t_start)/1000.0);\n\n#ifdef GGML_USE_OPENMP\n //if (!getenv(\"OMP_WAIT_POLICY\")) {\n // // set the wait policy to active, so that OpenMP threads don't sleep\n // setenv(\"OMP_WAIT_POLICY\", \"active\", 0)\n //}\n\n if (!getenv(\"KMP_BLOCKTIME\")) {\n // set the time to wait before sleeping a thread\n // this is less aggressive than setting the wait policy to active, but should achieve similar results in most cases\n#ifdef _WIN32\n _putenv_s(\"KMP_BLOCKTIME\", \"200\"); // 200ms\n#else\n setenv(\"KMP_BLOCKTIME\", \"200\", 0); // 200ms\n#endif\n }\n#endif\n }\n\n#if defined(__ARM_ARCH)\n ggml_init_arm_arch_features();\n#endif\n\n#if defined(__riscv)\n ggml_init_riscv_arch_features();\n#endif\n\n is_first_call = false;\n }\n\n ggml_critical_section_end();\n}\n" }, { "path": "src/ggml-cpu/ggml-cpu.cpp", "content": "#include \"ggml-backend.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"repack.h\"\n#include \"traits.h\"\n#include \"ggml-impl.h\"\n#include \"amx/amx.h\"\n\n#include \n#include \n#include \n\n#ifdef GGML_USE_CPU_HBM\n# include \"hbm.h\"\n#endif\n\n#ifdef GGML_USE_CPU_KLEIDIAI\n# include \"kleidiai/kleidiai.h\"\n#endif\n\n#ifdef GGML_USE_CPU_RISCV64_SPACEMIT\n# include \"spacemit/ime.h\"\n#endif\n\n#if defined(_WIN32)\n# define WIN32_LEAN_AND_MEAN\n# ifndef NOMINMAX\n# define NOMINMAX\n# endif\n# include \n#else\n# include \n#endif\n\n#if defined(__APPLE__)\n# include \n# include \n#endif\n\n// ggml-backend interface\n\nstd::vector & ggml_backend_cpu_get_extra_buffer_types() {\n static std::vector bufts = []() {\n std::vector bufts;\n\n#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n if (ggml_backend_amx_buffer_type()) {\n bufts.push_back(ggml_backend_amx_buffer_type());\n }\n#endif\n\n#ifdef GGML_USE_CPU_RISCV64_SPACEMIT\n if (ggml_backend_cpu_riscv64_spacemit_buffer_type()) {\n bufts.push_back(ggml_backend_cpu_riscv64_spacemit_buffer_type());\n }\n#endif\n\n#ifdef GGML_USE_CPU_KLEIDIAI\n if (ggml_backend_cpu_kleidiai_buffer_type()) {\n bufts.push_back(ggml_backend_cpu_kleidiai_buffer_type());\n }\n#endif\n\n#ifdef GGML_USE_CPU_REPACK\n if (ggml_backend_cpu_repack_buffer_type()) {\n bufts.push_back(ggml_backend_cpu_repack_buffer_type());\n }\n#endif\n\n return bufts;\n }();\n\n return bufts;\n}\n\nstatic ggml_backend_buffer_type_t * ggml_backend_cpu_device_get_extra_buffers_type(ggml_backend_dev_t device) {\n static std::vector extra_bufts = [] {\n std::vector bufts = ggml_backend_cpu_get_extra_buffer_types();\n bufts.push_back(nullptr);\n return bufts;\n }();\n\n return extra_bufts.data();\n\n GGML_UNUSED(device);\n}\n\nstatic bool ggml_backend_cpu_is_extra_buffer_type(ggml_backend_buffer_type_t buft) {\n for (auto * extra : ggml_backend_cpu_get_extra_buffer_types()) {\n if (extra == buft) {\n return true;\n }\n }\n return false;\n}\n\n// CPU backend - backend (stream)\n\nstruct ggml_backend_cpu_context {\n int n_threads;\n ggml_threadpool_t threadpool;\n\n uint8_t * work_data;\n size_t work_size;\n\n ggml_abort_callback abort_callback;\n void * abort_callback_data;\n\n bool use_ref; // use reference implementation\n};\n\nstatic const char * ggml_backend_cpu_get_name(ggml_backend_t backend) {\n return \"CPU\";\n\n GGML_UNUSED(backend);\n}\n\nstatic void ggml_backend_cpu_free(ggml_backend_t backend) {\n struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;\n delete[] cpu_ctx->work_data;\n delete cpu_ctx;\n delete backend;\n}\n\nstruct ggml_backend_plan_cpu {\n struct ggml_cplan cplan;\n struct ggml_cgraph cgraph;\n};\n\nstatic ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, const struct ggml_cgraph * cgraph) {\n struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;\n\n struct ggml_backend_plan_cpu * cpu_plan = new ggml_backend_plan_cpu;\n\n cpu_plan->cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);\n cpu_plan->cgraph = *cgraph; // FIXME: deep copy\n\n if (cpu_plan->cplan.work_size > 0) {\n cpu_plan->cplan.work_data = new uint8_t[cpu_plan->cplan.work_size];\n if (cpu_plan->cplan.work_data == NULL) {\n delete cpu_plan;\n return NULL;\n }\n }\n\n cpu_plan->cplan.abort_callback = cpu_ctx->abort_callback;\n cpu_plan->cplan.abort_callback_data = cpu_ctx->abort_callback_data;\n cpu_plan->cplan.use_ref = cpu_ctx->use_ref;\n\n return cpu_plan;\n}\n\nstatic void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {\n struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan;\n\n delete[] cpu_plan->cplan.work_data;\n delete cpu_plan;\n\n GGML_UNUSED(backend);\n}\n\nstatic enum ggml_status ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {\n struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan;\n\n return ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan);\n\n GGML_UNUSED(backend);\n}\n\nstatic enum ggml_status ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;\n\n struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);\n\n if (cpu_ctx->work_size < cplan.work_size) {\n delete[] cpu_ctx->work_data;\n cpu_ctx->work_data = new uint8_t[cplan.work_size];\n if (cpu_ctx->work_data == NULL) {\n cpu_ctx->work_size = 0;\n return GGML_STATUS_ALLOC_FAILED;\n }\n cpu_ctx->work_size = cplan.work_size;\n }\n cplan.work_data = (uint8_t *)cpu_ctx->work_data;\n\n cplan.abort_callback = cpu_ctx->abort_callback;\n cplan.abort_callback_data = cpu_ctx->abort_callback_data;\n cplan.use_ref = cpu_ctx->use_ref;\n\n return ggml_graph_compute(cgraph, &cplan);\n}\n\nstatic const struct ggml_backend_i ggml_backend_cpu_i = {\n /* .get_name = */ ggml_backend_cpu_get_name,\n /* .free = */ ggml_backend_cpu_free,\n /* .set_tensor_async = */ NULL,\n /* .get_tensor_async = */ NULL,\n /* .cpy_tensor_async = */ NULL,\n /* .synchronize = */ NULL,\n /* .graph_plan_create = */ ggml_backend_cpu_graph_plan_create,\n /* .graph_plan_free = */ ggml_backend_cpu_graph_plan_free,\n /* .graph_plan_update = */ NULL,\n /* .graph_plan_compute = */ ggml_backend_cpu_graph_plan_compute,\n /* .graph_compute = */ ggml_backend_cpu_graph_compute,\n /* .event_record = */ NULL,\n /* .event_wait = */ NULL,\n /* .graph_optimize = */ NULL,\n};\n\nstatic ggml_guid_t ggml_backend_cpu_guid(void) {\n static ggml_guid guid = { 0xaa, 0x67, 0xc7, 0x43, 0x96, 0xe6, 0xa3, 0x8a, 0xe3, 0xaf, 0xea, 0x92, 0x36, 0xbc, 0xfc, 0x89 };\n return &guid;\n}\n\nggml_backend_t ggml_backend_cpu_init(void) {\n // initialize CPU backend now to avoid slowing the first graph computation\n ggml_cpu_init();\n\n struct ggml_backend_cpu_context * ctx = new ggml_backend_cpu_context;\n if (ctx == NULL) {\n return NULL;\n }\n\n ctx->n_threads = GGML_DEFAULT_N_THREADS;\n ctx->threadpool = NULL;\n ctx->work_data = NULL;\n ctx->work_size = 0;\n ctx->abort_callback = NULL;\n ctx->abort_callback_data = NULL;\n ctx->use_ref = false;\n\n ggml_backend_t cpu_backend = new ggml_backend {\n /* .guid = */ ggml_backend_cpu_guid(),\n /* .iface = */ ggml_backend_cpu_i,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n /* .context = */ ctx,\n };\n\n if (cpu_backend == NULL) {\n delete ctx;\n return NULL;\n }\n\n return cpu_backend;\n}\n\nbool ggml_backend_is_cpu(ggml_backend_t backend) {\n return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cpu_guid());\n}\n\nvoid ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {\n GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));\n\n struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;\n ctx->n_threads = n_threads;\n}\n\nvoid ggml_backend_cpu_set_threadpool(ggml_backend_t backend_cpu, ggml_threadpool_t threadpool) {\n GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));\n\n struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;\n\n if (ctx->threadpool && ctx->threadpool != threadpool) {\n // already had a different threadpool, pause/suspend it before switching\n ggml_threadpool_pause(ctx->threadpool);\n }\n ctx->threadpool = threadpool;\n}\n\nvoid ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data) {\n GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));\n\n struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;\n ctx->abort_callback = abort_callback;\n ctx->abort_callback_data = abort_callback_data;\n}\n\nvoid ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref) {\n GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));\n\n struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;\n ctx->use_ref = use_ref;\n}\n\n// CPU backend - device\n\nstruct ggml_backend_cpu_device_context {\n std::string description = \"CPU\";\n\n ggml_backend_cpu_device_context() {\n#ifdef __APPLE__\n size_t len = 0;\n if (!sysctlbyname(\"machdep.cpu.brand_string\", NULL, &len, NULL, 0)) {\n description.resize(len);\n sysctlbyname(\"machdep.cpu.brand_string\", &description[0], &len, NULL, 0); // NOLINT\n }\n#elif defined(__linux__)\n FILE * f = fopen(\"/proc/cpuinfo\", \"r\");\n if (f) {\n char buf[1024];\n while (fgets(buf, sizeof(buf), f)) {\n if (strncmp(buf, \"model name\", 10) == 0) {\n char * p = strchr(buf, ':');\n if (p) {\n p++;\n while (std::isspace(*p)) {\n p++;\n }\n while (std::isspace(p[strlen(p) - 1])) {\n p[strlen(p) - 1] = '\\0';\n }\n description = p;\n break;\n }\n }\n }\n fclose(f);\n }\n#elif defined(_WIN32)\n HKEY hKey;\n if (RegOpenKeyEx(HKEY_LOCAL_MACHINE,\n TEXT(\"HARDWARE\\\\DESCRIPTION\\\\System\\\\CentralProcessor\\\\0\"),\n 0,\n KEY_READ,\n &hKey) == ERROR_SUCCESS) {\n DWORD cpu_brand_size = 0;\n if (RegQueryValueExA(hKey,\n \"ProcessorNameString\",\n NULL,\n NULL,\n NULL,\n &cpu_brand_size) == ERROR_SUCCESS) {\n description.resize(cpu_brand_size);\n if (RegQueryValueExA(hKey,\n \"ProcessorNameString\",\n NULL,\n NULL,\n (LPBYTE)&description[0], // NOLINT\n &cpu_brand_size) == ERROR_SUCCESS) {\n if (description.find('\\0') != std::string::npos) {\n description.resize(description.find('\\0'));\n }\n }\n }\n RegCloseKey(hKey);\n }\n#endif\n }\n};\n\nstatic const char * ggml_backend_cpu_device_get_name(ggml_backend_dev_t dev) {\n return \"CPU\";\n\n GGML_UNUSED(dev);\n}\n\nstatic const char * ggml_backend_cpu_device_get_description(ggml_backend_dev_t dev) {\n struct ggml_backend_cpu_device_context * ctx = (struct ggml_backend_cpu_device_context *)dev->context;\n\n return ctx->description.c_str();\n}\n\nstatic void ggml_backend_cpu_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n#ifdef _WIN32\n MEMORYSTATUSEX status;\n status.dwLength = sizeof(status);\n GlobalMemoryStatusEx(&status);\n *total = status.ullTotalPhys;\n *free = status.ullAvailPhys;\n#else\n long pages = sysconf(_SC_PHYS_PAGES);\n long page_size = sysconf(_SC_PAGE_SIZE);\n *total = pages * page_size;\n\n // \"free\" system memory is ill-defined, for practical purposes assume that all of it is free:\n *free = *total;\n#endif // _WIN32\n\n GGML_UNUSED(dev);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_cpu_device_get_type(ggml_backend_dev_t dev) {\n return GGML_BACKEND_DEVICE_TYPE_CPU;\n\n GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {\n props->name = ggml_backend_cpu_device_get_name(dev);\n props->description = ggml_backend_cpu_device_get_description(dev);\n props->type = ggml_backend_cpu_device_get_type(dev);\n ggml_backend_cpu_device_get_memory(dev, &props->memory_free, &props->memory_total);\n props->caps = {\n /* .async = */ false,\n /* .host_buffer = */ false,\n /* .buffer_from_host_ptr = */ true,\n /* .events = */ false,\n };\n}\n\nstatic ggml_backend_t ggml_backend_cpu_device_init_backend(ggml_backend_dev_t dev, const char * params) {\n return ggml_backend_cpu_init();\n\n GGML_UNUSED(dev);\n GGML_UNUSED(params);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cpu_device_get_buffer_type(ggml_backend_dev_t dev) {\n return ggml_backend_cpu_buffer_type();\n\n GGML_UNUSED(dev);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {\n return ggml_backend_cpu_buffer_from_ptr(ptr, size);\n\n GGML_UNUSED(dev);\n GGML_UNUSED(max_tensor_size);\n}\n\nstatic bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n\n if (op->op == GGML_OP_NONE || op->op == GGML_OP_RESHAPE || op->op == GGML_OP_VIEW || op->op == GGML_OP_PERMUTE || op->op == GGML_OP_TRANSPOSE) {\n return true;\n }\n\n // check extra buffer types\n // note: only the first sources are checked for extra buffer types to reduce overhead, increase if necessary\n for (int i = 0; i < 4; i++) {\n if (op->src[i] && op->src[i]->buffer &&\n ggml_backend_cpu_is_extra_buffer_type(op->src[i]->buffer->buft)) {\n auto * buf_extra = (ggml::cpu::extra_buffer_type *) op->src[i]->buffer->buft->context;\n return buf_extra->supports_op(dev, op);\n }\n }\n\n switch (op->op) {\n case GGML_OP_CPY:\n case GGML_OP_SET_ROWS:\n return\n op->type != GGML_TYPE_IQ3_XXS &&\n op->type != GGML_TYPE_IQ3_S &&\n op->type != GGML_TYPE_IQ2_XXS &&\n op->type != GGML_TYPE_IQ2_XS &&\n op->type != GGML_TYPE_IQ2_S &&\n op->type != GGML_TYPE_IQ1_S &&\n op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float\n case GGML_OP_MUL_MAT:\n return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type;\n case GGML_OP_SOFT_MAX_BACK: {\n if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) {\n return false;\n }\n float max_bias = 0.0f;\n\n memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float));\n\n return max_bias == 0.0f;\n }\n case GGML_OP_IM2COL_BACK:\n return src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32;\n case GGML_OP_GET_ROWS_BACK:\n return src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16;\n case GGML_OP_OUT_PROD:\n return (src0->type == GGML_TYPE_F32 || (ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) &&\n src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;\n default:\n return true;\n }\n}\n\nstatic bool ggml_backend_cpu_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n return ggml_backend_buft_is_host(buft) || ggml_backend_cpu_is_extra_buffer_type(buft);\n GGML_UNUSED(dev);\n}\n\nstatic const struct ggml_backend_device_i ggml_backend_cpu_device_i = {\n /* .get_name = */ ggml_backend_cpu_device_get_name,\n /* .get_description = */ ggml_backend_cpu_device_get_description,\n /* .get_memory = */ ggml_backend_cpu_device_get_memory,\n /* .get_type = */ ggml_backend_cpu_device_get_type,\n /* .get_props = */ ggml_backend_cpu_device_get_props,\n /* .init_backend = */ ggml_backend_cpu_device_init_backend,\n /* .get_buffer_type = */ ggml_backend_cpu_device_get_buffer_type,\n /* .get_host_buffer_type = */ NULL,\n /* .buffer_from_host_ptr = */ ggml_backend_cpu_device_buffer_from_host_ptr,\n /* .supports_op = */ ggml_backend_cpu_device_supports_op,\n /* .supports_buft = */ ggml_backend_cpu_device_supports_buft,\n /* .offload_op = */ NULL,\n /* .event_new = */ NULL,\n /* .event_free = */ NULL,\n /* .event_synchronize = */ NULL,\n};\n\n// CPU backend - backend (reg)\n\nstatic const char * ggml_backend_cpu_reg_get_name(ggml_backend_reg_t reg) {\n return \"CPU\";\n\n GGML_UNUSED(reg);\n}\n\nstatic size_t ggml_backend_cpu_reg_get_device_count(ggml_backend_reg_t reg) {\n return 1;\n\n GGML_UNUSED(reg);\n}\n\nstatic ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n GGML_ASSERT(index == 0);\n\n static ggml_backend_cpu_device_context ctx;\n static ggml_backend_device ggml_backend_cpu_device = {\n /* .iface = */ ggml_backend_cpu_device_i,\n /* .reg = */ reg,\n /* .context = */ &ctx,\n };\n\n return &ggml_backend_cpu_device;\n}\n\n// This is intended to replace the the ggml_cpu_has_* functions when loading the CPU backend dynamically,\n// and additionally to allow other backends to expose their own list of features that applications can query using the same API\nstatic ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t reg) {\n static std::vector features = []() {\n ggml_cpu_init();\n\n std::vector features;\n if (ggml_cpu_has_sse3()) {\n features.push_back({ \"SSE3\", \"1\" });\n }\n if (ggml_cpu_has_ssse3()) {\n features.push_back({ \"SSSE3\", \"1\" });\n }\n if (ggml_cpu_has_avx()) {\n features.push_back({ \"AVX\", \"1\" });\n }\n if (ggml_cpu_has_avx_vnni()) {\n features.push_back({ \"AVX_VNNI\", \"1\" });\n }\n if (ggml_cpu_has_avx2()) {\n features.push_back({ \"AVX2\", \"1\" });\n }\n if (ggml_cpu_has_f16c()) {\n features.push_back({ \"F16C\", \"1\" });\n }\n if (ggml_cpu_has_fma()) {\n features.push_back({ \"FMA\", \"1\" });\n }\n if (ggml_cpu_has_bmi2()) {\n features.push_back({ \"BMI2\", \"1\" });\n }\n if (ggml_cpu_has_avx512()) {\n features.push_back({ \"AVX512\", \"1\" });\n }\n if (ggml_cpu_has_avx512_vbmi()) {\n features.push_back({ \"AVX512_VBMI\", \"1\" });\n }\n if (ggml_cpu_has_avx512_vnni()) {\n features.push_back({ \"AVX512_VNNI\", \"1\" });\n }\n if (ggml_cpu_has_avx512_bf16()) {\n features.push_back({ \"AVX512_BF16\", \"1\" });\n }\n if (ggml_cpu_has_amx_int8()) {\n features.push_back({ \"AMX_INT8\", \"1\" });\n }\n if (ggml_cpu_has_neon()) {\n features.push_back({ \"NEON\", \"1\" });\n }\n if (ggml_cpu_has_arm_fma()) {\n features.push_back({ \"ARM_FMA\", \"1\" });\n }\n if (ggml_cpu_has_fp16_va()) {\n features.push_back({ \"FP16_VA\", \"1\" });\n }\n if (ggml_cpu_has_matmul_int8()) {\n features.push_back({ \"MATMUL_INT8\", \"1\" });\n }\n if (ggml_cpu_has_sve()) {\n features.push_back({ \"SVE\", \"1\" });\n }\n if (ggml_cpu_has_dotprod()) {\n features.push_back({ \"DOTPROD\", \"1\" });\n }\n if (ggml_cpu_get_sve_cnt() > 0) {\n static std::string sve_cnt = std::to_string(ggml_cpu_get_sve_cnt());\n features.push_back({ \"SVE_CNT\", sve_cnt.c_str() });\n }\n if (ggml_cpu_has_sme()) {\n features.push_back({ \"SME\", \"1\" });\n }\n if (ggml_cpu_has_riscv_v()) {\n features.push_back({ \"RISCV_V\", \"1\" });\n }\n if (ggml_cpu_get_rvv_vlen() > 0) {\n static std::string rvv_vlen = std::to_string(ggml_cpu_get_rvv_vlen());\n features.push_back({ \"RVV_VLEN\", rvv_vlen.c_str() });\n }\n if (ggml_cpu_has_vsx()) {\n features.push_back({ \"VSX\", \"1\" });\n }\n if (ggml_cpu_has_vxe()) {\n features.push_back({ \"VXE\", \"1\" });\n }\n if (ggml_cpu_has_wasm_simd()) {\n features.push_back({ \"WASM_SIMD\", \"1\" });\n }\n if (ggml_cpu_has_llamafile()) {\n features.push_back({ \"LLAMAFILE\", \"1\" });\n }\n #ifdef GGML_USE_ACCELERATE\n features.push_back({ \"ACCELERATE\", \"1\" });\n #endif\n #ifdef GGML_USE_CPU_HBM\n features.push_back({ \"CPU_HBM\", \"1\" });\n #endif\n #ifdef GGML_USE_OPENMP\n features.push_back({ \"OPENMP\", \"1\" });\n #endif\n #ifdef GGML_USE_CPU_KLEIDIAI\n features.push_back({ \"KLEIDIAI\", \"1\" });\n #endif\n #ifdef GGML_USE_CPU_REPACK\n features.push_back({ \"REPACK\", \"1\" });\n #endif\n\n features.push_back({ nullptr, nullptr });\n\n return features;\n }();\n\n return features.data();\n\n GGML_UNUSED(reg);\n}\n\nstatic void * ggml_backend_cpu_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n if (strcmp(name, \"ggml_backend_set_n_threads\") == 0) {\n ggml_backend_set_n_threads_t fct = ggml_backend_cpu_set_n_threads;\n return (void *)fct;\n }\n if (strcmp(name, \"ggml_backend_dev_get_extra_bufts\") == 0) {\n ggml_backend_dev_get_extra_bufts_t fct = ggml_backend_cpu_device_get_extra_buffers_type;\n return (void *)fct;\n }\n if (strcmp(name, \"ggml_backend_get_features\") == 0) {\n return (void *)ggml_backend_cpu_get_features;\n }\n if (strcmp(name, \"ggml_backend_set_abort_callback\") == 0) {\n return (void *)ggml_backend_cpu_set_abort_callback;\n }\n if (strcmp(name, \"ggml_backend_cpu_numa_init\") == 0) {\n return (void *)ggml_numa_init;\n }\n if (strcmp(name, \"ggml_backend_cpu_is_numa\") == 0) {\n return (void *)ggml_is_numa;\n }\n if (strcmp(name, \"ggml_backend_cpu_set_use_ref\") == 0) {\n return (void *)ggml_backend_cpu_set_use_ref;\n }\n\n // threadpool - TODO: move to ggml-base\n if (strcmp(name, \"ggml_threadpool_new\") == 0) {\n return (void *)ggml_threadpool_new;\n }\n if (strcmp(name, \"ggml_threadpool_free\") == 0) {\n return (void *)ggml_threadpool_free;\n }\n if (strcmp(name, \"ggml_backend_cpu_set_threadpool\") == 0) {\n return (void *)ggml_backend_cpu_set_threadpool;\n }\n\n return NULL;\n\n GGML_UNUSED(reg);\n}\n\nstatic const struct ggml_backend_reg_i ggml_backend_cpu_reg_i = {\n /* .get_name = */ ggml_backend_cpu_reg_get_name,\n /* .get_device_count = */ ggml_backend_cpu_reg_get_device_count,\n /* .get_device = */ ggml_backend_cpu_reg_get_device,\n /* .get_proc_address = */ ggml_backend_cpu_get_proc_address,\n};\n\nggml_backend_reg_t ggml_backend_cpu_reg(void) {\n // init CPU feature detection\n ggml_cpu_init();\n\n static struct ggml_backend_reg ggml_backend_cpu_reg = {\n /* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_cpu_reg_i,\n /* .context = */ NULL,\n };\n\n return &ggml_backend_cpu_reg;\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_cpu_reg)\n" }, { "path": "src/ggml-cpu/hbm.cpp", "content": "#ifdef GGML_USE_CPU_HBM\n\n#include \"ggml-backend.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-impl.h\"\n\n#include \"hbm.h\"\n\n// buffer type HBM\n\n#include \n\nstatic const char * ggml_backend_cpu_hbm_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n return \"CPU_HBM\";\n\n GGML_UNUSED(buft);\n}\n\nstatic void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n hbw_free(buffer->context);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,\n size_t size) {\n void * ptr;\n int result = hbw_posix_memalign(&ptr, ggml_backend_cpu_buffer_type_get_alignment(buft), size);\n if (result != 0) {\n GGML_LOG_ERROR(\"failed to allocate HBM buffer of size %zu\\n\", size);\n return NULL;\n }\n\n ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);\n buffer->buft = buft;\n buffer->iface.free_buffer = ggml_backend_cpu_hbm_buffer_free_buffer;\n\n return buffer;\n}\n\nggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) {\n static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_hbm = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_cpu_hbm_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,\n /* .get_max_size = */ nullptr, // defaults to SIZE_MAX\n /* .get_alloc_size = */ nullptr, // defaults to ggml_nbytes\n /* .is_host = */ ggml_backend_cpu_buffer_type_is_host,\n },\n /* .context = */ nullptr,\n };\n\n return &ggml_backend_cpu_buffer_type_hbm;\n}\n#endif\n" }, { "path": "src/ggml-cpu/hbm.h", "content": "#pragma once\n\n#include \"ggml-backend.h\"\n#include \"ggml.h\"\n\n// GGML CPU internal header\n\nggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);\n" }, { "path": "src/ggml-cpu/kleidiai/kernels.cpp", "content": "// SPDX-FileCopyrightText: Copyright 2025-2026 Arm Limited and/or its affiliates \n// SPDX-License-Identifier: MIT\n//\n\n// KleidiAI micro-kernels\n#include \"kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.h\"\n#include \"kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h\"\n#include \"kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h\"\n\n#include \"kai_lhs_pack_bf16p2vlx2_f32_sme.h\"\n#include \"kai_lhs_quant_pack_qsi8d32p_f32.h\"\n#include \"kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.h\"\n#include \"kai_lhs_quant_pack_qsi8d32p_f32_neon.h\"\n#include \"kai_lhs_quant_pack_qai8dxp_f32.h\"\n\n#include \"kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h\"\n#include \"kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h\"\n#include \"kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.h\"\n#include \"kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.h\"\n#include \"kai_lhs_pack_f16pmrx2_f32_neon.h\"\n\n#include \"kai_common.h\"\n\n#include \"simd-mappings.h\"\n\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n\n#include \"kernels.h\"\n\n#define NELEMS(x) (sizeof(x) / sizeof(*x))\n\ntemplate\nstatic inline size_t kernel_offs_fn3(size_t a, size_t b, size_t c) {\n return Fn(a, b, c);\n}\n\ntemplate\nstatic inline size_t kernel_offs_fn2(size_t a, size_t b, size_t) {\n return Fn(a, b);\n}\n\ntemplate\nstatic inline void kernel_run_fn11(size_t m, size_t n, size_t k, size_t bl,\n const void* lhs, const void* rhs, void* dst,\n size_t dst_stride_row, size_t dst_stride_col,\n float clamp_min, float clamp_max) {\n Fn(m, n, k, bl, lhs, rhs, static_cast(dst), dst_stride_row, dst_stride_col, clamp_min, clamp_max);\n}\n\ntemplate\nstatic inline void kernel_run_fn10(size_t m, size_t n, size_t k, size_t /*bl*/,\n const void* lhs, const void* rhs, void* dst,\n size_t dst_stride_row, size_t dst_stride_col,\n float clamp_min, float clamp_max) {\n Fn(m, n, k, lhs, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max);\n}\n\ntemplate\nstatic inline void kernel_run_float_fn10(size_t m, size_t n, size_t k, size_t /*bl*/,\n const void* lhs, const void* rhs, void* dst,\n size_t dst_stride_row, size_t dst_stride_col,\n float clamp_min, float clamp_max) {\n Fn(m, n, k, lhs, rhs, static_cast(dst), dst_stride_row, dst_stride_col, clamp_min, clamp_max);\n}\n\ntemplate\nstatic inline size_t lhs_ps_fn6(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {\n return Fn(m, k, bl, mr, kr, sr);\n}\n\ntemplate\nstatic inline size_t lhs_ps_fn5(size_t m, size_t k, size_t /*bl*/, size_t mr, size_t kr, size_t sr) {\n return Fn(m, k, mr, kr, sr);\n}\n\ntemplate\nstatic inline size_t lhs_offs_fn6(size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {\n return Fn(m_idx, k, bl, mr, kr, sr);\n}\n\ntemplate\nstatic inline size_t lhs_offs_fn5(size_t m_idx, size_t k, size_t /*bl*/, size_t mr, size_t kr, size_t sr) {\n return Fn(m_idx, k, mr, kr, sr);\n}\n\ntemplate\nstatic inline void lhs_pack_float_fn10(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr,\n size_t m_idx_start, const void* lhs, size_t lhs_stride, void* lhs_packed) {\n Fn(m, k, bl, mr, kr, sr, m_idx_start, static_cast(lhs), lhs_stride, lhs_packed);\n}\n\ntemplate\nstatic inline void lhs_pack_void_fn10(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr,\n size_t m_idx_start, const void* lhs, size_t lhs_stride, void* lhs_packed) {\n Fn(m, k, bl, mr, kr, sr, m_idx_start, lhs, lhs_stride, lhs_packed);\n}\n\ntemplate\nstatic inline void lhs_pack_void_fn9(size_t m, size_t k, size_t /*bl*/, size_t mr, size_t kr, size_t sr,\n size_t m_idx_start, const void* lhs, size_t lhs_stride, void* lhs_packed) {\n Fn(m, k, mr, kr, sr, m_idx_start, lhs, lhs_stride, lhs_packed);\n}\n\ntemplate\nstatic inline void lhs_pack_float_fn9_no_bl(size_t m, size_t k, size_t /*bl*/, size_t mr, size_t kr, size_t sr,\n size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed) {\n Fn(m, k, mr, kr, sr, m_idx_start, static_cast(lhs), lhs_stride, lhs_packed);\n}\n\ntemplate\nstatic inline size_t rhs_ps_fn5(size_t n, size_t k, size_t nr, size_t kr, size_t bl) {\n return Fn(n, k, nr, kr, bl);\n}\n\ntemplate\nstatic inline size_t rhs_ps_fn2(size_t n, size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) {\n return Fn(n, k);\n}\n\ntemplate\nstatic inline size_t rhs_stride_fn4(size_t k, size_t nr, size_t kr, size_t bl) {\n return Fn(k, nr, kr, bl);\n}\n\ntemplate\nstatic inline size_t rhs_stride_fn1(size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) {\n return Fn(k);\n}\n\ntemplate\nstatic inline void rhs_pack_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,\n size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* /*scale*/,\n void* rhs_packed, size_t extra_bytes, const void* params) {\n Fn(num_groups, n, k, nr, kr, sr, bl,\n static_cast(rhs),\n static_cast(bias),\n rhs_packed, extra_bytes,\n static_cast(params));\n}\n\ntemplate\nstatic inline void rhs_pack_scale_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t /*bl*/,\n size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* scale,\n void* rhs_packed, size_t extra_bytes, const void* params) {\n Fn(num_groups, n, k, nr, kr, sr,\n static_cast(rhs),\n static_cast(bias),\n static_cast(scale),\n rhs_packed, extra_bytes,\n static_cast(params));\n}\n\ntemplate\nstatic inline void rhs_pack_fn13(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t /*bl*/,\n size_t rhs_stride, const void* rhs, const void* bias, const void* scale,\n void* rhs_packed, size_t extra_bytes, const void* params) {\n Fn(num_groups, n, k, nr, kr, sr, rhs_stride, rhs, bias, scale, rhs_packed, extra_bytes, params);\n}\n\nstatic const size_t INT4_PER_BYTE = 2;\nstatic const size_t INT4_BITS = 4;\nstatic const int Q4_0_ZERO_POINT = 8;\nconst size_t INT4_PER_UINT16 = 4;\n\nstatic void dequantize_row_qsi4c32pscalef16(\n const void *packed_data,\n int32_t row_idx,\n int64_t nc,\n float *out,\n size_t nr_pack,\n size_t packed_row_stride,\n size_t kr,\n size_t bl,\n size_t num_bytes_multiplier\n) {\n size_t group_idx = row_idx / nr_pack;\n size_t row_in_group = row_idx % nr_pack;\n const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;\n size_t num_blocks = nc / bl;\n const uint8_t *block_ptr = packed_group;\n\n for (size_t b = 0; b < num_blocks; ++b) {\n uint16_t scale_f16 = *((const uint16_t *)(block_ptr + row_in_group * num_bytes_multiplier));\n float scale = GGML_CPU_FP16_TO_FP32(scale_f16);\n\n const uint8_t *segment_ptr = block_ptr + nr_pack * num_bytes_multiplier;\n size_t num_segments = bl / kr;\n size_t num_bytes_per_segment = kr / INT4_PER_BYTE;\n\n for (size_t s = 0; s < num_segments; ++s) {\n const uint8_t *seg_base = segment_ptr + s * nr_pack * num_bytes_per_segment;\n const uint8_t *qbytes = seg_base + row_in_group * num_bytes_per_segment;\n for (size_t k = 0; k < num_bytes_per_segment; ++k) {\n uint8_t byte = qbytes[k] ^ 0x88;\n int x0 = (byte & 0x0F) - Q4_0_ZERO_POINT;\n int x1 = (byte >> INT4_BITS) - Q4_0_ZERO_POINT;\n out[b * bl + s * num_bytes_per_segment + k] = x0 * scale;\n out[b * bl + s * num_bytes_per_segment + k + bl/2] = x1 * scale;\n }\n }\n block_ptr += nr_pack * num_bytes_multiplier + num_segments * nr_pack * num_bytes_per_segment;\n }\n}\n\nstatic void dequantize_row_qsi4c32ps1s0scalef16(\n const void *packed_data,\n int32_t row_idx,\n int64_t k,\n float *out,\n size_t nr,\n size_t packed_row_stride,\n size_t kr,\n size_t bl,\n size_t num_bytes_multiplier\n) {\n const size_t num_blocks = k / bl;\n const size_t bl4 = bl / INT4_PER_UINT16;\n\n size_t group_idx = row_idx / nr;\n size_t row_in_group = row_idx % nr;\n\n const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;\n const uint16_t *qdata = (const uint16_t *)packed_group;\n const uint16_t *scales = (const uint16_t *)(packed_group + packed_row_stride - (nr * num_blocks * num_bytes_multiplier));\n\n for (size_t block_idx = 0; block_idx < num_blocks; ++block_idx) {\n uint16_t scale_f16 = scales[row_in_group + block_idx * nr];\n float scale = GGML_CPU_FP16_TO_FP32(scale_f16);\n\n for (size_t bl4_idx = 0; bl4_idx < bl4; ++bl4_idx) {\n uint16_t q = qdata[(block_idx * bl4 + bl4_idx) * nr + row_in_group];\n\n for (size_t qidx = 0; qidx < INT4_PER_UINT16; ++qidx) {\n int v = ((q >> (qidx * 4)) & 0xF) - Q4_0_ZERO_POINT;\n out[block_idx * bl + bl4_idx * INT4_BITS + qidx] = v * scale;\n }\n }\n }\n GGML_UNUSED(kr);\n}\n\nstatic void dequantize_row_qsi8cxp(\n const void *packed_data,\n int32_t row_idx,\n int64_t k,\n float *out,\n size_t nr,\n size_t packed_row_stride,\n size_t kr,\n size_t bl,\n size_t num_bytes_multiplier\n) {\n GGML_UNUSED(bl);\n GGML_UNUSED(num_bytes_multiplier);\n\n const size_t k_internal = ((size_t) k + QK8_0 - 1) / QK8_0 * QK8_0;\n const size_t group_idx = row_idx / nr;\n const size_t row_in_group = row_idx % nr;\n\n const uint8_t * group_ptr = static_cast(packed_data) + group_idx * packed_row_stride;\n const int8_t * data_base = reinterpret_cast(group_ptr);\n\n const size_t num_blocks = k_internal / kr;\n\n for (size_t block = 0; block < num_blocks; ++block) {\n const int8_t * block_ptr = data_base + (block * nr + row_in_group) * kr;\n for (size_t i = 0; i < kr; ++i) {\n const size_t k_idx = block * kr + i;\n if (k_idx < (size_t) k) {\n out[k_idx] = static_cast(block_ptr[i]);\n }\n }\n }\n\n const uint8_t * sums_ptr = group_ptr + nr * k_internal;\n GGML_UNUSED(sums_ptr);\n\n const float * scale_ptr = reinterpret_cast(sums_ptr + nr * sizeof(int32_t));\n const float scale = scale_ptr[row_in_group];\n\n if (scale == 0.0f) {\n for (size_t i = 0; i < (size_t) k; ++i) {\n out[i] = 0.0f;\n }\n return;\n }\n\n for (size_t i = 0; i < (size_t) k; ++i) {\n out[i] *= scale;\n }\n}\n\nstatic ggml_kleidiai_kernels gemm_gemv_kernels[] = {\n#if defined(__ARM_FEATURE_SME)\n {\n /* SME GEMM */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_void_fn10,\n },\n /* SME GEMV */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32_neon,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon,\n /* .to_float = */ dequantize_row_qsi4c32ps1s0scalef16,\n /* .packed_size_ex = */ &rhs_ps_fn5,\n /* .packed_stride_ex = */ &rhs_stride_fn4,\n /* .pack_func_ex = */ &rhs_pack_fn12,\n },\n /* .required_cpu = */ CPU_FEATURE_SME,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_Q4_0,\n /* .op_type = */ GGML_TYPE_F32,\n },\n {\n /* SME GEMM */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn2,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2,\n /* .run_kernel_ex = */ &kernel_run_fn10,\n },\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_pack_bf16p2vlx2_f32_sme,\n /* .get_packed_offset_ex = */ &lhs_offs_fn5,\n /* .packed_size_ex = */ &lhs_ps_fn5,\n /* .pack_func_ex = */ &lhs_pack_void_fn9,\n },\n /* SME GEMV */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n /* .get_lhs_offset_ex = */ nullptr,\n /* .get_rhs_packed_offset_ex = */ nullptr,\n /* .run_kernel_ex = */ nullptr,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_pack_bf16p2vlx2_f32_sme,\n /* .get_packed_offset_ex = */ &lhs_offs_fn5,\n /* .packed_size_ex = */ &lhs_ps_fn5,\n /* .pack_func_ex = */ &lhs_pack_void_fn9,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ nullptr,\n /* .to_float = */ nullptr,\n /* .packed_size_ex = */ &rhs_ps_fn2,\n /* .packed_stride_ex = */ &rhs_stride_fn1,\n /* .pack_func_ex = */ &rhs_pack_fn13,\n },\n /* .required_cpu = */ CPU_FEATURE_SME,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_F16,\n /* .op_type = */ GGML_TYPE_F32,\n },\n#endif\n#if defined(__APPLE__)\n#if defined(__ARM_FEATURE_DOTPROD)\n {\n /* DOTPROD GEMM */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* DOTPROD GEMV */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n /* .to_float = */ dequantize_row_qsi4c32pscalef16,\n /* .packed_size_ex = */ &rhs_ps_fn5,\n /* .packed_stride_ex = */ &rhs_stride_fn4,\n /* .pack_func_ex = */ &rhs_pack_fn12,\n },\n /* .required_cpu = */ CPU_FEATURE_DOTPROD,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_Q4_0,\n /* .op_type = */ GGML_TYPE_F32,\n },\n#endif\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n {\n /* i8mm GEMM */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p4x8sb_f32_neon,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* i8mm GEMV */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n /* .to_float = */ dequantize_row_qsi4c32pscalef16,\n /* .packed_size_ex = */ &rhs_ps_fn5,\n /* .packed_stride_ex = */ &rhs_stride_fn4,\n /* .pack_func_ex = */ &rhs_pack_fn12,\n },\n /* .required_cpu = */ CPU_FEATURE_I8MM,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_Q4_0,\n /* .op_type = */ GGML_TYPE_F32,\n },\n#endif\n#else\n#if defined(__ARM_FEATURE_SVE)\n {\n /* SVE i8mm GEMM */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p4x8sb_f32_neon,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* SVE dotprod GEMV */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n /* .to_float = */ dequantize_row_qsi4c32pscalef16,\n /* .packed_size_ex = */ &rhs_ps_fn5,\n /* .packed_stride_ex = */ &rhs_stride_fn4,\n /* .pack_func_ex = */ &rhs_pack_fn12,\n },\n /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_Q4_0,\n /* .op_type = */ GGML_TYPE_F32,\n },\n#endif\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n {\n /* i8mm GEMM */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p4x8sb_f32_neon,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* i8mm GEMV */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n /* .to_float = */ dequantize_row_qsi4c32pscalef16,\n /* .packed_size_ex = */ &rhs_ps_fn5,\n /* .packed_stride_ex = */ &rhs_stride_fn4,\n /* .pack_func_ex = */ &rhs_pack_fn12,\n },\n /* .required_cpu = */ CPU_FEATURE_I8MM,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_Q4_0,\n /* .op_type = */ GGML_TYPE_F32,\n },\n#endif // __ARM_FEATURE_MATMUL_INT8\n#if defined(__ARM_FEATURE_DOTPROD)\n {\n /* DOTPROD GEMM */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* DOTPROD GEMV */\n /* .kern_info = */ {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn3,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3,\n /* .run_kernel_ex = */ &kernel_run_fn11,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn6,\n /* .packed_size_ex = */ &lhs_ps_fn6,\n /* .pack_func_ex = */ &lhs_pack_float_fn10,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n /* .to_float = */ dequantize_row_qsi4c32pscalef16,\n /* .packed_size_ex = */ &rhs_ps_fn5,\n /* .packed_stride_ex = */ &rhs_stride_fn4,\n /* .pack_func_ex = */ &rhs_pack_fn12,\n },\n /* .required_cpu = */ CPU_FEATURE_DOTPROD,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_Q4_0,\n /* .op_type = */ GGML_TYPE_F32,\n },\n#endif\n#endif\n { /* Sentinel */ }\n};\n\nstatic ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {\n#if defined(__ARM_FEATURE_SME)\n {\n /* SME GEMM */\n {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn2,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2,\n /* .run_kernel_ex = */ &kernel_run_float_fn10,\n },\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn5,\n /* .packed_size_ex = */ &lhs_ps_fn5,\n /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl,\n },\n /* SME GEMV */\n {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn2,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2,\n /* .run_kernel_ex = */ &kernel_run_float_fn10,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn5,\n /* .packed_size_ex = */ &lhs_ps_fn5,\n /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,\n /* .to_float = */ dequantize_row_qsi8cxp,\n /* .packed_size_ex = */ &rhs_ps_fn5,\n /* .packed_stride_ex = */ &rhs_stride_fn4,\n /* .pack_func_ex = */ &rhs_pack_scale_fn12,\n },\n /* .required_cpu = */ CPU_FEATURE_SME,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_Q8_0,\n /* .op_type = */ GGML_TYPE_F32,\n },\n#endif\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n {\n /* I8MM GEMM */\n {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn2,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2,\n /* .run_kernel_ex = */ &kernel_run_float_fn10,\n },\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn5,\n /* .packed_size_ex = */ &lhs_ps_fn5,\n /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl,\n },\n /* I8MM GEMV (dotprod fallback) */\n {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn2,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2,\n /* .run_kernel_ex = */ &kernel_run_float_fn10,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn5,\n /* .packed_size_ex = */ &lhs_ps_fn5,\n /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,\n /* .to_float = */ dequantize_row_qsi8cxp,\n /* .packed_size_ex = */ &rhs_ps_fn5,\n /* .packed_stride_ex = */ &rhs_stride_fn4,\n /* .pack_func_ex = */ &rhs_pack_scale_fn12,\n },\n /* .required_cpu = */ CPU_FEATURE_I8MM,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_Q8_0,\n /* .op_type = */ GGML_TYPE_F32,\n },\n#endif\n#if defined(__ARM_FEATURE_DOTPROD)\n {\n /* DOTPROD GEMM */\n {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn2,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2,\n /* .run_kernel_ex = */ &kernel_run_float_fn10,\n },\n /* .gemm_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn5,\n /* .packed_size_ex = */ &lhs_ps_fn5,\n /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl,\n },\n /* DOTPROD GEMV */\n {\n /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n /* .get_lhs_offset_ex = */ &kernel_offs_fn2,\n /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2,\n /* .run_kernel_ex = */ &kernel_run_float_fn10,\n },\n /* .gemv_lhs_info = */ {\n /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n /* .get_packed_offset_ex = */ &lhs_offs_fn5,\n /* .packed_size_ex = */ &lhs_ps_fn5,\n /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl,\n },\n /* .rhs_info = */ {\n /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,\n /* .to_float = */ dequantize_row_qsi8cxp,\n /* .packed_size_ex = */ &rhs_ps_fn5,\n /* .packed_stride_ex = */ &rhs_stride_fn4,\n /* .pack_func_ex = */ &rhs_pack_scale_fn12,\n },\n /* .required_cpu = */ CPU_FEATURE_DOTPROD,\n /* .lhs_type = */ GGML_TYPE_F32,\n /* .rhs_type = */ GGML_TYPE_Q8_0,\n /* .op_type = */ GGML_TYPE_F32,\n },\n#endif\n { /* Sentinel */ }\n};\n\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor) {\n ggml_kleidiai_kernels * kernel = nullptr;\n\n if (tensor->op == GGML_OP_MUL_MAT && tensor->src[0] != nullptr && tensor->src[1] != nullptr) {\n#if defined(__ARM_FEATURE_SME) || \\\n defined(__ARM_FEATURE_DOTPROD) || \\\n defined(__ARM_FEATURE_MATMUL_INT8) || \\\n defined(__ARM_FEATURE_SVE)\n auto try_table = [&](auto & table) {\n for (size_t i = 0; i < NELEMS(table) - 1; ++i) {\n if ((cpu_features & table[i].required_cpu) == table[i].required_cpu &&\n table[i].lhs_type == tensor->src[1]->type &&\n table[i].rhs_type == tensor->src[0]->type &&\n table[i].op_type == tensor->type) {\n kernel = &table[i];\n return true;\n }\n }\n return false;\n };\n\n if (tensor->src[0]->type == GGML_TYPE_Q8_0) {\n try_table(gemm_gemv_kernels_q8);\n } else {\n try_table(gemm_gemv_kernels);\n }\n#else\n GGML_UNUSED(gemm_gemv_kernels);\n GGML_UNUSED(gemm_gemv_kernels_q8);\n GGML_UNUSED(cpu_features);\n#endif\n }\n\n return kernel;\n}\n\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features) {\n ggml_kleidiai_kernels * kernels = nullptr;\n\n#if defined(__ARM_FEATURE_SME) || \\\n defined(__ARM_FEATURE_DOTPROD) || \\\n defined(__ARM_FEATURE_MATMUL_INT8) || \\\n defined(__ARM_FEATURE_SVE)\n for (size_t i = 0; i < NELEMS(gemm_gemv_kernels) - 1; ++i) {\n if ((features & gemm_gemv_kernels[i].required_cpu) == gemm_gemv_kernels[i].required_cpu) {\n kernels = &gemm_gemv_kernels[i];\n break;\n }\n }\n#else\n GGML_UNUSED(features);\n#endif\n\n return kernels;\n}\n\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features) {\n ggml_kleidiai_kernels * kernels = nullptr;\n\n#if defined(__ARM_FEATURE_SME) || defined(__ARM_FEATURE_DOTPROD) || defined(__ARM_FEATURE_MATMUL_INT8)\n for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) {\n if ((features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu) {\n kernels = &gemm_gemv_kernels_q8[i];\n break;\n }\n }\n#else\n GGML_UNUSED(features);\n#endif\n\n return kernels;\n}\n" }, { "path": "src/ggml-cpu/kleidiai/kernels.h", "content": "// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates \n// SPDX-License-Identifier: MIT\n//\n\n#pragma once\n\n#include \"ggml.h\"\n\nenum cpu_feature {\n CPU_FEATURE_NONE = 0,\n CPU_FEATURE_DOTPROD = 1,\n CPU_FEATURE_I8MM = 2,\n CPU_FEATURE_SVE = 4,\n CPU_FEATURE_SME = 8\n};\n\ninline cpu_feature& operator|=(cpu_feature& lhs, cpu_feature rhs) {\n lhs = static_cast(lhs | rhs);\n return lhs;\n}\ninline cpu_feature operator|(cpu_feature lhs, cpu_feature rhs) {\n return static_cast(static_cast(lhs) | static_cast(rhs));\n}\n\nstruct kernel_info {\n size_t (*get_m_step)(void);\n size_t (*get_n_step)(void);\n size_t (*get_mr)(void);\n size_t (*get_nr)(void);\n size_t (*get_kr)(void);\n size_t (*get_sr)(void);\n\n size_t (*get_dst_offset)(size_t m_idx, size_t n_idx, size_t stride);\n size_t (*get_dst_size)(size_t m, size_t n);\n\n size_t (*get_lhs_offset_ex)(size_t m_idx, size_t k, size_t bl);\n\n size_t (*get_rhs_packed_offset_ex)(size_t n_idx, size_t k, size_t bl);\n\n void (*run_kernel_ex)(\n size_t m, size_t n, size_t k, size_t bl,\n const void* lhs_packed, const void* rhs_packed,\n void* dst, size_t dst_stride_row, size_t dst_stride_col,\n float clamp_min, float clamp_max);\n};\n\nstruct lhs_packing_info {\n size_t (*get_offset)(size_t m_idx, size_t lhs_stride);\n\n size_t (*get_packed_offset_ex)(size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr);\n\n size_t (*packed_size_ex)(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr);\n\n void (*pack_func_ex)(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr,\n size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed);\n};\n\nstruct rhs_packing_info {\n size_t (*packed_stride)(size_t k, size_t nr, size_t kr, size_t bl);\n\n void (*to_float)(const void *packed_data, int32_t row_idx, int64_t nc, float *out,\n size_t nr_pack, size_t packed_row_stride, size_t kr, size_t bl,\n size_t num_bytes_multiplier);\n\n size_t (*packed_size_ex)(size_t n, size_t k, size_t nr, size_t kr, size_t bl);\n\n size_t (*packed_stride_ex)(size_t k, size_t nr, size_t kr, size_t bl);\n\n void (*pack_func_ex)(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,\n size_t rhs_stride, const void * rhs, const void * bias, const void * scale, void * rhs_packed, size_t extra_bytes, const void * params);\n};\n\nstruct ggml_kleidiai_kernels {\n kernel_info gemm;\n lhs_packing_info gemm_lhs_info;\n\n kernel_info gemv;\n lhs_packing_info gemv_lhs_info;\n\n rhs_packing_info rhs_info;\n\n cpu_feature required_cpu;\n ggml_type lhs_type;\n ggml_type rhs_type;\n ggml_type op_type;\n};\n\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor);\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features);\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features);\n" }, { "path": "src/ggml-cpu/kleidiai/kleidiai.cpp", "content": "// SPDX-FileCopyrightText: Copyright 2025-2026 Arm Limited and/or its affiliates \n// SPDX-License-Identifier: MIT\n//\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#if defined(__linux__)\n#include \n#include \n#include \n#include \n#include \n#elif defined(__APPLE__)\n#include \n#include \n#include \n#elif defined(_WIN32)\n#include \n#include \n#endif\n\n#include \"kleidiai.h\"\n\n#include \"ggml-cpu.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-threading.h\"\n#include \"traits.h\"\n\n#include \"kernels.h\"\n\n#include \"kai_common.h\"\n\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n\nstatic constexpr int GGML_KLEIDIAI_MAX_KERNEL_SLOTS = 2;\nstatic constexpr uint32_t GGML_KLEIDIAI_PACK_MAGIC = 0x4b4c4149; // \"KLAI\"\nstatic constexpr uint16_t GGML_KLEIDIAI_PACK_VERSION = 1;\nstatic constexpr size_t GGML_KLEIDIAI_PACK_ALIGN = 64;\n\nstruct ggml_kleidiai_context {\n cpu_feature features;\n ggml_kleidiai_kernels * kernels_q4;\n ggml_kleidiai_kernels * kernels_q8;\n int sme_thread_cap; // <= 0 means “SME disabled/unknown”;\n int thread_hint; // <= 0 means “no hint”\n} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, 0, -1 };\n\nstatic const char* cpu_feature_to_string(cpu_feature f) {\n if (f == CPU_FEATURE_NONE) {\n return \"NONE\";\n } else if ((f & CPU_FEATURE_SME) == CPU_FEATURE_SME) {\n return \"SME\";\n } else if ((f & CPU_FEATURE_SVE) == CPU_FEATURE_SVE) {\n return \"SVE\";\n }\n else if ((f & CPU_FEATURE_I8MM) == CPU_FEATURE_I8MM) {\n return \"I8MM\";\n } else if ((f & CPU_FEATURE_DOTPROD) == CPU_FEATURE_DOTPROD) {\n return \"DOTPROD\";\n }\n else {\n return \"UNKNOWN\";\n }\n}\n\nstatic size_t detect_num_smcus() {\n if (!ggml_cpu_has_sme()) {\n return 0;\n }\n\n#if defined(__linux__) && defined(__aarch64__)\n // Linux/aarch64: Best-effort count of Streaming Mode Compute Units (SMCUs) via SMIDR_EL1 sysfs.\n size_t num_private = 0;\n std::set shared_ids;\n\n for (size_t cpu = 0;; ++cpu) {\n const std::string path =\n \"/sys/devices/system/cpu/cpu\" + std::to_string(cpu) +\n \"/regs/identification/smidr_el1\";\n\n std::ifstream file(path);\n if (!file.is_open()) {\n break;\n }\n\n uint64_t smidr = 0;\n if (!(file >> std::hex >> smidr)) {\n continue;\n }\n\n // Arm ARM: SMIDR_EL1\n const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3);\n // Build an \"affinity-like\" identifier for shared SMCUs.\n // Keep the original packing logic, but isolate it here.\n const uint32_t id = (uint32_t)((smidr & 0xFFFu) | ((smidr >> 20) & 0xFFFFF000u));\n\n switch (sh) {\n case 0b10: // private SMCU\n ++num_private;\n break;\n case 0b11: // shared SMCU\n shared_ids.emplace(id);\n break;\n case 0b00:\n // Ambiguous / implementation-defined. Be conservative:\n // treat id==0 as private, otherwise as shared.\n if (id == 0) ++num_private;\n else shared_ids.emplace(id);\n break;\n default:\n break;\n }\n }\n\n return num_private + shared_ids.size();\n\n#elif defined(__APPLE__) && defined(__aarch64__)\n // table for known M4 variants. Users can override via GGML_KLEIDIAI_SME=.\n char chip_name[256] = {};\n size_t size = sizeof(chip_name);\n\n if (sysctlbyname(\"machdep.cpu.brand_string\", chip_name, &size, nullptr, 0) == 0) {\n const std::string brand(chip_name);\n\n struct ModelSMCU { const char *match; size_t smcus; };\n static const ModelSMCU table[] = {\n { \"M4 Ultra\", 2 },\n { \"M4 Max\", 2 },\n { \"M4 Pro\", 2 },\n { \"M4\", 1 },\n };\n\n for (const auto &e : table) {\n if (brand.find(e.match) != std::string::npos) {\n return e.smcus;\n }\n }\n }\n return 1;\n\n#else\n return 1;\n#endif\n}\n\nstatic int parse_uint_env(const char *s, const char *name, bool *ok) {\n if (!s) { *ok = false; return 0; }\n char *end = nullptr;\n long v = strtol(s, &end, 10);\n if (end == s || *end != '\\0') {\n GGML_LOG_WARN(\"kleidiai: invalid %s='%s' (expected integer)\\n\", name, s);\n *ok = false;\n return 0;\n }\n if (v < 0 || v > INT_MAX) {\n GGML_LOG_WARN(\"kleidiai: out-of-range %s='%s'\\n\", name, s);\n *ok = false;\n return 0;\n }\n *ok = true;\n return (int)v;\n}\n\nstatic void init_kleidiai_context(void) {\n ggml_critical_section_start();\n static bool initialized = false;\n\n if (!initialized) {\n initialized = true;\n\n const char *env_sme = getenv(\"GGML_KLEIDIAI_SME\");\n const char *env_threads = getenv(\"GGML_TOTAL_THREADS\");\n\n const bool cpu_has_sme = ggml_cpu_has_sme();\n size_t detected_smcus = 0;\n\n ctx.features = (ggml_cpu_has_dotprod() ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |\n (ggml_cpu_has_matmul_int8() ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) |\n ((ggml_cpu_has_sve() && ggml_cpu_get_sve_cnt() == QK8_0) ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);\n\n if (env_threads) {\n bool ok = false;\n int hint = parse_uint_env(env_threads, \"GGML_TOTAL_THREADS\", &ok);\n if (ok && hint > 0) {\n ctx.thread_hint = hint;\n }\n }\n\n // SME policy:\n // - If CPU doesn't support SME: SME always off.\n // - Else:\n // - env unset => auto-detect cores; enable if detected > 0.\n // - env=0 => force off.\n // - env>0 => force N cores (skip detection).\n int sme_cores = 0;\n bool sme_env_ok = false;\n bool sme_env_set = (env_sme != nullptr);\n\n if (!cpu_has_sme) {\n if (sme_env_set) {\n bool ok = false;\n int req = parse_uint_env(env_sme, \"GGML_KLEIDIAI_SME\", &ok);\n if (ok && req > 0) {\n GGML_LOG_WARN(\"kleidiai: GGML_KLEIDIAI_SME=%d but SME is not supported on this CPU; disabling SME\\n\", req);\n }\n }\n sme_cores = 0;\n } else {\n if (sme_env_set) {\n bool ok = false;\n int v = parse_uint_env(env_sme, \"GGML_KLEIDIAI_SME\", &ok);\n sme_env_ok = ok;\n\n if (!ok) {\n GGML_LOG_WARN(\"kleidiai: GGML_KLEIDIAI_SME set but parsing failed; falling back to runtime SME-core detection\\n\");\n detected_smcus = detect_num_smcus();\n sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;\n } else if (v == 0) {\n sme_cores = 0;\n } else {\n sme_cores = v;\n }\n } else {\n detected_smcus = detect_num_smcus();\n sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;\n }\n\n if (!sme_env_set && sme_cores == 0) {\n GGML_LOG_WARN(\"kleidiai: SME supported but runtime SME-core detection returned 0; falling back to NEON\\n\");\n }\n\n if (sme_cores > 0) {\n ctx.features |= CPU_FEATURE_SME;\n }\n }\n\n // Kernel selection\n ctx.kernels_q4 = ggml_kleidiai_select_kernels_q4_0(ctx.features);\n ctx.kernels_q8 = ggml_kleidiai_select_kernels_q8_0(ctx.features);\n\n if (!ctx.kernels_q4) {\n GGML_LOG_INFO(\"kleidiai: no compatible q4 kernels found for CPU features mask %d\\n\", (int)ctx.features);\n } else {\n GGML_LOG_INFO(\"kleidiai: primary q4 kernel feature %s\\n\", cpu_feature_to_string(ctx.kernels_q4->required_cpu));\n }\n\n if (!ctx.kernels_q8) {\n GGML_LOG_INFO(\"kleidiai: no compatible q8 kernels found for CPU features mask %d\\n\", (int)ctx.features);\n } else {\n GGML_LOG_INFO(\"kleidiai: primary q8 kernel feature %s\\n\", cpu_feature_to_string(ctx.kernels_q8->required_cpu));\n }\n\n ctx.sme_thread_cap = (ctx.features & CPU_FEATURE_SME) ? sme_cores : 0;\n\n if (ctx.features & CPU_FEATURE_SME) {\n if (sme_env_set && sme_env_ok && sme_cores > 0) {\n GGML_LOG_INFO(\"kleidiai: SME enabled (GGML_KLEIDIAI_SME=%d override)\\n\", sme_cores);\n } else {\n GGML_LOG_INFO(\"kleidiai: SME enabled (runtime-detected SME cores=%d)\\n\", sme_cores);\n }\n } else {\n GGML_LOG_INFO(\"kleidiai: SME disabled\\n\");\n }\n }\n\n ggml_critical_section_end();\n}\n\nstatic inline int kleidiai_sme_thread_cap() {\n return ctx.sme_thread_cap;\n}\n\nstatic inline size_t align_up(size_t value, size_t alignment) {\n if (alignment == 0) {\n return value;\n }\n const size_t remainder = value % alignment;\n return remainder == 0 ? value : value + (alignment - remainder);\n}\n\nstatic inline bool kleidiai_pack_fallback_allowed() {\n if (ctx.sme_thread_cap <= 0) {\n return false;\n }\n if (ctx.thread_hint <= 0) {\n return true;\n }\n return ctx.thread_hint > ctx.sme_thread_cap;\n}\n\nstruct kleidiai_weight_header {\n uint32_t magic;\n uint16_t version;\n uint16_t slot_count;\n uint64_t offsets[GGML_KLEIDIAI_MAX_KERNEL_SLOTS];\n uint64_t sizes[GGML_KLEIDIAI_MAX_KERNEL_SLOTS];\n};\n\nstatic inline kleidiai_weight_header * kleidiai_weight_header_from_ptr(void * data) {\n return reinterpret_cast(data);\n}\n\nstatic inline const kleidiai_weight_header * kleidiai_weight_header_from_ptr(const void * data) {\n return reinterpret_cast(data);\n}\n\nstatic inline bool kleidiai_is_weight_header_valid(const kleidiai_weight_header * header) {\n if (!header) {\n return false;\n }\n if (header->magic != GGML_KLEIDIAI_PACK_MAGIC || header->version != GGML_KLEIDIAI_PACK_VERSION) {\n return false;\n }\n if (header->slot_count == 0 || header->slot_count > GGML_KLEIDIAI_MAX_KERNEL_SLOTS) {\n return false;\n }\n return true;\n}\n\nstatic inline uint8_t * kleidiai_weight_slot_ptr(kleidiai_weight_header * header, int slot) {\n if (!kleidiai_is_weight_header_valid(header)) {\n return nullptr;\n }\n if (slot < 0 || slot >= header->slot_count) {\n return nullptr;\n }\n return reinterpret_cast(header) + header->offsets[slot];\n}\n\nstatic inline const uint8_t * kleidiai_weight_slot_ptr(const kleidiai_weight_header * header, int slot) {\n if (!kleidiai_is_weight_header_valid(header)) {\n return nullptr;\n }\n if (slot < 0 || slot >= header->slot_count) {\n return nullptr;\n }\n return reinterpret_cast(header) + header->offsets[slot];\n}\n\nstatic inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4() {\n return ctx.kernels_q4;\n}\n\nstatic inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q8() {\n return ctx.kernels_q8;\n}\n\ntemplate \nstatic int kleidiai_collect_kernel_chain_common(\n ggml_kleidiai_kernels * primary,\n cpu_feature features,\n std::array & out,\n SelectFallback select_fallback) {\n int count = 0;\n if (!primary) {\n return 0;\n }\n out[count++] = primary;\n\n if ((primary->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) {\n const cpu_feature fallback_mask = static_cast(features & ~CPU_FEATURE_SME);\n if (fallback_mask != CPU_FEATURE_NONE) {\n ggml_kleidiai_kernels * fallback = select_fallback(fallback_mask);\n if (fallback && fallback != primary &&\n fallback->lhs_type == primary->lhs_type &&\n fallback->rhs_type == primary->rhs_type &&\n fallback->op_type == primary->op_type) {\n out[count++] = fallback;\n }\n }\n }\n\n return count;\n}\n\nstatic int kleidiai_collect_kernel_chain(const struct ggml_tensor * op,\n std::array & out) {\n ggml_kleidiai_kernels * primary = ggml_kleidiai_select_kernels(ctx.features, op);\n return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,\n [&](cpu_feature mask) { return ggml_kleidiai_select_kernels(mask, op); });\n}\n\nstatic int kleidiai_collect_q4_chain(std::array & out) {\n ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q4();\n return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,\n [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_0(mask); });\n}\n\nstatic int kleidiai_collect_q8_chain(std::array & out) {\n ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q8();\n return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,\n [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q8_0(mask); });\n}\n\nstatic inline int64_t ggml_ne(const ggml_tensor * tensor, int dim) {\n GGML_ASSERT(dim >= 0 && dim < GGML_MAX_DIMS);\n return tensor->ne[dim];\n}\n\nnamespace ggml::cpu::kleidiai {\n\nstatic size_t round_down(size_t x, size_t y) {\n return y == 0 ? x : x - (x % y);\n}\n\nstatic void transpose_f32kxn_f16nxk(size_t n, size_t k, float * dst, const uint16_t * src, size_t rhs_stride) {\n size_t src_stride = rhs_stride / sizeof(uint16_t);\n size_t dst_stride = n;\n\n for (size_t k_idx = 0; k_idx < k; ++k_idx) {\n for (size_t n_idx = 0; n_idx < n; ++n_idx) {\n uint16_t v = *(src + k_idx + n_idx * src_stride);\n *(dst + n_idx + k_idx * dst_stride) = kai_cast_f32_f16(v);\n }\n }\n}\n\nclass tensor_traits : public ggml::cpu::tensor_traits {\n bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n if (op->op != GGML_OP_MUL_MAT) {\n return false;\n }\n\n std::array kernel_chain;\n const int slot_count = kleidiai_collect_kernel_chain(op, kernel_chain);\n if (slot_count == 0) {\n return false;\n }\n\n const bool is_gemv = op->src[1]->ne[1] == 1;\n const size_t k = op->src[0]->ne[0];\n const size_t n = op->src[0]->ne[1];\n const size_t m = op->src[1]->ne[1];\n\n if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) {\n const size_t qk = (op->src[0]->type == GGML_TYPE_Q4_0) ? QK4_0 : QK8_0;\n\n size_t cursor = 0;\n bool any_slot = false;\n\n for (int slot = 0; slot < slot_count; ++slot) {\n ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n lhs_packing_info * lhs_info = is_gemv ? &kernels->gemv_lhs_info : &kernels->gemm_lhs_info;\n kernel_info * kernel = is_gemv ? &kernels->gemv : &kernels->gemm;\n\n if (!lhs_info || !lhs_info->packed_size_ex || !kernel) {\n return false;\n }\n\n const size_t mr = kernel->get_mr();\n const size_t kr = kernel->get_kr();\n const size_t sr = kernel->get_sr();\n\n const size_t packed = lhs_info->packed_size_ex(m, k, qk, mr, kr, sr);\n\n cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n cursor += packed;\n any_slot = true;\n }\n\n if (!any_slot) {\n return false;\n }\n\n size = cursor;\n return true;\n }\n\n if (op->src[0]->type == GGML_TYPE_F16) {\n const int64_t lhs_batch_size0 = op->src[1]->ne[2];\n const int64_t rhs_batch_size0 = op->src[0]->ne[2];\n GGML_ASSERT(rhs_batch_size0 > 0);\n const int64_t r = lhs_batch_size0 / rhs_batch_size0;\n\n size_t cursor = 0;\n bool any_slot = false;\n\n for (int slot = 0; slot < slot_count; ++slot) {\n ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n lhs_packing_info * lhs_info = is_gemv ? &kernels->gemv_lhs_info : &kernels->gemm_lhs_info;\n kernel_info * kernel = is_gemv ? &kernels->gemv : &kernels->gemm;\n if (!lhs_info || !lhs_info->packed_size_ex || !kernels->rhs_info.packed_size_ex || !kernel) {\n return false;\n }\n\n const size_t mr = kernel->get_mr();\n const size_t kr = kernel->get_kr();\n const size_t sr = kernel->get_sr();\n\n cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n cursor += lhs_info->packed_size_ex(m * r, k, 0, mr, kr, sr);\n any_slot = true;\n }\n\n for (int slot = 0; slot < slot_count; ++slot) {\n ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n kernel_info * kernel = is_gemv ? &kernels->gemv : &kernels->gemm;\n if (!kernel || !kernels->rhs_info.packed_size_ex) {\n return false;\n }\n cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n cursor += kernels->rhs_info.packed_size_ex(n, k, kernel->get_nr(), kernel->get_kr(), 0);\n }\n\n cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n cursor += k * n * sizeof(float);\n cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n cursor += n * sizeof(float);\n\n if (!any_slot) {\n return false;\n }\n\n size = cursor;\n return true;\n }\n\n return false;\n }\n\n bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * dst) override {\n if (dst->op == GGML_OP_MUL_MAT) {\n if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {\n return compute_forward_qx(params, dst);\n } else if (dst->src[0]->type == GGML_TYPE_F16) {\n return compute_forward_fp16(params, dst);\n }\n } else if (dst->op == GGML_OP_GET_ROWS) {\n if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {\n return compute_forward_get_rows(params, dst);\n }\n }\n return false;\n }\n\n bool compute_forward_fp16(ggml_compute_params * params, struct ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n ggml_kleidiai_kernels *kernels = ggml_kleidiai_select_kernels(ctx.features, dst);\n if (!kernels) {\n return false;\n }\n\n const bool is_gemv = src1->ne[1] == 1;\n kernel_info * kernel = is_gemv ? &kernels->gemv : &kernels->gemm;\n lhs_packing_info * lhs_info = is_gemv ? &kernels->gemv_lhs_info : &kernels->gemm_lhs_info;\n GGML_ASSERT(kernel);\n if (!kernels->rhs_info.pack_func_ex ||\n !kernel->get_lhs_offset_ex || !kernel->get_rhs_packed_offset_ex || !kernel->run_kernel_ex) {\n return false;\n }\n\n const int nth = params->nth;\n const int ith = params->ith;\n\n const int64_t lhs_batch_size0 = ne12;\n const int64_t rhs_batch_size0 = ne02;\n const int64_t batch_size = lhs_batch_size0;\n\n GGML_ASSERT(rhs_batch_size0 > 0);\n GGML_ASSERT(lhs_batch_size0 % rhs_batch_size0 == 0);\n const int64_t r = lhs_batch_size0 / rhs_batch_size0;\n\n const int64_t m_group = ne11;\n const int64_t m = m_group;\n const int64_t n = ne01;\n const int64_t k = ne00;\n\n const size_t lhs_stride = src1->nb[1];\n const size_t rhs_stride = src0->nb[1];\n const size_t dst_stride = dst->nb[1];\n\n const int64_t mr = (int64_t) kernel->get_mr();\n const int64_t nr = (int64_t) kernel->get_nr();\n const int64_t kr = (int64_t) kernel->get_kr();\n const int64_t sr = (int64_t) kernel->get_sr();\n\n const size_t lhs_packed_size = lhs_info->packed_size_ex(m, k, 0, mr, kr, sr);\n const size_t rhs_packed_size = kernels->rhs_info.packed_size_ex(n, k, nr, kr, 0);\n const size_t kxn_size = k * n * sizeof(float);\n const size_t bias_size = n * sizeof(float);\n\n const size_t wsize_required = lhs_packed_size + rhs_packed_size + kxn_size + bias_size;\n GGML_ASSERT(wsize_required <= params->wsize);\n\n uint8_t * lhs_packed = static_cast(params->wdata);\n uint8_t * rhs_packed = lhs_packed + lhs_packed_size;\n uint8_t * rhs_kxn = rhs_packed + rhs_packed_size;\n uint8_t * bias = rhs_kxn + kxn_size;\n\n for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) {\n const int64_t rhs_batch_idx = batch_idx / r;\n const uint8_t * rhs_batch_base = static_cast(src0->data) + rhs_batch_idx * src0->nb[2];\n uint8_t * dst_batch_base = static_cast(dst->data) + batch_idx * dst->nb[2];\n\n // LHS packing (threaded over m, honoring mr alignment and KV groups)\n {\n const int64_t m_roundup_mr = kai_roundup(m, mr);\n const int64_t num_threads = KAI_MIN(m_roundup_mr / mr, nth);\n\n if (ith < num_threads) {\n const int64_t num_m_per_thread0 = round_down((size_t)(m_roundup_mr / num_threads), (size_t)mr);\n const int64_t num_m_per_threadN_1 = m - (num_threads - 1) * num_m_per_thread0;\n\n const int64_t m_start = ith * num_m_per_thread0;\n const int64_t m_count = (ith == num_threads - 1) ? num_m_per_threadN_1 : num_m_per_thread0;\n\n // Base packed offset (aligned) and per-row stride in bytes\n const size_t base_packed_off = lhs_info->get_packed_offset_ex(m_start, k, 0, mr, kr, sr);\n const size_t next_block_off = lhs_info->get_packed_offset_ex(m_start + mr, k, 0, mr, kr, sr);\n const size_t row_stride_bytes = (next_block_off - base_packed_off) / (size_t)mr;\n\n int64_t remaining = m_count;\n int64_t cur = m_start;\n\n while (remaining > 0) {\n const int64_t row_in_group = cur;\n const int64_t avail = m_group - row_in_group;\n const int64_t take = std::min(avail, remaining);\n\n const uint8_t * lhs_batch_base = static_cast(src1->data) + batch_idx * src1->nb[2];\n const void * src_ptr = lhs_batch_base + (size_t)row_in_group * lhs_stride;\n const size_t dst_off = base_packed_off + (size_t)(cur - m_start) * row_stride_bytes;\n void * dst_ptr = lhs_packed + dst_off;\n\n lhs_info->pack_func_ex(take, k, 0, mr, kr, sr, 0, src_ptr, lhs_stride, dst_ptr);\n\n cur += take;\n remaining -= take;\n }\n }\n }\n\n // RHS packing (single thread), then synchronize\n if (ith == 0) {\n memset(bias, 0, (size_t)n * sizeof(float));\n transpose_f32kxn_f16nxk((size_t)n, (size_t)k,\n reinterpret_cast(rhs_kxn),\n reinterpret_cast(rhs_batch_base),\n rhs_stride);\n\n kernels->rhs_info.pack_func_ex(1, n, k, nr, kr, sr, 0, n * sizeof(float),\n rhs_kxn, bias, nullptr, rhs_packed, 0, nullptr);\n }\n\n ggml_barrier(params->threadpool);\n\n // Matmul (threaded over n)\n {\n const int64_t n_step = (int64_t) kernel->get_n_step();\n int64_t num_threads_n = KAI_MIN(n / n_step, nth);\n if (num_threads_n <= 0) {\n num_threads_n = 1;\n }\n\n if (ith < num_threads_n) {\n const int64_t num_n_per_thread0 = round_down((size_t)(n / num_threads_n), (size_t)n_step);\n const int64_t num_n_per_threadN_1 = n - (num_threads_n - 1) * num_n_per_thread0;\n\n const int64_t n_start = ith * num_n_per_thread0;\n const int64_t n_to_process = (ith == num_threads_n - 1) ? num_n_per_threadN_1 : num_n_per_thread0;\n\n // LHS packed base at row 0 (consistent with packing above)\n const size_t lhs_packed_offset0 = lhs_info->get_packed_offset_ex(0, k, 0, mr, kr, sr);\n const size_t rhs_packed_offset = kernel->get_rhs_packed_offset_ex(n_start, k, 0);\n const size_t dst_offset = kernel->get_dst_offset((size_t)0, (size_t)n_start, dst_stride);\n\n const void * lhs_ptr = lhs_packed + lhs_packed_offset0;\n const void * rhs_ptr = rhs_packed + rhs_packed_offset;\n float * dst_ptr = reinterpret_cast(dst_batch_base + dst_offset);\n\n kernel->run_kernel_ex(m, n_to_process, k, 0, lhs_ptr, rhs_ptr, dst_ptr, dst_stride, sizeof(float), -FLT_MAX, FLT_MAX);\n }\n }\n\n if (batch_idx != batch_size - 1) {\n ggml_barrier(params->threadpool);\n }\n }\n\n return true;\n }\n\n bool compute_forward_qx(struct ggml_compute_params * params, struct ggml_tensor * dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data);\n const bool has_header = kleidiai_is_weight_header_valid(header);\n const bool is_gemv = src1->ne[1] == 1;\n std::array kernel_chain;\n const int slot_total = kleidiai_collect_kernel_chain(dst, kernel_chain);\n\n auto weight_for_slot = [&](int slot_index, size_t & size_out) -> const uint8_t * {\n if (slot_index < 0 || slot_index >= slot_total) {\n return nullptr;\n }\n if (has_header) {\n if (slot_index < header->slot_count) {\n size_out = static_cast(header->sizes[slot_index]);\n return kleidiai_weight_slot_ptr(header, slot_index);\n }\n return nullptr;\n }\n if (slot_index == 0) {\n size_out = ggml_nbytes(src0);\n return static_cast(src0->data);\n }\n return nullptr;\n };\n\n struct runtime_slot {\n int slot_index;\n ggml_kleidiai_kernels * kernels;\n kernel_info * kernel;\n lhs_packing_info * lhs_info;\n size_t mr;\n size_t nr;\n size_t kr;\n size_t sr;\n size_t n_step;\n size_t lhs_packed_size;\n size_t lhs_offset;\n size_t n_offset;\n size_t n_cols;\n int assigned_threads;\n int thread_begin;\n int thread_end;\n const uint8_t * rhs_base;\n };\n\n std::array runtime{};\n int runtime_count = 0;\n\n for (int slot = 0; slot < slot_total && runtime_count < GGML_KLEIDIAI_MAX_KERNEL_SLOTS; ++slot) {\n ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n kernel_info * kinfo = is_gemv ? &kernels->gemv : &kernels->gemm;\n lhs_packing_info * linfo = is_gemv ? &kernels->gemv_lhs_info : &kernels->gemm_lhs_info;\n if (!kinfo || !linfo || !linfo->packed_size_ex || !linfo->pack_func_ex || !linfo->get_offset ||\n !kinfo->get_rhs_packed_offset_ex || !kinfo->run_kernel_ex || !kinfo->get_dst_offset) {\n continue;\n }\n\n size_t rhs_size = 0;\n const uint8_t * rhs_ptr = weight_for_slot(slot, rhs_size);\n if (!rhs_ptr || rhs_size == 0) {\n continue;\n }\n\n runtime[runtime_count] = {\n slot,\n kernels,\n kinfo,\n linfo,\n kinfo->get_mr(),\n kinfo->get_nr(),\n kinfo->get_kr(),\n kinfo->get_sr(),\n kinfo->get_n_step(),\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n rhs_ptr\n };\n ++runtime_count;\n }\n\n if (runtime_count == 0) {\n ggml_kleidiai_kernels * fallback = ggml_kleidiai_select_kernels(ctx.features, dst);\n if (!fallback) {\n return false;\n }\n kernel_info * kinfo = is_gemv ? &fallback->gemv : &fallback->gemm;\n lhs_packing_info * linfo = is_gemv ? &fallback->gemv_lhs_info : &fallback->gemm_lhs_info;\n rhs_packing_info * rinfo = &fallback->rhs_info;\n if (!kinfo || !linfo || !linfo->packed_size_ex || !linfo->pack_func_ex ||\n !kinfo->get_rhs_packed_offset_ex || !kinfo->run_kernel_ex || !kinfo->get_dst_offset ||\n !rinfo || !rinfo->pack_func_ex || !rinfo->packed_size_ex) {\n return false;\n }\n kernel_chain[0] = fallback;\n runtime[0] = {\n 0,\n fallback,\n kinfo,\n linfo,\n kinfo->get_mr(),\n kinfo->get_nr(),\n kinfo->get_kr(),\n kinfo->get_sr(),\n kinfo->get_n_step(),\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n nullptr\n };\n size_t rhs_size_fallback = 0;\n const uint8_t * rhs_base = weight_for_slot(0, rhs_size_fallback);\n if (!rhs_base) {\n rhs_base = static_cast(src0->data);\n }\n runtime[0].rhs_base = rhs_base;\n runtime_count = 1;\n }\n\n const int nth_total = params->nth > 0 ? params->nth : 1;\n const int ith_total = params->ith;\n\n int sme_slot = -1;\n for (int i = 0; i < runtime_count; ++i) {\n if ((runtime[i].kernels->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) {\n sme_slot = i;\n break;\n }\n }\n\n const int sme_cap_limit = ctx.sme_thread_cap;\n const bool use_hybrid = sme_cap_limit > 0 &&\n runtime_count > 1 &&\n nth_total > sme_cap_limit;\n // Heuristic: disable hybrid for very small workloads where per-slot overhead dominates.\n // If rows are small or average columns per thread are small, keep single-slot.\n size_t min_cols_per_thread = 0;\n if (runtime_count > 0 && nth_total > 0) {\n min_cols_per_thread = (size_t) std::max(1, (int64_t)ne01 / (int64_t)nth_total);\n }\n const bool too_small_for_hybrid = (min_cols_per_thread < 2) || (ne11 < 128);\n\n const bool hybrid_enabled = use_hybrid && !too_small_for_hybrid;\n\n if (!hybrid_enabled) {\n int chosen_slot = 0;\n if (too_small_for_hybrid && sme_slot != -1) {\n chosen_slot = sme_slot;\n } else if (runtime_count > 1 && ctx.sme_thread_cap > 0 && nth_total > ctx.sme_thread_cap) {\n chosen_slot = 1;\n }\n if (chosen_slot != 0 && chosen_slot < runtime_count) {\n runtime[0] = runtime[chosen_slot];\n }\n runtime_count = runtime_count > 0 ? 1 : 0;\n\n // Recompute SME slot based on the collapsed runtime[0]\n sme_slot = -1;\n if (runtime_count > 0 &&\n (runtime[0].kernels->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) {\n sme_slot = 0;\n }\n }\n\n int sme_cap = kleidiai_sme_thread_cap();\n if (sme_cap < 0) {\n sme_cap = nth_total;\n }\n sme_cap = std::min(sme_cap, nth_total);\n\n int threads_remaining = nth_total;\n if (sme_slot != -1) {\n int sme_threads = std::min(std::max(sme_cap, 0), threads_remaining);\n runtime[sme_slot].assigned_threads = sme_threads;\n threads_remaining -= sme_threads;\n }\n\n int fallback_indices[GGML_KLEIDIAI_MAX_KERNEL_SLOTS];\n int fallback_count = 0;\n for (int i = 0; i < runtime_count; ++i) {\n if (i == sme_slot) {\n continue;\n }\n fallback_indices[fallback_count++] = i;\n }\n\n for (int fi = 0; fi < fallback_count; ++fi) {\n if (threads_remaining <= 0) {\n break;\n }\n const int slot_index = fallback_indices[fi];\n const int slots_left = fallback_count - fi;\n int share = (threads_remaining + slots_left - 1) / slots_left;\n share = std::min(share, threads_remaining);\n runtime[slot_index].assigned_threads = share;\n threads_remaining -= share;\n }\n\n if (threads_remaining > 0) {\n const int fallback_slot = (sme_slot != -1) ? sme_slot : 0;\n runtime[fallback_slot].assigned_threads += threads_remaining;\n threads_remaining = 0;\n }\n\n int thread_cursor = 0;\n for (int i = 0; i < runtime_count; ++i) {\n runtime[i].thread_begin = thread_cursor;\n thread_cursor += runtime[i].assigned_threads;\n runtime[i].thread_end = thread_cursor;\n }\n\n if (thread_cursor < nth_total && runtime_count > 0) {\n runtime[runtime_count - 1].assigned_threads += nth_total - thread_cursor;\n runtime[runtime_count - 1].thread_end = nth_total;\n }\n\n int local_slot = -1;\n int local_ith = 0;\n for (int i = 0; i < runtime_count; ++i) {\n if (ith_total >= runtime[i].thread_begin && ith_total < runtime[i].thread_end) {\n local_slot = i;\n local_ith = ith_total - runtime[i].thread_begin;\n break;\n }\n }\n if (local_slot == -1) {\n return false;\n }\n\n const size_t k = ne00;\n const size_t m = ne11;\n const size_t n = ne01;\n\n size_t cursor = 0;\n for (int i = 0; i < runtime_count; ++i) {\n const ggml_type slot_rhs_type = runtime[i].kernels->rhs_type;\n const size_t slot_pack_size_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n slot_rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : 0;\n runtime[i].lhs_packed_size = runtime[i].lhs_info->packed_size_ex(m, k, slot_pack_size_arg, runtime[i].mr, runtime[i].kr, runtime[i].sr);\n cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n runtime[i].lhs_offset = cursor;\n cursor += runtime[i].lhs_packed_size;\n }\n\n GGML_ASSERT(cursor <= params->wsize);\n uint8_t * scratch = static_cast(params->wdata);\n\n size_t assigned_cols = 0;\n uint64_t weighted_total = 0;\n if (runtime_count > 1 && sme_slot != -1) {\n for (int i = 0; i < runtime_count; ++i) {\n const uint64_t weight = (i == sme_slot) ? (sme_cap << 1) : 1;\n weighted_total += (uint64_t)runtime[i].assigned_threads * weight;\n }\n }\n for (int i = 0; i < runtime_count; ++i) {\n runtime[i].n_offset = assigned_cols;\n if (runtime[i].assigned_threads == 0) {\n runtime[i].n_cols = 0;\n continue;\n }\n const size_t remaining_cols = n - assigned_cols;\n if (remaining_cols == 0) {\n runtime[i].n_cols = 0;\n continue;\n }\n const size_t step = runtime[i].n_step ? runtime[i].n_step : 1;\n size_t target = 0;\n if (weighted_total > 0) {\n const uint64_t weight = (i == sme_slot) ? (sme_cap << 1) : 1;\n target = (size_t)(((uint64_t)n * runtime[i].assigned_threads * weight) / weighted_total);\n } else {\n target = (size_t)(((uint64_t)n * runtime[i].assigned_threads) / nth_total);\n }\n target = std::min(target, remaining_cols);\n size_t aligned = round_down(target, step);\n if (aligned == 0 && remaining_cols >= step) {\n aligned = step;\n }\n runtime[i].n_cols = aligned;\n assigned_cols += aligned;\n }\n\n if (assigned_cols < n) {\n for (int i = runtime_count - 1; i >= 0; --i) {\n if (runtime[i].assigned_threads > 0) {\n runtime[i].n_cols += n - assigned_cols;\n break;\n }\n }\n }\n const size_t dst_stride = dst->nb[1];\n\n for (int64_t batch_idx = 0; batch_idx < ne12; ++batch_idx) {\n const uint8_t * lhs_batch_base = static_cast(src1->data) + batch_idx * src1->nb[2];\n uint8_t * dst_batch_base = static_cast(dst->data) + batch_idx * dst->nb[2];\n\n if (runtime[local_slot].assigned_threads > 0) {\n runtime_slot & slot = runtime[local_slot];\n const ggml_type slot_rhs_type = slot.kernels->rhs_type;\n const size_t slot_lhs_exec_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0;\n const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)slot.mr);\n int64_t max_threads = slot.mr ? (m_roundup_mr / (int64_t)slot.mr) : slot.assigned_threads;\n max_threads = std::max(1, max_threads);\n const int64_t use_threads = std::min(slot.assigned_threads, max_threads);\n\n if (local_ith < use_threads) {\n const int64_t num_m_per_thread0 = round_down((size_t)(m_roundup_mr / use_threads), slot.mr);\n const int64_t num_m_per_threadN_1 = (int64_t)m - (use_threads - 1) * num_m_per_thread0;\n\n const int64_t m_start = (int64_t)local_ith * num_m_per_thread0;\n const int64_t m_count = (local_ith == use_threads - 1) ? num_m_per_threadN_1 : num_m_per_thread0;\n\n const size_t base_packed_off = slot.lhs_info->get_packed_offset_ex(m_start, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr);\n const size_t next_block_off = slot.lhs_info->get_packed_offset_ex(m_start + slot.mr, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr);\n const size_t row_stride_bytes = slot.mr ? (next_block_off - base_packed_off) / slot.mr : 0;\n\n int64_t remaining = m_count;\n int64_t cur = m_start;\n\n uint8_t * lhs_packed = scratch + slot.lhs_offset;\n while (remaining > 0) {\n const int64_t row_in_group = cur;\n const int64_t avail = (int64_t)m - row_in_group;\n const int64_t take = std::min(avail, remaining);\n\n const size_t src_off = slot.lhs_info->get_offset(row_in_group, src1->nb[1]);\n const void * src_ptr = lhs_batch_base + src_off;\n const size_t dst_off = base_packed_off + (size_t)(cur - m_start) * row_stride_bytes;\n void * dst_ptr = lhs_packed + dst_off;\n\n slot.lhs_info->pack_func_ex(take, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr, 0, src_ptr, src1->nb[1], dst_ptr);\n\n cur += take;\n remaining -= take;\n }\n }\n }\n\n ggml_barrier(params->threadpool);\n\n runtime_slot & slot = runtime[local_slot];\n if (slot.n_cols > 0 && slot.assigned_threads > 0) {\n int64_t active_threads = slot.assigned_threads;\n const int64_t max_threads = slot.n_step ? (slot.n_cols / slot.n_step) : slot.assigned_threads;\n if (max_threads > 0) {\n active_threads = std::min(active_threads, std::max(1, max_threads));\n }\n active_threads = std::max(1, active_threads);\n\n if (local_ith < active_threads) {\n const size_t step = slot.n_step ? slot.n_step : 1;\n const size_t chunk0 = round_down((size_t)(slot.n_cols / active_threads), step);\n const size_t chunkN = slot.n_cols - (active_threads - 1) * chunk0;\n const size_t local_start = (size_t)local_ith * chunk0;\n const size_t cols = (local_ith == active_threads - 1) ? chunkN : chunk0;\n\n if (cols > 0) {\n const ggml_type slot_rhs_type = slot.kernels->rhs_type;\n const size_t slot_lhs_exec_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0;\n const size_t slot_rhs_block_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0;\n const size_t global_start = slot.n_offset + local_start;\n const size_t lhs_packed_offset = slot.lhs_info->get_packed_offset_ex(0, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr);\n const size_t rhs_packed_offset = slot.kernel->get_rhs_packed_offset_ex(global_start, k, slot_rhs_block_arg);\n const size_t dst_offset = slot.kernel->get_dst_offset(0, global_start, dst_stride);\n\n const uint8_t * lhs_ptr = scratch + slot.lhs_offset + lhs_packed_offset;\n const uint8_t * rhs_ptr = slot.rhs_base + rhs_packed_offset;\n float * dst_ptr = reinterpret_cast(dst_batch_base + dst_offset);\n\n slot.kernel->run_kernel_ex(m, cols, k, slot_rhs_block_arg,\n lhs_ptr,\n rhs_ptr,\n dst_ptr,\n dst_stride,\n sizeof(float),\n -FLT_MAX,\n FLT_MAX);\n }\n }\n }\n\n if (batch_idx != ne12 - 1) {\n ggml_barrier(params->threadpool);\n }\n }\n\n return true;\n }\n\n bool compute_forward_get_rows(struct ggml_compute_params * params, struct ggml_tensor * dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data);\n const bool has_header = kleidiai_is_weight_header_valid(header);\n\n std::array kernel_chain;\n const bool want_q8 = src0->type == GGML_TYPE_Q8_0;\n const int chain_count = want_q8 ? kleidiai_collect_q8_chain(kernel_chain)\n : kleidiai_collect_q4_chain(kernel_chain);\n\n ggml_kleidiai_kernels * kernels = nullptr;\n const uint8_t * packed_base = static_cast(src0->data);\n\n if (has_header && chain_count > 0) {\n int select_slot = 0;\n if (select_slot >= header->slot_count) {\n select_slot = header->slot_count - 1;\n }\n if (select_slot >= 0 && select_slot < chain_count) {\n kernels = kernel_chain[select_slot];\n const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, select_slot);\n if (slot_ptr) {\n packed_base = slot_ptr;\n }\n }\n }\n\n if (!kernels && chain_count > 0) {\n kernels = kernel_chain[0];\n if (has_header) {\n const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, 0);\n if (slot_ptr) {\n packed_base = slot_ptr;\n }\n }\n }\n\n if (!kernels) {\n return false;\n }\n\n rhs_packing_info * rhs_info = &kernels->rhs_info;\n kernel_info * kernel = &kernels->gemm;\n if (!rhs_info->to_float || !kernel->get_nr) {\n return false;\n }\n\n const int64_t nc = ne00;\n const int64_t nr = ggml_nelements(src1);\n\n const ggml_type rhs_type = kernels->rhs_type;\n size_t block_len = 0;\n size_t num_bytes_multiplier = 0;\n if (rhs_type == GGML_TYPE_Q4_0) {\n block_len = QK4_0;\n num_bytes_multiplier = sizeof(uint16_t);\n } else if (rhs_type == GGML_TYPE_Q8_0) {\n block_len = QK8_0;\n num_bytes_multiplier = sizeof(float);\n } else {\n return false;\n }\n\n const size_t block_rows = kernel->get_nr();\n const size_t kr = kernel->get_kr();\n\n const size_t packed_stride = rhs_info->packed_stride(nc, block_rows, kr, block_len);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int dr = (nr + nth - 1) / nth;\n const int ir0 = dr * ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int64_t i = ir0; i < ir1; ++i) {\n GGML_ASSERT(src1->type == GGML_TYPE_I32);\n int64_t row_idx = ((const int32_t *)src1->data)[i];\n GGML_ASSERT(row_idx >= 0 && row_idx < src0->ne[1]);\n\n float *out = (float *)((char *)dst->data + i * nb1);\n rhs_info->to_float(packed_base, row_idx, nc, out, block_rows, packed_stride, kr, block_len, num_bytes_multiplier);\n }\n\n return true;\n }\n\npublic:\n int repack(struct ggml_tensor * tensor, const void * data, size_t data_size) {\n GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0);\n const size_t n = tensor->ne[1];\n const size_t k = tensor->ne[0];\n\n kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(tensor->data);\n if (!header) {\n return -1;\n }\n\n header->magic = GGML_KLEIDIAI_PACK_MAGIC;\n header->version = GGML_KLEIDIAI_PACK_VERSION;\n header->slot_count = 0;\n\n uint8_t * base_ptr = static_cast(tensor->data);\n size_t cursor = sizeof(kleidiai_weight_header);\n cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n\n std::array kernel_chain;\n const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;\n const int slot_total = want_q8 ? kleidiai_collect_q8_chain(kernel_chain)\n : kleidiai_collect_q4_chain(kernel_chain);\n const bool allow_fallback = kleidiai_pack_fallback_allowed();\n\n std::vector qdata;\n std::vector scales;\n\n if (want_q8 && slot_total > 0) {\n qdata.resize(n * k, 0);\n scales.resize(n, 0.0f);\n\n const size_t row_stride = tensor->nb[1];\n const size_t k_blocks = (k + QK8_0 - 1) / QK8_0;\n\n for (size_t row = 0; row < n; ++row) {\n const auto * row_blocks = reinterpret_cast(\n static_cast(data) + row * row_stride);\n\n float max_abs = 0.0f;\n for (size_t block = 0; block < k_blocks; ++block) {\n const block_q8_0 & blk = row_blocks[block];\n const float d = GGML_FP16_TO_FP32(blk.d);\n for (size_t l = 0; l < QK8_0; ++l) {\n const size_t linear_idx = block * QK8_0 + l;\n if (linear_idx >= k) {\n break;\n }\n const float value = d * static_cast(blk.qs[l]);\n max_abs = std::max(max_abs, std::fabs(value));\n }\n }\n\n float scale = max_abs > 0.0f ? max_abs / 127.0f : 0.0f;\n scales[row] = scale;\n const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f;\n\n for (size_t block = 0; block < k_blocks; ++block) {\n const block_q8_0 & blk = row_blocks[block];\n const float d = GGML_FP16_TO_FP32(blk.d);\n for (size_t l = 0; l < QK8_0; ++l) {\n const size_t linear_idx = block * QK8_0 + l;\n if (linear_idx >= k) {\n break;\n }\n const float value = d * static_cast(blk.qs[l]);\n int32_t q = scale > 0.0f ? static_cast(std::lround(value * inv_scale)) : 0;\n q = std::clamp(q, -127, 127);\n qdata[row * k + linear_idx] = static_cast(q);\n }\n }\n }\n }\n\n for (int slot = 0; slot < slot_total && slot < GGML_KLEIDIAI_MAX_KERNEL_SLOTS; ++slot) {\n if (!allow_fallback && slot > 0) {\n break;\n }\n ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n kernel_info * kernel = &kernels->gemm;\n rhs_packing_info * rhs_info = &kernels->rhs_info;\n if (!rhs_info || !rhs_info->pack_func_ex || !rhs_info->packed_size_ex || !kernel) {\n continue;\n }\n\n const size_t nr = kernel->get_nr();\n const size_t kr = kernel->get_kr();\n const size_t sr = kernel->get_sr();\n const ggml_type rhs_type = kernels->rhs_type;\n const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :\n rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : 0;\n if (block_len == 0) {\n continue;\n }\n\n const size_t packed_size = rhs_info->packed_size_ex(n, k, nr, kr, block_len);\n const size_t aligned_cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n\n uint8_t * dst_ptr = base_ptr + aligned_cursor;\n\n if (rhs_type == GGML_TYPE_Q4_0) {\n struct kai_rhs_pack_qs4cxs1s0_param params;\n params.lhs_zero_point = 1;\n params.rhs_zero_point = 8;\n rhs_info->pack_func_ex(1, n, k, nr, kr, sr, QK4_0, 0,\n static_cast(data), nullptr, nullptr,\n dst_ptr, 0, ¶ms);\n } else if (rhs_type == GGML_TYPE_Q8_0) {\n struct kai_rhs_pack_qsi8cx_params params;\n params.lhs_zero_point = 1;\n params.scale_multiplier = 1.0f;\n rhs_info->pack_func_ex(1, n, k, nr, kr, sr, 0, 0,\n qdata.data(), nullptr, scales.data(),\n dst_ptr, 0, ¶ms);\n } else {\n continue;\n }\n\n header->offsets[header->slot_count] = aligned_cursor;\n header->sizes[header->slot_count] = packed_size;\n ++header->slot_count;\n\n cursor = aligned_cursor + packed_size;\n }\n\n if (header->slot_count == 0) {\n header->magic = 0;\n header->version = 0;\n memcpy(tensor->data, data, data_size);\n }\n\n return 0;\n }\n};\n\nstatic ggml::cpu::tensor_traits * get_tensor_traits(ggml_backend_buffer_t, struct ggml_tensor *) {\n static tensor_traits traits;\n return &traits;\n}\n} // namespace ggml::cpu::kleidiai\n\nstatic enum ggml_status ggml_backend_cpu_kleidiai_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {\n tensor->extra = (void *) ggml::cpu::kleidiai::get_tensor_traits(buffer, tensor);\n\n return GGML_STATUS_SUCCESS;\n GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_cpu_kleidiai_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,\n const void * data, size_t offset, size_t size) {\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(size == ggml_nbytes(tensor));\n\n auto tensor_traits = (ggml::cpu::kleidiai::tensor_traits *) tensor->extra;\n auto OK = tensor_traits->repack(tensor, data, size);\n\n GGML_ASSERT(OK == 0);\n GGML_UNUSED(buffer);\n}\n\nstatic const char * ggml_backend_cpu_kleidiai_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n GGML_UNUSED(buft);\n return \"CPU_KLEIDIAI\";\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_kleidiai_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n\n if (buffer == nullptr) {\n return nullptr;\n }\n\n buffer->buft = buft;\n buffer->iface.init_tensor = ggml_backend_cpu_kleidiai_buffer_init_tensor;\n buffer->iface.set_tensor = ggml_backend_cpu_kleidiai_buffer_set_tensor;\n buffer->iface.get_tensor = nullptr;\n buffer->iface.cpy_tensor = nullptr;\n return buffer;\n}\n\nstatic size_t ggml_backend_cpu_kleidiai_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n GGML_UNUSED(buft);\n return TENSOR_ALIGNMENT;\n}\n\nstatic size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {\n GGML_UNUSED(buft);\n\n if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q8_0) {\n return ggml_nbytes(tensor);\n }\n\n const size_t n = tensor->ne[1];\n const size_t k = tensor->ne[0];\n\n size_t cursor = sizeof(kleidiai_weight_header);\n cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n\n std::array kernel_chain;\n const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;\n const int slot_total = want_q8 ? kleidiai_collect_q8_chain(kernel_chain)\n : kleidiai_collect_q4_chain(kernel_chain);\n const bool allow_fallback = kleidiai_pack_fallback_allowed();\n\n size_t slot_count = 0;\n for (int slot = 0; slot < slot_total; ++slot) {\n if (!allow_fallback && slot > 0) {\n break;\n }\n ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n if (!kernels) {\n continue;\n }\n kernel_info * kernel = &kernels->gemm;\n rhs_packing_info * rhs_info = &kernels->rhs_info;\n if (!kernel || !rhs_info || !rhs_info->packed_size_ex) {\n continue;\n }\n\n const ggml_type rhs_type = kernels->rhs_type;\n const size_t block_len = rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : 0;\n if (block_len == 0) {\n continue;\n }\n\n cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n cursor += rhs_info->packed_size_ex(n, k, kernel->get_nr(), kernel->get_kr(), block_len);\n ++slot_count;\n }\n\n if (slot_count == 0) {\n return ggml_nbytes(tensor);\n }\n\n return std::max(cursor, ggml_nbytes(tensor));\n}\n\nnamespace ggml::cpu::kleidiai {\nclass extra_buffer_type : ggml::cpu::extra_buffer_type {\n bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {\n std::array kernel_chain;\n const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain);\n if ((op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) &&\n (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) &&\n op->src[0]->buffer &&\n (ggml_n_dims(op->src[0]) == 2) &&\n op->src[0]->buffer->buft == ggml_backend_cpu_kleidiai_buffer_type() &&\n slot_total > 0) {\n if (op->src[0]->type == GGML_TYPE_Q4_0 && ctx.kernels_q4 == nullptr) {\n return false;\n }\n if (op->src[0]->type == GGML_TYPE_Q8_0 && ctx.kernels_q8 == nullptr) {\n return false;\n }\n if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {\n return false;\n }\n if ((op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_I32) &&\n ggml_ne(op->src[1], 2) == 1 && ggml_ne(op->src[1], 3) == 1) {\n return true;\n }\n }\n return false;\n }\n\n ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {\n if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) {\n if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_kleidiai_buffer_type()) {\n return (ggml::cpu::tensor_traits *) op->src[0]->extra;\n } else {\n std::array kernel_chain;\n const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain);\n const bool has_kernel = slot_total > 0;\n if (has_kernel && op->src[1]->ne[1] > 1) {\n if ((op->src[0]->nb[1] * op->src[0]->ne[1] != op->src[0]->nb[2]) ||\n (op->src[1]->nb[1] * op->src[1]->ne[1] != op->src[1]->nb[2])) {\n return nullptr;\n }\n return ggml::cpu::kleidiai::get_tensor_traits(NULL, NULL);\n }\n }\n }\n return nullptr;\n }\n};\n} // namespace ggml::cpu::kleidiai\n\nggml_backend_buffer_type_t ggml_backend_cpu_kleidiai_buffer_type(void) {\n static ggml::cpu::kleidiai::extra_buffer_type ctx;\n static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_kleidiai = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_cpu_kleidiai_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_cpu_kleidiai_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cpu_kleidiai_buffer_type_get_alignment,\n /* .get_max_size = */ nullptr, // defaults to SIZE_MAX\n /* .get_alloc_size = */ ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size,\n /* .is_host = */ nullptr,\n },\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n /* .context = */ &ctx,\n };\n\n init_kleidiai_context();\n\n return &ggml_backend_cpu_buffer_type_kleidiai;\n}\n" }, { "path": "src/ggml-cpu/kleidiai/kleidiai.h", "content": "// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates \n// SPDX-License-Identifier: MIT\n//\n\n#pragma once\n\n#include \"ggml-alloc.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nggml_backend_buffer_type_t ggml_backend_cpu_kleidiai_buffer_type(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cpu/llamafile/sgemm.cpp", "content": "// Copyright 2024 Mozilla Foundation\n//\n// Permission is hereby granted, free of charge, to any person obtaining\n// a copy of this software and associated documentation files (the\n// \"Software\"), to deal in the Software without restriction, including\n// without limitation the rights to use, copy, modify, merge, publish,\n// distribute, sublicense, and/or sell copies of the Software, and to\n// permit persons to whom the Software is furnished to do so, subject to\n// the following conditions:\n//\n// The above copyright notice and this permission notice shall be\n// included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\n// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\n// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\n// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\n// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\n// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n// SOFTWARE.\n\n//\n// _ _ ___ _ _ ___\n// | |_(_)_ _ _ _| _ ) | /_\\ / __|\n// | _| | ' \\ || | _ \\ |__ / _ \\\\__ \\.\n// \\__|_|_||_\\_, |___/____/_/ \\_\\___/\n// |__/\n//\n// BASIC LINEAR ALGEBRA SUBPROGRAMS\n//\n//\n// This file implements multithreaded CPU matrix multiplication for the\n// common contiguous use case C = Aᵀ * B. These kernels are designed to\n// have excellent performance[1] for matrices that fit in the CPU cache\n// without imposing any overhead such as cache filling or malloc calls.\n//\n// This implementation does not guarantee any upper bound with rounding\n// errors, which grow along with k. Our goal's to maximally exploit the\n// hardware for performance, and then use whatever resources remain for\n// improving numerical accuracy.\n//\n// [1] J. Tunney, ‘LLaMA Now Goes Faster on CPUs’, Mar. 2024. [Online].\n// Available: https://justine.lol/matmul/. [Accessed: 29-Mar-2024].\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Wpedantic\"\n#pragma GCC diagnostic ignored \"-Wignored-attributes\"\n#endif\n\n#include \"sgemm.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"ggml-quants.h\"\n#include \"simd-mappings.h\"\n\n#include \n#include \n\n#ifdef _MSC_VER\n#define NOINLINE __declspec(noinline)\n#else\n#define NOINLINE __attribute__((__noinline__))\n#endif\n\n#if defined(__ARM_NEON) || defined(__AVX512F__) || defined(__VXE__) || defined(__VXE2__)\n#define VECTOR_REGISTERS 32\n#else\n#define VECTOR_REGISTERS 16\n#endif\n\n#if defined(__riscv_v_intrinsic)\n#define LMUL 4\n#endif\n\n#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)\n\nnamespace {\n\ninline float unhalf(ggml_fp16_t d) {\n return GGML_CPU_FP16_TO_FP32(d);\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// VECTORIZED ARITHMETIC OPERATIONS\n\n#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ninline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }\ninline __m128 sub(__m128 x, __m128 y) { return _mm_sub_ps(x, y); }\ninline __m128 mul(__m128 x, __m128 y) { return _mm_mul_ps(x, y); }\n#endif // __SSE__\n\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ninline __m256 add(__m256 x, __m256 y) { return _mm256_add_ps(x, y); }\ninline __m256 sub(__m256 x, __m256 y) { return _mm256_sub_ps(x, y); }\ninline __m256 mul(__m256 x, __m256 y) { return _mm256_mul_ps(x, y); }\n#endif // __AVX__\n\n#if defined(__AVX512F__)\ninline __m512 add(__m512 x, __m512 y) { return _mm512_add_ps(x, y); }\ninline __m512 sub(__m512 x, __m512 y) { return _mm512_sub_ps(x, y); }\ninline __m512 mul(__m512 x, __m512 y) { return _mm512_mul_ps(x, y); }\n#endif // __AVX512F__\n\n#if defined(__ARM_NEON)\ninline float32x4_t add(float32x4_t x, float32x4_t y) { return vaddq_f32(x, y); }\ninline float32x4_t sub(float32x4_t x, float32x4_t y) { return vsubq_f32(x, y); }\ninline float32x4_t mul(float32x4_t x, float32x4_t y) { return vmulq_f32(x, y); }\n#endif // __ARM_NEON\n\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)\ninline float16x8_t add(float16x8_t x, float16x8_t y) { return vaddq_f16(x, y); }\ninline float16x8_t sub(float16x8_t x, float16x8_t y) { return vsubq_f16(x, y); }\ninline float16x8_t mul(float16x8_t x, float16x8_t y) { return vmulq_f16(x, y); }\n#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC\n\n#if defined(__VXE__) || defined(__VXE2__)\ninline float32x4_t add(float32x4_t x, float32x4_t y) { return vec_add(x, y); }\ninline float32x4_t sub(float32x4_t x, float32x4_t y) { return vec_sub(x, y); }\ninline float32x4_t mul(float32x4_t x, float32x4_t y) { return vec_mul(x, y); }\n#endif\n\n#if defined(__MMA__)\ntypedef vector unsigned char vec_t;\ntypedef __vector_quad acc_t;\n#endif\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// VECTORIZED FUSED MULTIPLY ADD\n\n/**\n * Computes a * b + c.\n */\ntemplate \ninline U madd(T a, T b, U c) {\n return add(mul(a, b), c);\n}\n\n#if defined(__FMA__)\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ntemplate <>\ninline __m256 madd(__m256 a, __m256 b, __m256 c) {\n return _mm256_fmadd_ps(a, b, c);\n}\n#endif\n#if defined(__AVX512F__)\ntemplate <>\ninline __m512 madd(__m512 a, __m512 b, __m512 c) {\n return _mm512_fmadd_ps(a, b, c);\n}\n#endif\n#if defined(__AVX512BF16__)\ntemplate <>\ninline __m512 madd(__m512bh a, __m512bh b, __m512 c) {\n return _mm512_dpbf16_ps(c, a, b);\n}\ntemplate <>\ninline __m256 madd(__m256bh a, __m256bh b, __m256 c) {\n return _mm256_dpbf16_ps(c, a, b);\n}\n#endif\n#endif\n\n#if defined(__ARM_FEATURE_FMA)\ntemplate <>\ninline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {\n return vfmaq_f32(c, b, a);\n}\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)\ntemplate <>\ninline float16x8_t madd(float16x8_t a, float16x8_t b, float16x8_t c) {\n return vfmaq_f16(c, b, a);\n}\n#endif\n#endif\n\n#if defined(__VXE__) || defined(__VXE2__)\ntemplate <>\ninline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {\n return vec_madd(a, b, c);\n}\n#endif\n\n#if defined(__riscv_zvfh)\ntemplate <>\ninline vfloat32m1_t madd(vfloat16mf2_t a, vfloat16mf2_t b, vfloat32m1_t c) {\n return __riscv_vfwmacc_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1());\n}\ninline vfloat32m2_t madd(vfloat16m1_t a, vfloat16m1_t b, vfloat32m2_t c) {\n return __riscv_vfwmacc_vv_f32m2(c, a, b, __riscv_vsetvlmax_e32m2());\n}\ninline vfloat32m4_t madd(vfloat16m2_t a, vfloat16m2_t b, vfloat32m4_t c) {\n return __riscv_vfwmacc_vv_f32m4(c, a, b, __riscv_vsetvlmax_e32m4());\n}\ninline vfloat32m8_t madd(vfloat16m4_t a, vfloat16m4_t b, vfloat32m8_t c) {\n return __riscv_vfwmacc_vv_f32m8(c, a, b, __riscv_vsetvlmax_e32m8());\n}\ninline vfloat32m1_t madd(vfloat32m1_t a, vfloat32m1_t b, vfloat32m1_t c) {\n return __riscv_vfmacc_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1());\n}\ninline vfloat32m2_t madd(vfloat32m2_t a, vfloat32m2_t b, vfloat32m2_t c) {\n return __riscv_vfmacc_vv_f32m2(c, a, b, __riscv_vsetvlmax_e32m2());\n}\ninline vfloat32m4_t madd(vfloat32m4_t a, vfloat32m4_t b, vfloat32m4_t c) {\n return __riscv_vfmacc_vv_f32m4(c, a, b, __riscv_vsetvlmax_e32m4());\n}\ninline vfloat32m8_t madd(vfloat32m8_t a, vfloat32m8_t b, vfloat32m8_t c) {\n return __riscv_vfmacc_vv_f32m8(c, a, b, __riscv_vsetvlmax_e32m8());\n}\n#endif\n\n#if defined(__riscv_zvfbfwma)\ninline vfloat32m1_t madd(vbfloat16mf2_t a, vbfloat16mf2_t b, vfloat32m1_t c) {\n return __riscv_vfwmaccbf16_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1());\n}\ninline vfloat32m2_t madd(vbfloat16m1_t a, vbfloat16m1_t b, vfloat32m2_t c) {\n return __riscv_vfwmaccbf16_vv_f32m2(c, a, b, __riscv_vsetvlmax_e32m2());\n}\ninline vfloat32m4_t madd(vbfloat16m2_t a, vbfloat16m2_t b, vfloat32m4_t c) {\n return __riscv_vfwmaccbf16_vv_f32m4(c, a, b, __riscv_vsetvlmax_e32m4());\n}\n#endif\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// VECTORIZED HORIZONTAL SUM\n\n#if defined(__ARM_NEON)\ninline float hsum(float32x4_t x) {\n return vaddvq_f32(x);\n}\n#endif // __ARM_NEON\n\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)\ninline float hsum(float16x8_t x) {\n return vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(x)),\n vcvt_f32_f16(vget_high_f16(x))));\n}\n#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC\n\n#if defined(__VXE__) || defined(__VXE2__)\ninline float hsum(float32x4_t x) {\n float32x4_t tmp = x + vec_reve(x);\n return tmp[0] + tmp[1];\n}\n#endif\n\n#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ninline float hsum(__m128 x) {\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\n x = _mm_add_ps(x, _mm_movehl_ps(x, x));\n x = _mm_add_ss(x, _mm_movehdup_ps(x));\n#else\n __m128 t;\n t = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));\n x = _mm_add_ps(x, t);\n t = _mm_movehl_ps(t, x);\n x = _mm_add_ss(x, t);\n#endif\n return _mm_cvtss_f32(x);\n}\n#endif\n\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ninline float hsum(__m256 x) {\n return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1),\n _mm256_castps256_ps128(x)));\n}\n#endif // __AVX__\n\n#if defined(__AVX512F__)\ninline float hsum(__m512 x) {\n return _mm512_reduce_add_ps(x);\n}\n#endif // __AVX512F__\n\n#if defined(__riscv_zvfh)\ninline float hsum(vfloat32m1_t x) {\n return __riscv_vfmv_f_s_f32m1_f32(\n __riscv_vfredusum_vs_f32m1_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m1()));\n}\ninline float hsum(vfloat32m2_t x) {\n return __riscv_vfmv_f_s_f32m1_f32(\n __riscv_vfredusum_vs_f32m2_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m2()));\n}\ninline float hsum(vfloat32m4_t x) {\n return __riscv_vfmv_f_s_f32m1_f32(\n __riscv_vfredusum_vs_f32m4_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m4()));\n}\ninline float hsum(vfloat32m8_t x) {\n return __riscv_vfmv_f_s_f32m1_f32(\n __riscv_vfredusum_vs_f32m8_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m8()));\n}\n#endif\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// VECTORIZED MEMORY LOADING\n\ntemplate T load(const U *);\n\n#if defined(__ARM_NEON)\ntemplate <> inline float32x4_t load(const float *p) {\n return vld1q_f32(p);\n}\n#if !defined(_MSC_VER)\n// FIXME: this should check for __ARM_FEATURE_FP16_VECTOR_ARITHMETIC\ntemplate <> inline float16x8_t load(const ggml_fp16_t *p) {\n return vld1q_f16((const float16_t *)p);\n}\ntemplate <> inline float32x4_t load(const ggml_fp16_t *p) {\n return vcvt_f32_f16(vld1_f16((const float16_t *)p));\n}\n#endif // _MSC_VER\n#endif // __ARM_NEON\n\n#if defined(__VXE__) || defined(__VXE2__)\ntemplate <> inline float32x4_t load(const ggml_fp16_t * p) {\n float tmp[4];\n\n for (int i = 0; i < 4; i++) {\n tmp[i] = GGML_CPU_FP16_TO_FP32(p[i]);\n }\n\n return vec_xl(0, (const float *)(tmp));\n}\ntemplate <> inline float32x4_t load(const float * p) {\n return vec_xl(0, p);\n}\n#endif\n\n#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ntemplate <> inline __m128 load(const float *p) {\n return _mm_loadu_ps(p);\n}\n#endif // __SSE__\n\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ntemplate <> inline __m256 load(const float *p) {\n return _mm256_loadu_ps(p);\n}\n#endif // __AVX__\n\n#if defined(__AVX2__) || defined(__AVX512F__)\ntemplate <> inline __m256 load(const ggml_bf16_t *p) {\n return _mm256_castsi256_ps(\n _mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p)), 16));\n}\n#endif // __AVX2__\n\n#if defined(__F16C__)\ntemplate <> inline __m256 load(const ggml_fp16_t *p) {\n return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));\n}\n#endif // __F16C__\n\n#if defined(__AVX512F__)\ntemplate <> inline __m512 load(const float *p) {\n return _mm512_loadu_ps(p);\n}\ntemplate <> inline __m512 load(const ggml_fp16_t *p) {\n return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));\n}\ntemplate <> inline __m512 load(const ggml_bf16_t *p) {\n return _mm512_castsi512_ps(\n _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)p)), 16));\n}\n#endif // __AVX512F__\n\n#if defined(__AVX512BF16__)\ntemplate <> inline __m512bh load(const ggml_bf16_t *p) {\n return (__m512bh)_mm512_loadu_ps((const float *)p);\n}\ntemplate <> inline __m256bh load(const ggml_bf16_t *p) {\n return (__m256bh)_mm256_loadu_ps((const float *)p);\n}\ntemplate <> inline __m512bh load(const float *p) {\n return _mm512_cvtne2ps_pbh(_mm512_loadu_ps(p + 16), _mm512_loadu_ps(p));\n}\ntemplate <> inline __m256bh load(const float *p) {\n return _mm512_cvtneps_pbh(_mm512_loadu_ps(p));\n}\n#endif\n\n#if defined(__riscv_zvfh)\ntemplate <> inline vfloat16mf2_t load(const ggml_fp16_t *p) {\n return __riscv_vle16_v_f16mf2(reinterpret_cast(p), __riscv_vsetvlmax_e16mf2());\n}\ntemplate <> inline vfloat16m1_t load(const ggml_fp16_t *p) {\n return __riscv_vle16_v_f16m1(reinterpret_cast(p), __riscv_vsetvlmax_e16m1());\n}\ntemplate <> inline vfloat16m2_t load(const ggml_fp16_t *p) {\n return __riscv_vle16_v_f16m2(reinterpret_cast(p), __riscv_vsetvlmax_e16m2());\n}\ntemplate <> inline vfloat16m4_t load(const ggml_fp16_t *p) {\n return __riscv_vle16_v_f16m4(reinterpret_cast(p), __riscv_vsetvlmax_e16m4());\n}\ntemplate <> inline vfloat32m1_t load(const float *p) {\n return __riscv_vle32_v_f32m1(p, __riscv_vsetvlmax_e32m1());\n}\ntemplate <> inline vfloat32m2_t load(const float *p) {\n return __riscv_vle32_v_f32m2(p, __riscv_vsetvlmax_e32m2());\n}\ntemplate <> inline vfloat32m4_t load(const float *p) {\n return __riscv_vle32_v_f32m4(p, __riscv_vsetvlmax_e32m4());\n}\ntemplate <> inline vfloat32m8_t load(const float *p) {\n return __riscv_vle32_v_f32m8(p, __riscv_vsetvlmax_e32m8());\n}\n#endif\n\n#if defined(__riscv_zvfbfwma)\ntemplate <> inline vbfloat16mf2_t load(const ggml_bf16_t *p) {\n return __riscv_vle16_v_bf16mf2(reinterpret_cast(p), __riscv_vsetvlmax_e16mf2());\n}\ntemplate <> inline vbfloat16m1_t load(const ggml_bf16_t *p) {\n return __riscv_vle16_v_bf16m1(reinterpret_cast(p), __riscv_vsetvlmax_e16m1());\n}\ntemplate <> inline vbfloat16m2_t load(const ggml_bf16_t *p) {\n return __riscv_vle16_v_bf16m2(reinterpret_cast(p), __riscv_vsetvlmax_e16m2());\n}\n#endif\n\n#if defined(__riscv_zvfh)\ntemplate T set_zero();\n\ntemplate <> inline vfloat16mf2_t set_zero() {\n return __riscv_vfmv_v_f_f16mf2(0, __riscv_vsetvlmax_e16mf2());\n}\ntemplate <> inline vfloat16m1_t set_zero() {\n return __riscv_vfmv_v_f_f16m1(0, __riscv_vsetvlmax_e16m1());\n}\ntemplate <> inline vfloat16m2_t set_zero() {\n return __riscv_vfmv_v_f_f16m2(0, __riscv_vsetvlmax_e16m2());\n}\ntemplate <> inline vfloat16m4_t set_zero() {\n return __riscv_vfmv_v_f_f16m4(0, __riscv_vsetvlmax_e16m4());\n}\ntemplate <> inline vfloat32m1_t set_zero() {\n return __riscv_vfmv_v_f_f32m1(0.0f, __riscv_vsetvlmax_e32m1());\n}\ntemplate <> inline vfloat32m2_t set_zero() {\n return __riscv_vfmv_v_f_f32m2(0, __riscv_vsetvlmax_e32m2());\n}\ntemplate <> inline vfloat32m4_t set_zero() {\n return __riscv_vfmv_v_f_f32m4(0, __riscv_vsetvlmax_e32m4());\n}\ntemplate <> inline vfloat32m8_t set_zero() {\n return __riscv_vfmv_v_f_f32m8(0, __riscv_vsetvlmax_e32m8());\n}\n#endif\n\n#if defined(__riscv_v_intrinsic)\ntemplate size_t vlmax() {\n if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e16mf2(); }\n else if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e16m1(); }\n else if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e16m2(); }\n else if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e16m4(); }\n else if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e32m1(); }\n else if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e32m2(); }\n else if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e32m4(); }\n else if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e32m8(); }\n return 0;\n}\n#endif\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// FLOATING POINT MATRIX MULTIPLICATION\n\ntemplate \nstatic inline int64_t BLOCK_SIZE(size_t m) {\n const int64_t NB_BLOC_M = (m + M - 1) / M;\n return (m % NB_BLOC_M == 0) ? m / NB_BLOC_M : (m / NB_BLOC_M) + 1;\n}\n\nstatic constexpr inline int64_t BLOC_POS(int64_t ib, int64_t ibN, int64_t bloc_size) {\n return ib < ibN ? ib * bloc_size : ibN * bloc_size + (ib - ibN) * (bloc_size - 1);\n}\n\ntemplate \nclass tinyBLAS {\n public:\n tinyBLAS(const ggml_compute_params * params, int64_t k,\n const TA *A, int64_t lda,\n const TB *B, int64_t ldb,\n TC *C, int64_t ldc)\n : params(params), A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc) {\n }\n\n bool matmul(int64_t m, int64_t n) {\n if (k % KN != 0)\n return false;\n // compute RM for only need tile with size RM&RM-1\n#if VECTOR_REGISTERS == 32\n if (m % 16 == 0 && (m/16 >= params->nth)) {\n const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n mnpack<4, 6, 4>(m, n, SIZE_N, 12);\n return true;\n }\n if (m % 8 == 0 ) {\n const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n mnpack<4, 6, 2>(m, n, SIZE_N, 12);\n return true;\n }\n if (m % 4 == 0) {\n const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n mnpack<4, 6, 1>(m, n, SIZE_N, 12);\n return true;\n }\n#else // VECTOR_REGISTERS == 16\n if (m % 16 == 0 && (m/16 >= params->nth)) {\n const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n mnpack<4, 3, 4>(m, n, SIZE_N, 24);\n return true;\n }\n if (m % 8 == 0 ) {\n const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n mnpack<4, 3, 2>(m, n, SIZE_N, 24);\n return true;\n }\n if (m % 4 == 0) {\n const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n mnpack<4, 3, 1>(m, n, SIZE_N, 24);\n return true;\n }\n#endif\n return false;\n }\n\n private:\n template \n inline void mnpack(int64_t m, int64_t n, int64_t SIZE_N, int64_t BN) {\n if (SIZE_N == RN) {\n return gemm(m, n, BN);\n }\n if constexpr (RN > 1) {\n return mnpack(m, n, SIZE_N, BN);\n } else {\n GGML_LOG_ERROR(\"mnpack<%d, %d> block size not supported\\n\", RM, (int)SIZE_N);\n GGML_ASSERT(false); // we have miss something.\n }\n }\n\n template \n inline void gemm_bloc(int64_t ii, int64_t jj) {\n D Cv[RN][RM] = {};\n for (int64_t l = 0; l < k; l += KN) {\n // help compiler for op order.\n if constexpr (RM <= RN) {\n V Av[RM];\n for (int64_t i = 0; i < RM; ++i) {\n Av[i] = load(A + lda * (ii + i) + l);\n }\n for (int64_t j = 0; j < RN; ++j) {\n V Bv = load(B + ldb * (jj + j) + l);\n for (int64_t i = 0; i < RM; ++i) {\n Cv[j][i] = madd(Av[i], Bv, Cv[j][i]);\n }\n }\n } else {\n V Bv[RN];\n for (int64_t j = 0; j < RN; ++j) {\n Bv[j] = load(B + ldb * (jj + j) + l);\n }\n for (int64_t i = 0; i < RM; ++i) {\n V Av = load(A + lda * (ii + i) + l);\n for (int64_t j = 0; j < RN; ++j) {\n Cv[j][i] = madd(Av, Bv[j], Cv[j][i]);\n }\n }\n }\n }\n for (int64_t j = 0; j < RN; ++j)\n for (int64_t i = 0; i < RM; ++i)\n C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n }\n\n template \n NOINLINE void gemm(int64_t m, int64_t n, int64_t BN) {\n GGML_ASSERT(m % (RM * BM) == 0);\n const int64_t ytiles = m / (RM * BM);\n const int64_t xtiles = (n + RN -1) / RN;\n const int64_t jj_RN = (xtiles - (xtiles * RN - n));\n\n // \"round\" bloc_size to \"nearest\" BN\n const int64_t NB_BN = xtiles < BN ? 1 : (xtiles + BN / 2) / BN;\n const int64_t SIZE_BN = xtiles % NB_BN == 0 ? xtiles / NB_BN : xtiles / NB_BN + 1;\n const int64_t jj_BN = (NB_BN - (NB_BN * SIZE_BN - xtiles));\n const int64_t nb_job = ytiles * NB_BN;\n\n if (params->ith == 0) {\n GGML_ASSERT( jj_BN * SIZE_BN + (NB_BN - jj_BN) * (SIZE_BN - 1) == xtiles);\n // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.\n ggml_threadpool_chunk_set(params->threadpool, params->nth);\n }\n\n ggml_barrier(params->threadpool);\n\n int64_t job = params->ith;\n while (job < nb_job) {\n const int64_t ii = (job % ytiles) * RM * BM;\n const int64_t jb = job / ytiles;\n const int64_t jr0 = BLOC_POS(jb , jj_BN, SIZE_BN);\n const int64_t jrN = BLOC_POS(jb+1, jj_BN, SIZE_BN);\n\n const int64_t jj0 = BLOC_POS(jr0, jj_RN, RN);\n const int64_t jj2 = BLOC_POS(jrN, jj_RN, RN);\n const int64_t jj1 = jj2 < jj_RN * RN ? jj2 : jj_RN * RN;\n\n for (int64_t bi = 0; bi < BM * RM; bi += RM) {\n int64_t jj = jj0;\n for (; jj < jj1; jj += RN) {\n gemm_bloc(ii + bi, jj);\n }\n if constexpr (RN > 1) {\n for (; jj < jj2; jj += RN - 1) {\n gemm_bloc(ii + bi, jj);\n }\n }\n GGML_ASSERT(jj == jj2);\n }\n\n job = ggml_threadpool_chunk_add(params->threadpool, 1);\n }\n\n ggml_barrier(params->threadpool);\n return;\n }\n\n const ggml_compute_params * params;\n const TA *const A;\n const TB *const B;\n TC *const C;\n const int64_t k;\n const int64_t lda;\n const int64_t ldb;\n const int64_t ldc;\n};\n\n#if defined(__riscv_v_intrinsic)\ntemplate \nclass tinyBLAS_RVV {\n public:\n tinyBLAS_RVV(const ggml_compute_params * params, int64_t k,\n const TA *A, int64_t lda,\n const TB *B, int64_t ldb,\n TC *C, int64_t ldc)\n : params(params), A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc) {\n }\n\n bool matmul(int64_t m, int64_t n) {\n if (k % vlmax() != 0) {\n return false;\n }\n\n#if LMUL == 1\n if (m % 16 == 0 && (m/16 >= params->nth)) {\n const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n mnpack<4, 6, 4>(m, n, SIZE_N, 12);\n return true;\n }\n if (m % 8 == 0 ) {\n const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n mnpack<4, 6, 2>(m, n, SIZE_N, 12);\n return true;\n }\n if (m % 4 == 0) {\n const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n mnpack<4, 6, 1>(m, n, SIZE_N, 12);\n return true;\n }\n#elif LMUL == 2\n if (m % 16 == 0 && (m/16 >= params->nth)) {\n const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n mnpack<4, 3, 4>(m, n, SIZE_N, 24);\n return true;\n }\n if (m % 8 == 0 ) {\n const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n mnpack<4, 3, 2>(m, n, SIZE_N, 24);\n return true;\n }\n if (m % 4 == 0) {\n const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n mnpack<4, 3, 1>(m, n, SIZE_N, 24);\n return true;\n }\n#else // LMUL = 4\n if (m % 16 == 0 && (m/16 >= params->nth)) {\n const int64_t SIZE_N = BLOCK_SIZE<2>(n);\n mnpack<2, 2, 8>(m, n, SIZE_N, 36);\n return true;\n }\n if (m % 8 == 0 ) {\n const int64_t SIZE_N = BLOCK_SIZE<2>(n);\n mnpack<2, 2, 4>(m, n, SIZE_N, 36);\n return true;\n }\n if (m % 4 == 0) {\n const int64_t SIZE_N = BLOCK_SIZE<2>(n);\n mnpack<2, 2, 2>(m, n, SIZE_N, 36);\n return true;\n }\n#endif\n return false;\n }\n\n private:\n template\n inline void mnpack(int64_t m, int64_t n, int64_t SIZE_N, int64_t BN) {\n if (SIZE_N == RN) {\n return gemm(m, n, BN);\n }\n if constexpr (RN > 1) {\n return mnpack(m, n, SIZE_N, BN);\n } else {\n GGML_LOG_ERROR(\"mnpack<%d, %d> block size not supported\\n\", RM, (int)SIZE_N);\n GGML_ASSERT(false); // we have miss something.\n }\n }\n\n inline void gemm_bloc_4x6(int64_t ii, int64_t jj) {\n size_t vl = vlmax();\n D Cv00 = set_zero();\n D Cv01 = set_zero();\n D Cv02 = set_zero();\n D Cv03 = set_zero();\n D Cv10 = set_zero();\n D Cv11 = set_zero();\n D Cv12 = set_zero();\n D Cv13 = set_zero();\n D Cv20 = set_zero();\n D Cv21 = set_zero();\n D Cv22 = set_zero();\n D Cv23 = set_zero();\n D Cv30 = set_zero();\n D Cv31 = set_zero();\n D Cv32 = set_zero();\n D Cv33 = set_zero();\n D Cv40 = set_zero();\n D Cv41 = set_zero();\n D Cv42 = set_zero();\n D Cv43 = set_zero();\n D Cv50 = set_zero();\n D Cv51 = set_zero();\n D Cv52 = set_zero();\n D Cv53 = set_zero();\n\n for (int64_t l = 0; l < k; l += vl) {\n V Bv0 = load(B + ldb * (jj + 0) + l);\n V Bv1 = load(B + ldb * (jj + 1) + l);\n V Bv2 = load(B + ldb * (jj + 2) + l);\n V Bv3 = load(B + ldb * (jj + 3) + l);\n V Bv4 = load(B + ldb * (jj + 4) + l);\n V Bv5 = load(B + ldb * (jj + 5) + l);\n\n V Av0 = load(A + lda * (ii + 0) + l);\n Cv00 = madd(Av0, Bv0, Cv00);\n Cv10 = madd(Av0, Bv1, Cv10);\n Cv20 = madd(Av0, Bv2, Cv20);\n Cv30 = madd(Av0, Bv3, Cv30);\n Cv40 = madd(Av0, Bv4, Cv40);\n Cv50 = madd(Av0, Bv5, Cv50);\n\n V Av1 = load(A + lda * (ii + 1) + l);\n Cv01 = madd(Av1, Bv0, Cv01);\n Cv11 = madd(Av1, Bv1, Cv11);\n Cv21 = madd(Av1, Bv2, Cv21);\n Cv31 = madd(Av1, Bv3, Cv31);\n Cv41 = madd(Av1, Bv4, Cv41);\n Cv51 = madd(Av1, Bv5, Cv51);\n\n V Av2 = load(A + lda * (ii + 2) + l);\n Cv02 = madd(Av2, Bv0, Cv02);\n Cv12 = madd(Av2, Bv1, Cv12);\n Cv22 = madd(Av2, Bv2, Cv22);\n Cv32 = madd(Av2, Bv3, Cv32);\n Cv42 = madd(Av2, Bv4, Cv42);\n Cv52 = madd(Av2, Bv5, Cv52);\n\n V Av3 = load(A + lda * (ii + 3) + l);\n Cv03 = madd(Av3, Bv0, Cv03);\n Cv13 = madd(Av3, Bv1, Cv13);\n Cv23 = madd(Av3, Bv2, Cv23);\n Cv33 = madd(Av3, Bv3, Cv33);\n Cv43 = madd(Av3, Bv4, Cv43);\n Cv53 = madd(Av3, Bv5, Cv53);\n }\n\n C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n C[ldc * (jj + 2) + (ii + 0)] = hsum(Cv20);\n C[ldc * (jj + 2) + (ii + 1)] = hsum(Cv21);\n C[ldc * (jj + 2) + (ii + 2)] = hsum(Cv22);\n C[ldc * (jj + 2) + (ii + 3)] = hsum(Cv23);\n C[ldc * (jj + 3) + (ii + 0)] = hsum(Cv30);\n C[ldc * (jj + 3) + (ii + 1)] = hsum(Cv31);\n C[ldc * (jj + 3) + (ii + 2)] = hsum(Cv32);\n C[ldc * (jj + 3) + (ii + 3)] = hsum(Cv33);\n C[ldc * (jj + 4) + (ii + 0)] = hsum(Cv40);\n C[ldc * (jj + 4) + (ii + 1)] = hsum(Cv41);\n C[ldc * (jj + 4) + (ii + 2)] = hsum(Cv42);\n C[ldc * (jj + 4) + (ii + 3)] = hsum(Cv43);\n C[ldc * (jj + 5) + (ii + 0)] = hsum(Cv50);\n C[ldc * (jj + 5) + (ii + 1)] = hsum(Cv51);\n C[ldc * (jj + 5) + (ii + 2)] = hsum(Cv52);\n C[ldc * (jj + 5) + (ii + 3)] = hsum(Cv53);\n }\n\n inline void gemm_bloc_4x5(int64_t ii, int64_t jj) {\n size_t vl = vlmax();\n D Cv00 = set_zero();\n D Cv01 = set_zero();\n D Cv02 = set_zero();\n D Cv03 = set_zero();\n D Cv10 = set_zero();\n D Cv11 = set_zero();\n D Cv12 = set_zero();\n D Cv13 = set_zero();\n D Cv20 = set_zero();\n D Cv21 = set_zero();\n D Cv22 = set_zero();\n D Cv23 = set_zero();\n D Cv30 = set_zero();\n D Cv31 = set_zero();\n D Cv32 = set_zero();\n D Cv33 = set_zero();\n D Cv40 = set_zero();\n D Cv41 = set_zero();\n D Cv42 = set_zero();\n D Cv43 = set_zero();\n\n for (int64_t l = 0; l < k; l += vl) {\n V Bv0 = load(B + ldb * (jj + 0) + l);\n V Bv1 = load(B + ldb * (jj + 1) + l);\n V Bv2 = load(B + ldb * (jj + 2) + l);\n V Bv3 = load(B + ldb * (jj + 3) + l);\n V Bv4 = load(B + ldb * (jj + 4) + l);\n\n V Av0 = load(A + lda * (ii + 0) + l);\n Cv00 = madd(Av0, Bv0, Cv00);\n Cv10 = madd(Av0, Bv1, Cv10);\n Cv20 = madd(Av0, Bv2, Cv20);\n Cv30 = madd(Av0, Bv3, Cv30);\n Cv40 = madd(Av0, Bv4, Cv40);\n\n V Av1 = load(A + lda * (ii + 1) + l);\n Cv01 = madd(Av1, Bv0, Cv01);\n Cv11 = madd(Av1, Bv1, Cv11);\n Cv21 = madd(Av1, Bv2, Cv21);\n Cv31 = madd(Av1, Bv3, Cv31);\n Cv41 = madd(Av1, Bv4, Cv41);\n\n V Av2 = load(A + lda * (ii + 2) + l);\n Cv02 = madd(Av2, Bv0, Cv02);\n Cv12 = madd(Av2, Bv1, Cv12);\n Cv22 = madd(Av2, Bv2, Cv22);\n Cv32 = madd(Av2, Bv3, Cv32);\n Cv42 = madd(Av2, Bv4, Cv42);\n\n V Av3 = load(A + lda * (ii + 3) + l);\n Cv03 = madd(Av3, Bv0, Cv03);\n Cv13 = madd(Av3, Bv1, Cv13);\n Cv23 = madd(Av3, Bv2, Cv23);\n Cv33 = madd(Av3, Bv3, Cv33);\n Cv43 = madd(Av3, Bv4, Cv43);\n }\n\n C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n C[ldc * (jj + 2) + (ii + 0)] = hsum(Cv20);\n C[ldc * (jj + 2) + (ii + 1)] = hsum(Cv21);\n C[ldc * (jj + 2) + (ii + 2)] = hsum(Cv22);\n C[ldc * (jj + 2) + (ii + 3)] = hsum(Cv23);\n C[ldc * (jj + 3) + (ii + 0)] = hsum(Cv30);\n C[ldc * (jj + 3) + (ii + 1)] = hsum(Cv31);\n C[ldc * (jj + 3) + (ii + 2)] = hsum(Cv32);\n C[ldc * (jj + 3) + (ii + 3)] = hsum(Cv33);\n C[ldc * (jj + 4) + (ii + 0)] = hsum(Cv40);\n C[ldc * (jj + 4) + (ii + 1)] = hsum(Cv41);\n C[ldc * (jj + 4) + (ii + 2)] = hsum(Cv42);\n C[ldc * (jj + 4) + (ii + 3)] = hsum(Cv43);\n }\n\n inline void gemm_bloc_4x4(int64_t ii, int64_t jj) {\n size_t vl = vlmax();\n D Cv00 = set_zero();\n D Cv01 = set_zero();\n D Cv02 = set_zero();\n D Cv03 = set_zero();\n D Cv10 = set_zero();\n D Cv11 = set_zero();\n D Cv12 = set_zero();\n D Cv13 = set_zero();\n D Cv20 = set_zero();\n D Cv21 = set_zero();\n D Cv22 = set_zero();\n D Cv23 = set_zero();\n D Cv30 = set_zero();\n D Cv31 = set_zero();\n D Cv32 = set_zero();\n D Cv33 = set_zero();\n\n for (int64_t l = 0; l < k; l += vl) {\n V Av0 = load(A + lda * (ii + 0) + l);\n V Av1 = load(A + lda * (ii + 1) + l);\n V Av2 = load(A + lda * (ii + 2) + l);\n V Av3 = load(A + lda * (ii + 3) + l);\n\n V Bv0 = load(B + ldb * (jj + 0) + l);\n Cv00 = madd(Av0, Bv0, Cv00);\n Cv01 = madd(Av1, Bv0, Cv01);\n Cv02 = madd(Av2, Bv0, Cv02);\n Cv03 = madd(Av3, Bv0, Cv03);\n\n V Bv1 = load(B + ldb * (jj + 1) + l);\n Cv10 = madd(Av0, Bv1, Cv10);\n Cv11 = madd(Av1, Bv1, Cv11);\n Cv12 = madd(Av2, Bv1, Cv12);\n Cv13 = madd(Av3, Bv1, Cv13);\n\n V Bv2 = load(B + ldb * (jj + 2) + l);\n Cv20 = madd(Av0, Bv2, Cv20);\n Cv21 = madd(Av1, Bv2, Cv21);\n Cv22 = madd(Av2, Bv2, Cv22);\n Cv23 = madd(Av3, Bv2, Cv23);\n\n V Bv3 = load(B + ldb * (jj + 3) + l);\n Cv30 = madd(Av0, Bv3, Cv30);\n Cv31 = madd(Av1, Bv3, Cv31);\n Cv32 = madd(Av2, Bv3, Cv32);\n Cv33 = madd(Av3, Bv3, Cv33);\n }\n\n C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n C[ldc * (jj + 2) + (ii + 0)] = hsum(Cv20);\n C[ldc * (jj + 2) + (ii + 1)] = hsum(Cv21);\n C[ldc * (jj + 2) + (ii + 2)] = hsum(Cv22);\n C[ldc * (jj + 2) + (ii + 3)] = hsum(Cv23);\n C[ldc * (jj + 3) + (ii + 0)] = hsum(Cv30);\n C[ldc * (jj + 3) + (ii + 1)] = hsum(Cv31);\n C[ldc * (jj + 3) + (ii + 2)] = hsum(Cv32);\n C[ldc * (jj + 3) + (ii + 3)] = hsum(Cv33);\n }\n\n inline void gemm_bloc_4x3(int64_t ii, int64_t jj) {\n size_t vl = vlmax();\n D Cv00 = set_zero();\n D Cv01 = set_zero();\n D Cv02 = set_zero();\n D Cv03 = set_zero();\n D Cv10 = set_zero();\n D Cv11 = set_zero();\n D Cv12 = set_zero();\n D Cv13 = set_zero();\n D Cv20 = set_zero();\n D Cv21 = set_zero();\n D Cv22 = set_zero();\n D Cv23 = set_zero();\n\n for (int64_t l = 0; l < k; l += vl) {\n V Av0 = load(A + lda * (ii + 0) + l);\n V Av1 = load(A + lda * (ii + 1) + l);\n V Av2 = load(A + lda * (ii + 2) + l);\n V Av3 = load(A + lda * (ii + 3) + l);\n\n V Bv0 = load(B + ldb * (jj + 0) + l);\n Cv00 = madd(Av0, Bv0, Cv00);\n Cv01 = madd(Av1, Bv0, Cv01);\n Cv02 = madd(Av2, Bv0, Cv02);\n Cv03 = madd(Av3, Bv0, Cv03);\n\n V Bv1 = load(B + ldb * (jj + 1) + l);\n Cv10 = madd(Av0, Bv1, Cv10);\n Cv11 = madd(Av1, Bv1, Cv11);\n Cv12 = madd(Av2, Bv1, Cv12);\n Cv13 = madd(Av3, Bv1, Cv13);\n\n V Bv2 = load(B + ldb * (jj + 2) + l);\n Cv20 = madd(Av0, Bv2, Cv20);\n Cv21 = madd(Av1, Bv2, Cv21);\n Cv22 = madd(Av2, Bv2, Cv22);\n Cv23 = madd(Av3, Bv2, Cv23);\n }\n\n C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n C[ldc * (jj + 2) + (ii + 0)] = hsum(Cv20);\n C[ldc * (jj + 2) + (ii + 1)] = hsum(Cv21);\n C[ldc * (jj + 2) + (ii + 2)] = hsum(Cv22);\n C[ldc * (jj + 2) + (ii + 3)] = hsum(Cv23);\n }\n\n inline void gemm_bloc_4x2(int64_t ii, int64_t jj) {\n size_t vl = vlmax();\n D Cv00 = set_zero();\n D Cv01 = set_zero();\n D Cv02 = set_zero();\n D Cv03 = set_zero();\n D Cv10 = set_zero();\n D Cv11 = set_zero();\n D Cv12 = set_zero();\n D Cv13 = set_zero();\n\n for (int64_t l = 0; l < k; l += vl) {\n V Av0 = load(A + lda * (ii + 0) + l);\n V Av1 = load(A + lda * (ii + 1) + l);\n V Av2 = load(A + lda * (ii + 2) + l);\n V Av3 = load(A + lda * (ii + 3) + l);\n\n V Bv0 = load(B + ldb * (jj + 0) + l);\n Cv00 = madd(Av0, Bv0, Cv00);\n Cv01 = madd(Av1, Bv0, Cv01);\n Cv02 = madd(Av2, Bv0, Cv02);\n Cv03 = madd(Av3, Bv0, Cv03);\n\n V Bv1 = load(B + ldb * (jj + 1) + l);\n Cv10 = madd(Av0, Bv1, Cv10);\n Cv11 = madd(Av1, Bv1, Cv11);\n Cv12 = madd(Av2, Bv1, Cv12);\n Cv13 = madd(Av3, Bv1, Cv13);\n }\n\n C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n }\n\n inline void gemm_bloc_4x1(int64_t ii, int64_t jj) {\n size_t vl = vlmax();\n D Cv00 = set_zero();\n D Cv01 = set_zero();\n D Cv02 = set_zero();\n D Cv03 = set_zero();\n\n for (int64_t l = 0; l < k; l += vl) {\n V Av0 = load(A + lda * (ii + 0) + l);\n V Av1 = load(A + lda * (ii + 1) + l);\n V Av2 = load(A + lda * (ii + 2) + l);\n V Av3 = load(A + lda * (ii + 3) + l);\n\n V Bv0 = load(B + ldb * (jj + 0) + l);\n Cv00 = madd(Av0, Bv0, Cv00);\n Cv01 = madd(Av1, Bv0, Cv01);\n Cv02 = madd(Av2, Bv0, Cv02);\n Cv03 = madd(Av3, Bv0, Cv03);\n }\n\n C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n }\n\n inline void gemm_bloc_2x2(int64_t ii, int64_t jj) {\n size_t vl = vlmax();\n D Cv00 = set_zero();\n D Cv01 = set_zero();\n D Cv10 = set_zero();\n D Cv11 = set_zero();\n\n for (int64_t l = 0; l < k; l += vl) {\n V Av0 = load(A + lda * (ii + 0) + l);\n V Av1 = load(A + lda * (ii + 1) + l);\n\n V Bv0 = load(B + ldb * (jj + 0) + l);\n Cv00 = madd(Av0, Bv0, Cv00);\n Cv01 = madd(Av1, Bv0, Cv01);\n\n V Bv1 = load(B + ldb * (jj + 1) + l);\n Cv10 = madd(Av0, Bv1, Cv10);\n Cv11 = madd(Av1, Bv1, Cv11);\n }\n\n C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n }\n\n inline void gemm_bloc_2x1(int64_t ii, int64_t jj) {\n size_t vl = vlmax();\n D Cv00 = set_zero();\n D Cv01 = set_zero();\n\n for (int64_t l = 0; l < k; l += vl) {\n V Av0 = load(A + lda * (ii + 0) + l);\n V Av1 = load(A + lda * (ii + 1) + l);\n\n V Bv0 = load(B + ldb * (jj + 0) + l);\n Cv00 = madd(Av0, Bv0, Cv00);\n Cv01 = madd(Av1, Bv0, Cv01);\n }\n\n C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n }\n\n template \n inline void gemm_bloc(int64_t ii, int64_t jj) {\n if constexpr (RM == 4) {\n if constexpr (RN == 6) { return gemm_bloc_4x6(ii, jj); }\n if constexpr (RN == 5) { return gemm_bloc_4x5(ii, jj); }\n if constexpr (RN == 4) { return gemm_bloc_4x4(ii, jj); }\n if constexpr (RN == 3) { return gemm_bloc_4x3(ii, jj); }\n if constexpr (RN == 2) { return gemm_bloc_4x2(ii, jj); }\n if constexpr (RN == 1) { return gemm_bloc_4x1(ii, jj); }\n } else if constexpr (RM == 2) {\n if constexpr (RN == 2) { return gemm_bloc_2x2(ii, jj); }\n if constexpr (RN == 1) { return gemm_bloc_2x1(ii, jj); }\n }\n }\n\n template \n NOINLINE void gemm(int64_t m, int64_t n, int64_t BN) {\n GGML_ASSERT(m % (RM * BM) == 0);\n const int64_t ytiles = m / (RM * BM);\n const int64_t xtiles = (n + RN -1) / RN;\n const int64_t jj_RN = (xtiles - (xtiles * RN - n));\n\n // \"round\" bloc_size to \"nearest\" BN\n const int64_t NB_BN = xtiles < BN ? 1 : (xtiles + BN / 2) / BN;\n const int64_t SIZE_BN = xtiles % NB_BN == 0 ? xtiles / NB_BN : xtiles / NB_BN + 1;\n const int64_t jj_BN = (NB_BN - (NB_BN * SIZE_BN - xtiles));\n const int64_t nb_job = ytiles * NB_BN;\n\n if (params->ith == 0) {\n GGML_ASSERT( jj_BN * SIZE_BN + (NB_BN - jj_BN) * (SIZE_BN - 1) == xtiles);\n // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.\n ggml_threadpool_chunk_set(params->threadpool, params->nth);\n }\n\n ggml_barrier(params->threadpool);\n\n int64_t job = params->ith;\n while (job < nb_job) {\n const int64_t ii = (job % ytiles) * RM * BM;\n const int64_t jb = job / ytiles;\n const int64_t jr0 = BLOC_POS(jb , jj_BN, SIZE_BN);\n const int64_t jrN = BLOC_POS(jb+1, jj_BN, SIZE_BN);\n\n const int64_t jj0 = BLOC_POS(jr0, jj_RN, RN);\n const int64_t jj2 = BLOC_POS(jrN, jj_RN, RN);\n const int64_t jj1 = jj2 < jj_RN * RN ? jj2 : jj_RN * RN;\n\n for (int64_t bi = 0; bi < BM * RM; bi += RM) {\n int64_t jj = jj0;\n for (; jj < jj1; jj += RN) {\n gemm_bloc(ii + bi, jj);\n }\n if constexpr (RN > 1) {\n for (; jj < jj2; jj += RN - 1) {\n gemm_bloc(ii + bi, jj);\n }\n }\n GGML_ASSERT(jj == jj2);\n }\n\n job = ggml_threadpool_chunk_add(params->threadpool, 1);\n }\n\n ggml_barrier(params->threadpool);\n return;\n }\n\n const ggml_compute_params * params;\n const TA *const A;\n const TB *const B;\n TC *const C;\n const int64_t k;\n const int64_t lda;\n const int64_t ldb;\n const int64_t ldc;\n};\n#endif\n\n//////////////////////////////////////////////////////////////////////////////////////////\n// QUANT ZERO MATRIX MULTIPLICATION\n\n#if defined(__ARM_FEATURE_DOTPROD)\ntemplate \nclass tinyBLAS_Q0_ARM {\n public:\n tinyBLAS_Q0_ARM(int64_t k,\n const TA *A, int64_t lda,\n const block_q8_0 *B, int64_t ldb,\n float *C, int64_t ldc,\n int ith, int nth)\n : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n }\n\n void matmul(int64_t m, int64_t n) {\n mnpack(0, m, 0, n);\n }\n\n private:\n NOINLINE void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t mc, nc, mp, np;\n switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3ll)) {\n case 0x33:\n mc = 3;\n nc = 3;\n gemm<3, 3>(m0, m, n0, n);\n break;\n case 0x32:\n mc = 3;\n nc = 2;\n gemm<3, 2>(m0, m, n0, n);\n break;\n case 0x23:\n mc = 2;\n nc = 3;\n gemm<2, 3>(m0, m, n0, n);\n break;\n case 0x22:\n mc = 2;\n nc = 2;\n gemm<2, 2>(m0, m, n0, n);\n break;\n case 0x31:\n mc = 3;\n nc = 1;\n gemm<3, 1>(m0, m, n0, n);\n break;\n case 0x13:\n mc = 1;\n nc = 3;\n gemm<1, 3>(m0, m, n0, n);\n break;\n case 0x21:\n mc = 2;\n nc = 1;\n gemm<2, 1>(m0, m, n0, n);\n break;\n case 0x12:\n mc = 1;\n nc = 2;\n gemm<1, 2>(m0, m, n0, n);\n break;\n case 0x11:\n mc = 1;\n nc = 1;\n gemm<1, 1>(m0, m, n0, n);\n break;\n default:\n return;\n }\n mp = m0 + (m - m0) / mc * mc;\n np = n0 + (n - n0) / nc * nc;\n mnpack(mp, m, n0, np);\n mnpack(m0, m, np, n);\n }\n\n template \n NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * RN;\n float32x4_t Cv[RN][RM] = {};\n for (int64_t l = 0; l < k; ++l)\n for (int64_t j = 0; j < RN; ++j)\n for (int64_t i = 0; i < RM; ++i)\n Cv[j][i] = vmlaq_n_f32(Cv[j][i],\n vcvtq_f32_s32(vdotq_s32(\n vdotq_s32(vdupq_n_s32(0),\n load_lo(A + lda * (ii + i) + l),\n load_lo(B + ldb * (jj + j) + l)),\n load_hi(A + lda * (ii + i) + l),\n load_hi(B + ldb * (jj + j) + l))),\n unhalf(A[lda * (ii + i) + l].d) *\n unhalf(B[ldb * (jj + j) + l].d));\n for (int64_t j = 0; j < RN; ++j)\n for (int64_t i = 0; i < RM; ++i)\n C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n }\n }\n\n inline int8x16_t load_lo(const block_q8_0 *b) {\n return vld1q_s8(b->qs);\n }\n\n inline int8x16_t load_hi(const block_q8_0 *b) {\n return vld1q_s8(b->qs + 16);\n }\n\n inline int8x16_t load_lo(const block_q4_0 *b) {\n return vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vld1q_u8(b->qs),\n vdupq_n_u8(0x0f))),\n vdupq_n_s8(0x8));\n }\n\n inline int8x16_t load_hi(const block_q4_0 *b) {\n return vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(vld1q_u8(b->qs), 4)),\n vdupq_n_s8(0x8));\n }\n\n const TA *const A;\n const block_q8_0 *const B;\n float *const C;\n const int64_t k;\n const int64_t lda;\n const int64_t ldb;\n const int64_t ldc;\n const int ith;\n const int nth;\n};\n#endif // __ARM_FEATURE_DOTPROD\n\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\ntemplate \nclass tinyBLAS_Q0_AVX {\n public:\n tinyBLAS_Q0_AVX(int64_t k,\n const TA *A, int64_t lda,\n const TB *B, int64_t ldb,\n TC *C, int64_t ldc,\n int ith, int nth)\n : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n const int8_t kvalues_iq4nl[16] = {\n -127, -104, -83, -65,\n -49, -35, -22, -10,\n 1, 13, 25, 38,\n 53, 69, 89, 113\n };\n\n iq4nlt = _mm_loadu_si128((const __m128i *)kvalues_iq4nl);\n }\n\n void matmul(int64_t m, int64_t n) {\n mnpack(0, m, 0, n);\n }\n\n private:\n void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t mc, nc, mp, np;\n switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 4)) {\n#if VECTOR_REGISTERS == 32\n case 0x44:\n mc = 4;\n nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n gemm4xN<4>(m0, m, n0, n);\n#else\n gemm<4, 4>(m0, m, n0, n);\n#endif\n break;\n case 0x43:\n mc = 4;\n nc = 3;\n#if defined(__AVX2__) && defined(__F16C__)\n gemm4xN<3>(m0, m, n0, n);\n#else\n gemm<4, 3>(m0, m, n0, n);\n#endif\n break;\n case 0x34:\n mc = 3;\n nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n gemmMx4<3>(m0, m, n0, n);\n#else\n gemm<3, 4>(m0, m, n0, n);\n#endif\n break;\n case 0x33:\n mc = 3;\n nc = 3;\n gemm<3, 3>(m0, m, n0, n);\n break;\n case 0x42:\n mc = 4;\n nc = 2;\n#if defined(__AVX2__) && defined(__F16C__)\n gemm4xN<2>(m0, m, n0, n);\n#else\n gemm<4, 2>(m0, m, n0, n);\n#endif\n break;\n case 0x24:\n mc = 2;\n nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n gemmMx4<2>(m0, m, n0, n);\n#else\n gemm<2, 4>(m0, m, n0, n);\n#endif\n break;\n#else\n case 0x44:\n case 0x43:\n case 0x42:\n mc = 4;\n nc = 2;\n#if defined(__AVX2__) && defined(__F16C__)\n gemm4xN<2>(m0, m, n0, n);\n#else\n gemm<4, 2>(m0, m, n0, n);\n#endif\n break;\n case 0x34:\n case 0x24:\n mc = 2;\n nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n gemmMx4<2>(m0, m, n0, n);\n#else\n gemm<2, 4>(m0, m, n0, n);\n#endif\n break;\n case 0x33:\n#endif\n case 0x32:\n mc = 3;\n nc = 2;\n gemm<3, 2>(m0, m, n0, n);\n break;\n case 0x23:\n mc = 2;\n nc = 3;\n gemm<2, 3>(m0, m, n0, n);\n break;\n case 0x41:\n mc = 4;\n nc = 1;\n#if defined(__AVX2__) && defined(__F16C__)\n gemm4xN<1>(m0, m, n0, n);\n#else\n gemm<4, 1>(m0, m, n0, n);\n#endif\n break;\n case 0x22:\n mc = 2;\n nc = 2;\n gemm<2, 2>(m0, m, n0, n);\n break;\n case 0x14:\n mc = 1;\n nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n gemmMx4<1>(m0, m, n0, n);\n#else\n gemm<1, 4>(m0, m, n0, n);\n#endif\n break;\n case 0x31:\n mc = 3;\n nc = 1;\n gemm<3, 1>(m0, m, n0, n);\n break;\n case 0x13:\n mc = 1;\n nc = 3;\n gemm<1, 3>(m0, m, n0, n);\n break;\n case 0x21:\n mc = 2;\n nc = 1;\n gemm<2, 1>(m0, m, n0, n);\n break;\n case 0x12:\n mc = 1;\n nc = 2;\n gemm<1, 2>(m0, m, n0, n);\n break;\n case 0x11:\n mc = 1;\n nc = 1;\n gemm<1, 1>(m0, m, n0, n);\n break;\n default:\n return;\n }\n mp = m0 + (m - m0) / mc * mc;\n np = n0 + (n - n0) / nc * nc;\n mnpack(mp, m, n0, np);\n mnpack(m0, m, np, n);\n }\n\n#if defined(__AVX2__) && defined(__F16C__)\n// Templated functions for gemm of dimensions 4xN\n template \n NOINLINE void gemm4xN(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t ytiles = (m - m0) / 4;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * 4;\n int64_t jj = n0 + job % xtiles * RN;\n __m256 Cv[RN][4] = {};\n for (int64_t l = 0; l < k; ++l) {\n uint64_t a_delta = ((uint64_t)A[lda * (ii + 3) + l].d << 48) | ((uint64_t)A[lda * (ii + 2) + l].d << 32) | ((uint64_t)A[lda * (ii + 1) + l].d << 16) | (A[lda * (ii + 0) + l].d);\n // Convert delta values for four blocks to float values\n __m128 da = _mm_cvtph_ps(_mm_set_epi64x(0, a_delta));\n __m256i avec0 = load(A + lda * (ii + 0) + l);\n __m256i avec1 = load(A + lda * (ii + 1) + l);\n __m256i avec2 = load(A + lda * (ii + 2) + l);\n __m256i avec3 = load(A + lda * (ii + 3) + l);\n for (int64_t j = 0; j < RN; ++j) {\n __m128 db = _mm_set1_ps(unhalf(B[ldb * (jj + j) + l].d));\n // Computation of product of delta values for four blocks and replicate it across 256 bit lane\n __m256 dvec = _mm256_castps128_ps256(_mm_mul_ps(da, db));\n dvec = _mm256_permute2f128_ps(dvec ,dvec, 0);\n // Computation of dot product and multiplication with appropriate delta value products\n Cv[j][0] = madd(_mm256_shuffle_ps(dvec, dvec, 0),\n updot(_mm256_sign_epi8(avec0, avec0),\n _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec0)),\n Cv[j][0]);\n Cv[j][1] = madd(_mm256_shuffle_ps(dvec, dvec, 85),\n updot(_mm256_sign_epi8(avec1, avec1),\n _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec1)),\n Cv[j][1]);\n Cv[j][2] = madd(_mm256_shuffle_ps(dvec, dvec, 170),\n updot(_mm256_sign_epi8(avec2, avec2),\n _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec2)),\n Cv[j][2]);\n Cv[j][3] = madd(_mm256_shuffle_ps(dvec, dvec, 255),\n updot(_mm256_sign_epi8(avec3, avec3),\n _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec3)),\n Cv[j][3]);\n }\n }\n\n for (int64_t j = 0; j < RN; ++j)\n for (int64_t i = 0; i < 4; ++i)\n C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n }\n }\n\n // Templated functions for gemm of dimensions Mx4\n template \n NOINLINE void gemmMx4(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / 4;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * 4;\n __m256 Cv[4][RM] = {};\n for (int64_t l = 0; l < k; ++l) {\n uint64_t b_delta = ((uint64_t)B[ldb * (jj + 3) + l].d << 48) | ((uint64_t)B[ldb * (jj + 2) + l].d << 32) | ((uint64_t)B[ldb * (jj + 1) + l].d << 16) | (B[ldb * (jj + 0) + l].d);\n // Convert delta values for four blocks to float values\n __m128 db = _mm_cvtph_ps(_mm_set_epi64x(0, b_delta));\n __m256i bvec0 = load(B + ldb * (jj + 0) + l);\n __m256i bvec1 = load(B + ldb * (jj + 1) + l);\n __m256i bvec2 = load(B + ldb * (jj + 2) + l);\n __m256i bvec3 = load(B + ldb * (jj + 3) + l);\n for (int64_t i = 0; i < RM; ++i) {\n __m128 da = _mm_set1_ps(unhalf((A[lda * (ii + i) + l].d)));\n // Computation of product of delta values for four blocks and replicate it across 256 bit lane\n __m256 dvec = _mm256_castps128_ps256(_mm_mul_ps(da, db));\n dvec = _mm256_permute2f128_ps(dvec ,dvec, 0);\n // Computation of dot product and multiplication with appropriate delta value products\n Cv[0][i] = madd(_mm256_shuffle_ps(dvec, dvec, 0),\n updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n load(A + lda * (ii + i) + l)),\n _mm256_sign_epi8(bvec0, load(A + lda * (ii + i) + l))),\n Cv[0][i]);\n Cv[1][i] = madd(_mm256_shuffle_ps(dvec, dvec, 85),\n updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n load(A + lda * (ii + i) + l)),\n _mm256_sign_epi8(bvec1, load(A + lda * (ii + i) + l))),\n Cv[1][i]);\n Cv[2][i] = madd(_mm256_shuffle_ps(dvec, dvec, 170),\n updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n load(A + lda * (ii + i) + l)),\n _mm256_sign_epi8(bvec2, load(A + lda * (ii + i) + l))),\n Cv[2][i]);\n Cv[3][i] = madd(_mm256_shuffle_ps(dvec, dvec, 255),\n updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n load(A + lda * (ii + i) + l)),\n _mm256_sign_epi8(bvec3, load(A + lda * (ii + i) + l))),\n Cv[3][i]);\n }\n }\n for (int64_t j = 0; j < 4; ++j)\n for (int64_t i = 0; i < RM; ++i)\n C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n }\n }\n#endif\n\n template \n NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * RN;\n __m256 Cv[RN][RM] = {};\n for (int64_t l = 0; l < k; ++l)\n for (int64_t j = 0; j < RN; ++j)\n for (int64_t i = 0; i < RM; ++i) {\n#if defined(__AVX2__)\n __m256 udTmp = updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n load(A + lda * (ii + i) + l)),\n _mm256_sign_epi8(load(B + ldb * (jj + j) + l),\n load(A + lda * (ii + i) + l)));\n#else\n __m128i ali0 = load0(A + lda * (ii + i) + l);\n __m128i ali1 = load1(A + lda * (ii + i) + l);\n __m128i blj0 = load0(B + ldb * (jj + j) + l);\n __m128i blj1 = load1(B + ldb * (jj + j) + l);\n\n __m128i sepAA0 = _mm_sign_epi8(ali0, ali0);\n __m128i sepAA1 = _mm_sign_epi8(ali1, ali1);\n __m128i sepBA0 = _mm_sign_epi8(blj0, ali0);\n __m128i sepBA1 = _mm_sign_epi8(blj1, ali1);\n\n // updot\n const __m128i oneFill = _mm_set1_epi16(1);\n __m128i mad0 = _mm_maddubs_epi16(sepAA0, sepBA0);\n __m128i mad1 = _mm_maddubs_epi16(sepAA1, sepBA1);\n __m256 udTmp = _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_madd_epi16(oneFill, mad1), _mm_madd_epi16(oneFill, mad0)));\n#endif\n Cv[j][i] = madd(_mm256_set1_ps(unhalf(A[lda * (ii + i) + l].d) *\n unhalf(B[ldb * (jj + j) + l].d)),\n udTmp,\n Cv[j][i]);\n }\n for (int64_t j = 0; j < RN; ++j)\n for (int64_t i = 0; i < RM; ++i)\n C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n }\n }\n\n inline __m256i load(const block_q8_0 *b) {\n return _mm256_loadu_si256((const __m256i *)b->qs);\n }\n\n inline __m128i load0(const block_q8_0 *b) {\n return _mm_loadu_si128((const __m128i *)b->qs);\n }\n\n inline __m128i load1(const block_q8_0 *b) {\n return _mm_loadu_si128(((const __m128i *)b->qs) + 1);\n }\n\n inline __m256i load(const block_q4_0 *b) {\n return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));\n }\n\n inline __m128i load0(const block_q4_0 *b) {\n const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n return _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), x), _mm_set1_epi8(8));\n }\n\n inline __m128i load1(const block_q4_0 *b) {\n const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n return _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4)), _mm_set1_epi8(8));\n }\n\n inline __m256i load(const block_q5_0 *b) {\n return _mm256_or_si256(denibble(b->qs), bittobyte(b->qh));\n }\n\n inline __m128i load0(const block_q5_0* b) {\n const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n uint32_t x32;\n memcpy(&x32, b->qh, sizeof(uint32_t));\n __m128i qxl = _mm_and_si128(_mm_set1_epi8(15), x);\n __m128i bytesl = _mm_cmpeq_epi8(_mm_set1_epi64x(-1),\n _mm_or_si128(_mm_set1_epi64x(0x7fbfdfeff7fbfdfe),\n _mm_shuffle_epi8(_mm_set1_epi32(x32),\n _mm_set_epi64x(0x0101010101010101, 0x0000000000000000))));\n bytesl = _mm_andnot_si128(bytesl, _mm_set1_epi8((char)0xF0));\n return _mm_or_si128(qxl, bytesl);\n }\n\n inline __m128i load1(const block_q5_0* b) {\n const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n uint32_t x32;\n memcpy(&x32, b->qh, sizeof(uint32_t));\n __m128i qxh = _mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4));\n __m128i bytesh = _mm_cmpeq_epi8(_mm_set1_epi64x(-1),\n _mm_or_si128(_mm_set1_epi64x(0x7fbfdfeff7fbfdfe),\n _mm_shuffle_epi8(_mm_set1_epi32(x32),\n _mm_set_epi64x(0x0303030303030303, 0x0202020202020202))));\n bytesh = _mm_andnot_si128(bytesh, _mm_set1_epi8((char)0xF0));\n return _mm_or_si128(qxh, bytesh);\n }\n\n inline __m256i load(const block_iq4_nl *b) {\n return MM256_SET_M128I(load1(b), load0(b));\n }\n\n inline __m128i load0(const block_iq4_nl *b) {\n const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n return _mm_shuffle_epi8(iq4nlt, _mm_and_si128(_mm_set1_epi8(15), x));\n }\n\n inline __m128i load1(const block_iq4_nl *b) {\n const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n return _mm_shuffle_epi8(iq4nlt, _mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4)));\n }\n\n inline __m256 updot(__m256i u, __m256i s) {\n __m256i res;\n#if defined(__AVX512VNNI__) && defined(__AVX512VL__)\n res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);\n#elif defined(__AVXVNNI__)\n res = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), u, s);\n#else\n res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));\n#endif\n return _mm256_cvtepi32_ps(res);\n }\n\n static inline __m256i denibble(const uint8_t *p) {\n __m128i x = _mm_loadu_si128((const __m128i *)p);\n return _mm256_and_si256(_mm256_set1_epi8(15),\n _mm256_insertf128_si256(_mm256_castsi128_si256(x),\n _mm_srli_epi16(x, 4), 1));\n }\n\n static inline __m256i bittobyte(const uint8_t *p) {\n uint32_t x32;\n memcpy(&x32, p, sizeof(uint32_t));\n __m256i bytes = _mm256_cmpeq_epi8(_mm256_set1_epi64x(-1),\n _mm256_or_si256(_mm256_set1_epi64x(0x7fbfdfeff7fbfdfe),\n _mm256_shuffle_epi8(_mm256_set1_epi32(x32),\n _mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,\n 0x0101010101010101, 0x0000000000000000))));\n return _mm256_andnot_si256(bytes, _mm256_set1_epi8((char)0xF0));\n }\n\n const TA *const A;\n const TB *const B;\n TC *const C;\n const int64_t k;\n const int64_t lda;\n const int64_t ldb;\n const int64_t ldc;\n const int ith;\n const int nth;\n __m128i iq4nlt;\n};\n#endif // __AVX__\n\n//PPC Implementation\n#if defined(__MMA__)\n\n#define SAVE_ACC(ACC, ii, jj) \\\n __builtin_mma_disassemble_acc(vec_C, ACC); \\\n for (int I = 0; I < 4; I++) { \\\n for (int J = 0; J < 4; J++) { \\\n *((float*)(C+ii+((jj+J)*ldc)+I)) = *((float*)&vec_C[I]+J); \\\n } \\\n } \\\n\ntemplate\nstruct mma_instr;\n\ntemplate<>\nstruct mma_instr {\n static inline void outer_product(acc_t *acc, vec_t a, vec_t b) {\n __builtin_mma_xvbf16ger2pp(acc, a, b);\n }\n};\n\ntemplate<>\nstruct mma_instr {\n static inline void outer_product(acc_t *acc, vec_t a, vec_t b) {\n __builtin_mma_xvf16ger2pp(acc, a, b);\n }\n};\n\ntemplate \nclass tinyBLAS_HP16_PPC {\n public:\n tinyBLAS_HP16_PPC(int64_t k,\n const TA *A, int64_t lda,\n const TB *B, int64_t ldb,\n TC *C, int64_t ldc,\n int ith, int nth)\n : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n }\n\n void matmul(int64_t m, int64_t n) {\n mnpack(0, m, 0, n);\n }\n\n private:\n void vector_permute_store(vec_t *c, int numVec, unsigned char *vecOffset) {\n vec_t t[8], s[8];\n vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};\n vec_t swiz2 = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};\n vec_t swiz3 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};\n vec_t swiz4 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};\n\n if (numVec == 2) {\n t[0] = vec_perm(c[0], c[1], swiz1);\n t[1] = vec_perm(c[2], c[3], swiz1);\n s[0] = vec_perm(t[0], t[1], swiz3);\n s[1] = vec_perm(t[0], t[1], swiz4);\n vec_xst(s[0], 0, (vec_t*)vecOffset);\n vec_xst(s[1], 0, (vec_t*)(vecOffset + 16));\n } else if (numVec == 4) {\n t[0] = vec_perm(c[0], c[1], swiz1);\n t[1] = vec_perm(c[0], c[1], swiz2);\n t[2] = vec_perm(c[2], c[3], swiz1);\n t[3] = vec_perm(c[2], c[3], swiz2);\n s[0] = vec_perm(t[0], t[2], swiz3);\n s[1] = vec_perm(t[0], t[2], swiz4);\n s[2] = vec_perm(t[1], t[3], swiz3);\n s[3] = vec_perm(t[1], t[3], swiz4);\n for (int i = 0; i < 4; ++i)\n vec_xst(s[i], 0, (vec_t*)(vecOffset + i * 16));\n } else if (numVec == 8) {\n for (int i = 0; i < 4; i += 2) {\n t[i+0] = vec_perm(c[i+0], c[i+1], swiz1);\n t[i+1] = vec_perm(c[i+0], c[i+1], swiz2);\n }\n for (int i = 4; i < 8; i += 2) {\n t[i+0] = vec_perm(c[i+0], c[i+1], swiz1);\n t[i+1] = vec_perm(c[i+0], c[i+1], swiz2);\n }\n s[0] = vec_perm(t[0], t[2], swiz3);\n s[1] = vec_perm(t[0], t[2], swiz4);\n s[2] = vec_perm(t[1], t[3], swiz3);\n s[3] = vec_perm(t[1], t[3], swiz4);\n s[4] = vec_perm(t[4], t[6], swiz3);\n s[5] = vec_perm(t[4], t[6], swiz4);\n s[6] = vec_perm(t[5], t[7], swiz3);\n s[7] = vec_perm(t[5], t[7], swiz4);\n for (int i = 0; i < 8; ++i)\n vec_xst(s[i], 0, (vec_t*)(vecOffset + i * 16));\n }\n }\n\n void packNormal(const TA* a, int64_t lda, int rows, int cols, unsigned char* vec) {\n int64_t i, j;\n TA *aoffset = NULL;\n unsigned char *vecOffset = NULL;\n TA * aoffsets[8];\n vector unsigned char c_arr[8];\n aoffset = const_cast(a);\n vecOffset = vec;\n j = (rows >> 3);\n if (j > 0) {\n do {\n if (cols == 4) {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 4; ++it)\n aoffsets[it] = aoffsets[it-1] + lda;\n aoffset += 4 * lda;\n for (int i = 0; i < 4; ++i)\n c_arr[i] = vec_xl(0, (vector unsigned char*)aoffsets[i]);\n vector_permute_store(c_arr, 4, vecOffset);\n for (int i = 0; i<4; i++)\n aoffsets[i] = aoffsets[i]+lda;\n vecOffset +=64;\n }\n i = (cols >> 3);\n if (i > 0) {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 8; ++it) {\n aoffsets[it] = aoffsets[it-1] + lda;\n }\n aoffset += 8 * lda;\n do {\n for (int it = 0; it < 8; ++it)\n c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]);\n vector_permute_store(c_arr, 8, vecOffset);\n for (int it = 0; it < 8; ++it)\n aoffsets[it] = aoffsets[it] + 8*lda;\n vecOffset += 128;\n i--;\n } while(i > 0);\n }\n j--;\n } while(j > 0);\n }\n if (rows & 4) {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 4; ++it)\n aoffsets[it] = aoffsets[it-1] + lda;\n aoffset += 4 * lda;\n if (cols == 4) {\n for (int it = 0; it < 4; ++it)\n c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]);\n vector_permute_store(c_arr, 2, vecOffset);\n for (int it = 0; it< 4; it++)\n aoffsets[it] = aoffsets[it] + lda;\n vecOffset += 32;\n }\n i = (cols >> 3);\n if (i > 0) {\n do {\n for (int it = 0; it < 4; ++it)\n c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]);\n vector_permute_store(c_arr, 4, vecOffset);\n for (int it = 0; it< 4; it++)\n aoffsets[it] = aoffsets[it] + 8*lda;\n vecOffset += 64;\n i--;\n } while(i > 0);\n }\n }\n if (rows & 3) {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 4; ++it)\n aoffsets[it] = aoffsets[it-1] + lda;\n if (cols == 4) {\n switch(rows) {\n case 3: c_arr[2] = vec_xl(0, (vector unsigned char*)aoffsets[2]);\n case 2: c_arr[1] = vec_xl(0, (vector unsigned char*)aoffsets[1]);\n case 1: c_arr[0] = vec_xl(0, (vector unsigned char*)aoffsets[0]);\n break;\n }\n vector_permute_store(c_arr, 2, vecOffset);\n for (int it = 0; it< 4; it++)\n aoffsets[it] = aoffsets[it] + lda;\n vecOffset += 32;\n }\n i = (cols >> 3);\n if (i > 0) {\n do {\n switch(rows) {\n case 3: c_arr[2] = vec_xl(0, (vector unsigned char*)aoffsets[2]);\n case 2: c_arr[1] = vec_xl(0, (vector unsigned char*)aoffsets[1]);\n case 1: c_arr[0] = vec_xl(0, (vector unsigned char*)aoffsets[0]);\n break;\n }\n vector_permute_store(c_arr, 4, vecOffset);\n for (int it = 0; it <4; it++)\n aoffsets[it] = aoffsets[it] + 8* lda;\n vecOffset += 64;\n i--;\n } while(i > 0);\n }\n }\n }\n\n void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t mc, nc, mp, np;\n int m_rem = MIN(m - m0, 8);\n int n_rem = MIN(n - n0, 8);\n\n if (m_rem >= 8 && n_rem >= 8) {\n mc = 8;\n nc = 8;\n gemm<8,8>(m0, m, n0, n);\n } else if (m_rem >= 4 && n_rem >= 8) {\n mc = 4;\n nc = 8;\n gemm<4,8>(m0, m, n0, n);\n } else if (m_rem >=8 && n_rem >=4){\n mc = 8;\n nc = 4;\n gemm<8,4>(m0, m, n0, n);\n } else if ((m_rem < 4) && (n_rem >= 8)) {\n nc = 8;\n switch(m_rem) {\n case 1:\n mc = 1;\n gemm_Mx8<1>(m0, m, n0, n);\n break;\n case 2:\n mc = 2;\n gemm_Mx8<2>(m0, m, n0, n);\n break;\n case 3:\n mc = 3;\n gemm_Mx8<3>(m0, m, n0, n);\n break;\n default:\n return;\n }\n } else if (m_rem >= 4 && n_rem >= 4) {\n mc = 4;\n nc = 4;\n gemm_small<4, 4>(m0, m, n0, n);\n } else if ((m_rem > 4) && (n_rem < 4)) {\n mc = 4;\n switch(n_rem) {\n case 1:\n nc = 1;\n gemm_small<4, 1>(m0, m, n0, n);\n break;\n case 2:\n nc = 2;\n gemm_small<4, 2>(m0, m, n0, n);\n break;\n case 3:\n nc = 3;\n gemm_small<4, 3>(m0, m, n0, n);\n break;\n\n default:\n return;\n }\n } else {\n switch((m_rem << 4) | n_rem) {\n case 0x43:\n mc = 4;\n nc = 3;\n gemm_small<4, 3>(m0, m, n0, n);\n break;\n case 0x42:\n mc = 4;\n nc = 2;\n gemm_small<4, 2>(m0, m, n0, n);\n break;\n case 0x41:\n mc = 4;\n nc = 1;\n gemm_small<4, 1>(m0, m, n0, n);\n break;\n case 0x34:\n mc = 3;\n nc = 4;\n gemm_small<3, 4>(m0, m, n0, n);\n break;\n case 0x33:\n mc = 3;\n nc = 3;\n gemm_small<3, 3>(m0, m, n0, n);\n break;\n case 0x32:\n mc = 3;\n nc = 2;\n gemm_small<3, 2>(m0, m, n0, n);\n break;\n case 0x31:\n mc = 3;\n nc = 1;\n gemm_small<3, 1>(m0, m, n0, n);\n break;\n case 0x24:\n mc = 2;\n nc = 4;\n gemm_small<2,4>(m0, m, n0, n);\n break;\n case 0x23:\n mc = 2;\n nc = 3;\n gemm_small<2, 3>(m0, m, n0, n);\n break;\n case 0x22:\n mc = 2;\n nc = 2;\n gemm_small<2, 2>(m0, m, n0, n);\n break;\n case 0x21:\n mc = 2;\n nc = 1;\n gemm_small<2, 1>(m0, m, n0, n);\n break;\n case 0x14:\n mc = 1;\n nc = 4;\n gemm_small<1, 4>(m0, m, n0, n);\n break;\n case 0x13:\n mc = 1;\n nc = 3;\n gemm_small<1, 3>(m0, m, n0, n);\n break;\n case 0x12:\n mc = 1;\n nc = 2;\n gemm_small<1, 2>(m0, m, n0, n);\n break;\n case 0x11:\n mc = 1;\n nc = 1;\n gemm_small<1, 1>(m0, m, n0, n);\n break;\n default:\n return;\n }\n }\n mp = m0 + (m - m0) / mc * mc;\n np = n0 + (n - n0) / nc * nc;\n mnpack(mp, m, n0, np);\n mnpack(m0, m, np, n);\n }\n\n void KERNEL_4x8(int64_t ii, int64_t jj) {\n vec_t vec_A[4], vec_B[8] , vec_C[4];\n acc_t acc_0, acc_1;\n __builtin_mma_xxsetaccz(&acc_0);\n __builtin_mma_xxsetaccz(&acc_1);\n for (int l = 0; l < k; l+=8) {\n packNormal((A+(ii*lda)+l), lda, 4, 8, (uint8_t*)vec_A);\n packNormal((B+(jj*ldb)+l), ldb, 8, 8, (uint8_t*)vec_B);\n for (int x = 0; x < 4; x++) {\n mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]);\n mma_instr::outer_product(&acc_1, vec_A[x], vec_B[x+4]);\n }\n }\n SAVE_ACC(&acc_0, ii, jj);\n SAVE_ACC(&acc_1, ii, jj+4);\n }\n\n void KERNEL_8x4(int64_t ii, int64_t jj) {\n vec_t vec_A[8], vec_B[4] , vec_C[4];\n acc_t acc_0, acc_1;\n __builtin_mma_xxsetaccz(&acc_0);\n __builtin_mma_xxsetaccz(&acc_1);\n for (int l = 0; l < k; l+=8) {\n packNormal((A+(ii*lda)+l), lda, 8, 8, (uint8_t*)vec_A);\n packNormal((B+(jj*ldb)+l), ldb, 8, 4, (uint8_t*)vec_B);\n for (int x = 0; x < 4; x++) {\n mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]);\n mma_instr::outer_product(&acc_1, vec_A[x+4], vec_B[x]);\n }\n }\n SAVE_ACC(&acc_0, ii, jj);\n SAVE_ACC(&acc_1, ii+4, jj);\n }\n\n\n void KERNEL_8x8(int64_t ii, int64_t jj) {\n vec_t vec_A[8], vec_B[8], vec_C[4];\n acc_t acc_0, acc_1, acc_2, acc_3;\n __builtin_mma_xxsetaccz(&acc_0);\n __builtin_mma_xxsetaccz(&acc_1);\n __builtin_mma_xxsetaccz(&acc_2);\n __builtin_mma_xxsetaccz(&acc_3);\n for (int l = 0; l < k; l+=8) {\n packNormal(A+(ii*lda)+l, lda, 8, 8, (uint8_t*)vec_A);\n packNormal(B+(jj*ldb)+l, ldb, 8, 8, (uint8_t*)vec_B);\n for (int x = 0; x < 4; x++) {\n mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]);\n mma_instr::outer_product(&acc_1, vec_A[x], vec_B[x+4]);\n mma_instr::outer_product(&acc_2, vec_A[x+4], vec_B[x]);\n mma_instr::outer_product(&acc_3, vec_A[x+4], vec_B[x+4]);\n }\n }\n\n SAVE_ACC(&acc_0, ii, jj);\n SAVE_ACC(&acc_1, ii, jj+4);\n SAVE_ACC(&acc_2, ii+4, jj);\n SAVE_ACC(&acc_3, ii+4, jj+4);\n }\n\n template\n void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * RN;\n vec_t vec_C[4];\n acc_t acc_0;\n __builtin_mma_xxsetaccz(&acc_0);\n vec_t vec_A[2], vec_B[2];\n for (int l=0; l::outer_product(&acc_0, vec_A[x], vec_B[x]);\n }\n }\n __builtin_mma_disassemble_acc(vec_C, &acc_0);\n for (int I = 0; I < RM; I++) {\n for (int J = 0; J < RN; J++) {\n *((TC*)(C+ii+((jj+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);\n }\n }\n }\n }\n\n template\n void gemm_Mx8(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int RN = 8;\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * RN;\n vec_t vec_C[4];\n acc_t acc_0, acc_1;\n __builtin_mma_xxsetaccz(&acc_0);\n __builtin_mma_xxsetaccz(&acc_1);\n vec_t vec_A[4], vec_B[8];\n for (int l=0; l::outer_product(&acc_0, vec_A[x], vec_B[x]);\n mma_instr::outer_product(&acc_1, vec_A[x], vec_B[x+4]);\n }\n }\n __builtin_mma_disassemble_acc(vec_C, &acc_0);\n for (int I = 0; I < RM; I++) {\n for (int J = 0; J < 4; J++) {\n *((TC*)(C+ii+((jj+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);\n }\n }\n __builtin_mma_disassemble_acc(vec_C, &acc_1);\n for (int I = 0; I < RM; I++) {\n for (int J = 0; J < 4; J++) {\n *((TC*)(C+ii+((jj+4+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);\n }\n }\n }\n }\n\n template\n inline void kernel(int64_t ii, int64_t jj) {\n if constexpr(RM == 4 && RN == 8) {\n KERNEL_4x8(ii,jj);\n } else if constexpr(RM == 8 && RN == 8) {\n KERNEL_8x8(ii,jj);\n } else if constexpr(RM == 8 && RN == 4) {\n KERNEL_8x4(ii,jj);\n } else {\n assert(false && \"RN/RM values not supported\");\n }\n }\n\n template \n NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * RN;\n kernel(ii, jj);\n }\n }\n\n const TA *const A;\n const TB *const B;\n TC *C;\n const int64_t k;\n const int64_t lda;\n const int64_t ldb;\n const int64_t ldc;\n const int ith;\n const int nth;\n};\n\ntemplate \nclass tinyBLAS_Q0_PPC {\n public:\n tinyBLAS_Q0_PPC(int64_t k,\n const TA * A, int64_t lda,\n const block_q8_0 * B, int64_t ldb,\n float * C, int64_t ldc,\n int ith, int nth)\n : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n }\n\n void matmul(int64_t m, int64_t n) {\n const int64_t mc = 64;\n const int64_t kc = 64;\n int64_t nc = 64;\n int64_t n_aligned = 0;\n if (n % 64 == 0) {\n n_aligned = n;\n } else if (n == 4) {\n n_aligned = 4;\n } else if (n < 64) {\n n_aligned = (n / 8) * 8;\n } else {\n n_aligned = (n / 64) * 64;\n }\n\n if (n_aligned > 0) {\n if (n_aligned % 64 == 0) nc = 64;\n else if (n_aligned == n) nc = n;\n else if (n_aligned % 32 == 0) nc = 32;\n else if (n_aligned % 24 == 0) nc = 24;\n else if (n_aligned % 16 == 0) nc = 16;\n else nc = 8;\n }\n bool can_use_tiled = n_aligned > 0 && (m % mc == 0) && (k % kc == 0);\n if (can_use_tiled) {\n matmul_tiled(m, n_aligned, mc, nc, kc);\n if (n > n_aligned) {\n mnpack(0, m, n_aligned, n);\n }\n } else {\n mnpack(0, m, 0, n);\n }\n }\n\n private:\n inline void save_res(int ii, int jj, int idx, vector float * fin_res, int RM = 4, int RN = 4) {\n for (int I = 0; I < RM; I++) {\n for (int J = 0; J < RN; J++) {\n *((float *)(C + ii + ((jj + J) * ldc) + I)) = *((float *)&fin_res[idx + I] + J);\n }\n }\n }\n\n inline void save_acc(acc_t * ACC, int64_t ii, int64_t jj) {\n vec_t vec_C[4];\n __builtin_mma_disassemble_acc(vec_C, ACC);\n for (int I = 0; I < 4; I++) {\n for (int J = 0; J < 4; J++) {\n *((float *)(C + ii + ((jj + J) * ldc) + I)) = *((float *)&vec_C[I] + J);\n }\n }\n }\n\n inline void add_save_acc(acc_t * ACC, int64_t ii, int64_t jj) {\n vec_t vec_C[4];\n __builtin_mma_disassemble_acc(vec_C, ACC);\n for (int I = 0; I < 4; I++) {\n for (int J = 0; J < 4; J++) {\n float * c_ptr = (float *)(C + ii+ ((jj + J) * ldc) + I);\n *c_ptr += *((float *)&vec_C[I] + J);\n }\n }\n }\n\n template\n inline void compute(acc_t * ACC, int c_idx, int s_idx, ArrayType & comparray, vector float * vs, vector float * fin_res) {\n vector signed int vec_C[4];\n vector float CA[4] = {0};\n vector float res[4] = {0};\n __builtin_mma_disassemble_acc(vec_C, ACC);\n for (int i = 0; i < 4; i++) {\n CA[i] = vec_splats((float)(((double)comparray[c_idx + i]) * -128.0));\n res[i] = vec_add(vec_ctf(vec_C[i], 0), CA[i]);\n fin_res[s_idx + i] = vec_madd(res[i], vs[s_idx + i], fin_res[s_idx + i]);\n }\n }\n\n inline void process_q4_elements(vector signed char (&c)[2], int * ca) {\n const vector signed char lowMask = vec_splats((signed char)0xF);\n const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n const vector signed char v8 = vec_splats((signed char)0x8);\n vector signed int vsum = {0};\n vector signed int vsum2 = {0};\n c[0] = vec_and(c[1], lowMask);\n c[1] = vec_sr(c[1], v4);\n c[0] = vec_sub(c[0], v8);\n c[1] = vec_sub(c[1], v8);\n vsum = vec_sum4s(c[0], vsum);\n vsum2 = vec_sum4s(c[1], vsum2);\n vsum = vec_add(vsum, vsum2);\n *(ca) = vsum[0] + vsum[1] + vsum[2] + vsum[3];\n }\n\n template \n inline void vector_permute_store(V2 & s1, V2 & s2, V2 & s3, V2 & s4, V1 * vecOffset, bool flip) {\n vector unsigned char swiz1 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};\n vector unsigned char swiz2 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};\n vector unsigned char swiz3 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27};\n vector unsigned char swiz4 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};\n V2 t1, t2, t3, t4, t5, t6, t7, t8;\n vector unsigned char xor_vector;\n uint8_t flip_vec = 0x80;\n xor_vector = vec_splats(flip_vec);\n t1 = vec_perm(s1, s2, swiz1);\n t2 = vec_perm(s1, s2, swiz2);\n t3 = vec_perm(s3, s4, swiz1);\n t4 = vec_perm(s3, s4, swiz2);\n t5 = vec_perm(t1, t3, swiz3);\n t6 = vec_perm(t1, t3, swiz4);\n t7 = vec_perm(t2, t4, swiz3);\n t8 = vec_perm(t2, t4, swiz4);\n if (flip == true) {\n t5 = vec_xor(t5, xor_vector);\n t6 = vec_xor(t6, xor_vector);\n t7 = vec_xor(t7, xor_vector);\n t8 = vec_xor(t8, xor_vector);\n }\n vec_xst(t5, 0, vecOffset);\n vec_xst(t6, 0, vecOffset + 16);\n vec_xst(t7, 0, vecOffset + 32);\n vec_xst(t8, 0, vecOffset + 48);\n }\n\n inline void unpack_q4_to_q8(vector signed char packed, vector signed char & lo, vector signed char & hi) {\n const vector signed char lowMask = vec_splats((signed char)0x0F);\n const vector signed char v8 = vec_splats((signed char)0x08);\n const vector unsigned char v4 = vec_splats((unsigned char)4);\n lo = vec_and(packed, lowMask);\n hi = vec_sr(packed, v4);\n lo = vec_sub(lo, v8);\n hi = vec_sub(hi, v8);\n }\n\n inline void vector_permute_store_fp16(vec_t * c, unsigned char * vecOffset) {\n vec_t t[8], s[8];\n vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};\n vec_t swiz2 = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};\n vec_t swiz3 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};\n vec_t swiz4 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};\n for (int i = 0; i < 4; i += 2) {\n t[i + 0] = vec_perm(c[i + 0], c[i + 1], swiz1);\n t[i + 1] = vec_perm(c[i + 0], c[i + 1], swiz2);\n }\n for (int i = 4; i < 8; i += 2) {\n t[i + 0] = vec_perm(c[i + 0], c[i + 1], swiz1);\n t[i + 1] = vec_perm(c[i + 0], c[i + 1], swiz2);\n }\n s[0] = vec_perm(t[0], t[2], swiz3);\n s[1] = vec_perm(t[0], t[2], swiz4);\n s[2] = vec_perm(t[1], t[3], swiz3);\n s[3] = vec_perm(t[1], t[3], swiz4);\n s[4] = vec_perm(t[4], t[6], swiz3);\n s[5] = vec_perm(t[4], t[6], swiz4);\n s[6] = vec_perm(t[5], t[7], swiz3);\n s[7] = vec_perm(t[5], t[7], swiz4);\n for (int i = 0; i < 8; ++i) {\n vec_xst(s[i], 0, (vec_t *)(vecOffset + i * 16));\n }\n }\n\n static inline void convert_and_scale_q8(vector signed char raw, vector float v_scale, vector unsigned short & out_hi, vector unsigned short & out_lo) {\n vector signed short i16_hi = vec_unpackh(raw);\n vector signed short i16_lo = vec_unpackl(raw);\n\n vector float f_hi_h = vec_ctf(vec_unpackh(i16_hi), 0);\n vector float f_hi_l = vec_ctf(vec_unpackl(i16_hi), 0);\n vector float f_lo_h = vec_ctf(vec_unpackh(i16_lo), 0);\n vector float f_lo_l = vec_ctf(vec_unpackl(i16_lo), 0);\n out_hi = vec_pack_to_short_fp32(vec_mul(f_hi_h, v_scale), vec_mul(f_hi_l, v_scale));\n out_lo = vec_pack_to_short_fp32(vec_mul(f_lo_h, v_scale), vec_mul(f_lo_l, v_scale));\n }\n\n void packNormal_q4_fp16(const block_q4_0 * a, int64_t lda, int rows, int blocks, unsigned char * vec) {\n unsigned char * vecOffset = vec;\n for (int i = 0; i < rows; i += 8) {\n const block_q4_0 * rows_base[8];\n for (int r = 0; r < 8; r++) {\n rows_base[r] = a + (i + r) * lda;\n }\n for (int blk = 0; blk < blocks; blk++) {\n vector unsigned short hp_res[8][4];\n for (int r = 0; r < 8; r++) {\n const block_q4_0 * current_blk = rows_base[r] + blk;\n vector float v_scale = vec_extract_fp32_from_shorth(vec_splats(current_blk->d));\n vector signed char v_qs = vec_xl(0, (const vector signed char *)current_blk->qs);\n vector signed char c1, c2;\n unpack_q4_to_q8(v_qs, c1, c2);\n convert_and_scale_q8(c1, v_scale, hp_res[r][0], hp_res[r][1]);\n convert_and_scale_q8(c2, v_scale, hp_res[r][2], hp_res[r][3]);\n }\n for (int c = 0; c < 4; c++) {\n vector unsigned char c_arr[8];\n for (int r = 0; r < 8; r++) {\n c_arr[r] = (vector unsigned char)hp_res[r][c];\n }\n vector_permute_store_fp16((vec_t *)c_arr, vecOffset);\n vecOffset += 128;\n }\n }\n }\n }\n\n template \n static inline void pack_q8_block(const block_q8_0 * a, int64_t lda, int rows, int blocks, unsigned char * vec) {\n unsigned char * vecOffset = vec;\n const vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};\n const vec_t swiz2 = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};\n const vec_t swiz3 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};\n const vec_t swiz4 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};\n\n for (int i = 0; i < rows; i += chunk_size) {\n const block_q8_0 * rows_base[chunk_size];\n for (int r = 0; r < chunk_size; r++) {\n rows_base[r] = a + (i + r) * lda;\n }\n for (int blk = 0; blk < blocks; blk++) {\n vector unsigned short hp_res[chunk_size][4];\n for (int r = 0; r < chunk_size; r++) {\n const block_q8_0 * b = rows_base[r] + blk;\n vector float v_scale = vec_extract_fp32_from_shorth(vec_splats(b->d));\n vector signed char c[2];\n __vector_pair pair = __builtin_vsx_lxvp(0, (__vector_pair *)b->qs);\n __builtin_vsx_disassemble_pair(c, & pair);\n convert_and_scale_q8(c[0], v_scale, hp_res[r][0], hp_res[r][1]);\n convert_and_scale_q8(c[1], v_scale, hp_res[r][2], hp_res[r][3]);\n }\n for (int col = 0; col < 4; col++) {\n if constexpr (chunk_size == 8) {\n vec_t t[8];\n t[0] = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz1);\n t[1] = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz2);\n t[2] = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz1);\n t[3] = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz2);\n t[4] = vec_perm((vec_t)hp_res[4][col], (vec_t)hp_res[5][col], swiz1);\n t[5] = vec_perm((vec_t)hp_res[4][col], (vec_t)hp_res[5][col], swiz2);\n t[6] = vec_perm((vec_t)hp_res[6][col], (vec_t)hp_res[7][col], swiz1);\n t[7] = vec_perm((vec_t)hp_res[6][col], (vec_t)hp_res[7][col], swiz2);\n\n vec_xst(vec_perm(t[0], t[2], swiz3), 0, (vec_t *)(vecOffset + 0));\n vec_xst(vec_perm(t[0], t[2], swiz4), 0, (vec_t *)(vecOffset + 16));\n vec_xst(vec_perm(t[1], t[3], swiz3), 0, (vec_t *)(vecOffset + 32));\n vec_xst(vec_perm(t[1], t[3], swiz4), 0, (vec_t *)(vecOffset + 48));\n vec_xst(vec_perm(t[4], t[6], swiz3), 0, (vec_t *)(vecOffset + 64));\n vec_xst(vec_perm(t[4], t[6], swiz4), 0, (vec_t *)(vecOffset + 80));\n vec_xst(vec_perm(t[5], t[7], swiz3), 0, (vec_t *)(vecOffset + 96));\n vec_xst(vec_perm(t[5], t[7], swiz4), 0, (vec_t *)(vecOffset + 112));\n vecOffset += 128;\n } else {\n vec_t t0 = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz1);\n vec_t t1 = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz2);\n vec_t t2 = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz1);\n vec_t t3 = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz2);\n\n vec_xst(vec_perm(t0, t2, swiz3), 0, (vec_t *)(vecOffset + 0));\n vec_xst(vec_perm(t0, t2, swiz4), 0, (vec_t *)(vecOffset + 16));\n vec_xst(vec_perm(t1, t3, swiz3), 0, (vec_t *)(vecOffset + 32));\n vec_xst(vec_perm(t1, t3, swiz4), 0, (vec_t *)(vecOffset + 48));\n vecOffset += 64;\n }\n }\n }\n }\n }\n\n void packNormal_q8_fp16(const block_q8_0 * a, int64_t lda, int rows, int blocks, unsigned char * vec) {\n if (rows == 4) {\n pack_q8_block<4>(a, lda, rows, blocks, vec);\n } else {\n pack_q8_block<8>(a, lda, rows, blocks, vec);\n }\n }\n\n template\n void packNormalInt4(const TA * a, int64_t lda, int rows, int cols, int8_t * vec, std::array & comparray) {\n int64_t i, j;\n TA * aoffset = NULL;\n int8_t * vecOffset = NULL;\n TA * aoffset1 = NULL, * aoffset2 = NULL, * aoffset3 = NULL, * aoffset4 = NULL;\n TA * aoffset5 = NULL, * aoffset6 = NULL, * aoffset7 = NULL, * aoffset8 = NULL;\n vector signed char c1[2] = {0}, c2[2] = {0}, c3[2] = {0}, c4[2] = {0};\n vector signed char c5[2] = {0}, c6[2] = {0}, c7[2] = {0}, c8[2] = {0};\n aoffset = const_cast(a);\n vecOffset = vec;\n j = (rows >> 3);\n if (j > 0) {\n do {\n aoffset1 = aoffset;\n aoffset2 = aoffset1 + lda;\n aoffset3 = aoffset2 + lda;\n aoffset4 = aoffset3 + lda;\n aoffset5 = aoffset4 + lda;\n aoffset6 = aoffset5 + lda;\n aoffset7 = aoffset6 + lda;\n aoffset8 = aoffset7 + lda;\n aoffset += 8 * lda;\n i = (cols >> 2);\n if (i > 0) {\n do {\n c1[1] = vec_xl(0, (const vector signed char *)aoffset1->qs);\n c2[1] = vec_xl(0, (const vector signed char *)aoffset2->qs);\n c3[1] = vec_xl(0, (const vector signed char *)aoffset3->qs);\n c4[1] = vec_xl(0, (const vector signed char *)aoffset4->qs);\n c5[1] = vec_xl(0, (const vector signed char *)aoffset5->qs);\n c6[1] = vec_xl(0, (const vector signed char *)aoffset6->qs);\n c7[1] = vec_xl(0, (const vector signed char *)aoffset7->qs);\n c8[1] = vec_xl(0, (const vector signed char *)aoffset8->qs);\n\n process_q4_elements(c1, & comparray[0]);\n process_q4_elements(c2, & comparray[1]);\n process_q4_elements(c3, & comparray[2]);\n process_q4_elements(c4, & comparray[3]);\n process_q4_elements(c5, & comparray[4]);\n process_q4_elements(c6, & comparray[5]);\n process_q4_elements(c7, & comparray[6]);\n process_q4_elements(c8, & comparray[7]);\n vector_permute_store(c1[0], c2[0], c3[0], c4[0], vecOffset, false);\n vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false);\n vector_permute_store(c5[0], c6[0], c7[0], c8[0], vecOffset + 128, false);\n vector_permute_store(c5[1], c6[1], c7[1], c8[1], vecOffset + 192, false);\n aoffset1 += lda;\n aoffset2 += lda;\n aoffset3 += lda;\n aoffset4 += lda;\n aoffset5 += lda;\n aoffset6 += lda;\n aoffset7 += lda;\n aoffset8 += lda;\n vecOffset += 256;\n i--;\n } while (i > 0);\n }\n j--;\n } while (j > 0);\n }\n\n if (rows & 4) {\n aoffset1 = aoffset;\n aoffset2 = aoffset1 + lda;\n aoffset3 = aoffset2 + lda;\n aoffset4 = aoffset3 + lda;\n aoffset += 4 * lda;\n i = (cols >> 2);\n if (i > 0) {\n do {\n c1[1] = vec_xl(0, (const vector signed char *)aoffset1->qs);\n c2[1] = vec_xl(0, (const vector signed char *)aoffset2->qs);\n c3[1] = vec_xl(0, (const vector signed char *)aoffset3->qs);\n c4[1] = vec_xl(0, (const vector signed char *)aoffset4->qs);\n\n process_q4_elements(c1, & comparray[0]);\n process_q4_elements(c2, & comparray[1]);\n process_q4_elements(c3, & comparray[2]);\n process_q4_elements(c4, & comparray[3]);\n vector_permute_store(c1[0], c2[0], c3[0], c4[0], vecOffset, false);\n vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false);\n aoffset1 += lda;\n aoffset2 += lda;\n aoffset3 += lda;\n aoffset4 += lda;\n vecOffset += 128;\n i--;\n } while (i > 0);\n }\n }\n\n if (rows & 3) {\n aoffset1 = aoffset;\n aoffset2 = aoffset1 + lda;\n aoffset3 = aoffset2 + lda;\n i = (cols >> 2);\n if (i > 0) {\n do {\n switch(rows) {\n case 3: c3[1] = vec_xl(0, (const vector signed char *)aoffset3->qs);\n case 2: c2[1] = vec_xl(0, (const vector signed char *)aoffset2->qs);\n case 1: c1[1] = vec_xl(0, (const vector signed char *)aoffset1->qs);\n break;\n }\n process_q4_elements(c1, & comparray[0]);\n process_q4_elements(c2, & comparray[1]);\n process_q4_elements(c3, & comparray[2]);\n process_q4_elements(c4, & comparray[3]);\n vector_permute_store(c1[0], c2[0], c3[0], c4[0], vecOffset, false);\n vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false);\n aoffset1 += lda;\n aoffset2 += lda;\n aoffset3 += lda;\n vecOffset += 128;\n i--;\n } while(i > 0);\n }\n }\n }\n\n template\n void packNormal(const block_q8_0 * a, int64_t lda, int rows, int cols, VA * vec, bool flip) {\n int64_t i, j;\n block_q8_0 * aoffset = NULL;\n VA * vecOffset = NULL;\n block_q8_0 * aoffsets[8];\n __vector_pair arr[8];\n VB c[8][2] = {0};\n VB c1[8] = {0}; VB c2[8] = {0};\n aoffset = const_cast(a);\n vecOffset = vec;\n j = (rows >> 3);\n if (j > 0) {\n do {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 8; it++)\n aoffsets[it] = aoffsets[it - 1] + lda;\n aoffset += 8 * lda;\n\n i = (cols >> 3);\n if (i > 0) {\n do {\n for (int it = 0; it < 8; it++) {\n arr[it] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[it]->qs);\n __builtin_vsx_disassemble_pair(c[it], & arr[it]);\n c1[it] = c[it][0];\n c2[it] = c[it][1];\n }\n vector_permute_store(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);\n vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset + 64, flip);\n vector_permute_store(c1[4], c1[5], c1[6], c1[7], vecOffset + 128, flip);\n vector_permute_store(c2[4], c2[5], c2[6], c2[7], vecOffset + 192, flip);\n for (int it = 0; it < 8; it++)\n aoffsets[it] += lda;\n vecOffset += 256;\n i--;\n } while(i > 0);\n }\n j--;\n } while(j > 0);\n }\n if (rows & 4) {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 4; it++ )\n aoffsets[it] = aoffsets[it-1] + lda;\n aoffset += 4 * lda;\n i = (cols >> 3);\n if (i > 0) {\n do {\n for (int it = 0; it < 4; it++) {\n arr[it] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[it]->qs);\n __builtin_vsx_disassemble_pair(c[it], & arr[it]);\n c1[it] = c[it][0];\n c2[it] = c[it][1];\n }\n vector_permute_store(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);\n vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset + 64, flip);\n for (int it = 0; it < 4; it++) {\n aoffsets[it] += lda;\n }\n vecOffset += 128;\n i--;\n } while(i > 0);\n }\n }\n\n if (rows & 3) {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 3; it++ )\n aoffsets[it] = aoffsets[it - 1] + lda;\n i = (cols >> 3);\n if (i > 0) {\n do {\n switch(rows) {\n case 3: arr[2] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[2]->qs);\n __builtin_vsx_disassemble_pair(c[2], & arr[2]);\n c1[2] = c[2][0]; c2[2] = c[2][1];\n case 2: arr[1] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[1]->qs);\n __builtin_vsx_disassemble_pair(c[1], & arr[1]);\n c1[1] = c[1][0]; c2[1] = c[1][1];\n case 1: arr[0] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[0]->qs);\n __builtin_vsx_disassemble_pair(c[0], & arr[0]);\n c1[0] = c[0][0]; c2[0] = c[0][1];\n break;\n }\n vector_permute_store(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);\n vector_permute_store(c2[0], c2[1], c2[2], c2[3], vecOffset + 64, flip);\n for (int it = 0; it < 3; it++)\n aoffsets[it] += lda;\n vecOffset += 128;\n i--;\n } while(i > 0);\n }\n }\n }\n\n void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int m_rem = MIN(m - m0, 16);\n int n_rem = MIN(n - n0, 16);\n\n int mc = 0, nc = 0;\n\n if (m_rem >= 8 && n_rem >= 8) {\n mc = 8;\n nc = 8;\n gemm<8, 8>(m0, m, n0, n);\n } else if (m_rem >= 4 && n_rem >= 8) {\n mc = 4;\n nc = 8;\n gemm<4, 8>(m0, m, n0, n);\n } else if (m_rem >= 8 && n_rem >= 4) {\n mc = 8;\n nc = 4;\n gemm<8, 4>(m0, m, n0, n);\n } else if (m_rem >= 4 && n_rem >= 4) {\n mc = 4;\n nc = 4;\n gemm_small(m0, m, n0, n, mc, nc);\n } else {\n mc = (m_rem >= 4) ? 4 : m_rem;\n nc = (n_rem >= 4) ? 4 : n_rem;\n if (mc == 0 || nc == 0)\n return;\n gemm_small(m0, m, n0, n, mc, nc);\n }\n\n int64_t mp = m0 + ((m - m0) / mc) * mc;\n int64_t np = n0 + ((n - n0) / nc) * nc;\n mnpack(mp, m, n0, np);\n mnpack(m0, m, np, n);\n }\n\n\n void KERNEL_4x8(int64_t ii, int64_t jj) {\n vec_t vec_A[8], vec_B[16] = {0};\n acc_t acc_0, acc_1;\n std::array comparray {};\n vector float fin_res[8] = {0};\n vector float vs[8] = {0};\n bool isAblock_q4 = std::is_same_v;\n for (int l = 0; l < k; l++) {\n __builtin_mma_xxsetaccz(& acc_0);\n __builtin_mma_xxsetaccz(& acc_1);\n if (std::is_same_v) {\n packNormalInt4<4>((A + (ii * lda) + l), lda, 4, 4, (int8_t *)vec_A, comparray);\n } else {\n packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 4, 8, (int8_t *)vec_A, false);\n }\n packNormal((B + (jj * ldb) + l), ldb, 8, 8, (uint8_t *)vec_B, true);\n for(int x = 0; x < 8; x++) {\n __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);\n __builtin_mma_xvi8ger4pp(& acc_1, vec_A[x], vec_B[x+8]);\n }\n for (int I = 0; I<4; I++) {\n for (int J = 0; J<4; J++) {\n *((float *)& vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));\n *((float *)& vs[I + 4] + J) = (unhalf((A +((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J + 4) * ldb) + l)->d));\n }\n }\n if (!isAblock_q4) {\n auto aoffset = A + (ii * lda) + l;\n for (int i = 0; i < 4; i++) {\n comparray[i] = 0;\n int ca = 0;\n auto *at = aoffset->qs;\n for (int j = 0; j < 32; j++)\n ca += (int)*at++;\n comparray[i] = ca;\n aoffset += lda;\n }\n }\n compute(& acc_0, 0, 0, comparray, vs, fin_res);\n compute(& acc_1, 0, 4, comparray, vs, fin_res);\n }\n save_res(ii, jj, 0, fin_res);\n save_res(ii, jj + 4, 4, fin_res);\n }\n\n void KERNEL_8x4(int64_t ii, int64_t jj) {\n vec_t vec_A[16], vec_B[8] = {0};\n acc_t acc_0, acc_1;\n std::array comparray {};\n vector float fin_res[8] = {0};\n vector float vs[8] = {0};\n bool isAblock_q4 = std::is_same_v;\n for (int l = 0; l < k; l++) {\n __builtin_mma_xxsetaccz(& acc_0);\n __builtin_mma_xxsetaccz(& acc_1);\n if (std::is_same_v) {\n packNormalInt4<8>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray);\n } else {\n packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 8, 8, (int8_t *)vec_A, false);\n }\n packNormal((B + (jj * ldb) + l), ldb, 4, 8, (uint8_t *)vec_B, true);\n for(int x = 0; x < 8; x++) {\n __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);\n __builtin_mma_xvi8ger4pp(& acc_1, vec_A[x + 8], vec_B[x]);\n }\n for (int I = 0; I < 8; I++) {\n for (int J = 0; J < 4; J++) {\n *((float *)&vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));\n }\n }\n if (!isAblock_q4) {\n auto aoffset = A + (ii * lda) + l;\n for (int i = 0; i < 8; i++) {\n comparray[i] = 0;\n int ca = 0;\n auto *at = aoffset->qs;\n for (int j = 0; j < 32; j++)\n ca += (int)*at++;\n comparray[i] = ca;\n aoffset += lda;\n }\n }\n compute(& acc_0, 0, 0, comparray, vs, fin_res);\n compute(& acc_1, 4, 4, comparray, vs, fin_res);\n }\n save_res(ii, jj, 0, fin_res);\n save_res(ii + 4, jj, 4, fin_res);\n }\n\n void KERNEL_8x8(int64_t ii, int64_t jj) {\n vec_t vec_A[16], vec_B[16] = {0};\n acc_t acc_0, acc_1, acc_2, acc_3;\n acc_t acc_4, acc_5, acc_6, acc_7;\n std::array comparray {};\n vector float fin_res[16] = {0};\n vector float vs[16] = {0};\n bool isAblock_q4 = std::is_same_v;\n for (int l = 0; l < k; l++) {\n __builtin_mma_xxsetaccz(& acc_0);\n __builtin_mma_xxsetaccz(& acc_1);\n __builtin_mma_xxsetaccz(& acc_2);\n __builtin_mma_xxsetaccz(& acc_3);\n if (std::is_same_v) {\n packNormalInt4<8>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray);\n } else {\n packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 8, 8, (int8_t *)vec_A, false);\n }\n packNormal((B + (jj * ldb) + l), ldb, 8, 8, (uint8_t *)vec_B, true);\n for(int x = 0; x < 8; x++) {\n __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);\n __builtin_mma_xvi8ger4pp(& acc_1, vec_A[x + 8], vec_B[x]);\n __builtin_mma_xvi8ger4pp(& acc_2, vec_A[x], vec_B[x + 8]);\n __builtin_mma_xvi8ger4pp(& acc_3, vec_A[x + 8], vec_B[x + 8]);\n }\n for (int I = 0; I < 8 ; I++) {\n for (int J = 0; J < 4; J++) {\n *((float *)& vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));\n *((float *)& vs[I + 8] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J + 4) * ldb) + l)->d));\n }\n }\n if (!isAblock_q4) {\n auto aoffset = A + (ii * lda) + l;\n for (int i = 0; i < 8; i++) {\n comparray[i] = 0;\n int ca = 0;\n auto *at = aoffset->qs;\n for (int j = 0; j < 32; j++)\n ca += (int)*at++;\n comparray[i] = ca;\n aoffset += lda;\n }\n }\n compute(& acc_0, 0, 0, comparray, vs, fin_res);\n compute(& acc_1, 4, 4, comparray, vs, fin_res);\n compute(& acc_2, 0, 8, comparray, vs, fin_res);\n compute(& acc_3, 4, 12, comparray, vs, fin_res);\n }\n save_res(ii, jj, 0, fin_res);\n save_res(ii + 4, jj, 4, fin_res);\n save_res(ii, jj + 4, 8, fin_res);\n save_res(ii + 4, jj + 4, 12, fin_res);\n }\n\n void KERNEL_Q0(int64_t ii, int64_t jj, int64_t mc, int64_t nc, int64_t kc, int64_t l, vec_t * vec_A, vec_t * vec_B) {\n acc_t acc[8];\n for (int i = 0; i < mc ; i += 16) {\n for (int j = 0; j < nc; j += 8) {\n int A0_base = (i / 16) * (2 * 32 * kc);\n int B0_base = (j / 8) * (32 * kc);\n for (int x = 0; x < 8; x++) {\n __builtin_mma_xxsetaccz(&acc[x]);\n }\n for (int64_t kk = 0; kk < kc; kk++) {\n int A0_block_idx = A0_base + kk * 32;\n int B0_block_idx = B0_base + kk * 32;\n int A1_block_idx = A0_block_idx + 32 * kc;\n int B1_block_idx = B0_block_idx + 32 * kc;\n vec_t * A0_block = & vec_A[A0_block_idx];\n vec_t * B0_block = & vec_B[B0_block_idx];\n vec_t * A1_block = & vec_A[A1_block_idx];\n for (int it = 0; it < 4; it++) {\n for (int x = 0; x < 4; x++) {\n __builtin_mma_xvf16ger2pp(& acc[0], A0_block[8 * it + x], B0_block[8 * it + x]);\n __builtin_mma_xvf16ger2pp(& acc[1], A0_block[8 * it + x], B0_block[8 * it + x + 4]);\n __builtin_mma_xvf16ger2pp(& acc[2], A0_block[8 * it + x + 4], B0_block[8 * it + x]);\n __builtin_mma_xvf16ger2pp(& acc[3], A0_block[8 * it + x + 4], B0_block[8 * it + x + 4]);\n __builtin_mma_xvf16ger2pp(& acc[4], A1_block[8 * it + x], B0_block[8 * it + x]);\n __builtin_mma_xvf16ger2pp(& acc[5], A1_block[8 * it + x], B0_block[8 * it+ x + 4]);\n __builtin_mma_xvf16ger2pp(& acc[6], A1_block[8 * it + x + 4], B0_block[8 * it + x]);\n __builtin_mma_xvf16ger2pp(& acc[7], A1_block[8 * it + x + 4], B0_block[8 * it + x + 4]);\n }\n }\n }\n if (l == 0) {\n save_acc(& acc[0], ii + i, jj + j);\n save_acc(& acc[1], ii + i, jj + j + 4);\n save_acc(& acc[2], ii + i + 4, jj + j);\n save_acc(& acc[3], ii + i + 4, jj + j + 4);\n save_acc(& acc[4], ii + i + 8, jj + j);\n save_acc(& acc[5], ii + i + 8, jj + j + 4);\n save_acc(& acc[6], ii + i + 12, jj + j);\n save_acc(& acc[7], ii + i + 12, jj + j + 4);\n } else {\n add_save_acc(& acc[0], ii + i, jj + j);\n add_save_acc(& acc[1], ii + i, jj + j + 4);\n add_save_acc(& acc[2], ii + i + 4, jj + j);\n add_save_acc(& acc[3], ii + i + 4, jj + j + 4);\n add_save_acc(& acc[4], ii + i + 8, jj + j);\n add_save_acc(& acc[5], ii + i + 8, jj + j + 4);\n add_save_acc(& acc[6], ii + i + 12, jj + j);\n add_save_acc(& acc[7], ii + i + 12, jj + j + 4);\n }\n }\n }\n }\n\n void matmul_tiled(int64_t m, int64_t n, int64_t mc, int64_t nc, int64_t kc) {\n vec_t A_pack[mc * kc * 4];\n vec_t B_pack[nc * kc * 4];\n constexpr bool is_Ablock_q4 = std::is_same_v;\n int64_t ytiles = m / mc;\n int64_t xtiles = n / nc;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles) {\n end = tiles;\n }\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = (job / xtiles) * mc;\n int64_t jj = (job % xtiles) * nc;\n for (int64_t kk = 0; kk < k; kk += kc) {\n if constexpr(is_Ablock_q4) {\n packNormal_q4_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack);\n } else {\n packNormal_q8_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack);\n }\n packNormal_q8_fp16(B + jj * ldb + kk, ldb, nc, kc, (uint8_t *)B_pack);\n KERNEL_Q0(ii, jj, mc, nc, kc, kk, A_pack, B_pack);\n }\n }\n }\n\n void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n, int RM, int RN) {\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n vec_t vec_A[8] = {0}, vec_B[8] = {0};\n vector signed int vec_C[4];\n acc_t acc_0;\n bool isAblock_q4 = std::is_same_v;\n\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * RN;\n std::array comparray{};\n vector float res[4] = {0};\n vector float fin_res[4] = {0};\n vector float vs[4] = {0};\n vector float CA[4] = {0};\n __builtin_prefetch((A + (ii * lda) + 0)->qs, 0, 1); // prefetch first value\n __builtin_prefetch((B + (jj * ldb) + 0)->qs, 0, 1); // prefetch first value\n for (int l = 0; l < k; l++) {\n __builtin_prefetch((A + (ii * lda) + (l + 1))->qs, 0, 1); // prefetch one loop ahead\n __builtin_prefetch((B + (jj * ldb) + (l + 1))->qs, 0, 1); // prefetch one loop ahead\n __builtin_mma_xxsetaccz(& acc_0);\n if (isAblock_q4) {\n packNormalInt4<4>((A + (ii * lda) + l), lda, RM, 4, (int8_t *)vec_A, comparray);\n } else {\n packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, RM, 8, (int8_t *)vec_A, false);\n }\n packNormal((B + (jj * ldb) + l), ldb, RN, 8, (uint8_t *)vec_B, true);\n for (int x = 0; x < 8; x += 4) {\n __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);\n __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x + 1], vec_B[x + 1]);\n __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x + 2], vec_B[x + 2]);\n __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x + 3], vec_B[x + 3]);\n }\n for (int I = 0; I < RM; I++) {\n for (int J = 0; J < RN; J++) {\n *((float*)&vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));\n }\n }\n __builtin_mma_disassemble_acc(vec_C, & acc_0);\n if (!isAblock_q4) {\n auto aoffset = A + (ii * lda) + l;\n for (int i = 0; i < RM; i++) {\n comparray[i] = 0;\n int ca = 0;\n auto *at = aoffset->qs;\n for (int j = 0; j < 32; j++)\n ca += (int)*at++;\n comparray[i] = ca;\n aoffset += lda;\n }\n }\n for (int i = 0; i < RM; i++) {\n CA[i] = vec_splats((float)(((double)comparray[i]) * -128.0));\n res[i] = vec_add(vec_ctf(vec_C[i], 0), CA[i]);\n fin_res[i] = vec_madd(res[i], vs[i], fin_res[i]);\n }\n }\n save_res(ii, jj, 0, fin_res, RM, RN);\n }\n }\n\n template\n inline void kernel(int64_t ii, int64_t jj) {\n if constexpr(RM == 4 && RN == 8) {\n KERNEL_4x8(ii,jj);\n } else if constexpr(RM == 8 && RN == 4) {\n KERNEL_8x4(ii,jj);\n } else if constexpr(RM == 8 && RN == 8) {\n KERNEL_8x8(ii,jj);\n } else {\n assert(false && \"RN/RM values not supported\");\n }\n }\n\n template \n NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * RN;\n kernel(ii, jj);\n }\n }\n const TA * const A;\n const block_q8_0 * const B;\n float * C;\n const int64_t k;\n int64_t kc;\n const int64_t lda;\n const int64_t ldb;\n const int64_t ldc;\n const int ith;\n const int nth;\n};\n\nclass tinyBLAS_PPC {\n public:\n tinyBLAS_PPC(int64_t k,\n const float * A, int64_t lda,\n const float * B, int64_t ldb,\n float * C, int64_t ldc,\n int ith, int nth)\n : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n }\n\n void matmul(int64_t m, int64_t n) {\n int64_t mc = 256; int64_t nc = 256; int64_t kc = 256;\n if (m % mc == 0 && n % nc == 0 && k % kc == 0) {\n matmul_tiled(m, n, mc, nc, kc);\n } else {\n mnpack(0, m, 0, n);\n }\n }\n\n private:\n\n inline void save_acc(acc_t * ACC, int64_t ii, int64_t jj) {\n vec_t vec_C[4];\n __builtin_mma_disassemble_acc(vec_C, ACC);\n for (int I = 0; I < 4; I++) {\n for (int J = 0; J < 4; J++) {\n *((float *)(C+ii+((jj+J)*ldc)+I)) = *((float *)&vec_C[I]+J);\n }\n }\n }\n\n inline void add_save_acc(acc_t * ACC, int64_t ii, int64_t jj) {\n vec_t vec_C[4];\n __builtin_mma_disassemble_acc(vec_C, ACC);\n for (int I = 0; I < 4; I++) {\n for (int J = 0; J < 4; J++) {\n float * c_ptr = (float *)(C+ii+((jj+J)*ldc)+I);\n *c_ptr += *((float *)&vec_C[I]+J);\n }\n }\n }\n\n inline void vector_permute_store_4(vector float * src, float * vecOffset) {\n vector float t1, t2, t3, t4, t5, t6, t7, t8;\n t1 = vec_mergeh(src[0], src[1]);\n t2 = vec_mergeh(src[2], src[3]);\n t3 = vec_mergel(src[0], src[1]);\n t4 = vec_mergel(src[2], src[3]);\n\n t5 = vec_xxpermdi(t1, t2, 0);\n t6 = vec_xxpermdi(t1, t2, 3);\n t7 = vec_xxpermdi(t3, t4, 0);\n t8 = vec_xxpermdi(t3, t4, 3);\n\n vec_xst(t5, 0, vecOffset);\n vec_xst(t6, 0, vecOffset + 4);\n vec_xst(t7, 0, vecOffset + 8);\n vec_xst(t8, 0, vecOffset + 12);\n }\n\n inline void vector_permute_store_8(vector float * src, float * vecOffset) {\n vector float t1, t2, t3, t4, t5, t6, t7, t8;\n t1 = vec_mergeh(src[0], src[1]);\n t2 = vec_mergeh(src[2], src[3]);\n t3 = vec_mergeh(src[4], src[5]);\n t4 = vec_mergeh(src[6], src[7]);\n\n t5 = vec_xxpermdi(t1, t2, 0);\n t6 = vec_xxpermdi(t3, t4, 0);\n t7 = vec_xxpermdi(t1, t2, 3);\n t8 = vec_xxpermdi(t3, t4, 3);\n\n vec_xst(t5, 0, vecOffset);\n vec_xst(t6, 0, vecOffset + 4);\n vec_xst(t7, 0, vecOffset + 8);\n vec_xst(t8, 0, vecOffset + 12);\n\n t1 = vec_mergel(src[0], src[1]);\n t2 = vec_mergel(src[2], src[3]);\n t3 = vec_mergel(src[4], src[5]);\n t4 = vec_mergel(src[6], src[7]);\n\n t5 = vec_xxpermdi(t1, t2, 0);\n t6 = vec_xxpermdi(t3, t4, 0);\n t7 = vec_xxpermdi(t1, t2, 3);\n t8 = vec_xxpermdi(t3, t4, 3);\n\n vec_xst(t5, 0, vecOffset + 16);\n vec_xst(t6, 0, vecOffset + 20);\n vec_xst(t7, 0, vecOffset + 24);\n vec_xst(t8, 0, vecOffset + 28);\n }\n\n void packTranspose(const float * a, int64_t lda, int rows, int cols, float * vec) {\n int64_t i, j;\n float * aoffsets[8];\n float * aoffset = NULL, * boffset = NULL;\n __vector_pair arr[8];\n vector float c[8][2] = {0};\n vector float c1[8] = {0};\n vector float c2[8] = {0};\n aoffset = const_cast(a);\n boffset = vec;\n j = (rows >> 3);\n if (j > 0) {\n do {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 8; it++)\n aoffsets[it] = aoffsets[it-1] + lda;\n aoffset += 8 * lda;\n i = (cols >> 3);\n if (i > 0) {\n do {\n for (int it = 0; it < 8; it++) {\n arr[it] = __builtin_vsx_lxvp(0, (__vector_pair*)aoffsets[it]);\n __builtin_vsx_disassemble_pair(c[it], &arr[it]);\n c1[it] = c[it][0];\n c2[it] = c[it][1];\n }\n\n vector_permute_store_8(c1, boffset);\n vector_permute_store_8(c2, boffset + 32);\n boffset += 64;\n i--;\n if (i > 0) {\n for (int it = 0; it < 8; it++) {\n aoffsets[it] = aoffsets[it] + 8;\n }\n }\n } while(i > 0);\n }\n if (cols & 4) {\n for (int it = 0; it < 8 ; it++)\n c1[it] = vec_xl(0, aoffsets[it]);\n vector_permute_store_8(c1, boffset);\n }\n j--;\n } while(j > 0);\n }\n\n if (rows & 4) {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 4; it++)\n aoffsets[it] = aoffsets[it-1] + lda;\n aoffset += 4 * lda;\n i = (cols >> 3);\n if (i > 0) {\n do {\n for (int it = 0; it < 4; it++) {\n arr[it] = __builtin_vsx_lxvp(0, (__vector_pair*)aoffsets[it]);\n __builtin_vsx_disassemble_pair(c[it], &arr[it]);\n c1[it] = c[it][0];\n c2[it] = c[it][1];\n }\n vector_permute_store_4(c1, boffset);\n vector_permute_store_4(c2, boffset + 16);\n for (int it = 0; it < 4; it++)\n aoffsets[it] += 8 * lda;\n boffset += 32;\n i--;\n } while(i > 0);\n }\n\n if (cols & 4) {\n for (int it = 0; it < 4; it++)\n c1[it] = vec_xl(0, aoffsets[it]);\n vector_permute_store_4(c1, boffset);\n }\n }\n if (rows & 3) {\n aoffsets[0] = aoffset;\n for (int it = 1; it < 3; it++)\n aoffsets[it] = aoffsets[it-1] + lda;\n if (cols & 4) {\n for (int it = 0; it < 3; it++)\n c1[it] = vec_xl(0, aoffsets[it]);\n vector_permute_store_4(c1, boffset);\n }\n }\n }\n\n void KERNEL_4x4(int64_t ii, int64_t jj) {\n vec_t vec_A[4], vec_B[4], vec_C[4];\n acc_t acc_0;\n __builtin_mma_xxsetaccz(&acc_0);\n for (int l = 0; l < k; l += 4) {\n packTranspose(A + (ii * lda) + l, lda, 4, 4, (float *)vec_A);\n packTranspose(B + (jj * ldb) + l, ldb, 4, 4, (float *)vec_B);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[0], vec_B[0]);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[1], vec_B[1]);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[2], vec_B[2]);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[3], vec_B[3]);\n }\n save_acc(&acc_0, ii, jj);\n }\n\n void KERNEL_4x8(int64_t ii, int64_t jj) {\n vec_t vec_A[4], vec_B[8], vec_C[4];\n acc_t acc_0, acc_1;\n __builtin_mma_xxsetaccz(&acc_0);\n __builtin_mma_xxsetaccz(&acc_1);\n for (int64_t l = 0; l < k; l += 4) {\n packTranspose(A + (ii * lda) + l, lda, 4, 4, (float *)vec_A);\n packTranspose(B + (jj * ldb) + l, ldb, 8, 4, (float *)vec_B);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[0], (vec_t)vec_B[0]);\n __builtin_mma_xvf32gerpp(&acc_1, vec_A[0], (vec_t)vec_B[1]);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[1], (vec_t)vec_B[2]);\n __builtin_mma_xvf32gerpp(&acc_1, vec_A[1], (vec_t)vec_B[3]);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[2], (vec_t)vec_B[4]);\n __builtin_mma_xvf32gerpp(&acc_1, vec_A[2], (vec_t)vec_B[5]);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[3], (vec_t)vec_B[6]);\n __builtin_mma_xvf32gerpp(&acc_1, vec_A[3], (vec_t)vec_B[7]);\n }\n save_acc(&acc_0, ii, jj);\n save_acc(&acc_1, ii, jj + 4);\n }\n\n void KERNEL_8x4(int64_t ii, int64_t jj) {\n vec_t vec_A[8], vec_B[4], vec_C[4];\n acc_t acc_0, acc_1;\n __builtin_mma_xxsetaccz(&acc_0);\n __builtin_mma_xxsetaccz(&acc_1);\n for (int64_t l = 0; l < k; l += 4) {\n packTranspose(A + (ii * lda) + l, lda, 8, 4, (float *)vec_A);\n packTranspose(B + (jj * ldb) + l, ldb, 4, 4, (float *)vec_B);\n __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[0], vec_B[0]);\n __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[1], vec_B[0]);\n __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[2], vec_B[1]);\n __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[3], vec_B[1]);\n __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[4], vec_B[2]);\n __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[5], vec_B[2]);\n __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[6], vec_B[3]);\n __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[7], vec_B[3]);\n }\n save_acc(&acc_0, ii, jj);\n save_acc(&acc_1, ii + 4, jj);\n }\n\n void KERNEL_8x8(int64_t ii, int64_t jj) {\n vec_t vec_A[16], vec_B[16], vec_C[4];\n acc_t acc_0, acc_1, acc_2, acc_3;\n __builtin_mma_xxsetaccz(&acc_0);\n __builtin_mma_xxsetaccz(&acc_1);\n __builtin_mma_xxsetaccz(&acc_2);\n __builtin_mma_xxsetaccz(&acc_3);\n for (int l = 0; l < k; l+=8) {\n packTranspose(A + (ii * lda) + l, lda, 8, 8, (float *)vec_A);\n packTranspose(B + (jj * ldb) + l, ldb, 8, 8, (float *)vec_B);\n for(int x = 0; x < 16; x+=2) {\n __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[x], vec_B[x]);\n __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[x], vec_B[x + 1]);\n __builtin_mma_xvf32gerpp(&acc_2, (vec_t)vec_A[x + 1], vec_B[x]);\n __builtin_mma_xvf32gerpp(&acc_3, (vec_t)vec_A[x + 1], vec_B[x + 1]);\n }\n }\n save_acc(&acc_0, ii, jj);\n save_acc(&acc_1, ii, jj + 4);\n save_acc(&acc_2, ii + 4, jj);\n save_acc(&acc_3, ii + 4, jj + 4);\n }\n\n inline void MMA_16x8(vec_t * vec_A0, vec_t * vec_A1, vec_t * vec_B, acc_t * acc) {\n for (int x = 0; x < 16; x += 2) {\n __builtin_mma_xvf32gerpp(&acc[0], vec_A0[x + 0], vec_B[x]);\n __builtin_mma_xvf32gerpp(&acc[1], vec_A0[x + 0], vec_B[x + 1]);\n __builtin_mma_xvf32gerpp(&acc[2], vec_A0[x + 1], vec_B[x]);\n __builtin_mma_xvf32gerpp(&acc[3], vec_A0[x + 1], vec_B[x + 1]);\n __builtin_mma_xvf32gerpp(&acc[4], vec_A1[x + 0], vec_B[x]);\n __builtin_mma_xvf32gerpp(&acc[5], vec_A1[x + 0], vec_B[x + 1]);\n __builtin_mma_xvf32gerpp(&acc[6], vec_A1[x + 1], vec_B[x]);\n __builtin_mma_xvf32gerpp(&acc[7], vec_A1[x + 1], vec_B[x + 1]);\n }\n }\n\n void KERNEL(int64_t ii, int64_t jj, int64_t mc, int64_t nc, int64_t kc, vec_t * vec_A, vec_t * vec_B, int64_t kk) {\n for (int64_t i = 0; i < mc; i += 16) {\n int A_base_addr = (mc / 8) * (i / 8) * 16;\n for (int64_t j = 0; j < nc; j += 8) {\n int B_base_addr = (nc / 8) * (j / 8) * 16;\n acc_t acc[8];\n vec_t A0_block[16]; vec_t A1_block[16];\n for (int x = 0; x < 8; x++)\n __builtin_mma_xxsetaccz(&acc[x]);\n for (int64_t l = 0; l < kc; l += 8) {\n int A0_block_idx = A_base_addr + (l / 8) * 16;\n int A1_block_idx = A0_block_idx + (mc / 8) * 16;\n int B_block_idx = B_base_addr + (l / 8) * 16;\n vec_t* A0_block = &vec_A[A0_block_idx];\n vec_t* A1_block = &vec_A[A1_block_idx];\n vec_t* B_block = &vec_B[B_block_idx];\n MMA_16x8(A0_block, A1_block, B_block, acc);\n }\n if (kk == 0) {\n save_acc(&acc[0], ii + i, jj + j);\n save_acc(&acc[1], ii + i, jj + j + 4);\n save_acc(&acc[2], ii + i + 4, jj + j);\n save_acc(&acc[3], ii + i + 4, jj + j + 4);\n save_acc(&acc[4], ii + i + 8, jj + j);\n save_acc(&acc[5], ii + i + 8, jj + j + 4);\n save_acc(&acc[6], ii + i + 12, jj + j);\n save_acc(&acc[7], ii + i + 12, jj + j + 4);\n } else {\n add_save_acc(&acc[0], ii + i, jj + j);\n add_save_acc(&acc[1], ii + i, jj + j + 4);\n add_save_acc(&acc[2], ii + i + 4, jj + j);\n add_save_acc(&acc[3], ii + i + 4, jj + j + 4);\n add_save_acc(&acc[4], ii + i + 8, jj + j);\n add_save_acc(&acc[5], ii + i + 8, jj + j + 4);\n add_save_acc(&acc[6], ii + i + 12, jj + j);\n add_save_acc(&acc[7], ii + i + 12, jj + j + 4);\n }\n }\n }\n }\n\n void matmul_tiled(int64_t m , int64_t n, int64_t mc, int64_t nc, int64_t kc) {\n int64_t ytiles = m / mc;\n int64_t xtiles = n / nc;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles) {\n end = tiles;\n }\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = (job / xtiles) * mc;\n int64_t jj = (job % xtiles) * nc;\n for (int64_t kk = 0; kk < k; kk += kc) {\n vec_t A_pack[kc * mc / 4];\n vec_t B_pack[kc * nc / 4];\n packTranspose(A + (ii * lda) + kk, lda, kc, mc, (float *)A_pack);\n packTranspose(B + (jj * ldb) + kk, ldb, kc, nc, (float *)B_pack);\n KERNEL(ii, jj, mc, nc, kc, A_pack, B_pack, kk);\n }\n }\n }\n\n void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int m_rem = MIN(m - m0, 8);\n int n_rem = MIN(n - n0, 8);\n int mc = 0, nc = 0;\n if (m_rem >= 8 && n_rem >= 8) {\n mc = 8;\n nc = 8;\n gemm<8, 8>(m0, m, n0, n);\n } else if (m_rem >= 4 && n_rem >= 8) {\n mc = 4;\n nc = 8;\n gemm<4, 8>(m0, m, n0, n);\n } else if (m_rem >= 8 && n_rem >= 4) {\n mc = 8;\n nc = 4;\n gemm<8, 4>(m0, m, n0, n);\n } else if (m_rem >= 4 && n_rem >= 4) {\n mc = 4;\n nc = 4;\n gemm<4, 4>(m0, m, n0, n);\n } else {\n mc = (m_rem >= 4) ? 4 : m_rem;\n nc = (n_rem >= 4) ? 4 : n_rem;\n if (mc == 0 || nc == 0)\n return;\n gemm_small(m0, m, n0, n, mc, nc);\n }\n int64_t mp = m0 + ((m - m0) / mc) * mc;\n int64_t np = n0 + ((n - n0) / nc) * nc;\n mnpack(mp, m, n0, np);\n mnpack(m0, m, np, n);\n }\n\n void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n, int RM, int RN) {\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * RN;\n vec_t vec_C[4];\n acc_t acc_0;\n __builtin_mma_xxsetaccz(&acc_0);\n vec_t vec_A[4] = {0}, vec_B[4] = {0};\n for (int l = 0; l < k; l += 4) {\n /* 'GEMV Forwarding' concept is used in first two conditional loops.\n * when one of the matrix has a single row/column, the elements are\n * broadcasted, instead of using packing routine to prepack the\n * matrix elements.\n */\n if (RM == 1) {\n float * a = const_cast(A + (ii) * lda + l);\n packTranspose(B + (jj * ldb) + l, ldb, RN, 4, (float *)vec_B);\n vec_A[0] = (vec_t)vec_xl(0,a);\n vec_A[1] = (vec_t)vec_splats(*((float *)&vec_A+1));\n vec_A[2] = (vec_t)vec_splats(*((float *)&vec_A+2));\n vec_A[3] = (vec_t)vec_splats(*((float *)&vec_A+3));\n } else if (RN == 1) {\n packTranspose(A + (ii * lda) + l, lda, RM, 4, (float *)vec_A);\n float * b = const_cast(B + (jj) * ldb + l);\n vec_B[0] = (vec_t)vec_xl(0,b);\n vec_B[1] = (vec_t)vec_splats(*((float *)&vec_B+1));\n vec_B[2] = (vec_t)vec_splats(*((float *)&vec_B+2));\n vec_B[3] = (vec_t)vec_splats(*((float *)&vec_B+3));\n } else {\n packTranspose(A + (ii * lda) + l, lda, RM, 4, (float *)vec_A);\n packTranspose(B + (jj * ldb) + l, ldb, RN, 4, (float *)vec_B);\n }\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[0], vec_B[0]);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[1], vec_B[1]);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[2], vec_B[2]);\n __builtin_mma_xvf32gerpp(&acc_0, vec_A[3], vec_B[3]);\n }\n __builtin_mma_disassemble_acc(vec_C, &acc_0);\n for (int I = 0; I < RM; I++) {\n for (int J = 0; J < RN; J++) {\n *((float *)(C+ii+((jj+J)*ldc)+I)) = *((float *)&vec_C[I]+J);\n }\n }\n }\n }\n\n template\n inline void kernel(int64_t ii, int64_t jj) {\n if constexpr(RM == 4 && RN == 4) {\n KERNEL_4x4(ii, jj);\n } else if constexpr(RM == 4 && RN == 8) {\n KERNEL_4x8(ii, jj);\n } else if constexpr(RM == 8 && RN == 4) {\n KERNEL_8x4(ii, jj);\n } else if constexpr(RM == 8 && RN == 8) {\n KERNEL_8x8(ii, jj);\n } else {\n static_assert(false, \"RN/RM values not supported\");\n }\n }\n\n template \n NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n int64_t ytiles = (m - m0) / RM;\n int64_t xtiles = (n - n0) / RN;\n int64_t tiles = xtiles * ytiles;\n int64_t duty = (tiles + nth - 1) / nth;\n int64_t start = duty * ith;\n int64_t end = start + duty;\n if (end > tiles)\n end = tiles;\n for (int64_t job = start; job < end; ++job) {\n int64_t ii = m0 + job / xtiles * RM;\n int64_t jj = n0 + job % xtiles * RN;\n kernel(ii, jj);\n }\n }\n\n const float * const A;\n const float * const B;\n float * C;\n const int64_t k;\n const int64_t lda;\n const int64_t ldb;\n const int64_t ldc;\n const int ith;\n const int nth;\n};\n#endif\n} // namespace\n\n/**\n * Performs optimized matrix multiplication on CPU.\n *\n * This subroutine may compute C = Aᵀ * B with column major ordering.\n * Despite its name, this isn't a generalized implementation. Work is\n * only performed when a handwritten kernel is written and available.\n * Otherwise the caller should fall back to a general matmul routine.\n *\n * For example, for single-threaded single-precision GEMM you can say\n *\n * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc,\n * 0, 1,\n * GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);\n *\n * @param m is rows in `A` and `C`\n * @param n is cols in `B` and `C`\n * @param k is cols in `A` and rows in `B`\n * @param A is first input matrix (always transposed)\n * @param lda is row stride of `A`\n * @param B is second input matrix (never transposed)\n * @param ldb is row stride of `B`\n * @param C is input/output array of output matrices\n * @param ldc is row stride of `C`\n * @param ith is thread id (must be less than `nth`)\n * @param nth is number of threads (must be greater than zero)\n * @param Atype is GGML data type of `A`\n * @param Btype is GGML data type of `B`\n * @param Ctype is GGML data type of `C`\n * @return true if this function was able to service the matmul request\n */\nbool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64_t n, int64_t k,\n const void *A, int64_t lda, const void *B, int64_t ldb, void *C,\n int64_t ldc, int Atype, int Btype, int Ctype) {\n\n assert(m >= 0);\n assert(n >= 0);\n assert(k >= 0);\n assert(lda >= k);\n assert(ldb >= k);\n assert(ldc >= m);\n assert(params->nth > 0);\n assert(params->ith < params->nth);\n\n // only enable sgemm for prompt processing\n#if !defined(__MMA__)\n if (n < 2)\n return false;\n#endif\n\n if (Ctype != GGML_TYPE_F32)\n return false;\n\n switch (Atype) {\n\n case GGML_TYPE_F32: {\n if (Btype != GGML_TYPE_F32)\n return false;\n#if defined(__AVX512F__)\n tinyBLAS<16, __m512, __m512, float, float, float> tb{ params,\n k, (const float *)A, lda,\n (const float *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n#elif defined(__AVX__) || defined(__AVX2__)\n tinyBLAS<8, __m256, __m256, float, float, float> tb{ params,\n k, (const float *)A, lda,\n (const float *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n#elif defined(__ARM_NEON)\n if (n < 4)\n return false;\n tinyBLAS<4, float32x4_t, float32x4_t, float, float, float> tb{ params,\n k, (const float *)A, lda,\n (const float *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n#elif defined(__VXE__) || defined(__VXE2__)\n if (n < 4)\n return false;\n tinyBLAS<4, float32x4_t, float32x4_t, float, float, float> tb{ params,\n k, (const float *)A, lda,\n (const float *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n#elif defined(__MMA__)\n if (k % 8)\n return false;\n tinyBLAS_PPC tb{\n k, (const float *)A, lda,\n (const float *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth};\n tb.matmul(m, n);\n return true;\n#elif defined(__riscv_zvfh)\n #if LMUL == 1\n tinyBLAS_RVV tb{ params,\n k, (const float *)A, lda,\n (const float *)B, ldb,\n (float *)C, ldc};\n #elif LMUL == 2\n tinyBLAS_RVV tb{ params,\n k, (const float *)A, lda,\n (const float *)B, ldb,\n (float *)C, ldc};\n #else // LMUL = 4\n tinyBLAS_RVV tb{ params,\n k, (const float *)A, lda,\n (const float *)B, ldb,\n (float *)C, ldc};\n #endif\n return tb.matmul(m, n);\n#else\n return false;\n#endif\n }\n\n case GGML_TYPE_BF16: {\n#if defined(__AVX512BF16__)\n if (Btype == GGML_TYPE_BF16) {\n tinyBLAS<32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,\n (const ggml_bf16_t *)A, lda,\n (const ggml_bf16_t *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n }\n#elif defined(__AVX512F__)\n if (Btype == GGML_TYPE_BF16) {\n tinyBLAS<16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,\n (const ggml_bf16_t *)A, lda,\n (const ggml_bf16_t *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n }\n#elif defined(__AVX2__)\n if (Btype == GGML_TYPE_BF16) {\n tinyBLAS<8, __m256, __m256, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,\n (const ggml_bf16_t *)A, lda,\n (const ggml_bf16_t *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n }\n#elif defined(__MMA__)\n if (k % 8) {\n return false;\n }\n\n if (Btype == GGML_TYPE_BF16) {\n tinyBLAS_HP16_PPC tb{ k,\n (const ggml_bf16_t *)A, lda,\n (const ggml_bf16_t *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth };\n\n tb.matmul(m, n);\n return true;\n }\n#elif defined(__riscv_zvfbfwma)\n #if LMUL == 1\n tinyBLAS_RVV tb{ params,\n k, (const ggml_bf16_t *)A, lda,\n (const ggml_bf16_t *)B, ldb,\n (float *)C, ldc};\n #elif LMUL == 2\n tinyBLAS_RVV tb{ params,\n k, (const ggml_bf16_t *)A, lda,\n (const ggml_bf16_t *)B, ldb,\n (float *)C, ldc};\n #else // LMUL = 4\n tinyBLAS_RVV tb{ params,\n k, (const ggml_bf16_t *)A, lda,\n (const ggml_bf16_t *)B, ldb,\n (float *)C, ldc};\n #endif\n return tb.matmul(m, n);\n#endif\n return false;\n }\n\n case GGML_TYPE_F16: {\n#if defined(__AVX512F__)\n if (Btype == GGML_TYPE_F16) {\n tinyBLAS<16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float> tb{ params, k,\n (const ggml_fp16_t *)A, lda,\n (const ggml_fp16_t *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n }\n#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)\n if (Btype == GGML_TYPE_F16) {\n tinyBLAS<8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{ params, k,\n (const ggml_fp16_t *)A, lda,\n (const ggml_fp16_t *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n }\n#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)\n if (n < 8)\n return false;\n if (Btype == GGML_TYPE_F16) {\n tinyBLAS<8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{ params,\n k, (const ggml_fp16_t *)A, lda,\n (const ggml_fp16_t *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n }\n#elif defined(__ARM_NEON) && !defined(_MSC_VER)\n if (Btype == GGML_TYPE_F32) {\n tinyBLAS<4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{ params,\n k, (const ggml_fp16_t *)A, lda,\n (const float *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n }\n#elif defined(__VXE__) || defined(__VXE2__)\n if (n < 4)\n return false;\n if (Btype == GGML_TYPE_F16) {\n tinyBLAS<4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, float> tb{ params,\n k, (const ggml_fp16_t *)A, lda,\n (const ggml_fp16_t *)B, ldb,\n (float *)C, ldc};\n return tb.matmul(m, n);\n }\n#elif defined(__riscv_zvfh)\n if (Btype == GGML_TYPE_F16) {\n #if LMUL == 1\n tinyBLAS_RVV tb{ params,\n k, (const ggml_fp16_t *)A, lda,\n (const ggml_fp16_t *)B, ldb,\n (float *)C, ldc};\n #elif LMUL == 2\n tinyBLAS_RVV tb{ params,\n k, (const ggml_fp16_t *)A, lda,\n (const ggml_fp16_t *)B, ldb,\n (float *)C, ldc};\n #else // LMUL = 4\n tinyBLAS_RVV tb{ params,\n k, (const ggml_fp16_t *)A, lda,\n (const ggml_fp16_t *)B, ldb,\n (float *)C, ldc};\n #endif\n return tb.matmul(m, n);\n }\n#elif defined(__MMA__)\n if (k % 8) {\n return false;\n }\n\n if (Btype == GGML_TYPE_F16) {\n tinyBLAS_HP16_PPC tb{ k,\n (const ggml_fp16_t *)A, lda,\n (const ggml_fp16_t *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth };\n\n tb.matmul(m, n);\n return true;\n }\n#endif\n return false;\n }\n\n case GGML_TYPE_Q8_0: {\n if (Btype != GGML_TYPE_Q8_0)\n return false;\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\n tinyBLAS_Q0_AVX tb{\n k, (const block_q8_0 *)A, lda,\n (const block_q8_0 *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth};\n tb.matmul(m, n);\n return true;\n#elif defined(__ARM_FEATURE_DOTPROD)\n tinyBLAS_Q0_ARM tb{\n k, (const block_q8_0 *)A, lda,\n (const block_q8_0 *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth};\n tb.matmul(m, n);\n return true;\n#elif defined(__MMA__)\n //TO-DO: Remove this condition once gemv forwarding is enabled.\n if (n < 8 && n != 4)\n return false;\n if (m < 8 && m != 4)\n return false;\n tinyBLAS_Q0_PPC tb{\n k, (const block_q8_0 *)A, lda,\n (const block_q8_0 *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth};\n tb.matmul(m, n);\n return true;\n#else\n return false;\n#endif\n }\n\n case GGML_TYPE_Q4_0: {\n if (Btype != GGML_TYPE_Q8_0)\n return false;\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\n tinyBLAS_Q0_AVX tb{\n k, (const block_q4_0 *)A, lda,\n (const block_q8_0 *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth};\n tb.matmul(m, n);\n return true;\n#elif defined(__ARM_FEATURE_DOTPROD)\n tinyBLAS_Q0_ARM tb{\n k, (const block_q4_0 *)A, lda,\n (const block_q8_0 *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth};\n tb.matmul(m, n);\n return true;\n#elif defined(__MMA__)\n //TO-DO: Remove this condition once gemv forwarding is enabled.\n if (n < 8 && n != 4)\n return false;\n if (m < 8 && m != 4)\n return false;\n tinyBLAS_Q0_PPC tb{\n k, (const block_q4_0 *)A, lda,\n (const block_q8_0 *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth};\n tb.matmul(m, n);\n return true;\n#else\n return false;\n#endif\n }\n\n case GGML_TYPE_Q5_0: {\n if (Btype != GGML_TYPE_Q8_0)\n return false;\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\n tinyBLAS_Q0_AVX tb{\n k, (const block_q5_0 *)A, lda,\n (const block_q8_0 *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth};\n tb.matmul(m, n);\n return true;\n#else\n return false;\n#endif\n }\n\n case GGML_TYPE_IQ4_NL: {\n if (Btype != GGML_TYPE_Q8_0)\n return false;\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\n tinyBLAS_Q0_AVX tb{\n k, (const block_iq4_nl *)A, lda,\n (const block_q8_0 *)B, ldb,\n (float *)C, ldc,\n params->ith, params->nth};\n tb.matmul(m, n);\n return true;\n#else\n return false;\n#endif\n }\n\n default:\n return false;\n }\n\n (void)params;\n (void)m;\n (void)n;\n (void)k;\n (void)A;\n (void)lda;\n (void)B;\n (void)ldb;\n (void)C;\n (void)ldc;\n (void)Atype;\n (void)Btype;\n (void)Ctype;\n}\n" }, { "path": "src/ggml-cpu/llamafile/sgemm.h", "content": "#pragma once\n#include \n#include \n\n#if defined(__VXE__) || defined(__VXE2__)\n#include \n#endif\n\n#ifdef _MSC_VER\n#define NOINLINE __declspec(noinline)\n#else\n#define NOINLINE __attribute__((__noinline__))\n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nbool llamafile_sgemm(const struct ggml_compute_params * params, int64_t, int64_t, int64_t,\n const void *, int64_t, const void *, int64_t, void *, int64_t,\n int, int, int);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cpu/ops.cpp", "content": "#include \"ops.h\"\n\n#include \"ggml-cpu.h\"\n#include \"ggml-impl.h\"\n#include \"binary-ops.h\"\n#include \"simd-gemm.h\"\n#include \"ggml.h\"\n#include \"unary-ops.h\"\n#include \"vec.h\"\n\n#include \n#include \n#include \n\n// ggml_compute_forward_dup\n\nstatic void ggml_compute_forward_dup_same_cont(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n GGML_ASSERT(src0->type == dst->type);\n\n const size_t nb0 = ggml_type_size(src0->type);\n\n const int ith = params->ith; // thread index\n const int nth = params->nth; // number of threads\n\n // parallelize by blocks\n const int nk = ggml_nelements(src0)/ggml_blck_size(src0->type);\n const int dr = (nk + nth - 1) / nth;\n const int k0 = dr * ith;\n const int k1 = MIN(k0 + dr, nk);\n\n if (k0 < k1) {\n memcpy(\n ((char *) dst->data + k0*nb0),\n ((char *) src0->data + k0*nb0),\n (k1 - k0) * nb0);\n }\n}\n\ntemplate\nstatic void ggml_compute_forward_dup_flt(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n GGML_ASSERT(!ggml_is_quantized(src0->type) && !ggml_is_quantized(dst->type));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n const int ith = params->ith; // thread index\n const int nth = params->nth; // number of threads\n\n // parallelize by rows\n const int nr = ne01;\n // number of rows per thread\n const int dr = (nr + nth - 1) / nth;\n // row range for this thread\n const int ir0 = dr * ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n // case: type & row size equal\n if (src0->type == dst->type &&\n ne00 == ne0 &&\n nb00 == ggml_type_size(src0->type) && nb0 == ggml_type_size(dst->type)) {\n // copy by rows\n const size_t rs = ne00*nb00;\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = ir0; i01 < ir1; i01++) {\n memcpy(\n ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),\n ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),\n rs);\n }\n }\n }\n return;\n }\n\n // case: dst tensor is contiguous\n if (ggml_is_contiguous(dst)) {\n if (nb00 == sizeof(src_t)) {\n if constexpr (std::is_same_v) {\n // same type\n size_t id = 0;\n const size_t rs = ne00 * nb00;\n char * dst_ptr = (char *) dst->data;\n\n for (int i03 = 0; i03 < ne03; i03++) {\n for (int i02 = 0; i02 < ne02; i02++) {\n id += rs * ir0;\n for (int i01 = ir0; i01 < ir1; i01++) {\n const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;\n memcpy(dst_ptr + id, src0_ptr, rs);\n id += rs;\n }\n id += rs * (ne01 - ir1);\n }\n }\n } else {\n // casting between non-quantized types\n size_t id = 0;\n dst_t * dst_ptr = (dst_t *) dst->data;\n\n for (int i03 = 0; i03 < ne03; i03++) {\n for (int i02 = 0; i02 < ne02; i02++) {\n id += ne00 * ir0;\n for (int i01 = ir0; i01 < ir1; i01++) {\n const src_t * src0_ptr = (src_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n for (int i00 = 0; i00 < ne00; i00++) {\n float tmp = type_conversion_table::to_f32(src0_ptr[i00]);\n dst_ptr[id] = type_conversion_table::from_f32(tmp);\n id++;\n }\n }\n id += ne00 * (ne01 - ir1);\n }\n }\n }\n } else {\n //printf(\"%s: this is not optimal - fix me\\n\", __func__);\n\n size_t id = 0;\n dst_t * dst_ptr = (dst_t *) dst->data;\n\n for (int i03 = 0; i03 < ne03; i03++) {\n for (int i02 = 0; i02 < ne02; i02++) {\n id += ne00 * ir0;\n for (int i01 = ir0; i01 < ir1; i01++) {\n for (int i00 = 0; i00 < ne00; i00++) {\n const src_t * src0_ptr = (src_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n\n float tmp = type_conversion_table::to_f32(*src0_ptr);\n dst_ptr[id] = type_conversion_table::from_f32(tmp);\n id++;\n }\n }\n id += ne00 * (ne01 - ir1);\n }\n }\n }\n return;\n }\n\n // dst counters\n int64_t i10 = 0;\n int64_t i11 = 0;\n int64_t i12 = 0;\n int64_t i13 = 0;\n\n if constexpr (std::is_same_v) {\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n i10 += ne00 * ir0;\n while (i10 >= ne0) {\n i10 -= ne0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n for (int64_t i01 = ir0; i01 < ir1; i01++) {\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);\n\n memcpy(dst_ptr, src0_ptr, sizeof(dst_t));\n\n if (++i10 == ne00) {\n i10 = 0;\n if (++i11 == ne01) {\n i11 = 0;\n if (++i12 == ne02) {\n i12 = 0;\n if (++i13 == ne03) {\n i13 = 0;\n }\n }\n }\n }\n }\n }\n i10 += ne00 * (ne01 - ir1);\n while (i10 >= ne0) {\n i10 -= ne0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n }\n }\n\n } else {\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n i10 += ne00 * ir0;\n while (i10 >= ne0) {\n i10 -= ne0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n for (int64_t i01 = ir0; i01 < ir1; i01++) {\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);\n\n float tmp = type_conversion_table::to_f32(*(const src_t *) src0_ptr);\n *(dst_t *) dst_ptr = type_conversion_table::from_f32(tmp);\n\n if (++i10 == ne0) {\n i10 = 0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n }\n }\n i10 += ne00 * (ne01 - ir1);\n while (i10 >= ne0) {\n i10 -= ne0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n }\n }\n }\n}\n\n\ntemplate\nstatic void ggml_compute_forward_dup_to_q(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n GGML_ASSERT(!ggml_is_quantized(src0->type));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n const int ith = params->ith; // thread index\n const int nth = params->nth; // number of threads\n\n // parallelize by rows\n const int nr = ne01;\n // number of rows per thread\n const int dr = (nr + nth - 1) / nth;\n // row range for this thread\n const int ir0 = dr * ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n if (ggml_is_contiguous(dst) &&\n nb00 == sizeof(src_t) &&\n ggml_get_type_traits_cpu(dst->type)->from_float) {\n // casting non-quantized types --> intermediate f32 --> quantized\n ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(dst->type)->from_float;\n float * src0_f32 = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;\n\n size_t id = 0;\n size_t rs = nb0 * (ne00 / ggml_blck_size(dst->type));\n char * dst_ptr = (char *) dst->data;\n\n for (int i03 = 0; i03 < ne03; i03++) {\n for (int i02 = 0; i02 < ne02; i02++) {\n id += rs * ir0;\n for (int i01 = ir0; i01 < ir1; i01++) {\n const src_t * src0_ptr = (src_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n\n for (int i00 = 0; i00 < ne00; i00++) {\n src0_f32[i00] = type_conversion_table::to_f32(src0_ptr[i00]);\n }\n\n quantize_row_q(src0_f32, dst_ptr + id, ne00);\n id += rs;\n }\n id += rs * (ne01 - ir1);\n }\n }\n } else {\n // printf(\"%s %s\\n\", ggml_type_name(src0->type), ggml_type_name(dst->type));\n GGML_ABORT(\"not implemented\");\n }\n}\n\n// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.\nstatic void ggml_compute_forward_dup_bytes(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n GGML_ASSERT(src0->type == dst->type);\n\n GGML_TENSOR_UNARY_OP_LOCALS;\n\n if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {\n ggml_compute_forward_dup_same_cont(params, dst);\n return;\n }\n\n const size_t type_size = ggml_type_size(src0->type);\n\n const int ith = params->ith; // thread index\n const int nth = params->nth; // number of threads\n\n // parallelize by rows\n const int nr = ne01;\n // number of rows per thread\n const int dr = (nr + nth - 1) / nth;\n // row range for this thread\n const int ir0 = dr * ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n if (src0->type == dst->type &&\n ggml_are_same_shape(src0, dst) &&\n nb00 == type_size && nb0 == type_size) {\n // copy by rows\n const size_t rs = ggml_row_size(src0->type, ne00);\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = ir0; i01 < ir1; i01++) {\n memcpy(\n ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),\n ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),\n rs);\n }\n }\n }\n return;\n }\n\n if (ggml_is_contiguous(dst)) {\n size_t id = 0;\n char * dst_ptr = (char *) dst->data;\n const size_t rs = ne00 * type_size;\n\n if (nb00 == type_size) {\n // src0 is contiguous on first dimension, copy by rows\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n id += rs * ir0;\n for (int64_t i01 = ir0; i01 < ir1; i01++) {\n const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;\n memcpy(dst_ptr + id, src0_ptr, rs);\n id += rs;\n }\n id += rs * (ne01 - ir1);\n }\n }\n } else {\n //printf(\"%s: this is not optimal - fix me\\n\", __func__);\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n id += rs * ir0;\n for (int64_t i01 = ir0; i01 < ir1; i01++) {\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n const char * src0_ptr = (char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03;\n memcpy(dst_ptr + id, src0_ptr, type_size);\n\n id += type_size;\n }\n }\n id += rs * (ne01 - ir1);\n }\n }\n }\n\n return;\n }\n\n // dst counters\n int64_t k10 = 0;\n int64_t i11 = 0;\n int64_t i12 = 0;\n int64_t i13 = 0;\n\n // number of blocks in a row\n const int64_t nk00 = ne00 / ggml_blck_size(src0->type);\n const int64_t nk0 = ne0 / ggml_blck_size(dst->type);\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n k10 += nk00 * ir0;\n while (k10 >= nk0) {\n k10 -= nk0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n for (int64_t i01 = ir0; i01 < ir1; i01++) {\n for (int64_t k00 = 0; k00 < nk00; k00++) {\n const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n char * dst_ptr = ((char *) dst->data + k10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);\n\n memcpy(dst_ptr, src0_ptr, type_size);\n\n if (++k10 == nk0) {\n k10 = 0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n }\n }\n k10 += nk00 * (ne01 - ir1);\n while (k10 >= nk0) {\n k10 -= nk0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n }\n }\n}\n\nstatic void ggml_compute_forward_dup_from_q(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const ggml_type type = src0->type;\n ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n\n size_t qk = ggml_blck_size(type);\n const int64_t nr = ggml_nelements(src1) / qk;\n\n // destination must be contiguous in the first dimension\n GGML_ASSERT(nb10 == ggml_type_size(dst->type));\n // must either have first dimension large enough to hold a row, or fully contiguous\n GGML_ASSERT((ne10 % qk) == 0 || ggml_is_contiguous(dst));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n\n uint32_t i = ir * qk;\n\n const int64_t i03 = i/(ne00 * ne01 * ne02);\n const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);\n const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;\n const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;\n const int64_t x_offset = (i00/qk)*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;\n\n const int64_t i13 = i/(ne10 * ne11 * ne12);\n const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);\n const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;\n const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;\n const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13;\n\n dequantize_row_q(\n (const void *) ((char *) src0->data + x_offset),\n (float *) ((char *) dst->data + dst_offset), qk);\n }\n}\n\nvoid ggml_compute_forward_dup(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (src0->type == dst->type) {\n ggml_compute_forward_dup_bytes(params, dst);\n return;\n }\n\n switch (src0->type) {\n case GGML_TYPE_F16:\n {\n /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt(params, dst);\n else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt(params, dst);\n else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt(params, dst);\n else ggml_compute_forward_dup_to_q(params, dst);\n } break;\n case GGML_TYPE_BF16:\n {\n /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt(params, dst);\n else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt(params, dst);\n else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt(params, dst);\n else ggml_compute_forward_dup_to_q(params, dst);\n } break;\n case GGML_TYPE_F32:\n {\n /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt(params, dst);\n else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt(params, dst);\n else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt(params, dst);\n else if (dst->type == GGML_TYPE_I32) ggml_compute_forward_dup_flt(params, dst);\n else ggml_compute_forward_dup_to_q(params, dst);\n } break;\n case GGML_TYPE_I32:\n {\n if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt(params, dst);\n else GGML_ABORT(\"not implemented\");\n } break;\n default:\n {\n if (ggml_is_quantized(src0->type) && dst->type == GGML_TYPE_F32) {\n ggml_compute_forward_dup_from_q(params, dst);\n break;\n }\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_add\n\nstatic void ggml_compute_forward_add_q_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const ggml_type type = src0->type;\n const ggml_type dtype = dst->type;\n ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(dtype)->from_float;\n\n // we don't support permuted src0 or src1\n GGML_ASSERT(nb00 == ggml_type_size(type));\n GGML_ASSERT(nb10 == sizeof(float));\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n GGML_ASSERT(ggml_is_quantized(src0->type));\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n float * wdata = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 indices\n const int i03 = ir/(ne02*ne01);\n const int i02 = (ir - i03*ne02*ne01)/ne01;\n const int i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n // src1 and dst are same shape as src0 => same indices\n const int i13 = i03;\n const int i12 = i02;\n const int i11 = i01;\n\n const int i3 = i03;\n const int i2 = i02;\n const int i1 = i01;\n\n void * src0_row = (void *) ((char *) src0->data + (i01*nb01 + i02*nb02 + i03*nb03));\n float * src1_row = (float *)((char *) src1->data + (i11*nb11 + i12*nb12 + i13*nb13));\n void * dst_row = (void *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));\n\n assert(ne00 % 32 == 0);\n\n // unquantize row from src0 to temp buffer\n dequantize_row_q(src0_row, wdata, ne00);\n // add src1\n ggml_vec_acc_f32(ne00, wdata, src1_row);\n // quantize row to dst\n if (quantize_row_q != NULL) {\n quantize_row_q(wdata, dst_row, ne00);\n } else {\n memcpy(dst_row, wdata, ne0*nb0);\n }\n }\n}\n\nvoid ggml_compute_forward_add(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n {\n ggml_compute_forward_add_non_quantized(params, dst);\n } break;\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_NVFP4:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_TQ1_0:\n case GGML_TYPE_TQ2_0:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ2_S:\n {\n ggml_compute_forward_add_q_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_add_id\n\nstatic void ggml_compute_forward_add_id_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const ggml_tensor * src2 = dst->src[2];\n\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(src2->type == GGML_TYPE_I32);\n\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n GGML_ASSERT(src1->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_TERNARY_OP_LOCALS\n\n GGML_ASSERT( nb0 == sizeof(float));\n GGML_ASSERT(nb10 == sizeof(float));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 indices\n const int i3 = ir/(ne2*ne1);\n const int i2 = (ir - i3*ne2*ne1)/ne1;\n const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n // src1 indices\n const int i11 = *(int32_t *) ((char *) src2->data + i1*nb20 + i2*nb21);\n\n GGML_ASSERT(i11 >= 0 && i11 < ne11);\n\n ggml_vec_add_f32(ne0,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ),\n (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),\n (float *) ((char *) src1->data + i11*nb11));\n }\n}\n\nvoid ggml_compute_forward_add_id(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_add_id_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"unsupported type for ggml_compute_forward_add_id: %s\", ggml_type_name(src0->type));\n }\n }\n}\n\n// ggml_compute_forward_add1\n\nstatic void ggml_compute_forward_add1_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_is_scalar(src1));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT( nb0 == sizeof(float));\n GGML_ASSERT(nb00 == sizeof(float));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 and dst are same shape => same indices\n const int i3 = ir/(ne2*ne1);\n const int i2 = (ir - i3*ne2*ne1)/ne1;\n const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n#ifdef GGML_USE_ACCELERATE\n GGML_UNUSED(ggml_vec_add1_f32);\n\n vDSP_vadd(\n (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), 1,\n (float *) ((char *) src1->data), 0,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ), 1,\n ne0);\n#else\n ggml_vec_add1_f32(ne0,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ),\n (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),\n *(float *) src1->data);\n#endif\n }\n}\n\nstatic void ggml_compute_forward_add1_f16_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_is_scalar(src1));\n\n // scalar to add\n const float v = *(float *) src1->data;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT(src0->type == GGML_TYPE_F16);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F16);\n\n GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));\n GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 and dst are same shape => same indices\n const int i3 = ir/(ne2*ne1);\n const int i2 = (ir - i3*ne2*ne1)/ne1;\n const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );\n ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n for (int i = 0; i < ne0; i++) {\n dst_ptr[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(src0_ptr[i]) + v);\n }\n }\n}\n\nstatic void ggml_compute_forward_add1_f16_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_is_scalar(src1));\n\n // scalar to add\n const float v = GGML_CPU_FP16_TO_FP32(*(ggml_fp16_t *) src1->data);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT(src0->type == GGML_TYPE_F16);\n GGML_ASSERT(src1->type == GGML_TYPE_F16);\n GGML_ASSERT(dst->type == GGML_TYPE_F16);\n\n GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));\n GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 and dst are same shape => same indices\n const int i3 = ir/(ne2*ne1);\n const int i2 = (ir - i3*ne2*ne1)/ne1;\n const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );\n ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n for (int i = 0; i < ne0; i++) {\n dst_ptr[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(src0_ptr[i]) + v);\n }\n }\n}\n\nstatic void ggml_compute_forward_add1_q_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_is_scalar(src1));\n\n // scalar to add\n const float v = *(float *) src1->data;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n const ggml_type type = src0->type;\n ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(type)->from_float;\n\n // we don't support permuted src0\n GGML_ASSERT(nb00 == ggml_type_size(type));\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n GGML_ASSERT(ggml_is_quantized(src0->type));\n GGML_ASSERT(dst->type == src0->type);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith;\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 and dst are same shape => same indices\n const int i3 = ir/(ne2*ne1);\n const int i2 = (ir - i3*ne2*ne1)/ne1;\n const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n void * src0_row = (void *) ((char *) src0->data + (i1*nb01 + i2*nb02 + i3*nb03));\n void * dst_row = (void *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb0 ));\n\n assert(ne0 % 32 == 0);\n\n // unquantize row from src0 to temp buffer\n dequantize_row_q(src0_row, wdata, ne0);\n // add src1\n ggml_vec_acc1_f32(ne0, wdata, v);\n // quantize row to dst\n quantize_row_q(wdata, dst_row, ne0);\n }\n}\n\nstatic void ggml_compute_forward_add1_bf16_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_is_scalar(src1));\n\n // scalar to add\n const float v = *(float *) src1->data;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT(src0->type == GGML_TYPE_BF16);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_BF16);\n\n GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));\n GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 and dst are same shape => same indices\n const int i3 = ir/(ne2*ne1);\n const int i2 = (ir - i3*ne2*ne1)/ne1;\n const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );\n ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n for (int i = 0; i < ne0; i++) {\n dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);\n }\n }\n}\n\nstatic void ggml_compute_forward_add1_bf16_bf16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_is_scalar(src1));\n\n // scalar to add\n const float v = GGML_BF16_TO_FP32(*(ggml_bf16_t *) src1->data);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT(src0->type == GGML_TYPE_BF16);\n GGML_ASSERT(src1->type == GGML_TYPE_BF16);\n GGML_ASSERT(dst->type == GGML_TYPE_BF16);\n\n GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));\n GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 and dst are same shape => same indices\n const int i3 = ir/(ne2*ne1);\n const int i2 = (ir - i3*ne2*ne1)/ne1;\n const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );\n ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n for (int i = 0; i < ne0; i++) {\n dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);\n }\n }\n}\n\nvoid ggml_compute_forward_add1(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_add1_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n if (src1->type == GGML_TYPE_F16) {\n ggml_compute_forward_add1_f16_f16(params, dst);\n }\n else if (src1->type == GGML_TYPE_F32) {\n ggml_compute_forward_add1_f16_f32(params, dst);\n }\n else {\n GGML_ABORT(\"fatal error\");\n }\n } break;\n case GGML_TYPE_BF16:\n {\n if (src1->type == GGML_TYPE_BF16) {\n ggml_compute_forward_add1_bf16_bf16(params, dst);\n }\n else if (src1->type == GGML_TYPE_F32) {\n ggml_compute_forward_add1_bf16_f32(params, dst);\n }\n else {\n GGML_ABORT(\"fatal error\");\n }\n } break;\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q8_1:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_NVFP4:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_TQ1_0:\n case GGML_TYPE_TQ2_0:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ2_S:\n {\n ggml_compute_forward_add1_q_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_acc\n\nstatic void ggml_compute_forward_acc_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n\n // view src0 and dst with these strides and data offset inbytes during acc\n // nb0 is implicitly element_size because src0 and dst are contiguous\n size_t nb1 = ((int32_t *) dst->op_params)[0];\n size_t nb2 = ((int32_t *) dst->op_params)[1];\n size_t nb3 = ((int32_t *) dst->op_params)[2];\n size_t offset = ((int32_t *) dst->op_params)[3];\n bool inplace = (bool) ((int32_t *) dst->op_params)[4];\n\n if (!inplace) {\n if (params->ith == 0) {\n // memcpy needs to be synchronized across threads to avoid race conditions.\n // => do it in INIT phase\n memcpy(\n ((char *) dst->data),\n ((char *) src0->data),\n ggml_nbytes(dst));\n }\n ggml_barrier(params->threadpool);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src1);\n const int nc = src1->ne[0];\n\n GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)\n GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)\n\n // src0 and dst as viewed during acc\n const size_t nb0 = ggml_element_size(src0);\n\n const size_t nb00 = nb0;\n const size_t nb01 = nb1;\n const size_t nb02 = nb2;\n const size_t nb03 = nb3;\n\n GGML_ASSERT(offset + (ne10 == 0 ? 0 : ne10-1)*nb0 + (ne11 == 0 ? 0 : ne11-1)*nb1 + (ne12 == 0 ? 0 : ne12-1)*nb2 + (ne13 == 0 ? 0 : ne13-1)*nb3 < ggml_nbytes(dst));\n GGML_ASSERT(offset + (ne10 == 0 ? 0 : ne10-1)*nb00 + (ne11 == 0 ? 0 : ne11-1)*nb01 + (ne12 == 0 ? 0 : ne12-1)*nb02 + (ne13 == 0 ? 0 : ne13-1)*nb03 < ggml_nbytes(src0));\n\n GGML_ASSERT(nb10 == sizeof(float));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 and dst are viewed with shape of src1 and offset\n // => same indices\n const int i3 = ir/(ne12*ne11);\n const int i2 = (ir - i3*ne12*ne11)/ne11;\n const int i1 = (ir - i3*ne12*ne11 - i2*ne11);\n\n#ifdef GGML_USE_ACCELERATE\n vDSP_vadd(\n (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + offset), 1,\n (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11), 1,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset), 1, nc);\n#else\n ggml_vec_add_f32(nc,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset),\n (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + offset),\n (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));\n#endif\n }\n}\n\nvoid ggml_compute_forward_acc(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_acc_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q8_1:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_NVFP4:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_TQ1_0:\n case GGML_TYPE_TQ2_0:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ2_S:\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_sum\n\nstatic void ggml_compute_forward_sum_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n assert(ggml_is_scalar(dst));\n assert(src0->nb[0] == sizeof(float));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n\n ggml_float sum = 0;\n ggml_float row_sum = 0;\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n ggml_vec_sum_f32_ggf(ne00,\n &row_sum,\n (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));\n sum += row_sum;\n }\n }\n }\n ((float *) dst->data)[0] = sum;\n}\n\nstatic void ggml_compute_forward_sum_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n assert(ggml_is_scalar(dst));\n\n assert(src0->nb[0] == sizeof(ggml_fp16_t));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n\n float sum = 0;\n float row_sum = 0;\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n ggml_vec_sum_f16_ggf(ne00,\n &row_sum,\n (ggml_fp16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));\n sum += row_sum;\n }\n }\n }\n ((ggml_fp16_t *) dst->data)[0] = GGML_CPU_FP32_TO_FP16(sum);\n}\n\nstatic void ggml_compute_forward_sum_bf16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n assert(ggml_is_scalar(dst));\n\n assert(src0->nb[0] == sizeof(ggml_bf16_t));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n\n float sum = 0;\n float row_sum = 0;\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n ggml_vec_sum_bf16_ggf(ne00,\n &row_sum,\n (ggml_bf16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));\n sum += row_sum;\n }\n }\n }\n ((ggml_bf16_t *) dst->data)[0] = GGML_FP32_TO_BF16(sum);\n}\n\nvoid ggml_compute_forward_sum(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_sum_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_sum_f16(params, dst);\n } break;\n case GGML_TYPE_BF16:\n {\n ggml_compute_forward_sum_bf16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_cumsum\n\nstatic void ggml_compute_forward_cumsum_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n GGML_ASSERT(dst->nb[0] == sizeof(float));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT(ne0 == ne00);\n GGML_ASSERT(ne1 == ne01);\n GGML_ASSERT(ne2 == ne02);\n GGML_ASSERT(ne3 == ne03);\n\n const auto [ir0, ir1] = get_thread_range(params, src0);\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir/(ne02*ne01);\n const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n float * src_row = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n float * dst_row = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n\n ggml_vec_cumsum_f32(ne00, dst_row, src_row);\n }\n}\n\nvoid ggml_compute_forward_cumsum(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_cumsum_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_sum_rows\n\nstatic void ggml_compute_forward_sum_rows_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n GGML_ASSERT(dst->nb[0] == sizeof(float));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT(ne0 == 1);\n GGML_ASSERT(ne1 == ne01);\n GGML_ASSERT(ne2 == ne02);\n GGML_ASSERT(ne3 == ne03);\n\n for (int64_t i3 = 0; i3 < ne03; i3++) {\n for (int64_t i2 = 0; i2 < ne02; i2++) {\n for (int64_t i1 = 0; i1 < ne01; i1++) {\n float * src_row = (float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);\n float * dst_row = (float *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);\n float row_sum = 0;\n ggml_vec_sum_f32(ne00, &row_sum, src_row);\n dst_row[0] = row_sum;\n }\n }\n }\n}\n\nvoid ggml_compute_forward_sum_rows(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_sum_rows_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_mean\n\nstatic void ggml_compute_forward_mean_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n assert(src0->nb[0] == sizeof(float));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n assert(ne0 == 1);\n assert(ne1 == ne01);\n assert(ne2 == ne02);\n assert(ne3 == ne03);\n\n GGML_UNUSED(ne0);\n GGML_UNUSED(ne1);\n GGML_UNUSED(ne2);\n GGML_UNUSED(ne3);\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n ggml_vec_sum_f32(ne00,\n (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),\n (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));\n\n *(float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3) /= (float) ne00;\n }\n }\n }\n}\n\nvoid ggml_compute_forward_mean(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_mean_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_argmax\n\nstatic void ggml_compute_forward_argmax_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n assert(src0->nb[0] == sizeof(float));\n assert(dst->nb[0] == sizeof(float));\n\n const int64_t ne00 = src0->ne[0];\n const int64_t ne01 = src0->ne[1];\n\n const size_t nb01 = src0->nb[1];\n const size_t nb0 = dst->nb[0];\n\n for (int64_t i1 = 0; i1 < ne01; i1++) {\n float * src = (float *) ((char *) src0->data + i1*nb01);\n int32_t * dst_ = (int32_t *) ((char *) dst->data + i1*nb0);\n int v = 0;\n ggml_vec_argmax_f32(ne00, &v, src);\n dst_[0] = v;\n }\n}\n\nvoid ggml_compute_forward_argmax(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_argmax_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_count_equal\n\nstatic void ggml_compute_forward_count_equal_i32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n GGML_ASSERT(src0->type == GGML_TYPE_I32);\n GGML_ASSERT(src1->type == GGML_TYPE_I32);\n GGML_ASSERT(ggml_are_same_shape(src0, src1));\n GGML_ASSERT(ggml_is_scalar(dst));\n GGML_ASSERT(dst->type == GGML_TYPE_I64);\n\n const int64_t nr = ggml_nrows(src0);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n int64_t * sums = (int64_t *) params->wdata;\n int64_t sum_thread = 0;\n\n // rows per thread\n const int64_t dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int64_t ir0 = dr*ith;\n const int64_t ir1 = MIN(ir0 + dr, nr);\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir / (ne02*ne01);\n const int64_t i02 = (ir - i03*ne03) / ne01;\n const int64_t i01 = ir - i03*ne03 - i02*ne02;\n\n const char * data0 = (const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01;\n const char * data1 = (const char *) src1->data + i03*nb13 + i02*nb12 + i01*nb11;\n\n for (int64_t i00 = 0; i00 < ne00; ++i00) {\n const int32_t val0 = *((const int32_t *) (data0 + i00*nb00));\n const int32_t val1 = *((const int32_t *) (data1 + i00*nb10));\n\n sum_thread += val0 == val1;\n }\n }\n if (ith != 0) {\n sums[ith] = sum_thread;\n }\n ggml_barrier(params->threadpool);\n\n if (ith != 0) {\n return;\n }\n\n for (int ith_other = 1; ith_other < nth; ++ith_other) {\n sum_thread += sums[ith_other];\n }\n *((int64_t *) dst->data) = sum_thread;\n}\n\nvoid ggml_compute_forward_count_equal(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_I32:\n {\n ggml_compute_forward_count_equal_i32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_repeat\n\nstatic void ggml_compute_forward_repeat_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n GGML_ASSERT(ggml_can_repeat(src0, dst));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n // guaranteed to be an integer due to the check in ggml_can_repeat\n const int nr0 = (int)(ne0/ne00);\n const int nr1 = (int)(ne1/ne01);\n const int nr2 = (int)(ne2/ne02);\n const int nr3 = (int)(ne3/ne03);\n\n // TODO: support for transposed / permuted tensors\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb00 == sizeof(float));\n\n // TODO: maybe this is not optimal?\n for (int i3 = 0; i3 < nr3; i3++) {\n for (int k3 = 0; k3 < ne03; k3++) {\n for (int i2 = 0; i2 < nr2; i2++) {\n for (int k2 = 0; k2 < ne02; k2++) {\n for (int i1 = 0; i1 < nr1; i1++) {\n for (int k1 = 0; k1 < ne01; k1++) {\n for (int i0 = 0; i0 < nr0; i0++) {\n ggml_vec_cpy_f32(ne00,\n (float *) ((char *) dst->data + (i3*ne03 + k3)*nb3 + (i2*ne02 + k2)*nb2 + (i1*ne01 + k1)*nb1 + (i0*ne00)*nb0),\n (float *) ((char *) src0->data + ( k3)*nb03 + ( k2)*nb02 + ( k1)*nb01));\n }\n }\n }\n }\n }\n }\n }\n}\n\nstatic void ggml_compute_forward_repeat_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n GGML_ASSERT(ggml_can_repeat(src0, dst));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n // guaranteed to be an integer due to the check in ggml_can_repeat\n const int nr0 = (int)(ne0/ne00);\n const int nr1 = (int)(ne1/ne01);\n const int nr2 = (int)(ne2/ne02);\n const int nr3 = (int)(ne3/ne03);\n\n // TODO: support for transposed / permuted tensors\n GGML_ASSERT(nb0 == sizeof(ggml_fp16_t));\n GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n\n // TODO: maybe this is not optimal?\n for (int i3 = 0; i3 < nr3; i3++) {\n for (int k3 = 0; k3 < ne03; k3++) {\n for (int i2 = 0; i2 < nr2; i2++) {\n for (int k2 = 0; k2 < ne02; k2++) {\n for (int i1 = 0; i1 < nr1; i1++) {\n for (int k1 = 0; k1 < ne01; k1++) {\n for (int i0 = 0; i0 < nr0; i0++) {\n ggml_fp16_t * y = (ggml_fp16_t *) ((char *) dst->data + (i3*ne03 + k3)*nb3 + (i2*ne02 + k2)*nb2 + (i1*ne01 + k1)*nb1 + (i0*ne00)*nb0);\n ggml_fp16_t * x = (ggml_fp16_t *) ((char *) src0->data + ( k3)*nb03 + ( k2)*nb02 + ( k1)*nb01);\n // ggml_vec_cpy_f16(ne00, y, x)\n for (int i = 0; i < ne00; ++i) {\n y[i] = x[i];\n }\n }\n }\n }\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_repeat(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n case GGML_TYPE_I16:\n {\n ggml_compute_forward_repeat_f16(params, dst);\n } break;\n case GGML_TYPE_F32:\n case GGML_TYPE_I32:\n {\n ggml_compute_forward_repeat_f32(params, dst);\n } break;\n // TODO: templateify the implementation and support for I64\n // ref https://github.com/ggml-org/llama.cpp/pull/14274#discussion_r2169492225\n //case GGML_TYPE_I64:\n // {\n // ggml_compute_forward_repeat_i64(params, dst);\n // } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_repeat_back\n\nstatic void ggml_compute_forward_repeat_back_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n GGML_ASSERT(ggml_can_repeat(dst, src0));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n // guaranteed to be an integer due to the check in ggml_can_repeat\n const int nr0 = (int)(ne00/ne0);\n const int nr1 = (int)(ne01/ne1);\n const int nr2 = (int)(ne02/ne2);\n const int nr3 = (int)(ne03/ne3);\n\n // TODO: support for transposed / permuted tensors\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb00 == sizeof(float));\n\n if (ggml_is_contiguous(dst)) {\n ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);\n } else {\n for (int k3 = 0; k3 < ne3; k3++) {\n for (int k2 = 0; k2 < ne2; k2++) {\n for (int k1 = 0; k1 < ne1; k1++) {\n ggml_vec_set_f32(ne0,\n (float *) ((char *) dst->data + k1*nb1 + k2*nb2 + k3*nb3),\n 0);\n }\n }\n }\n }\n\n // TODO: maybe this is not optimal?\n for (int i3 = 0; i3 < nr3; i3++) {\n for (int k3 = 0; k3 < ne3; k3++) {\n for (int i2 = 0; i2 < nr2; i2++) {\n for (int k2 = 0; k2 < ne2; k2++) {\n for (int i1 = 0; i1 < nr1; i1++) {\n for (int k1 = 0; k1 < ne1; k1++) {\n for (int i0 = 0; i0 < nr0; i0++) {\n ggml_vec_acc_f32(ne0,\n (float *) ((char *) dst->data + ( k3)*nb3 + ( k2)*nb2 + ( k1)*nb1),\n (float *) ((char *) src0->data + (i3*ne3 + k3)*nb03 + (i2*ne2 + k2)*nb02 + (i1*ne1 + k1)*nb01 + (i0*ne0)*nb00));\n }\n }\n }\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_repeat_back(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_repeat_back_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_concat\n\nstatic void ggml_compute_forward_concat_any(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n const size_t len = ggml_type_size(src0->type);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n GGML_ASSERT(dim >= 0 && dim < 4);\n\n int64_t o[4] = {0, 0, 0, 0};\n o[dim] = src0->ne[dim];\n\n const char * x;\n\n // TODO: smarter multi-theading\n for (int i3 = 0; i3 < ne3; i3++) {\n for (int i2 = ith; i2 < ne2; i2 += nth) {\n for (int i1 = 0; i1 < ne1; i1++) {\n for (int i0 = 0; i0 < ne0; i0++) {\n if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n x = (const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03;\n } else {\n x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13;\n }\n\n char * y = (char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3;\n\n memcpy(y, x, len);\n }\n }\n }\n }\n}\n\nstatic void ggml_compute_forward_concat_i8(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n GGML_ASSERT(dim >= 0 && dim < 4);\n\n int64_t o[4] = {0, 0, 0, 0};\n o[dim] = src0->ne[dim];\n\n const int8_t * x;\n\n // TODO: smarter multi-theading\n for (int i3 = 0; i3 < ne3; i3++) {\n for (int i2 = ith; i2 < ne2; i2 += nth) {\n for (int i1 = 0; i1 < ne1; i1++) {\n for (int i0 = 0; i0 < ne0; i0++) {\n if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n x = (const int8_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);\n } else {\n x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);\n }\n\n int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n\n *y = *x;\n }\n }\n }\n }\n}\n\nstatic void ggml_compute_forward_concat_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n GGML_ASSERT(dim >= 0 && dim < 4);\n\n int64_t o[4] = {0, 0, 0, 0};\n o[dim] = src0->ne[dim];\n\n const ggml_fp16_t * x;\n\n // TODO: smarter multi-theading\n for (int i3 = 0; i3 < ne3; i3++) {\n for (int i2 = ith; i2 < ne2; i2 += nth) {\n for (int i1 = 0; i1 < ne1; i1++) {\n for (int i0 = 0; i0 < ne0; i0++) {\n if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n x = (const ggml_fp16_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);\n } else {\n x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);\n }\n\n ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n\n *y = *x;\n }\n }\n }\n }\n}\n\nstatic void ggml_compute_forward_concat_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n GGML_ASSERT(dim >= 0 && dim < 4);\n\n int64_t o[4] = {0, 0, 0, 0};\n o[dim] = src0->ne[dim];\n\n const float * x;\n\n // TODO: smarter multi-theading\n for (int i3 = 0; i3 < ne3; i3++) {\n for (int i2 = ith; i2 < ne2; i2 += nth) {\n for (int i1 = 0; i1 < ne1; i1++) {\n for (int i0 = 0; i0 < ne0; i0++) {\n if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n x = (const float *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);\n } else {\n x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);\n }\n\n float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n\n *y = *x;\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_concat(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n case GGML_TYPE_I16:\n {\n ggml_compute_forward_concat_f16(params, dst);\n } break;\n case GGML_TYPE_I8:\n {\n ggml_compute_forward_concat_i8(params, dst);\n } break;\n case GGML_TYPE_F32:\n case GGML_TYPE_I32:\n {\n ggml_compute_forward_concat_f32(params, dst);\n } break;\n default:\n {\n ggml_compute_forward_concat_any(params, dst);\n }\n }\n}\n\n// ggml_compute_forward_gelu\n\nstatic void ggml_compute_forward_gelu_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n assert(ggml_is_contiguous_rows(src0));\n assert(ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n const int i3 = ir/(ne02*ne01);\n const int i2 = (ir - i3*ne02*ne01)/ne01;\n const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n ggml_vec_gelu_f32(nc,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),\n (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_gelu_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n assert(ggml_is_contiguous_rows(src0));\n assert(ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n const int i3 = ir/(ne02*ne01);\n const int i2 = (ir - i3*ne02*ne01)/ne01;\n const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n ggml_vec_gelu_f16(nc,\n (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),\n (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];\n const float v = GGML_CPU_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_gelu(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_gelu_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_gelu_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_fill\n\nstatic void ggml_compute_forward_fill_f32(const ggml_compute_params * params, ggml_tensor * dst) {\n const float c = ggml_get_op_params_f32(dst, 0);\n\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne);\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb);\n\n const auto [ir0, ir1] = get_thread_range(params, dst);\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir/(ne2*ne1);\n const int64_t i02 = (ir - i03*ne2*ne1)/ne1;\n const int64_t i01 = (ir - i03*ne2*ne1 - i02*ne1);\n\n float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);\n\n ggml_vec_set_f32(ne0, dst_ptr, c);\n }\n}\n\nvoid ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor * dst) {\n ggml_compute_forward_fill_f32(params, dst);\n}\n\n// ggml_compute_tri\n\nstatic void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0);\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n const auto [ir0, ir1] = get_thread_range(params, src0);\n\n bool (*bipred)(int, int);\n\n switch (ttype) {\n case GGML_TRI_TYPE_LOWER: bipred = [](int i, int r) { return i < r; }; break;\n case GGML_TRI_TYPE_LOWER_DIAG: bipred = [](int i, int r) { return i <= r; }; break;\n case GGML_TRI_TYPE_UPPER: bipred = [](int i, int r) { return i > r; }; break;\n case GGML_TRI_TYPE_UPPER_DIAG: bipred = [](int i, int r) { return i >= r; }; break;\n default: GGML_ABORT(\"invalid tri type\");\n }\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir/(ne02*ne01);\n const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n const float * src_ptr = (const float *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);\n float * dst_ptr = ( float *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);\n\n for (int i0 = 0; i0 < ne0; ++i0) {\n dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : 0.0f;\n }\n }\n}\n\nvoid ggml_compute_forward_tri(const ggml_compute_params * params, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_tri_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_gelu_erf\n\nstatic void ggml_compute_forward_gelu_erf_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n assert(ggml_is_contiguous_rows(src0));\n assert(ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n const int i3 = ir/(ne02*ne01);\n const int i2 = (ir - i3*ne02*ne01)/ne01;\n const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n ggml_vec_gelu_erf_f32(nc,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),\n (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_gelu_erf_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n assert(ggml_is_contiguous_rows(src0));\n assert(ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n const int i3 = ir/(ne02*ne01);\n const int i2 = (ir - i3*ne02*ne01)/ne01;\n const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n ggml_vec_gelu_erf_f16(nc,\n (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),\n (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];\n const float v = GGML_CPU_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_gelu_erf(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_gelu_erf_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_gelu_erf_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_gelu_quick\n\nstatic void ggml_compute_forward_gelu_quick_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n assert(ggml_is_contiguous_rows(src0));\n assert(ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n const int i3 = ir/(ne02*ne01);\n const int i2 = (ir - i3*ne02*ne01)/ne01;\n const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n ggml_vec_gelu_quick_f32(nc,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),\n (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_gelu_quick_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n assert(ggml_is_contiguous_rows(src0));\n assert(ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n const int i3 = ir/(ne02*ne01);\n const int i2 = (ir - i3*ne02*ne01)/ne01;\n const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n ggml_vec_gelu_quick_f16(nc,\n (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),\n (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];\n const float v = GGML_CPU_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_gelu_quick(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_gelu_quick_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_gelu_quick_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_silu\n\nstatic void ggml_compute_forward_silu_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n assert(ggml_is_contiguous_rows(src0));\n assert(ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n const int i3 = ir/(ne02*ne01);\n const int i2 = (ir - i3*ne02*ne01)/ne01;\n const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n ggml_vec_silu_f32(nc,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),\n (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_silu_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n assert(ggml_is_contiguous_rows(src0));\n assert(ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n const int i3 = ir/(ne02*ne01);\n const int i2 = (ir - i3*ne02*ne01)/ne01;\n const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n ggml_vec_silu_f16(nc,\n (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),\n (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];\n const float v = GGML_CPU_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_silu(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_silu_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_silu_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n// ggml_compute_forward_leaky_relu\n\nstatic void ggml_compute_forward_leaky_relu_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n assert(ggml_is_contiguous_1(src0));\n assert(ggml_is_contiguous_1(dst));\n assert(ggml_are_same_shape(src0, dst));\n\n const int n = ggml_nrows(src0);\n const int nc = src0->ne[0];\n\n float negative_slope;\n memcpy(&negative_slope, dst->op_params, sizeof(float));\n\n assert(dst->nb[0] == sizeof(float));\n assert(src0->nb[0] == sizeof(float));\n\n for (int i = 0; i < n; i++) {\n ggml_vec_leaky_relu_f32(nc,\n (float *) ((char *) dst->data + i*( dst->nb[1])),\n (float *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);\n }\n}\n\nstatic void ggml_compute_forward_leaky_relu_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n assert(ggml_is_contiguous_1(src0));\n assert(ggml_is_contiguous_1(dst));\n assert(ggml_are_same_shape(src0, dst));\n\n const int n = ggml_nrows(src0);\n const int nc = src0->ne[0];\n\n float negative_slope;\n memcpy(&negative_slope, dst->op_params, sizeof(float));\n\n assert(dst->nb[0] == sizeof(ggml_fp16_t));\n assert(src0->nb[0] == sizeof(ggml_fp16_t));\n\n for (int i = 0; i < n; i++) {\n ggml_vec_leaky_relu_f16(nc,\n (ggml_fp16_t *) ((char *) dst->data + i*( dst->nb[1])),\n (ggml_fp16_t *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);\n }\n}\n\nvoid ggml_compute_forward_leaky_relu(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_leaky_relu_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_leaky_relu_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_silu_back\n\nstatic void ggml_compute_forward_silu_back_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * grad = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n assert(ggml_is_contiguous_1(grad));\n assert(ggml_is_contiguous_1(src1));\n assert(ggml_is_contiguous_1(dst));\n assert(ggml_are_same_shape(src1, dst));\n assert(ggml_are_same_shape(src1, grad));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1->ne[0];\n const int nr = ggml_nrows(src1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n ggml_vec_silu_backward_f32(nc,\n (float *) ((char *) dst->data + i1*( dst->nb[1])),\n (float *) ((char *) src1->data + i1*(src1->nb[1])),\n (float *) ((char *) grad->data + i1*(grad->nb[1])));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_silu_back_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * grad = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n assert(ggml_is_contiguous_1(grad));\n assert(ggml_is_contiguous_1(src1));\n assert(ggml_is_contiguous_1(dst));\n assert(ggml_are_same_shape(src1, dst));\n assert(ggml_are_same_shape(src1, grad));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1->ne[0];\n const int nr = ggml_nrows(src1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n ggml_vec_silu_backward_f16(nc,\n (ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])),\n (ggml_fp16_t *) ((char *) src1->data + i1*(src1->nb[1])),\n (ggml_fp16_t *) ((char *) grad->data + i1*(grad->nb[1])));\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n const float v = GGML_CPU_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nvoid ggml_compute_forward_silu_back(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_silu_back_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_silu_back_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_reglu\n\nstatic void ggml_compute_forward_reglu_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n float * src0_p = (float *) (src0_d + i1*src0_o);\n float * src1_p = (float *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_reglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_reglu_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_reglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n const float v = GGML_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_reglu(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_reglu_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_reglu_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_geglu\n\nstatic void ggml_compute_forward_geglu_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n float * src0_p = (float *) (src0_d + i1*src0_o);\n float * src1_p = (float *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_geglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_geglu_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_geglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n const float v = GGML_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_geglu(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_geglu_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_geglu_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_swiglu\n\nstatic void ggml_compute_forward_swiglu_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n float * src0_p = (float *) (src0_d + i1*src0_o);\n float * src1_p = (float *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_swiglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_swiglu_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_swiglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n const float v = GGML_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_swiglu(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_swiglu_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_swiglu_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_swiglu_oai\n\nstatic void ggml_compute_forward_swiglu_oai_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n const float alpha = ggml_get_op_params_f32(dst, 2);\n const float limit = ggml_get_op_params_f32(dst, 3);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n float * src0_p = (float *) (src0_d + i1*src0_o);\n float * src1_p = (float *) (src1_d + i1*src1_o);\n float * dst_p = (float *) ((char *) dst->data + i1*(dst->nb[1]));\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n for (int k = 0; k < nc; k++) {\n const float x = std::min(src0_p[k], limit);\n const float y = std::clamp(src1_p[k], -limit, limit);\n const float out_glu = x / (1.f + expf(alpha * (-x)));\n dst_p[k] = out_glu * (y + 1.f);\n }\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = dst_p[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_swiglu_oai(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_swiglu_oai_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_geglu_erf\n\nstatic void ggml_compute_forward_geglu_erf_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n float * src0_p = (float *) (src0_d + i1*src0_o);\n float * src1_p = (float *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_geglu_erf_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_geglu_erf_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_geglu_erf_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n const float v = GGML_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_geglu_erf(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_geglu_erf_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_geglu_erf_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_geglu_quick\n\nstatic void ggml_compute_forward_geglu_quick_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n float * src0_p = (float *) (src0_d + i1*src0_o);\n float * src1_p = (float *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_geglu_quick_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n GGML_UNUSED(x);\n assert(!isnan(x));\n assert(!isinf(x));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_geglu_quick_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n char * src0_d = (char *) src0->data;\n char * src1_d = (char *) (src1 ? src1->data : src0->data);\n const size_t src0_o = src0->nb[1];\n const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n const int nr = ggml_nrows(src0);\n\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == nr);\n\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n ggml_vec_geglu_quick_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n for (int k = 0; k < nc; k++) {\n const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n const float v = GGML_FP16_TO_FP32(x);\n GGML_UNUSED(v);\n assert(!isnan(v));\n assert(!isinf(v));\n }\n#endif // NDEBUG\n }\n}\n\nstatic void ggml_compute_forward_geglu_quick(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_geglu_quick_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_geglu_quick_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_norm\n\nstatic void ggml_compute_forward_norm_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n\n GGML_ASSERT(eps >= 0.0f);\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n\n float sum = 0.0;\n ggml_vec_sum_f32(ne00, &sum, x);\n float mean = sum/ne00;\n\n float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n float variance = 0;\n\n#ifdef GGML_USE_ACCELERATE\n mean = -mean;\n vDSP_vsadd(x, 1, &mean, y, 1, ne00);\n vDSP_measqv(y, 1, &variance, ne00);\n#else\n variance = ggml_vec_cvar_f32(ne00, y, x, mean);\n#endif //GGML_USE_ACCELERATE\n\n const float scale = 1.0f/sqrtf(variance + eps);\n ggml_vec_scale_f32(ne00, y, scale);\n }\n }\n }\n}\n\nvoid ggml_compute_forward_norm(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_norm_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_group_rms_norm\n\nstatic void ggml_compute_forward_rms_norm_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n\n GGML_ASSERT(eps >= 0.0f);\n\n // TODO: optimize\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n\n ggml_float sum = 0.0;\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n sum += (ggml_float)(x[i00] * x[i00]);\n }\n\n const float mean = sum/ne00;\n\n float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n\n memcpy(y, x, ne00 * sizeof(float));\n // for (int i00 = 0; i00 < ne00; i00++) {\n // y[i00] = x[i00];\n // }\n\n const float scale = 1.0f/sqrtf(mean + eps);\n\n // if you hit this, likely you got an inf somewhere earlier\n assert(scale > 0.0f);\n\n ggml_vec_scale_f32(ne00, y, scale);\n }\n }\n }\n}\n\nvoid ggml_compute_forward_rms_norm(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_rms_norm_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nstatic void ggml_compute_forward_rms_norm_back_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0]; // gradients from forward pass output\n const ggml_tensor * src1 = dst->src[1]; // src1 from forward pass\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1));\n\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n GGML_ASSERT(src1->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n\n // TODO: optimize\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n // src1 is same shape as src0 => same indices\n const int64_t i11 = i01;\n const int64_t i12 = i02;\n const int64_t i13 = i03;\n\n const float * dz = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n const float * x = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13);\n\n ggml_float sum_xx = 0.0;\n ggml_float sum_xdz = 0.0;\n\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n sum_xx += (ggml_float)(x[i00] * x[i00]);\n sum_xdz += (ggml_float)(x[i00] * dz[i00]);\n }\n\n //const float mean = (float)(sum_xx)/ne00;\n const float mean_eps = (float)(sum_xx)/ne00 + eps;\n const float sum_eps = (float)(sum_xx) + eps*ne00;\n //const float mean_xdz = (float)(sum_xdz)/ne00;\n // we could cache rms from forward pass to improve performance.\n // to do this implement ggml_rms and compose ggml_rms_norm using ggml_rms.\n //const float rms = sqrtf(mean_eps);\n const float rrms = 1.0f / sqrtf(mean_eps);\n //const float scale = -rrms/(ne00 * mean_eps); // -1/(n*rms**3)\n\n {\n // z = rms_norm(x)\n //\n // rms_norm(src1) =\n // scale(\n // src1,\n // div(\n // 1,\n // sqrt(\n // add(\n // scale(\n // sum(\n // sqr(\n // src1)),\n // (1.0/N)),\n // eps))));\n\n // postorder:\n // ## op args grad\n // 00 param src1 grad[#00]\n // 01 const 1\n // 02 sqr (#00) grad[#02]\n // 03 sum (#02) grad[#03]\n // 04 const 1/N\n // 05 scale (#03, #04) grad[#05]\n // 06 const eps\n // 07 add (#05, #06) grad[#07]\n // 08 sqrt (#07) grad[#08]\n // 09 div (#01,#08) grad[#09]\n // 10 scale (#00,#09) grad[#10]\n //\n // backward pass, given grad[#10]\n // #10: scale\n // grad[#00] += scale(grad[#10],#09)\n // grad[#09] += sum(mul(grad[#10],#00))\n // #09: div\n // grad[#08] += neg(mul(grad[#09], div(#09,#08)))\n // #08: sqrt\n // grad[#07] += mul(grad[#08], div(0.5, #08))\n // #07: add\n // grad[#05] += grad[#07]\n // #05: scale\n // grad[#03] += scale(grad[#05],#04)\n // #03: sum\n // grad[#02] += repeat(grad[#03], #02)\n // #02:\n // grad[#00] += scale(mul(#00, grad[#02]), 2.0)\n //\n // substitute and simplify:\n // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, grad[#02]), 2.0)\n // grad[#02] = repeat(grad[#03], #02)\n // grad[#02] = repeat(scale(grad[#05],#04), #02)\n // grad[#02] = repeat(scale(grad[#07],#04), #02)\n // grad[#02] = repeat(scale(mul(grad[#08], div(0.5, #08)),#04), #02)\n // grad[#02] = repeat(scale(mul(neg(mul(grad[#09], div(#09,#08))), div(0.5, #08)),#04), #02)\n // grad[#02] = repeat(scale(mul(neg(mul(sum(mul(grad[#10],#00)), div(#09,#08))), div(0.5, #08)),#04), #02)\n // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(#09,#08) * div(0.5, #08) * (1/N)), #02)\n // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(div(#01,#08),#08) * div(0.5, #08) * (1/N)), #02)\n // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(1,#08*#08) * div(0.5, #08) * (1/N)), #02)\n // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N)), #02)\n // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, grad[#02]), 2.0)\n // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, repeat(-(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N)), #02)), 2.0)\n // grad[#00] = scale(grad(#10), #09) + scale(scale(#00, -(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N))), 2.0)\n // grad[#00] = scale(grad(#10), #09) + scale(#00, -(sum(mul(grad[#10],#00)) * div(1,#07) * div(1,#08) * (1/N)))\n // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,#07*#08) * (-1/N))\n // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,#07*#08) * (-1/N))\n // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,mean_eps*rms) * (-1/N))\n // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*mean_eps))\n // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*(sum_xx/N+eps)))\n // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*sum_xx+rms*N*eps))\n // grad[#00] = scale(dz, rrms) + scale(x, sum(mul(dz,x)) * div(-1,rms*N*mean_eps))\n // grad[#00] = scale(dz, rrms) + scale(x, sum_xdz * div(-1,rms*N*mean_eps))\n // a = b*c + d*e\n // a = b*c*f/f + d*e*f/f\n // a = (b*c*f + d*e*f)*(1/f)\n // a = (b*c*(1/c) + d*e*(1/c))*(1/(1/c))\n // a = (b + d*e/c)*c\n // b = dz, c = rrms, d = x, e = sum_xdz * div(-1,rms*N*mean_eps)\n // a = (dz + x*sum_xdz * div(-1,rms*N*mean_eps)/rrms)*rrms\n // a = (dz + x*sum_xdz * div(-1,rms*N*mean_eps)*rms)*rrms\n // a = (dz + x*sum_xdz * div(-rms,rms*N*mean_eps))*rrms\n // a = (dz + x*sum_xdz * div(-1,N*mean_eps))*rrms\n // a = (dz + x*div(-sum_xdz,N*mean_eps))*rrms\n // a = (dz + x*div(-mean_xdz,mean_eps))*rrms\n // grad[#00] = scale(dz + scale(x, div(-mean_xdz,mean_eps)),rrms)\n // grad[#00] = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)\n // dx = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)\n }\n // dx = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)\n // post-order:\n // dx := x\n // dx := scale(dx,-mean_xdz/mean_eps)\n // dx := add(dx, dz)\n // dx := scale(dx, rrms)\n float * dx = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n\n // dx[i00] = (x*(-sum_xdz/sum_eps) + dz) / sqrtf(mean_eps)\n ggml_vec_cpy_f32 (ne00, dx, x);\n // ggml_vec_scale_f32(ne00, dx, -mean_xdz/mean_eps);\n ggml_vec_scale_f32(ne00, dx, (float)(-sum_xdz)/sum_eps);\n ggml_vec_acc_f32 (ne00, dx, dz);\n ggml_vec_scale_f32(ne00, dx, rrms);\n }\n }\n }\n}\n\nvoid ggml_compute_forward_rms_norm_back(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_rms_norm_back_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_group_norm\n\nstatic void ggml_compute_forward_group_norm_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n // TODO: optimize\n\n float eps;\n memcpy(&eps, dst->op_params + 1, sizeof(float));\n\n int n_channels = src0->ne[2];\n int n_groups = dst->op_params[0];\n int n_channels_per_group = (n_channels + n_groups - 1) / n_groups;\n for (int i = ith; i < n_groups; i += nth) {\n int start = i * n_channels_per_group;\n int end = start + n_channels_per_group;\n if (end > n_channels) {\n end = n_channels;\n }\n int step = end - start;\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n ggml_float sum = 0.0;\n for (int64_t i02 = start; i02 < end; i02++) {\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);\n\n ggml_float sumr = 0.0;\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n sumr += (ggml_float)x[i00];\n }\n sum += sumr;\n }\n }\n const float mean = sum / (ne00 * ne01 * step);\n\n ggml_float sum2 = 0.0;\n for (int64_t i02 = start; i02 < end; i02++) {\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);\n\n float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3);\n\n ggml_float sumr = 0.0;\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n float v = x[i00] - mean;\n y[i00] = v;\n sumr += (ggml_float)(v * v);\n }\n sum2 += sumr;\n }\n }\n const float variance = sum2 / (ne00 * ne01 * step);\n const float scale = 1.0f / sqrtf(variance + eps);\n\n for (int64_t i02 = start; i02 < end; i02++) {\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3);\n ggml_vec_scale_f32(ne00, y, scale);\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_group_norm(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_group_norm_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_l2_norm\n\nstatic void ggml_compute_forward_l2_norm_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n\n GGML_ASSERT(eps >= 0.0f);\n\n // TODO: optimize\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n\n ggml_float sum = 0.0;\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n sum += (ggml_float)(x[i00] * x[i00]);\n }\n\n float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n\n memcpy(y, x, ne00 * sizeof(float));\n\n const float scale = 1.0f/fmaxf(sqrtf(sum), eps);\n\n ggml_vec_scale_f32(ne00, y, scale);\n }\n }\n }\n}\n\nvoid ggml_compute_forward_l2_norm(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_l2_norm_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_out_prod\n\nstatic void ggml_compute_forward_out_prod_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_ASSERT(ne0 == ne00);\n GGML_ASSERT(ne1 == ne10);\n GGML_ASSERT(ne2 == ne12);\n GGML_ASSERT(ne3 == ne13);\n\n GGML_ASSERT(ne2 % ne02 == 0);\n GGML_ASSERT(ne3 % ne03 == 0);\n\n // we don't support permuted src0 or src1\n GGML_ASSERT(nb00 == sizeof(float));\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n // GGML_ASSERT(nb0 <= nb1);\n // GGML_ASSERT(nb1 <= nb2);\n // GGML_ASSERT(nb2 <= nb3);\n\n // nb01 >= nb00 - src0 is not transposed\n // compute by src0 rows\n\n if (ith == 0) {\n ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);\n }\n ggml_barrier(params->threadpool);\n\n // dst[:,:,:,:] = 0\n // for i2,i3:\n // for i1:\n // for i01:\n // for i0:\n // dst[i0,i1,i2,i3] += src0[i0,i01,i2,i3] * src1[i1,i01,i2,i3]\n\n // parallelize by last three dimensions\n\n // total rows in dst\n const int64_t nr = ne1*ne2*ne3;\n\n // rows per thread\n const int64_t dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int64_t ir0 = dr*ith;\n const int64_t ir1 = MIN(ir0 + dr, nr);\n\n // block-tiling attempt\n const int64_t blck_0 = MAX(GGML_VEC_MAD_UNROLL, 32);\n const int64_t blck_1 = 16;\n\n // dps == dst per src0, used for group query attention\n const int64_t dps2 = ne2 / ne02;\n const int64_t dps3 = ne3 / ne03;\n\n for (int64_t bir = ir0; bir < ir1; bir += blck_1) {\n const int64_t bir1 = MIN(bir + blck_1, ir1);\n for (int64_t bi01 = 0; bi01 < ne01; bi01 += blck_0) {\n const int64_t bne01 = MIN(bi01 + blck_0, ne01);\n for (int64_t ir = bir; ir < bir1; ++ir) {\n // dst indices\n const int64_t i3 = ir/(ne2*ne1);\n const int64_t i2 = (ir - i3*ne2*ne1)/ne1;\n const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n const int64_t i02 = i2 / dps2;\n const int64_t i03 = i3 / dps3;\n\n //const int64_t i10 = i1;\n const int64_t i12 = i2;\n const int64_t i13 = i3;\n\n#if GGML_VEC_MAD_UNROLL > 2\n const int64_t bne01_unroll = bne01 - (bne01 % GGML_VEC_MAD_UNROLL);\n for (int64_t i01 = bi01; i01 < bne01_unroll; i01 += GGML_VEC_MAD_UNROLL) {\n const int64_t i11 = i01;\n\n float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));\n float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));\n float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));\n\n ggml_vec_mad_f32_unroll(ne0, nb01, nb11, d, s0, s1);\n }\n for (int64_t i01 = bne01_unroll; i01 < bne01; ++i01) {\n const int64_t i11 = i01;\n\n float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));\n float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));\n float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));\n\n ggml_vec_mad_f32(ne0, d, s0, *s1);\n }\n#else\n for (int64_t i01 = bi01; i01 < bne01; ++i01) {\n const int64_t i11 = i01;\n\n float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));\n float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));\n float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));\n\n ggml_vec_mad_f32(ne0, d, s0, *s1);\n }\n#endif\n }\n }\n }\n}\n\nstatic void ggml_compute_forward_out_prod_q_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const ggml_type type = src0->type;\n ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n\n GGML_ASSERT(ne02 == ne12);\n GGML_ASSERT(ne03 == ne13);\n GGML_ASSERT(ne2 == ne12);\n GGML_ASSERT(ne3 == ne13);\n\n // we don't support permuted src0 dim0\n GGML_ASSERT(nb00 == ggml_type_size(type));\n\n // dst dim0 cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n // GGML_ASSERT(nb0 <= nb1);\n // GGML_ASSERT(nb1 <= nb2);\n // GGML_ASSERT(nb2 <= nb3);\n\n GGML_ASSERT(ne0 == ne00);\n GGML_ASSERT(ne1 == ne10);\n GGML_ASSERT(ne2 == ne02);\n GGML_ASSERT(ne3 == ne03);\n\n // nb01 >= nb00 - src0 is not transposed\n // compute by src0 rows\n\n if (ith == 0) {\n ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);\n }\n ggml_barrier(params->threadpool);\n\n // parallelize by last three dimensions\n\n // total rows in dst\n const int64_t nr = ne1*ne2*ne3;\n\n // rows per thread\n const int64_t dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int64_t ir0 = dr*ith;\n const int64_t ir1 = MIN(ir0 + dr, nr);\n\n // dst[:,:,:,:] = 0\n // for i2,i3:\n // for i1:\n // for i01:\n // for i0:\n // dst[i0,i1,i2,i3] += src0[i0,i01,i2,i3] * src1[i1,i01,i2,i3]\n\n float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith;\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n // dst indices\n const int64_t i3 = ir/(ne2*ne1);\n const int64_t i2 = (ir - i3*ne2*ne1)/ne1;\n const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n const int64_t i02 = i2;\n const int64_t i03 = i3;\n\n //const int64_t i10 = i1;\n const int64_t i12 = i2;\n const int64_t i13 = i3;\n\n for (int64_t i01 = 0; i01 < ne01; ++i01) {\n const int64_t i11 = i01;\n\n float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));\n float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));\n float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));\n\n dequantize_row_q(s0, wdata, ne0);\n ggml_vec_mad_f32(ne0, d, wdata, *s1);\n }\n }\n}\n\nvoid ggml_compute_forward_out_prod(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_NVFP4:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_TQ1_0:\n case GGML_TYPE_TQ2_0:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ2_S:\n {\n ggml_compute_forward_out_prod_q_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n GGML_ABORT(\"fatal error\"); // todo\n // ggml_compute_forward_out_prod_f16_f32(params, dst);\n }\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_out_prod_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_scale\n\nstatic void ggml_compute_forward_scale_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n float s; // scale factor\n float b; // bias\n\n memcpy(&s, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&b, (float *) dst->op_params + 1, sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n const size_t nb01 = src0->nb[1];\n\n const size_t nb1 = dst->nb[1];\n\n if (b == 0.0f) {\n for (int i1 = ir0; i1 < ir1; i1++) {\n if (dst->data != src0->data) {\n // src0 is same shape as dst => same indices\n // TODO: add x parameter to ggml_vec_scale_f32 and remove this memcpy\n memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float));\n }\n ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), s);\n }\n } else {\n for (int i1 = ir0; i1 < ir1; i1++) {\n ggml_vec_mad1_f32(nc,\n (float *) ((char *) dst->data + i1*nb1),\n (float *) ((char *) src0->data + i1*nb1),\n s, b);\n }\n }\n}\n\nvoid ggml_compute_forward_scale(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_scale_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_set\n\nstatic void ggml_compute_forward_set_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n\n // view src0 and dst with these strides and data offset inbytes during set\n // nb0 is implicitly element_size because src0 and dst are contiguous\n size_t nb1 = ((int32_t *) dst->op_params)[0];\n size_t nb2 = ((int32_t *) dst->op_params)[1];\n size_t nb3 = ((int32_t *) dst->op_params)[2];\n size_t offset = ((int32_t *) dst->op_params)[3];\n bool inplace = (bool) ((int32_t *) dst->op_params)[4];\n\n if (!inplace) {\n if (params->ith == 0) {\n // memcpy needs to be synchronized across threads to avoid race conditions.\n // => do it in INIT phase\n memcpy(\n ((char *) dst->data),\n ((char *) src0->data),\n ggml_nbytes(dst));\n }\n ggml_barrier(params->threadpool);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src1);\n const int nc = src1->ne[0];\n\n GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)\n GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)\n\n // src0 and dst as viewed during set\n const size_t nb0 = ggml_element_size(src0);\n\n const int im0 = (ne10 == 0 ? 0 : ne10-1);\n const int im1 = (ne11 == 0 ? 0 : ne11-1);\n const int im2 = (ne12 == 0 ? 0 : ne12-1);\n const int im3 = (ne13 == 0 ? 0 : ne13-1);\n\n GGML_ASSERT(offset + im0*nb0 + im1*nb1 + im2*nb2 + im3*nb3 <= ggml_nbytes(dst));\n\n GGML_ASSERT(nb10 == sizeof(float));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 and dst are viewed with shape of src1 and offset\n // => same indices\n const int i3 = ir/(ne12*ne11);\n const int i2 = (ir - i3*ne12*ne11)/ne11;\n const int i1 = (ir - i3*ne12*ne11 - i2*ne11);\n\n ggml_vec_cpy_f32(nc,\n (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset),\n (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));\n }\n}\n\nstatic void ggml_compute_forward_set_i32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n\n // view src0 and dst with these strides and data offset inbytes during set\n // nb0 is implicitly element_size because src0 and dst are contiguous\n size_t nb1 = ((int32_t *) dst->op_params)[0];\n size_t nb2 = ((int32_t *) dst->op_params)[1];\n size_t nb3 = ((int32_t *) dst->op_params)[2];\n size_t offset = ((int32_t *) dst->op_params)[3];\n bool inplace = (bool) ((int32_t *) dst->op_params)[4];\n\n if (!inplace) {\n if (params->ith == 0) {\n // memcpy needs to be synchronized across threads to avoid race conditions.\n // => do it in INIT phase\n memcpy(\n ((char *) dst->data),\n ((char *) src0->data),\n ggml_nbytes(dst));\n }\n ggml_barrier(params->threadpool);\n }\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src1);\n const int nc = src1->ne[0];\n\n GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)\n GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)\n\n // src0 and dst as viewed during set\n const size_t nb0 = ggml_element_size(src0);\n\n const int im0 = (ne10 == 0 ? 0 : ne10-1);\n const int im1 = (ne11 == 0 ? 0 : ne11-1);\n const int im2 = (ne12 == 0 ? 0 : ne12-1);\n const int im3 = (ne13 == 0 ? 0 : ne13-1);\n\n GGML_ASSERT(offset + im0*nb0 + im1*nb1 + im2*nb2 + im3*nb3 <= ggml_nbytes(dst));\n\n GGML_ASSERT(nb10 == sizeof(int32_t));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // src0 and dst are viewed with shape of src1 and offset\n // => same indices\n const int i3 = ir/(ne12*ne11);\n const int i2 = (ir - i3*ne12*ne11)/ne11;\n const int i1 = (ir - i3*ne12*ne11 - i2*ne11);\n\n ggml_vec_cpy_i32(nc,\n (int32_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset),\n (int32_t *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));\n }\n}\n\nvoid ggml_compute_forward_set(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_set_f32(params, dst);\n } break;\n case GGML_TYPE_I32:\n {\n ggml_compute_forward_set_i32(params, dst);\n } break;\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q8_1:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_NVFP4:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_TQ1_0:\n case GGML_TYPE_TQ2_0:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ2_S:\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_cpy\n\nvoid ggml_compute_forward_cpy(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n ggml_compute_forward_dup(params, dst);\n}\n\n// ggml_compute_forward_cont\n\nvoid ggml_compute_forward_cont(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n ggml_compute_forward_dup(params, dst);\n}\n\n// ggml_compute_forward_get_rows\n\nstatic void ggml_compute_forward_get_rows_q(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int64_t nc = ne00;\n const int64_t nr = ggml_nelements(src1);\n\n const ggml_type type = src0->type;\n ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n\n assert(ne0 == nc);\n assert(ne02 == ne11);\n assert(nb00 == ggml_type_size(type));\n assert(ggml_nrows(dst) == nr);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int64_t i = ir0; i < ir1; ++i) {\n const int64_t i12 = i/(ne11*ne10);\n const int64_t i11 = (i - i12*ne11*ne10)/ne10;\n const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);\n const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n GGML_ASSERT(i01 >= 0 && i01 < ne01);\n\n dequantize_row_q(\n (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),\n (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);\n }\n}\n\nstatic void ggml_compute_forward_get_rows_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int64_t nc = ne00;\n const int64_t nr = ggml_nelements(src1);\n\n assert(ne0 == nc);\n assert(ne02 == ne11);\n assert(nb00 == sizeof(ggml_fp16_t));\n assert(ggml_nrows(dst) == nr);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int64_t i = ir0; i < ir1; ++i) {\n const int64_t i12 = i/(ne11*ne10);\n const int64_t i11 = (i - i12*ne11*ne10)/ne10;\n const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);\n const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n GGML_ASSERT(i01 >= 0 && i01 < ne01);\n\n ggml_cpu_fp16_to_fp32(\n (const ggml_fp16_t*) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),\n (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);\n }\n}\n\nstatic void ggml_compute_forward_get_rows_bf16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int64_t nc = ne00;\n const int64_t nr = ggml_nelements(src1);\n\n assert(ne0 == nc);\n assert(ne02 == ne11);\n assert(nb00 == sizeof(ggml_bf16_t));\n assert(ggml_nrows(dst) == nr);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int64_t i = ir0; i < ir1; ++i) {\n const int64_t i12 = i/(ne11*ne10);\n const int64_t i11 = (i - i12*ne11*ne10)/ne10;\n const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);\n const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n GGML_ASSERT(i01 >= 0 && i01 < ne01);\n\n ggml_cpu_bf16_to_fp32(\n (const ggml_bf16_t *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),\n (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);\n }\n}\n\nstatic void ggml_compute_forward_get_rows_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int64_t nc = ne00;\n const int64_t nr = ggml_nelements(src1);\n\n assert(ne0 == nc);\n assert(ne02 == ne11);\n assert(nb00 == sizeof(float));\n assert(ggml_nrows(dst) == nr);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int64_t i = ir0; i < ir1; ++i) {\n const int64_t i12 = i/(ne11*ne10);\n const int64_t i11 = (i - i12*ne11*ne10)/ne10;\n const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);\n const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n GGML_ASSERT(i01 >= 0 && i01 < ne01);\n\n ggml_vec_cpy_f32(nc,\n (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3),\n (float *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03));\n }\n}\n\nvoid ggml_compute_forward_get_rows(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q8_1:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_NVFP4:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_TQ1_0:\n case GGML_TYPE_TQ2_0:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ2_S:\n {\n ggml_compute_forward_get_rows_q(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_get_rows_f16(params, dst);\n } break;\n case GGML_TYPE_BF16:\n {\n ggml_compute_forward_get_rows_bf16(params, dst);\n } break;\n case GGML_TYPE_F32:\n case GGML_TYPE_I32:\n {\n ggml_compute_forward_get_rows_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n\n //static bool first = true;\n //printf(\"ne0 = %d, ne1 = %d, ne2 = %d\\n\", dst->ne[0], dst->ne[1], dst->ne[2]);\n //if (first) {\n // first = false;\n //} else {\n // for (int k = 0; k < dst->ne[1]; ++k) {\n // for (int j = 0; j < dst->ne[0]/16; ++j) {\n // for (int i = 0; i < 16; ++i) {\n // printf(\"%8.4f \", ((float *) dst->data)[k*dst->ne[0] + j*16 + i]);\n // }\n // printf(\"\\n\");\n // }\n // printf(\"\\n\");\n // }\n // printf(\"\\n\");\n // exit(0);\n //}\n}\n\ntemplate\nstatic void ggml_compute_forward_set_rows_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int64_t nc = ne00;\n const int64_t nr = ne01;\n\n assert(ne0 == nc);\n assert(ne2 == ne02);\n assert(ne3 == ne03);\n assert(src0->type == GGML_TYPE_F32);\n assert(ne02 % ne11 == 0);\n assert(ne03 % ne12 == 0);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n // rows per thread\n const int64_t dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int64_t ir0 = dr*ith;\n const int64_t ir1 = std::min(ir0 + dr, nr);\n\n ggml_from_float_t const from_float = ggml_get_type_traits_cpu(dst->type)->from_float;\n\n for (int64_t i03 = 0; i03 < ne03; ++i03) {\n for (int64_t i02 = 0; i02 < ne02; ++i02) {\n for (int64_t i = ir0; i < ir1; ++i) {\n const int64_t i12 = i03%ne12;\n const int64_t i11 = i02%ne11;\n const int64_t i10 = i;\n\n const int64_t i1 = *(idx_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n GGML_ASSERT(i1 >= 0 && i1 < ne1);\n\n from_float(\n (const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),\n ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);\n }\n }\n }\n}\n\nvoid ggml_compute_forward_set_rows(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n if (src1->type == GGML_TYPE_I64) {\n ggml_compute_forward_set_rows_f32(params, dst);\n } else if (src1->type == GGML_TYPE_I32) {\n ggml_compute_forward_set_rows_f32(params, dst);\n } else {\n GGML_ABORT(\"src1->type = %d (%s) not supported\", src1->type, ggml_type_name(src1->type));\n }\n } break;\n default:\n {\n GGML_ABORT(\"src0->type = %d (%s) not supported\", src0->type, ggml_type_name(src0->type));\n }\n }\n}\n\n// ggml_compute_forward_get_rows_back\n\nstatic void ggml_compute_forward_get_rows_back_f32_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n if (params->ith != 0) {\n return;\n }\n\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n // ggml_compute_forward_dup_same_cont(params, opt0, dst);\n\n memset(dst->data, 0, ggml_nbytes(dst));\n\n const int nc = src0->ne[0];\n const int nr = ggml_nelements(src1);\n\n GGML_ASSERT( dst->ne[0] == nc);\n GGML_ASSERT(src0->nb[0] == sizeof(ggml_fp16_t));\n\n for (int i = 0; i < nr; ++i) {\n const int r = ((int32_t *) src1->data)[i];\n\n for (int j = 0; j < nc; ++j) {\n ggml_fp16_t v = ((ggml_fp16_t *) ((char *) src0->data + i*src0->nb[1]))[j];\n ((float *) ((char *) dst->data + r*dst->nb[1]))[j] += GGML_CPU_FP16_TO_FP32(v);\n }\n }\n}\n\nstatic void ggml_compute_forward_get_rows_back_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n if (params->ith != 0) {\n return;\n }\n\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n // ggml_compute_forward_dup_same_cont(params, opt0, dst);\n\n memset(dst->data, 0, ggml_nbytes(dst));\n\n const int nc = src0->ne[0];\n const int nr = ggml_nelements(src1);\n\n GGML_ASSERT( dst->ne[0] == nc);\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n for (int i = 0; i < nr; ++i) {\n const int r = ((int32_t *) src1->data)[i];\n\n ggml_vec_add_f32(nc,\n (float *) ((char *) dst->data + r*dst->nb[1]),\n (float *) ((char *) dst->data + r*dst->nb[1]),\n (float *) ((char *) src0->data + i*src0->nb[1]));\n }\n}\n\nvoid ggml_compute_forward_get_rows_back(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_get_rows_back_f32_f16(params, dst);\n } break;\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_get_rows_back_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n\n //static bool first = true;\n //printf(\"ne0 = %d, ne1 = %d, ne2 = %d\\n\", dst->ne[0], dst->ne[1], dst->ne[2]);\n //if (first) {\n // first = false;\n //} else {\n // for (int k = 0; k < dst->ne[1]; ++k) {\n // for (int j = 0; j < dst->ne[0]/16; ++j) {\n // for (int i = 0; i < 16; ++i) {\n // printf(\"%8.4f \", ((float *) dst->data)[k*dst->ne[0] + j*16 + i]);\n // }\n // printf(\"\\n\");\n // }\n // printf(\"\\n\");\n // }\n // printf(\"\\n\");\n // exit(0);\n //}\n}\n\n// ggml_compute_forward_diag\n\nstatic void ggml_compute_forward_diag_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n if (params->ith != 0) {\n return;\n }\n\n // TODO: handle transposed/permuted matrices\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT(ne00 == ne0);\n GGML_ASSERT(ne00 == ne1);\n GGML_ASSERT(ne01 == 1);\n GGML_ASSERT(ne02 == ne2);\n GGML_ASSERT(ne03 == ne3);\n\n GGML_ASSERT(nb00 == sizeof(float));\n GGML_ASSERT(nb0 == sizeof(float));\n\n for (int i3 = 0; i3 < ne3; i3++) {\n for (int i2 = 0; i2 < ne2; i2++) {\n for (int i1 = 0; i1 < ne1; i1++) {\n float * d = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);\n float * s = (float *)((char *) src0->data + i3*nb03 + i2*nb02);\n for (int i0 = 0; i0 < i1; i0++) {\n d[i0] = 0;\n }\n d[i1] = s[i1];\n for (int i0 = i1+1; i0 < ne0; i0++) {\n d[i0] = 0;\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_diag(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_diag_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_diag_mask_inf\n\nstatic void ggml_compute_forward_diag_mask_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst,\n const float value) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int n_past = ((int32_t *) dst->op_params)[0];\n const bool inplace = src0->data == dst->data;\n\n GGML_ASSERT(n_past >= 0);\n\n if (!inplace) {\n if (ith == 0) {\n // memcpy needs to be synchronized across threads to avoid race conditions.\n // => do it in INIT phase\n GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n memcpy(\n ((char *) dst->data),\n ((char *) src0->data),\n ggml_nbytes(dst));\n }\n ggml_barrier(params->threadpool);\n }\n\n // TODO: handle transposed/permuted matrices\n\n const int n = ggml_nrows(src0);\n const int nc = src0->ne[0];\n const int nr = src0->ne[1];\n const int nz = n/nr;\n\n GGML_ASSERT( dst->nb[0] == sizeof(float));\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n for (int k = 0; k < nz; k++) {\n for (int j = ith; j < nr; j += nth) {\n for (int i = n_past; i < nc; i++) {\n if (i > n_past + j) {\n *(float *)((char *) dst->data + k*dst->nb[2] + j*dst->nb[1] + i*dst->nb[0]) = value;\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_diag_mask_inf(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_diag_mask_f32(params, dst, -INFINITY);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nvoid ggml_compute_forward_diag_mask_zero(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_diag_mask_f32(params, dst, 0);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_soft_max\n\nstatic void ggml_compute_forward_soft_max_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const ggml_tensor * src2 = dst->src[2];\n\n assert(ggml_is_contiguous(dst));\n assert(ggml_are_same_shape(src0, dst));\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n\n memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n const int64_t nb11 = src1 ? src1->nb[1] : 1;\n const int64_t nb12 = src1 ? src1->nb[2] : 1;\n const int64_t nb13 = src1 ? src1->nb[3] : 1;\n\n const int64_t ne12 = src1 ? src1->ne[2] : 1;\n const int64_t ne13 = src1 ? src1->ne[3] : 1;\n\n // TODO: is this supposed to be ceil instead of floor?\n // https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L370\n const uint32_t n_head = ne02;\n const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n\n const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n float * wp = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;\n\n const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);\n\n // sinks\n const float * sk = src2 ? (float *)((char *) src2->data) : nullptr;\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n const int64_t i11 = i01;\n const int64_t i12 = i02%ne12;\n const int64_t i13 = i03%ne13;\n\n // ALiBi\n const uint32_t h = i02; // head\n const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;\n\n float * sp = (float *)((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n float * dp = (float *)((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n\n // broadcast the mask across rows\n ggml_fp16_t * mp_f16 = src1 ? (ggml_fp16_t *)((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13) : NULL;\n float * mp_f32 = src1 ? (float *)((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13) : NULL;\n\n ggml_vec_cpy_f32 (ne00, wp, sp);\n ggml_vec_scale_f32(ne00, wp, scale);\n if (mp_f32) {\n if (use_f16) {\n for (int i = 0; i < ne00; ++i) {\n wp[i] += slope*GGML_CPU_FP16_TO_FP32(mp_f16[i]);\n }\n } else {\n for (int i = 0; i < ne00; ++i) {\n wp[i] += slope*mp_f32[i];\n }\n }\n }\n\n#ifndef NDEBUG\n for (int i = 0; i < ne00; ++i) {\n //printf(\"p[%d] = %f\\n\", i, p[i]);\n assert(!isnan(wp[i]));\n }\n#endif // NDEBUG\n\n float max = -INFINITY;\n ggml_vec_max_f32(ne00, &max, wp);\n\n // if we have sinks, make a correction as if they were included in the softmax\n if (sk) {\n max = MAX(max, sk[i02]);\n }\n\n ggml_float sum = ggml_vec_soft_max_f32(ne00, dp, wp, max);\n assert(sum > 0.0);\n\n if (sk) {\n sum += (ggml_float) expf(sk[i02] - max);\n }\n\n sum = 1.0/sum;\n ggml_vec_scale_f32(ne00, dp, sum);\n\n#ifndef NDEBUG\n for (int i = 0; i < ne00; ++i) {\n assert(!isnan(dp[i]));\n assert(!isinf(dp[i]));\n }\n#endif // NDEBUG\n }\n }\n }\n}\n\nvoid ggml_compute_forward_soft_max(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_soft_max_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n\n// ggml_compute_forward_soft_max_ext_back\n\nstatic void ggml_compute_forward_soft_max_ext_back_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src1));\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(ggml_are_same_shape(src1, dst));\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n\n memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float));\n memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));\n\n GGML_ASSERT(max_bias == 0.0f);\n\n // TODO: handle transposed/permuted matrices\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src0->ne[0];\n const int nr = ggml_nrows(src0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n float *dy = (float *)((char *) src0->data + i1*src0->nb[1]);\n float *y = (float *)((char *) src1->data + i1*src1->nb[1]);\n float *dx = (float *)((char *) dst->data + i1*dst->nb[1]);\n\n#ifndef NDEBUG\n for (int i = 0; i < nc; ++i) {\n //printf(\"p[%d] = %f\\n\", i, p[i]);\n assert(!isnan(dy[i]));\n assert(!isnan(y[i]));\n }\n#endif // NDEBUG\n // Jii = yi - yi*yi\n // Jij = -yi*yj\n // J = diag(y)-y.T*y\n // dx = J * dy\n // dxk = sum_i(Jki * dyi)\n // dxk = sum_i(-yk*yi * dyi) - (-yk*yk)*dyk + (yk - yk*yk)*dyk\n // dxk = sum_i(-yk*yi * dyi) + yk*yk*dyk + yk*dyk - yk*yk*dyk\n // dxk = sum_i(-yk*yi * dyi) + yk*dyk\n // dxk = -yk * sum_i(yi * dyi) + yk*dyk\n // dxk = -yk * dot(y, dy) + yk*dyk\n // dxk = yk * (- dot(y, dy) + dyk)\n // dxk = yk * (dyk - dot(y, dy))\n //\n // post-order:\n // dot_y_dy := dot(y, dy)\n // dx := dy\n // dx := dx - dot_y_dy\n // dx := dx * y\n\n // linear runtime, no additional memory\n float dot_y_dy = 0;\n ggml_vec_dot_f32 (nc, &dot_y_dy, 0, y, 0, dy, 0, 1);\n ggml_vec_cpy_f32 (nc, dx, dy);\n ggml_vec_acc1_f32 (nc, dx, -dot_y_dy);\n ggml_vec_mul_f32 (nc, dx, dx, y);\n ggml_vec_scale_f32(nc, dx, scale);\n\n#ifndef NDEBUG\n for (int i = 0; i < nc; ++i) {\n assert(!isnan(dx[i]));\n assert(!isinf(dx[i]));\n }\n#endif // NDEBUG\n }\n}\n\nvoid ggml_compute_forward_soft_max_ext_back(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_soft_max_ext_back_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_clamp\n\nstatic void ggml_compute_forward_clamp_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n float min;\n float max;\n memcpy(&min, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&max, (float *) dst->op_params + 1, sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int n = ggml_nrows(src0);\n const int nc = src0->ne[0];\n\n const size_t nb00 = src0->nb[0];\n const size_t nb01 = src0->nb[1];\n\n const size_t nb0 = dst->nb[0];\n const size_t nb1 = dst->nb[1];\n\n GGML_ASSERT( nb0 == sizeof(float));\n GGML_ASSERT(nb00 == sizeof(float));\n\n for (int j = ith; j < n; j += nth) {\n float * dst_ptr = (float *) ((char *) dst->data + j*nb1);\n float * src0_ptr = (float *) ((char *) src0->data + j*nb01);\n\n for (int i = 0; i < nc; i++) {\n dst_ptr[i] = MAX(MIN(src0_ptr[i], max), min);\n }\n }\n}\n\nstatic void ggml_compute_forward_clamp_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n float min;\n float max;\n memcpy(&min, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&max, (float *) dst->op_params + 1, sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int n = ggml_nrows(src0);\n const int nc = src0->ne[0];\n\n const size_t nb00 = src0->nb[0];\n const size_t nb01 = src0->nb[1];\n\n const size_t nb0 = dst->nb[0];\n const size_t nb1 = dst->nb[1];\n\n GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));\n GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n\n for (int j = ith; j < n; j += nth) {\n ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + j*nb1);\n ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + j*nb01);\n\n for (int i = 0; i < nc; i++) {\n float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i]);\n dst_ptr[i] = GGML_CPU_FP32_TO_FP16(MAX(MIN(v, max), min));\n }\n }\n}\n\nvoid ggml_compute_forward_clamp(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_clamp_f32(params, dst);\n } break;\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_clamp_f16(params, dst);\n } break;\n case GGML_TYPE_BF16:\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q8_1:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_NVFP4:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_TQ1_0:\n case GGML_TYPE_TQ2_0:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ2_S:\n case GGML_TYPE_Q8_K:\n case GGML_TYPE_I8:\n case GGML_TYPE_I16:\n case GGML_TYPE_I32:\n case GGML_TYPE_I64:\n case GGML_TYPE_F64:\n case GGML_TYPE_COUNT:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_rope\n\nstatic float rope_yarn_ramp(const float low, const float high, const int i0) {\n const float y = (i0 / 2 - low) / MAX(0.001f, high - low);\n return 1 - MIN(1, MAX(0, y));\n}\n\n// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn\n// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.\nstatic void rope_yarn(\n float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,\n float * cos_theta, float * sin_theta) {\n // Get n-d rotational scaling corrected for extrapolation\n float theta_interp = freq_scale * theta_extrap;\n float theta = theta_interp;\n if (ext_factor != 0.0f) {\n float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;\n theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;\n\n // Get n-d magnitude scaling corrected for interpolation\n mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);\n }\n *cos_theta = cosf(theta) * mscale;\n *sin_theta = sinf(theta) * mscale;\n}\n\nstatic void ggml_rope_cache_init(\n float theta_base, float freq_scale, const float * freq_factors, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,\n float * cache, float sin_sign, float theta_scale) {\n // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py\n float theta = theta_base;\n for (int64_t i0 = 0; i0 < ne0; i0 += 2) {\n const float ff = freq_factors ? freq_factors[i0/2] : 1.0f;\n rope_yarn(\n theta/ff, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]\n );\n cache[i0 + 1] *= sin_sign;\n\n theta *= theta_scale;\n }\n}\n\nstatic void ggml_mrope_cache_init(\n float theta_base_t, float theta_base_h, float theta_base_w, float theta_base_e, int sections[4], bool is_imrope, bool indep_sects,\n float freq_scale, const float * freq_factors, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,\n float * cache, float sin_sign, float theta_scale) {\n // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py\n float theta_t = theta_base_t;\n float theta_h = theta_base_h;\n float theta_w = theta_base_w;\n float theta_e = theta_base_e; // extra position id for vision encoder\n int sect_dims = sections[0] + sections[1] + sections[2] + sections[3];\n int sec_w = sections[1] + sections[0];\n int sec_e = sections[2] + sec_w;\n GGML_ASSERT(sect_dims <= ne0);\n\n for (int64_t i0 = 0; i0 < ne0; i0 += 2) {\n const float ff = freq_factors ? freq_factors[i0/2] : 1.0f;\n\n int sector = (i0 / 2) % sect_dims;\n if (indep_sects) {\n // compute theta independently for each dim sections\n // (i.e. reset corresponding theta when `i0` go from one section to another)\n if (sector == 0) {\n theta_t = theta_base_t;\n }\n else if (sector == sections[0]) {\n theta_h = theta_base_h;;\n }\n else if (sector == sec_w) {\n theta_w = theta_base_w;\n }\n else if (sector == sec_e) {\n theta_e = theta_base_e;\n }\n }\n\n float theta = theta_t;\n if (is_imrope) { // qwen3vl apply interleaved mrope\n if (sector % 3 == 1 && sector < 3 * sections[1]) {\n theta = theta_h;\n } else if (sector % 3 == 2 && sector < 3 * sections[2]) {\n theta = theta_w;\n } else if (sector % 3 == 0 && sector < 3 * sections[0]) {\n theta = theta_t;\n } else {\n theta = theta_e;\n }\n } else {\n if (sector >= sections[0] && sector < sec_w) {\n theta = theta_h;\n }\n else if (sector >= sec_w && sector < sec_w + sections[2]) {\n theta = theta_w;\n }\n else if (sector >= sec_w + sections[2]) {\n theta = theta_e;\n }\n }\n\n rope_yarn(\n theta/ff, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]\n );\n cache[i0 + 1] *= sin_sign;\n\n theta_t *= theta_scale;\n theta_w *= theta_scale;\n theta_h *= theta_scale;\n theta_e *= theta_scale;\n }\n}\n\n\ntemplate\nstatic void rotate_pairs(const int64_t n, const int64_t n_offset, const float * cache, const T * src_data, T * dst_data, const int scale = 2) {\n for (int64_t i0 = 0; i0 < n; i0 += 2) {\n const int64_t ic = i0/scale; // hack for GGML_ROPE_TYPE_NORMAL, where we need ic = i0; for all other cases, ic = i0/2\n\n const float cos_theta = cache[i0 + 0];\n const float sin_theta = cache[i0 + 1];\n\n const T * const src = src_data + ic;\n T * dst = dst_data + ic;\n\n const float x0 = type_conversion_table::to_f32(src[0]);\n const float x1 = type_conversion_table::to_f32(src[n_offset]);\n\n dst[0] = type_conversion_table::from_f32(x0*cos_theta - x1*sin_theta);\n dst[n_offset] = type_conversion_table::from_f32(x0*sin_theta + x1*cos_theta);\n }\n}\n\ntemplate //float or ggml_fp16_t\nstatic void ggml_compute_forward_rope_flt(\n const ggml_compute_params * params,\n ggml_tensor * dst,\n const bool forward) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const ggml_tensor * src2 = dst->src[2];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n GGML_ASSERT(src1->type == GGML_TYPE_I32);\n\n float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;\n int sections[4];\n\n //const int n_past = ((int32_t *) dst->op_params)[0];\n const int n_dims = ((int32_t *) dst->op_params)[1];\n const int mode = ((int32_t *) dst->op_params)[2];\n //const int n_ctx = ((int32_t *) dst->op_params)[3];\n const int n_ctx_orig = ((int32_t *) dst->op_params)[4];\n\n memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));\n memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));\n memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));\n memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));\n memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));\n memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));\n memcpy(§ions, (int32_t *) dst->op_params + 11, sizeof(int)*4);\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n //printf(\"ne0: %d, ne1: %d, ne2: %d, ne3: %d\\n\", ne0, ne1, ne2, ne3);\n //printf(\"n_past = %d, ne2 = %d\\n\", n_past, ne2);\n\n GGML_ASSERT(nb0 == nb00);\n GGML_ASSERT(nb0 == sizeof(T));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(dst);\n\n GGML_ASSERT(n_dims <= ne0);\n GGML_ASSERT(n_dims % 2 == 0);\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n // row index used to determine which thread to use\n int ir = 0;\n\n const float theta_scale = powf(freq_base, -2.0f/n_dims);\n\n float corr_dims[2];\n ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);\n\n const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; // qwen3vl apply interleaved mrope\n const bool mrope_used = mode & GGML_ROPE_TYPE_MROPE; // ggml_rope_multi, note: also true for vision (24 & 8 == true) and for imrope\n const bool is_vision = mode == GGML_ROPE_TYPE_VISION;\n\n if (mrope_used) {\n GGML_ASSERT(sections[0] > 0 || sections[1] > 0 || sections[2] > 0);\n }\n\n if (is_vision) {\n GGML_ASSERT(n_dims == ne0/2);\n }\n\n const float * freq_factors = NULL;\n if (src2 != NULL) {\n GGML_ASSERT(src2->type == GGML_TYPE_F32);\n GGML_ASSERT(src2->ne[0] >= n_dims / 2);\n freq_factors = (const float *) src2->data;\n }\n\n // backward process uses inverse rotation by cos and sin.\n // cos and sin build a rotation matrix, where the inverse is the transpose.\n // this essentially just switches the sign of sin.\n const float sin_sign = forward ? 1.0f : -1.0f;\n\n const int32_t * pos = (const int32_t *) src1->data;\n\n int64_t last_i2 = -1;\n\n for (int64_t i3 = 0; i3 < ne3; i3++) { // batch\n for (int64_t i2 = 0; i2 < ne2; i2++) { // seq-len\n for (int64_t i1 = 0; i1 < ne1; i1++) { // attn-heads\n if (ir++ < ir0) continue; // skip rows mapped to other threads\n if (ir > ir1) break;\n\n float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;\n if (last_i2 != i2) {\n if (!mrope_used) {\n const int64_t p = pos[i2];\n ggml_rope_cache_init(p, freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);\n }\n else {\n const int64_t p_t = pos[i2];\n const int64_t p_h = pos[i2 + ne2];\n const int64_t p_w = pos[i2 + ne2 * 2];\n const int64_t p_e = pos[i2 + ne2 * 3];\n ggml_mrope_cache_init(\n p_t, p_h, p_w, p_e, sections, is_imrope, is_vision,\n freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);\n }\n\n last_i2 = i2;\n }\n\n T * src = (T *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n T * dst_data = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);\n\n switch (mode) {\n case GGML_ROPE_TYPE_NORMAL:\n rotate_pairs(n_dims, 1, cache, src, dst_data, 1);\n break;\n case GGML_ROPE_TYPE_NEOX:\n case GGML_ROPE_TYPE_MROPE:\n case GGML_ROPE_TYPE_IMROPE:\n rotate_pairs(n_dims, n_dims/2, cache, src, dst_data);\n break;\n case GGML_ROPE_TYPE_VISION:\n rotate_pairs(ne0, n_dims, cache, src, dst_data);\n break;\n default:\n GGML_ABORT(\"rope type not supported\");\n }\n\n if (!is_vision) {\n // fill the remain channels with data from src tensor\n for (int64_t i0 = n_dims; i0 < ne0; i0 += 2) {\n const T * const src = (T *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n T * dst_data = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n } //attn-heads\n }\n }\n}\n\nvoid ggml_compute_forward_rope(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_rope_flt(params, dst, true);\n } break;\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_rope_flt(params, dst, true);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_rope_back\n\nvoid ggml_compute_forward_rope_back(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_rope_flt(params, dst, false);\n } break;\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_rope_flt(params, dst, false);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_conv_transpose_1d\n\nstatic void ggml_compute_forward_conv_transpose_1d_f16_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F16);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nk = ne00*ne01*ne02;\n\n GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n GGML_ASSERT(nb10 == sizeof(float));\n\n if (ith == 0) {\n memset(params->wdata, 0, params->wsize);\n\n // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)\n {\n ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;\n\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i02*nb02 + i01*nb01);\n ggml_fp16_t * dst_data = wdata + i01*ne00*ne02;\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n dst_data[i00*ne02 + i02] = src[i00];\n }\n }\n }\n }\n\n // permute source data (src1) from (L x Cin) to (Cin x L)\n {\n ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;\n ggml_fp16_t * dst_data = wdata;\n\n for (int64_t i11 = 0; i11 < ne11; i11++) {\n const float * const src = (float *)((char *) src1->data + i11*nb11);\n for (int64_t i10 = 0; i10 < ne10; i10++) {\n dst_data[i10*ne11 + i11] = GGML_CPU_FP32_TO_FP16(src[i10]);\n }\n }\n }\n\n // need to zero dst since we are accumulating into it\n memset(dst->data, 0, ggml_nbytes(dst));\n }\n ggml_barrier(params->threadpool);\n\n const int32_t s0 = ((const int32_t*)(dst->op_params))[0];\n\n // total rows in dst\n const int nr = ne1;\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;\n ggml_fp16_t * const wdata_src = wdata + nk;\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n float * dst_data = (float *)((char *) dst->data + i1*nb1);\n ggml_fp16_t * wdata_kernel = wdata + i1*ne02*ne00;\n for (int i10 = 0; i10 < ne10; i10++) {\n const int i1n = i10*ne11;\n for (int i00 = 0; i00 < ne00; i00++) {\n float v = 0;\n ggml_vec_dot_f16(ne02, &v, 0,\n (ggml_fp16_t *) wdata_src + i1n, 0,\n (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1);\n dst_data[i10*s0 + i00] += v;\n }\n }\n }\n}\n\nstatic void ggml_compute_forward_conv_transpose_1d_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nk = ne00*ne01*ne02;\n\n GGML_ASSERT(nb00 == sizeof(float));\n GGML_ASSERT(nb10 == sizeof(float));\n\n if (ith == 0) {\n memset(params->wdata, 0, params->wsize);\n\n // prepare kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)\n {\n float * const wdata = (float *) params->wdata + 0;\n\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n const float * const src = (float *)((char *) src0->data + i02*nb02 + i01*nb01);\n float * dst_data = wdata + i01*ne00*ne02;\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n dst_data[i00*ne02 + i02] = src[i00];\n }\n }\n }\n }\n\n // prepare source data (src1)\n {\n float * const wdata = (float *) params->wdata + nk;\n float * dst_data = wdata;\n\n for (int64_t i11 = 0; i11 < ne11; i11++) {\n const float * const src = (float *)((char *) src1->data + i11*nb11);\n for (int64_t i10 = 0; i10 < ne10; i10++) {\n dst_data[i10*ne11 + i11] = src[i10];\n }\n }\n }\n\n // need to zero dst since we are accumulating into it\n memset(dst->data, 0, ggml_nbytes(dst));\n }\n ggml_barrier(params->threadpool);\n\n const int32_t s0 = ((const int32_t*)(dst->op_params))[0];\n\n // total rows in dst\n const int nr = ne1;\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n float * const wdata = (float *) params->wdata + 0;\n float * const wdata_src = wdata + nk;\n\n for (int i1 = ir0; i1 < ir1; i1++) {\n float * dst_data = (float *)((char *) dst->data + i1*nb1);\n float * wdata_kernel = wdata + i1*ne02*ne00;\n for (int i10 = 0; i10 < ne10; i10++) {\n const int i1n = i10*ne11;\n for (int i00 = 0; i00 < ne00; i00++) {\n float v = 0;\n ggml_vec_dot_f32(ne02, &v, 0,\n wdata_src + i1n, 0,\n wdata_kernel + i00*ne02, 0, 1);\n dst_data[i10*s0 + i00] += v;\n }\n }\n }\n}\n\nvoid ggml_compute_forward_conv_transpose_1d(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_conv_transpose_1d_f16_f32(params, dst);\n } break;\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_conv_transpose_1d_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_im2col_f32\n// src0: kernel [OC, IC, KH, KW]\n// src1: image [N, IC, IH, IW]\n// dst: result [N, OH, OW, IC*KH*KW]\nstatic void ggml_compute_forward_im2col_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n const int32_t p0 = ((const int32_t *)(dst->op_params))[2];\n const int32_t p1 = ((const int32_t *)(dst->op_params))[3];\n const int32_t d0 = ((const int32_t *)(dst->op_params))[4];\n const int32_t d1 = ((const int32_t *)(dst->op_params))[5];\n const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t N = is_2D ? ne13 : ne12;\n const int64_t IC = is_2D ? ne12 : ne11;\n const int64_t IH = is_2D ? ne11 : 1;\n const int64_t IW = ne10;\n\n const int64_t KH = is_2D ? ne01 : 1;\n const int64_t KW = ne00;\n\n const int64_t OH = is_2D ? ne2 : 1;\n const int64_t OW = ne1;\n\n int ofs0 = is_2D ? nb13 : nb12;\n int ofs1 = is_2D ? nb12 : nb11;\n\n GGML_ASSERT(nb10 == sizeof(float));\n\n // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]\n {\n float * const wdata = (float *) dst->data;\n\n for (int64_t in = 0; in < N; in++) {\n for (int64_t ioh = 0; ioh < OH; ioh++) { // 1\n for (int64_t iow = 0; iow < OW; iow++) {\n for (int64_t iic = ith; iic < IC; iic += nth) {\n\n // micro kernel\n float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]\n const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]\n\n for (int64_t ikh = 0; ikh < KH; ikh++) { // 1\n for (int64_t ikw = 0; ikw < KW; ikw++) {\n const int64_t iiw = iow*s0 + ikw*d0 - p0;\n const int64_t iih = ioh*s1 + ikh*d1 - p1;\n\n if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {\n dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;\n } else {\n dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]);\n }\n }\n }\n }\n }\n }\n }\n }\n}\n\n\n// ggml_compute_forward_im2col_f16\n// src0: kernel [OC, IC, KH, KW]\n// src1: image [N, IC, IH, IW]\n// dst: result [N, OH, OW, IC*KH*KW]\nstatic void ggml_compute_forward_im2col_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F16);\n GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F16);\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n const int32_t p0 = ((const int32_t *)(dst->op_params))[2];\n const int32_t p1 = ((const int32_t *)(dst->op_params))[3];\n const int32_t d0 = ((const int32_t *)(dst->op_params))[4];\n const int32_t d1 = ((const int32_t *)(dst->op_params))[5];\n const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t N = is_2D ? ne13 : ne12;\n const int64_t IC = is_2D ? ne12 : ne11;\n const int64_t IH = is_2D ? ne11 : 1;\n const int64_t IW = ne10;\n\n const int64_t KH = is_2D ? ne01 : 1;\n const int64_t KW = ne00;\n\n const int64_t OH = is_2D ? ne2 : 1;\n const int64_t OW = ne1;\n\n int ofs0 = is_2D ? nb13 : nb12;\n int ofs1 = is_2D ? nb12 : nb11;\n\n GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]\n {\n ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;\n\n for (int64_t in = 0; in < N; in++) {\n for (int64_t ioh = 0; ioh < OH; ioh++) { // 1\n for (int64_t iow = 0; iow < OW; iow++) {\n for (int64_t iic = ith; iic < IC; iic += nth) {\n\n // micro kernel\n ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]\n const float * const src_data_f32 = src1->type == GGML_TYPE_F32\n ? (const float *)((const char *) src1->data + in*ofs0 + iic*ofs1)\n : nullptr; // [IH, IW]\n const ggml_fp16_t * const src_data_f16 = src1->type == GGML_TYPE_F16\n ? (const ggml_fp16_t *)((const char *) src1->data + in*ofs0 + iic*ofs1)\n : nullptr; // [IH, IW]\n\n for (int64_t ikh = 0; ikh < KH; ikh++) { // 1\n for (int64_t ikw = 0; ikw < KW; ikw++) {\n const int64_t iiw = iow*s0 + ikw*d0 - p0;\n const int64_t iih = ioh*s1 + ikh*d1 - p1;\n\n if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {\n dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;\n } else {\n if (src_data_f32 != nullptr) {\n dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_CPU_FP32_TO_FP16(src_data_f32[iih*IW + iiw]);\n } else {\n dst_data[iic*(KH*KW) + ikh*KW + ikw] = src_data_f16[iih*IW + iiw];\n }\n }\n }\n }\n }\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_im2col(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n switch (dst->type) {\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_im2col_f16(params, dst);\n } break;\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_im2col_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_im2col_back_f32\n\nvoid ggml_compute_forward_im2col_back_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0]; // gradients of forward pass output\n const ggml_tensor * src1 = dst->src[1]; // convolution kernel\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n const int32_t p0 = ((const int32_t *)(dst->op_params))[2];\n const int32_t p1 = ((const int32_t *)(dst->op_params))[3];\n const int32_t d0 = ((const int32_t *)(dst->op_params))[4];\n const int32_t d1 = ((const int32_t *)(dst->op_params))[5];\n const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t N = is_2D ? ne3 : ne2;\n const int64_t IC = is_2D ? ne2 : ne1;\n const int64_t IH = is_2D ? ne1 : 1;\n const int64_t IW = ne0;\n\n const int64_t KH = is_2D ? ne11 : 1;\n const int64_t KW = ne10;\n\n const int64_t OH = is_2D ? ne02 : 1;\n const int64_t OW = ne01;\n\n int ofs0 = is_2D ? nb3 : nb2;\n int ofs1 = is_2D ? nb2 : nb1;\n\n GGML_ASSERT(nb0 == sizeof(float));\n\n // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]\n {\n float * const wdata = (float *) dst->data;\n\n for (int64_t in = 0; in < N; in++) {\n for (int64_t iic = ith; iic < IC; iic += nth) {\n for (int64_t iih = 0; iih < IH; iih++) {\n for (int64_t iiw = 0; iiw < IW; iiw++) {\n\n // micro kernel\n float grad = 0.0f;\n for (int64_t ikh = 0; ikh < KH; ikh++) {\n for (int64_t ikw = 0; ikw < KW; ikw++) {\n // For s0 > 1 some values were skipped over in the forward pass.\n // These values have tmpw % s0 != 0 and need to be skipped in the backwards pass as well.\n const int64_t tmpw = (iiw + p0 - ikw*d0);\n if (tmpw % s0 != 0) {\n continue;\n }\n const int64_t iow = tmpw / s0;\n\n // Equivalent logic as above except for s1.\n int64_t ioh;\n if (is_2D) {\n const int64_t tmph = iih + p1 - ikh*d1;\n\n if (tmph % s1 != 0) {\n continue;\n }\n\n ioh = tmph / s1;\n } else {\n ioh = 0;\n }\n\n if (iow < 0 || iow >= OW || ioh < 0 || ioh >= OH) {\n continue;\n }\n\n const float * const grad_in = (const float *) src0->data\n + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]\n grad += grad_in[iic*(KH*KW) + ikh*KW + ikw];\n }\n }\n float * dst_data = (float *)((char *) wdata + (in*ofs0 + iic*ofs1)); // [IH, IW]\n dst_data[iih*IW + iiw] = grad;\n }\n }\n }\n }\n }\n}\n\n\n// ggml_compute_forward_im2col_3d_f16\n// src0: kernel [OC*IC, KD, KH, KW]\n// src1: image [N*IC, ID, IH, IW]\n// dst: result [N*OD, OH, OW, IC * KD * KH * KW]\nstatic void ggml_compute_forward_im2col_3d_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F16);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F16);\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n const int32_t s2 = ((const int32_t *)(dst->op_params))[2];\n const int32_t p0 = ((const int32_t *)(dst->op_params))[3];\n const int32_t p1 = ((const int32_t *)(dst->op_params))[4];\n const int32_t p2 = ((const int32_t *)(dst->op_params))[5];\n const int32_t d0 = ((const int32_t *)(dst->op_params))[6];\n const int32_t d1 = ((const int32_t *)(dst->op_params))[7];\n const int32_t d2 = ((const int32_t *)(dst->op_params))[8];\n const int32_t IC = ((const int32_t *)(dst->op_params))[9];\n\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t N = ne13 / IC;\n const int64_t ID = ne12;\n const int64_t IH = ne11;\n const int64_t IW = ne10;\n\n const int64_t OC = ne03 / IC;\n GGML_UNUSED(OC);\n const int64_t KD = ne02;\n const int64_t KH = ne01;\n const int64_t KW = ne00;\n\n const int64_t OD = ne3 / N;\n const int64_t OH = ne2;\n const int64_t OW = ne1;\n const int64_t OH_OW = OH*OW;\n const int64_t KD_KH_KW = KD*KH*KW;\n const int64_t KH_KW = KH*KW;\n const int64_t IC_KD_KH_KW = IC*KD*KH*KW;\n\n GGML_ASSERT(nb10 == sizeof(float));\n\n // im2col: [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW]\n {\n ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;\n\n for (int64_t in = 0; in < N; in++) {\n for (int64_t iod = 0; iod < OD; iod++) {\n for (int64_t ioh = 0; ioh < OH; ioh++) {\n for (int64_t iow = 0; iow < OW; iow++) {\n for (int64_t iic = ith; iic < IC; iic += nth) {\n\n // micro kernel\n ggml_fp16_t * dst_data = wdata + (in*OD*OH_OW + iod*OH_OW + ioh*OW + iow)*IC_KD_KH_KW; // [IC, KD, KH, KW]\n const float * const src_data = (const float *) ((const char *)src1->data + (in*IC + iic)*nb13); // [ID, IH, IW]\n\n for (int64_t ikd = 0; ikd < KD; ikd++) {\n for (int64_t ikh = 0; ikh < KH; ikh++) {\n for (int64_t ikw = 0; ikw < KW; ikw++) {\n const int64_t iiw = iow*s0 + ikw*d0 - p0;\n const int64_t iih = ioh*s1 + ikh*d1 - p1;\n const int64_t iid = iod*s2 + ikd*d2 - p2;\n\n if (iid < 0 || iid >= ID || iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {\n dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = 0;\n } else {\n const float * const s = (const float *) ((const char *)src_data + iid*nb12 + iih*nb11 + iiw*nb10); // [ID, IH, IW]\n dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = GGML_CPU_FP32_TO_FP16(*s);\n }\n }\n }\n }\n }\n }\n }\n }\n }\n }\n}\n\n// ggml_compute_forward_im2col_3d_f32\n// src0: kernel [OC*IC, KD, KH, KW]\n// src1: image [N*IC, ID, IH, IW]\n// dst: result [N*OD, OH, OW, IC * KD * KH * KW]\nstatic void ggml_compute_forward_im2col_3d_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n const int32_t s2 = ((const int32_t *)(dst->op_params))[2];\n const int32_t p0 = ((const int32_t *)(dst->op_params))[3];\n const int32_t p1 = ((const int32_t *)(dst->op_params))[4];\n const int32_t p2 = ((const int32_t *)(dst->op_params))[5];\n const int32_t d0 = ((const int32_t *)(dst->op_params))[6];\n const int32_t d1 = ((const int32_t *)(dst->op_params))[7];\n const int32_t d2 = ((const int32_t *)(dst->op_params))[8];\n const int32_t IC = ((const int32_t *)(dst->op_params))[9];\n\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t N = ne13 / IC;\n const int64_t ID = ne12;\n const int64_t IH = ne11;\n const int64_t IW = ne10;\n\n const int64_t OC = ne03 / IC;\n GGML_UNUSED(OC);\n const int64_t KD = ne02;\n const int64_t KH = ne01;\n const int64_t KW = ne00;\n\n const int64_t OD = ne3 / N;\n const int64_t OH = ne2;\n const int64_t OW = ne1;\n\n const int64_t OH_OW = OH*OW;\n const int64_t KD_KH_KW = KD*KH*KW;\n const int64_t KH_KW = KH*KW;\n const int64_t IC_KD_KH_KW = IC*KD*KH*KW;\n\n GGML_ASSERT(nb10 == sizeof(float));\n\n // im2col: [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW]\n {\n float * const wdata = (float *) dst->data;\n\n for (int64_t in = 0; in < N; in++) {\n for (int64_t iod = 0; iod < OD; iod++) {\n for (int64_t ioh = 0; ioh < OH; ioh++) {\n for (int64_t iow = 0; iow < OW; iow++) {\n for (int64_t iic = ith; iic < IC; iic += nth) {\n\n // micro kernel\n float * dst_data = wdata + (in*OD*OH_OW + iod*OH_OW + ioh*OW + iow)*IC_KD_KH_KW; // [IC, KD, KH, KW]\n const float * const src_data = (const float *) ((const char *)src1->data + (in*IC + iic)*nb13); // [ID, IH, IW]\n\n for (int64_t ikd = 0; ikd < KD; ikd++) {\n for (int64_t ikh = 0; ikh < KH; ikh++) {\n for (int64_t ikw = 0; ikw < KW; ikw++) {\n const int64_t iiw = iow*s0 + ikw*d0 - p0;\n const int64_t iih = ioh*s1 + ikh*d1 - p1;\n const int64_t iid = iod*s2 + ikd*d2 - p2;\n\n if (iid < 0 || iid >= ID || iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) {\n dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = 0;\n } else {\n const float * const s = (const float *) ((const char *)src_data + iid*nb12 + iih*nb11 + iiw*nb10); // [ID, IH, IW]\n dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = *s;\n }\n }\n }\n }\n }\n }\n }\n }\n }\n }\n}\n\n\nvoid ggml_compute_forward_im2col_3d(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n switch (dst->type) {\n case GGML_TYPE_F16:\n {\n ggml_compute_forward_im2col_3d_f16(params, dst);\n } break;\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_im2col_3d_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nstatic void ggml_call_mul_mat(ggml_type type, const ggml_compute_params * params, int64_t m, int64_t n, int64_t k,\n void * a, void * b, float * c) {\n const ggml_type_traits * traits = ggml_get_type_traits(type);\n struct ggml_tensor src1 = {};\n src1.type = type;\n src1.ne[0] = k;\n src1.ne[1] = m;\n src1.ne[2] = 1;\n src1.ne[3] = 1;\n src1.nb[0] = traits->type_size;\n src1.nb[1] = k * traits->type_size;\n src1.nb[2] = src1.nb[1];\n src1.nb[3] = src1.nb[2];\n src1.data = a;\n\n struct ggml_tensor src0 = {};\n src0.type = type;\n src0.ne[0] = k;\n src0.ne[1] = n;\n src0.ne[2] = 1;\n src0.ne[3] = 1;\n src0.nb[0] = traits->type_size;\n src0.nb[1] = k * traits->type_size;\n src0.nb[2] = src0.nb[1];\n src0.nb[3] = src0.nb[2];\n src0.data = b;\n\n struct ggml_tensor dst = {};\n dst.ne[0] = n;\n dst.ne[1] = m;\n dst.ne[2] = 1;\n dst.ne[3] = 1;\n dst.nb[0] = sizeof(float);\n dst.nb[1] = n * sizeof(float);\n dst.nb[2] = dst.nb[1];\n dst.nb[3] = dst.nb[2];\n dst.data = c;\n dst.src[0] = &src0;\n dst.src[1] = &src1;\n\n ggml_compute_forward_mul_mat(params, &dst);\n}\n\nstatic inline int64_t ggml_wrap_around(int64_t coord, int64_t size) {\n return (coord + size) % size; // adding size avoids negative number weirdness\n}\n\n// ggml_compute_forward_conv_2d\n\n\nstatic void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params,\n const ggml_tensor * kernel, // [KW, KH, IC, OC]\n const ggml_tensor * src, // [W, H, C, N]\n ggml_tensor * dst, // [OW, OH, OC, N]\n ggml_type kernel_type) {\n\n GGML_ASSERT(ggml_is_contiguous(kernel));\n GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32);\n GGML_ASSERT(kernel->type == kernel_type);\n\n const ggml_type_traits * traits = ggml_get_type_traits(kernel_type);\n\n const int32_t stride_x = dst->op_params[0];\n const int32_t stride_y = dst->op_params[1];\n const int32_t pad_x = dst->op_params[2];\n const int32_t pad_y = dst->op_params[3];\n const int32_t dilation_x = dst->op_params[4];\n const int32_t dilation_y = dst->op_params[5];\n\n const int64_t c_in = src->ne[2];\n const int64_t c_out = kernel->ne[3];\n GGML_ASSERT(c_in == kernel->ne[2]);\n\n const int64_t src_w = src->ne[0];\n const int64_t src_h = src->ne[1];\n const int64_t knl_w = kernel->ne[0];\n const int64_t knl_h = kernel->ne[1];\n const int64_t dst_w = dst->ne[0];\n const int64_t dst_h = dst->ne[1];\n\n const float * src_data = (float *) src->data;\n void * knl_data = kernel->data;\n float * dst_data = (float *) dst->data;\n\n const int64_t knl_n = knl_w * knl_h * c_in;\n const int64_t patch_total = dst->ne[3] * dst_w * dst_h;\n\n const int64_t space_per_patch = knl_n * traits->type_size + c_out * sizeof(float);\n const int64_t batch_size = params->wsize / space_per_patch;\n const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size;\n const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch;\n\n GGML_ASSERT(patches_per_batch > 0 && batch_size >= 1);\n\n void * tmp = params->wdata;\n\n for (int64_t batch_i = 0; batch_i < batch_n; ++batch_i) {\n\n const int64_t patch_start_batch = batch_i * patches_per_batch;\n const int64_t patch_end_batch = std::min(patch_start_batch + patches_per_batch,\n patch_total);\n const int64_t patch_n = patch_end_batch - patch_start_batch;\n\n const int64_t patch_per_thread = (patch_n + params->nth - 1) / params->nth;\n const int64_t patch_start = patch_start_batch + params->ith * patch_per_thread;\n const int64_t patch_end = std::min(patch_start + patch_per_thread, patch_end_batch);\n\n //im2col for a patch\n for (int64_t p = patch_start; p < patch_end; ++p) {\n const int64_t batch_n = p / (dst_w * dst_h);\n const int64_t src_x = (p / dst_w) % dst_h;\n const int64_t src_y = p % dst_w;\n\n const float * src_base = (const float *)((const char *)src_data + batch_n * src->nb[3]);\n char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n * traits->type_size;\n\n for (int64_t ic = 0; ic < c_in; ++ic) {\n for (int64_t ky = 0; ky < knl_h; ++ky) {\n for (int64_t kx = 0; kx < knl_w; ++kx) {\n const int64_t sy = src_x * stride_y + ky * dilation_y - pad_y;\n const int64_t sx = src_y * stride_x + kx * dilation_x - pad_x;\n\n int64_t dst_idx = ic * (knl_h * knl_w) + ky * knl_w + kx;\n\n float src_val;\n if (sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) {\n src_val = 0.0f;\n } else {\n const float * src_ptr = (const float *)((const char *)src_base + sx * src->nb[0] + sy * src->nb[1] + ic * src->nb[2]);\n src_val = *src_ptr;\n }\n\n char * element_ptr = dst_row + dst_idx * traits->type_size;\n if (kernel_type == GGML_TYPE_F32) {\n *(float *) element_ptr = src_val;\n } else if (kernel_type == GGML_TYPE_F16) {\n *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val);\n }\n }\n }\n }\n } // patches handled by this thread\n\n ggml_barrier(params->threadpool);\n\n float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n * traits->type_size);\n\n GGML_ASSERT(gemm_output + patch_n * c_out <= (float*)tmp + params->wsize);\n\n // GEMM: patches[patch_n, knl_n] × kernel[knl_n, c_out] = output[patch_n, c_out]\n ggml_call_mul_mat(kernel_type, params, patch_n, c_out, knl_n, tmp, knl_data, gemm_output);\n\n ggml_barrier(params->threadpool);\n\n\n //permute back [OC, N, OH, OW] to [N, OC, OH, OW]\n const int64_t permute_per_thread = (patch_n + params->nth - 1) / params->nth;\n const int64_t permute_start = params->ith * permute_per_thread;\n const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n);\n\n for (int64_t i = permute_start; i < permute_end; ++i) {\n const int64_t p = patch_start_batch + i;\n const int64_t batch_n = p / (dst_w * dst_h);\n const int64_t dst_y = (p / dst_w) % dst_h;\n const int64_t dst_x = p % dst_w;\n\n for (int64_t oc = 0; oc < c_out; ++oc) {\n const float value = gemm_output[i * c_out + oc];\n float * dst_ptr = (float *)((char *)dst_data + dst_x * dst->nb[0] + dst_y * dst->nb[1] + oc * dst->nb[2] + batch_n * dst->nb[3]);\n *dst_ptr = value;\n }\n }\n }\n}\n\nvoid ggml_compute_forward_conv_2d(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n ggml_compute_forward_conv_2d_impl(params, src0, src1, dst, src0->type);\n}\n\n// ggml_compute_forward_conv_3d\n\nstatic void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params,\n const ggml_tensor * kernel,\n const ggml_tensor * src,\n ggml_tensor * dst,\n ggml_type kernel_type) {\n\n GGML_ASSERT(ggml_is_contiguous(kernel));\n GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32);\n GGML_ASSERT(kernel->type == kernel_type);\n\n const ggml_type_traits * traits = ggml_get_type_traits(kernel_type);\n\n const int32_t s0 = dst->op_params[0];\n const int32_t s1 = dst->op_params[1];\n const int32_t s2 = dst->op_params[2];\n const int32_t p0 = dst->op_params[3];\n const int32_t p1 = dst->op_params[4];\n const int32_t p2 = dst->op_params[5];\n const int32_t d0 = dst->op_params[6];\n const int32_t d1 = dst->op_params[7];\n const int32_t d2 = dst->op_params[8];\n const int32_t c = dst->op_params[9];\n const int32_t n = dst->op_params[10];\n const int32_t oc = dst->op_params[11];\n\n const int64_t src_w = src->ne[0];\n const int64_t src_h = src->ne[1];\n const int64_t src_d = src->ne[2];\n const int64_t knl_w = kernel->ne[0];\n const int64_t knl_h = kernel->ne[1];\n const int64_t knl_d = kernel->ne[2];\n const int64_t dst_w = dst->ne[0];\n const int64_t dst_h = dst->ne[1];\n const int64_t dst_d = dst->ne[2];\n\n const float * src_data = (float *) src->data;\n void * knl_data = kernel->data;\n float * dst_data = (float *) dst->data;\n\n const int64_t knl_n_per_channel = knl_w * knl_h * knl_d;\n const int64_t knl_n_total = knl_n_per_channel * c;\n const int64_t patch_total = n * dst_w * dst_h * dst_d;\n\n const int64_t space_per_patch = knl_n_total * traits->type_size + oc * sizeof(float);\n const int64_t batch_size = params->wsize / space_per_patch;\n const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size;\n const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch;\n\n GGML_ASSERT(patches_per_batch > 0 && batch_size >= 1);\n\n void * tmp = params->wdata;\n\n for (int64_t batch_i = 0; batch_i < batch_n; ++batch_i) {\n const int64_t patch_start_batch = batch_i * patches_per_batch;\n const int64_t patch_end_batch = std::min(patch_start_batch + patches_per_batch, patch_total);\n const int64_t patch_n_in_batch = patch_end_batch - patch_start_batch;\n\n const int64_t patch_per_thread = (patch_n_in_batch + params->nth - 1) / params->nth;\n const int64_t patch_start = patch_start_batch + params->ith * patch_per_thread;\n const int64_t patch_end = std::min(patch_start + patch_per_thread, patch_end_batch);\n\n for (int64_t p = patch_start; p < patch_end; ++p) {\n const int64_t p_in_batch = p % (dst_w * dst_h * dst_d);\n const int64_t p_in_depth = p_in_batch % (dst_w * dst_h);\n const int64_t batch_idx = p / (dst_w * dst_h * dst_d);\n const int64_t dst_z = p_in_batch / (dst_w * dst_h);\n const int64_t dst_y = p_in_depth / dst_w;\n const int64_t dst_x = p_in_depth % dst_w;\n\n char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n_total * traits->type_size;\n\n for (int64_t ic = 0; ic < c; ++ic) {\n for (int64_t kz = 0; kz < knl_d; ++kz) {\n for (int64_t ky = 0; ky < knl_h; ++ky) {\n for (int64_t kx = 0; kx < knl_w; ++kx) {\n const int64_t sz = dst_z * s2 + kz * d2 - p2;\n const int64_t sy = dst_y * s1 + ky * d1 - p1;\n const int64_t sx = dst_x * s0 + kx * d0 - p0;\n\n int64_t dst_idx = ic * knl_n_per_channel + kz * (knl_h * knl_w) + ky * knl_w + kx;\n\n float src_val;\n if (sz < 0 || sz >= src_d || sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) {\n src_val = 0.0f;\n } else {\n const int64_t cn_idx = batch_idx * c + ic;\n const float * src_ptr = (const float *)((const char *)src_data + sx*src->nb[0] + sy*src->nb[1] + sz*src->nb[2] + cn_idx*src->nb[3]);\n src_val = *src_ptr;\n }\n\n char * element_ptr = dst_row + dst_idx * traits->type_size;\n if (kernel_type == GGML_TYPE_F32) {\n *(float *)element_ptr = src_val;\n } else if (kernel_type == GGML_TYPE_F16) {\n *(ggml_fp16_t *)element_ptr = GGML_CPU_FP32_TO_FP16(src_val);\n }\n }\n }\n }\n }\n }\n\n ggml_barrier(params->threadpool);\n\n float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n_total * traits->type_size);\n ggml_call_mul_mat(kernel_type, params, patch_n_in_batch, oc, knl_n_total, tmp, knl_data, gemm_output);\n\n ggml_barrier(params->threadpool);\n\n const int64_t permute_per_thread = (patch_n_in_batch + params->nth - 1) / params->nth;\n const int64_t permute_start = params->ith * permute_per_thread;\n const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n_in_batch);\n\n for (int64_t i = permute_start; i < permute_end; ++i) {\n const int64_t p = patch_start_batch + i;\n const int64_t p_in_batch = p % (dst_w * dst_h * dst_d);\n const int64_t p_in_depth = p_in_batch % (dst_w * dst_h);\n const int64_t batch_idx = p / (dst_w * dst_h * dst_d);\n const int64_t dst_z = p_in_batch / (dst_w * dst_h);\n const int64_t dst_y = p_in_depth / dst_w;\n const int64_t dst_x = p_in_depth % dst_w;\n\n for (int64_t ioc = 0; ioc < oc; ++ioc) {\n const float value = gemm_output[i * oc + ioc];\n const int64_t ocn_idx = batch_idx * oc + ioc;\n float * dst_ptr = (float *)((char *)dst_data + dst_x*dst->nb[0] + dst_y*dst->nb[1] + dst_z*dst->nb[2] + ocn_idx*dst->nb[3]);\n *dst_ptr = value;\n }\n }\n }\n}\n\nvoid ggml_compute_forward_conv_3d(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n ggml_compute_forward_conv_3d_impl(params, src0, src1, dst, src0->type);\n}\n\n// ggml_compute_forward_conv_transpose_2d\n\nvoid ggml_compute_forward_conv_transpose_2d(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F16);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nk = ne00*ne01*ne02*ne03;\n\n GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n GGML_ASSERT(nb10 == sizeof(float));\n\n if (ith == 0) {\n memset(params->wdata, 0, params->wsize);\n\n // permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout)\n {\n ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i03*nb03 + i02*nb02);\n ggml_fp16_t * dst_data = wdata + i02*ne01*ne00*ne03;\n for (int64_t i01 = 0; i01 < ne01; i01++) {\n for (int64_t i00 = 0; i00 < ne00; i00++) {\n dst_data[i01*ne00*ne03 + i00*ne03 + i03] = src[i01 * ne00 + i00];\n }\n }\n }\n }\n }\n\n // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh)\n {\n ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;\n for (int i12 = 0; i12 < ne12; i12++) {\n for (int i11 = 0; i11 < ne11; i11++) {\n const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);\n ggml_fp16_t * dst_data = wdata + i11*ne10*ne12;\n for (int i10 = 0; i10 < ne10; i10++) {\n dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);\n }\n }\n }\n }\n\n memset(dst->data, 0, ggml_nbytes(dst));\n }\n ggml_barrier(params->threadpool);\n\n const int32_t stride = ggml_get_op_params_i32(dst, 0);\n\n // total patches in dst\n const int np = ne2;\n\n // patches per thread\n const int dp = (np + nth - 1)/nth;\n\n // patch range for this thread\n const int ip0 = dp*ith;\n const int ip1 = MIN(ip0 + dp, np);\n\n ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;\n ggml_fp16_t * const wdata_src = wdata + nk;\n\n for (int i2 = ip0; i2 < ip1; i2++) { // Cout\n float * dst_data = (float *)((char *) dst->data + i2*nb2);\n ggml_fp16_t * wdata_kernel = wdata + i2*ne01*ne00*ne03;\n for (int i11 = 0; i11 < ne11; i11++) {\n for (int i10 = 0; i10 < ne10; i10++) {\n const int i1n = i11*ne10*ne12 + i10*ne12;\n for (int i01 = 0; i01 < ne01; i01++) {\n for (int i00 = 0; i00 < ne00; i00++) {\n float v = 0;\n ggml_vec_dot_f16(ne03, &v, 0,\n wdata_src + i1n, 0,\n wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);\n dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;\n }\n }\n }\n }\n }\n}\n\n// ggml_compute_forward_conv_2d_dw\n\nstruct ggml_conv_2d_dw_params {\n int64_t channels;\n int64_t batch;\n int64_t src_w;\n int64_t src_h;\n int64_t dst_w;\n int64_t dst_h;\n int64_t knl_w;\n int64_t knl_h;\n int stride_x;\n int stride_y;\n int pad_x;\n int pad_y;\n int dilation_x;\n int dilation_y;\n};\n\nstatic void ggml_compute_forward_conv_2d_dw_cwhn(\n const ggml_compute_params * params,\n const ggml_tensor * src,\n const ggml_tensor * kernel,\n ggml_tensor * dst,\n const ggml_conv_2d_dw_params & p) {\n\n const int64_t c = p.channels;\n const float * knl_data = (const float *)kernel->data;\n\n const int64_t rows_total = p.dst_h * p.batch;\n const int64_t rows_per_thread = (rows_total + params->nth - 1) / params->nth;\n const int64_t row_start = params->ith * rows_per_thread;\n const int64_t row_end = MIN(row_start + rows_per_thread, rows_total);\n\n#ifdef GGML_SIMD\n #if defined(__ARM_FEATURE_SVE)\n const int64_t pkg_size = svcntw();\n #else\n const int64_t pkg_size = GGML_F32_EPR;\n #endif\n const int64_t pkg_count = c / pkg_size;\n const int64_t c_pkg_end = pkg_count * pkg_size;\n#else\n const int64_t c_pkg_end = 0;\n#endif\n\n for (int64_t row = row_start; row < row_end; ++row) {\n const int64_t dst_y = row % p.dst_h;\n const float * src_data = (const float *)src->data + (row / p.dst_h) * p.src_w * p.src_h * c;\n for (int64_t dst_x = 0; dst_x < p.dst_w; ++dst_x) {\n float * dst_data = (float *)dst->data + (row * p.dst_w + dst_x) * c;\n const int64_t src_y_base = dst_y * p.stride_y - p.pad_y;\n const int64_t src_x_base = dst_x * p.stride_x - p.pad_x;\n\n#ifdef GGML_SIMD\n // Vectorized loop\n for (int64_t c_i = 0; c_i < c_pkg_end; c_i += pkg_size) {\n GGML_F32_VEC sum = GGML_F32_VEC_ZERO;\n for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) {\n const int64_t src_y = src_y_base + knl_y * p.dilation_y;\n if (src_y < 0 || src_y >= p.src_h) {\n continue;\n }\n for (int64_t knl_x = 0; knl_x < p.knl_w; ++knl_x) {\n const int64_t src_x = src_x_base + knl_x * p.dilation_x;\n if (src_x < 0 || src_x >= p.src_w) {\n continue;\n }\n GGML_F32_VEC k = GGML_F32_VEC_LOAD(knl_data + (knl_y * p.knl_w + knl_x) * c + c_i);\n GGML_F32_VEC s = GGML_F32_VEC_LOAD(src_data + (src_y * p.src_w + src_x) * c + c_i);\n sum = GGML_F32_VEC_FMA(sum, k, s);\n }\n }\n GGML_F32_VEC_STORE(dst_data + c_i, sum);\n }\n#endif\n // Scalar loop\n for (int64_t c_i = c_pkg_end; c_i < c; ++c_i) {\n float sum = 0.0f;\n for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) {\n const int64_t src_y = src_y_base + knl_y * p.dilation_y;\n if (src_y < 0 || src_y >= p.src_h) {\n continue;\n }\n for (int64_t knl_x = 0; knl_x < p.knl_w; ++knl_x) {\n const int64_t src_x = src_x_base + knl_x * p.dilation_x;\n if (src_x < 0 || src_x >= p.src_w) {\n continue;\n }\n sum += knl_data[(knl_y * p.knl_w + knl_x) * c + c_i]\n * src_data[(src_y * p.src_w + src_x) * c + c_i];\n }\n }\n dst_data[c_i] = sum;\n }\n }\n }\n}\n\nstatic void ggml_compute_forward_conv_2d_dw_whcn(\n const ggml_compute_params * params,\n const ggml_tensor * src,\n const ggml_tensor * kernel,\n ggml_tensor * dst,\n const ggml_conv_2d_dw_params & p) {\n\n const int64_t n = p.channels * p.batch;\n const int64_t per_thread = (n + params->nth - 1) / params->nth;\n const int64_t start = params->ith * per_thread;\n const int64_t end = MIN(start + per_thread, n);\n\n for (int64_t i = start; i < end; ++i) {\n const float * knl_data = (const float *)kernel->data + (i % p.channels) * p.knl_w * p.knl_h;\n const float * src_data = (const float *)src->data + i * p.src_w * p.src_h;\n float * dst_data = (float *)dst->data + i * p.dst_w * p.dst_h;\n\n for (int64_t dst_y = 0; dst_y < p.dst_h; ++dst_y) {\n for (int64_t dst_x = 0; dst_x < p.dst_w; ++dst_x) {\n\n float sum = 0.0f;\n for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) {\n const int64_t src_y = dst_y * p.stride_y + knl_y * p.dilation_y - p.pad_y;\n if (src_y < 0 || src_y >= p.src_h) {\n continue;\n }\n for (int64_t knl_x = 0; knl_x < p.knl_w; ++knl_x) {\n const int64_t src_x = dst_x * p.stride_x + knl_x * p.dilation_x - p.pad_x;\n if (src_x < 0 || src_x >= p.src_w) {\n continue;\n }\n sum += knl_data[knl_y * p.knl_w + knl_x]\n * src_data[src_y * p.src_w + src_x];\n }\n }\n dst_data[dst_y * p.dst_w + dst_x] = sum;\n }\n }\n }\n}\n\nvoid ggml_compute_forward_conv_2d_dw(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * kernel = dst->src[0];\n const ggml_tensor * src = dst->src[1];\n ggml_conv_2d_dw_params p;\n p.channels = src->ne[2];\n p.batch = src->ne[3];\n p.src_w = src->ne[0];\n p.src_h = src->ne[1];\n p.dst_w = dst->ne[0];\n p.dst_h = dst->ne[1];\n p.knl_w = kernel->ne[0];\n p.knl_h = kernel->ne[1];\n p.stride_x = dst->op_params[0];\n p.stride_y = dst->op_params[1];\n p.pad_x = dst->op_params[2];\n p.pad_y = dst->op_params[3];\n p.dilation_x = dst->op_params[4];\n p.dilation_y = dst->op_params[5];\n\n GGML_ASSERT(kernel->ne[3] == p.channels);\n GGML_ASSERT(dst->ne[3] == p.batch);\n\n if (ggml_is_contiguous(src)) {\n ggml_compute_forward_conv_2d_dw_whcn(params, src, kernel, dst, p);\n } else if (ggml_is_contiguous_channels(src)) {\n // kernel should also have channels most contiguous in memory\n GGML_ASSERT(kernel->nb[0] >= kernel->nb[2] && kernel->nb[1] >= kernel->nb[0]);\n ggml_compute_forward_conv_2d_dw_cwhn(params, src, kernel, dst, p);\n } else {\n GGML_ABORT(\"non-contiguous memory layout not supported\");\n }\n}\n\n// ggml_compute_forward_pool_1d_ksp\nstatic void ggml_compute_forward_pool_1d_ksp(\n const ggml_compute_params * params,\n const ggml_op_pool op,\n const int k,\n const int s,\n const int p,\n ggml_tensor * dst) {\n\n const ggml_tensor * src = dst->src[0];\n\n assert(src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16);\n\n if (params->ith != 0) {\n return;\n }\n\n const int64_t IW = src->ne[0];\n const int64_t OW = dst->ne[0];\n\n const int64_t nr = ggml_nrows(src);\n\n for (int64_t ir = 0; ir < nr; ++ir) {\n const char * srow_bytes = (const char *) src->data + ir * src->nb[1];\n float * drow = (float *) (( char *) dst->data + ir * dst->nb[1]);\n\n for (int64_t ow = 0; ow < OW; ++ow) {\n float res = 0;\n switch (op) {\n case GGML_OP_POOL_AVG: res = 0.0f; break;\n case GGML_OP_POOL_MAX: res = -FLT_MAX; break;\n case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n }\n\n int count = 0;\n const int base = (int) ow * s - p;\n\n for (int ki = 0; ki < k; ++ki) {\n const int j = base + ki;\n if (j < 0 || j >= (int) IW) {\n continue;\n }\n\n float v;\n if (src->type == GGML_TYPE_F32) {\n v = ((const float *) srow_bytes)[j];\n } else {\n v = GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t *) srow_bytes)[j]);\n }\n\n switch (op) {\n case GGML_OP_POOL_AVG: res += v; break;\n case GGML_OP_POOL_MAX: res = std::max(v, res); break;\n case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n }\n\n ++count;\n }\n\n switch (op) {\n case GGML_OP_POOL_AVG: res = (count > 0) ? (res / count) : 0.0f; break;\n case GGML_OP_POOL_MAX: break;\n case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n }\n\n drow[ow] = res;\n }\n }\n}\n\n// ggml_compute_forward_pool_1d\n\nvoid ggml_compute_forward_pool_1d(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const int32_t * opts = (const int32_t *)dst->op_params;\n ggml_op_pool op = static_cast(opts[0]);\n const int k0 = opts[1];\n const int s0 = opts[2];\n const int p0 = opts[3];\n\n ggml_compute_forward_pool_1d_ksp(params, op, k0, s0, p0, dst);\n}\n\n// ggml_compute_forward_pool_2d\n\nvoid ggml_compute_forward_pool_2d(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src = dst->src[0];\n\n assert(src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16);\n\n if (params->ith != 0) {\n return;\n }\n\n const int32_t * opts = (const int32_t *)dst->op_params;\n\n ggml_op_pool op = static_cast(opts[0]);\n const int k0 = opts[1];\n const int k1 = opts[2];\n const int s0 = opts[3];\n const int s1 = opts[4];\n const int p0 = opts[5];\n const int p1 = opts[6];\n const char * cdata = (const char*)src->data;\n const char * const data_end = cdata + ggml_nbytes(src);\n\n const int64_t px = dst->ne[0];\n const int64_t py = dst->ne[1];\n const int64_t pa = px * py;\n\n float * dplane = (float *)dst->data;\n\n const int ka = k0 * k1;\n const int offset0 = -p0;\n const int offset1 = -p1;\n\n while (cdata < data_end) {\n for (int oy = 0; oy < py; ++oy) {\n float * const drow = dplane + oy * px;\n float * const out = drow;\n\n for (int ox = 0; ox < px; ++ox) {\n float res = 0;\n switch (op) {\n case GGML_OP_POOL_AVG: res = 0; break;\n case GGML_OP_POOL_MAX: res = -FLT_MAX; break;\n case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n }\n\n const int ix = offset0 + ox * s0;\n const int iy = offset1 + oy * s1;\n\n for (int ky = 0; ky < k1; ++ky) {\n if (iy + ky < 0 || iy + ky >= src->ne[1]) {\n continue;\n }\n\n const void * srow = (const void *)(cdata + src->nb[1] * (iy + ky));\n for (int kx = 0; kx < k0; ++kx) {\n int j = ix + kx;\n if (j < 0 || j >= src->ne[0]) {\n continue;\n }\n\n const float srow_j = (src->type == GGML_TYPE_F32) ? ((const float*)srow)[j] : GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t*)srow)[j]);\n switch (op) {\n case GGML_OP_POOL_AVG: res += srow_j; break;\n case GGML_OP_POOL_MAX: res = std::max(srow_j, res); break;\n case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n }\n }\n }\n switch (op) {\n case GGML_OP_POOL_AVG: res /= ka; break;\n case GGML_OP_POOL_MAX: break;\n case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n }\n\n out[ox] = res;\n }\n }\n\n cdata += src->nb[2];\n dplane += pa;\n }\n}\n\n// ggml_compute_forward_pool_2d_back\n\nvoid ggml_compute_forward_pool_2d_back(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src = dst->src[0];\n const ggml_tensor * dstf = dst->src[1]; // forward tensor of dst\n\n assert(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n\n if (params->ith != 0) {\n return;\n }\n\n const int32_t * opts = (const int32_t *)dst->op_params;\n ggml_op_pool op = static_cast(opts[0]);\n const int k0 = opts[1];\n const int k1 = opts[2];\n const int s0 = opts[3];\n const int s1 = opts[4];\n const int p0 = opts[5];\n const int p1 = opts[6];\n\n char * cdata = (char *) dst->data;\n const char * cdataf = (const char *) dstf->data;\n const char * const data_end = cdata + ggml_nbytes(dst);\n\n GGML_ASSERT(params->ith == 0);\n memset(cdata, 0, ggml_nbytes(dst));\n\n const int64_t px = src->ne[0];\n const int64_t py = src->ne[1];\n const int64_t pa = px * py;\n\n const float * splane = (const float *) src->data;\n\n const int ka = k0 * k1;\n const int offset0 = -p0;\n const int offset1 = -p1;\n\n while (cdata < data_end) {\n for (int oy = 0; oy < py; ++oy) {\n const float * const srow = splane + oy * px;\n for (int ox = 0; ox < px; ++ox) {\n const float grad0 = srow[ox];\n\n const int ix = offset0 + ox * s0;\n const int iy = offset1 + oy * s1;\n\n if (op == GGML_OP_POOL_MAX) {\n float maxval = -FLT_MAX;\n int kxmax = -1;\n int kymax = -1;\n\n for (int ky = 0; ky < k1; ++ky) {\n if (iy + ky < 0 || iy + ky >= dst->ne[1]) {\n continue;\n }\n const void * drowf = (const void *)(cdataf + dst->nb[1] * (iy + ky));\n for (int kx = 0; kx < k0; ++kx) {\n int j = ix + kx;\n if (j < 0 || j >= dst->ne[0]) {\n continue;\n }\n\n const float val = dst->type == GGML_TYPE_F32 ?\n ((const float *) drowf)[j] : GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t *) drowf)[j]);\n if (val <= maxval) {\n continue;\n }\n\n maxval = val;\n kxmax = kx;\n kymax = ky;\n }\n }\n\n if (kxmax == -1 || kymax == -1) {\n continue;\n }\n\n void * drow = (void *)(cdata + dst->nb[1] * (iy + kymax));\n const int j = ix + kxmax;\n if (dst->type == GGML_TYPE_F32) {\n ((float *) drow)[j] += grad0;\n } else {\n ((ggml_fp16_t *) drow)[j] = GGML_CPU_FP32_TO_FP16(grad0 + GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t *) drow)[j]));\n }\n } else if (op == GGML_OP_POOL_AVG) {\n const float grad = grad0 / ka;\n\n for (int ky = 0; ky < k1; ++ky) {\n if (iy + ky < 0 || iy + ky >= dst->ne[1]) {\n continue;\n }\n void * drow = (void *)(cdata + dst->nb[1] * (iy + ky));\n for (int kx = 0; kx < k0; ++kx) {\n int j = ix + kx;\n if (j < 0 || j >= dst->ne[0]) {\n continue;\n }\n\n if (dst->type == GGML_TYPE_F32) {\n ((float *) drow)[j] += grad;\n } else {\n ((ggml_fp16_t *) drow)[j] += GGML_CPU_FP32_TO_FP16(grad);\n }\n }\n }\n } else {\n GGML_ASSERT(false);\n }\n }\n }\n\n cdata += dst->nb[2];\n cdataf += dst->nb[2];\n splane += pa;\n }\n}\n\n// ggml_compute_forward_upscale\n\nstatic void ggml_compute_forward_upscale_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n float sf0 = (float)ne0/src0->ne[0];\n float sf1 = (float)ne1/src0->ne[1];\n float sf2 = (float)ne2/src0->ne[2];\n float sf3 = (float)ne3/src0->ne[3];\n float pixel_offset = 0.5f;\n\n const int32_t mode_flags = ggml_get_op_params_i32(dst, 0);\n const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF);\n\n if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) {\n pixel_offset = 0.0f;\n sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0;\n sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1;\n }\n\n if (mode == GGML_SCALE_MODE_NEAREST) {\n for (int64_t i3 = 0; i3 < ne3; i3++) {\n const int64_t i03 = i3 / sf3;\n for (int64_t i2 = ith; i2 < ne2; i2 += nth) {\n const int64_t i02 = i2 / sf2;\n for (int64_t i1 = 0; i1 < ne1; i1++) {\n const int64_t i01 = i1 / sf1;\n for (int64_t i0 = 0; i0 < ne0; i0++) {\n const int64_t i00 = i0 / sf0;\n\n const float * x = (float *)((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n float * y = (float *)((char *) dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n\n *y = *x;\n }\n }\n }\n }\n } else if (mode == GGML_SCALE_MODE_BILINEAR && (mode_flags & GGML_SCALE_FLAG_ANTIALIAS)) {\n // Similar to F.interpolate(..., mode=\"bilinear\", align_corners=False, antialias=True)\n // https://github.com/pytorch/pytorch/blob/8871ff29b743948d1225389d5b7068f37b22750b/aten/src/ATen/native/cpu/UpSampleKernel.cpp\n auto triangle_filter = [](float x) -> float {\n return std::max(1.0f - fabsf(x), 0.0f);\n };\n\n // support and invscale, minimum 1 pixel for bilinear\n const float support1 = std::max(1.0f, 1.0f / sf1);\n const float invscale1 = 1.0f / support1;\n const float support0 = std::max(1.0f, 1.0f / sf0);\n const float invscale0 = 1.0f / support0;\n\n for (int64_t i3 = 0; i3 < ne3; i3++) {\n const int64_t i03 = i3 / sf3;\n for (int64_t i2 = ith; i2 < ne2; i2 += nth) {\n const int64_t i02 = i2 / sf2;\n for (int64_t i1 = 0; i1 < ne1; i1++) {\n const float y = ((float) i1 + pixel_offset) / sf1;\n for (int64_t i0 = 0; i0 < ne0; i0++) {\n const float x = ((float) i0 + pixel_offset) / sf0;\n\n // the range of source pixels that contribute\n const int64_t x_min = std::max(x - support0 + pixel_offset, 0);\n const int64_t x_max = std::min(x + support0 + pixel_offset, ne00);\n const int64_t y_min = std::max(y - support1 + pixel_offset, 0);\n const int64_t y_max = std::min(y + support1 + pixel_offset, ne01);\n\n // bilinear filter with antialiasing\n float val = 0.0f;\n float total_weight = 0.0f;\n\n for (int64_t sy = y_min; sy < y_max; sy++) {\n const float weight_y = triangle_filter((sy - y + pixel_offset) * invscale1);\n\n for (int64_t sx = x_min; sx < x_max; sx++) {\n const float weight_x = triangle_filter((sx - x + pixel_offset) * invscale0);\n const float weight = weight_x * weight_y;\n\n if (weight <= 0.0f) {\n continue;\n }\n\n const float pixel = *(const float *)((const char *)src0->data + sx*nb00 + sy*nb01 + i02*nb02 + i03*nb03);\n val += pixel * weight;\n total_weight += weight;\n }\n }\n\n if (total_weight > 0.0f) {\n val /= total_weight;\n }\n\n float * dst_ptr = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n *dst_ptr = val;\n }\n }\n }\n }\n } else if (mode == GGML_SCALE_MODE_BILINEAR) {\n for (int64_t i3 = 0; i3 < ne3; i3++) {\n const int64_t i03 = i3 / sf3;\n for (int64_t i2 = ith; i2 < ne2; i2 += nth) {\n const int64_t i02 = i2 / sf2;\n for (int64_t i1 = 0; i1 < ne1; i1++) {\n const float y = ((float)i1 + pixel_offset) / sf1 - pixel_offset;\n int64_t y0 = (int64_t)floorf(y);\n int64_t y1 = y0 + 1;\n\n y0 = std::max(int64_t(0), std::min(y0, ne01 - 1));\n y1 = std::max(int64_t(0), std::min(y1, ne01 - 1));\n\n float dy = y - (float)y0;\n dy = std::max(0.0f, std::min(dy, 1.0f));\n\n for (int64_t i0 = 0; i0 < ne0; i0++) {\n const float x = ((float)i0 + pixel_offset) / sf0 - pixel_offset;\n int64_t x0 = (int64_t)floorf(x);\n int64_t x1 = x0 + 1;\n\n x0 = std::max(int64_t(0), std::min(x0, ne00 - 1));\n x1 = std::max(int64_t(0), std::min(x1, ne00 - 1));\n\n float dx = x - (float)x0;\n dx = std::max(0.0f, std::min(dx, 1.0f));\n\n // fetch the four surrounding pixel values and interpolate\n const float a = *(const float *)((const char *)src0->data + x0*nb00 + y0*nb01 + i02*nb02 + i03*nb03);\n const float b = *(const float *)((const char *)src0->data + x1*nb00 + y0*nb01 + i02*nb02 + i03*nb03);\n const float c = *(const float *)((const char *)src0->data + x0*nb00 + y1*nb01 + i02*nb02 + i03*nb03);\n const float d = *(const float *)((const char *)src0->data + x1*nb00 + y1*nb01 + i02*nb02 + i03*nb03);\n\n const float val = a*(1 - dx)*(1 - dy) + b*dx*(1 - dy) + c*(1 - dx)*dy + d*dx*dy;\n\n float * y_dst = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n *y_dst = val;\n }\n }\n }\n }\n } else if (mode == GGML_SCALE_MODE_BICUBIC) {\n // https://en.wikipedia.org/wiki/Bicubic_interpolation#Bicubic_convolution_algorithm\n const float a = -0.75f; // use alpha = -0.75 (same as PyTorch)\n auto weight1 = [a](float x) { return ((a + 2) * x - (a + 3)) * x * x + 1; };\n auto weight2 = [a](float x) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; };\n auto bicubic = [=](float p0, float p1, float p2, float p3, float x) {\n const float w0 = weight2(x + 1);\n const float w1 = weight1(x + 0);\n const float w2 = weight1(1 - x);\n const float w3 = weight2(2 - x);\n return p0*w0 + p1*w1 + p2*w2 + p3*w3;\n };\n\n for (int64_t i3 = 0; i3 < ne3; i3++) {\n const int64_t i03 = i3 / sf3;\n for (int64_t i2 = ith; i2 < ne2; i2 += nth) {\n const int64_t i02 = i2 / sf2;\n for (int64_t i1 = 0; i1 < ne1; i1++) {\n const float y = ((float)i1 + pixel_offset) / sf1 - pixel_offset;\n const int64_t y0 = (int64_t)floorf(y);\n const float dy = y - (float)y0;\n\n for (int64_t i0 = 0; i0 < ne0; i0++) {\n const float x = ((float)i0 + pixel_offset) / sf0 - pixel_offset;\n const int64_t x0 = (int64_t)floorf(x);\n const float dx = x - (float)x0;\n\n auto p = [=](int64_t x_off, int64_t y_off) -> float {\n int64_t i00 = std::max(int64_t(0), std::min(x0 + x_off, ne00 - 1));\n int64_t i01 = std::max(int64_t(0), std::min(y0 + y_off, ne01 - 1));\n return *(const float *)((const char *)src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n };\n\n const float val = bicubic(\n bicubic(p(-1,-1), p(0,-1), p(1,-1), p(2,-1), dx),\n bicubic(p(-1, 0), p(0, 0), p(1, 0), p(2, 0), dx),\n bicubic(p(-1, 1), p(0, 1), p(1, 1), p(2, 1), dx),\n bicubic(p(-1, 2), p(0, 2), p(1, 2), p(2, 2), dx), dy);\n\n float * y_dst = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n *y_dst = val;\n }\n }\n }\n }\n } else {\n GGML_ABORT(\"unsupported upscale mode\");\n }\n}\n\nvoid ggml_compute_forward_upscale(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_upscale_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n\n// ggml_compute_forward_pad\n\ntemplate\nstatic void ggml_compute_forward_pad_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n assert(dst->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n float * dst_ptr = (float *) dst->data;\n const int32_t lp0 = ggml_get_op_params_i32(dst, 0);\n const int32_t rp0 = ggml_get_op_params_i32(dst, 1);\n const int32_t lp1 = ggml_get_op_params_i32(dst, 2);\n const int32_t rp1 = ggml_get_op_params_i32(dst, 3);\n const int32_t lp2 = ggml_get_op_params_i32(dst, 4);\n const int32_t rp2 = ggml_get_op_params_i32(dst, 5);\n const int32_t lp3 = ggml_get_op_params_i32(dst, 6);\n const int32_t rp3 = ggml_get_op_params_i32(dst, 7);\n\n // TODO: optimize\n\n for (int64_t i2 = 0; i2 < ne2; ++i2) {\n for (int64_t i1 = ith; i1 < ne1; i1 += nth) {\n for (int64_t i0 = 0; i0 < ne0; ++i0) {\n for (int64_t i3 = 0; i3 < ne3; ++i3) {\n // circular means wrap around on a torus, so x and y loop around\n if constexpr (circular_t) {\n const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;\n const int64_t src_i0 = ggml_wrap_around(i0 - lp0, ne00);\n const int64_t src_i1 = ggml_wrap_around(i1 - lp1, ne01);\n const int64_t src_i2 = ggml_wrap_around(i2 - lp2, ne02);\n const int64_t src_i3 = ggml_wrap_around(i3 - lp3, ne03);\n\n const int64_t src_idx =\n src_i3*nb03 +\n src_i2*nb02 +\n src_i1*nb01 +\n src_i0*nb00;\n\n const float * src_ptr = (const float *)((char *) src0->data + src_idx);\n dst_ptr[dst_idx] = *src_ptr;\n } else {\n const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;\n if ((i0 >= lp0 && i0 < ne0 - rp0) \\\n && (i1 >= lp1 && i1 < ne1 - rp1) \\\n && (i2 >= lp2 && i2 < ne2 - rp2) \\\n && (i3 >= lp3 && i3 < ne3 - rp3)) {\n const int64_t src_idx = (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01 + (i0 - lp0)*nb00;\n const float * src_ptr = (const float *)((char *) src0->data + src_idx);\n dst_ptr[dst_idx] = *src_ptr;\n } else {\n dst_ptr[dst_idx] = 0;\n }\n }\n }\n }\n }\n }\n}\n\n\nvoid ggml_compute_forward_pad(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const bool circular = (bool) ggml_get_op_params_i32(dst, 8);\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n if (circular) {\n ggml_compute_forward_pad_f32(params, dst);\n } else {\n ggml_compute_forward_pad_f32(params, dst);\n }\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_pad_reflect_1d\n\nvoid ggml_compute_forward_pad_reflect_1d(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int32_t * opts = (const int32_t *) dst->op_params;\n const int p0 = opts[0];\n const int p1 = opts[1];\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n for (int64_t i3 = 0; i3 < ne3; i3++) {\n for (int64_t i2 = 0; i2 < ne2; i2++) {\n for (int64_t i1 = ith; i1 < ne1; i1 += nth) {\n float * left = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + p0*nb0);\n float * right = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + (ne0-p1-1)*nb0);\n\n ggml_vec_cpy_f32(ne00, left, (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n for (int i0 = 1; i0 <= p0; i0++) { left[-i0] = left[i0]; }\n for (int i0 = 1; i0 <= p1; i0++) { right[i0] = right[-i0]; }\n }\n }\n }\n}\n\n// ggml_compute_forward_roll\n\nstatic int64_t ggml_wrap_index(int64_t i, int64_t ne) {\n if (i < 0) {\n return i + ne;\n } else if (i >= ne) {\n return i - ne;\n }\n return i;\n}\n\nstatic void ggml_compute_forward_roll_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const float * src_data = (const float *) src0->data;\n float * dst_data = (float *) dst->data;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n const int s0 = ggml_get_op_params_i32(dst, 0);\n const int s1 = ggml_get_op_params_i32(dst, 1);\n const int s2 = ggml_get_op_params_i32(dst, 2);\n const int s3 = ggml_get_op_params_i32(dst, 3);\n\n const int64_t total = ne1 * ne2 * ne3;\n const int64_t per_thread = (total + params->nth) / params->nth;\n const int64_t start = params->ith * per_thread;\n const int64_t end = std::min(start + per_thread, total);\n\n for (int64_t i = start; i < end; ++i) {\n const int64_t i1 = i % ne1;\n const int64_t i2 = (i / ne1) % ne2;\n const int64_t i3 = i / (ne2 * ne1);\n float * dst_row = dst_data + (i3*nb3 + i2*nb2 + i1*nb1) / sizeof(float);\n\n const int64_t i01 = ggml_wrap_index(i1 - s1, ne01);\n const int64_t i02 = ggml_wrap_index(i2 - s2, ne02);\n const int64_t i03 = ggml_wrap_index(i3 - s3, ne03);\n const float * src_row = src_data + (i03*nb03 + i02*nb02 + i01*nb01) / sizeof(float);\n\n const int64_t s = ggml_wrap_index(-s0, ne00);\n const int64_t n = ne00 - s;\n ggml_vec_cpy_f32(n, dst_row, src_row + s);\n ggml_vec_cpy_f32(s, dst_row + n, src_row);\n }\n}\n\nvoid ggml_compute_forward_roll(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_roll_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_arange\n\nstatic void ggml_compute_forward_arange_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n GGML_ASSERT(dst->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const float start = ggml_get_op_params_f32(dst, 0);\n const float stop = ggml_get_op_params_f32(dst, 1);\n const float step = ggml_get_op_params_f32(dst, 2);\n\n const int64_t steps = (int64_t) ceilf((stop - start) / step);\n\n GGML_ASSERT(ggml_nelements(dst) == steps);\n\n for (int64_t i = ith; i < steps; i+= nth) {\n float value = start + step * i;\n ((float *)dst->data)[i] = value;\n }\n}\n\nvoid ggml_compute_forward_arange(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n switch (dst->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_arange_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nstatic void ggml_compute_forward_timestep_embedding_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n const int dim = ggml_get_op_params_i32(dst, 0);\n const int max_period = ggml_get_op_params_i32(dst, 1);\n\n int half = dim / 2;\n\n for (int64_t i = 0; i < ne00; i++) {\n float * embed_data = (float *)((char *) dst->data + i*nb1);\n for (int64_t j = ith; j < half; j += nth) {\n float timestep = ((float *)src0->data)[i];\n float freq = (float)expf(-logf(max_period) * j / half);\n float arg = timestep * freq;\n embed_data[j] = cosf(arg);\n embed_data[j + half] = sinf(arg);\n }\n if (dim % 2 != 0 && ith == 0) {\n embed_data[2 * half] = 0.f;\n }\n }\n}\n\nvoid ggml_compute_forward_timestep_embedding(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_timestep_embedding_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_argsort\n\ntemplate\nstruct cmp_argsort {\n const float * data;\n bool operator()(int32_t a, int32_t b) const {\n if constexpr (order == GGML_SORT_ORDER_ASC) {\n return data[a] < data[b];\n } else {\n return data[a] > data[b];\n }\n }\n};\n\nstatic void ggml_compute_forward_argsort_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT(nb0 == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t nr = ggml_nrows(src0);\n\n ggml_sort_order order = (ggml_sort_order) ggml_get_op_params_i32(dst, 0);\n\n for (int64_t i = ith; i < nr; i += nth) {\n const float * src_data = (float *)((char *) src0->data + i*nb01);\n\n int32_t * dst_data = (int32_t *)((char *) dst->data + i*nb1);\n\n for (int64_t j = 0; j < ne0; j++) {\n dst_data[j] = j;\n }\n\n switch (order) {\n case GGML_SORT_ORDER_ASC:\n std::sort(dst_data, dst_data + ne0, cmp_argsort{src_data});\n break;\n\n case GGML_SORT_ORDER_DESC:\n std::sort(dst_data, dst_data + ne0, cmp_argsort{src_data});\n break;\n\n default:\n GGML_ABORT(\"invalid sort order\");\n }\n }\n}\n\nvoid ggml_compute_forward_argsort(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_argsort_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_top_k\n\nstruct cmp_top_k {\n const float * data;\n bool operator()(int32_t a, int32_t b) const {\n return data[a] > data[b];\n }\n};\n\nstatic void ggml_compute_forward_top_k_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT(nb0 == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t nr = ggml_nrows(src0);\n\n const int top_k = ne0;\n\n int32_t * tmp = (int32_t *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;\n\n for (int64_t i = ith; i < nr; i += nth) {\n const float * src_data = (float *)((char *) src0->data + i*nb01);\n\n for (int64_t j = 0; j < ne00; j++) {\n tmp[j] = j;\n }\n\n std::partial_sort(tmp, tmp + top_k, tmp + ne00, cmp_top_k{src_data});\n\n int32_t * dst_data = (int32_t *)((char *) dst->data + i*nb1);\n\n std::copy(tmp, tmp + top_k, dst_data);\n\n // emphasize that the order is not important\n if (top_k > 1) {\n std::swap(dst_data[0], dst_data[1]);\n }\n }\n}\n\nvoid ggml_compute_forward_top_k(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_top_k_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nstatic void ggml_compute_forward_flash_attn_ext_f16_one_chunk(\n const ggml_compute_params * params,\n ggml_tensor * dst,\n int ir0, int ir1,\n int64_t ic_start, int64_t ic_end,\n float * partials, int64_t partial_stride) {\n\n const bool write_partials = (partials != nullptr);\n const ggml_tensor * q = dst->src[0];\n const ggml_tensor * k = dst->src[1];\n const ggml_tensor * v = dst->src[2];\n const ggml_tensor * mask = dst->src[3];\n const ggml_tensor * sinks = dst->src[4];\n\n GGML_TENSOR_LOCALS(int64_t, neq, q, ne)\n GGML_TENSOR_LOCALS(size_t, nbq, q, nb)\n GGML_TENSOR_LOCALS(int64_t, nek, k, ne)\n GGML_TENSOR_LOCALS(size_t, nbk, k, nb)\n GGML_TENSOR_LOCALS(int64_t, nev, v, ne)\n GGML_TENSOR_LOCALS(size_t, nbv, v, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int64_t DK = nek0;\n const int64_t DV = nev0;\n const int64_t N = neq1;\n\n GGML_ASSERT(ne0 == DV);\n GGML_ASSERT(ne2 == N);\n\n // input tensor rows must be contiguous\n GGML_ASSERT(nbq0 == ggml_type_size(q->type));\n GGML_ASSERT(nbk0 == ggml_type_size(k->type));\n GGML_ASSERT(nbv0 == ggml_type_size(v->type));\n\n GGML_ASSERT(neq0 == DK);\n GGML_ASSERT(nek0 == DK);\n GGML_ASSERT(nev0 == DV);\n\n GGML_ASSERT(neq1 == N);\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n // broadcast factors\n const int64_t rk2 = neq2/nek2;\n const int64_t rk3 = neq3/nek3;\n\n const int64_t rv2 = neq2/nev2;\n const int64_t rv3 = neq3/nev3;\n\n // parallelize by q rows using ggml_vec_dot_f32\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n float logit_softcap = 0.0f;\n\n memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));\n memcpy(&logit_softcap, (float *) dst->op_params + 2, sizeof(float));\n\n if (logit_softcap != 0) {\n scale /= logit_softcap;\n }\n\n const uint32_t n_head = neq2;\n const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n\n const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n ggml_type const k_vec_dot_type = ggml_get_type_traits_cpu(k->type)->vec_dot_type;\n ggml_from_float_t const q_to_vec_dot = ggml_get_type_traits_cpu(k_vec_dot_type)->from_float;\n ggml_vec_dot_t const kq_vec_dot = ggml_get_type_traits_cpu(k->type)->vec_dot;\n ggml_to_float_t const v_to_float = ggml_get_type_traits(v->type)->to_float;\n\n GGML_ASSERT(( q_to_vec_dot) && \"fattn: unsupported K-type\");\n GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float ) && \"fattn: unsupported V-type\");\n\n int ith = params->ith;\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // q indices\n const int iq3 = ir/(neq2*neq1);\n const int iq2 = (ir - iq3*neq2*neq1)/neq1;\n const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);\n\n const uint32_t h = iq2; // head index\n const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;\n\n float S = 0.0f; // sum\n float M = -INFINITY; // maximum KQ value\n\n float * VKQ32 = (float *) params->wdata + ith*(1*DK + 2*DV + CACHE_LINE_SIZE_F32); // FP32 VKQ accumulator\n float * V32 = (VKQ32 + 1*DV); // (temporary) FP32 V buffer\n ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulator\n ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16\n\n if (v->type == GGML_TYPE_F16) {\n memset(VKQ16, 0, DV*sizeof(ggml_fp16_t));\n } else {\n memset(VKQ32, 0, DV*sizeof(float));\n }\n\n const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL;\n\n // k indices\n const int ik3 = iq3 / rk3;\n const int ik2 = iq2 / rk2;\n\n // v indices\n const int iv3 = iq3 / rv3;\n const int iv2 = iq2 / rv2;\n\n const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));\n q_to_vec_dot(pq, Q_q, DK);\n\n // online softmax / attention\n // loop over n_kv and n_head_kv\n // ref: https://arxiv.org/pdf/2112.05682.pdf\n\n for (int64_t ic = ic_start; ic < ic_end; ++ic) {\n const float mv = mp ? slope*GGML_CPU_FP16_TO_FP32(mp[ic]) : 0.0f;\n if (mv == -INFINITY) {\n continue;\n }\n\n float s; // KQ value\n\n const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);\n kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1);\n\n s = s*scale; // scale KQ value\n\n if (logit_softcap != 0.0f) {\n s = logit_softcap*tanhf(s);\n }\n\n s += mv; // apply mask\n\n const float Mold = M;\n\n float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value\n float vs = 1.0f; // post-softmax KQ value, expf(s - M)\n\n const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));\n\n if (v->type == GGML_TYPE_F16) {\n if (s > M) {\n // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f\n M = s;\n ms = expf(Mold - M);\n\n // V = V*expf(Mold - M)\n ggml_vec_scale_f16(DV, VKQ16, ms);\n } else {\n // no new maximum, ms == 1.0f, vs != 1.0f\n vs = expf(s - M);\n }\n\n // V += v*expf(s - M)\n ggml_vec_mad_f16(DV, VKQ16, (const ggml_fp16_t *) v_data, vs);\n } else {\n if (s > M) {\n // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f\n M = s;\n ms = expf(Mold - M);\n\n // V = V*expf(Mold - M)\n ggml_vec_scale_f32(DV, VKQ32, ms);\n } else {\n // no new maximum, ms == 1.0f, vs != 1.0f\n vs = expf(s - M);\n }\n\n // V += v*expf(s - M)\n if (v_to_float) {\n v_to_float(v_data, V32, DV);\n ggml_vec_mad_f32(DV, VKQ32, V32, vs);\n } else {\n // V is F32\n ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs);\n }\n }\n\n S = S*ms + vs; // scale and increment sum with partial sum\n }\n\n if (v->type == GGML_TYPE_F16) {\n for (int64_t d = 0; d < DV; ++d) {\n VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d]);\n }\n }\n\n // sinks - apply only on the first kv-chunk\n if (sinks && ic_start == 0) {\n const float s = ((float *)((char *) sinks->data))[h];\n\n float ms = 1.0f;\n float vs = 1.0f;\n\n if (s > M) {\n ms = expf(M - s);\n M = s;\n ggml_vec_scale_f32(DV, VKQ32, ms);\n } else {\n vs = expf(s - M);\n }\n\n S = S*ms + vs;\n }\n\n if (write_partials) {\n // Write M, S, VKQ to partials for later reduction\n // partials layout: [M, S, VKQ[DV]] per query head\n float * partial = partials + ir * partial_stride;\n partial[0] = M;\n partial[1] = S;\n memcpy(partial + 2, VKQ32, DV * sizeof(float));\n } else {\n // V /= S\n const float S_inv = S == 0.0f ? 0.0f : 1.0f/S;\n ggml_vec_scale_f32(DV, VKQ32, S_inv);\n\n // dst indices\n const int i1 = iq1;\n const int i2 = iq2;\n const int i3 = iq3;\n\n // permute(0, 2, 1, 3)\n memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, VKQ32, nb1);\n }\n }\n}\n\nstatic void ggml_compute_forward_flash_attn_ext_tiled(\n const ggml_compute_params * params,\n ggml_tensor * dst,\n int ir0, int ir1) {\n const ggml_tensor * q = dst->src[0];\n const ggml_tensor * k = dst->src[1];\n const ggml_tensor * v = dst->src[2];\n const ggml_tensor * mask = dst->src[3];\n const ggml_tensor * sinks = dst->src[4];\n\n GGML_TENSOR_LOCALS(int64_t, neq, q, ne)\n GGML_TENSOR_LOCALS(size_t, nbq, q, nb)\n GGML_TENSOR_LOCALS(int64_t, nek, k, ne)\n GGML_TENSOR_LOCALS(size_t, nbk, k, nb)\n GGML_TENSOR_LOCALS(int64_t, nev, v, ne)\n GGML_TENSOR_LOCALS(size_t, nbv, v, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int64_t DK = nek0;\n const int64_t DV = nev0;\n const int64_t N = neq1;\n\n GGML_ASSERT(ne0 == DV);\n GGML_ASSERT(ne2 == N);\n\n // input tensor rows must be contiguous\n GGML_ASSERT(nbq0 == ggml_type_size(q->type));\n GGML_ASSERT(nbk0 == ggml_type_size(k->type));\n GGML_ASSERT(nbv0 == ggml_type_size(v->type));\n\n GGML_ASSERT(neq0 == DK);\n GGML_ASSERT(nek0 == DK);\n GGML_ASSERT(nev0 == DV);\n\n GGML_ASSERT(neq1 == N);\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n GGML_ASSERT(k->type == v->type);\n const ggml_type kv_type = k->type;\n\n\n // broadcast factors\n const int64_t rk2 = neq2/nek2;\n const int64_t rk3 = neq3/nek3;\n\n const int64_t rv2 = neq2/nev2;\n const int64_t rv3 = neq3/nev3;\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n float logit_softcap = 0.0f;\n\n memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));\n memcpy(&logit_softcap, (float *) dst->op_params + 2, sizeof(float));\n\n if (logit_softcap != 0) {\n scale /= logit_softcap;\n }\n\n const uint32_t n_head = neq2;\n const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n\n const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n int ith = params->ith;\n\n static constexpr int Q_TILE_SZ = ggml_fa_tile_config::Q;\n static constexpr int KV_TILE_SZ = ggml_fa_tile_config::KV;\n\n int ir = ir0;\n while (ir < ir1) {\n // q indices for the start of this tile\n const int iq3 = ir/(neq2*neq1);\n const int iq2 = (ir - iq3*neq2*neq1)/neq1;\n const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);\n\n // Number of valid rows in this tile:\n // - limited by tile size (Q_TILE_SZ)\n // - limited by chunk boundary (ir1 - ir)\n // - limited by head boundary (neq1 - iq1) to avoid crossing into next head\n const int tile_rows = MIN(Q_TILE_SZ, MIN((int)(ir1 - ir), (int)(neq1 - iq1)));\n GGML_ASSERT(tile_rows > 0);\n\n const uint32_t h = iq2; // head index\n const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;\n\n float S[Q_TILE_SZ];\n float M[Q_TILE_SZ];\n\n for (int i = 0 ; i < Q_TILE_SZ; ++i) {\n S[i] = 0.;\n M[i] = -INFINITY;\n }\n\n // Per-thread scratch layout:\n // Q_q: Q_TILE_SZ * DK (converted Q tile — F32 for GEMM, KV type for scalar)\n // KQ: Q_TILE_SZ * KV_TILE_SZ (attention scores in float)\n // mask: Q_TILE_SZ * KV_TILE_SZ (mask in float)\n // VKQ32: Q_TILE_SZ * DV (FP32 output accumulator)\n // V32: KV_TILE_SZ * DV (F32 buffer for V tile)\n // K_f32: KV_TILE_SZ * DK (F32 buffer for K tile — GEMM path)\n float * base = (float *) params->wdata + ith*(Q_TILE_SZ*DK + 2*Q_TILE_SZ*KV_TILE_SZ + Q_TILE_SZ*DV + KV_TILE_SZ*DV + KV_TILE_SZ*DK + CACHE_LINE_SIZE_F32);\n\n void * Q_q = base;\n float * KQ = (float *)((char *)base + Q_TILE_SZ * DK * sizeof(float));\n float * mask32 = KQ + Q_TILE_SZ * KV_TILE_SZ;\n float * VKQ32 = mask32 + Q_TILE_SZ * KV_TILE_SZ;\n float * V32 = VKQ32 + Q_TILE_SZ * DV;\n float * K_f32 = V32 + KV_TILE_SZ * DV;\n\n memset(VKQ32, 0, Q_TILE_SZ * DV * sizeof(float));\n memset(mask32, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));\n\n // k indices\n const int ik3 = iq3 / rk3;\n const int ik2 = iq2 / rk2;\n\n // v indices\n const int iv3 = iq3 / rv3;\n const int iv2 = iq2 / rv2;\n\n {\n float * Q_f32 = (float *)Q_q;\n for (int tq = 0; tq < tile_rows; tq++) {\n const float * pq = (const float *) ((char *) q->data + ((iq1 + tq)*nbq1 + iq2*nbq2 + iq3*nbq3));\n memcpy(Q_f32 + tq * DK, pq, DK * sizeof(float));\n }\n for (int tq = tile_rows; tq < Q_TILE_SZ; tq++) {\n memset(Q_f32 + tq * DK, 0, DK * sizeof(float));\n }\n }\n\n memset(K_f32, 0, DK * KV_TILE_SZ * sizeof(float));\n memset(V32, 0, KV_TILE_SZ * DV * sizeof(float));\n\n for (int64_t ic = 0; ic < nek1; ic += KV_TILE_SZ) {\n const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, nek1 - ic);\n\n // skip the tile entirely if all the masks are -inf\n if (mask) {\n bool can_skip = true;\n for (int tq = 0; tq < tile_rows; tq++) {\n const ggml_fp16_t * mp_row = (const ggml_fp16_t *)((const char *) mask->data + (iq1 + tq)*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]);\n for (int tk = 0; tk < kv_tile; tk++) {\n mask32[tq * KV_TILE_SZ + tk] = slope * GGML_CPU_FP16_TO_FP32(mp_row[ic + tk]);\n if (mask32[tq * KV_TILE_SZ + tk] != -INFINITY) {\n can_skip = false;\n }\n }\n // Pad remaining mask entries with -inf\n for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) {\n mask32[tq * KV_TILE_SZ + tk] = -INFINITY;\n }\n }\n\n if (can_skip) {\n continue;\n }\n }\n\n // Pack K tile transposed: K_f32[dk][kv] so KV_TILE is contiguous (SIMD dim)\n // Zero-pad the last tile so the GEMM always operates on KV_TILE_SZ columns\n for (int tk = 0; tk < kv_tile; tk++) {\n const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;\n if (kv_type == GGML_TYPE_F16) {\n const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;\n for (int64_t dk = 0; dk < DK; dk++) {\n K_f32[dk * KV_TILE_SZ + tk] = GGML_CPU_FP16_TO_FP32(k_f16[dk]);\n }\n } else {\n const float * k_f32_src = (const float *)k_data;\n for (int64_t dk = 0; dk < DK; dk++) {\n K_f32[dk * KV_TILE_SZ + tk] = k_f32_src[dk];\n }\n }\n }\n memset(KQ, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));\n simd_gemm(KQ, (const float *)Q_q, K_f32, Q_TILE_SZ, DK, KV_TILE_SZ);\n ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale);\n\n // Set padded KQ entries to -inf so softmax gives them zero weight\n if (kv_tile < KV_TILE_SZ) {\n for (int tq = 0; tq < Q_TILE_SZ; tq++) {\n for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) {\n KQ[tq * KV_TILE_SZ + tk] = -INFINITY;\n }\n }\n }\n\n if (logit_softcap != 0.0f) {\n ggml_vec_tanh_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, KQ);\n ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, logit_softcap);\n }\n\n if (mask) {\n ggml_vec_add_f32(tile_rows * KV_TILE_SZ, KQ, KQ, mask32);\n }\n\n bool skip[Q_TILE_SZ] = {};\n\n for (int tq = 0; tq < Q_TILE_SZ; tq++) {\n float * kq_row = KQ + tq * KV_TILE_SZ;\n\n float tile_max;\n ggml_vec_max_f32(KV_TILE_SZ, &tile_max, kq_row);\n\n if (tile_max == -INFINITY) {\n skip[tq] = true;\n continue;\n }\n\n const float Mold = M[tq];\n const float Mnew = fmaxf(Mold, tile_max);\n\n if (Mnew > Mold) {\n const float ms = expf(Mold - Mnew);\n ggml_vec_scale_f32(DV, VKQ32 + tq * DV, ms);\n S[tq] *= ms;\n }\n M[tq] = Mnew;\n\n\n S[tq] += ggml_vec_soft_max_f32(KV_TILE_SZ, kq_row, kq_row, Mnew);\n }\n\n // V accumulation: VKQ32 += softmax(KQ) * V\n // Pack V tile to contiguous F32, zero-padded\n for (int tk = 0; tk < kv_tile; tk++) {\n const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;\n if (kv_type == GGML_TYPE_F16) {\n ggml_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);\n } else {\n memcpy(V32 + tk * DV, v_data, DV * sizeof(float));\n }\n }\n for (int tq = 0; tq < Q_TILE_SZ; tq++) {\n if (skip[tq]) {\n memset(KQ + tq * KV_TILE_SZ, 0, KV_TILE_SZ * sizeof(float));\n }\n }\n simd_gemm(VKQ32, KQ, V32, Q_TILE_SZ, KV_TILE_SZ, DV);\n }\n\n // sinks (apply only to valid rows in the tile)\n if (sinks) {\n const float s = ((float *)((char *) sinks->data))[h];\n\n for (int tq = 0; tq < tile_rows; tq++) {\n float ms = 1.0f;\n float vs = 1.0f;\n\n if (s > M[tq]) {\n ms = expf(M[tq] - s);\n ggml_vec_scale_f32(DV, VKQ32 + tq * DV, ms);\n } else {\n vs = expf(s - M[tq]);\n }\n\n S[tq] = S[tq] * ms + vs;\n }\n }\n\n for (int tq = 0; tq < tile_rows; tq++) {\n // V /= S\n const float S_inv = S[tq] == 0.0f ? 0.0f : 1.0f / S[tq];\n ggml_vec_scale_f32(DV, VKQ32 + tq * DV, S_inv);\n\n // dst indices\n const int i1 = iq1 + tq;\n const int i2 = iq2;\n const int i3 = iq3;\n\n // permute(0, 2, 1, 3)\n memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, VKQ32 + tq * DV, nb1);\n }\n\n ir += tile_rows;\n }\n}\n\n// Reduction function: combines partial results across KV chunks\n// Partials layout in wdata: [n_q_heads][n_chunks][2 + DV]\nstatic void ggml_flash_attn_ext_reduce_partials(\n const ggml_compute_params * params,\n ggml_tensor * dst,\n const int64_t n_chunks,\n const int64_t chunk_size) {\n\n const ggml_tensor * q = dst->src[0];\n const ggml_tensor * k = dst->src[1];\n const ggml_tensor * v = dst->src[2];\n\n const int64_t DK = k->ne[0];\n const int64_t DV = v->ne[0];\n const int64_t nek1 = k->ne[1];\n const int64_t n_q_heads = q->ne[2];\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t wdata_per_thread = DK + 2*DV + CACHE_LINE_SIZE_F32;\n float * thread_wdata = (float *) params->wdata + ith * wdata_per_thread;\n\n const int64_t partials_offset = nth * (DK + 2*DV + CACHE_LINE_SIZE_F32);\n const int64_t partial_size = 2 + DV;\n const float * partials_base = (const float *) params->wdata + partials_offset;\n\n // Output layout\n const int64_t ne1 = dst->ne[1];\n const int64_t ne2 = dst->ne[2];\n const size_t nb1 = dst->nb[1];\n\n // Each thread reduces a subset of query heads\n for (int64_t q_head = ith; q_head < n_q_heads; q_head += nth) {\n float M_final = -INFINITY;\n float S_final = 0.0f;\n float * VKQ_final = thread_wdata;\n memset(VKQ_final, 0, DV * sizeof(float));\n\n // Combine partials from all chunks\n for (int64_t chunk_idx = 0; chunk_idx < n_chunks; ++chunk_idx) {\n const int64_t ic_start = chunk_idx * chunk_size;\n if (ic_start >= nek1) continue;\n\n const float * partial = partials_base + (q_head * n_chunks + chunk_idx) * partial_size;\n const float M_chunk = partial[0];\n const float S_chunk = partial[1];\n const float * VKQ_chunk = partial + 2;\n\n if (S_chunk == 0.0f) continue;\n\n const float M_new = fmaxf(M_final, M_chunk);\n const float scale_old = expf(M_final - M_new);\n const float scale_new = expf(M_chunk - M_new);\n\n for (int64_t d = 0; d < DV; ++d) {\n VKQ_final[d] = VKQ_final[d] * scale_old + VKQ_chunk[d] * scale_new;\n }\n S_final = S_final * scale_old + S_chunk * scale_new;\n M_final = M_new;\n }\n\n // Normalize and write to output\n if (S_final != 0.0f) {\n const float S_inv = 1.0f / S_final;\n ggml_vec_scale_f32(DV, VKQ_final, S_inv);\n }\n // iq1=0, iq3=0 for decode\n memcpy((char *) dst->data + (0*ne2*ne1 + q_head + 0*ne1)*nb1, VKQ_final, nb1);\n }\n}\n\nstatic void ggml_compute_forward_flash_attn_ext_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * q = dst->src[0];\n const ggml_tensor * k = dst->src[1];\n const ggml_tensor * v = dst->src[2];\n\n GGML_TENSOR_LOCALS(int64_t, neq, q, ne)\n GGML_TENSOR_LOCALS(size_t, nbq, q, nb)\n GGML_TENSOR_LOCALS(int64_t, nek, k, ne)\n GGML_TENSOR_LOCALS(size_t, nbk, k, nb)\n GGML_TENSOR_LOCALS(int64_t, nev, v, ne)\n GGML_TENSOR_LOCALS(size_t, nbv, v, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int64_t DK = nek0;\n const int64_t DV = nev0;\n const int64_t N = neq1;\n\n\n GGML_ASSERT(ne0 == DV);\n GGML_ASSERT(ne2 == N);\n\n // input tensor rows must be contiguous\n GGML_ASSERT(nbq0 == ggml_type_size(q->type));\n GGML_ASSERT(nbk0 == ggml_type_size(k->type));\n GGML_ASSERT(nbv0 == ggml_type_size(v->type));\n\n GGML_ASSERT(neq0 == DK);\n GGML_ASSERT(nek0 == DK);\n GGML_ASSERT(nev0 == DV);\n\n GGML_ASSERT(neq1 == N);\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n // When use_ref is set, force the vec-only reference implementation (no tiling, no KV-chunking)\n const bool use_ref = params->use_ref;\n\n const bool kv_is_f32_or_f16 = (k->type == GGML_TYPE_F32 || k->type == GGML_TYPE_F16);\n const bool use_split_kv_path = !use_ref && (neq1 == 1 && neq3 == 1) && kv_is_f32_or_f16 && (k->type == v->type) && q->type == GGML_TYPE_F32 && nek1 >= 512;\n\n if (use_split_kv_path) {\n const int64_t chunk_size = (nek1 + nth - 1) / nth;\n\n // Partials buffer layout: [q_head][kv_chunk][M, S, VKQ]\n const int64_t partial_size = 2 + DV;\n float * partials_base = (float *) params->wdata + nth * (DK + 2*DV + CACHE_LINE_SIZE_F32);\n\n const int64_t ic_start = ith * chunk_size;\n const int64_t ic_end = std::min(ic_start + chunk_size, nek1);\n\n const int64_t partial_stride = nth * partial_size;\n float * chunk_partials = partials_base + ith * partial_size;\n\n if (ic_start < nek1) {\n for (int64_t q_head = 0; q_head < neq2; q_head++) {\n ggml_compute_forward_flash_attn_ext_f16_one_chunk(\n params, dst, q_head, q_head + 1, ic_start, ic_end,\n chunk_partials, partial_stride);\n }\n } else {\n for (int64_t q_head = 0; q_head < neq2; q_head++) {\n float * q_partials = chunk_partials + q_head * partial_stride;\n q_partials[0] = -INFINITY; // M\n q_partials[1] = 0.0f; // S\n }\n }\n\n ggml_barrier(params->threadpool);\n ggml_flash_attn_ext_reduce_partials(params, dst, nth, chunk_size);\n } else {\n\n // total rows in q\n const int64_t nr = neq1*neq2*neq3;\n\n // disable for NUMA\n const bool disable_chunking = ggml_is_numa();\n\n // 4x chunks per thread\n int nth_scaled = nth * 4;\n int64_t chunk_size = (nr + nth_scaled - 1) / nth_scaled;\n int64_t nchunk = (nr + chunk_size - 1) / chunk_size;\n\n if (nth == 1 || nchunk < nth || disable_chunking) {\n nchunk = nth;\n }\n\n if (ith == 0) {\n ggml_threadpool_chunk_set(params->threadpool, nth);\n }\n\n ggml_barrier(params->threadpool);\n\n const int64_t dr = (nr + nchunk - 1) / nchunk;\n\n static constexpr int64_t Q_TILE_SZ = ggml_fa_tile_config::Q;\n bool use_tiled = !use_ref &&\n (q->type == GGML_TYPE_F32 &&\n kv_is_f32_or_f16 &&\n k->type == v->type &&\n neq1 >= Q_TILE_SZ);\n#ifdef GGML_SIMD\n use_tiled &= (DV % GGML_F32_EPR == 0);\n#endif\n int current_chunk = ith;\n\n while (current_chunk < nchunk) {\n const int64_t ir0 = dr * current_chunk;\n const int64_t ir1 = MIN(ir0 + dr, nr);\n\n if (use_tiled) {\n ggml_compute_forward_flash_attn_ext_tiled(params, dst, ir0, ir1);\n } else {\n ggml_compute_forward_flash_attn_ext_f16_one_chunk(params, dst, ir0, ir1, 0, nek1, nullptr, 0);\n }\n\n current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);\n }\n }\n}\n\nvoid ggml_compute_forward_flash_attn_ext(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n switch (dst->op_params[3]) {\n case GGML_PREC_DEFAULT:\n case GGML_PREC_F32:\n {\n // uses F32 accumulators\n ggml_compute_forward_flash_attn_ext_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_flash_attn_back\n\nstatic void ggml_compute_forward_flash_attn_back_f32(\n const ggml_compute_params * params,\n const bool masked,\n ggml_tensor * dst) {\n\n const ggml_tensor * q = dst->src[0];\n const ggml_tensor * k = dst->src[1];\n const ggml_tensor * v = dst->src[2];\n const ggml_tensor * d = dst->src[3];\n\n GGML_TENSOR_LOCALS(int64_t, neq, q, ne)\n GGML_TENSOR_LOCALS(size_t, nbq, q, nb)\n GGML_TENSOR_LOCALS(int64_t, nek, k, ne)\n GGML_TENSOR_LOCALS(size_t, nbk, k, nb)\n GGML_TENSOR_LOCALS(int64_t, nev, v, ne)\n GGML_TENSOR_LOCALS(size_t, nbv, v, nb)\n GGML_TENSOR_LOCALS(int64_t, ned, d, ne)\n GGML_TENSOR_LOCALS(size_t, nbd, d, nb)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n GGML_TENSOR_LOCALS(size_t, nb, dst, nb)\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t D = neq0;\n const int64_t N = neq1;\n const int64_t P = nek1 - N;\n const int64_t M = P + N;\n\n const int Mup = ggml_up(M, GGML_SOFT_MAX_UNROLL);\n const int mxDM = MAX(D, Mup);\n\n // GGML_ASSERT(ne0 == D);\n // GGML_ASSERT(ne1 == N);\n GGML_ASSERT(P >= 0);\n\n GGML_ASSERT(nbq0 == sizeof(float));\n GGML_ASSERT(nbk0 == sizeof(float));\n GGML_ASSERT(nbv0 == sizeof(float));\n\n GGML_ASSERT(neq0 == D);\n GGML_ASSERT(nek0 == D);\n GGML_ASSERT(nev1 == D);\n GGML_ASSERT(ned0 == D);\n\n GGML_ASSERT(neq1 == N);\n GGML_ASSERT(nek1 == N + P);\n GGML_ASSERT(nev1 == D);\n GGML_ASSERT(ned1 == N);\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n if (ith == 0) {\n memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3);\n }\n ggml_barrier(params->threadpool);\n\n const int64_t elem_q = ggml_nelements(q);\n const int64_t elem_k = ggml_nelements(k);\n\n ggml_type result_type = dst->type;\n GGML_ASSERT(ggml_blck_size(result_type) == 1);\n const size_t tsize = ggml_type_size(result_type);\n\n const size_t offs_q = 0;\n const size_t offs_k = offs_q + GGML_PAD(elem_q * tsize, GGML_MEM_ALIGN);\n const size_t offs_v = offs_k + GGML_PAD(elem_k * tsize, GGML_MEM_ALIGN);\n\n void * grad_q = (char *) dst->data;\n void * grad_k = (char *) dst->data + offs_k;\n void * grad_v = (char *) dst->data + offs_v;\n\n const size_t nbgq1 = nb0*neq0;\n const size_t nbgq2 = nb0*neq0*neq1;\n const size_t nbgq3 = nb0*neq0*neq1*neq2;\n\n const size_t nbgk1 = nb0*nek0;\n const size_t nbgk2 = nb0*nek0*nek1;\n const size_t nbgk3 = nb0*nek0*nek1*neq2;\n\n const size_t nbgv1 = nb0*nev0;\n const size_t nbgv2 = nb0*nev0*nev1;\n const size_t nbgv3 = nb0*nev0*nev1*neq2;\n\n // parallelize by k rows using ggml_vec_dot_f32\n\n // total rows in k\n const int nr = nek2*nek3;\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n const float scale = 1.0f/sqrtf(D);\n\n //printf(\"P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\\n\", P, N, D, ir0, ir1, scale);\n\n // how often k2 (and v2) is repeated in q2\n int nrep = neq2/nek2;\n\n for (int ir = ir0; ir < ir1; ++ir) {\n // q indices\n const int ik3 = ir/(nek2);\n const int ik2 = ir - ik3*nek2;\n\n const int iq3 = ik3;\n const int id3 = ik3;\n const int iv3 = ik3;\n const int iv2 = ik2;\n\n for (int irep = 0; irep < nrep; ++irep) {\n const int iq2 = ik2 + irep*nek2;\n const int id2 = iq2;\n\n // (ik2 + irep*nek2) % nek2 == ik2\n for (int iq1 = 0; iq1 < neq1; ++iq1) {\n const int id1 = iq1;\n\n // not sure about CACHE_LINE_SIZE_F32..\n // - maybe it must not be multiplied by 2 and excluded from .. in SM 1*(..) offset?\n float * S = (float *) params->wdata + ith*2*(mxDM + CACHE_LINE_SIZE_F32) + 0*(mxDM+CACHE_LINE_SIZE_F32);\n float * SM = (float *) params->wdata + ith*2*(mxDM + CACHE_LINE_SIZE_F32) + 1*(mxDM+CACHE_LINE_SIZE_F32);\n\n for (int i = M; i < Mup; ++i) {\n S[i] = -INFINITY;\n }\n\n const int64_t masked_begin = masked ? (P + iq1 + 1) : M;\n for (int64_t ic = 0; ic < masked_begin; ++ic) {\n // k indices\n const int ik1 = ic;\n\n // S indices\n const int i1 = ik1;\n\n ggml_vec_dot_f32(neq0,\n S + i1, 0,\n (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,\n (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);\n }\n\n // scale\n ggml_vec_scale_f32(masked_begin, S, scale);\n\n for (int64_t i = masked_begin; i < M; i++) {\n S[i] = -INFINITY;\n }\n\n // softmax\n // exclude known -INF S[..] values from max and loop\n // dont forget to set their SM values to zero\n {\n float max = -INFINITY;\n ggml_vec_max_f32(masked_begin, &max, S);\n\n ggml_float sum = 0.0;\n {\n#ifdef GGML_SOFT_MAX_ACCELERATE\n max = -max;\n vDSP_vsadd(SM, 1, &max, SM, 1, Mup);\n vvexpf(SM, SM, &Mup);\n ggml_vec_sum_f32(Mup, &sum, SM);\n#else\n sum = ggml_vec_soft_max_f32(Mup, SM, S, max);\n#endif\n }\n\n assert(sum > 0.0);\n\n sum = 1.0/sum;\n ggml_vec_scale_f32(masked_begin, SM, sum);\n\n }\n\n // step-by-step explanation\n {\n // forward-process shape grads from backward process\n // parallel_for ik2,ik3:\n // for irep:\n // iq2 = ik2 + irep*nek2\n // k[:D,:M,:,:] [D,M,:,:] grad[k][:D,:M,ik2,ik3] += grad[kcur]\n // q[:D,:N,:,:] [D,N,:,:] grad[q][:D,iq1,iq2,iq3] += grad[qcur]\n // v[:M,:D,:,:] [M,D,:,:] grad[v][:M,:D,iv2,iv3] += grad[vcur]\n // for iq1:\n // kcur = k[:D,:M,ik2,ik3] [D,M,1,1] grad[kcur] = grad[S1].T @ qcur\n // qcur = q[:D,iq1,iq2,iq3] [D,1,1,1] grad[qcur] = grad[S1] @ kcur\n // vcur = v[:M,:D,iv2,iv3] [M,D,1,1] grad[vcur] = grad[S5].T @ S4\n // S0 = -Inf [D,1,1,1]\n // ~S1[i] = dot(kcur[:D,i], qcur)\n // S1 = qcur @ kcur.T [M,1,1,1] grad[S1] = grad[S2] * scale\n // S2 = S1 * scale [M,1,1,1] grad[S2] = diag_mask_zero(grad[S3], P)\n // S3 = diag_mask_inf(S2, P) [M,1,1,1] grad[S3] = S4 * (grad[S4] - dot(S4, grad[S4]))\n // S4 = softmax(S3) [M,1,1,1] grad[S4] = grad[S5] @ vcur\n // ~S5[i] = dot(vcur[:,i], S4)\n // S5 = S4 @ vcur.T [D,1,1,1] grad[S5] = d[:D,id1,id2,id3]\n // ~dst[i,iq1,iq2,iq3] = S5[i] ^\n // dst[:D,iq1,iq2,iq3] = S5 | grad[dst[:D,iq1,iq2,iq3]] = d[:D,id1,id2,id3]\n // dst backward-/ grad[dst] = d\n //\n // output gradients with their dependencies:\n //\n // grad[kcur] = grad[S1].T @ qcur\n // grad[S1] = diag_mask_zero(grad[S3], P) * scale\n // grad[S3] = S4 * (grad[S4] - dot(S4, grad[S4]))\n // grad[S4] = grad[S5] @ vcur\n // grad[S4] = d[:D,id1,id2,id3] @ vcur\n // grad[qcur] = grad[S1] @ kcur\n // grad[vcur] = grad[S5].T @ S4\n // grad[vcur] = d[:D,id1,id2,id3].T @ S4\n //\n // in post-order:\n //\n // S1 = qcur @ kcur.T\n // S2 = S1 * scale\n // S3 = diag_mask_inf(S2, P)\n // S4 = softmax(S3)\n // grad[S4] = d[:D,id1,id2,id3] @ vcur\n // grad[S3] = S4 * (grad[S4] - dot(S4, grad[S4]))\n // grad[S1] = diag_mask_zero(grad[S3], P) * scale\n // grad[qcur] = grad[S1] @ kcur\n // grad[kcur] = grad[S1].T @ qcur\n // grad[vcur] = d[:D,id1,id2,id3].T @ S4\n //\n // using less variables (SM=S4):\n //\n // S = diag_mask_inf(qcur @ kcur.T * scale, P)\n // SM = softmax(S)\n // S = d[:D,iq1,iq2,iq3] @ vcur\n // dot_SM_gradSM = dot(SM, S)\n // S = SM * (S - dot(SM, S))\n // S = diag_mask_zero(S, P) * scale\n //\n // grad[q][:D,iq1,iq2,iq3] += S @ kcur\n // grad[k][:D,:M,ik2,ik3] += S.T @ qcur\n // grad[v][:M,:D,iv2,iv3] += d[:D,id1,id2,id3].T @ SM\n }\n\n // S = gradSM = d[:D,id1,id2,id3] @ vcur[:,:,iv2,iv3]\n // S = d[:D,id1,id2,id3] @ vcur[:,:,iv2,iv3]\n // for ic:\n // S[:M] += vcur[:M,ic,iv2,iv3] * d[ic,id1,id2,id3]\n // exclude known future zero S[..] values from operation\n ggml_vec_set_f32(masked_begin, S, 0);\n for (int64_t ic = 0; ic < D; ++ic) {\n ggml_vec_mad_f32(masked_begin,\n S,\n (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)),\n *(float *) ((char *) d->data + (ic*nbd0 + id1*nbd1 + id2*nbd2 + id3*nbd3)));\n }\n\n // S = SM * (S - dot(SM, S))\n float dot_SM_gradSM = 0;\n ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, 0, SM, 0, S, 0, 1);\n ggml_vec_acc1_f32(M, S, -dot_SM_gradSM);\n ggml_vec_mul_f32 (masked_begin, S, S, SM);\n\n // S = diag_mask_zero(S, P) * scale\n // already done by above ggml_vec_set_f32\n\n // exclude known zero S[..] values from operation\n ggml_vec_scale_f32(masked_begin, S, scale);\n\n // S shape [M,1]\n // SM shape [M,1]\n // kcur shape [D,M]\n // qcur shape [D,1]\n // vcur shape [M,D]\n\n // grad[q][:D,iq1,iq2,iq3] += S @ kcur\n // grad[q][:D,iq1,iq2,iq3] += shape[M,1] @ shape[D,M]\n // for ic:\n // grad[q][:D,iq1,iq2,iq3] += S[ic] * kcur[:D,ic,ik2,ik3]\n // exclude known zero S[..] values from loop\n for (int64_t ic = 0; ic < masked_begin; ++ic) {\n ggml_vec_mad_f32(D,\n (float *) ((char *) grad_q + (iq1*nbgq1 + iq2*nbgq2 + iq3*nbgq3)),\n (float *) ((char *) k->data + (ic*nbk1 + ik2*nbk2 + ik3*nbk3)),\n S[ic]);\n }\n\n // grad[k][:D,:M,iq2,iq3] += S.T @ qcur\n // for ic:\n // grad[k][:D,ic,iq2,iq3] += S.T[0,ic] * qcur[:D,0]\n // grad[k][:D,ic,iq2,iq3] += S[ic] * qcur[:D,0]\n // exclude known zero S[..] values from loop\n for (int64_t ic = 0; ic < masked_begin; ++ic) {\n ggml_vec_mad_f32(D,\n (float *) ((char *) grad_k + (ic*nbgk1 + ik2*nbgk2 + ik3*nbgk3)),\n (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)),\n S[ic]);\n }\n\n // grad[v][:M,:D,iv2,iv3] += d[:D,id1,id2,id3].T @ SM\n // for ic:\n // grad[v][:M,ic,iv2,iv3] += d[:D,id1,id2,id3].T[0,ic] * SM[:M]\n // grad[v][:M,ic,iv2,iv3] += d[ic,id1,id2,id3] * SM[:M]\n // exclude known zero SM[..] values from mad\n for (int64_t ic = 0; ic < D; ++ic) {\n ggml_vec_mad_f32(masked_begin,\n (float *) ((char *) grad_v + ( ic*nbgv1 + iv2*nbgv2 + iv3*nbgv3)),\n SM,\n *(float *) ((char *) d->data + (ic*nbd0 + id1*nbd1 + id2*nbd2 + id3*nbd3)));\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_flash_attn_back(\n const ggml_compute_params * params,\n const bool masked,\n ggml_tensor * dst) {\n\n const ggml_tensor * q = dst->src[0];\n\n switch (q->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_flash_attn_back_f32(params, masked, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_ssm_conv\n\nstatic void ggml_compute_forward_ssm_conv_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0]; // conv_x\n const ggml_tensor * src1 = dst->src[1]; // conv1d.weight\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nc = src1->ne[0]; // d_conv\n const int ncs = src0->ne[0]; // d_conv - 1 + n_t\n const int nr = src0->ne[1]; // d_inner\n const int n_t = dst->ne[1]; // tokens per sequence\n const int n_s = dst->ne[2]; // number of sequences in the batch\n\n GGML_ASSERT( dst->ne[0] == nr);\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n GGML_ASSERT(src1->nb[0] == sizeof(float));\n GGML_ASSERT(src0->nb[1] == src0->ne[0]*sizeof(float));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n const int ir = ir1 - ir0;\n\n for (int i3 = 0; i3 < n_s; ++i3) {\n for (int i2 = 0; i2 < n_t; ++i2) {\n // {d_conv - 1 + n_t, d_inner, n_seqs}\n // sliding window\n const float * s = (const float *) ((const char *) src0->data + ir0*(src0->nb[1]) + i2*(src0->nb[0]) + i3*(src0->nb[2])); // {d_conv, d_inner, n_s}\n const float * c = (const float *) ((const char *) src1->data + ir0*(src1->nb[1])); // {d_conv, d_inner}\n float * x = (float *) ((char *) dst->data + ir0*(dst->nb[0]) + i2*(dst->nb[1]) + i3*(dst->nb[2])); // {d_inner, n_t, n_s}\n\n // TODO: transpose the output for smaller strides for big batches?\n // d_inner\n for (int i1 = 0; i1 < ir; ++i1) {\n // rowwise dot product\n // NOTE: not using ggml_vec_dot_f32, because its sum is in double precision\n float sumf = 0.0f;\n\n // d_conv\n for (int i0 = 0; i0 < nc; ++i0) {\n sumf += s[i0 + i1*ncs] * c[i0 + i1*nc];\n }\n x[i1] = sumf;\n }\n }\n }\n}\n\nvoid ggml_compute_forward_ssm_conv(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n switch (dst->src[0]->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_ssm_conv_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_ssm_scan\n\nstatic void ggml_compute_forward_ssm_scan_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0]; // s {d_state, dim, n_head, n_seqs+}\n const ggml_tensor * src1 = dst->src[1]; // x {dim, n_head, n_seq_tokens, n_seqs}\n const ggml_tensor * src2 = dst->src[2]; // dt {n_head, n_seq_tokens, n_seqs}\n const ggml_tensor * src3 = dst->src[3]; // A {d_state, n_head} or {1, n_head}\n const ggml_tensor * src4 = dst->src[4]; // B {d_state, n_group, n_seq_tokens, n_seqs}\n const ggml_tensor * src5 = dst->src[5]; // C {d_state, n_group, n_seq_tokens, n_seqs}\n const ggml_tensor * src6 = dst->src[6]; // ids {n_seqs}\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t nc = src0->ne[0]; // d_state\n const int64_t nr = src0->ne[1]; // dim\n const int64_t nh = src1->ne[1]; // n_head\n const int64_t ng = src4->ne[1];\n const int64_t nt = src1->ne[2]; // number of tokens per sequence\n const int64_t ns = src1->ne[3]; // number of sequences in the batch\n\n // can't use ggml_nbytes because src1 is not necessarily contiguous\n const int64_t s_off = ggml_nelements(src1) * ggml_element_size(src1);\n\n GGML_ASSERT(ggml_nelements(src1) + nc*nr*nh*ns == ggml_nelements(dst));\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n GGML_ASSERT(src1->nb[0] == sizeof(float));\n GGML_ASSERT(src2->nb[0] == sizeof(float));\n GGML_ASSERT(src3->nb[0] == sizeof(float));\n GGML_ASSERT(src4->nb[0] == sizeof(float));\n GGML_ASSERT(src5->nb[0] == sizeof(float));\n GGML_ASSERT(src6->nb[0] == sizeof(int32_t));\n GGML_ASSERT(nh % ng == 0);\n\n // heads per thread\n const int dh = (nh + nth - 1)/nth;\n\n // head range for this thread\n const int ih0 = dh*ith;\n const int ih1 = MIN(ih0 + dh, nh);\n\n const int32_t * ids = (const int32_t *) src6->data;\n\n for (int i3 = 0; i3 < ns; ++i3) {\n const float * s0 = (const float *) ((const char *) src0->data + ids[i3]*(src0->nb[3])); // {d_state, dim, nh, ns}\n float * s = ( float *) (( char *) dst->data + i3*(src0->nb[3]) + s_off); // {d_state, dim, nh, ns}\n\n for (int i2 = 0; i2 < nt; ++i2) {\n const float * x = (const float *) ((const char *) src1->data + i2*(src1->nb[2]) + i3*(src1->nb[3])); // {dim, nh, nt, ns}\n const float * dt = (const float *) ((const char *) src2->data + i2*(src2->nb[1]) + i3*(src2->nb[2])); // {nh, nt, ns}\n const float * A = (const float *) ((const char *) src3->data); // {d_state, nh} or {1, nh}\n const float * B = (const float *) ((const char *) src4->data + i2*(src4->nb[2]) + i3*(src4->nb[3])); // {d_state, ng, nt, ns}\n const float * C = (const float *) ((const char *) src5->data + i2*(src5->nb[2]) + i3*(src5->nb[3])); // {d_state, ng, nt, ns}\n float * y = ( float *) (( char *) dst->data + i2*(nh*nr*sizeof(float)) + i3*(nt*nh*nr*sizeof(float))); // {dim, nh, nt, ns}\n\n if (src3->ne[0] == 1) {\n // Mamba-2 has a scalar decay factor per head; dA can be outside the state-wise loop\n\n // n_head\n for (int h = ih0; h < ih1; ++h) {\n // ref: https://github.com/state-spaces/mamba/blob/62db608da60f6fc790b8ed9f4b3225e95ca15fde/mamba_ssm/ops/triton/softplus.py#L16\n const float dt_soft_plus = ggml_compute_softplus_f32(dt[h]);\n const float dA = expf(dt_soft_plus * A[h]);\n const int g = h / (nh / ng); // repeat_interleave\n\n // dim\n for (int i1 = 0; i1 < nr; ++i1) {\n const int ii = i1 + h*nr;\n const float x_dt = x[ii] * dt_soft_plus;\n float sumf = 0.0f;\n#if defined(GGML_SIMD)\n #if defined(__ARM_FEATURE_SVE)\n const int ggml_f32_epr = svcntw();\n const int ggml_f32_step = 1 * ggml_f32_epr;\n\n const int np = (nc & ~(ggml_f32_step - 1));\n\n GGML_F32_VEC sum = GGML_F32_VEC_ZERO;\n\n GGML_F32_VEC adA = GGML_F32_VEC_SET1(dA);\n GGML_F32_VEC axdt = GGML_F32_VEC_SET1(x_dt);\n\n for (int i = 0; i < np; i += ggml_f32_step) {\n // TODO: maybe unroll more?\n for (int j = 0; j < 1; j++) {\n GGML_F32_VEC t0 = GGML_F32_VEC_LOAD(s0 + i + j*ggml_f32_epr + ii*nc);\n GGML_F32_VEC t1 = GGML_F32_VEC_LOAD(B + i + j*ggml_f32_epr + g*nc);\n GGML_F32_VEC t2 = GGML_F32_VEC_LOAD(C + i + j*ggml_f32_epr + g*nc);\n\n t0 = GGML_F32_VEC_MUL(t0, adA);\n t1 = GGML_F32_VEC_MUL(t1, axdt);\n\n t0 = GGML_F32_VEC_ADD(t0, t1);\n\n sum = GGML_F32_VEC_FMA(sum, t0, t2);\n\n GGML_F32_VEC_STORE(s + i + j*ggml_f32_epr + ii*nc, t0);\n }\n }\n\n sumf = GGML_F32xt_REDUCE_ONE(sum);\n #elif defined(__riscv_v_intrinsic)\n // todo: RVV implementation\n const int np = 0;\n #else\n const int np = (nc & ~(GGML_F32_STEP - 1));\n\n GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };\n\n GGML_F32_VEC adA = GGML_F32_VEC_SET1(dA);\n GGML_F32_VEC axdt = GGML_F32_VEC_SET1(x_dt);\n\n GGML_F32_VEC ax[GGML_F32_ARR];\n GGML_F32_VEC ay[GGML_F32_ARR];\n GGML_F32_VEC az[GGML_F32_ARR];\n\n for (int i = 0; i < np; i += GGML_F32_STEP) {\n for (int j = 0; j < GGML_F32_ARR; j++) {\n ax[j] = GGML_F32_VEC_LOAD(s0 + i + j*GGML_F32_EPR + ii*nc);\n ay[j] = GGML_F32_VEC_LOAD(B + i + j*GGML_F32_EPR + g*nc);\n az[j] = GGML_F32_VEC_LOAD(C + i + j*GGML_F32_EPR + g*nc);\n\n ax[j] = GGML_F32_VEC_MUL(ax[j], adA);\n ay[j] = GGML_F32_VEC_MUL(ay[j], axdt);\n\n ax[j] = GGML_F32_VEC_ADD(ax[j], ay[j]);\n\n sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], az[j]);\n\n GGML_F32_VEC_STORE(s + i + j*GGML_F32_EPR + ii*nc, ax[j]);\n }\n }\n\n // reduce sum0..sum3 to sum0\n GGML_F32_VEC_REDUCE(sumf, sum);\n #endif\n#else\n const int np = 0;\n#endif\n // d_state\n for (int i0 = np; i0 < nc; ++i0) {\n const int i = i0 + ii*nc;\n const int ig = i0 + g*nc;\n // state = prev_state * dA + dB * x\n const float state = (s0[i] * dA) + (B[ig] * x_dt);\n // y = rowwise_dotprod(state, C)\n sumf += state * C[ig];\n s[i] = state;\n }\n y[ii] = sumf;\n }\n }\n } else {\n // Mamba-1 has an element-wise decay factor for the states\n\n // n_head\n for (int h = ih0; h < ih1; ++h) {\n // ref: https://github.com/state-spaces/mamba/blob/62db608da60f6fc790b8ed9f4b3225e95ca15fde/mamba_ssm/ops/triton/softplus.py#L16\n const float dt_soft_plus = ggml_compute_softplus_f32(dt[h]);\n const int g = h / (nh / ng); // repeat_interleave\n\n // dim\n for (int i1 = 0; i1 < nr; ++i1) {\n const int ii = i1 + h*nr;\n const float x_dt = x[ii] * dt_soft_plus;\n#if defined(__ARM_FEATURE_SVE)\n svfloat32_t vx_dt = GGML_F32_VEC_SET1(x_dt);\n svfloat32_t vdt_soft_plus = GGML_F32_VEC_SET1(dt_soft_plus);\n svfloat32_t r1_vector = GGML_F32_VEC_ZERO;\n\n // d_state\n // TODO: what happens when (d_state % svcntw()) != 0?\n for (int64_t k = 0; k < nc; k += svcntw()) {\n svfloat32_t vA = GGML_F32_VEC_LOAD(&A[h*nc + k]);\n svfloat32_t vB = GGML_F32_VEC_LOAD(&B[k + g*nc]);\n svfloat32_t vC = GGML_F32_VEC_LOAD(&C[k + g*nc]);\n svfloat32_t vs0 = GGML_F32_VEC_LOAD(&s0[ii*nc + k]);\n\n svfloat32_t t1 = GGML_F32_VEC_MUL(vdt_soft_plus, vA);\n t1 = exp_ps_sve(svptrue_b32(), t1);\n svfloat32_t t2 = GGML_F32_VEC_MUL(vx_dt, vB);\n\n vs0 = GGML_F32_VEC_FMA(t2, vs0, t1);\n r1_vector = GGML_F32_VEC_ADD(GGML_F32_VEC_MUL(vs0, vC), r1_vector);\n\n GGML_F32_VEC_STORE(&s[ii*nc + k], vs0);\n }\n y[ii] = GGML_F32xt_REDUCE_ONE(r1_vector);\n#else\n float sumf = 0.0f;\n // NOTE: can't really use GGML_SIMD here because d_state is usually 16\n // and also because expf is used within the loop.\n // d_state\n for (int i0 = 0; i0 < nc; ++i0) {\n const int i = i0 + ii*nc;\n const int ig = i0 + g*nc;\n // state = prev_state * dA + dB * x\n const float state = (s0[i] * expf(dt_soft_plus * A[i0 + h*nc])) + (B[ig] * x_dt);\n // y = rowwise_dotprod(state, C)\n sumf += state * C[ig];\n s[i] = state;\n }\n y[ii] = sumf;\n#endif\n }\n }\n }\n // use the output as the source when it's not the first token-wise iteration\n s0 = s;\n }\n }\n}\n\nvoid ggml_compute_forward_ssm_scan(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n switch (dst->src[0]->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_ssm_scan_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_win_part\n\nstatic void ggml_compute_forward_win_part_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n GGML_UNUSED(params);\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n\n const int32_t nep0 = ((const int32_t *)(dst->op_params))[0];\n const int32_t nep1 = ((const int32_t *)(dst->op_params))[1];\n const int32_t w = ((const int32_t *)(dst->op_params))[2];\n\n assert(ne00 == ne0);\n assert(ne3 == nep0*nep1);\n\n // TODO: optimize / multi-thread\n for (int py = 0; py < nep1; ++py) {\n for (int px = 0; px < nep0; ++px) {\n const int64_t i3 = py*nep0 + px;\n for (int64_t i2 = 0; i2 < ne2; ++i2) {\n for (int64_t i1 = 0; i1 < ne1; ++i1) {\n for (int64_t i0 = 0; i0 < ne0; ++i0) {\n const int64_t i02 = py*w + i2;\n const int64_t i01 = px*w + i1;\n const int64_t i00 = i0;\n\n const int64_t i = i3*ne2*ne1*ne0 + i2*ne1*ne0 + i1*ne0 + i0;\n const int64_t j = i02*ne01*ne00 + i01*ne00 + i00;\n\n if (py*w + i2 >= ne02 || px*w + i1 >= ne01) {\n ((float *) dst->data)[i] = 0.0f;\n } else {\n ((float *) dst->data)[i] = ((float *) src0->data)[j];\n }\n }\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_win_part(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_win_part_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_win_unpart\n\nstatic void ggml_compute_forward_win_unpart_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n GGML_UNUSED(params);\n\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)\n\n const int32_t w = ((const int32_t *)(dst->op_params))[0];\n\n // padding\n const int px = (w - ne1%w)%w;\n //const int py = (w - ne2%w)%w;\n\n const int npx = (px + ne1)/w;\n //const int npy = (py + ne2)/w;\n\n assert(ne0 == ne00);\n\n // TODO: optimize / multi-thread\n for (int64_t i2 = 0; i2 < ne2; ++i2) {\n for (int64_t i1 = 0; i1 < ne1; ++i1) {\n for (int64_t i0 = 0; i0 < ne0; ++i0) {\n const int ip2 = i2/w;\n const int ip1 = i1/w;\n\n const int64_t i02 = i2%w;\n const int64_t i01 = i1%w;\n const int64_t i00 = i0;\n\n const int64_t i = (ip2*npx + ip1)*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00 + i00;\n const int64_t j = i2*ne1*ne0 + i1*ne0 + i0;\n\n ((float *) dst->data)[j] = ((float *) src0->data)[i];\n }\n }\n }\n}\n\nvoid ggml_compute_forward_win_unpart(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_win_unpart_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n//ggml_compute_forward_unary\n\nvoid ggml_compute_forward_unary(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_unary_op op = ggml_get_unary_op(dst);\n\n switch (op) {\n case GGML_UNARY_OP_ABS:\n {\n ggml_compute_forward_abs(params, dst);\n } break;\n case GGML_UNARY_OP_SGN:\n {\n ggml_compute_forward_sgn(params, dst);\n } break;\n case GGML_UNARY_OP_NEG:\n {\n ggml_compute_forward_neg(params, dst);\n } break;\n case GGML_UNARY_OP_STEP:\n {\n ggml_compute_forward_step(params, dst);\n } break;\n case GGML_UNARY_OP_TANH:\n {\n ggml_compute_forward_tanh(params, dst);\n } break;\n case GGML_UNARY_OP_ELU:\n {\n ggml_compute_forward_elu(params, dst);\n } break;\n case GGML_UNARY_OP_RELU:\n {\n ggml_compute_forward_relu(params, dst);\n } break;\n case GGML_UNARY_OP_SIGMOID:\n {\n ggml_compute_forward_sigmoid(params, dst);\n } break;\n case GGML_UNARY_OP_GELU:\n {\n ggml_compute_forward_gelu(params, dst);\n } break;\n case GGML_UNARY_OP_GELU_ERF:\n {\n ggml_compute_forward_gelu_erf(params, dst);\n } break;\n case GGML_UNARY_OP_GELU_QUICK:\n {\n ggml_compute_forward_gelu_quick(params, dst);\n } break;\n case GGML_UNARY_OP_SILU:\n {\n ggml_compute_forward_silu(params, dst);\n } break;\n case GGML_UNARY_OP_HARDSWISH:\n {\n ggml_compute_forward_hardswish(params, dst);\n } break;\n case GGML_UNARY_OP_HARDSIGMOID:\n {\n ggml_compute_forward_hardsigmoid(params, dst);\n } break;\n case GGML_UNARY_OP_EXP:\n {\n ggml_compute_forward_exp(params, dst);\n } break;\n case GGML_UNARY_OP_FLOOR:\n {\n ggml_compute_forward_floor(params, dst);\n } break;\n case GGML_UNARY_OP_CEIL:\n {\n ggml_compute_forward_ceil(params, dst);\n } break;\n case GGML_UNARY_OP_ROUND:\n {\n ggml_compute_forward_round(params, dst);\n } break;\n case GGML_UNARY_OP_TRUNC:\n {\n ggml_compute_forward_trunc(params, dst);\n } break;\n case GGML_UNARY_OP_XIELU:\n {\n ggml_compute_forward_xielu(params, dst);\n } break;\n case GGML_UNARY_OP_EXPM1:\n {\n ggml_compute_forward_expm1(params, dst);\n } break;\n case GGML_UNARY_OP_SOFTPLUS:\n {\n ggml_compute_forward_softplus(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n//ggml_compute_forward_glu\n\nvoid ggml_compute_forward_glu(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_glu_op op = ggml_get_glu_op(dst);\n\n switch (op) {\n case GGML_GLU_OP_REGLU:\n {\n ggml_compute_forward_reglu(params, dst);\n } break;\n case GGML_GLU_OP_GEGLU:\n {\n ggml_compute_forward_geglu(params, dst);\n } break;\n case GGML_GLU_OP_SWIGLU:\n {\n ggml_compute_forward_swiglu(params, dst);\n } break;\n case GGML_GLU_OP_SWIGLU_OAI:\n {\n ggml_compute_forward_swiglu_oai(params, dst);\n } break;\n case GGML_GLU_OP_GEGLU_ERF:\n {\n ggml_compute_forward_geglu_erf(params, dst);\n } break;\n case GGML_GLU_OP_GEGLU_QUICK:\n {\n ggml_compute_forward_geglu_quick(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_get_rel_pos\n\nstatic void ggml_compute_forward_get_rel_pos_f16(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n GGML_UNUSED(params);\n\n const ggml_tensor * src0 = dst->src[0];\n\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L292-L322\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n const int64_t w = ne1;\n\n ggml_fp16_t * src0_data = (ggml_fp16_t *) src0->data;\n ggml_fp16_t * dst_data = (ggml_fp16_t *) dst->data;\n\n for (int64_t i2 = 0; i2 < ne2; ++i2) {\n for (int64_t i1 = 0; i1 < ne1; ++i1) {\n const int64_t pos = (w - i1 - 1) + i2;\n for (int64_t i0 = 0; i0 < ne0; ++i0) {\n dst_data[i2*ne1*ne0 + i1*ne0 + i0] = src0_data[pos*ne00 + i0];\n }\n }\n }\n}\n\nvoid ggml_compute_forward_get_rel_pos(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n {\n ggml_compute_forward_get_rel_pos_f16(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_add_rel_pos\n\nstatic void ggml_compute_forward_add_rel_pos_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const ggml_tensor * src2 = dst->src[2];\n\n const bool inplace = (bool) ((int32_t *) dst->op_params)[0];\n if (!inplace) {\n if (params->ith == 0) {\n memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));\n }\n ggml_barrier(params->threadpool);\n }\n // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L357-L359\n\n float * src1_data = (float *) src1->data;\n float * src2_data = (float *) src2->data;\n float * dst_data = (float *) dst->data;\n\n const int64_t ne10 = src1->ne[0];\n const int64_t ne11 = src1->ne[1];\n const int64_t ne12 = src1->ne[2];\n const int64_t ne13 = src1->ne[3];\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n // total patches in dst\n const int np = ne13;\n\n // patches per thread\n const int dp = (np + nth - 1)/nth;\n\n // patch range for this thread\n const int ip0 = dp*ith;\n const int ip1 = MIN(ip0 + dp, np);\n\n for (int64_t i13 = ip0; i13 < ip1; ++i13) {\n for (int64_t i12 = 0; i12 < ne12; ++i12) {\n for (int64_t i11 = 0; i11 < ne11; ++i11) {\n const int64_t jp1 = i13*ne12*ne11*ne10 + i12*ne11*ne10 + i11*ne10;\n for (int64_t i10 = 0; i10 < ne10; ++i10) {\n const int64_t jp0 = jp1 + i10;\n const float src1_e = src1_data[jp0];\n const float src2_e = src2_data[jp0];\n\n const int64_t jdh = jp0 * ne10;\n const int64_t jdw = jdh - (ne10 - 1) * i10;\n\n for (int64_t j = 0; j < ne10; ++j) {\n dst_data[jdh + j ] += src2_e;\n dst_data[jdw + j*ne10] += src1_e;\n }\n }\n }\n }\n }\n}\n\nvoid ggml_compute_forward_add_rel_pos(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_add_rel_pos_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_rwkv_wkv6\n\nstatic void ggml_compute_forward_rwkv_wkv6_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n const int64_t T = dst->src[1]->ne[2];\n const int64_t C = dst->ne[0];\n const int64_t HEADS = dst->src[1]->ne[1];\n const int64_t n_seqs = dst->src[5]->ne[1];\n const int64_t head_size = C / HEADS;\n\n float * dst_data = (float *) dst->data;\n float * state = ((float *) dst->data) + C * T;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n if (ith >= HEADS) {\n return;\n }\n\n const int h_start = (HEADS * ith) / nth;\n const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ?\n (HEADS * (ith + 1)) / nth : HEADS;\n\n float * k = (float *) dst->src[0]->data;\n float * v = (float *) dst->src[1]->data;\n float * r = (float *) dst->src[2]->data;\n float * time_faaaa = (float *) dst->src[3]->data;\n float * time_decay = (float *) dst->src[4]->data;\n\n size_t t_stride = HEADS * head_size; // Same to C\n\n size_t h_stride = C / HEADS;\n GGML_ASSERT(C % HEADS == 0); // C must be divisible by HEADS\n size_t h_stride_2d = head_size * head_size;\n\n if (ith == 0) {\n memset(dst_data, 0, T * C * sizeof(float));\n }\n ggml_barrier(params->threadpool);\n\n\n #if defined(__AVX__) && !defined(__AVX512F__)\n #define GGML_F32X GGML_F32x8\n #define GGML_F32X_SET1 GGML_F32x8_SET1\n #define GGML_F32X_LOAD GGML_F32x8_LOAD\n #define GGML_F32X_STORE GGML_F32x8_STORE\n #define GGML_F32X_MUL GGML_F32x8_MUL\n #define GGML_F32X_FMA GGML_F32x8_FMA\n #define WKV_VECTOR_SIZE 8\n #elif defined(__AVX512F__)\n #define GGML_F32X GGML_F32x16\n #define GGML_F32X_SET1 GGML_F32x16_SET1\n #define GGML_F32X_LOAD GGML_F32x16_LOAD\n #define GGML_F32X_STORE GGML_F32x16_STORE\n #define GGML_F32X_MUL GGML_F32x16_MUL\n #define GGML_F32X_FMA GGML_F32x16_FMA\n #define WKV_VECTOR_SIZE 16\n #elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n #define GGML_F32X GGML_F32xt\n #define GGML_F32X_SET1 GGML_F32xt_SET1\n #define GGML_F32X_LOAD GGML_F32xt_LOAD\n #define GGML_F32X_STORE GGML_F32xt_STORE\n #define GGML_F32X_MUL GGML_F32xt_MUL\n #define GGML_F32X_FMA GGML_F32xt_FMA\n #define WKV_VECTOR_SIZE 8\n #elif defined(__ARM_NEON) && defined(__aarch64__)\n #define GGML_F32X GGML_F32x4\n #define GGML_F32X_SET1 GGML_F32x4_SET1\n #define GGML_F32X_LOAD GGML_F32x4_LOAD\n #define GGML_F32X_STORE GGML_F32x4_STORE\n #define GGML_F32X_MUL GGML_F32x4_MUL\n #define GGML_F32X_FMA GGML_F32x4_FMA\n #define WKV_VECTOR_SIZE 4\n #endif\n\n #ifdef WKV_VECTOR_SIZE\n int wkv_vector_size;\n #if defined(__ARM_FEATURE_SVE)\n wkv_vector_size = svcntw();\n #else\n wkv_vector_size = WKV_VECTOR_SIZE;\n #endif\n const int64_t vec_count = head_size / wkv_vector_size;\n\n for (int64_t t = 0; t < T; t++) {\n size_t t_offset = t * t_stride;\n size_t state_offset = head_size * C * (t / (T / n_seqs));\n float * state_cur = state + state_offset;\n float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[5]->data + state_offset;\n\n for (int64_t h = h_start; h < h_end; h++) {\n size_t h_offset = h * h_stride;\n size_t t_h_offset = t_offset + h_offset;\n size_t h_2d_offset = h * h_stride_2d;\n\n for (int64_t i = 0; i < head_size; i++) {\n size_t t_h_i_offset = t_h_offset + i;\n size_t h_i_offset = h_offset + i;\n size_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n float k_val = k[t_h_i_offset];\n float r_val = r[t_h_i_offset];\n float time_faaaa_val = time_faaaa[h_i_offset];\n float time_decay_val = time_decay[t_h_i_offset];\n\n // Broadcast scalar values to vectors\n GGML_F32X k_vec = GGML_F32X_SET1(k_val);\n GGML_F32X r_vec = GGML_F32X_SET1(r_val);\n GGML_F32X time_faaaa_vec = GGML_F32X_SET1(time_faaaa_val);\n GGML_F32X time_decay_vec = GGML_F32X_SET1(time_decay_val);\n\n for (int64_t j = 0; j < vec_count; j++) {\n size_t base_j = j * wkv_vector_size;\n size_t t_h_j_offset = t_h_offset + base_j;\n size_t h_2d_i_j_offset = h_2d_i_offset + base_j;\n\n // Load x elements at once\n GGML_F32X v_vec = GGML_F32X_LOAD(&v[t_h_j_offset]);\n GGML_F32X prev_state_vec = GGML_F32X_LOAD(&state_prev[h_2d_i_j_offset]);\n GGML_F32X dst_vec = GGML_F32X_LOAD(&dst_data[t_h_j_offset]);\n\n // Compute kv = v * k\n GGML_F32X kv_vec = GGML_F32X_MUL(v_vec, k_vec);\n\n // Compute temp = kv * time_faaaa + prev_state\n GGML_F32X temp_vec = GGML_F32X_FMA(prev_state_vec, kv_vec, time_faaaa_vec);\n\n // Update dst: dst += temp * r\n dst_vec = GGML_F32X_FMA(dst_vec, temp_vec, r_vec);\n GGML_F32X_STORE(&dst_data[t_h_j_offset], dst_vec);\n\n // Update state: state = prev_state * time_decay + kv\n GGML_F32X new_state_vec = GGML_F32X_FMA(kv_vec, prev_state_vec, time_decay_vec);\n GGML_F32X_STORE(&state_cur[h_2d_i_j_offset], new_state_vec);\n }\n\n // Handle remaining elements, this will not be used.\n for (int64_t j = vec_count * wkv_vector_size; j < head_size; j++) {\n size_t t_h_j_offset = t_h_offset + j;\n size_t h_2d_i_j_offset = h_2d_i_offset + j;\n float v_val = v[t_h_j_offset];\n float kv_val = v_val * k_val;\n float prev_state_val = state_prev[h_2d_i_j_offset];\n float temp_val = kv_val * time_faaaa_val + prev_state_val;\n dst_data[t_h_j_offset] += temp_val * r_val;\n state_cur[h_2d_i_j_offset] = prev_state_val * time_decay_val + kv_val;\n }\n }\n }\n }\n\n #else\n // basically fused operations:\n // dst = r @ (time_faaaa * (k @ v) + state),\n // state = time_decay * state + (k @ v),\n // recursive through each token\n for (int64_t t = 0; t < T; t++) {\n size_t t_offset = t * t_stride;\n size_t state_offset = head_size * C * (t / (T / n_seqs));\n float * state_cur = state + state_offset;\n float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[5]->data + state_offset;\n\n for (int64_t h = h_start; h < h_end; h++) {\n size_t h_offset = h * h_stride;\n size_t t_h_offset = t_offset + h_offset;\n size_t h_2d_offset = h * h_stride_2d;\n\n for (int64_t i = 0; i < head_size; i++) {\n size_t t_h_i_offset = t_h_offset + i;\n size_t h_i_offset = h_offset + i;\n size_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n float k_val = k[t_h_i_offset];\n float r_val = r[t_h_i_offset];\n float time_faaaa_val = time_faaaa[h_i_offset];\n // RWKV v6: different time_decay for each token.\n float time_decay_val = time_decay[t_h_i_offset];\n\n for (int64_t j = 0; j < head_size; j++) {\n size_t t_h_j_offset = t_h_offset + j;\n size_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n float v_val = v[t_h_j_offset];\n float kv_val = v_val * k_val;\n float prev_state_val = state_prev[h_2d_i_j_offset];\n float temp_val = kv_val * time_faaaa_val + prev_state_val;\n dst_data[t_h_j_offset] += temp_val * r_val;\n state_cur[h_2d_i_j_offset] = prev_state_val * time_decay_val + kv_val;\n }\n }\n }\n }\n #endif\n}\n\n\nvoid ggml_compute_forward_rwkv_wkv6(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_rwkv_wkv6_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_gla\n\nstatic void ggml_compute_forward_gla_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n const int64_t T = dst->src[1]->ne[2];\n const int64_t C = dst->ne[0];\n const int64_t HEADS = dst->src[1]->ne[1];\n const int64_t n_seqs = dst->src[4]->ne[1];\n const int64_t head_size = C / HEADS;\n const float scale = ggml_get_op_params_f32(dst, 0);\n\n float * dst_data = (float *) dst->data;\n float * state = ((float *) dst->data) + C * T;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n if (ith >= HEADS) {\n return;\n }\n\n const int h_start = (HEADS * ith) / nth;\n const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ?\n (HEADS * (ith + 1)) / nth : HEADS;\n\n float * k = (float *) dst->src[0]->data;\n float * v = (float *) dst->src[1]->data;\n float * q = (float *) dst->src[2]->data;\n float * g = (float *) dst->src[3]->data;\n\n size_t t_stride = HEADS * head_size; // Same to C\n\n size_t h_stride = C / HEADS;\n GGML_ASSERT(C % HEADS == 0); // C must be divisible by HEADS\n size_t h_stride_2d = head_size * head_size;\n\n if (ith == 0) {\n memset(dst_data, 0, T * C * sizeof(float));\n }\n ggml_barrier(params->threadpool);\n\n\n #if defined(__AVX__) && !defined(__AVX512F__)\n #define GGML_F32X GGML_F32x8\n #define GGML_F32X_SET1 GGML_F32x8_SET1\n #define GGML_F32X_LOAD GGML_F32x8_LOAD\n #define GGML_F32X_STORE GGML_F32x8_STORE\n #define GGML_F32X_MUL GGML_F32x8_MUL\n #define GGML_F32X_FMA GGML_F32x8_FMA\n #define GLA_VECTOR_SIZE 8\n #elif defined(__AVX512F__)\n #define GGML_F32X GGML_F32x16\n #define GGML_F32X_SET1 GGML_F32x16_SET1\n #define GGML_F32X_LOAD GGML_F32x16_LOAD\n #define GGML_F32X_STORE GGML_F32x16_STORE\n #define GGML_F32X_MUL GGML_F32x16_MUL\n #define GGML_F32X_FMA GGML_F32x16_FMA\n #define GLA_VECTOR_SIZE 16\n #elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n #define GGML_F32X GGML_F32xt\n #define GGML_F32X_SET1 GGML_F32xt_SET1\n #define GGML_F32X_LOAD GGML_F32xt_LOAD\n #define GGML_F32X_STORE GGML_F32xt_STORE\n #define GGML_F32X_MUL GGML_F32xt_MUL\n #define GGML_F32X_FMA GGML_F32xt_FMA\n #define GLA_VECTOR_SIZE 8\n #elif defined(__ARM_NEON) && defined(__aarch64__)\n #define GGML_F32X GGML_F32x4\n #define GGML_F32X_SET1 GGML_F32x4_SET1\n #define GGML_F32X_LOAD GGML_F32x4_LOAD\n #define GGML_F32X_STORE GGML_F32x4_STORE\n #define GGML_F32X_MUL GGML_F32x4_MUL\n #define GGML_F32X_FMA GGML_F32x4_FMA\n #define GLA_VECTOR_SIZE 4\n #endif\n\n #ifdef GLA_VECTOR_SIZE\n int gla_vector_size;\n #if defined(__ARM_FEATURE_SVE)\n gla_vector_size = svcntw();\n #else\n gla_vector_size = GLA_VECTOR_SIZE;\n #endif\n const int64_t vec_count = head_size / gla_vector_size;\n\n for (int64_t t = 0; t < T; t++) {\n size_t t_offset = t * t_stride;\n size_t state_offset = head_size * C * (t / (T / n_seqs));\n float * state_cur = state + state_offset;\n float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[4]->data + state_offset;\n\n for (int64_t h = h_start; h < h_end; h++) {\n size_t h_offset = h * h_stride;\n size_t t_h_offset = t_offset + h_offset;\n size_t h_2d_offset = h * h_stride_2d;\n\n for (int64_t i = 0; i < head_size; i++) {\n size_t t_h_i_offset = t_h_offset + i;\n size_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n float k_val = k[t_h_i_offset];\n float q_val = q[t_h_i_offset] * scale;\n float g_val = g[t_h_i_offset];\n\n // Broadcast scalar values to vectors\n GGML_F32X k_vec = GGML_F32X_SET1(k_val);\n GGML_F32X q_vec = GGML_F32X_SET1(q_val);\n GGML_F32X g_vec = GGML_F32X_SET1(g_val);\n\n for (int64_t j = 0; j < vec_count; j++) {\n size_t base_j = j * gla_vector_size;\n size_t t_h_j_offset = t_h_offset + base_j;\n size_t h_2d_i_j_offset = h_2d_i_offset + base_j;\n\n // Load x elements at once\n GGML_F32X v_vec = GGML_F32X_LOAD(&v[t_h_j_offset]);\n GGML_F32X prev_state_vec = GGML_F32X_LOAD(&state_prev[h_2d_i_j_offset]);\n GGML_F32X dst_vec = GGML_F32X_LOAD(&dst_data[t_h_j_offset]);\n\n // Compute kv = v * k\n GGML_F32X kv_vec = GGML_F32X_MUL(v_vec, k_vec);\n\n // Compute temp = prev_state * g + kv\n GGML_F32X temp_vec = GGML_F32X_FMA(kv_vec, prev_state_vec, g_vec);\n\n // Update dst: dst += temp * q\n dst_vec = GGML_F32X_FMA(dst_vec, temp_vec, q_vec);\n GGML_F32X_STORE(&dst_data[t_h_j_offset], dst_vec);\n\n // Update state\n GGML_F32X_STORE(&state_cur[h_2d_i_j_offset], temp_vec);\n }\n\n // Handle remaining elements, this will not be used.\n for (int64_t j = vec_count * gla_vector_size; j < head_size; j++) {\n size_t t_h_j_offset = t_h_offset + j;\n size_t h_2d_i_j_offset = h_2d_i_offset + j;\n float v_val = v[t_h_j_offset];\n float kv_val = v_val * k_val;\n float prev_state_val = state_prev[h_2d_i_j_offset];\n float temp_val = kv_val + prev_state_val * g_val;\n dst_data[t_h_j_offset] += temp_val * q_val;\n state_cur[h_2d_i_j_offset] = temp_val;\n }\n }\n }\n }\n\n #else\n for (int64_t t = 0; t < T; t++) {\n size_t t_offset = t * t_stride;\n size_t state_offset = head_size * C * (t / (T / n_seqs));\n float * state_cur = state + state_offset;\n float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[4]->data + state_offset;\n\n for (int64_t h = h_start; h < h_end; h++) {\n size_t h_offset = h * h_stride;\n size_t t_h_offset = t_offset + h_offset;\n size_t h_2d_offset = h * h_stride_2d;\n\n for (int64_t i = 0; i < head_size; i++) {\n size_t t_h_i_offset = t_h_offset + i;\n size_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n float k_val = k[t_h_i_offset];\n float q_val = q[t_h_i_offset] * scale;\n float g_val = g[t_h_i_offset];\n\n for (int64_t j = 0; j < head_size; j++) {\n size_t t_h_j_offset = t_h_offset + j;\n size_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n float v_val = v[t_h_j_offset];\n float kv_val = v_val * k_val;\n float prev_state_val = state_prev[h_2d_i_j_offset];\n float temp_val = prev_state_val * g_val + kv_val;\n dst_data[t_h_j_offset] += temp_val * q_val;\n state_cur[h_2d_i_j_offset] = temp_val;\n }\n }\n }\n }\n #endif\n}\n\n\nvoid ggml_compute_forward_gla(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_gla_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nstatic void ggml_compute_forward_solve_tri_f32(const struct ggml_compute_params * params, struct ggml_tensor * dst) {\n const struct ggml_tensor * src0 = dst->src[0]; // A (lower triangular)\n const struct ggml_tensor * src1 = dst->src[1]; // B (RHS)\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ne00 == ne01); // A must be square\n GGML_ASSERT(ne0 == ne10); // solution cols == B cols\n GGML_ASSERT(ne1 == ne11); // solution rows == B rows\n\n GGML_ASSERT(ne02 == ne12 && ne12 == ne2);\n GGML_ASSERT(ne03 == ne13 && ne13 == ne3);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int64_t k = ne10; // number of RHS columns\n const int64_t n = ne11; // A is n×n\n const int64_t nr = ne02 * ne03 * k; // we're parallelizing on columns here, so seq x token x column will be the unit\n\n // chunks per thread\n const int64_t dr = (nr + nth - 1)/nth;\n\n // chunk range for this thread\n const int64_t ir0 = dr*ith;\n const int64_t ir1 = MIN(ir0 + dr, nr);\n\n const float * A = (const float *) src0->data; // [n, n, B1, B2]\n const float * B = (const float *) src1->data; // [n, k, B1, B2]\n float * X = ( float *) dst->data; // [n, k, B1, B2]\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir/(ne02*k);\n const int64_t i02 = (ir - i03*ne02*k)/k;\n const int64_t i01 = (ir - i03*ne02*k - i02*k);\n\n const float * A_batch = A + i02 * nb02 / sizeof(float) + i03 * nb03 / sizeof(float);\n const float * B_batch = B + i02 * nb12 / sizeof(float) + i03 * nb13 / sizeof(float);\n\n float * X_batch = X + i02 * nb2 / sizeof(float) + i03 * nb3 / sizeof(float);\n\n for (int64_t i00 = 0; i00 < n; ++i00) {\n float sum = 0.0f;\n for (int64_t t = 0; t < i00; ++t) {\n sum += A_batch[i00 * n + t] * X_batch[t * k + i01];\n }\n\n const float diag = A_batch[i00 * n + i00];\n assert(diag != 0.0f && \"Zero diagonal in triangular matrix\");\n\n X_batch[i00 * k + i01] = (B_batch[i00 * k + i01] - sum) / diag;\n }\n }\n}\n\nvoid ggml_compute_forward_solve_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {\n ggml_compute_forward_solve_tri_f32(params, dst);\n } else {\n GGML_ABORT(\"fatal error\");\n }\n}\n\n// ggml_compute_forward_gated_delta_net\nstatic void ggml_compute_forward_gated_delta_net_one_chunk(\n const ggml_compute_params * params,\n ggml_tensor * dst,\n int64_t ir0,\n int64_t ir1) {\n\n ggml_tensor * src_q = dst->src[0];\n ggml_tensor * src_k = dst->src[1];\n ggml_tensor * src_v = dst->src[2];\n ggml_tensor * src_g = dst->src[3];\n ggml_tensor * src_beta = dst->src[4];\n ggml_tensor * src_state = dst->src[5];\n\n const int64_t S_v = src_v->ne[0];\n const int64_t H = src_v->ne[1];\n const int64_t n_tokens = src_v->ne[2];\n const int64_t n_seqs = src_v->ne[3];\n\n GGML_ASSERT(ggml_is_contiguous_rows(src_q));\n GGML_ASSERT(ggml_is_contiguous_rows(src_k));\n GGML_ASSERT(ggml_is_contiguous_rows(src_v));\n GGML_ASSERT(ggml_is_contiguous(src_g));\n GGML_ASSERT(ggml_is_contiguous(src_beta));\n GGML_ASSERT(ggml_is_contiguous(src_state));\n\n GGML_ASSERT(src_g->ne[0] == 1 || src_g->ne[0] == S_v);\n GGML_ASSERT(src_beta->ne[0] == 1);\n\n GGML_TENSOR_LOCALS(int64_t, neq, src_q, ne);\n GGML_TENSOR_LOCALS(size_t, nbq, src_q, nb);\n GGML_TENSOR_LOCALS(int64_t, nek, src_k, ne);\n GGML_TENSOR_LOCALS(size_t, nbk, src_k, nb);\n GGML_TENSOR_LOCALS(int64_t, nev, src_v, ne);\n GGML_TENSOR_LOCALS(size_t, nbv, src_v, nb);\n GGML_TENSOR_LOCALS(int64_t, neg, src_g, ne);\n GGML_TENSOR_LOCALS(size_t, nbg, src_g, nb);\n GGML_TENSOR_LOCALS(size_t, nbb, src_beta, nb);\n\n const bool kda = (neg0 == S_v);\n\n // scratch layout per thread: [delta(S_v)]\n const int64_t scratch_per_thread = S_v;\n const int ith = params->ith;\n\n float * delta = (float *)params->wdata + ith * scratch_per_thread + CACHE_LINE_SIZE_F32;\n\n // output layout: [attn_scores | new_states]\n // attn_scores: S_v * H * n_tokens * n_seqs floats\n // new_states: S_v * S_v * H * n_seqs floats\n const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs;\n float * attn_out_base = (float *)dst->data;\n float * state_out_base = (float *)dst->data + attn_score_elems;\n\n const float * state_in_base = (const float *)src_state->data;\n\n //const int64_t rq1 = nev1 / neq1;\n //const int64_t rk1 = nev1 / nek1;\n const int64_t rq3 = nev3 / neq3;\n const int64_t rk3 = nev3 / nek3;\n\n const float scale = 1.0f / sqrtf((float) S_v);\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n const int64_t iv1 = ir % H; // head_index\n const int64_t iv3 = ir / H; // sequence\n\n const int64_t iq1 = iv1 % neq1;\n const int64_t ik1 = iv1 % nek1;\n\n const int64_t iq3 = iv3 / rq3;\n const int64_t ik3 = iv3 / rk3;\n\n float * s_out = state_out_base + (iv3 * H + iv1) * S_v * S_v;\n\n // copy input state into output buffer and operate in-place\n const float * s_in = state_in_base + (iv3 * H + iv1) * S_v * S_v;\n memcpy(s_out, s_in, S_v * S_v * sizeof(float));\n\n // attn output pointer for first token of this (head, seq)\n float * attn_data = attn_out_base + (iv3 * n_tokens * H + iv1) * S_v;\n\n for (int64_t t = 0; t < n_tokens; t++) {\n const float * q_d = (const float *)((const char *)src_q->data + iq3 * nbq3 + t * nbq2 + iq1 * nbq1);\n const float * k_d = (const float *)((const char *)src_k->data + ik3 * nbk3 + t * nbk2 + ik1 * nbk1);\n const float * v_d = (const float *)((const char *)src_v->data + iv3 * nbv3 + t * nbv2 + iv1 * nbv1);\n\n const float beta_val = *(const float *)((const char *)src_beta->data + iv3 * nbb3 + t * nbb2 + iv1 * nbb1);\n const float * g_d = (const float *)((const char *)src_g->data + iv3 * nbg3 + t * nbg2 + iv1 * nbg1);\n\n // state is stored transposed: s_out[j*S_v + i] = S[i][j]\n // so row j of s_out = column j of S (contiguous access)\n\n if (kda) {\n // precompute exp(g) into delta scratch (reused below)\n for (int64_t i = 0; i < S_v; ++i) {\n delta[i] = expf(g_d[i]);\n }\n // S[i][:] *= exp(g[i]) => for each row j of M: M[j][i] *= exp(g[i])\n for (int64_t j = 0; j < S_v; ++j) {\n ggml_vec_mul_f32(S_v, &s_out[j * S_v], &s_out[j * S_v], delta);\n }\n } else {\n ggml_vec_scale_f32(S_v * S_v, s_out, expf(g_d[0]));\n }\n\n // delta[j] = sum_i S[i][j] * k[i] = dot(row j of M, k)\n for (int64_t j = 0; j < S_v; ++j) {\n float sum = 0.0f;\n ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, k_d, 0, 1);\n delta[j] = (v_d[j] - sum) * beta_val;\n }\n\n // outer product: S[i][j] += k[i] * delta[j] => M[j][i] += delta[j] * k[i]\n for (int64_t j = 0; j < S_v; ++j) {\n ggml_vec_mad_f32(S_v, &s_out[j * S_v], k_d, delta[j]);\n }\n\n // attn_out[j] = sum_i S[i][j] * q[i] = dot(row j of M, q)\n for (int64_t j = 0; j < S_v; ++j) {\n float sum = 0.0f;\n ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, q_d, 0, 1);\n attn_data[j] = sum * scale;\n }\n\n attn_data += S_v * H; // advance to next token\n }\n }\n}\n\n\nstatic void ggml_compute_forward_gated_delta_net_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n ggml_tensor * V = dst->src[2];\n int64_t nr = V->ne[1] * V->ne[3];\n\n // disable for NUMA\n const bool disable_chunking = ggml_is_numa();\n\n int nth = params->nth;\n int ith = params->ith;\n\n // 4x chunks per thread\n int nth_scaled = nth * 4;\n int64_t chunk_size = (nr + nth_scaled - 1) / nth_scaled;\n int64_t nchunk = (nr + chunk_size - 1) / chunk_size;\n\n if (nth == 1 || nchunk < nth || disable_chunking) {\n nchunk = nth;\n }\n\n if (ith == 0) {\n ggml_threadpool_chunk_set(params->threadpool, nth);\n }\n\n ggml_barrier(params->threadpool);\n\n const int64_t dr = (nr + nchunk - 1) / nchunk;\n\n int current_chunk = ith;\n\n while (current_chunk < nchunk) {\n const int64_t ir0 = dr * current_chunk;\n const int64_t ir1 = MIN(ir0 + dr, nr);\n\n ggml_compute_forward_gated_delta_net_one_chunk(params, dst, ir0, ir1);\n current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);\n }\n}\n\nvoid ggml_compute_forward_gated_delta_net(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_gated_delta_net_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_rwkv_wkv7\n\nstatic void ggml_compute_forward_rwkv_wkv7_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n const int64_t T = dst->src[1]->ne[2];\n const int64_t C = dst->ne[0];\n const int64_t HEADS = dst->src[1]->ne[1];\n const int64_t n_seqs = dst->src[6]->ne[1];\n const int64_t head_size = C / HEADS;\n\n float * dst_data = (float *) dst->data;\n float * state = ((float *) dst->data) + C * T;\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n if (ith >= HEADS) {\n return;\n }\n\n const int h_start = (HEADS * ith) / nth;\n const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ?\n (HEADS * (ith + 1)) / nth : HEADS;\n\n float * r = (float *) dst->src[0]->data;\n float * w = (float *) dst->src[1]->data;\n float * k = (float *) dst->src[2]->data;\n float * v = (float *) dst->src[3]->data;\n float * a = (float *) dst->src[4]->data;\n float * b = (float *) dst->src[5]->data;\n\n int64_t t_stride = HEADS * head_size; // Same to C\n\n int64_t h_stride = C / HEADS;\n GGML_ASSERT(C % HEADS == 0); // C must be divisible by HEADS\n int64_t h_stride_2d = head_size * head_size;\n\n #if defined(GGML_SIMD)\n #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v_intrinsic)\n // scalar Route to scalar implementation //TODO: Write SVE code and RVV code\n for (int64_t t = 0; t < T; t++) {\n int64_t t_offset = t * t_stride;\n int64_t state_offset = head_size * C * (t / (T / n_seqs));\n float * state_cur = state + state_offset;\n float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset;\n\n for (int64_t h = h_start; h < h_end; h++) {\n int64_t h_offset = h * h_stride;\n int64_t t_h_offset = t_offset + h_offset;\n int64_t h_2d_offset = h * h_stride_2d;\n\n for (int64_t i = 0; i < head_size; i++) {\n int64_t t_h_i_offset = t_h_offset + i;\n int64_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n float v_val = v[t_h_i_offset];\n\n float sa = 0, result = 0;\n for (int64_t j = 0; j < head_size; j++) {\n sa += a[t_h_offset + j] * state_prev[h_2d_i_offset + j];\n }\n\n for (int64_t j = 0; j < head_size; j++) {\n int64_t t_h_j_offset = t_h_offset + j;\n int64_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n float r_val = r[t_h_j_offset];\n float w_val = w[t_h_j_offset];\n float k_val = k[t_h_j_offset];\n float b_val = b[t_h_j_offset];\n float kv_val = v_val * k_val;\n float prev_state_val = state_prev[h_2d_i_j_offset];\n state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val;\n result += state_cur[h_2d_i_j_offset] * r_val;\n }\n dst_data[t_h_i_offset] = result;\n }\n }\n }\n #else\n for (int64_t t = 0; t < T; t++) {\n int64_t t_offset = t * t_stride;\n int64_t state_offset = head_size * C * (t / (T / n_seqs));\n float * state_cur = state + state_offset;\n float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset;\n\n for (int64_t h = h_start; h < h_end; h++) {\n int64_t h_offset = h * h_stride;\n int64_t t_h_offset = t_offset + h_offset;\n int64_t h_2d_offset = h * h_stride_2d;\n\n for (int64_t ii = 0; ii < head_size; ii++) {\n int64_t t_h_i_offset = t_h_offset + ii;\n int64_t h_2d_i_offset = h_2d_offset + ii * h_stride;\n\n GGML_F32_VEC v_vec = GGML_F32_VEC_SET1(v[t_h_i_offset]);\n\n float sa = 0;\n {\n GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };\n GGML_F32_VEC ax[GGML_F32_ARR];\n GGML_F32_VEC ay[GGML_F32_ARR];\n for (int64_t j = 0; j < head_size; j += GGML_F32_STEP) {\n for (int64_t kk = 0; kk < GGML_F32_ARR; kk++) {\n ax[kk] = GGML_F32_VEC_LOAD(&a[t_h_offset + j + kk * GGML_F32_EPR]);\n ay[kk] = GGML_F32_VEC_LOAD(&state_prev[h_2d_i_offset + j + kk * GGML_F32_EPR]);\n sum[kk] = GGML_F32_VEC_FMA(sum[kk], ax[kk], ay[kk]);\n }\n }\n GGML_F32_VEC_REDUCE(sa, sum);\n }\n\n GGML_F32_VEC sa_vec = GGML_F32_VEC_SET1(sa);\n\n int64_t j = 0;\n GGML_F32_VEC result_vec[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };\n for (; j < head_size; j += GGML_F32_STEP) {\n for (int64_t kk = 0; kk < GGML_F32_ARR; kk++) {\n int64_t t_h_j_offset = t_h_offset + j + kk * GGML_F32_EPR;\n int64_t h_2d_i_j_offset = h_2d_i_offset + j + kk * GGML_F32_EPR;\n\n GGML_F32_VEC r_vec = GGML_F32_VEC_LOAD(&r[t_h_j_offset]);\n GGML_F32_VEC w_vec = GGML_F32_VEC_LOAD(&w[t_h_j_offset]);\n GGML_F32_VEC k_vec = GGML_F32_VEC_LOAD(&k[t_h_j_offset]);\n GGML_F32_VEC b_vec = GGML_F32_VEC_LOAD(&b[t_h_j_offset]);\n\n k_vec = GGML_F32_VEC_MUL(v_vec, k_vec);\n\n GGML_F32_VEC state_vec = GGML_F32_VEC_LOAD(&state_prev[h_2d_i_j_offset]);\n // kv + s * decay + sa * b\n state_vec = GGML_F32_VEC_FMA(k_vec, state_vec, w_vec);\n state_vec = GGML_F32_VEC_FMA(state_vec, sa_vec, b_vec);\n GGML_F32_VEC_STORE(&state_cur[h_2d_i_j_offset], state_vec);\n\n result_vec[kk] = GGML_F32_VEC_FMA(result_vec[kk], state_vec, r_vec);\n }\n }\n GGML_F32_VEC_REDUCE(dst_data[t_h_i_offset], result_vec);\n\n // There shouldn't be left-overs though.\n for (; j < head_size; j++) {\n int64_t t_h_j_offset = t_h_offset + j;\n int64_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n float r_val = r[t_h_j_offset];\n float w_val = w[t_h_j_offset];\n float k_val = k[t_h_j_offset];\n float b_val = b[t_h_j_offset];\n float kv_val = v[t_h_i_offset] * k_val;\n\n float prev_state_val = state_prev[h_2d_i_j_offset];\n state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val;\n dst_data[t_h_i_offset] += state_cur[h_2d_i_j_offset] * r_val;\n }\n }\n }\n }\n #endif\n #else\n for (int64_t t = 0; t < T; t++) {\n int64_t t_offset = t * t_stride;\n int64_t state_offset = head_size * C * (t / (T / n_seqs));\n float * state_cur = state + state_offset;\n float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset;\n\n for (int64_t h = h_start; h < h_end; h++) {\n int64_t h_offset = h * h_stride;\n int64_t t_h_offset = t_offset + h_offset;\n int64_t h_2d_offset = h * h_stride_2d;\n\n for (int64_t i = 0; i < head_size; i++) {\n int64_t t_h_i_offset = t_h_offset + i;\n int64_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n float v_val = v[t_h_i_offset];\n\n float sa = 0, result = 0;\n for (int64_t j = 0; j < head_size; j++) {\n sa += a[t_h_offset + j] * state_prev[h_2d_i_offset + j];\n }\n\n for (int64_t j = 0; j < head_size; j++) {\n int64_t t_h_j_offset = t_h_offset + j;\n int64_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n float r_val = r[t_h_j_offset];\n float w_val = w[t_h_j_offset];\n float k_val = k[t_h_j_offset];\n float b_val = b[t_h_j_offset];\n float kv_val = v_val * k_val;\n float prev_state_val = state_prev[h_2d_i_j_offset];\n state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val;\n result += state_cur[h_2d_i_j_offset] * r_val;\n }\n dst_data[t_h_i_offset] = result;\n }\n }\n }\n #endif\n}\n\n\nvoid ggml_compute_forward_rwkv_wkv7(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_rwkv_wkv7_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_map_custom1\n\nvoid ggml_compute_forward_map_custom1(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * a = dst->src[0];\n\n struct ggml_map_custom1_op_params p;\n memcpy(&p, dst->op_params, sizeof(p));\n\n p.fun(dst, a, params->ith, params->nth, p.userdata);\n}\n\n// ggml_compute_forward_map_custom2\n\nvoid ggml_compute_forward_map_custom2(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * a = dst->src[0];\n const ggml_tensor * b = dst->src[1];\n\n struct ggml_map_custom2_op_params p;\n memcpy(&p, dst->op_params, sizeof(p));\n\n p.fun(dst, a, b, params->ith, params->nth, p.userdata);\n}\n\n// ggml_compute_forward_map_custom3\n\nvoid ggml_compute_forward_map_custom3(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * a = dst->src[0];\n const ggml_tensor * b = dst->src[1];\n const ggml_tensor * c = dst->src[2];\n\n struct ggml_map_custom3_op_params p;\n memcpy(&p, dst->op_params, sizeof(p));\n\n p.fun(dst, a, b, c, params->ith, params->nth, p.userdata);\n}\n\n// ggml_compute_forward_custom\n\nvoid ggml_compute_forward_custom(\n const struct ggml_compute_params * params,\n struct ggml_tensor * dst) {\n\n struct ggml_custom_op_params p;\n memcpy(&p, dst->op_params, sizeof(p));\n\n p.fun(dst, params->ith, params->nth, p.userdata);\n}\n\n// ggml_compute_forward_cross_entropy_loss\n\nstatic void ggml_compute_forward_cross_entropy_loss_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));\n GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));\n GGML_ASSERT(ggml_are_same_shape(src0, src1));\n GGML_ASSERT(ggml_is_scalar(dst));\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n // TODO: handle transposed/permuted matrices\n const int64_t nc = src0->ne[0];\n const int64_t nr = ggml_nrows(src0);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n float * sums = (float *) params->wdata;\n float * st = ((float *) params->wdata) + nth + ith*nc;\n float sum_thread = 0.0f;\n\n GGML_ASSERT(params->wsize >= sizeof(float) * (nth + nth * nc));\n\n // rows per thread\n const int64_t dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int64_t ir0 = dr*ith;\n const int64_t ir1 = MIN(ir0 + dr, nr);\n\n for (int64_t i1 = ir0; i1 < ir1; ++i1) {\n const float * s0 = (const float *)((const char *) src0->data + i1*src0->nb[1]);\n const float * s1 = (const float *)((const char *) src1->data + i1*src1->nb[1]);\n\n#ifndef NDEBUG\n for (int64_t i = 0; i < nc; ++i) {\n //printf(\"p[%d] = %f\\n\", i, p[i]);\n assert(!isnan(s0[i]));\n assert(!isnan(s1[i]));\n }\n#endif // NDEBUG\n\n float max = -INFINITY;\n ggml_vec_max_f32(nc, &max, s0);\n const ggml_float sum_softmax = ggml_vec_log_soft_max_f32(nc, st, s0, max);\n assert(sum_softmax >= 0.0);\n\n ggml_vec_add1_f32(nc, st, st, -sum_softmax);\n ggml_vec_mul_f32(nc, st, st, s1);\n\n float sum_st = 0.0f;\n ggml_vec_sum_f32(nc, &sum_st, st);\n sum_thread += sum_st;\n\n#ifndef NDEBUG\n for (int64_t i = 0; i < nc; ++i) {\n assert(!isnan(st[i]));\n assert(!isinf(st[i]));\n }\n#endif // NDEBUG\n }\n sums[ith] = sum_thread;\n ggml_barrier(params->threadpool);\n\n if (ith == 0) {\n float * dp = (float *) dst->data;\n ggml_vec_sum_f32(nth, dp, sums);\n dp[0] *= -1.0f / (float) nr;\n }\n}\n\nvoid ggml_compute_forward_cross_entropy_loss(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_cross_entropy_loss_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\n// ggml_compute_forward_cross_entropy_loss_back\n\nstatic void ggml_compute_forward_cross_entropy_loss_back_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * grad = dst->src[0]; // gradient of forward pass output\n const ggml_tensor * src0f = dst->src[1]; // src0 of forward pass\n const ggml_tensor * src1f = dst->src[2]; // src1 of forward pass\n\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_is_contiguous(src0f));\n GGML_ASSERT(ggml_is_contiguous(src1f));\n GGML_ASSERT(ggml_is_contiguous(grad));\n GGML_ASSERT(ggml_are_same_shape(src0f, src1f) && ggml_are_same_shape(src0f, dst));\n\n const int64_t ith = params->ith;\n const int64_t nth = params->nth;\n\n // TODO: handle transposed/permuted matrices\n const int64_t nc = src0f->ne[0];\n const int64_t nr = ggml_nrows(src0f);\n\n // rows per thread\n const int64_t dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int64_t ir0 = dr*ith;\n const int64_t ir1 = MIN(ir0 + dr, nr);\n\n const float d_by_nr = ((const float *) grad->data)[0] / (float) nr;\n\n for (int64_t i1 = ir0; i1 < ir1; i1++) {\n float * ds0 = (float *)((char *) dst->data + i1*dst->nb[1]);\n const float * s0 = (const float *)((const char *) src0f->data + i1*src0f->nb[1]);\n const float * s1 = (const float *)((const char *) src1f->data + i1*src1f->nb[1]);\n\n#ifndef NDEBUG\n for (int64_t i = 0; i < nc; ++i) {\n //printf(\"p[%d] = %f\\n\", i, p[i]);\n assert(!isnan(s0[i]));\n assert(!isnan(s1[i]));\n }\n#endif // NDEBUG\n\n // soft_max\n float max = -INFINITY;\n ggml_vec_max_f32(nc, &max, s0);\n const ggml_float sum = ggml_vec_soft_max_f32(nc, ds0, s0, max);\n assert(sum > 0.0);\n ggml_vec_scale_f32(nc, ds0, 1.0/sum);\n\n // grad(src0f) = (softmax(src0f) - src1f) * grad(cross_entropy_loss(src0f, src1f)) / nr\n ggml_vec_sub_f32(nc, ds0, ds0, s1);\n ggml_vec_scale_f32(nc, ds0, d_by_nr);\n\n#ifndef NDEBUG\n for (int64_t i = 0; i < nc; ++i) {\n assert(!isnan(ds0[i]));\n assert(!isinf(ds0[i]));\n }\n#endif // NDEBUG\n }\n}\n\nvoid ggml_compute_forward_cross_entropy_loss_back(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_cross_entropy_loss_back_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nstatic void ggml_compute_forward_opt_step_adamw_f32(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src0_grad = dst->src[1];\n const ggml_tensor * src0_grad_m = dst->src[2];\n const ggml_tensor * src0_grad_v = dst->src[3];\n const ggml_tensor * adamw_params = dst->src[4];\n\n GGML_ASSERT(ggml_are_same_shape(src0, src0_grad));\n GGML_ASSERT(ggml_are_same_shape(src0, src0_grad_m));\n GGML_ASSERT(ggml_are_same_shape(src0, src0_grad_v));\n GGML_ASSERT(ggml_nelements(adamw_params) == 7);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_UNARY_OP_LOCALS\n GGML_ASSERT(nb00 == sizeof(float));\n\n // rows per thread\n const int dr = (nr + nth - 1)/nth;\n\n // row range for this thread\n const int ir0 = dr*ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n const float * adamw_params_ptr = ggml_get_data_f32(adamw_params);\n\n const float alpha = adamw_params_ptr[0];\n const float beta1 = adamw_params_ptr[1];\n const float beta2 = adamw_params_ptr[2];\n const float eps = adamw_params_ptr[3];\n const float wd = adamw_params_ptr[4];\n const float beta1h = adamw_params_ptr[5];\n const float beta2h = adamw_params_ptr[6];\n const float keep = 1.f - alpha * wd;\n for (int ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir/(ne02*ne01);\n const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n const size_t offset = i03*nb03 + i02*nb02 + i01*nb01;\n\n float * w = (float *) ((char *) src0->data + offset); // weight\n const float * g = (const float *) ((const char *) src0_grad->data + offset); // grad\n float * m = (float *) ((char *) src0_grad_m->data + offset);\n float * v = (float *) ((char *) src0_grad_v->data + offset);\n\n for (int i00 = 0; i00 < ne00; ++i00) {\n m[i00] = m[i00]*beta1 + g[i00]*(1.0f - beta1);\n v[i00] = v[i00]*beta2 + g[i00]*g[i00]*(1.0f - beta2);\n\n const float mh = m[i00]*beta1h;\n const float vh = sqrtf(v[i00]*beta2h) + eps;\n\n // The weight decay is applied independently of the Adam momenta m and v.\n // This is NOT equivalent to l2 regularization that adds w[i00]*w[i00] to the loss.\n // See: https://arxiv.org/pdf/1711.05101v3.pdf\n w[i00] = w[i00] * keep - alpha * mh / vh;\n }\n }\n}\n\nvoid ggml_compute_forward_opt_step_adamw(\n const ggml_compute_params * params,\n ggml_tensor * dst) {\n\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_opt_step_adamw_f32(params, dst);\n } break;\n default:\n {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nstatic void ggml_compute_forward_opt_step_sgd_f32(const ggml_compute_params * params, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src0_grad = dst->src[1];\n const ggml_tensor * sgd_params = dst->src[2];\n\n GGML_ASSERT(ggml_are_same_shape(src0, src0_grad));\n GGML_ASSERT(ggml_nelements(sgd_params) == 2);\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const int nr = ggml_nrows(src0);\n\n GGML_TENSOR_UNARY_OP_LOCALS\n GGML_ASSERT(nb00 == sizeof(float));\n\n // rows per thread\n const int dr = (nr + nth - 1) / nth;\n\n // row range for this thread\n const int ir0 = dr * ith;\n const int ir1 = MIN(ir0 + dr, nr);\n\n // using adamw param subset we care about - alpha, wd - could have a separate struct\n const float * sgd_params_ptr = ggml_get_data_f32(sgd_params);\n const float alpha = sgd_params_ptr[0];\n const float keep = 1.f - alpha * sgd_params_ptr[1];\n\n for (int ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir / (ne02 * ne01);\n const int64_t i02 = (ir - i03 * ne02 * ne01) / ne01;\n const int64_t i01 = (ir - i03 * ne02 * ne01 - i02 * ne01);\n\n const size_t offset = i03 * nb03 + i02 * nb02 + i01 * nb01;\n\n float * w = (float *) ((char *) src0->data + offset); // weight\n const float * g = (const float *) ((const char *) src0_grad->data + offset); // grad\n\n for (int i00 = 0; i00 < ne00; ++i00) {\n w[i00] = w[i00] * keep - alpha * g[i00];\n }\n }\n}\n\nvoid ggml_compute_forward_opt_step_sgd(const ggml_compute_params * params, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n {\n ggml_compute_forward_opt_step_sgd_f32(params, dst);\n }\n break;\n default:\n {\n GGML_ABORT(\"fatal error - sgd is F32 only\");\n }\n }\n}\n" }, { "path": "src/ggml-cpu/ops.h", "content": "#pragma once\n\n#include \"ggml.h\"\n\n//\n// cache line\n//\n\n#if defined(__cpp_lib_hardware_interference_size)\n#define CACHE_LINE_SIZE std::hardware_destructive_interference_size\n#else\n#if defined(__POWER9_VECTOR__)\n#define CACHE_LINE_SIZE 128\n#elif defined(__VXE__) || defined(__VXE2__)\n#define CACHE_LINE_SIZE 256\n#else\n#define CACHE_LINE_SIZE 64\n#endif\n#endif\n\nstatic const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);\n\n// Work buffer size for im2col operations in CONV2D\n#define GGML_IM2COL_WORK_SIZE (16 * 1024 * 1024)\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nvoid ggml_compute_forward_dup(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_add(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_add_id(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_add1(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_acc(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sum(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sum_rows(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cumsum(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_mean(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_argmax(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_count_equal(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_repeat(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_repeat_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_concat(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_silu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rms_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rms_norm_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_group_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_l2_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_out_prod(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_scale(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_set(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cpy(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cont(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_get_rows(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_get_rows_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_set_rows(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_diag(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_diag_mask_inf(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_diag_mask_zero(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_soft_max(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_soft_max_ext_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rope(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rope_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_clamp(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_transpose_1d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_im2col(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_im2col_back_f32(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_im2col_3d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_2d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_3d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_transpose_2d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_2d_dw(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pool_1d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pool_2d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pool_2d_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_upscale(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pad(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pad_reflect_1d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_roll(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_arange(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_timestep_embedding(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_argsort(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_top_k(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_fill(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_flash_attn_ext(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_flash_attn_back(\n const struct ggml_compute_params * params,\n const bool masked,\n struct ggml_tensor * dst);\nvoid ggml_compute_forward_ssm_conv(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_ssm_scan(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_win_part(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_win_unpart(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_unary(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_glu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_get_rel_pos(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_add_rel_pos(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rwkv_wkv6(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rwkv_wkv7(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_solve_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_gla(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_gated_delta_net(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_map_custom1(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_map_custom2(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_map_custom3(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_custom(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cross_entropy_loss(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cross_entropy_loss_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_opt_step_adamw(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_mul_mat(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_opt_step_sgd(const struct ggml_compute_params * params, struct ggml_tensor * dst);\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cpu/quants.c", "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"ggml-quants.h\"\n#include \"quants.h\"\n\n#include \"arch-fallback.h\"\n\n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\nvoid quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_q4_0_ref(x, y, k);\n}\n\nvoid quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_q4_1_ref(x, y, k);\n}\n\nvoid quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_q5_0_ref(x, y, k);\n}\n\nvoid quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_q5_1_ref(x, y, k);\n}\n\nvoid quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_q8_0_ref(x, y, k);\n}\n\nvoid quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_q8_1_ref(x, y, k);\n}\n\nvoid quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_mxfp4_ref(x, y, k);\n}\n\nvoid quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_nvfp4_ref(x, y, k);\n}\n\n//\n// 2-6 bit quantization in super-blocks\n//\n\n//========================- 2-bit (de)-quantization\n\nvoid quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n quantize_row_q2_K_ref(x, vy, k);\n}\n\n//========================= 3-bit (de)-quantization\n\nvoid quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n quantize_row_q3_K_ref(x, vy, k);\n}\n\n// ====================== 4-bit (de)-quantization\n\nvoid quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK_K == 0);\n block_q4_K * GGML_RESTRICT y = vy;\n quantize_row_q4_K_ref(x, y, k);\n}\n\n// ====================== 5-bit (de)-quantization\n\nvoid quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK_K == 0);\n block_q5_K * GGML_RESTRICT y = vy;\n quantize_row_q5_K_ref(x, y, k);\n}\n\n// ====================== 6-bit (de)-quantization\n\nvoid quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK_K == 0);\n block_q6_K * GGML_RESTRICT y = vy;\n quantize_row_q6_K_ref(x, y, k);\n}\n\n// ====================== Ternary (de)-quantization (BitNet b1.58 and TriLMs)\n\nvoid quantize_row_tq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK_K == 0);\n block_tq1_0 * GGML_RESTRICT y = vy;\n quantize_row_tq1_0_ref(x, y, k);\n}\n\nvoid quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(k % QK_K == 0);\n block_tq2_0 * GGML_RESTRICT y = vy;\n quantize_row_tq2_0_ref(x, y, k);\n}\n\n//===================================== Q8_K ==============================================\n\nvoid quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n quantize_row_q8_K_ref(x, y, k);\n}\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n for (; ib < nb; ++ib) {\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n const int v1 = (x[ib].qs[j] >> 4) - 8;\n\n sumi0 += (v0 * y[ib].qs[j]);\n sumi1 += (v1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n }\n\n *s = sumf;\n}\n\n// TODO: add WASM SIMD\nvoid ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n for (; ib < nb; ++ib) {\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const int v0 = (x[ib].qs[j] & 0x0F);\n const int v1 = (x[ib].qs[j] >> 4);\n\n sumi0 += (v0 * y[ib].qs[j]);\n sumi1 += (v1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_MXFP4 == 0);\n static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n const block_mxfp4 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_MXFP4;\n\n int ib = 0;\n float sumf = 0;\n\n for (; ib < nb; ++ib) {\n const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_E8M0_TO_FP32_HALF(x[ib].e);\n\n int sumi1 = 0;\n int sumi2 = 0;\n for (int j = 0; j < QK_MXFP4/2; ++j) {\n sumi1 += y[ib].qs[j + 0] * kvalues_mxfp4[x[ib].qs[j] & 0xf];\n sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >> 4];\n }\n sumf += d * (sumi1 + sumi2);\n }\n *s = sumf;\n}\n\n// NVFP4: super-block of 64 elements = 4 sub-blocks of 16 = 2 q8_0 blocks\nvoid ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_NVFP4 == 0);\n\n const block_nvfp4 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_NVFP4;\n\n float sumf = 0;\n\n for (int ib = 0; ib < nb; ++ib) {\n for (int s_idx = 0; s_idx < 4; ++s_idx) {\n const float d = ggml_ue4m3_to_fp32(x[ib].d[s_idx]);\n const int q8_block = s_idx / 2;\n const int q8_off = (s_idx % 2) * QK_NVFP4_SUB;\n const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8_block].d);\n\n int sumi_lo = 0, sumi_hi = 0;\n for (int j = 0; j < QK_NVFP4_SUB/2; ++j) {\n const uint8_t qv = x[ib].qs[s_idx*(QK_NVFP4_SUB/2) + j];\n sumi_lo += y[2*ib + q8_block].qs[q8_off + j + 0] * kvalues_mxfp4[qv & 0xf];\n sumi_hi += y[2*ib + q8_block].qs[q8_off + j + QK_NVFP4_SUB/2] * kvalues_mxfp4[qv >> 4];\n }\n\n sumf += dy * d * (sumi_lo + sumi_hi);\n }\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n for (; ib < nb; ++ib) {\n uint32_t qh;\n memcpy(&qh, x[ib].qh, sizeof(qh));\n\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;\n const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));\n\n const int32_t x0 = (int8_t)(((x[ib].qs[j] & 0x0F) | xh_0) - 16);\n const int32_t x1 = (int8_t)(((x[ib].qs[j] >> 4) | xh_1) - 16);\n\n sumi0 += (x0 * y[ib].qs[j]);\n sumi1 += (x1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d)) * sumi;\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_1;\n const int nb = n / qk;\n\n int ib = 0;\n float sumf = 0;\n\n assert(n % qk == 0);\n assert(qk == QK5_1);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_1 * GGML_RESTRICT x = vx;\n const block_q8_1 * GGML_RESTRICT y = vy;\n\n for (; ib < nb; ++ib) {\n uint32_t qh;\n memcpy(&qh, x[ib].qh, sizeof(qh));\n\n int sumi0 = 0;\n int sumi1 = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;\n const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;\n\n const int32_t x0 = (x[ib].qs[j] & 0xF) | xh_0;\n const int32_t x1 = (x[ib].qs[j] >> 4) | xh_1;\n\n sumi0 += (x0 * y[ib].qs[j]);\n sumi1 += (x1 * y[ib].qs[j + qk/2]);\n }\n\n int sumi = sumi0 + sumi1;\n sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n\n assert(n % qk == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q8_0 * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n int ib = 0;\n float sumf = 0;\n\n for (; ib < nb; ++ib) {\n int sumi = 0;\n\n for (int j = 0; j < qk; j++) {\n sumi += x[ib].qs[j]*y[ib].qs[j];\n }\n\n sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_tq1_0 * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243};\n\n float sumf = 0.0f;\n\n for (int i = 0; i < nb; ++i) {\n int sum = 0;\n\n for (size_t j = 0; j < sizeof(x->qs) - sizeof(x->qs) % 32; j += 32) {\n for (size_t l = 0; l < 5; ++l) {\n for (size_t m = 0; m < 32; ++m) {\n uint8_t q = x[i].qs[j + m] * pow3[l];\n uint16_t xi = ((uint16_t) q * 3) >> 8;\n sum += (xi - 1) * y[i].qs[j*5 + l*32 + m];\n }\n }\n }\n for (size_t j = sizeof(x->qs) - sizeof(x->qs) % 32; j < sizeof(x->qs); j += 16) {\n for (size_t l = 0; l < 5; ++l) {\n for (size_t m = 0; m < 16; ++m) {\n uint8_t q = x[i].qs[j + m] * pow3[l];\n uint16_t xi = ((uint16_t) q * 3) >> 8;\n sum += (xi - 1) * y[i].qs[j*5 + l*16 + m];\n }\n }\n }\n\n for (size_t l = 0; l < 4; ++l) {\n for (size_t j = 0; j < sizeof(x->qh); ++j) {\n uint8_t q = x[i].qh[j] * pow3[l];\n uint16_t xi = ((uint16_t) q * 3) >> 8;\n sum += (xi - 1) * y[i].qs[sizeof(x->qs)*5 + l*sizeof(x->qh) + j];\n }\n }\n\n sumf += (float) sum * (GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_tq2_0 * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n float sumf = 0.0f;\n\n for (int i = 0; i < nb; ++i) {\n int32_t sumi = 0;\n\n for (size_t j = 0; j < sizeof(x->qs); j += 32) {\n for (size_t l = 0; l < 4; ++l) {\n for (size_t k = 0; k < 32; ++k) {\n sumi += y[i].qs[j*4 + l*32 + k] * (((x[i].qs[j + k] >> (l*2)) & 3) - 1);\n }\n }\n }\n\n const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n sumf += (float) sumi * d;\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q2_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n float sumf = 0;\n\n for (int i = 0; i < nb; ++i) {\n\n const uint8_t * q2 = x[i].qs;\n const int8_t * q8 = y[i].qs;\n const uint8_t * sc = x[i].scales;\n\n int summs = 0;\n for (int j = 0; j < 16; ++j) {\n summs += y[i].bsums[j] * (sc[j] >> 4);\n }\n\n const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n int isum = 0;\n int is = 0;\n int d;\n for (int k = 0; k < QK_K/128; ++k) {\n int shift = 0;\n for (int j = 0; j < 4; ++j) {\n d = sc[is++] & 0xF;\n int isuml = 0;\n for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);\n isum += d * isuml;\n d = sc[is++] & 0xF;\n isuml = 0;\n for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);\n isum += d * isuml;\n shift += 2;\n q8 += 32;\n }\n q2 += 32;\n }\n sumf += dall * isum - dmin * summs;\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const uint32_t kmask1 = 0x03030303;\n const uint32_t kmask2 = 0x0f0f0f0f;\n\n const block_q3_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n // scalar version\n // This function is written like this so the compiler can manage to vectorize most of it\n // Using -Ofast, GCC and clang manage to produce code that is within a factor of 2 or so from the\n // manually vectorized version above. Every other version I tried would run at least 4 times slower.\n // The ideal situation would be if we could just write the code once, and the compiler would\n // automatically produce the best possible set of machine instructions, instead of us having to manually\n // write vectorized versions for AVX, ARM_NEON, etc.\n\n int8_t aux8[QK_K];\n int16_t aux16[8];\n float sums [8];\n int32_t aux32[8];\n memset(sums, 0, 8*sizeof(float));\n\n uint32_t auxs[4];\n const int8_t * scales = (const int8_t*)auxs;\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT hm = x[i].hmask;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n memset(aux32, 0, 8*sizeof(int32_t));\n int8_t * GGML_RESTRICT a = aux8;\n uint8_t m = 1;\n for (int j = 0; j < QK_K; j += 128) {\n for (int l = 0; l < 32; ++l) a[l] = q3[l] & 3;\n for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);\n a += 32; m <<= 1;\n for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 2) & 3;\n for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);\n a += 32; m <<= 1;\n for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 4) & 3;\n for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);\n a += 32; m <<= 1;\n for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 6) & 3;\n for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);\n a += 32; m <<= 1;\n q3 += 32;\n }\n a = aux8;\n\n memcpy(auxs, x[i].scales, 12);\n uint32_t tmp = auxs[2];\n auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);\n auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);\n auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);\n auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);\n for (int j = 0; j < QK_K/16; ++j) {\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];\n q8 += 8; a += 8;\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];\n q8 += 8; a += 8;\n }\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];\n }\n for (int l = 0; l < 8; ++l) sumf += sums[l];\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q4_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n const uint8_t * scales = (const uint8_t*)&utmp[0];\n const uint8_t * mins = (const uint8_t*)&utmp[2];\n\n int8_t aux8[QK_K];\n int16_t aux16[8];\n float sums [8];\n int32_t aux32[8];\n memset(sums, 0, 8*sizeof(float));\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n memset(aux32, 0, 8*sizeof(int32_t));\n int8_t * GGML_RESTRICT a = aux8;\n for (int j = 0; j < QK_K/64; ++j) {\n for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);\n a += 32;\n for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);\n a += 32; q4 += 32;\n }\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n int sumi = 0;\n for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];\n a = aux8;\n int is = 0;\n for (int j = 0; j < QK_K/32; ++j) {\n int32_t scale = scales[is++];\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n }\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];\n const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;\n sumf -= dmin * sumi;\n }\n for (int l = 0; l < 8; ++l) sumf += sums[l];\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q5_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n uint32_t utmp[4];\n\n const uint8_t * scales = (const uint8_t*)&utmp[0];\n const uint8_t * mins = (const uint8_t*)&utmp[2];\n\n int8_t aux8[QK_K];\n int16_t aux16[8];\n float sums [8];\n int32_t aux32[8];\n memset(sums, 0, 8*sizeof(float));\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n const uint8_t * GGML_RESTRICT hm = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n memset(aux32, 0, 8*sizeof(int32_t));\n int8_t * GGML_RESTRICT a = aux8;\n uint8_t m = 1;\n for (int j = 0; j < QK_K/64; ++j) {\n for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);\n for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);\n a += 32; m <<= 1;\n for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);\n for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);\n a += 32; m <<= 1;\n q4 += 32;\n }\n memcpy(utmp, x[i].scales, 12);\n utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n const uint32_t uaux = utmp[1] & kmask1;\n utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n utmp[2] = uaux;\n utmp[0] &= kmask1;\n\n int sumi = 0;\n for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];\n a = aux8;\n int is = 0;\n for (int j = 0; j < QK_K/32; ++j) {\n int32_t scale = scales[is++];\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n }\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];\n const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;\n sumf -= dmin * sumi;\n }\n for (int l = 0; l < 8; ++l) sumf += sums[l];\n *s = sumf;\n}\n\nvoid ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_q6_K * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n int8_t aux8[QK_K];\n int16_t aux16[8];\n float sums [8];\n int32_t aux32[8];\n memset(sums, 0, 8*sizeof(float));\n\n float sumf = 0;\n for (int i = 0; i < nb; ++i) {\n const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n memset(aux32, 0, 8*sizeof(int32_t));\n int8_t * GGML_RESTRICT a = aux8;\n for (int j = 0; j < QK_K; j += 128) {\n for (int l = 0; l < 32; ++l) {\n a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;\n a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;\n a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;\n a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;\n }\n a += 128;\n q4 += 64;\n qh += 32;\n }\n a = aux8;\n int is = 0;\n for (int j = 0; j < QK_K/16; ++j) {\n int scale = x[i].scales[is++];\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n q8 += 8; a += 8;\n }\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];\n }\n for (int l = 0; l < 8; ++l) sumf += sums[l];\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n uint32_t aux32[2];\n const uint8_t * aux8 = (const uint8_t *)aux32;\n\n float sumf = 0.f;\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n int32_t bsum = 0;\n for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n memcpy(aux32, q2, 2*sizeof(uint32_t));\n q2 += 4;\n const uint32_t ls = 2*(aux32[1] >> 28) + 1;\n int32_t sumi = 0;\n for (int l = 0; l < 4; ++l) {\n const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);\n const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];\n for (int j = 0; j < 8; ++j) {\n sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);\n }\n q8 += 8;\n }\n bsum += sumi * ls;\n }\n sumf += d * bsum;\n }\n *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n float sumf = 0.f;\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n const uint8_t * GGML_RESTRICT sc = x[i].scales;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n int32_t bsum = 0;\n for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n const uint16_t ls1 = 2*(sc[ib32] & 0xf) + 1;\n const uint16_t ls2 = 2*(sc[ib32] >> 4) + 1;\n int32_t sumi = 0;\n for (int l = 0; l < 2; ++l) {\n const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));\n const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];\n for (int j = 0; j < 8; ++j) {\n sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);\n }\n q8 += 8;\n }\n bsum += sumi * ls1;\n sumi = 0;\n for (int l = 2; l < 4; ++l) {\n const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));\n const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];\n for (int j = 0; j < 8; ++j) {\n sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);\n }\n q8 += 8;\n }\n bsum += sumi * ls2;\n q2 += 4;\n }\n sumf += d * bsum;\n }\n *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq2_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n float sumf = 0;\n for (int i = 0; i < nb; i++) {\n\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint8_t * qh = x[i].qh;\n const uint8_t * signs = qs + QK_K/8;\n\n int bsum = 0;\n for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n int ls1 = 1 + 2*(x[i].scales[ib32] & 0xf);\n int ls2 = 1 + 2*(x[i].scales[ib32] >> 4);\n int sumi1 = 0, sumi2 = 0;\n for (int l = 0; l < 2; ++l) {\n const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));\n for (int j = 0; j < 8; ++j) {\n sumi1 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);\n }\n q8 += 8;\n }\n for (int l = 2; l < 4; ++l) {\n const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));\n for (int j = 0; j < 8; ++j) {\n sumi2 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);\n }\n q8 += 8;\n }\n bsum += ls1 * sumi1 + ls2 * sumi2;\n qs += 4;\n signs += 4;\n }\n\n sumf += d * bsum;\n }\n\n *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_xxs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n uint32_t aux32;\n\n float sumf = 0.f;\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n int32_t bsum = 0;\n for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n memcpy(&aux32, gas, sizeof(uint32_t)); gas += sizeof(uint32_t);\n const uint32_t ls = 2*(aux32 >> 28) + 1;\n int32_t sumi = 0;\n for (int l = 0; l < 4; ++l) {\n const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*l+0]);\n const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*l+1]);\n const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];\n for (int j = 0; j < 4; ++j) {\n sumi += grid1[j] * q8[j+0] * (signs & kmask_iq2xs[j+0] ? -1 : 1);\n sumi += grid2[j] * q8[j+4] * (signs & kmask_iq2xs[j+4] ? -1 : 1);\n }\n q8 += 8;\n }\n q3 += 8;\n bsum += sumi * ls;\n }\n sumf += d * bsum;\n }\n *s = 0.25f * sumf;\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq3_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n float sumf = 0.f;\n for (int i = 0; i < nb; ++i) {\n const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n const uint8_t * GGML_RESTRICT qs = x[i].qs;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const uint8_t * GGML_RESTRICT signs = x[i].signs;\n const int8_t * GGML_RESTRICT q8 = y[i].qs;\n int32_t bsum = 0;\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const uint32_t ls1 = 2*(x[i].scales[ib32/2] & 0xf) + 1;\n const uint32_t ls2 = 2*(x[i].scales[ib32/2] >> 4) + 1;\n int32_t sumi = 0;\n for (int l = 0; l < 4; ++l) {\n const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256)));\n const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256)));\n for (int j = 0; j < 4; ++j) {\n sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);\n sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);\n }\n q8 += 8;\n }\n qs += 8;\n signs += 4;\n bsum += sumi * ls1;\n sumi = 0;\n for (int l = 0; l < 4; ++l) {\n const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256)));\n const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256)));\n for (int j = 0; j < 4; ++j) {\n sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);\n sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);\n }\n q8 += 8;\n }\n qs += 8;\n signs += 4;\n bsum += sumi * ls2;\n }\n sumf += d * bsum;\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_s * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n float sumf = 0;\n for (int i = 0; i < nb; i++) {\n\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint16_t * qh = x[i].qh;\n\n int sumi = 0, sumi1 = 0;\n for (int ib = 0; ib < QK_K/32; ++ib) {\n const int ls = 2*((qh[ib] >> 12) & 7) + 1;\n const int delta = qh[ib] & 0x8000 ? -1 : 1;\n int lsum = 0;\n for (int l = 0; l < 4; ++l) {\n const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));\n for (int j = 0; j < 8; ++j) {\n lsum += q8[j] * grid[j];\n }\n q8 += 8;\n }\n sumi += ls * lsum;\n sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]);\n qs += 4;\n }\n\n sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(n % QK_K == 0);\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n\n const block_iq1_m * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n iq1m_scale_t scale;\n\n int sum1[2], sum2[2], delta[4];\n\n float sumf = 0;\n for (int i = 0; i < nb; i++) {\n\n const int8_t * q8 = y[i].qs;\n const uint8_t * qs = x[i].qs;\n const uint8_t * qh = x[i].qh;\n const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n\n int sumi1 = 0, sumi2 = 0;\n for (int ib = 0; ib < QK_K/32; ++ib) {\n delta[0] = qh[0] & 0x08 ? -1 : 1;\n delta[1] = qh[0] & 0x80 ? -1 : 1;\n delta[2] = qh[1] & 0x08 ? -1 : 1;\n delta[3] = qh[1] & 0x80 ? -1 : 1;\n sum1[0] = sum1[1] = sum2[0] = sum2[1] = 0;\n for (int l = 0; l < 4; ++l) {\n const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((uint16_t)qh[l/2] << (8 - 4*(l%2))) & 0x700)));\n int lsum1 = 0, lsum2 = 0;\n for (int j = 0; j < 8; ++j) {\n lsum1 += q8[j] * grid[j];\n lsum2 += q8[j];\n }\n q8 += 8;\n sum1[l/2] += lsum1;\n sum2[l/2] += lsum2*delta[l];\n }\n\n const int ls1 = 2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1;\n const int ls2 = 2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1;\n\n sumi1 += sum1[0] * ls1 + sum1[1] * ls2;\n sumi2 += sum2[0] * ls1 + sum2[1] * ls2;\n qs += 4;\n qh += 2;\n }\n\n sumf += GGML_CPU_FP16_TO_FP32(scale.f16) * y[i].d * (sumi1 + IQ1M_DELTA * sumi2);\n }\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK4_NL == 0);\n static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n const block_iq4_nl * GGML_RESTRICT x = vx;\n const block_q8_0 * GGML_RESTRICT y = vy;\n\n const int nb = n / QK4_NL;\n\n int ib = 0;\n float sumf = 0;\n\n for (; ib < nb; ++ib) {\n const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);\n int sumi1 = 0, sumi2 = 0;\n for (int j = 0; j < QK4_NL/2; ++j) {\n sumi1 += y[ib].qs[j+ 0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];\n sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >> 4];\n }\n sumf += d * (sumi1 + sumi2);\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n assert(nrc == 1);\n UNUSED(nrc);\n UNUSED(bx);\n UNUSED(by);\n UNUSED(bs);\n assert(n % QK_K == 0);\n\n const block_iq4_xs * GGML_RESTRICT x = vx;\n const block_q8_K * GGML_RESTRICT y = vy;\n\n const int nb = n / QK_K;\n\n float sumf = 0;\n for (int ibl = 0; ibl < nb; ++ibl) {\n const float d4d8 = GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d;\n uint16_t h = x[ibl].scales_h;\n const uint8_t * qs = x[ibl].qs;\n const int8_t * q8 = y[ibl].qs;\n for (int ib = 0; ib < QK_K/32; ib += 2) {\n const uint8_t ls1 = (x[ibl].scales_l[ib/2] & 0xf) | ((h << 4) & 0x30);\n const uint8_t ls2 = (x[ibl].scales_l[ib/2] >> 4) | ((h << 2) & 0x30);\n h >>= 4;\n const float d1 = d4d8*(ls1 - 32);\n const float d2 = d4d8*(ls2 - 32);\n int sumi1 = 0, sumi2 = 0;\n for (int j = 0; j < 16; ++j) {\n sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];\n sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4];\n }\n sumf += d1 * (sumi1 + sumi2);\n qs += 16;\n q8 += 32;\n sumi1 = sumi2 = 0;\n for (int j = 0; j < 16; ++j) {\n sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];\n sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4];\n }\n sumf += d2 * (sumi1 + sumi2);\n qs += 16;\n q8 += 32;\n }\n }\n *s = sumf;\n}\n\n// ============================ 4-bit non-linear quants\n\nvoid quantize_row_iq4_nl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n assert(k % QK4_NL == 0);\n quantize_row_iq4_nl_ref(x, y, k);\n}\n\nvoid quantize_row_iq4_xs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n quantize_iq4_xs(x, y, 1, k, NULL);\n}\n" }, { "path": "src/ggml-cpu/quants.h", "content": "#pragma once\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n\n#include \"ggml.h\"\n\n// GGML CPU internal header\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// Quantization\nvoid quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\nvoid quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\nvoid quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\nvoid quantize_row_tq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\nvoid quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\n// Dot product\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\n// Generic implementation\nvoid quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cpu/repack.cpp", "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"traits.h\"\n\n#include \"arch-fallback.h\"\n\n#include \n#include \n#include \n#include // for GGML_ASSERT\n\n#include \"repack.h\"\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#endif\n\n#define UNUSED GGML_UNUSED\n\nstatic inline int nearest_int(float fval) {\n assert(fabsf(fval) <= 4194303.f);\n float val = fval + 12582912.f;\n int i; memcpy(&i, &val, sizeof(int));\n return (i & 0x007fffff) - 0x00400000;\n}\n\n// Functions to create the interleaved data layout formats\n\n// interleave 4 block_q4_0s in blocks of blck_size_interleave\n// returns an interleaved block_q4_0x4\n// in the interleaved block_q4_0x4, place deltas for 4 block_q4_0 blocks\n// first, then interleave quants from 4 block_q4_0s in blocks of blck_size_interleave\n//\n// - in : an array of block_q4_0 pointers\n// - blck_size_interleave : the block_q4_0 quants bytes are interleaved in blocks of\n// blck_size_interleave bytes\n// - xor_mask : the mask to convert the nibbles in block_q4_0 quants bytes\n// from bias offset form to pure sign form (this saves subtract\n// operations durin unpacking)\n//\n\nextern \"C\" {\n\n#if defined __riscv_zvfh\nvoid ggml_quantize_mat_q8_0_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n // scalar\n const int blck_size_interleave = 1;\n float srcv[4][QK8_0];\n float id[4];\n\n for (int i = 0; i < nb; i++) {\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK8_0; j++) {\n srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];\n amax = MAX(amax, fabsf(srcv[row_iter][j]));\n }\n\n const float d = amax / ((1 << 7) - 1);\n id[row_iter] = d ? 1.0f / d : 0.0f;\n\n y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n }\n\n for (int j = 0; j < QK8_0 * 4; j++) {\n int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n src_offset += (j % blck_size_interleave);\n\n float x0 = srcv[src_id][src_offset] * id[src_id];\n y[i].qs[j] = roundf(x0);\n }\n }\n}\n\nvoid ggml_quantize_mat_q8_K_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK_K == 256);\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy;\n\n const int blck_size_interleave = 1;\n float srcv[4][QK_K];\n float iscale[4];\n\n for (int i = 0; i < nb; i++) {\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n float amax = 0.0f; // absolute max\n float max = 0;\n\n for (int j = 0; j < QK_K; j++) {\n srcv[row_iter][j] = x[row_iter * k + i * QK_K + j];\n // Update the maximum value of the corresponding super block\n if(amax < fabsf(srcv[row_iter][j])) {\n amax = fabsf(srcv[row_iter][j]);\n max = srcv[row_iter][j];\n }\n }\n\n iscale[row_iter] = amax ? -127.f/max : 0;\n y[i].d[row_iter] = amax ? 1/iscale[row_iter] : 0;\n }\n\n for (int j = 0; j < QK_K / 4; j++) {\n y[i].bsums[j] = 0;\n }\n for (int j = 0; j < QK_K * 4; j++) {\n int src_id = j % 4;\n int src_offset = j / 4;\n int index = ((j >> 6) << 2) + (j & 3);\n\n float x0 = srcv[src_id][src_offset] * iscale[src_id];\n y[i].qs[j] = nearest_int(x0);\n y[i].bsums[index] += y[i].qs[j];\n }\n }\n}\n#endif\n\nvoid ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n // scalar\n const int blck_size_interleave = 4;\n float srcv[4][QK8_0];\n float id[4];\n\n for (int i = 0; i < nb; i++) {\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK8_0; j++) {\n srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];\n amax = MAX(amax, fabsf(srcv[row_iter][j]));\n }\n\n const float d = amax / ((1 << 7) - 1);\n id[row_iter] = d ? 1.0f / d : 0.0f;\n\n y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n }\n\n for (int j = 0; j < QK8_0 * 4; j++) {\n int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n src_offset += (j % blck_size_interleave);\n\n float x0 = srcv[src_id][src_offset] * id[src_id];\n y[i].qs[j] = roundf(x0);\n }\n }\n}\n\nvoid ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK8_0 == 32);\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n // scalar\n const int blck_size_interleave = 8;\n float srcv[4][QK8_0];\n float id[4];\n\n for (int i = 0; i < nb; i++) {\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK8_0; j++) {\n srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];\n amax = MAX(amax, fabsf(srcv[row_iter][j]));\n }\n\n const float d = amax / ((1 << 7) - 1);\n id[row_iter] = d ? 1.0f / d : 0.0f;\n\n y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n }\n\n for (int j = 0; j < QK8_0 * 4; j++) {\n int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n src_offset += (j % blck_size_interleave);\n\n float x0 = srcv[src_id][src_offset] * id[src_id];\n y[i].qs[j] = roundf(x0);\n }\n }\n}\n\nvoid ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK_K == 256);\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy;\n\n // scalar\n const int blck_size_interleave = 4;\n float srcv[4][QK_K];\n float iscale[4];\n\n for (int i = 0; i < nb; i++) {\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n float amax = 0.0f; // absolute max\n float max = 0;\n\n for (int j = 0; j < QK_K; j++) {\n srcv[row_iter][j] = x[row_iter * k + i * QK_K + j];\n // Update the maximum value of the corresponding super block\n if(amax < fabsf(srcv[row_iter][j])) {\n amax = fabsf(srcv[row_iter][j]);\n max = srcv[row_iter][j];\n }\n }\n\n iscale[row_iter] = amax ? -127.f/max : 0;\n\n y[i].d[row_iter] = amax ? 1/iscale[row_iter] : 0;\n }\n\n for (int j = 0; j < QK_K / 4; j++) {\n y[i].bsums[j] = 0;\n }\n\n // Quants values are interleaved in sequence of four bytes from corresponding super blocks\n // Bsums values are interleaved in sequence of four bsums from each super block taken for interleaving\n // i.e first four bsums from the first super block, followed by first four bsums from second super block and so on\n for (int j = 0; j < QK_K * 4; j++) {\n int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n src_offset += (j % blck_size_interleave);\n int index = (((j & 15) >> 2) << 2) + ((j >> 8) << 4) + ((j >> 6) & 3);\n\n float x0 = srcv[src_id][src_offset] * iscale[src_id];\n y[i].qs[j] = nearest_int(x0);\n y[i].bsums[index] += y[i].qs[j];\n }\n }\n}\n\nvoid ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n assert(QK_K == 256);\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy;\n\n // scalar\n const int blck_size_interleave = 8;\n float srcv[4][QK_K];\n float iscale[4];\n\n for (int i = 0; i < nb; i++) {\n for (int row_iter = 0; row_iter < 4; row_iter++) {\n float amax = 0.0f; // absolute max\n float max = 0;\n\n for (int j = 0; j < QK_K; j++) {\n srcv[row_iter][j] = x[row_iter * k + i * QK_K + j];\n // Update the maximum value of the corresponding super block\n if(amax < fabsf(srcv[row_iter][j])) {\n amax = fabsf(srcv[row_iter][j]);\n max = srcv[row_iter][j];\n }\n }\n\n iscale[row_iter] = amax ? -127.f/max : 0;\n\n y[i].d[row_iter] = amax ? 1/iscale[row_iter] : 0;\n }\n\n for (int j = 0; j < QK_K / 4; j++) {\n y[i].bsums[j] = 0;\n }\n\n // Quants values are interleaved in sequence of eight bytes from corresponding super blocks\n // Bsums values are interleaved in sequence of four bsums from each super block taken for interleaving\n // i.e first four bsums from the first super block, followed by first four bsums from second super block and so on\n for (int j = 0; j < QK_K * 4; j++) {\n int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n src_offset += (j % blck_size_interleave);\n int index = (((j & 31) >> 3) << 2) + ((j >> 8) << 4) + ((j >> 6) & 3);\n\n float x0 = srcv[src_id][src_offset] * iscale[src_id];\n y[i].qs[j] = nearest_int(x0);\n y[i].bsums[index] += y[i].qs[j];\n }\n }\n}\n\n} // extern \"C\"\n\ntemplate \nvoid ggml_quantize_mat_t(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row);\n\ntemplate <> void ggml_quantize_mat_t<4, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n assert(nrow == 4);\n UNUSED(nrow);\n ggml_quantize_mat_q8_0_4x4(x, vy, n_per_row);\n}\n\ntemplate <> void ggml_quantize_mat_t<8, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n assert(nrow == 4);\n UNUSED(nrow);\n ggml_quantize_mat_q8_0_4x8(x, vy, n_per_row);\n}\n\ntemplate <> void ggml_quantize_mat_t<4, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n assert(nrow == 4);\n UNUSED(nrow);\n ggml_quantize_mat_q8_K_4x4(x, vy, n_per_row);\n}\n\ntemplate <> void ggml_quantize_mat_t<8, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n assert(nrow == 4);\n UNUSED(nrow);\n ggml_quantize_mat_q8_K_4x8(x, vy, n_per_row);\n}\n\n#if defined __riscv_zvfh\ntemplate <> void ggml_quantize_mat_t<1, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n assert(nrow == 4);\n UNUSED(nrow);\n ggml_quantize_mat_q8_0_4x1(x, vy, n_per_row);\n}\n\ntemplate <> void ggml_quantize_mat_t<1, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n assert(nrow == 4);\n UNUSED(nrow);\n ggml_quantize_mat_q8_K_4x1(x, vy, n_per_row);\n}\n#endif\n\ntemplate \nstatic void ggml_gemv_q6_K_NxM_q8_K_generic_impl(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int blocklen = M;\n constexpr int ncols_interleaved = N;\n const int qk = QK_K;\n const int nb = n / qk;\n const int blocks_per_half = 64 / blocklen;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[8];\n\n const block_q8_K * a_ptr = (const block_q8_K *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q6_Kx8 * b_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[j] = 0.0f;\n }\n\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n const int base_l = (k / blocks_per_half) * 128 + (k % blocks_per_half) * blocklen;\n const int base_h = base_l + 64;\n\n const int scale_idx_l = base_l / 16;\n const int scale_idx_h = base_h / 16;\n\n const int qh_shift_l = ((base_l % 128) / 32) * 2;\n const int qh_shift_h = ((base_h % 128) / 32) * 2;\n\n const int qh_half_l = (base_l / 128) * 32;\n const int qh_half_h = (base_h / 128) * 32;\n\n for (int j = 0; j < ncols_interleaved; j++) {\n const int8_t scale_l = b_ptr[l].scales[scale_idx_l * ncols_interleaved + j];\n const int8_t scale_h = b_ptr[l].scales[scale_idx_h * ncols_interleaved + j];\n\n int sumi_l = 0;\n int sumi_h = 0;\n\n for (int i = 0; i < blocklen; i++) {\n const int ql_pos = k * ncols_interleaved * blocklen + j * blocklen + i;\n const int l_4 = b_ptr[l].ql[ql_pos] & 0xF;\n const int hi_4 = (b_ptr[l].ql[ql_pos] >> 4) & 0xF;\n\n const int qh_idx_l = qh_half_l + ((base_l + i) % 32);\n const int qh_chunk_l = qh_idx_l / blocklen;\n const int qh_pos_l = qh_idx_l % blocklen;\n const int qh_offset_l = qh_chunk_l * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_l;\n const int hi_2_l = (b_ptr[l].qh[qh_offset_l] >> qh_shift_l) & 0x3;\n\n const int qh_idx_h = qh_half_h + ((base_h + i) % 32);\n const int qh_chunk_h = qh_idx_h / blocklen;\n const int qh_pos_h = qh_idx_h % blocklen;\n const int qh_offset_h = qh_chunk_h * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_h;\n const int hi_2_h = (b_ptr[l].qh[qh_offset_h] >> qh_shift_h) & 0x3;\n\n const int q_l = ((hi_2_l << 4) | l_4) - 32;\n const int q_h = ((hi_2_h << 4) | hi_4) - 32;\n\n const int8_t a_l = a_ptr[l].qs[base_l + i];\n const int8_t a_h = a_ptr[l].qs[base_h + i];\n\n sumi_l += q_l * a_l;\n sumi_h += q_h * a_h;\n }\n\n sumf[j] +=\n (sumi_l * scale_l + sumi_h * scale_h) * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n }\n }\n }\n\n for (int j = 0; j < ncols_interleaved; j++) {\n s[x * ncols_interleaved + j] = sumf[j];\n }\n }\n}\n\ntemplate \nstatic void ggml_gemm_q6_K_NxM_q8_K_generic_impl(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int blocklen = M;\n constexpr int ncols_interleaved = N;\n const int qk = QK_K;\n const int nb = n / qk;\n const int blocks_per_half = 64 / blocklen;\n const int q8_half_stride = 512;\n const int q8_low_high_step = 256;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n\n float sumf[4][8];\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q6_Kx8 * b_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[m][j] = 0.0f;\n }\n }\n\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n const int base_l = (k / blocks_per_half) * 128 + (k % blocks_per_half) * blocklen;\n const int base_h = base_l + 64;\n\n const int scale_idx_l = base_l / 16;\n const int scale_idx_h = base_h / 16;\n\n const int qh_shift_l = ((base_l % 128) / 32) * 2;\n const int qh_shift_h = ((base_h % 128) / 32) * 2;\n\n const int qh_half_l = (base_l / 128) * 32;\n const int qh_half_h = (base_h / 128) * 32;\n\n const int q8_base = (k / blocks_per_half) * q8_half_stride + (k % blocks_per_half) * (blocklen * 4);\n\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n const int8_t scale_l = b_ptr[l].scales[scale_idx_l * ncols_interleaved + j];\n const int8_t scale_h = b_ptr[l].scales[scale_idx_h * ncols_interleaved + j];\n\n int sumi_l = 0;\n int sumi_h = 0;\n\n for (int i = 0; i < blocklen; i++) {\n const int ql_pos = k * ncols_interleaved * blocklen + j * blocklen + i;\n const int l_4 = b_ptr[l].ql[ql_pos] & 0xF;\n const int hi_4 = (b_ptr[l].ql[ql_pos] >> 4) & 0xF;\n\n const int qh_idx_l = qh_half_l + ((base_l + i) % 32);\n const int qh_chunk_l = qh_idx_l / blocklen;\n const int qh_pos_l = qh_idx_l % blocklen;\n const int qh_offset_l =\n qh_chunk_l * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_l;\n const int hi_2_l = (b_ptr[l].qh[qh_offset_l] >> qh_shift_l) & 0x3;\n\n const int qh_idx_h = qh_half_h + ((base_h + i) % 32);\n const int qh_chunk_h = qh_idx_h / blocklen;\n const int qh_pos_h = qh_idx_h % blocklen;\n const int qh_offset_h =\n qh_chunk_h * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_h;\n const int hi_2_h = (b_ptr[l].qh[qh_offset_h] >> qh_shift_h) & 0x3;\n\n const int q_l = ((hi_2_l << 4) | l_4) - 32;\n const int q_h = ((hi_2_h << 4) | hi_4) - 32;\n\n const int8_t q8_l = a_ptr[l].qs[q8_base + m * blocklen + i];\n const int8_t q8_h = a_ptr[l].qs[q8_base + m * blocklen + i + q8_low_high_step];\n\n sumi_l += q_l * q8_l;\n sumi_h += q_h * q8_h;\n }\n\n sumf[m][j] += (sumi_l * scale_l + sumi_h * scale_h) * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) *\n a_ptr[l].d[m];\n }\n }\n }\n }\n\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n }\n}\n\ntemplate \nstatic void ggml_gemv_q5_K_NxM_q8_K_generic_impl(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int blocklen = M;\n constexpr int ncols_interleaved = N;\n const int qk = QK_K;\n const int nb = n / qk;\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[ncols_interleaved];\n float sum_minf[ncols_interleaved];\n uint32_t utmp[32];\n int sumi1;\n int sumi2;\n int sumi;\n\n const block_q8_K * a_ptr = (const block_q8_K *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q5_Kx8 * b_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[j] = 0.0;\n sum_minf[j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int sb = 0; sb < 8; sb++) {\n memcpy(utmp + sb * 4, b_ptr[l].scales + sb * K_SCALE_SIZE, K_SCALE_SIZE);\n utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n utmp[sb * 4 + 2] = uaux_0;\n utmp[sb * 4 + 0] &= kmask1;\n }\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n constexpr int scale_stride = 32;\n uint8_t * scales_0 = (uint8_t *) utmp + (k / (32 / blocklen)) * scale_stride;\n uint8_t * scales_1 = (uint8_t *) utmp + (k / (32 / blocklen)) * scale_stride + 16;\n\n const int qh_shift = (k / (32 / blocklen)) * 2;\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int b_qs_offset = k * ncols_interleaved * blocklen + j * blocklen + i;\n\n const int qh_idx = (k * blocklen + i) % 32;\n const int qh_chunk = qh_idx / blocklen;\n const int qh_pos = qh_idx % blocklen;\n const int b_qh_offset = qh_chunk * (blocklen * ncols_interleaved) + j * blocklen + qh_pos;\n\n const uint8_t qh_val = b_ptr[l].qh[b_qh_offset];\n const uint8_t h0 = (qh_val >> qh_shift) & 1;\n const uint8_t h1 = (qh_val >> (qh_shift + 1)) & 1;\n\n const int v0 = (int8_t) ((b_ptr[l].qs[b_qs_offset] & 0xF) | (h0 << 4));\n const int v1 = (int8_t) ((b_ptr[l].qs[b_qs_offset] >> 4) | (h1 << 4));\n\n const int q8_offset = (k / (32 / blocklen)) * 64 + (k % (32 / blocklen)) * blocklen + i;\n\n sumi1 = (v0 * a_ptr[l].qs[q8_offset]);\n sumi2 = (v1 * a_ptr[l].qs[q8_offset + 32]);\n sumi1 = sumi1 * scales_0[j];\n sumi2 = sumi2 * scales_1[j];\n sumi += sumi1 + sumi2;\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n }\n }\n for (int sb = 0; sb < 8; sb++) {\n uint8_t * mins = (uint8_t *) utmp + 8 + sb * 16;\n for (int j = 0; j < ncols_interleaved; j++) {\n sum_minf[j] += mins[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) *\n GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) {\n s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n }\n }\n}\n\ntemplate \nstatic void ggml_gemm_q5_K_NxM_q8_K_generic_impl(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n constexpr int blocklen = M;\n constexpr int ncols_interleaved = N;\n const int qk = QK_K;\n const int nb = n / qk;\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n float sumf[4][ncols_interleaved];\n float sum_minf[4][ncols_interleaved];\n uint32_t utmp[32];\n int sumi1;\n int sumi2;\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q5_Kx8 * b_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[m][j] = 0.0;\n sum_minf[m][j] = 0.0;\n }\n }\n for (int l = 0; l < nb; l++) {\n for (int sb = 0; sb < 8; sb++) {\n memcpy(utmp + sb * 4, b_ptr[l].scales + sb * K_SCALE_SIZE, K_SCALE_SIZE);\n utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n utmp[sb * 4 + 2] = uaux_0;\n utmp[sb * 4 + 0] &= kmask1;\n }\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n constexpr int scale_stride = 32;\n uint8_t * scales_0 = (uint8_t *) utmp + (k / (32 / blocklen)) * scale_stride;\n uint8_t * scales_1 = (uint8_t *) utmp + (k / (32 / blocklen)) * scale_stride + 16;\n\n const int qh_shift = (k / (32 / blocklen)) * 2;\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int b_qs_offset = k * ncols_interleaved * blocklen + j * blocklen + i;\n\n const int qh_idx = (k * blocklen + i) % 32;\n const int qh_chunk = qh_idx / blocklen;\n const int qh_pos = qh_idx % blocklen;\n const int b_qh_offset =\n qh_chunk * (blocklen * ncols_interleaved) + j * blocklen + qh_pos;\n\n const uint8_t qh_val = b_ptr[l].qh[b_qh_offset];\n const uint8_t h0 = (qh_val >> qh_shift) & 1;\n const uint8_t h1 = (qh_val >> (qh_shift + 1)) & 1;\n\n const int v0 = (int8_t) ((b_ptr[l].qs[b_qs_offset] & 0xF) | (h0 << 4));\n const int v1 = (int8_t) ((b_ptr[l].qs[b_qs_offset] >> 4) | (h1 << 4));\n\n const int q8_offset = (k / (32 / blocklen)) * 256 +\n (k % (32 / blocklen)) * 4 * blocklen + m * blocklen + i;\n\n sumi1 = (v0 * a_ptr[l].qs[q8_offset]);\n sumi2 = (v1 * a_ptr[l].qs[q8_offset + 128]);\n sumi1 = sumi1 * scales_0[j];\n sumi2 = sumi2 * scales_1[j];\n sumi += sumi1 + sumi2;\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n }\n }\n }\n for (int sb = 0; sb < 8; sb++) {\n uint8_t * mins = (uint8_t *) utmp + 8 + sb * 16;\n for (int m = 0; m < 4; m++) {\n const int16_t * bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) * 6);\n for (int j = 0; j < ncols_interleaved; j++) {\n sum_minf[m][j] += mins[j] * (bsums[0] + bsums[1]) *\n GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n }\n }\n }\n }\n}\n\nextern \"C\" {\n\nvoid ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert(nr == 1);\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[4];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n }\n}\n\nvoid ggml_gemv_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[4];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n }\n}\n\nvoid ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[8];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n }\n}\n\nvoid ggml_gemv_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 4;\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[8];\n float sum_minf[8];\n uint32_t utmp[32];\n int sumi1;\n int sumi2;\n int sumi;\n\n const block_q8_K * a_ptr = (const block_q8_K *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[j] = 0.0;\n sum_minf[j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int sb = 0; sb < 8; sb++) {\n memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12);\n utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n utmp[sb * 4 + 2] = uaux_0;\n utmp[sb * 4 + 0] &= kmask1;\n }\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n uint8_t * scales_0 = (uint8_t *) utmp + (k / 8) * 32;\n uint8_t * scales_1 = (uint8_t *) utmp + (k / 8) * 32 + 16;\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4);\n sumi1 = (v0 * a_ptr[l].qs[(k / 8) * 64 + (k % 8) * blocklen + i]);\n sumi2 = (v1 * a_ptr[l].qs[(k / 8) * 64 + (k % 8) * blocklen + i + 32]);\n sumi1 = sumi1 * scales_0[j];\n sumi2 = sumi2 * scales_1[j];\n sumi += sumi1 + sumi2;\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n }\n }\n for (int sb = 0; sb < 8; sb++) {\n uint8_t * mins = (uint8_t *) utmp + 8 + sb * 16;\n for (int j = 0; j < ncols_interleaved; j++) {\n sum_minf[j] += mins[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) {\n s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n }\n }\n}\n\nvoid ggml_gemv_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[8];\n float sum_minf[8];\n uint32_t utmp[32];\n int sumi1;\n int sumi2;\n int sumi;\n\n const block_q8_K * a_ptr = (const block_q8_K *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[j] = 0.0;\n sum_minf[j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int sb = 0; sb < 8; sb++) {\n memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12);\n utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n utmp[sb * 4 + 2] = uaux_0;\n utmp[sb * 4 + 0] &= kmask1;\n }\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n uint8_t *scales_0 = (uint8_t*) utmp + (k / 4) * 32;\n uint8_t *scales_1 = (uint8_t*) utmp + (k / 4) * 32 + 16;\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4);\n sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 64 + (k % 4) * blocklen + i]);\n sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 64 + (k % 4) * blocklen + i + 32]);\n sumi1 = sumi1 * scales_0[j];\n sumi2 = sumi2 * scales_1[j];\n sumi += sumi1 + sumi2;\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n }\n }\n for (int sb = 0; sb < 8; sb++) {\n uint8_t *mins = (uint8_t*) utmp + 8 + sb * 16;\n for (int j = 0; j < ncols_interleaved; j++) {\n sum_minf[j] += mins[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) {\n s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n }\n }\n}\n\nvoid ggml_gemv_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[8];\n float sum_minf[8];\n int sumi1,sumi2,sumi3,sumi4;\n int sumi;\n\n const block_q8_K * a_ptr = (const block_q8_K *)vy;\n for(int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q2_Kx8 * b_ptr = (const block_q2_Kx8 *) vx + (x * nb);\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[j] = 0.0;\n sum_minf[j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (4 * blocklen)); k++) {\n const uint8_t *scales_0 = b_ptr[l].scales + (k / 4) * 64 ;\n const uint8_t *scales_1 = b_ptr[l].scales + (k / 4) * 64 + 16;\n const uint8_t *scales_2 = b_ptr[l].scales + (k / 4) * 64 + 32;\n const uint8_t *scales_3 = b_ptr[l].scales + (k / 4) * 64 + 48;\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi3 = 0;\n sumi4 = 0;\n sumi = 0;\n int offset = ((k / 2) % 2) + j * 2;\n for (int i = 0; i < blocklen; ++i){\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 3);\n const int v1 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 2 ) & 3);\n const int v2 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4 ) & 3);\n const int v3 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 6 ) & 3);\n sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 128 + (k % 4) * blocklen + i]);\n sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 128 + (k % 4) * blocklen + i + 32]);\n sumi3 = (v2 * a_ptr[l].qs[(k >> 2) * 128 + (k % 4) * blocklen + i + 64]);\n sumi4 = (v3 * a_ptr[l].qs[(k >> 2) * 128 + (k % 4) * blocklen + i + 96]);\n\n sumi1 = sumi1 * (scales_0[offset] & 0xF);\n sumi2 = sumi2 * (scales_1[offset] & 0xF);\n sumi3 = sumi3 * (scales_2[offset] & 0xF);\n sumi4 = sumi4 * (scales_3[offset] & 0xF);\n sumi += sumi1 + sumi2 + sumi3 + sumi4;\n }\n sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n }\n }\n for(int sb = 0; sb < 8; sb++) {\n const uint8_t *mins = b_ptr[l].scales + sb * 16;\n for(int j = 0; j < ncols_interleaved; j++){\n sum_minf[j] += ((mins[j * 2] >> 4) * a_ptr[l].bsums[sb * 2] + (mins[(j * 2)+ 1] >> 4) * a_ptr[l].bsums[sb * 2 + 1]) * GGML_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) {\n s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n }\n }\n}\n\nvoid ggml_gemv_q5_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n ggml_gemv_q5_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n ggml_gemv_q5_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\n\nvoid ggml_gemv_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n ggml_gemv_q6_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n ggml_gemv_q6_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert(nr == 1);\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[4];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx4 * b_ptr = (const block_iq4_nlx4 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n }\n}\n\nvoid ggml_gemv_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert(nr == 1);\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[8];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx8 * b_ptr = (const block_iq4_nlx8 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n }\n}\n\nvoid ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert(nr == 1);\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[4];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n }\n sumf[j] += sumi * GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n }\n}\n\nvoid ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert(nr == 1);\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[8];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_mxfp4x8 * b_ptr = (const block_mxfp4x8 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n }\n sumf[j] += sumi * GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n }\n}\n\nvoid ggml_gemv_q8_0_4x4_q8_0_generic(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert(nr == 1);\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[4];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / blocklen); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n sumi += v0 * a_ptr[l].qs[k * blocklen + i];\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) {\n s[x * ncols_interleaved + j] = sumf[j];\n }\n }\n}\n\nvoid ggml_gemv_q8_0_4x8_q8_0_generic(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 8;\n\n assert(nr == 1);\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[4];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / blocklen); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n sumi += v0 * a_ptr[l].qs[k * blocklen + i];\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) {\n s[x * ncols_interleaved + j] = sumf[j];\n }\n }\n}\n\n#if defined __riscv_zvfh\nvoid ggml_gemv_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[16];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x16 * b_ptr = (const block_q4_0x16 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n }\n}\n\nvoid ggml_gemv_q4_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n assert (n % qk == 0);\n assert (nc % ncols_interleaved == 0);\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n float sumf[16];\n float sum_minf[16];\n uint8_t scales[128];\n uint8_t mins[128];\n int sumi1;\n int sumi2;\n int sumi;\n const block_q8_K * a_ptr = (const block_q8_K *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx16 * b_ptr = (const block_q4_Kx16 *) vx + (x * nb);\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[j] = 0.0f;\n sum_minf[j] = 0.0f;\n }\n for (int l = 0; l < nb; l++) {\n for (int i = 0; i < 128; i++) {\n scales[i] = b_ptr[l].scales[i] & 0x0F;\n mins[i] = b_ptr[l].scales[i] >> 4;\n }\n for (int i = 0; i < 64; i++) {\n scales[i] |= (b_ptr[l].scales[128 + i] & 0x03) << 4;\n mins[i] |= (b_ptr[l].scales[128 + i] & 0x0C) << 2;\n scales[i + 64] |= (b_ptr[l].scales[128 + i] & 0x30);\n mins[i + 64] |= (b_ptr[l].scales[128 + i] & 0xC0) >> 2;\n }\n for (int sb = 0; sb < 8; sb++) {\n uint8_t *min = &mins[sb * 16];\n for (int j = 0; j < ncols_interleaved; j++) {\n sum_minf[j] += min[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n }\n }\n for (int sb = 0; sb < 8; sb += 2) {\n uint8_t *scales_0 = &scales[sb * 16];\n uint8_t *scales_1 = &scales[(sb + 1) * 16];\n for (int i = 0; i < QK4_0; i++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi = 0;\n const int v0 = (int8_t) (b_ptr[l].qs[sb * 256 + i * 16 + j] & 0xF);\n const int v1 = (int8_t) (b_ptr[l].qs[sb * 256 + i * 16 + j] >> 4);\n sumi1 = (v0 * a_ptr[l].qs[sb * 32 + i]);\n sumi2 = (v1 * a_ptr[l].qs[sb * 32 + 32 + i]);\n sumi1 = sumi1 * scales_0[j];\n sumi2 = sumi2 * scales_1[j];\n sumi += sumi1 + sumi2;\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n }\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) {\n s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n }\n }\n}\n\nvoid ggml_gemv_iq4_nl_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert(nr == 1);\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[16];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx16 * b_ptr = (const block_iq4_nlx16 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n }\n}\n\nvoid ggml_gemv_q8_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert(nr == 1);\n assert(n % qk == 0);\n assert(nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n UNUSED(nr);\n\n float sumf[16];\n int sumi;\n\n const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q8_0x16 * b_ptr = (const block_q8_0x16 *) vx + (x * nb);\n\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / blocklen); k++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n sumi += v0 * a_ptr[l].qs[k * blocklen + i];\n }\n sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n }\n }\n }\n for (int j = 0; j < ncols_interleaved; j++) {\n s[x * ncols_interleaved + j] = sumf[j];\n }\n }\n}\n\nvoid ggml_gemv_q2_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n assert(n % QK_K == 0);\n assert(nr == 1);\n assert(nc % 16 == 0);\n\n UNUSED(bs);\n\n const int nb = n / QK_K;\n const block_q2_Kx16 * x = (const block_q2_Kx16 *)vx;\n const block_q8_K * y = (const block_q8_K *)vy;\n\n // Layout: Even-Low(0,2,4,6), Odd-Low(1,3,5,7), Even-High(8...), Odd-High(9...)\n const int sb_perm[16] = {\n 0, 4, 1, 5, 2, 6, 3, 7, // 0-7\n 8, 12, 9, 13, 10, 14, 11, 15 // 8-15\n };\n\n for (int col_tile = 0; col_tile < nc; col_tile += 16) {\n const block_q2_Kx16 * x_ptr = x + (col_tile / 16) * nb;\n const block_q8_K * y_ptr = y;\n\n float sumf[16] = {0};\n\n // Loop over K-blocks\n for (int k_block = 0; k_block < nb; ++k_block) {\n int32_t isum[16] = {0};\n int32_t summs[16] = {0};\n\n const uint8_t * qs_rhs = x_ptr[k_block].qs;\n const uint8_t * sc_rhs = x_ptr[k_block].scales;\n const int8_t * qs_lhs = y_ptr[k_block].qs;\n const int16_t * bs_lhs = y_ptr[k_block].bsums;\n\n // Iterate over sub-blocks 0..15\n for (int sb = 0; sb < 16; ++sb) {\n // Correction Term\n int16_t bsum = bs_lhs[sb];\n int scale_offset = sb_perm[sb] * 16;\n\n for (int col = 0; col < 16; ++col) {\n uint8_t sc_val = sc_rhs[scale_offset + col];\n summs[col] += bsum * (sc_val >> 4); // Min is high 4 bits\n }\n\n // Main Dot Product\n // Calculate base offsets for Q2 unpacking based on SB\n int byte_base;\n if (sb < 8) byte_base = (sb % 2 == 0) ? 0 : 16;\n else byte_base = (sb % 2 == 0) ? 32 : 48;\n\n int shift = ((sb / 2) % 4) * 2;\n\n for (int col = 0; col < 16; ++col) {\n uint8_t sc_val = sc_rhs[scale_offset + col];\n int32_t d_sb = sc_val & 0xF; // Scale is low 4 bits\n\n // Process 16 elements (l=0..15)\n for (int l = 0; l < 16; ++l) {\n // Q2: Interleaved by column. Byte `l` contains 4 k-values.\n int qs_idx = (byte_base + l) * 16 + col;\n uint8_t q2_val = (qs_rhs[qs_idx] >> shift) & 3;\n\n // Q8: Linear access\n int k = sb * 16 + l;\n int8_t q8_val = qs_lhs[k];\n\n isum[col] += q8_val * q2_val * d_sb;\n }\n }\n }\n\n // Finalize K-Block\n for (int col = 0; col < 16; ++col) {\n float d_lhs = y_ptr[k_block].d;\n float d_rhs = GGML_FP16_TO_FP32(x_ptr[k_block].d[col]);\n float dm_rhs = GGML_FP16_TO_FP32(x_ptr[k_block].dmin[col]);\n\n float d_all = d_lhs * d_rhs;\n float d_min = d_lhs * dm_rhs;\n\n sumf[col] += (isum[col] * d_all) - (summs[col] * d_min);\n }\n }\n\n for (int col = 0; col < 16; ++col) {\n s[col_tile + col] = sumf[col];\n }\n }\n}\n#endif\n\nvoid ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n {\n float sumf[4][4];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++)\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n }\n}\n\nvoid ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[4][4];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++)\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n}\n\nvoid ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[4][8];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++)\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n}\n\nvoid ggml_gemm_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 4;\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[4][8];\n float sum_minf[4][8];\n uint32_t utmp[32];\n int sumi1;\n int sumi2;\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[m][j] = 0.0;\n sum_minf[m][j] = 0.0;\n }\n }\n for (int l = 0; l < nb; l++) {\n for (int sb = 0; sb < 8; sb++) {\n memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12);\n utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n utmp[sb * 4 + 2] = uaux_0;\n utmp[sb * 4 + 0] &= kmask1;\n }\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n uint8_t * scales_0 = (uint8_t *) utmp + (k / 8) * 32;\n uint8_t * scales_1 = (uint8_t *) utmp + (k / 8) * 32 + 16;\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4);\n sumi1 = (v0 * a_ptr[l].qs[(k / 8) * 256 + (k % 8) * 4 * blocklen + m * blocklen + i]);\n sumi2 = (v1 * a_ptr[l].qs[(k / 8) * 256 + (k % 8) * 4 * blocklen + m * blocklen + i + 128]);\n sumi1 = sumi1 * scales_0[j];\n sumi2 = sumi2 * scales_1[j];\n sumi += sumi1 + sumi2;\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n }\n }\n }\n for (int sb = 0; sb < 8; sb++) {\n uint8_t * mins = (uint8_t *) utmp + 8 + sb * 16;\n for(int m = 0; m < 4; m++) {\n const int16_t * bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) * 6);\n for(int j = 0; j < ncols_interleaved; j++) {\n sum_minf[m][j] += mins[j] * (bsums[0] + bsums[1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n }\n }\n }\n }\n}\n\nvoid ggml_gemm_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n static const uint32_t kmask1 = 0x3f3f3f3f;\n static const uint32_t kmask2 = 0x0f0f0f0f;\n static const uint32_t kmask3 = 0x03030303;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(bs);\n\n float sumf[4][8];\n float sum_minf[4][8];\n uint32_t utmp[32];\n int sumi1;\n int sumi2;\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[m][j] = 0.0;\n sum_minf[m][j] = 0.0;\n }\n }\n for (int l = 0; l < nb; l++) {\n for (int sb = 0; sb < 8; sb++) {\n memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12);\n utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n utmp[sb * 4 + 2] = uaux_0;\n utmp[sb * 4 + 0] &= kmask1;\n }\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n uint8_t *scales_0 = (uint8_t*) utmp + (k / 4) * 32;\n uint8_t *scales_1 = (uint8_t*) utmp + (k / 4) * 32 + 16;\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4);\n sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 256 + (k % 4) * 4 * blocklen + m * blocklen + i]);\n sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 256 + (k % 4) * 4 * blocklen + m * blocklen + i + 128]);\n sumi1 = sumi1 * scales_0[j];\n sumi2 = sumi2 * scales_1[j];\n sumi += sumi1 + sumi2;\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n }\n }\n }\n for (int sb = 0; sb < 8; sb++) {\n uint8_t *mins = (uint8_t*) utmp + 8 + sb * 16;\n for(int m = 0; m < 4; m++) {\n const int16_t *bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) * 6);\n for(int j = 0; j < ncols_interleaved; j++) {\n sum_minf[m][j] += mins[j] * (bsums[0] + bsums[1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n }\n }\n }\n }\n}\n\nvoid ggml_gemm_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[4][8];\n float sum_minf[4][8];\n int sumi1, sumi2, sumi3, sumi4;\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q2_Kx8 * b_ptr = (const block_q2_Kx8 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[m][j] = 0.0;\n sum_minf[m][j] = 0.0;\n }\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (4 * blocklen)); k++) {\n\n const uint8_t *scales_0 = b_ptr[l].scales + (k / 4) * 64 ;\n const uint8_t *scales_1 = b_ptr[l].scales + (k / 4) * 64 + 16;\n const uint8_t *scales_2 = b_ptr[l].scales + (k / 4) * 64 + 32;\n const uint8_t *scales_3 = b_ptr[l].scales + (k / 4) * 64 + 48;\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi3 = 0;\n sumi4 = 0;\n sumi = 0;\n int offset = ((k / 2) % 2) + j * 2;\n for (int i = 0; i < blocklen; ++i){\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 3);\n const int v1 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 2 ) & 3);\n const int v2 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4 ) & 3);\n const int v3 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 6 ) & 3);\n sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 512 + (k % 4) * 4 * blocklen + m * blocklen + i]);\n sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 512 + (k % 4) * 4 * blocklen + m * blocklen + i + 128]);\n sumi3 = (v2 * a_ptr[l].qs[(k >> 2) * 512 + (k % 4) * 4 * blocklen + m * blocklen + i + 256]);\n sumi4 = (v3 * a_ptr[l].qs[(k >> 2) * 512 + (k % 4) * 4 * blocklen + m * blocklen + i + 384]);\n sumi1 = sumi1 * (scales_0[offset] & 0xF);\n sumi2 = sumi2 * (scales_1[offset] & 0xF);\n sumi3 = sumi3 * (scales_2[offset] & 0xF);\n sumi4 = sumi4 * (scales_3[offset] & 0xF);\n sumi += sumi1 + sumi2 + sumi3 + sumi4;\n }\n sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n }\n }\n }\n for(int sb = 0; sb < 8; sb++) {\n const uint8_t *mins = b_ptr[l].scales + sb * 16;\n for(int m = 0; m < 4; m++) {\n const int16_t *bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) * 6);\n for(int j = 0; j < ncols_interleaved; j++) {\n int mins_prod = ((mins[j * 2] >> 4) * bsums[0] + (mins[(j * 2)+ 1] >> 4) * bsums[1]);\n sum_minf[m][j] += (mins_prod) * GGML_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n }\n }\n }\n }\n\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n }\n }\n }\n }\n}\n\nvoid ggml_gemm_q5_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n ggml_gemm_q5_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n ggml_gemm_q5_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n ggml_gemm_q6_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n ggml_gemm_q6_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n {\n float sumf[4][4];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx4 * b_ptr = (const block_iq4_nlx4 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4]));\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++)\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n }\n}\n\nvoid ggml_gemm_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n float sumf[4][8];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx8 * b_ptr = (const block_iq4_nlx8 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4]));\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++)\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n}\n\nvoid ggml_gemm_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n float sumf[4][4];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4]));\n }\n sumf[m][j] += sumi * GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++)\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n}\n\nvoid ggml_gemm_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 8;\n const int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n float sumf[4][8];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_mxfp4x8 * b_ptr = (const block_mxfp4x8 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4]));\n }\n sumf[m][j] += sumi * GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++)\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n}\n\nvoid ggml_gemm_q8_0_4x4_q8_0_generic(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 4;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n float sumf[4][4];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[m][j] = 0.0;\n }\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / blocklen); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n sumi += v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i];\n }\n sumf[m][j] +=\n sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n }\n}\n\n\n\nvoid ggml_gemm_q8_0_4x8_q8_0_generic(int n,\n float * GGML_RESTRICT s,\n size_t bs,\n const void * GGML_RESTRICT vx,\n const void * GGML_RESTRICT vy,\n int nr,\n int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 4;\n const int blocklen = 8;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n float sumf[4][4];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[m][j] = 0.0;\n }\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / blocklen); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n sumi += v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i];\n }\n sumf[m][j] +=\n sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n }\n}\n\n#if defined __riscv_zvfh\nvoid ggml_gemm_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[4][16];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_0x16 * b_ptr = (const block_q4_0x16 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++)\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n}\n\nvoid ggml_gemm_q4_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK_K;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert (n % qk == 0);\n assert (nr % 4 == 0);\n assert (nc % ncols_interleaved == 0);\n\n UNUSED(s);\n UNUSED(bs);\n UNUSED(vx);\n UNUSED(vy);\n UNUSED(nr);\n UNUSED(nc);\n UNUSED(nb);\n UNUSED(ncols_interleaved);\n UNUSED(blocklen);\n\n float sumf[4][16];\n float sum_minf[4][16];\n uint8_t scales[128];\n uint8_t mins[128];\n int sumi1;\n int sumi2;\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q4_Kx16 * b_ptr = (const block_q4_Kx16 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[m][j] = 0.0;\n sum_minf[m][j] = 0.0;\n }\n }\n for (int l = 0; l < nb; l++) {\n for (int i = 0; i < 128; i++) {\n scales[i] = b_ptr[l].scales[i] & 0x0F;\n mins[i] = b_ptr[l].scales[i] >> 4;\n }\n for (int i = 0; i < 64; i++) {\n scales[i] |= (b_ptr[l].scales[128 + i] & 0x03) << 4;\n mins[i] |= (b_ptr[l].scales[128 + i] & 0x0C) << 2;\n scales[i + 64] |= (b_ptr[l].scales[128 + i] & 0x30);\n mins[i + 64] |= (b_ptr[l].scales[128 + i] & 0xC0) >> 2;\n }\n\n for (int sb = 0; sb < 8; sb++) {\n uint8_t *min = &mins[sb * 16];\n for(int m = 0; m < 4; m++) {\n const int16_t bsums = a_ptr[l].bsums[sb * 8 + m] + a_ptr[l].bsums[sb * 8 + m + 4];\n for(int j = 0; j < ncols_interleaved; j++) {\n sum_minf[m][j] += min[j] * bsums * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n }\n }\n }\n\n for (int sb = 0; sb < 8; sb += 2) {\n uint8_t *scales_0 = &scales[sb * 16];\n uint8_t *scales_1 = &scales[(sb + 1) * 16];\n\n for (int i = 0; i < QK4_0; i++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi1 = 0;\n sumi2 = 0;\n sumi = 0;\n\n const int v0 = (int8_t) (b_ptr[l].qs[sb * 256 + i * 16 + j] & 0xF);\n const int v1 = (int8_t) (b_ptr[l].qs[sb * 256 + i * 16 + j] >> 4);\n sumi1 = (v0 * a_ptr[l].qs[sb * 4 * 32 + i * 4 + m]);\n sumi2 = (v1 * a_ptr[l].qs[sb * 4 * 32 + 32 * 4 + i * 4 + m]);\n sumi1 = sumi1 * scales_0[j];\n sumi2 = sumi2 * scales_1[j];\n sumi += sumi1 + sumi2;\n\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n }\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n }\n }\n }\n }\n}\n\nvoid ggml_gemm_iq4_nl_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n float sumf[4][16];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_iq4_nlx16 * b_ptr = (const block_iq4_nlx16 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + (qk / 2) * 4]));\n }\n sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++)\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n}\n\nvoid ggml_gemm_q8_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n const int qk = QK8_0;\n const int nb = n / qk;\n const int ncols_interleaved = 16;\n const int blocklen = 1;\n\n assert(n % qk == 0);\n assert(nr % 4 == 0);\n assert(nc % ncols_interleaved == 0);\n\n float sumf[4][16];\n int sumi;\n\n for (int y = 0; y < nr / 4; y++) {\n const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n for (int x = 0; x < nc / ncols_interleaved; x++) {\n const block_q8_0x16 * b_ptr = (const block_q8_0x16 *) vx + (x * nb);\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumf[m][j] = 0.0;\n }\n }\n for (int l = 0; l < nb; l++) {\n for (int k = 0; k < (qk / blocklen); k++) {\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n sumi = 0;\n for (int i = 0; i < blocklen; ++i) {\n const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n sumi += v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i];\n }\n sumf[m][j] +=\n sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n }\n }\n }\n }\n for (int m = 0; m < 4; m++) {\n for (int j = 0; j < ncols_interleaved; j++) {\n s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n }\n }\n }\n }\n}\n\n\nvoid ggml_gemm_q2_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n assert(n % QK_K == 0);\n assert(nr % 4 == 0);\n assert(nc % 16 == 0);\n const int nb = n / QK_K;\n const block_q2_Kx16 * x = (const block_q2_Kx16 *)vx;\n const block_q8_Kx4 * y = (const block_q8_Kx4 *)vy;\n\n const int sb_perm[16] = {\n 0, 4, 1, 5, 2, 6, 3, 7,\n 8, 12, 9, 13, 10, 14, 11, 15\n };\n\n // Iterate Rows in tiles of 4\n for (int row_tile = 0; row_tile < nr; row_tile += 4) {\n // Iterate Columns in tiles of 16\n for (int col_tile = 0; col_tile < nc; col_tile += 16) {\n\n const block_q2_Kx16 * x_ptr = x + (col_tile / 16) * nb;\n const block_q8_Kx4 * y_ptr = y + (row_tile / 4) * nb;\n\n float sumf[4][16];\n memset(sumf, 0, sizeof(sumf));\n\n for (int k_block = 0; k_block < nb; ++k_block) {\n int32_t isum[4][16];\n int32_t summs[4][16];\n memset(isum, 0, sizeof(isum));\n memset(summs, 0, sizeof(summs));\n\n const uint8_t * qs_rhs = x_ptr[k_block].qs;\n const uint8_t * sc_rhs = x_ptr[k_block].scales;\n const int8_t * qs_lhs = y_ptr[k_block].qs;\n const int16_t * bs_lhs = y_ptr[k_block].bsums;\n\n for (int sb = 0; sb < 16; ++sb) {\n int scale_offset = sb_perm[sb] * 16;\n\n int byte_base;\n if (sb < 8) byte_base = (sb % 2 == 0) ? 0 : 16;\n else byte_base = (sb % 2 == 0) ? 32 : 48;\n int shift = ((sb / 2) % 4) * 2;\n\n for (int col = 0; col < 16; ++col) {\n uint8_t sc_val = sc_rhs[scale_offset + col];\n int32_t d_sb = sc_val & 0xF;\n int32_t m_sb = sc_val >> 4;\n\n // Correction Term\n for (int r = 0; r < 4; ++r) {\n int bsum_idx = (sb / 4) * 16 + r * 4 + (sb % 4);\n summs[r][col] += bs_lhs[bsum_idx] * m_sb;\n }\n\n // Main Dot Product\n for (int l = 0; l < 16; ++l) {\n int qs_idx = (byte_base + l) * 16 + col;\n uint8_t q2_val = (qs_rhs[qs_idx] >> shift) & 3;\n\n // Calculate Q8 index for this specific k and row\n int k = sb * 16 + l;\n int q8_idx = (k / 4) * 16 + (k % 4);\n\n for (int r = 0; r < 4; ++r) {\n // Add r*4 to jump to the correct row within the 4x4 chunk\n int8_t q8_val = qs_lhs[q8_idx + r * 4];\n isum[r][col] += q8_val * q2_val * d_sb;\n }\n }\n }\n }\n\n // Finalize K-Block\n for (int col = 0; col < 16; ++col) {\n float d_rhs = GGML_FP16_TO_FP32(x_ptr[k_block].d[col]);\n float dm_rhs = GGML_FP16_TO_FP32(x_ptr[k_block].dmin[col]);\n\n for (int r = 0; r < 4; ++r) {\n float d_lhs = y_ptr[k_block].d[r];\n float d_all = d_lhs * d_rhs;\n float d_min = d_lhs * dm_rhs;\n sumf[r][col] += (isum[r][col] * d_all) - (summs[r][col] * d_min);\n }\n }\n }\n\n for (int r = 0; r < 4; ++r) {\n for (int col = 0; col < 16; ++col) {\n s[(row_tile + r) * bs + (col_tile + col)] = sumf[r][col];\n }\n }\n }\n }\n}\n#endif\n\n} // extern \"C\"\n\nstatic block_q8_0x4 make_block_q8_0x4(block_q8_0 * in, unsigned int blck_size_interleave) {\n block_q8_0x4 out;\n\n for (int i = 0; i < 4; i++) {\n out.d[i] = in[i].d;\n }\n\n const int end = QK8_0 * 4 / blck_size_interleave;\n for (int i = 0; i < end; ++i) {\n int src_id = i % 4;\n int src_offset = (i / 4) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], blck_size_interleave);\n }\n return out;\n}\n\nstatic block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave) {\n block_q4_0x4 out;\n\n for (int i = 0; i < 4; i++) {\n out.d[i] = in[i].d;\n }\n\n const int end = QK4_0 * 2 / blck_size_interleave;\n\n if (blck_size_interleave == 8) {\n const uint64_t xor_mask = 0x8888888888888888ULL;\n for (int i = 0; i < end; ++i) {\n int src_id = i % 4;\n int src_offset = (i / 4) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n uint64_t elems;\n // Using memcpy to avoid unaligned memory accesses\n memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t));\n elems ^= xor_mask;\n memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t));\n }\n } else if (blck_size_interleave == 4) {\n const uint32_t xor_mask = 0x88888888;\n for (int i = 0; i < end; ++i) {\n int src_id = i % 4;\n int src_offset = (i / 4) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n uint32_t elems;\n memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint32_t));\n elems ^= xor_mask;\n memcpy(&out.qs[dst_offset], &elems, sizeof(uint32_t));\n }\n } else {\n GGML_ASSERT(false);\n }\n\n return out;\n}\n\n// interleave 8 block_q4_0s in blocks of blck_size_interleave\n// returns an interleaved block_q4_0x8\n// in the interleaved block_q4_0x8, place deltas for 8 block_q4_0 blocks\n// first, then interleave quants from 8 block_q4_0s in blocks of blck_size_interleave\nstatic block_q4_0x8 make_block_q4_0x8(block_q4_0 * in, unsigned int blck_size_interleave) {\n block_q4_0x8 out;\n\n for (int i = 0; i < 8; i++) {\n out.d[i] = in[i].d;\n }\n\n const int end = QK4_0 * 4 / blck_size_interleave;\n const uint64_t xor_mask = 0x8888888888888888ULL;\n\n for (int i = 0; i < end; ++i) {\n int src_id = i % 8;\n int src_offset = (i / 8) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n uint64_t elems;\n memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t));\n elems ^= xor_mask;\n memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t));\n }\n\n return out;\n}\n\nstatic block_q4_0x16 make_block_q4_0x16(block_q4_0 * in, unsigned int blck_size_interleave) {\n block_q4_0x16 out;\n\n for (int i = 0; i < 16; i++) {\n out.d[i] = in[i].d;\n }\n\n const int end = QK4_0 * 8 / blck_size_interleave;\n\n if (blck_size_interleave == 1) {\n const uint8_t xor_mask = 0x88;\n for (int i = 0; i < end; ++i) {\n int src_id = i % 16;\n int src_offset = i / 16;\n int dst_offset = i;\n\n out.qs[dst_offset] = in[src_id].qs[src_offset] ^ xor_mask;\n }\n } else {\n GGML_ASSERT(false);\n }\n\n return out;\n}\n\nstatic block_q4_Kx8 make_block_q4_Kx8(block_q4_K * in, unsigned int blck_size_interleave) {\n block_q4_Kx8 out;\n //Delta(scale) and dmin values of the eight Q4_K structures are copied onto the output interleaved structure\n for (int i = 0; i < 8; i++) {\n out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n }\n\n for (int i = 0; i < 8; i++) {\n out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n }\n\n const int end = QK_K * 4 / blck_size_interleave;\n\n // Interleave Q4_K quants by taking 8 bytes at a time\n for (int i = 0; i < end; ++i) {\n int src_id = i % 8;\n int src_offset = (i / 8) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n // buffer large enough for the max interleave block size (8 bytes)\n uint64_t elems;\n memcpy(&elems, &in[src_id].qs[src_offset], blck_size_interleave);\n memcpy(&out.qs[dst_offset], &elems, blck_size_interleave);\n }\n\n // The below logic is designed so as to unpack and rearrange scales and mins values in Q4_K\n // Currently the Q4_K structure has 8 scales and 8 mins packed in 12 bytes ( 6 bits for each value)\n // The output Q4_Kx8 structure has 96 bytes\n // Every 12 byte is packed such that it contains scales and mins for corresponding sub blocks from Q4_K structure\n // For eg - First 12 bytes contains 8 scales and 8 mins - each of first sub block from different Q4_K structures\n uint8_t s[8], m[8];\n\n for (int i = 0; i < 4; i++) {\n for (int j = 0; j < 8; j++) {\n s[j] = in[j].scales[i] & 63;\n m[j] = in[j].scales[i + 4] & 63;\n }\n\n out.scales[i * 12] = (s[0] & 63) + ((s[4] & 48) << 2);\n out.scales[i * 12 + 1] = (s[1] & 63) + ((s[5] & 48) << 2);\n out.scales[i * 12 + 2] = (s[2] & 63) + ((s[6] & 48) << 2);\n out.scales[i * 12 + 3] = (s[3] & 63) + ((s[7] & 48) << 2);\n out.scales[i * 12 + 4] = (m[0] & 63) + ((m[4] & 48) << 2);\n out.scales[i * 12 + 5] = (m[1] & 63) + ((m[5] & 48) << 2);\n out.scales[i * 12 + 6] = (m[2] & 63) + ((m[6] & 48) << 2);\n out.scales[i * 12 + 7] = (m[3] & 63) + ((m[7] & 48) << 2);\n out.scales[i * 12 + 8] = (s[4] & 15) + ((m[4] & 15) << 4);\n out.scales[i * 12 + 9] = (s[5] & 15) + ((m[5] & 15) << 4);\n out.scales[i * 12 + 10] = (s[6] & 15) + ((m[6] & 15) << 4);\n out.scales[i * 12 + 11] = (s[7] & 15) + ((m[7] & 15) << 4);\n\n }\n\n for (int i = 0; i < 4; i++) {\n for (int j = 0; j < 8; j++) {\n s[j] = ((in[j].scales[i] & 192) >> 2) | (in[j].scales[i+8] & 15);\n m[j] = ((in[j].scales[i + 4] & 192) >> 2) | ((in[j].scales[i+8] & 240) >> 4);\n }\n\n out.scales[i * 12 + 48] = (s[0] & 63) + ((s[4] & 48) << 2);\n out.scales[i * 12 + 49] = (s[1] & 63) + ((s[5] & 48) << 2);\n out.scales[i * 12 + 50] = (s[2] & 63) + ((s[6] & 48) << 2);\n out.scales[i * 12 + 51] = (s[3] & 63) + ((s[7] & 48) << 2);\n out.scales[i * 12 + 52] = (m[0] & 63) + ((m[4] & 48) << 2);\n out.scales[i * 12 + 53] = (m[1] & 63) + ((m[5] & 48) << 2);\n out.scales[i * 12 + 54] = (m[2] & 63) + ((m[6] & 48) << 2);\n out.scales[i * 12 + 55] = (m[3] & 63) + ((m[7] & 48) << 2);\n out.scales[i * 12 + 56] = (s[4] & 15) + ((m[4] & 15) << 4);\n out.scales[i * 12 + 57] = (s[5] & 15) + ((m[5] & 15) << 4);\n out.scales[i * 12 + 58] = (s[6] & 15) + ((m[6] & 15) << 4);\n out.scales[i * 12 + 59] = (s[7] & 15) + ((m[7] & 15) << 4);\n\n }\n\n return out;\n}\n\nstatic block_q4_Kx16 make_block_q4_Kx16(block_q4_K * in, unsigned int blck_size_interleave) {\n block_q4_Kx16 out;\n //Delta(scale) and dmin values of the 16 Q4_K structures are copied onto the output interleaved structure\n for (int i = 0; i < 16; i++) {\n out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n }\n\n for (int i = 0; i < 16; i++) {\n out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n }\n\n const int end = QK_K * 8 / blck_size_interleave;\n\n if (blck_size_interleave == 1) {\n for (int i = 0; i < end; ++i) {\n int src_id = i % 16;\n int src_offset = i / 16;\n int dst_offset = i;\n\n out.qs[dst_offset] = in[src_id].qs[src_offset];\n }\n\n // RVV repacking.\n //\n // Extract sums and mins for all 8 sub-blocks for each block of Q4_K.\n uint8_t s[128], m[128];\n for (int i = 0; i < 4; i++) {\n for (int j = 0; j < 16; j++) {\n s[i * 16 + j] = in[j].scales[i] & 63;\n m[i * 16 + j] = in[j].scales[i + 4] & 63;\n }\n }\n for (int i = 0; i < 4; i++) {\n for (int j = 0; j < 16; j++) {\n s[64 + i * 16 + j] = ((in[j].scales[i] & 192) >> 2) | (in[j].scales[i+8] & 15);\n m[64 + i * 16 + j] = ((in[j].scales[i + 4] & 192) >> 2) | ((in[j].scales[i+8] & 240) >> 4);\n }\n }\n\n for (int i = 0; i < 128; i++) {\n out.scales[i] = (s[i] & 15) | ((m[i] & 15) << 4);\n }\n for (int i = 0; i < 64; i++) {\n out.scales[128 + i] = ((s[i] & 48) >> 4) | ((m[i] & 48) >> 2) | (s[64 + i] & 48) | ((m[64 + i] & 48) << 2);\n }\n } else {\n GGML_ASSERT(false);\n }\n\n return out;\n}\n\nstatic block_q2_Kx8 make_block_q2_Kx8(block_q2_K * in, unsigned int blck_size_interleave) {\n block_q2_Kx8 out;\n\n // Delta(scale) and dmin values of the eight Q2_K structures are copied onto the output interleaved structure\n for (int i = 0; i < 8; i++) {\n out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n }\n\n for (int i = 0; i < 8; i++) {\n out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n }\n\n const int end = QK_K * 2 / blck_size_interleave;\n\n // Interleave Q2_K quants by taking 8 bytes at a time\n for (int i = 0; i < end; ++i) {\n int src_id = i % 8;\n int src_offset = (i / 8) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n uint64_t elems;\n memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t));\n memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t));\n }\n\n // The below logic is designed so as to unpack and rearrange scales and mins values in Q2_K\n // Currently the Q2_K structure has 16 scales and 16 mins packed in 16 bytes ( 4 bits for each value)\n // The output Q2_Kx8 structure has 128 bytes for storing scales and mins\n // Every 16 byte is packed such that it contains scales and mins for corresponding sub blocks from Q2_K structure\n // For eg - First 16 bytes contains 16 scales and 16 mins - each of first and second sub blocks from different Q2_K structures\n\n for (int i = 0; i < 128; i++) {\n // Index for selecting which q2k super block\n int src1 = (i % 16) / 2;\n // Index for selecting scale\n int src2 = ((i / 16) * 2) + (i % 2);\n\n out.scales[i] = in[src1].scales[src2];\n }\n return out;\n}\n\nstatic block_q5_Kx8 make_block_q5_Kx8(block_q5_K * in, unsigned int blck_size_interleave) {\n block_q5_Kx8 out;\n //Delta(scale) and dmin values of the eight Q5_K structures are copied onto the output interleaved structure\n for (int i = 0; i < 8; i++) {\n out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n }\n\n for (int i = 0; i < 8; i++) {\n out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n }\n\n const int end = QK_K * 4 / blck_size_interleave;\n\n // Interleave Q5_K quants by taking blck_size_interleave bytes at a time\n for (int i = 0; i < end; ++i) {\n int src_id = i % 8;\n int src_offset = (i / 8) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], blck_size_interleave);\n }\n\n // Repeat for high bits with the same chunk size, since\n // the high bits are interleaved in Q5_K and the index is\n // qh_idx = (qs_idx % 32);\n // qh_val = qh[qh_idx] >> (qs_idx / 32);\n for (int i = 0; i < end / 4; ++i) {\n int src_id = i % 8;\n int src_offset = (i / 8) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n memcpy(&out.qh[dst_offset], &in[src_id].qh[src_offset], blck_size_interleave);\n }\n\n // The below logic is copied over from Q4_K\n // The point is to unpack all the scales and mins for each sub block every time we load 12 bytes.\n // Currently the Q5_K structure has 8 scales and 8 mins packed in 12 bytes ( 6 bits for each value)\n // The output Q5_Kx8 structure has 96 bytes\n // Every 12 byte is packed such that it contains scales and mins for corresponding sub blocks from Q5_K structure\n // For eg - First 12 bytes contains 8 scales and 8 mins - each of first sub block from different Q5_K structures\n uint8_t s[8], m[8];\n\n for (int i = 0; i < 4; i++) {\n for (int j = 0; j < 8; j++) {\n s[j] = in[j].scales[i] & 63;\n m[j] = in[j].scales[i + 4] & 63;\n }\n\n out.scales[i * 12] = (s[0] & 63) + ((s[4] & 48) << 2);\n out.scales[i * 12 + 1] = (s[1] & 63) + ((s[5] & 48) << 2);\n out.scales[i * 12 + 2] = (s[2] & 63) + ((s[6] & 48) << 2);\n out.scales[i * 12 + 3] = (s[3] & 63) + ((s[7] & 48) << 2);\n out.scales[i * 12 + 4] = (m[0] & 63) + ((m[4] & 48) << 2);\n out.scales[i * 12 + 5] = (m[1] & 63) + ((m[5] & 48) << 2);\n out.scales[i * 12 + 6] = (m[2] & 63) + ((m[6] & 48) << 2);\n out.scales[i * 12 + 7] = (m[3] & 63) + ((m[7] & 48) << 2);\n out.scales[i * 12 + 8] = (s[4] & 15) + ((m[4] & 15) << 4);\n out.scales[i * 12 + 9] = (s[5] & 15) + ((m[5] & 15) << 4);\n out.scales[i * 12 + 10] = (s[6] & 15) + ((m[6] & 15) << 4);\n out.scales[i * 12 + 11] = (s[7] & 15) + ((m[7] & 15) << 4);\n }\n\n for (int i = 0; i < 4; i++) {\n for (int j = 0; j < 8; j++) {\n s[j] = ((in[j].scales[i] & 192) >> 2) | (in[j].scales[i + 8] & 15);\n m[j] = ((in[j].scales[i + 4] & 192) >> 2) | ((in[j].scales[i + 8] & 240) >> 4);\n }\n\n out.scales[i * 12 + 48] = (s[0] & 63) + ((s[4] & 48) << 2);\n out.scales[i * 12 + 49] = (s[1] & 63) + ((s[5] & 48) << 2);\n out.scales[i * 12 + 50] = (s[2] & 63) + ((s[6] & 48) << 2);\n out.scales[i * 12 + 51] = (s[3] & 63) + ((s[7] & 48) << 2);\n out.scales[i * 12 + 52] = (m[0] & 63) + ((m[4] & 48) << 2);\n out.scales[i * 12 + 53] = (m[1] & 63) + ((m[5] & 48) << 2);\n out.scales[i * 12 + 54] = (m[2] & 63) + ((m[6] & 48) << 2);\n out.scales[i * 12 + 55] = (m[3] & 63) + ((m[7] & 48) << 2);\n out.scales[i * 12 + 56] = (s[4] & 15) + ((m[4] & 15) << 4);\n out.scales[i * 12 + 57] = (s[5] & 15) + ((m[5] & 15) << 4);\n out.scales[i * 12 + 58] = (s[6] & 15) + ((m[6] & 15) << 4);\n out.scales[i * 12 + 59] = (s[7] & 15) + ((m[7] & 15) << 4);\n }\n\n return out;\n}\n\nstatic block_q6_Kx8 make_block_q6_Kx8(block_q6_K * in, unsigned int blck_size_interleave) {\n block_q6_Kx8 out;\n constexpr int n_blocks = 8; // Kx8\n for (int i = 0; i < n_blocks; i++) {\n out.d[i] = in[i].d;\n }\n\n const int end_ls = QK_K * 4 / blck_size_interleave;\n // Interleave Q6_K quants by taking blck_size_interleave bytes at a time\n for (int i = 0; i < end_ls; ++i) {\n int src_id = i % n_blocks;\n int src_offset = (i / n_blocks) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n uint64_t elem_ls;\n memcpy(&elem_ls, &in[src_id].ql[src_offset], blck_size_interleave);\n memcpy(&out.ql[dst_offset], &elem_ls, blck_size_interleave);\n }\n\n // Interleave high bits using same chunk size as low bits\n const int end_hs = end_ls / 2;\n for (int i = 0; i < end_hs; ++i) {\n int src_id = i % n_blocks;\n int src_offset = (i / n_blocks) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n uint64_t elem_hs;\n memcpy(&elem_hs, &in[src_id].qh[src_offset], blck_size_interleave);\n memcpy(&out.qh[dst_offset], &elem_hs, blck_size_interleave);\n }\n\n // The below logic is designed so as to unpack and rearrange scales in Q6_K\n // The output Q6_Kx8 structure interleaves the 8 bit scales in the same fashion as the quants\n // Q6_K structure has an 8-bit scale per 16 elements -> 16 scales\n // scales: [0 bl0 0 bl1 ... 0 bl7][1 bl0 ... 1 bl7] ... [15 bl0 ... 15 bl7] (bl = block)\n constexpr int n_scales = QK_K / 16;\n\n for (int i = 0; i < n_blocks; i++) {\n for (int j = 0; j < n_scales; j++) {\n out.scales[j * n_blocks + i] = in[i].scales[j];\n }\n }\n\n return out;\n}\n\nstatic block_q2_Kx16 make_block_q2_Kx16(const block_q2_K * in, unsigned int blck_size_interleave) {\n block_q2_Kx16 out;\n constexpr int N_COLS = 16;\n\n // 1. Copy Super-Scales (d) and Super-Mins (dmin)\n for (int i = 0; i < N_COLS; i++) {\n out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n }\n\n // 2. Interleave Q2_K Data\n const int bytes_per_col = 64;\n const int total_bytes = N_COLS * bytes_per_col;\n const int end = total_bytes / blck_size_interleave;\n\n for (int i = 0; i < end; ++i) {\n int src_col_id = i % N_COLS;\n int src_offset = (i / N_COLS) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n memcpy(&out.qs[dst_offset], &in[src_col_id].qs[src_offset], blck_size_interleave);\n }\n\n // 3. Repack Scales into the Optimized \"Sequential-Parallel\" Layout\n int out_idx = 0;\n\n // Arrays define the sub-block order for each group\n const int even_low_sbs[] = {0, 2, 4, 6};\n const int odd_low_sbs[] = {1, 3, 5, 7};\n const int even_high_sbs[] = {8, 10, 12, 14};\n const int odd_high_sbs[] = {9, 11, 13, 15};\n\n // Pack Group 1: Even-Low\n for (int sb : even_low_sbs) {\n for (int col = 0; col < N_COLS; col++) {\n out.scales[out_idx++] = in[col].scales[sb];\n }\n }\n\n // Pack Group 2: Odd-Low\n for (int sb : odd_low_sbs) {\n for (int col = 0; col < N_COLS; col++) {\n out.scales[out_idx++] = in[col].scales[sb];\n }\n }\n\n // Pack Group 3: Even-High\n for (int sb : even_high_sbs) {\n for (int col = 0; col < N_COLS; col++) {\n out.scales[out_idx++] = in[col].scales[sb];\n }\n }\n\n // Pack Group 4: Odd-High\n for (int sb : odd_high_sbs) {\n for (int col = 0; col < N_COLS; col++) {\n out.scales[out_idx++] = in[col].scales[sb];\n }\n }\n\n return out;\n}\n\nstatic int repack_q4_0_to_q4_0_4_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q4_0);\n GGML_ASSERT(interleave_block == 4 || interleave_block == 8);\n constexpr int nrows_interleaved = 4;\n\n block_q4_0x4 * dst = (block_q4_0x4 *)t->data;\n const block_q4_0 * src = (const block_q4_0 *)data;\n block_q4_0 dst_tmp[4];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK4_0;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q4_0x4(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic int repack_q4_K_to_q4_K_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q4_K);\n GGML_ASSERT(interleave_block == 8 || interleave_block == 4);\n constexpr int nrows_interleaved = 8;\n\n block_q4_Kx8 * dst = (block_q4_Kx8*)t->data;\n const block_q4_K * src = (const block_q4_K*) data;\n block_q4_K dst_tmp[8];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK_K;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_K));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++ ) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q4_Kx8(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic int repack_q4_K_to_q4_K_16_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q4_K);\n constexpr int nrows_interleaved = 16;\n\n block_q4_Kx16 * dst = (block_q4_Kx16*)t->data;\n const block_q4_K * src = (const block_q4_K*) data;\n block_q4_K dst_tmp[16];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK_K;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_K));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++ ) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q4_Kx16(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic int repack_q2_K_to_q2_K_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q2_K);\n GGML_ASSERT(interleave_block == 8);\n constexpr int nrows_interleaved = 8;\n\n block_q2_Kx8 * dst = (block_q2_Kx8*)t->data;\n const block_q2_K * src = (const block_q2_K*) data;\n block_q2_K dst_tmp[8];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK_K;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q2_K));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q2_Kx8(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic int repack_q2_K_to_q2_K_16_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q2_K);\n constexpr int nrows_interleaved = 16;\n\n block_q2_Kx16 * dst = (block_q2_Kx16*)t->data;\n const block_q2_K * src = (const block_q2_K*) data;\n\n block_q2_K dst_tmp[nrows_interleaved];\n\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK_K;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q2_K));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n // This loop gathers 16 separate blocks (one from each column)\n // that correspond to the same K-dimension chunk.\n for (int i = 0; i < nrows_interleaved; i++ ) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n\n *dst++ = make_block_q2_Kx16(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic int repack_q4_0_to_q4_0_16_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q4_0);\n constexpr int nrows_interleaved = 16;\n\n block_q4_0x16 * dst = (block_q4_0x16*)t->data;\n const block_q4_0 * src = (const block_q4_0*) data;\n block_q4_0 dst_tmp[16];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK4_0;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++ ) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q4_0x16(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic int repack_q5_K_to_q5_K_8_bl(struct ggml_tensor * t,\n int interleave_block,\n const void * GGML_RESTRICT data,\n size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q5_K);\n GGML_ASSERT(interleave_block == 4 || interleave_block == 8);\n constexpr int nrows_interleaved = 8;\n\n block_q5_Kx8 * dst = (block_q5_Kx8 *) t->data;\n const block_q5_K * src = (const block_q5_K *) data;\n block_q5_K dst_tmp[8];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK_K;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q5_K));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q5_Kx8(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n}\n\nstatic int repack_q6_K_to_q6_K_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q6_K);\n GGML_ASSERT(interleave_block == 4 || interleave_block == 8);\n constexpr int nrows_interleaved = 8;\n\n block_q6_Kx8 * dst = (block_q6_Kx8 *)t->data;\n const block_q6_K * src = (const block_q6_K *) data;\n block_q6_K dst_tmp[8];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK_K;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q6_K));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q6_Kx8(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n}\n\nstatic int repack_q4_0_to_q4_0_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q4_0);\n GGML_ASSERT(interleave_block == 8);\n constexpr int nrows_interleaved = 8;\n\n block_q4_0x8 * dst = (block_q4_0x8*)t->data;\n const block_q4_0 * src = (const block_q4_0*) data;\n block_q4_0 dst_tmp[8];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK4_0;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++ ) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q4_0x8(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic int repack_q8_0_to_q8_0_4_bl(struct ggml_tensor * t,\n int interleave_block,\n const void * GGML_RESTRICT data,\n size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q8_0);\n GGML_ASSERT(interleave_block == 4 || interleave_block == 8);\n constexpr int nrows_interleaved = 4;\n\n block_q8_0x4 * dst = (block_q8_0x4 *) t->data;\n const block_q8_0 * src = (const block_q8_0 *) data;\n block_q8_0 dst_tmp[4];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK8_0;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q8_0));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q8_0x4(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n}\n\nstatic block_q8_0x16 make_block_q8_0x16(block_q8_0 * in, unsigned int blck_size_interleave) {\n block_q8_0x16 out;\n\n for (int i = 0; i < 16; i++) {\n out.d[i] = in[i].d;\n }\n\n const int end = QK8_0 * 16 / blck_size_interleave;\n\n if (blck_size_interleave == 1) {\n for (int i = 0; i < end; ++i) {\n int src_id = i % 16;\n int src_offset = i / 16;\n int dst_offset = i;\n out.qs[dst_offset] = in[src_id].qs[src_offset];\n }\n } else {\n GGML_ASSERT(false);\n }\n\n return out;\n}\n\nstatic int repack_q8_0_to_q8_0_16_bl(struct ggml_tensor * t,\n int interleave_block,\n const void * GGML_RESTRICT data,\n size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q8_0);\n constexpr int nrows_interleaved = 16;\n\n block_q8_0x16 * dst = (block_q8_0x16 *) t->data;\n const block_q8_0 * src = (const block_q8_0 *) data;\n block_q8_0 dst_tmp[16];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK8_0;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q8_0));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q8_0x16(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n}\n\nstatic block_iq4_nlx4 make_block_iq4_nlx4(block_iq4_nl * in, unsigned int blck_size_interleave) {\n block_iq4_nlx4 out;\n\n for (int i = 0; i < 4; i++) {\n out.d[i] = in[i].d;\n }\n\n const int end = QK4_NL * 2 / blck_size_interleave;\n\n // TODO: this branch seems wrong\n //if (blck_size_interleave == 8) {\n // for (int i = 0; i < end; ++i) {\n // int src_id = i % 4;\n // int src_offset = (i / 4) * blck_size_interleave;\n // int dst_offset = i * blck_size_interleave;\n\n // // Using memcpy to avoid unaligned memory accesses\n // memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint64_t));\n // }\n //} else\n if (blck_size_interleave == 4) {\n for (int i = 0; i < end; ++i) {\n int src_id = i % 4;\n int src_offset = (i / 4) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint32_t));\n }\n } else {\n GGML_ASSERT(false);\n }\n\n return out;\n}\n\nstatic int repack_iq4_nl_to_iq4_nl_4_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_IQ4_NL);\n GGML_ASSERT(interleave_block == 4);\n\n const block_iq4_nl * src = (const block_iq4_nl *)data;\n block_iq4_nlx4 * dst = ( block_iq4_nlx4 *)t->data;\n\n block_iq4_nl dst_tmp[4];\n\n int nrow = ggml_nrows(t);\n int nrows_interleaved = 4;\n int nblocks = t->ne[0] / QK4_NL;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_iq4_nl));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_iq4_nlx4(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic block_iq4_nlx8 make_block_iq4_nlx8(block_iq4_nl * in, unsigned int blck_size_interleave) {\n block_iq4_nlx8 out;\n\n for (int i = 0; i < 8; i++) {\n out.d[i] = in[i].d;\n }\n\n const int end = QK4_NL * 4 / blck_size_interleave;\n\n if (blck_size_interleave == 8) {\n for (int i = 0; i < end; ++i) {\n int src_id = i % 8;\n int src_offset = (i / 8) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint64_t));\n }\n } else {\n GGML_ASSERT(false);\n }\n\n return out;\n}\n\nstatic int repack_iq4_nl_to_iq4_nl_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_IQ4_NL);\n GGML_ASSERT(interleave_block == 8);\n\n const block_iq4_nl * src = (const block_iq4_nl *)data;\n block_iq4_nlx8 * dst = ( block_iq4_nlx8 *)t->data;\n\n block_iq4_nl dst_tmp[8];\n\n int nrow = ggml_nrows(t);\n int nrows_interleaved = 8;\n int nblocks = t->ne[0] / QK4_NL;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_iq4_nl));\n\n if (t->ne[1] % nrows_interleaved != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_iq4_nlx8(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic block_iq4_nlx16 make_block_iq4_nlx16(block_iq4_nl * in, unsigned int blck_size_interleave) {\n block_iq4_nlx16 out;\n\n for (int i = 0; i < 16; i++) {\n out.d[i] = in[i].d;\n }\n\n const int end = QK4_NL * 8 / blck_size_interleave;\n\n if (blck_size_interleave == 1) {\n for (int i = 0; i < end; ++i) {\n int src_id = i % 16;\n int src_offset = i / 16;\n int dst_offset = i;\n\n out.qs[dst_offset] = in[src_id].qs[src_offset];\n }\n } else {\n GGML_ASSERT(false);\n }\n\n return out;\n}\n\nstatic int repack_iq4_nl_to_iq4_nl_16_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_IQ4_NL);\n GGML_ASSERT(interleave_block == 1);\n\n const block_iq4_nl * src = (const block_iq4_nl *)data;\n block_iq4_nlx16 * dst = ( block_iq4_nlx16 *)t->data;\n\n block_iq4_nl dst_tmp[16];\n\n int nrow = ggml_nrows(t);\n int nrows_interleaved = 16;\n int nblocks = t->ne[0] / QK4_NL;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_iq4_nl));\n\n if (t->ne[1] % nrows_interleaved != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_iq4_nlx16(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic block_mxfp4x4 make_block_mxfp4x4(block_mxfp4 * in, unsigned int blck_size_interleave) {\n block_mxfp4x4 out;\n\n for (int i = 0; i < 4; i++) {\n out.e[i] = in[i].e;\n }\n\n const int end = QK_MXFP4 * 2 / blck_size_interleave;\n\n if (blck_size_interleave == 4) {\n for (int i = 0; i < end; ++i) {\n int src_id = i % 4;\n int src_offset = (i / 4) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint32_t));\n }\n } else {\n GGML_ASSERT(false);\n }\n\n return out;\n}\n\nstatic int repack_mxfp4_to_mxfp4_4_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_MXFP4);\n GGML_ASSERT(interleave_block == 4);\n\n const block_mxfp4 * src = (const block_mxfp4 *)data;\n block_mxfp4x4 * dst = ( block_mxfp4x4 *)t->data;\n\n block_mxfp4 dst_tmp[4];\n\n int nrow = ggml_nrows(t);\n int nrows_interleaved = 4;\n int nblocks = t->ne[0] / QK_MXFP4;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_mxfp4));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_mxfp4x4(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic block_mxfp4x8 make_block_mxfp4x8(block_mxfp4 * in, unsigned int blck_size_interleave) {\n block_mxfp4x8 out;\n\n for (int i = 0; i < 8; i++) {\n out.e[i] = in[i].e;\n }\n\n const int end = QK_MXFP4 * 4 / blck_size_interleave;\n\n if (blck_size_interleave == 8) {\n for (int i = 0; i < end; ++i) {\n int src_id = i % 8;\n int src_offset = (i / 8) * blck_size_interleave;\n int dst_offset = i * blck_size_interleave;\n\n memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint64_t));\n }\n } else {\n GGML_ASSERT(false);\n }\n\n return out;\n}\n\nstatic int repack_mxfp4_to_mxfp4_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_MXFP4);\n GGML_ASSERT(interleave_block == 8);\n\n const block_mxfp4 * src = (const block_mxfp4 *)data;\n block_mxfp4x8 * dst = ( block_mxfp4x8 *)t->data;\n\n block_mxfp4 dst_tmp[8];\n\n int nrow = ggml_nrows(t);\n int nrows_interleaved = 8;\n int nblocks = t->ne[0] / QK_MXFP4;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_mxfp4));\n\n if (t->ne[1] % nrows_interleaved != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_mxfp4x8(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nnamespace ggml::cpu::repack {\n// repack\ntemplate \nint repack(struct ggml_tensor *, const void *, size_t);\n\n// TODO: generalise.\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_0_to_q4_0_4_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_0_to_q4_0_4_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_0_to_q4_0_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_K_to_q4_K_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_K_to_q4_K_8_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q2_K_to_q2_K_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q5_K_to_q5_K_8_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q5_K_to_q5_K_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q6_K_to_q6_K_8_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q6_K_to_q6_K_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_iq4_nl_to_iq4_nl_4_bl(t, 4, data, data_size);\n}\n\n// TODO: needs to be revisited\n//template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n// return repack_iq4_nl_to_iq4_nl_4_bl(t, 8, data, data_size);\n//}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_iq4_nl_to_iq4_nl_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_mxfp4_to_mxfp4_4_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_mxfp4_to_mxfp4_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q8_0_to_q8_0_4_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q8_0_to_q8_0_4_bl(t, 8, data, data_size);\n}\n\n#if defined __riscv_zvfh\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_0_to_q4_0_16_bl(t, 1, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_K_to_q4_K_16_bl(t, 1, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_iq4_nl_to_iq4_nl_16_bl(t, 1, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q8_0_to_q8_0_16_bl(t, 1, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q2_K_to_q2_K_16_bl(t, 1, data, data_size);\n}\n#endif\n\n// gemv\ntemplate \nvoid gemv(int, float *, size_t, const void *, const void *, int, int);\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q4_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q4_0_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <>\nvoid gemv(int n,\n float * s,\n size_t bs,\n const void * vx,\n const void * vy,\n int nr,\n int nc) {\n ggml_gemv_q2_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q4_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q4_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q5_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q5_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q6_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q6_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_iq4_nl_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_mxfp4_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_mxfp4_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q8_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q8_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\n#if defined __riscv_zvfh\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q4_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q4_K_16x1_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_iq4_nl_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q8_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemv_q2_K_16x1_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n#endif\n\n// gemm\ntemplate \nvoid gemm(int, float *, size_t, const void *, const void *, int, int);\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q4_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <>\nvoid gemm(int n,\n float * s,\n size_t bs,\n const void * vx,\n const void * vy,\n int nr,\n int nc) {\n ggml_gemm_q4_0_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q2_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q4_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q4_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q5_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q5_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q6_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q6_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_iq4_nl_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_mxfp4_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_mxfp4_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q8_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q8_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\n#if defined __riscv_zvfh\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q4_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q4_K_16x1_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_iq4_nl_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q8_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n ggml_gemm_q2_K_16x1_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n#endif\n\nclass tensor_traits_base : public ggml::cpu::tensor_traits {\n public:\n virtual int repack(struct ggml_tensor * t, const void * data, size_t data_size) = 0;\n};\n\ntemplate class tensor_traits : public tensor_traits_base {\n\n bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n // not realy a GGML_TYPE_Q8_0 but same size.\n switch (op->op) {\n case GGML_OP_MUL_MAT:\n {\n size = ggml_row_size(PARAM_TYPE, ggml_nelements(op->src[1]));\n return true;\n }\n case GGML_OP_MUL_MAT_ID:\n {\n size = ggml_row_size(PARAM_TYPE, ggml_nelements(op->src[1]));\n size = GGML_PAD(size, sizeof(int64_t)); // + padding for next block.\n\n const int64_t ne02 = op->src[0]->ne[2]; // n_as, n_expert\n const int64_t ne12 = op->src[1]->ne[2]; // n_tokens\n\n const size_t sizeof_mmid_row_mapping = sizeof(int64_t);\n\n size += sizeof_mmid_row_mapping*ne02*(ne12 + 1);\n\n return true;\n }\n default:\n // GGML_ABORT(\"fatal error\");\n break;\n }\n return false;\n }\n\n bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override {\n switch (op->op) {\n case GGML_OP_MUL_MAT:\n forward_mul_mat(params, op);\n return true;\n case GGML_OP_MUL_MAT_ID:\n forward_mul_mat_id(params, op);\n return true;\n default:\n // GGML_ABORT(\"fatal error\");\n break;\n }\n return false;\n }\n\n void forward_mul_mat_one_chunk(ggml_compute_params * params,\n ggml_tensor * op,\n int64_t src0_start,\n int64_t src0_end,\n int64_t src1_start,\n int64_t src1_end) {\n const ggml_tensor * src0 = op->src[0];\n const ggml_tensor * src1 = op->src[1];\n ggml_tensor * dst = op;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const size_t src1_col_stride = ggml_row_size(PARAM_TYPE, ne10);\n\n GGML_ASSERT(ne03 == 1 && ne13 == 1);\n GGML_ASSERT(ne12 % ne02 == 0);\n const int64_t r2 = ne12 / ne02;\n\n const int64_t i12 = src1_start / ne1;\n const int64_t i11 = src1_start - i12 * ne1;\n\n // Determine batch index\n const int64_t i02 = i12 / r2;\n\n const int64_t i1 = i11;\n const int64_t i2 = i12;\n\n const char * src0_ptr = (const char *) src0->data + i02 * nb02;\n const char * src1_ptr = (const char *) params->wdata + (i11 + i12 * ne11) * src1_col_stride;\n char * dst_ptr = ((char *) dst->data + (i1 * nb1 + i2 * nb2));\n\n const int64_t nrows = src1_end - src1_start;\n const int64_t ncols = src0_end - src0_start;\n\n GGML_ASSERT(src1_ptr + src1_col_stride * nrows <= (const char *) params->wdata + params->wsize);\n\n // If there are more than three rows in src1, use gemm; otherwise, use gemv.\n if (nrows > 3) {\n gemm(ne00, (float *) (dst_ptr) + src0_start, nb1 / nb0,\n src0_ptr + src0_start * nb01, src1_ptr,\n nrows - (nrows % 4), ncols);\n }\n for (int iter = nrows - (nrows % 4); iter < nrows; iter++) {\n gemv(ne00, (float *) (dst_ptr + (iter * nb1)) + src0_start,\n ne01, src0_ptr + src0_start * nb01,\n src1_ptr + (src1_col_stride * iter), 1 /* nrows */, ncols);\n }\n }\n\n void forward_mul_mat(ggml_compute_params * params, ggml_tensor * op) {\n const ggml_tensor * src0 = op->src[0];\n const ggml_tensor * src1 = op->src[1];\n ggml_tensor * dst = op;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_ASSERT(ne0 == ne01);\n GGML_ASSERT(ne1 == ne11);\n GGML_ASSERT(ne2 == ne12);\n GGML_ASSERT(ne3 == ne13);\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n // TODO: General batched mul mat for 4D tensors\n // Currently only supports 3D tensors\n GGML_ASSERT(ne03 == 1);\n GGML_ASSERT(ne13 == 1);\n GGML_ASSERT(ne3 == 1);\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_n_dims(op->src[0]) == 2);\n // GGML_ASSERT(ggml_n_dims(op->src[1]) == 2);\n\n char * wdata = static_cast(params->wdata);\n const size_t nbw1 = ggml_row_size(PARAM_TYPE, ne10);\n const size_t nbw2 = nbw1 * ne11;\n\n assert(params->wsize >= nbw2 * ne12);\n\n const ggml_from_float_t from_float = ggml_get_type_traits_cpu(PARAM_TYPE)->from_float;\n\n // INFO: Quantization is done in planes to avoid extra complexity in chunking.\n // Flattening dimensions not multiple of INTER_SIZE would require extra handling depending on how\n // the planes are broadcast.\n for (int64_t i12 = 0; i12 < ne12; i12++) {\n char * data_ptr = (char *) src1->data + i12 * nb12;\n char * wdata_ptr = wdata + i12 * nbw2;\n\n for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) {\n ggml_quantize_mat_t((float *) (data_ptr + i11 * nb11),\n (void *) (wdata_ptr + i11 * nbw1), 4, ne10);\n }\n\n const int64_t i11_processed = ne11 - ne11 % 4;\n for (int64_t i11 = i11_processed + ith; i11 < ne11; i11 += nth) {\n from_float((float *) (data_ptr + i11 * nb11), (void *) (wdata_ptr + i11 * nbw1), ne10);\n }\n }\n\n // disable for NUMA\n const bool disable_chunking = ggml_is_numa();\n\n // 4x chunks per thread\n const int64_t nr0 = ggml_nrows(op->src[0]);\n\n int nth_scaled = nth * 4;\n int64_t chunk_size0 = (nr0 + nth_scaled - 1) / nth_scaled;\n int64_t nchunk0 = (nr0 + chunk_size0 - 1) / chunk_size0;\n\n // src1 is chunked only by full planes.\n // When we flatten we need to address dimensions not multiple of the q8 INTER_SIZE\n // to route them thorugh GEMV.\n // nchunk1 = ne12 also avoids messing the chunking for models with no 3d tensors\n // to avoid affecting their performance\n int64_t nchunk1 = ne12;\n\n // Ensure minimum chunk size to avoid alignment issues with high thread counts\n // Minimum chunk size should be at least NB_COLS to prevent overlapping chunks after alignment\n const int64_t min_chunk_size = NB_COLS;\n if (nchunk0 > 0 && (nr0 / nchunk0) < min_chunk_size && nr0 >= min_chunk_size) {\n nchunk0 = (nr0 + min_chunk_size - 1) / min_chunk_size;\n }\n\n int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;\n // Only increase nchunk0 to nth if it won't make chunks too small\n if (nth == 1 || ((nchunk0 < nth || disable_chunking) && (nr0 + nth - 1) / nth >= min_chunk_size)) {\n nchunk0 = nth;\n dr0 = (nr0 + nchunk0 - 1) / nchunk0;\n }\n\n // Ensure nchunk doesn't exceed the number of rows divided by minimum chunk size\n // This prevents creating too many tiny chunks that could overlap after alignment\n const int64_t max_nchunk = (nr0 + min_chunk_size - 1) / min_chunk_size;\n nchunk0 = MIN(nchunk0, max_nchunk);\n\n if (ith == 0) {\n // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.\n ggml_threadpool_chunk_set(params->threadpool, nth);\n }\n\n ggml_barrier(params->threadpool);\n\n // The first chunk comes from our thread_id, the rest will get auto-assigned.\n int current_chunk = ith;\n\n while (current_chunk < nchunk0 * nchunk1) {\n const int64_t ith0 = current_chunk % nchunk0;\n const int64_t ith1 = current_chunk / nchunk0;\n\n int64_t src0_start = dr0 * ith0;\n int64_t src0_end = MIN(src0_start + dr0, nr0);\n\n // full-plane range for src1\n int64_t src1_start = ith1 * ne11;\n int64_t src1_end = (ith1 + 1) * ne11;\n\n // Align boundaries to NB_COLS - round up to ensure all data is included\n // The chunk size limiting above ensures chunks are large enough to prevent overlaps\n src0_start = (src0_start % NB_COLS) ? src0_start + NB_COLS - (src0_start % NB_COLS) : src0_start;\n src0_end = (src0_end % NB_COLS) ? src0_end + NB_COLS - (src0_end % NB_COLS) : src0_end;\n src0_end = MIN(src0_end, ne01);\n\n // Make sure current plane is the last one before exiting\n if (src0_start >= src0_end) {\n current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);\n continue;\n }\n\n forward_mul_mat_one_chunk(params, dst, src0_start, src0_end, src1_start, src1_end);\n\n current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);\n }\n }\n\n void forward_mul_mat_id(ggml_compute_params * params, ggml_tensor * op) {\n const ggml_tensor * src0 = op->src[0];\n const ggml_tensor * src1 = op->src[1];\n const ggml_tensor * ids = op->src[2];\n ggml_tensor * dst = op;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n const ggml_from_float_t from_float = ggml_get_type_traits_cpu(PARAM_TYPE)->from_float;\n\n // we don't support permuted src0 or src1\n GGML_ASSERT(nb00 == ggml_type_size(src0->type));\n GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n // dst cannot be transposed or permuted\n GGML_ASSERT(nb0 == sizeof(float));\n GGML_ASSERT(nb0 <= nb1);\n GGML_ASSERT(nb1 <= nb2);\n GGML_ASSERT(nb2 <= nb3);\n\n GGML_ASSERT(ne03 == 1);\n GGML_ASSERT(ne13 == 1);\n GGML_ASSERT(ne3 == 1);\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n // row groups\n const int n_ids = ids->ne[0]; // n_expert_used\n const int n_as = ne02; // n_expert\n\n const size_t nbw1 = ggml_row_size(PARAM_TYPE, ne10);\n const size_t nbw2 = nbw1*ne11;\n const size_t nbw3 = nbw2*ne12;\n\n struct mmid_row_mapping {\n int32_t i1;\n int32_t i2;\n };\n\n GGML_ASSERT(params->wsize >=\n (GGML_PAD(nbw3, sizeof(int64_t)) +\n n_as*(ne12 + 1)*sizeof(mmid_row_mapping))\n );\n\n auto * wdata = (char *)params->wdata;\n auto * wdata_src1_end = (char *)wdata + GGML_PAD(nbw3, sizeof(int64_t));\n\n // total of [n_as][ne12 + 1] elements of type mmid_row_mapping (2*int32_t = int64_t)\n auto * matrix_row_counts = (int64_t *) (wdata_src1_end); // [n_as]\n struct mmid_row_mapping * matrix_rows = (struct mmid_row_mapping *) (matrix_row_counts + n_as); // [n_as][ne12]\n\n // src1: float32 => param type\n for (int64_t i12 = 0; i12 < ne12; ++i12) {\n for (int64_t i11 = ith; i11 < ne11; i11 += nth) {\n from_float((float *)((char *) src1->data + i12 * nb12 + i11 * nb11),\n (void *) (wdata + i12 * nbw2 + i11 * nbw1),\n ne10);\n }\n }\n\n#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * ne12 + (i1)]\n\n if (ith == 0) {\n // initialize matrix_row_counts\n memset(matrix_row_counts, 0, n_as * sizeof(int64_t));\n\n // group rows by src0 matrix\n for (int32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {\n for (int32_t id = 0; id < n_ids; ++id) {\n const int32_t i02 =\n *(const int32_t *) ((const char *) ids->data + iid1 * ids->nb[1] + id * ids->nb[0]);\n\n GGML_ASSERT(i02 >= 0 && i02 < n_as);\n\n MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = { id, iid1 };\n matrix_row_counts[i02] += 1;\n }\n }\n }\n\n ggml_barrier(params->threadpool);\n\n // compute each matrix multiplication in sequence\n for (int cur_a = 0; cur_a < n_as; ++cur_a) {\n const int64_t cne1 = matrix_row_counts[cur_a];\n\n if (cne1 == 0) {\n continue;\n }\n\n const auto * src0_cur = (const char *) src0->data + cur_a*nb02;\n\n //const int64_t nr0 = ne01; // src0 rows\n const int64_t nr1 = cne1; // src1 rows\n\n int64_t src0_cur_start = (ith * ne01) / nth;\n int64_t src0_cur_end = ((ith + 1) * ne01) / nth;\n\n // Align boundaries to NB_COLS - round up to ensure all data is included\n src0_cur_start = (src0_cur_start % NB_COLS) ? src0_cur_start + NB_COLS - (src0_cur_start % NB_COLS) : src0_cur_start;\n src0_cur_end = (src0_cur_end % NB_COLS) ? src0_cur_end + NB_COLS - (src0_cur_end % NB_COLS) : src0_cur_end;\n if (src0_cur_end > ne01) {\n src0_cur_end = ne01;\n }\n\n if (src0_cur_start >= src0_cur_end) {\n return;\n }\n\n for (int ir1 = 0; ir1 < nr1; ir1++) {\n struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, ir1);\n\n const int id = row_mapping.i1; // selected expert index\n\n const int64_t i11 = id % ne11;\n const int64_t i12 = row_mapping.i2; // row index in src1\n\n const int64_t i1 = id; // selected expert index\n const int64_t i2 = i12; // row\n\n const auto * src1_col = (const char *) wdata + (i11 * nbw1 + i12 * nbw2);\n\n gemv(\n ne00, (float *) ((char *) dst->data + (i1 * nb1 + i2 * nb2)) + src0_cur_start, ne01,\n src0_cur + src0_cur_start * nb01, src1_col, 1, src0_cur_end - src0_cur_start);\n }\n }\n#undef MMID_MATRIX_ROW\n }\n\n int repack(struct ggml_tensor * t, const void * data, size_t data_size) override {\n GGML_LOG_DEBUG(\"%s: repack tensor %s with %s_%dx%d\\n\", __func__, t->name, ggml_type_name(t->type),\n (int) NB_COLS, (int) INTER_SIZE);\n return ggml::cpu::repack::repack(t, data, data_size);\n }\n};\n\n} // namespace ggml::cpu::repack\n\nstatic const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(const struct ggml_tensor * cur) {\n // instance for Q4\n static const ggml::cpu::repack::tensor_traits q4_0_4x4_q8_0;\n static const ggml::cpu::repack::tensor_traits q4_0_4x8_q8_0;\n static const ggml::cpu::repack::tensor_traits q4_0_8x8_q8_0;\n\n // instance for Q4_K\n static const ggml::cpu::repack::tensor_traits q4_K_8x4_q8_K;\n static const ggml::cpu::repack::tensor_traits q4_K_8x8_q8_K;\n\n // instance for Q5_K\n static const ggml::cpu::repack::tensor_traits q5_K_8x4_q8_K;\n static const ggml::cpu::repack::tensor_traits q5_K_8x8_q8_K;\n\n // instance for Q6_K\n static const ggml::cpu::repack::tensor_traits q6_K_8x4_q8_K;\n static const ggml::cpu::repack::tensor_traits q6_K_8x8_q8_K;\n\n // instance for Q2\n static const ggml::cpu::repack::tensor_traits q2_K_8x8_q8_K;\n\n // instance for IQ4\n static const ggml::cpu::repack::tensor_traits iq4_nl_4x4_q8_0;\n static const ggml::cpu::repack::tensor_traits iq4_nl_8x8_q8_0;\n\n // instance for MXFP4\n static const ggml::cpu::repack::tensor_traits mxfp4_4x4_q8_0;\n static const ggml::cpu::repack::tensor_traits mxfp4_8x8_q8_0;\n\n // instance for Q8_0\n static const ggml::cpu::repack::tensor_traits q8_0_4x4_q8_0;\n static const ggml::cpu::repack::tensor_traits q8_0_4x8_q8_0;\n\n // instances for RISC-V\n //\n // These implement outer-product style matrix multiplication kernels with\n // an interleave of 1.\n#if defined __riscv_zvfh\n static const ggml::cpu::repack::tensor_traits q4_0_16x1_q8_0;\n static const ggml::cpu::repack::tensor_traits q4_K_16x1_q8_K;\n static const ggml::cpu::repack::tensor_traits iq4_nl_16x1_q8_0;\n static const ggml::cpu::repack::tensor_traits q8_0_16x1_q8_0;\n static const ggml::cpu::repack::tensor_traits q2_K_16x1_q8_K;\n#endif\n\n if (cur->type == GGML_TYPE_Q4_0) {\n if (ggml_cpu_has_avx2() || (ggml_cpu_has_sve() && ggml_cpu_has_matmul_int8() && ggml_cpu_get_sve_cnt() == QK8_0)) {\n if (cur->ne[1] % 8 == 0) {\n return &q4_0_8x8_q8_0;\n }\n }\n if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n if (cur->ne[1] % 4 == 0) {\n return &q4_0_4x8_q8_0;\n }\n }\n if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n if (cur->ne[1] % 4 == 0) {\n return &q4_0_4x4_q8_0;\n }\n }\n if (ggml_cpu_has_riscv_v()) {\n #if defined __riscv_zvfh\n switch (__riscv_vlenb() * 8) {\n case 128: { break; } // TODO\n case 256: { if (cur->ne[1] % 16 == 0) { return &q4_0_16x1_q8_0; } break; }\n case 512: { break; } // TODO\n case 1024: { break; } // TODO\n default: { return nullptr; }\n }\n #endif\n }\n } else if (cur->type == GGML_TYPE_Q4_K) {\n if (ggml_cpu_has_avx2()) {\n if (cur->ne[1] % 8 == 0) {\n return &q4_K_8x8_q8_K;\n }\n }\n if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n if (cur->ne[1] % 8 == 0) {\n return &q4_K_8x8_q8_K;\n }\n }\n if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n if (cur->ne[1] % 8 == 0) {\n return &q4_K_8x4_q8_K;\n }\n }\n if (ggml_cpu_has_riscv_v()) {\n #if defined __riscv_zvfh\n switch (__riscv_vlenb() * 8) {\n case 128: { break; } // TODO\n case 256: { if (cur->ne[1] % 16 == 0) { return &q4_K_16x1_q8_K; } break; }\n case 512: { break; } // TODO\n case 1024: { break; } // TODO\n default: { return nullptr; }\n }\n #endif\n }\n } else if (cur->type == GGML_TYPE_Q2_K) {\n if (ggml_cpu_has_avx512()) {\n if (cur->ne[1] % 8 == 0) {\n return &q2_K_8x8_q8_K;\n }\n }\n if (ggml_cpu_has_riscv_v()) {\n #if defined __riscv_zvfh\n switch (__riscv_vlenb() * 8) {\n case 128: { break; } // TODO\n case 256: { if (cur->ne[1] % 16 == 0) { return &q2_K_16x1_q8_K; } break; }\n case 512: { break; } // TODO\n case 1024: { break; } // TODO\n default: { return nullptr; }\n }\n #endif\n }\n } else if (cur->type == GGML_TYPE_Q5_K) {\n if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n if (cur->ne[1] % 8 == 0) {\n return &q5_K_8x8_q8_K;\n }\n }\n if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n if (cur->ne[1] % 8 == 0) {\n return &q5_K_8x4_q8_K;\n }\n }\n } else if (cur->type == GGML_TYPE_Q6_K) {\n if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n if (cur->ne[1] % 8 == 0) {\n return &q6_K_8x8_q8_K;\n }\n }\n if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n if (cur->ne[1] % 8 == 0) {\n return &q6_K_8x4_q8_K;\n }\n }\n } else if (cur->type == GGML_TYPE_IQ4_NL) {\n if (ggml_cpu_has_avx2()) {\n if (cur->ne[1] % 8 == 0) {\n return &iq4_nl_8x8_q8_0;\n }\n }\n if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n if (cur->ne[1] % 4 == 0) {\n return &iq4_nl_4x4_q8_0;\n }\n }\n if (ggml_cpu_has_riscv_v()) {\n #if defined __riscv_zvfh\n switch (__riscv_vlenb() * 8) {\n case 128: { break; } // TODO\n case 256: { if (cur->ne[1] % 16 == 0) { return &iq4_nl_16x1_q8_0; } break; }\n case 512: { break; } // TODO\n case 1024: { break; } // TODO\n default: { return nullptr; }\n }\n #endif\n }\n } else if (cur->type == GGML_TYPE_MXFP4) {\n if (ggml_cpu_has_avx2()) {\n if (cur->ne[1] % 8 == 0) {\n return &mxfp4_8x8_q8_0;\n }\n }\n if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n if (cur->ne[1] % 4 == 0) {\n return &mxfp4_4x4_q8_0;\n }\n }\n } else if (cur->type == GGML_TYPE_Q8_0) {\n if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n if (cur->ne[1] % 4 == 0) {\n return &q8_0_4x8_q8_0;\n }\n }\n if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n if (cur->ne[1] % 4 == 0) {\n return &q8_0_4x4_q8_0;\n }\n }\n if (ggml_cpu_has_riscv_v()) {\n #if defined __riscv_zvfh\n switch (__riscv_vlenb() * 8) {\n case 128: { break; } // TODO\n case 256: { if (cur->ne[1] % 16 == 0) { return &q8_0_16x1_q8_0; } break; }\n case 512: { break; } // TODO\n case 1024: { break; } // TODO\n default: { return nullptr; }\n }\n #endif\n }\n }\n\n return nullptr;\n}\n\nstatic enum ggml_status ggml_backend_cpu_repack_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {\n tensor->extra = (void *) const_cast(ggml_repack_get_optimal_repack_type(tensor));\n\n GGML_UNUSED(buffer);\n return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_cpu_repack_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,\n const void * data, size_t offset, size_t size) {\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(size == ggml_nbytes(tensor));\n\n auto tensor_traits = (ggml::cpu::repack::tensor_traits_base *) tensor->extra;\n auto OK = tensor_traits->repack(tensor, data, size);\n\n GGML_ASSERT(OK == 0);\n GGML_UNUSED(buffer);\n}\n\nstatic const char * ggml_backend_cpu_repack_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n return \"CPU_REPACK\";\n\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_repack_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n\n if (buffer == nullptr) {\n return nullptr;\n }\n\n buffer->buft = buft;\n buffer->iface.init_tensor = ggml_backend_cpu_repack_buffer_init_tensor;\n buffer->iface.set_tensor = ggml_backend_cpu_repack_buffer_set_tensor;\n buffer->iface.get_tensor = nullptr;\n buffer->iface.cpy_tensor = nullptr;\n return buffer;\n}\n\nstatic size_t ggml_backend_cpu_repack_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n return TENSOR_ALIGNMENT;\n\n GGML_UNUSED(buft);\n}\n\nnamespace ggml::cpu::repack {\nclass extra_buffer_type : ggml::cpu::extra_buffer_type {\n bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {\n if ( op->op == GGML_OP_MUL_MAT &&\n op->src[0]->buffer &&\n (ggml_n_dims(op->src[0]) == 2) &&\n op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type() &&\n ggml_repack_get_optimal_repack_type(op->src[0])\n ) {\n if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {\n return false;\n }\n if (op->src[1]->type == GGML_TYPE_F32) {\n return true;\n }\n //if (op->src[1]->type == GGML_TYPE_Q8_0) {\n // return true;\n //}\n // may be possible if Q8_0 packed...\n } else if (op->op == GGML_OP_MUL_MAT_ID\n && op->src[0]->buffer\n && (ggml_n_dims(op->src[0]) == 3)\n && op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type()\n && ggml_repack_get_optimal_repack_type(op->src[0])\n ) {\n if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {\n return false;\n }\n if (op->src[1]->type == GGML_TYPE_F32) {\n return true;\n }\n //if (op->src[1]->type == GGML_TYPE_Q8_0) {\n // return true;\n //}\n }\n return false;\n }\n\n ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {\n if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_MUL_MAT_ID) {\n if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type()) {\n return (ggml::cpu::tensor_traits *) op->src[0]->extra;\n }\n }\n return nullptr;\n }\n};\n} // namespace ggml::cpu::repack\n\nggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void) {\n static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_repack = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_cpu_repack_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_cpu_repack_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cpu_repack_buffer_type_get_alignment,\n /* .get_max_size = */ nullptr, // defaults to SIZE_MAX\n /* .get_alloc_size = */ nullptr, // defaults to ggml_nbytes\n /* .is_host = */ nullptr,\n },\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n /* .context = */ new ggml::cpu::repack::extra_buffer_type(),\n };\n\n return &ggml_backend_cpu_buffer_type_repack;\n}\n" }, { "path": "src/ggml-cpu/repack.h", "content": "#pragma once\n\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n\n#include \"traits.h\"\n#include \"ggml.h\"\n\n// GGML internal header\n\nggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void);\n\ntemplate constexpr int QK_0() {\n if constexpr (K == 4) {\n return QK4_0;\n }\n if constexpr (K == 8) {\n return QK8_0;\n }\n return -1;\n}\n\ntemplate struct block {\n ggml_half d[N]; // deltas for N qK_0 blocks\n int8_t qs[(QK_0() * N * K) / 8]; // quants for N qK_0 blocks\n};\n\n// control size\nstatic_assert(sizeof(block<4, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 2, \"wrong block<4,4> size/padding\");\nstatic_assert(sizeof(block<4, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 4, \"wrong block<4,8> size/padding\");\nstatic_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 8, \"wrong block<4,16> size/padding\");\nstatic_assert(sizeof(block<8, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 4, \"wrong block<8,4> size/padding\");\nstatic_assert(sizeof(block<8, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 8, \"wrong block<8,8> size/padding\");\nstatic_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 16, \"wrong block<8,16> size/padding\");\n\nusing block_q4_0x4 = block<4, 4>;\nusing block_q4_0x8 = block<4, 8>;\nusing block_q4_0x16 = block<4, 16>;\nusing block_q8_0x4 = block<8, 4>;\nusing block_q8_0x8 = block<8, 8>;\nusing block_q8_0x16 = block<8, 16>;\n\nstruct block_q4_Kx8 {\n ggml_half d[8]; // super-block scale for quantized scales\n ggml_half dmin[8]; // super-block scale for quantized mins\n uint8_t scales[96]; // scales and mins, quantized with 6 bits\n uint8_t qs[1024]; // 4--bit quants\n};\n\nstatic_assert(sizeof(block_q4_Kx8) == sizeof(ggml_half) * 16 + K_SCALE_SIZE * 8 + QK_K * 4, \"wrong q4_K block size/padding\");\nstruct block_q4_Kx16 {\n ggml_half d[16]; // super-block scale for quantized scales\n ggml_half dmin[16]; // super-block scale for quantized mins\n uint8_t scales[192]; // scales and mins, quantized with 6 bits\n uint8_t qs[2048]; // 4--bit quants\n};\n\nstatic_assert(sizeof(block_q4_Kx16) == sizeof(ggml_half) * 32 + K_SCALE_SIZE * 16 + QK_K * 8, \"wrong q4_K block size/padding\");\nstruct block_q2_Kx8 {\n ggml_half d[8]; // super-block scale for quantized scales\n ggml_half dmin[8]; // super-block scale for quantized mins\n uint8_t scales[128]; // scales and mins, quantized with 4 bits\n uint8_t qs[512]; // 2--bit quants\n};\n\nstatic_assert(sizeof(block_q2_Kx8) == sizeof(ggml_half) * 16 + QK_K/2 + QK_K * 2, \"wrong q2_K block size/padding\");\nstruct block_q2_Kx16 {\n ggml_half d[16]; // Super-block scale for quantized scales\n ggml_half dmin[16]; // Super-block scale for quantized mins\n uint8_t scales[256]; // Sub-block scales (16 cols * 16 sub-blocks)\n uint8_t qs[1024]; // Data (16 cols * 64 bytes per block)\n};\nstatic_assert(sizeof(block_q2_Kx16) == sizeof(ggml_half) * 32 + QK_K + QK_K * 4, \"wrong q2_K block size/padding\");\n\nstruct block_q5_Kx8 {\n ggml_half d[8]; // super-block scale for quantized scales\n ggml_half dmin[8]; // super-block scale for quantized mins\n uint8_t scales[96]; // scales and mins, quantized with 6 bits\n uint8_t qh[QK_K * 8 / 8]; // high bits of 5-bit quants\n uint8_t qs[QK_K * 8 / 2]; // low bits of 5-bit quants (in groups of 4)\n};\n\nstatic_assert(sizeof(block_q5_Kx8) == sizeof(ggml_half) * 16 + K_SCALE_SIZE * 8 + QK_K * 5,\n \"wrong q5_K block size/padding\");\n\nstruct block_q6_Kx8 {\n ggml_half d[8];\n int8_t scales[QK_K / 16 * 8];\n uint8_t ql[QK_K / 2 * 8]; // low bits of 6-bit quants (groups of 2)\n uint8_t qh[QK_K / 4 * 8]; // high bits of 6-bit quants (groups of 4)\n};\n\nstatic_assert(sizeof(block_q6_Kx8) == sizeof(ggml_half) * 8 + QK_K / 16 * 8 + 3 * QK_K / 4 * 8,\n \"wrong q6_K block size/padding\");\n\nstruct block_q8_Kx4 {\n float d[4]; // delta\n int8_t qs[QK_K * 4]; // quants\n int16_t bsums[QK_K / 4]; // sum of quants in groups of 16\n};\n\nstatic_assert(sizeof(block_q8_Kx4) == sizeof(float) * 4 + QK_K * 4 + (QK_K / 4) * sizeof(int16_t), \"wrong q8_K block size/padding\");\n\nstruct block_iq4_nlx4 {\n ggml_half d[4]; // deltas for 4 iq4_nl blocks\n uint8_t qs[QK4_NL * 2]; // nibbles / quants for 4 iq4_nl blocks\n};\n\nstatic_assert(sizeof(block_iq4_nlx4) == 4 * sizeof(ggml_half) + QK4_NL * 2, \"wrong iq4_nlx4 block size/padding\");\n\nstruct block_iq4_nlx8 {\n ggml_half d[8]; // deltas for 8 iq4_nl blocks\n uint8_t qs[QK4_NL * 4]; // nibbles / quants for 8 iq4_nl blocks\n};\n\nstatic_assert(sizeof(block_iq4_nlx8) == 8 * sizeof(ggml_half) + QK4_NL * 4, \"wrong iq4_nlx8 block size/padding\");\n\nstruct block_iq4_nlx16 {\n ggml_half d[16]; // deltas for 16 iq4_nl blocks\n uint8_t qs[QK4_NL * 8]; // nibbles / quants for 16 iq4_nl blocks\n};\n\nstatic_assert(sizeof(block_iq4_nlx16) == 16 * sizeof(ggml_half) + QK4_NL * 8, \"wrong iq4_nlx16 block size/padding\");\nstruct block_mxfp4x4 {\n uint8_t e[4];\n uint8_t qs[QK_MXFP4 * 2];\n};\nstatic_assert(sizeof(block_mxfp4x4) == 4 + QK_MXFP4 * 2, \"wrong mxfp4x4 block size/padding\");\n\nstruct block_mxfp4x8 {\n uint8_t e[8];\n uint8_t qs[QK_MXFP4 * 4];\n};\nstatic_assert(sizeof(block_mxfp4x8) == 8 + QK_MXFP4 * 4, \"wrong mxfp4x8 block size/padding\");\n\n#if defined(__cplusplus)\nextern \"C\" {\n#endif\n\nvoid ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q5_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q5_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q6_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q6_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q5_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q5_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q6_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q6_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\n#if defined __riscv_zvfh\nvoid ggml_quantize_mat_q8_0_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_gemv_q4_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\n#endif\n\n// Native implementations\nvoid ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q5_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q5_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\n#if defined __riscv_zvfh\nvoid ggml_quantize_mat_q8_0_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_gemv_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q2_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q2_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\n#endif\n\n#if defined(__cplusplus)\n} // extern \"C\"\n#endif\n" }, { "path": "src/ggml-cpu/simd-gemm.h", "content": "#pragma once\n\n// Computes C[M x N] += A[M x K] * B[K x N]\n\n#include \"simd-mappings.h\"\n\n// TODO: add support for sizeless vector types\n#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic)\n\n// TODO: untested on avx512\n// These are in units of GGML_F32_EPR\n#if defined(__AVX512F__) || defined (__ARM_NEON__)\n static constexpr int GEMM_RM = 4;\n static constexpr int GEMM_RN = 4; // 16+4+1 = 25/32\n#elif defined(__AVX2__) || defined(__AVX__)\n static constexpr int GEMM_RM = 6;\n static constexpr int GEMM_RN = 2; // 12+2+1 = 15/16\n#else\n static constexpr int GEMM_RM = 2;\n static constexpr int GEMM_RN = 2;\n#endif\n\ntemplate \nstatic inline void simd_gemm_ukernel(\n float * GGML_RESTRICT C,\n const float * GGML_RESTRICT A,\n const float * GGML_RESTRICT B,\n int K, int N)\n{\n static constexpr int KN = GGML_F32_EPR;\n\n GGML_F32_VEC acc[RM][RN];\n for (int64_t i = 0; i < RM; i++) {\n for (int r = 0; r < RN; r++) {\n acc[i][r] = GGML_F32_VEC_LOAD(C + i * N + r * KN);\n }\n }\n\n for (int64_t kk = 0; kk < K; kk++) {\n GGML_F32_VEC Bv[RN];\n for (int r = 0; r < RN; r++) {\n Bv[r] = GGML_F32_VEC_LOAD(B + kk * N + r * KN);\n }\n for (int64_t i = 0; i < RM; i++) {\n GGML_F32_VEC p = GGML_F32_VEC_SET1(A[i * K + kk]);\n for (int r = 0; r < RN; r++) {\n acc[i][r] = GGML_F32_VEC_FMA(acc[i][r], Bv[r], p);\n }\n }\n }\n\n for (int64_t i = 0; i < RM; i++) {\n for (int r = 0; r < RN; r++) {\n GGML_F32_VEC_STORE(C + i * N + r * KN, acc[i][r]);\n }\n }\n}\n\n// C[M x N] += A[M x K] * B[K x N]\nstatic void simd_gemm(\n float * GGML_RESTRICT C,\n const float * GGML_RESTRICT A,\n const float * GGML_RESTRICT B,\n int M, int K, int N)\n{\n static constexpr int KN = GGML_F32_EPR;\n\n int64_t ii = 0;\n for (; ii + GEMM_RM <= M; ii += GEMM_RM) {\n int64_t jj = 0;\n for (; jj + GEMM_RN * KN <= N; jj += GEMM_RN * KN) {\n simd_gemm_ukernel(C + jj, A, B + jj, K, N);\n }\n for (; jj + KN <= N; jj += KN) {\n simd_gemm_ukernel(C + jj, A, B + jj, K, N);\n }\n for (; jj < N; jj++) {\n for (int64_t i = 0; i < GEMM_RM; i++) {\n float a = C[i * N + jj];\n for (int64_t kk = 0; kk < K; kk++) {\n a += A[i + kk] * B[kk * N + jj];\n }\n C[i * N + jj] = a;\n }\n }\n\n A += GEMM_RM * K;\n C += GEMM_RM * N;\n }\n\n // Tail rows: one at a time\n for (; ii < M; ii++) {\n int64_t jj = 0;\n for (; jj + GEMM_RN * KN <= N; jj += GEMM_RN * KN) {\n simd_gemm_ukernel<1, GEMM_RN>(C + jj, A, B + jj, K, N);\n }\n for (; jj + KN <= N; jj += KN) {\n simd_gemm_ukernel<1, 1>(C + jj, A, B + jj, K, N);\n }\n for (; jj < N; jj++) {\n float a = C[jj];\n for (int64_t kk = 0; kk < K; kk++) {\n a += A[kk] * B[kk * N + jj];\n }\n C[jj] = a;\n }\n\n A += K;\n C += N;\n }\n}\n\n#if defined(__GNUC__) && !defined(__clang__)\n#pragma GCC diagnostic pop\n#endif\n\n#else // scalar path\n\nstatic void simd_gemm(\n float * GGML_RESTRICT C,\n const float * GGML_RESTRICT A,\n const float * GGML_RESTRICT B,\n int M, int K, int N)\n{\n for (int64_t i = 0; i < M; i++) {\n for (int64_t j = 0; j < N; j++) {\n float sum = C[i * N + j];\n for (int64_t kk = 0; kk < K; kk++) {\n sum += A[i * K + kk] * B[kk * N + j];\n }\n C[i * N + j] = sum;\n }\n }\n}\n\n#endif // GGML_SIMD\n" }, { "path": "src/ggml-cpu/simd-mappings.h", "content": "#pragma once\n\n#include \"ggml-cpu-impl.h\"\n\n#ifdef __ARM_FEATURE_SVE\n#include \n#endif // __ARM_FEATURE_SVE\n\n#if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__)\n// if YCM cannot find , make a symbolic link to it, for example:\n//\n// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/\n//\n#include \n#endif\n\n#if defined(__riscv_v_intrinsic)\n#include \n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n//\n// simd mappings\n//\n\n// FP16 to FP32 conversion\n\n// 16-bit float\n// on Arm, we use __fp16\n// on x86, we use uint16_t\n//\n// for old CUDA compilers (<= 11), we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/10616\n// for MUSA compilers , we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/11843\n//\n#if defined(__ARM_NEON) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11) && !defined(__MUSACC__)\n #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) neon_compute_fp16_to_fp32(x)\n #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) neon_compute_fp32_to_fp16(x)\n\n #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)\n\n static inline float neon_compute_fp16_to_fp32(ggml_fp16_t h) {\n __fp16 tmp;\n memcpy(&tmp, &h, sizeof(ggml_fp16_t));\n return (float)tmp;\n }\n\n static inline ggml_fp16_t neon_compute_fp32_to_fp16(float f) {\n ggml_fp16_t res;\n __fp16 tmp = f;\n memcpy(&res, &tmp, sizeof(ggml_fp16_t));\n return res;\n }\n#elif defined(__F16C__)\n #ifdef _MSC_VER\n #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)))\n #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0)\n #else\n #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) _cvtsh_ss(x)\n #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) _cvtss_sh(x, 0)\n #endif\n#elif defined(__POWER9_VECTOR__)\n #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) power_compute_fp16_to_fp32(x)\n #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) power_compute_fp32_to_fp16(x)\n /* the inline asm below is about 12% faster than the lookup method */\n #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)\n #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x)\n\n static inline float power_compute_fp16_to_fp32(ggml_fp16_t h) {\n float f;\n double d;\n __asm__(\n \"mtfprd %0,%2\\n\"\n \"xscvhpdp %0,%0\\n\"\n \"frsp %1,%0\\n\" :\n /* temp */ \"=d\"(d),\n /* out */ \"=f\"(f):\n /* in */ \"r\"(h));\n return f;\n }\n\n static inline ggml_fp16_t power_compute_fp32_to_fp16(float f) {\n double d;\n ggml_fp16_t r;\n __asm__( /* xscvdphp can work on double or single precision */\n \"xscvdphp %0,%2\\n\"\n \"mffprd %1,%0\\n\" :\n /* temp */ \"=d\"(d),\n /* out */ \"=r\"(r):\n /* in */ \"f\"(f));\n return r;\n }\n#elif defined(__riscv) && defined(__riscv_zfhmin)\n static inline float riscv_compute_fp16_to_fp32(ggml_fp16_t h) {\n _Float16 hf;\n memcpy(&hf, &h, sizeof(ggml_fp16_t));\n return hf;\n }\n\n static inline ggml_fp16_t riscv_compute_fp32_to_fp16(float f) {\n ggml_fp16_t res;\n _Float16 hf = (_Float16)f;\n memcpy(&res, &hf, sizeof(ggml_fp16_t));\n return res;\n }\n\n #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) riscv_compute_fp16_to_fp32(x)\n #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) riscv_compute_fp32_to_fp16(x)\n #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)\n #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x)\n#endif\n\n// precomputed f32 table for f16 (256 KB)\n// defined in ggml-cpu.c, initialized in ggml_cpu_init()\nextern float ggml_table_f32_f16[1 << 16];\n\n// precomputed f32 table for e8m0 half (1 KB)\n// defined in ggml-cpu.c, initialized in ggml_cpu_init()\nextern float ggml_table_f32_e8m0_half[1 << 8];\n\n// Use lookup table for E8M0 on x86 (faster than bit manipulation)\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\n#define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)]\n#else\n#define GGML_CPU_E8M0_TO_FP32_HALF(x) GGML_E8M0_TO_FP32_HALF(x)\n#endif\n\n// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32,\n// so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON.\n// This is also true for POWER9.\n#if !defined(GGML_CPU_FP16_TO_FP32)\ninline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) {\n uint16_t s;\n memcpy(&s, &f, sizeof(uint16_t));\n return ggml_table_f32_f16[s];\n}\n\n#define GGML_CPU_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x)\n#endif\n\n#if !defined(GGML_CPU_FP32_TO_FP16)\n#define GGML_CPU_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)\n#endif\n\n\n// we define a common set of C macros which map to specific intrinsics based on the current architecture\n// we then implement the fundamental computation operations below using only these macros\n// adding support for new architectures requires to define the corresponding SIMD macros\n//\n// GGML_F32_STEP / GGML_F16_STEP\n// number of elements to process in a single step\n//\n// GGML_F32_EPR / GGML_F16_EPR\n// number of elements to fit in a single register\n//\n\n#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_FMA)\n\n#define GGML_SIMD\n\n// F32 SVE\n#define GGML_F32_EPR 8\n#define DEFAULT_PG svptrue_b32()\n\n#define GGML_F32xt svfloat32_t\n#define GGML_F32xt_ZERO svdup_n_f32(0.0f)\n#define GGML_F32xt_SET1(x) svdup_n_f32(x)\n#define GGML_F32xt_LOAD_IMPL(pg, a) svld1_f32(pg, a)\n#define GGML_F32xt_LOAD(a) GGML_F32xt_LOAD_IMPL(DEFAULT_PG, a)\n#define GGML_F32xt_STORE_IMPL(pg, a, b) svst1_f32(pg, a, b)\n#define GGML_F32xt_STORE(a, b) GGML_F32xt_STORE_IMPL(DEFAULT_PG, a, b)\n#define GGML_F32xt_FMA_IMPL(pg, a, b, c) svmad_f32_m(pg, b, c, a)\n#define GGML_F32xt_FMA(a, b, c) GGML_F32xt_FMA_IMPL(DEFAULT_PG, a, b, c)\n#define GGML_F32xt_ADD_IMPL(pg, a, b) svadd_f32_m(pg, a, b)\n#define GGML_F32xt_ADD(a, b) GGML_F32xt_ADD_IMPL(DEFAULT_PG, a, b)\n#define GGML_F32xt_MUL_IMPL(pg, a, b) svmul_f32_m(pg, a, b)\n#define GGML_F32xt_MUL(a, b) GGML_F32xt_MUL_IMPL(DEFAULT_PG, a, b)\n#define GGML_F32xt_REDUCE_ONE_IMPL(pg, a) svaddv(pg, a)\n#define GGML_F32xt_REDUCE_ONE(a) GGML_F32xt_REDUCE_ONE_IMPL(DEFAULT_PG, a)\n#define GGML_F32xt_REDUCE_IMPL(pg, res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8) \\\n{ \\\n sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum2); \\\n sum3 = svadd_f32_m(DEFAULT_PG, sum3, sum4); \\\n sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum6); \\\n sum7 = svadd_f32_m(DEFAULT_PG, sum7, sum8); \\\n sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum3); \\\n sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum7); \\\n sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum5); \\\n (res) = (ggml_float) GGML_F32xt_REDUCE_ONE(sum1); \\\n}\n#define GGML_F32xt_REDUCE(res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8) \\\n GGML_F32xt_REDUCE_IMPL(DEFAULT_PG, res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8)\n\n#define GGML_F32_VEC GGML_F32xt\n#define GGML_F32_VEC_ZERO GGML_F32xt_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32xt_SET1\n#define GGML_F32_VEC_LOAD GGML_F32xt_LOAD\n#define GGML_F32_VEC_STORE GGML_F32xt_STORE\n#define GGML_F32_VEC_FMA GGML_F32xt_FMA\n#define GGML_F32_VEC_ADD GGML_F32xt_ADD\n#define GGML_F32_VEC_MUL GGML_F32xt_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32xt_REDUCE\n\n// F16 SVE\n#define DEFAULT_PG32 svptrue_b32()\n#define DEFAULT_PG16 svptrue_b16()\n\n#define GGML_F32Cxt svfloat16_t\n#define GGML_F32Cxt_ZERO svdup_n_f16(0.0f)\n#define GGML_F32Cxt_SET1(x) svdup_n_f16(x)\n#define GGML_F32Cxt_LOAD(p) svld1_f16(DEFAULT_PG16, (const __fp16 *)(p))\n#define GGML_F32Cxt_STORE(dst_ptr, src_vec) svst1_f16(DEFAULT_PG16, (__fp16 *)(dst_ptr), (src_vec))\n\n#define GGML_F32Cxt_FMA_IMPL(pg, a, b, c) svmad_f16_x(pg, b, c, a)\n#define GGML_F32Cxt_FMA(a, b, c) GGML_F32Cxt_FMA_IMPL(DEFAULT_PG16, a, b, c)\n#define GGML_F32Cxt_ADD_IMPL(pg, a, b) svadd_f16_x(pg, a, b)\n#define GGML_F32Cxt_ADD(a, b) GGML_F32Cxt_ADD_IMPL(DEFAULT_PG16, a, b)\n#define GGML_F32Cxt_MUL_IMPL(pg, a, b) svmul_f16_x(pg, a, b)\n#define GGML_F32Cxt_MUL(a, b) GGML_F32Cxt_MUL_IMPL(DEFAULT_PG16, a, b)\n#define GGML_F32Cxt_REDUCE GGML_F16xt_REDUCE_MIXED\n\n#define GGML_F16x_VEC GGML_F32Cxt\n#define GGML_F16x_VEC_ZERO GGML_F32Cxt_ZERO\n#define GGML_F16x_VEC_SET1 GGML_F32Cxt_SET1\n#define GGML_F16x_VEC_LOAD(p, i) GGML_F32Cxt_LOAD(p)\n#define GGML_F16x_VEC_STORE(p, r, i) GGML_F32Cxt_STORE((__fp16 *)(p), r)\n#define GGML_F16x_VEC_FMA GGML_F32Cxt_FMA\n#define GGML_F16x_VEC_ADD GGML_F32Cxt_ADD\n#define GGML_F16x_VEC_MUL GGML_F32Cxt_MUL\n#define GGML_F16x_VEC_REDUCE GGML_F32Cxt_REDUCE\n\n#define GGML_F16xt_REDUCE_ONE_IMPL(pg, a) svaddv_f16(pg, a)\n#define GGML_F16xt_REDUCE_ONE(a) GGML_F16xt_REDUCE_ONE_IMPL(DEFAULT_PG16, a)\n\n#define GGML_F16xt_REDUCE_MIXED_IMPL(pg16, res, sum1, sum2, sum3, sum4) \\\n{ \\\n sum1 = svadd_f16_x(pg16, sum1, sum2); \\\n sum3 = svadd_f16_x(pg16, sum3, sum4); \\\n sum1 = svadd_f16_x(pg16, sum1, sum3); \\\n __fp16 sum_f16 = svaddv_f16(pg16, sum1); \\\n (res) = (ggml_float) sum_f16; \\\n}\n#define GGML_F16xt_REDUCE_MIXED(res, sum1, sum2, sum3, sum4) \\\n GGML_F16xt_REDUCE_MIXED_IMPL(DEFAULT_PG16, res, sum1, sum2, sum3, sum4)\n\n// F16 NEON\n\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)\n #define GGML_F16_STEP 32\n #define GGML_F16_EPR 8\n\n #define GGML_F16x8 float16x8_t\n #define GGML_F16x8_ZERO vdupq_n_f16(0.0f)\n #define GGML_F16x8_SET1(x) vdupq_n_f16(x)\n #define GGML_F16x8_LOAD(x) vld1q_f16((const __fp16 *)(x))\n #define GGML_F16x8_STORE vst1q_f16\n #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)\n #define GGML_F16x8_ADD vaddq_f16\n #define GGML_F16x8_MUL vmulq_f16\n #define GGML_F16x8_REDUCE(res, x) \\\n do { \\\n int offset = GGML_F16_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \\\n } \\\n const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \\\n const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \\\n (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \\\n } while (0)\n\n #define GGML_F16_VEC GGML_F16x8\n #define GGML_F16_VEC_ZERO GGML_F16x8_ZERO\n #define GGML_F16_VEC_SET1 GGML_F16x8_SET1\n #define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p)\n #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((__fp16 *)(p), (r)[i])\n #define GGML_F16_VEC_FMA GGML_F16x8_FMA\n #define GGML_F16_VEC_ADD GGML_F16x8_ADD\n #define GGML_F16_VEC_MUL GGML_F16x8_MUL\n #define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE\n#else\n // if FP16 vector arithmetic is not supported, we use FP32 instead\n // and take advantage of the vcvt_ functions to convert to/from FP16\n\n #define GGML_F16_STEP 16\n #define GGML_F16_EPR 4\n\n #define GGML_F32Cx4 float32x4_t\n #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f)\n #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x)\n #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const __fp16 *)(x)))\n #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))\n #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)\n #define GGML_F32Cx4_ADD vaddq_f32\n #define GGML_F32Cx4_MUL vmulq_f32\n #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE\n\n #define GGML_F16_VEC GGML_F32Cx4\n #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO\n #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1\n #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)\n #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((__fp16 *)(p), r[i])\n #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA\n #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD\n #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL\n #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE\n#endif\n\n#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)\n\n#define GGML_SIMD\n\n// F32 NEON\n\n#define GGML_F32_STEP 16\n#define GGML_F32_EPR 4\n\n#define GGML_F32x4 float32x4_t\n#define GGML_F32x4_ZERO vdupq_n_f32(0.0f)\n#define GGML_F32x4_SET1(x) vdupq_n_f32(x)\n#define GGML_F32x4_LOAD vld1q_f32\n#define GGML_F32x4_STORE vst1q_f32\n#define GGML_F32x4_FMA(a, b, c) vfmaq_f32(a, b, c)\n#define GGML_F32x4_ADD vaddq_f32\n#define GGML_F32x4_MUL vmulq_f32\n#define GGML_F32x4_REDUCE_ONE(x) vaddvq_f32(x)\n#define GGML_F32x4_REDUCE(res, x) \\\n{ \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \\\n } \\\n (res) = (ggml_float) GGML_F32x4_REDUCE_ONE((x)[0]); \\\n}\n\n#define GGML_F32_VEC GGML_F32x4\n#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 NEON\n\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)\n #define GGML_F16_STEP 32\n #define GGML_F16_EPR 8\n\n #define GGML_F16x8 float16x8_t\n #define GGML_F16x8_ZERO vdupq_n_f16(0.0f)\n #define GGML_F16x8_SET1(x) vdupq_n_f16(x)\n #define GGML_F16x8_LOAD(x) vld1q_f16((const __fp16 *)(x))\n #define GGML_F16x8_STORE vst1q_f16\n #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)\n #define GGML_F16x8_ADD vaddq_f16\n #define GGML_F16x8_MUL vmulq_f16\n #define GGML_F16x8_REDUCE(res, x) \\\n do { \\\n int offset = GGML_F16_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \\\n } \\\n const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \\\n const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \\\n (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \\\n } while (0)\n\n #define GGML_F16_VEC GGML_F16x8\n #define GGML_F16_VEC_ZERO GGML_F16x8_ZERO\n #define GGML_F16_VEC_SET1 GGML_F16x8_SET1\n #define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p)\n #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((__fp16 *)(p), (r)[i])\n #define GGML_F16_VEC_FMA GGML_F16x8_FMA\n #define GGML_F16_VEC_ADD GGML_F16x8_ADD\n #define GGML_F16_VEC_MUL GGML_F16x8_MUL\n #define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE\n#else\n // if FP16 vector arithmetic is not supported, we use FP32 instead\n // and take advantage of the vcvt_ functions to convert to/from FP16\n\n #define GGML_F16_STEP 16\n #define GGML_F16_EPR 4\n\n #define GGML_F32Cx4 float32x4_t\n #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f)\n #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x)\n #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const __fp16 *)(x)))\n #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))\n #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)\n #define GGML_F32Cx4_ADD vaddq_f32\n #define GGML_F32Cx4_MUL vmulq_f32\n #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE\n\n #define GGML_F16_VEC GGML_F32Cx4\n #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO\n #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1\n #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)\n #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((__fp16 *)(p), r[i])\n #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA\n #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD\n #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL\n #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE\n#endif\n\n#elif defined(__AVX512F__)\n\n#define GGML_SIMD\n\n// F32 AVX512\n\n#define GGML_F32_STEP 64\n#define GGML_F32_EPR 16\n\n#define GGML_F32x16 __m512\n#define GGML_F32x16_ZERO _mm512_setzero_ps()\n#define GGML_F32x16_SET1(x) _mm512_set1_ps(x)\n#define GGML_F32x16_LOAD _mm512_loadu_ps\n#define GGML_F32x16_STORE _mm512_storeu_ps\n// _mm512_fmadd_ps is defined in AVX512F so no guard is required\n#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)\n#define GGML_F32x16_ADD _mm512_add_ps\n#define GGML_F32x16_MUL _mm512_mul_ps\n#define GGML_F32x16_REDUCE(res, x) \\\ndo { \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm512_add_ps(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm512_add_ps(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm512_add_ps(x[i], x[offset+i]); \\\n } \\\n res = (ggml_float) _mm512_reduce_add_ps(x[0]); \\\n} while (0)\n\n// TODO: is this optimal ?\n\n#define GGML_F32_VEC GGML_F32x16\n#define GGML_F32_VEC_ZERO GGML_F32x16_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x16_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x16_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x16_STORE\n#define GGML_F32_VEC_FMA GGML_F32x16_FMA\n#define GGML_F32_VEC_ADD GGML_F32x16_ADD\n#define GGML_F32_VEC_MUL GGML_F32x16_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE\n\n// F16 AVX512\n\n#if defined(__AVX512FP16__)\n\n#define GGML_F16_STEP 128\n#define GGML_F16_EPR 32\n\n#define GGML_F16x32 __m512h\n#define GGML_F16x32_ZERO _mm512_setzero_ph()\n#define GGML_F16x32_SET1(x) _mm512_set1_ph(__extension__(_Float16)(x))\n#define GGML_F16x32_LOAD(x) _mm512_loadu_ph(x)\n#define GGML_F16x32_STORE(x, y) _mm512_storeu_ph(x, y)\n#define GGML_F16x32_FMA(a, b, c) _mm512_fmadd_ph(b, c, a)\n#define GGML_F16x32_ADD _mm512_add_ph\n#define GGML_F16x32_MUL _mm512_mul_ph\n#define GGML_F16x32_REDUCE(res, x) \\\ndo { \\\n int offset = GGML_F16_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm512_add_ph(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm512_add_ph(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm512_add_ph(x[i], x[offset+i]); \\\n } \\\n res = (ggml_float) _mm512_reduce_add_ph(x[0]); \\\n} while (0)\n\n#define GGML_F16_VEC GGML_F16x32\n#define GGML_F16_VEC_ZERO GGML_F16x32_ZERO\n#define GGML_F16_VEC_SET1 GGML_F16x32_SET1\n#define GGML_F16_VEC_LOAD(p, i) GGML_F16x32_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x32_STORE(p, r[i])\n#define GGML_F16_VEC_FMA GGML_F16x32_FMA\n#define GGML_F16_VEC_ADD GGML_F16x32_ADD\n#define GGML_F16_VEC_MUL GGML_F16x32_MUL\n#define GGML_F16_VEC_REDUCE GGML_F16x32_REDUCE\n\n#else // Fallback FP16 <-> FP32\n\n#define GGML_F16_STEP 64\n#define GGML_F16_EPR 16\n\n#define GGML_F32Cx16 __m512\n#define GGML_F32Cx16_ZERO _mm512_setzero_ps()\n#define GGML_F32Cx16_SET1(x) _mm512_set1_ps(x)\n\n// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F\n// so F16C guard isn't required\n#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x)))\n#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0))\n\n#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)\n#define GGML_F32Cx16_ADD _mm512_add_ps\n#define GGML_F32Cx16_MUL _mm512_mul_ps\n#define GGML_F32Cx16_REDUCE(res, x) \\\ndo { \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm512_add_ps(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm512_add_ps(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm512_add_ps(x[i], x[offset+i]); \\\n } \\\n res = (ggml_float) _mm512_reduce_add_ps(x[0]); \\\n} while (0)\n\n#define GGML_F16_VEC GGML_F32Cx16\n#define GGML_F16_VEC_ZERO GGML_F32Cx16_ZERO\n#define GGML_F16_VEC_SET1 GGML_F32Cx16_SET1\n#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx16_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i])\n#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA\n#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD\n#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL\n\n#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE\n\n#endif // __AVX512FP16__\n#elif defined(__AVX__)\n\n#define GGML_SIMD\n\n// F32 AVX\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR 8\n\n#define GGML_F32x8 __m256\n#define GGML_F32x8_ZERO _mm256_setzero_ps()\n#define GGML_F32x8_SET1(x) _mm256_set1_ps(x)\n#define GGML_F32x8_LOAD _mm256_loadu_ps\n#define GGML_F32x8_STORE _mm256_storeu_ps\n#if defined(__FMA__)\n #define GGML_F32x8_FMA(a, b, c) _mm256_fmadd_ps(b, c, a)\n#else\n #define GGML_F32x8_FMA(a, b, c) _mm256_add_ps(_mm256_mul_ps(b, c), a)\n#endif\n#define GGML_F32x8_ADD _mm256_add_ps\n#define GGML_F32x8_MUL _mm256_mul_ps\n#define GGML_F32x8_REDUCE(res, x) \\\ndo { \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm256_add_ps(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm256_add_ps(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm256_add_ps(x[i], x[offset+i]); \\\n } \\\n const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \\\n _mm256_extractf128_ps(x[0], 1)); \\\n const __m128 t1 = _mm_hadd_ps(t0, t0); \\\n res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \\\n} while (0)\n// TODO: is this optimal ?\n\n#define GGML_F32_VEC GGML_F32x8\n#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x8_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x8_STORE\n#define GGML_F32_VEC_FMA GGML_F32x8_FMA\n#define GGML_F32_VEC_ADD GGML_F32x8_ADD\n#define GGML_F32_VEC_MUL GGML_F32x8_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE\n\n// F16 AVX\n\n#define GGML_F16_STEP 32\n#define GGML_F16_EPR 8\n\n// F16 arithmetic is not supported by AVX, so we use F32 instead\n\n#define GGML_F32Cx8 __m256\n#define GGML_F32Cx8_ZERO _mm256_setzero_ps()\n#define GGML_F32Cx8_SET1(x) _mm256_set1_ps(x)\n\n#if defined(__F16C__)\n// the _mm256_cvt intrinsics require F16C\n#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))\n#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))\n#else\nstatic inline __m256 __avx_f32cx8_load(const ggml_fp16_t * x) {\n float tmp[8];\n\n for (int i = 0; i < 8; i++) {\n tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n }\n\n return _mm256_loadu_ps(tmp);\n}\nstatic inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {\n float arr[8];\n\n _mm256_storeu_ps(arr, y);\n\n for (int i = 0; i < 8; i++)\n x[i] = GGML_CPU_FP32_TO_FP16(arr[i]);\n}\n#define GGML_F32Cx8_LOAD(x) __avx_f32cx8_load(x)\n#define GGML_F32Cx8_STORE(x, y) __avx_f32cx8_store(x, y)\n#endif\n\n#define GGML_F32Cx8_FMA GGML_F32x8_FMA\n#define GGML_F32Cx8_ADD _mm256_add_ps\n#define GGML_F32Cx8_MUL _mm256_mul_ps\n#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE\n\n#define GGML_F16_VEC GGML_F32Cx8\n#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO\n#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1\n#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])\n#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA\n#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD\n#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL\n#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE\n\n#elif defined(__POWER9_VECTOR__)\n\n#define GGML_SIMD\n\n// F32 POWER9\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR 4\n\n#define GGML_F32x4 vector float\n#define GGML_F32x4_ZERO {0.0f}\n#define GGML_F32x4_SET1 vec_splats\n#define GGML_F32x4_LOAD(p) vec_xl(0, p)\n#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p)\n#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)\n#define GGML_F32x4_ADD vec_add\n#define GGML_F32x4_MUL vec_mul\n#define GGML_F32x4_REDUCE(res, x) \\\n{ \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = vec_add(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = vec_add(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = vec_add(x[i], x[offset+i]); \\\n } \\\n res = vec_extract(x[0], 0) + \\\n vec_extract(x[0], 1) + \\\n vec_extract(x[0], 2) + \\\n vec_extract(x[0], 3); \\\n}\n#define GGML_F32x4_REDUCE_4(res, s0, s1, s2, s3) \\\n{ \\\n vector float v = vec_add(vec_add(s0, s1), \\\n vec_add(s2, s3)); \\\n v = vec_add(v, vec_sld(v, v, 8)); \\\n v = vec_add(v, vec_sld(v, v, 4)); \\\n res += (ggml_float) vec_extract(v, 0); \\\n}\n\n#define GGML_F32_VEC GGML_F32x4\n#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 POWER9\n#define GGML_F16_STEP GGML_F32_STEP\n#define GGML_F16_EPR GGML_F32_EPR\n#define GGML_F16_VEC GGML_F32x4\n#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO\n#define GGML_F16_VEC_SET1 GGML_F32x4_SET1\n#define GGML_F16_VEC_FMA GGML_F32x4_FMA\n#define GGML_F16_VEC_ADD GGML_F32x4_ADD\n#define GGML_F16_VEC_MUL GGML_F32x4_MUL\n#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE\n// Use vec_xl, not vec_ld, in case the load address is not aligned.\n#define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ? \\\n vec_extract_fp32_from_shorth(vec_xl(0, p - GGML_F16_EPR)) : \\\n vec_extract_fp32_from_shortl(vec_xl(0, p))\nstatic inline unsigned char ggml_endian_byte(int i) {\n uint16_t tmp_val = 1;\n return ((unsigned char *)&tmp_val)[i];\n}\n#define GGML_ENDIAN_BYTE(i) ggml_endian_byte(i)\n#define GGML_F16_VEC_STORE(p, r, i) \\\n if (i & 0x1) \\\n vec_xst(vec_pack_to_short_fp32(r[i - GGML_ENDIAN_BYTE(1)], \\\n r[i - GGML_ENDIAN_BYTE(0)]), \\\n 0, p - GGML_F16_EPR)\n\n//BF16 POWER9\n#define GGML_BF16_STEP 16\n#define GGML_BF16_EPR 8\n\n#define GGML_BF16x8 vector unsigned short\n#define GGML_BF16x8_ZERO vec_splats((unsigned short)0)\n#define GGML_BF16x8_LOAD(p) vec_xl(0, (const unsigned short *)(p))\n\n#define GGML_BF16_VEC GGML_BF16x8\n#define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO\n#define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD\n#if defined(__LITTLE_ENDIAN__)\n#define GGML_BF16_TO_F32_LO(v) ((vector float) vec_mergel(GGML_BF16_VEC_ZERO, (v)))\n#define GGML_BF16_TO_F32_HI(v) ((vector float) vec_mergeh(GGML_BF16_VEC_ZERO, (v)))\n#else\n#define GGML_BF16_TO_F32_LO(v) ((vector float) vec_mergel((v), GGML_BF16_VEC_ZERO))\n#define GGML_BF16_TO_F32_HI(v) ((vector float) vec_mergeh((v), GGML_BF16_VEC_ZERO))\n#endif\n#define GGML_BF16_FMA_LO(acc, x, y) \\\n (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y))\n#define GGML_BF16_FMA_HI(acc, x, y) \\\n (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y))\n\n#elif defined(__wasm_simd128__)\n\n#define GGML_SIMD\n\n// F32 WASM\n\n#define GGML_F32_STEP 16\n#define GGML_F32_EPR 4\n\n#define GGML_F32x4 v128_t\n#define GGML_F32x4_ZERO wasm_f32x4_splat(0.0f)\n#define GGML_F32x4_SET1(x) wasm_f32x4_splat(x)\n#define GGML_F32x4_LOAD wasm_v128_load\n#define GGML_F32x4_STORE wasm_v128_store\n#define GGML_F32x4_FMA(a, b, c) wasm_f32x4_add(wasm_f32x4_mul(b, c), a)\n#define GGML_F32x4_ADD wasm_f32x4_add\n#define GGML_F32x4_MUL wasm_f32x4_mul\n#define GGML_F32x4_REDUCE(res, x) \\\n{ \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = wasm_f32x4_add(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = wasm_f32x4_add(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = wasm_f32x4_add(x[i], x[offset+i]); \\\n } \\\n res = wasm_f32x4_extract_lane(x[0], 0) + \\\n wasm_f32x4_extract_lane(x[0], 1) + \\\n wasm_f32x4_extract_lane(x[0], 2) + \\\n wasm_f32x4_extract_lane(x[0], 3); \\\n}\n\n#define GGML_F32_VEC GGML_F32x4\n#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 WASM\n\n#define GGML_F16_STEP 16\n#define GGML_F16_EPR 4\n\ninline static v128_t __wasm_f16x4_load(const ggml_fp16_t * p) {\n float tmp[4];\n\n tmp[0] = GGML_CPU_FP16_TO_FP32(p[0]);\n tmp[1] = GGML_CPU_FP16_TO_FP32(p[1]);\n tmp[2] = GGML_CPU_FP16_TO_FP32(p[2]);\n tmp[3] = GGML_CPU_FP16_TO_FP32(p[3]);\n\n return wasm_v128_load(tmp);\n}\n\ninline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {\n float tmp[4];\n\n wasm_v128_store(tmp, x);\n\n p[0] = GGML_CPU_FP32_TO_FP16(tmp[0]);\n p[1] = GGML_CPU_FP32_TO_FP16(tmp[1]);\n p[2] = GGML_CPU_FP32_TO_FP16(tmp[2]);\n p[3] = GGML_CPU_FP32_TO_FP16(tmp[3]);\n}\n\n#define GGML_F16x4 v128_t\n#define GGML_F16x4_ZERO wasm_f32x4_splat(0.0f)\n#define GGML_F16x4_SET1(x) wasm_f32x4_splat(x)\n#define GGML_F16x4_LOAD(x) __wasm_f16x4_load(x)\n#define GGML_F16x4_STORE(x, y) __wasm_f16x4_store(x, y)\n#define GGML_F16x4_FMA GGML_F32x4_FMA\n#define GGML_F16x4_ADD wasm_f32x4_add\n#define GGML_F16x4_MUL wasm_f32x4_mul\n#define GGML_F16x4_REDUCE(res, x) \\\n{ \\\n int offset = GGML_F16_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = wasm_f32x4_add(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = wasm_f32x4_add(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = wasm_f32x4_add(x[i], x[offset+i]); \\\n } \\\n res = (ggml_float) (wasm_f32x4_extract_lane(x[0], 0) + \\\n wasm_f32x4_extract_lane(x[0], 1) + \\\n wasm_f32x4_extract_lane(x[0], 2) + \\\n wasm_f32x4_extract_lane(x[0], 3)); \\\n}\n\n#define GGML_F16_VEC GGML_F16x4\n#define GGML_F16_VEC_ZERO GGML_F16x4_ZERO\n#define GGML_F16_VEC_SET1 GGML_F16x4_SET1\n#define GGML_F16_VEC_LOAD(p, i) GGML_F16x4_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x4_STORE(p, r[i])\n#define GGML_F16_VEC_FMA GGML_F16x4_FMA\n#define GGML_F16_VEC_ADD GGML_F16x4_ADD\n#define GGML_F16_VEC_MUL GGML_F16x4_MUL\n#define GGML_F16_VEC_REDUCE GGML_F16x4_REDUCE\n\n#elif defined(__SSE3__)\n\n#define GGML_SIMD\n\n// F32 SSE\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR 4\n\n#define GGML_F32x4 __m128\n#define GGML_F32x4_ZERO _mm_setzero_ps()\n#define GGML_F32x4_SET1(x) _mm_set1_ps(x)\n#define GGML_F32x4_LOAD _mm_loadu_ps\n#define GGML_F32x4_STORE _mm_storeu_ps\n#if defined(__FMA__)\n // TODO: Does this work?\n #define GGML_F32x4_FMA(a, b, c) _mm_fmadd_ps(b, c, a)\n#else\n #define GGML_F32x4_FMA(a, b, c) _mm_add_ps(_mm_mul_ps(b, c), a)\n#endif\n#define GGML_F32x4_ADD _mm_add_ps\n#define GGML_F32x4_MUL _mm_mul_ps\n#define GGML_F32x4_REDUCE(res, x) \\\n{ \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm_add_ps(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm_add_ps(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = _mm_add_ps(x[i], x[offset+i]); \\\n } \\\n const __m128 t0 = _mm_hadd_ps(x[0], x[0]); \\\n res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \\\n}\n// TODO: is this optimal ?\n\n#define GGML_F32_VEC GGML_F32x4\n#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 SSE\n\n#define GGML_F16_STEP 32\n#define GGML_F16_EPR 4\n\nstatic inline __m128 __sse_f16x4_load(const ggml_fp16_t * x) {\n float tmp[4];\n\n tmp[0] = GGML_CPU_FP16_TO_FP32(x[0]);\n tmp[1] = GGML_CPU_FP16_TO_FP32(x[1]);\n tmp[2] = GGML_CPU_FP16_TO_FP32(x[2]);\n tmp[3] = GGML_CPU_FP16_TO_FP32(x[3]);\n\n return _mm_loadu_ps(tmp);\n}\n\nstatic inline void __sse_f16x4_store(ggml_fp16_t * x, __m128 y) {\n float arr[4];\n\n _mm_storeu_ps(arr, y);\n\n x[0] = GGML_CPU_FP32_TO_FP16(arr[0]);\n x[1] = GGML_CPU_FP32_TO_FP16(arr[1]);\n x[2] = GGML_CPU_FP32_TO_FP16(arr[2]);\n x[3] = GGML_CPU_FP32_TO_FP16(arr[3]);\n}\n\n#define GGML_F32Cx4 __m128\n#define GGML_F32Cx4_ZERO _mm_setzero_ps()\n#define GGML_F32Cx4_SET1(x) _mm_set1_ps(x)\n#define GGML_F32Cx4_LOAD(x) __sse_f16x4_load(x)\n#define GGML_F32Cx4_STORE(x, y) __sse_f16x4_store(x, y)\n#define GGML_F32Cx4_FMA GGML_F32x4_FMA\n#define GGML_F32Cx4_ADD _mm_add_ps\n#define GGML_F32Cx4_MUL _mm_mul_ps\n#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE\n\n#define GGML_F16_VEC GGML_F32Cx4\n#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO\n#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1\n#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])\n#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA\n#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD\n#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL\n#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE\n\n#elif defined(__loongarch_asx)\n\n#define GGML_SIMD\n\n// F32 LASX\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR 8\n\n#define GGML_F32x8 __m256\n#define GGML_F32x8_ZERO (__m256)__lasx_xvldi(0)\n#define GGML_F32x8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x))\n#define GGML_F32x8_LOAD(x) (__m256)__lasx_xvld((x), 0)\n#define GGML_F32x8_STORE(x,y) __lasx_xvst((y), (x), 0)\n#define GGML_F32x8_FMA(a, b, c) __lasx_xvfmadd_s(b, c, a)\n#define GGML_F32x8_ADD __lasx_xvfadd_s\n#define GGML_F32x8_MUL __lasx_xvfmul_s\n#define GGML_F32x8_REDUCE(res, x) \\\ndo { \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \\\n } \\\n float *tmp_p = (float *)&x[0]; \\\n res = tmp_p[0] + tmp_p[1] + tmp_p[2] + tmp_p[3] + tmp_p[4] + tmp_p[5] + tmp_p[6] + tmp_p[7]; \\\n} while (0)\n// TODO: is this optimal ?\n\n#define GGML_F32_VEC GGML_F32x8\n#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x8_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x8_STORE\n#define GGML_F32_VEC_FMA GGML_F32x8_FMA\n#define GGML_F32_VEC_ADD GGML_F32x8_ADD\n#define GGML_F32_VEC_MUL GGML_F32x8_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE\n\n// F16 LASX\n\n#define GGML_F16_STEP 32\n#define GGML_F16_EPR 8\n\n// F16 arithmetic is not supported by LASX, so we use F32 instead\n\n#define GGML_F32Cx8 __m256\n#define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0)\n#define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x))\n\nstatic inline __m256 __lasx_f32cx8_load(const ggml_fp16_t * x) {\n __m256i a;\n memcpy(&a, x, sizeof(ggml_fp16_t) * 8);\n a = __lasx_xvpermi_d(a, 0 | (1 << 4));\n return __lasx_xvfcvtl_s_h(a);\n}\n\nstatic inline void __lasx_f32cx8_store(ggml_fp16_t * x, __m256 y) {\n __m256i a = __lasx_xvfcvt_h_s(y, y);\n a = __lasx_xvpermi_d(a, 0 | (2 << 2));\n memcpy(x, &a, sizeof(ggml_fp16_t) * 8);\n}\n#define GGML_F32Cx8_LOAD(x) __lasx_f32cx8_load(x)\n#define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y)\n\n#define GGML_F32Cx8_FMA GGML_F32x8_FMA\n#define GGML_F32Cx8_ADD __lasx_xvfadd_s\n#define GGML_F32Cx8_MUL __lasx_xvfmul_s\n#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE\n\n#define GGML_F16_VEC GGML_F32Cx8\n#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO\n#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1\n#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])\n#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA\n#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD\n#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL\n#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE\n\n#elif defined(__loongarch_sx)\n\n#define GGML_SIMD\n\n// F32 LSX\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR 4\n\n#define GGML_F32x4 __m128\n#define GGML_F32x4_ZERO (__m128)__lsx_vldi(0)\n#define GGML_F32x4_SET1(x) (__m128)__lsx_vreplfr2vr_s((x))\n#define GGML_F32x4_LOAD(x) (__m128)__lsx_vld((x), 0)\n#define GGML_F32x4_STORE(x, y) __lsx_vst(y, x, 0)\n#define GGML_F32x4_FMA(a, b, c) __lsx_vfmadd_s(b, c, a)\n#define GGML_F32x4_ADD __lsx_vfadd_s\n#define GGML_F32x4_MUL __lsx_vfmul_s\n\n#define GGML_F32x4_REDUCE(res, x) \\\n{ \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \\\n } \\\n __m128i t0 = __lsx_vpickev_w((__m128i)x[0], (__m128i)x[0]); \\\n __m128i t1 = __lsx_vpickod_w((__m128i)x[0], (__m128i)x[0]); \\\n __m128 t2 = __lsx_vfadd_s((__m128)t0, (__m128)t1); \\\n __m128i t3 = __lsx_vpickev_w((__m128i)t2, (__m128i)t2); \\\n __m128i t4 = __lsx_vpickod_w((__m128i)t2, (__m128i)t2); \\\n __m128 t5 = __lsx_vfadd_s((__m128)t3, (__m128)t4); \\\n res = (ggml_float) ((v4f32)t5)[0]; \\\n}\n\n#define GGML_F32_VEC GGML_F32x4\n#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 LSX\n\n#define GGML_F16_STEP 32\n#define GGML_F16_EPR 4\n\nstatic inline __m128 __lsx_f16x4_load(const ggml_fp16_t * x) {\n float tmp[4];\n\n tmp[0] = GGML_CPU_FP16_TO_FP32(x[0]);\n tmp[1] = GGML_CPU_FP16_TO_FP32(x[1]);\n tmp[2] = GGML_CPU_FP16_TO_FP32(x[2]);\n tmp[3] = GGML_CPU_FP16_TO_FP32(x[3]);\n\n return (__m128)__lsx_vld(tmp, 0);\n}\n\nstatic inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) {\n float arr[4];\n\n __lsx_vst(y, arr, 0);\n\n x[0] = GGML_CPU_FP32_TO_FP16(arr[0]);\n x[1] = GGML_CPU_FP32_TO_FP16(arr[1]);\n x[2] = GGML_CPU_FP32_TO_FP16(arr[2]);\n x[3] = GGML_CPU_FP32_TO_FP16(arr[3]);\n}\n\n#define GGML_F32Cx4 __m128\n#define GGML_F32Cx4_ZERO (__m128)__lsx_vldi(0)\n#define GGML_F32Cx4_SET1(x) (__m128)__lsx_vreplfr2vr_s((x))\n#define GGML_F32Cx4_LOAD(x) (__m128)__lsx_f16x4_load(x)\n#define GGML_F32Cx4_STORE(x, y) __lsx_f16x4_store(x, y)\n#define GGML_F32Cx4_FMA GGML_F32x4_FMA\n#define GGML_F32Cx4_ADD __lsx_vfadd_s\n#define GGML_F32Cx4_MUL __lsx_vfmul_s\n#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE\n\n#define GGML_F16_VEC GGML_F32Cx4\n#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO\n#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1\n#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])\n#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA\n#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD\n#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL\n#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE\n\n#elif defined(__VXE__) || defined(__VXE2__)\n\n#define GGML_SIMD\n\n// F32 s390x\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR 4\n\n#define GGML_F32x4 float32x4_t\n#define GGML_F32x4_ZERO vec_splats(0.0f)\n#define GGML_F32x4_SET1 vec_splats\n#define GGML_F32x4_LOAD(p) vec_xl(0, p)\n#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p)\n#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)\n#define GGML_F32x4_ADD vec_add\n#define GGML_F32x4_MUL vec_mul\n#define GGML_F32x4_REDUCE(res, x) \\\n{ \\\n int offset = GGML_F32_ARR >> 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = vec_add(x[i], x[offset + i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = vec_add(x[i], x[offset + i]); \\\n } \\\n offset >>= 1; \\\n for (int i = 0; i < offset; ++i) { \\\n x[i] = vec_add(x[i], x[offset + i]); \\\n } \\\n float32x4_t tmp = x[0] + vec_reve(x[0]); \\\n res = tmp[0] + tmp[1]; \\\n}\n#define GGML_F32x4_REDUCE_4(res, s0, s1, s2, s3) \\\n{ \\\n float32x4_t v = vec_add(vec_add(s0, s1), \\\n vec_add(s2, s3)); \\\n v = vec_add(v, vec_sld(v, v, 8)); \\\n v = vec_add(v, vec_sld(v, v, 4)); \\\n res += (ggml_float)vec_extract(v, 0); \\\n}\n\n#define GGML_F32_VEC GGML_F32x4\n#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 s390x\n#define GGML_F16_STEP GGML_F32_STEP\n#define GGML_F16_EPR GGML_F32_EPR\n\nstatic inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) {\n float tmp[4];\n\n for (int i = 0; i < 4; i++) {\n tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n }\n\n // note: keep type-cast here to prevent compiler bugs\n // see: https://github.com/ggml-org/llama.cpp/issues/12846\n return vec_xl(0, (const float *)(tmp));\n}\n\nstatic inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) {\n float arr[4];\n\n // note: keep type-cast here to prevent compiler bugs\n // see: https://github.com/ggml-org/llama.cpp/issues/12846\n vec_xst(v_y, 0, (float *)(arr));\n\n for (int i = 0; i < 4; i++) {\n x[i] = GGML_CPU_FP32_TO_FP16(arr[i]);\n }\n}\n\n#define GGML_F16_VEC GGML_F32x4\n#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO\n#define GGML_F16_VEC_SET1 GGML_F32x4_SET1\n#define GGML_F16_VEC_LOAD(p, i) __lzs_f16cx4_load(p)\n#define GGML_F16_VEC_STORE(p, r, i) __lzs_f16cx4_store(p, r[i])\n#define GGML_F16_VEC_FMA GGML_F32x4_FMA\n#define GGML_F16_VEC_ADD GGML_F32x4_ADD\n#define GGML_F16_VEC_MUL GGML_F32x4_MUL\n#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE\n\n// BF16 s390x\n#define GGML_BF16_STEP 16\n#define GGML_BF16_EPR 8\n\n#define GGML_BF16x8 __vector unsigned short\n#define GGML_BF16x8_ZERO vec_splats((unsigned short)0)\n#define GGML_BF16x8_LOAD(p) vec_xl(0, (const unsigned short *)(p))\n\n#define GGML_BF16_VEC GGML_BF16x8\n#define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO\n#define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD\n#define GGML_BF16_TO_F32_LO(v) ((float32x4_t) vec_mergel((v), GGML_BF16_VEC_ZERO))\n#define GGML_BF16_TO_F32_HI(v) ((float32x4_t) vec_mergeh((v), GGML_BF16_VEC_ZERO))\n#define GGML_BF16_FMA_LO(acc, x, y) \\\n (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y))\n#define GGML_BF16_FMA_HI(acc, x, y) \\\n (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y))\n\n#elif defined(__riscv_v_intrinsic)\n\n// compatible with vlen >= 128\n\n#define GGML_SIMD\n\n// F32\n\n#define GGML_F32_STEP 16\n#define GGML_F32_EPR 4\n\n#define GGML_F32x4 vfloat32m1_t\n#define GGML_F32x4_ZERO __riscv_vfmv_v_f_f32m1(0.0f, GGML_F32_EPR)\n#define GGML_F32x4_SET1(x) __riscv_vfmv_v_f_f32m1(x, GGML_F32_EPR)\n#define GGML_F32x4_LOAD(x) __riscv_vle32_v_f32m1(x, GGML_F32_EPR)\n#define GGML_F32x4_STORE(b, v) __riscv_vse32_v_f32m1(b, v, GGML_F32_EPR)\n#define GGML_F32x4_FMA(a, b, c) __riscv_vfmacc_vv_f32m1(a, b, c, GGML_F32_EPR)\n#define GGML_F32x4_ADD(a, b) __riscv_vfadd_vv_f32m1(a, b, GGML_F32_EPR)\n#define GGML_F32x4_MUL(a, b) __riscv_vfmul_vv_f32m1(a, b, GGML_F32_EPR)\n\n#define GGML_F32_VEC GGML_F32x4\n#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1 GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n#endif\n\n// GGML_F32_ARR / GGML_F16_ARR\n// number of registers to use per step\n#ifdef GGML_SIMD\n#define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR)\n#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR)\n#endif\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cpu/spacemit/ime.cpp", "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n\n#include \"ime.h\"\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-common.h\"\n#include \"ggml-cpu.h\"\n#include \"ime_kernels.h\"\n#include \"traits.h\"\n\n#include \n#include \n#include \n#include // for GGML_ASSERT\n#include \n#include \n\n// clang-format off\n#if defined(__riscv)\n\n#if !defined(__riscv_v) || !defined(__riscv_v_intrinsic)\n#error \"riscv v extension or v_intrinsic not enabled\"\n#else\n#include \n#endif\n\n#if !defined(__riscv_zfh)\n#error \"riscv zfh extension not enabled\"\n#endif\n\n#if defined(RISCV64_SPACEMIT_IME1)\n#else\n#error \"RISCV64_SPACEMIT_IME1 not defined\"\n#endif\n\n#else\n\n#error \"riscv not enabled in this build\"\n\n#endif\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#pragma GCC diagnostic ignored \"-Wcast-qual\"\n#pragma GCC diagnostic ignored \"-Wunused-parameter\"\n#endif\n\n#if defined(RISCV64_SPACEMIT_IME1)\n#define QGEMM_STRIDEN_THREAD_ALIGN 16\n#else\n#define QGEMM_STRIDEN_THREAD_ALIGN 32\n#endif\n\n// clang-format on\n\nstruct qnbitgemm_spacemit_ime_args {\n const float * a_ptr = nullptr;\n size_t lda = 0;\n const std::byte * packed_quant_b_data = nullptr;\n const float * quant_b_scale = nullptr;\n const void * quant_b_zp = nullptr;\n const float * quant_b_blksum = nullptr;\n const float * bias = nullptr;\n float * c_ptr = nullptr;\n size_t ldc = 0;\n};\n\nconstexpr size_t div_round_up(size_t up, size_t down) {\n return (up + down - 1) / down;\n}\n\nconstexpr size_t q8_blk_size(size_t blk_len) {\n const size_t blk_size = sizeof(float) + blk_len * sizeof(int8_t);\n // Currently, the strictest alignment requirement of a block is for a float.\n // Ensure contiguous blocks are suitably aligned.\n assert(blk_size % alignof(float) == 0);\n return blk_size;\n}\n\nnamespace ggml::cpu::riscv64_spacemit {\n\nconst int num_ai_cores = std::thread::hardware_concurrency() / 2;\n\n} // namespace ggml::cpu::riscv64_spacemit\n\nstatic void sqnbitgemm_spacemit_ime_i8i4(const size_t blk_len,\n const size_t gemm_k,\n const qnbitgemm_spacemit_ime_args * gemm_args,\n void * const per_gemm_ws,\n const size_t m_start,\n const size_t m_count,\n const size_t n_start,\n const size_t n_count) {\n constexpr size_t scale_stride = sizeof(uint16_t);\n constexpr size_t blk_bitwidth = 4;\n\n const size_t k_blks = div_round_up(gemm_k, blk_len);\n\n const size_t lda = k_blks * q8_blk_size(blk_len);\n const size_t ldc = gemm_args->ldc;\n const size_t ldb = k_blks * (blk_len * blk_bitwidth / 8);\n const std::byte * quant_a_ptr = static_cast(per_gemm_ws) + m_start * lda;\n\n const size_t zero_point_stride = gemm_args->quant_b_zp != nullptr ? sizeof(uint8_t) : 0;\n const size_t packed_b_stride = ldb + k_blks * (scale_stride + zero_point_stride);\n const std::byte * packed_quant_b_data = gemm_args->packed_quant_b_data + n_start * packed_b_stride;\n\n float * c_ptr = gemm_args->c_ptr + m_start * ldc + n_start;\n\n size_t count_n = 0;\n const size_t compute_block_count_n = m_count == 1 ? n_count : 16;\n for (size_t n = 0; n < n_count; n += count_n) {\n count_n = std::min(n_count - n, compute_block_count_n);\n\n const std::byte * a_row = quant_a_ptr;\n const std::byte * b_col = packed_quant_b_data + n * packed_b_stride;\n const std::byte * b_col_zp = (zero_point_stride != 0) ? b_col : nullptr;\n float * c_blk = c_ptr + n;\n\n int32_t rows_remaining = m_count;\n\n while (rows_remaining > 0) {\n const auto rows_handled = sqnbitgemm_spacemit_ime::ime1::gemm_kernel_i8i4(\n blk_len, a_row, b_col, nullptr, b_col_zp, c_blk, rows_remaining, count_n, gemm_k, k_blks, ldc, nullptr,\n scale_stride);\n\n c_blk += rows_handled * ldc;\n a_row += rows_handled * lda;\n\n rows_remaining -= rows_handled;\n }\n }\n}\n\ntemplate constexpr int QK_0() {\n if constexpr (K == 4) {\n return QK4_0;\n }\n if constexpr (K == 8) {\n return QK8_0;\n }\n return -1;\n}\n\ntemplate struct block {\n ggml_half d[N]; // deltas for N qK_0 blocks\n uint8_t qs[(QK_0() * N * K) / 8]; // quants for N qK_0 blocks\n};\n\ntemplate struct block_with_zp {\n ggml_half d[N]; // deltas for N qK_1 blocks\n uint8_t zp[N]; // zero points for N qK_1 blocks\n uint8_t qs[(QK_0() * N * K) / 8]; // quants for N qK_1 blocks\n};\n\n// control size\nstatic_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 8, \"wrong block<4,16> size/padding\");\nstatic_assert(sizeof(block_with_zp<4, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 8 + 16 * sizeof(uint8_t),\n \"wrong block_with_zp<4,16> size/padding\");\nstatic_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 16, \"wrong block<8,16> size/padding\");\n\nusing block_q4_0x16 = block<4, 16>;\nusing block_q4_1x16 = block_with_zp<4, 16>;\nusing block_q8_0x16 = block<8, 16>;\n\nstatic block_q4_0x16 make_block_q4_0x16(block_q4_0 * in, unsigned int blck_size_interleave) {\n block_q4_0x16 out;\n GGML_ASSERT(QK4_0 / blck_size_interleave == 2);\n\n for (int i = 0; i < 16; i++) {\n out.d[i] = in[i].d;\n }\n\n for (int i = 0; i < 16; i++) {\n // [0, 15], in.d & 0x0F\n for (int j = 0; j < QK4_0 / 4; j++) {\n //src [b0 b16] ......... [b8 b24] ......... [b15 b31]\n //dst [b0 b8] ......... [b7 b15]\n out.qs[i * QK4_0 / 4 + j] = (in[i].qs[j] & 0x0F) | ((in[i].qs[j + QK4_0 / 4] & 0x0F) << 4);\n }\n }\n\n for (int i = 0; i < 16; i++) {\n // [16, 31], in.d & 0xF0\n for (int j = 0; j < QK4_0 / 4; j++) {\n //src [b0 b16] ......... [b8 b24] ......... [b15 b31]\n //dst [b16 b24] ......... [b23 b31]\n out.qs[4 * QK4_0 + i * QK4_0 / 4 + j] = ((in[i].qs[j] & 0xF0) >> 4) | (in[i].qs[j + QK4_0 / 4] & 0xF0);\n }\n }\n\n return out;\n}\n\nstatic block_q4_1x16 make_block_q4_1x16(block_q4_1 * in, unsigned int blck_size_interleave) {\n block_q4_1x16 out;\n GGML_ASSERT(QK4_1 / blck_size_interleave == 2);\n\n for (int i = 0; i < 16; i++) {\n float d = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d);\n float m = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m);\n float mid = -std::nearbyintf(m / d);\n mid = std::min(15.0f, std::max(0.0f, mid));\n out.d[i] = GGML_FP32_TO_FP16(d);\n out.zp[i] = static_cast(mid);\n }\n\n for (int i = 0; i < 16; i++) {\n // [0, 15], in.d & 0x0F\n for (int j = 0; j < QK4_1 / 4; j++) {\n //src [b0 b16] ......... [b8 b24] ......... [b15 b31]\n //dst [b0 b8] ......... [b7 b15]\n out.qs[i * QK4_1 / 4 + j] = (in[i].qs[j] & 0x0F) | ((in[i].qs[j + QK4_1 / 4] & 0x0F) << 4);\n }\n }\n\n for (int i = 0; i < 16; i++) {\n // [16, 31], in.d & 0xF0\n for (int j = 0; j < QK4_1 / 4; j++) {\n //src [b0 b16] ......... [b8 b24] ......... [b15 b31]\n //dst [b16 b24] ......... [b23 b31]\n out.qs[4 * QK4_1 + i * QK4_1 / 4 + j] = ((in[i].qs[j] & 0xF0) >> 4) | (in[i].qs[j + QK4_1 / 4] & 0xF0);\n }\n }\n\n return out;\n}\n\nstatic int repack_q4_0_to_q4_0_16_bl(struct ggml_tensor * t,\n int interleave_block,\n const void * GGML_RESTRICT data,\n size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q4_0);\n GGML_ASSERT(interleave_block == 16);\n\n constexpr int nrows_interleaved = 16;\n\n block_q4_0x16 * dst = (block_q4_0x16 *) t->data;\n const block_q4_0 * src = (const block_q4_0 *) data;\n block_q4_0 dst_tmp[16];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK4_0;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_0 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q4_0x16(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic int repack_q4_1_to_q4_1_16_bl(struct ggml_tensor * t,\n int interleave_block,\n const void * GGML_RESTRICT data,\n size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q4_1);\n GGML_ASSERT(interleave_block == 16);\n\n constexpr int nrows_interleaved = 16;\n\n block_q4_1x16 * dst = (block_q4_1x16 *) t->data;\n const block_q4_1 * src = (const block_q4_1 *) data;\n block_q4_1 dst_tmp[16];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK4_1;\n\n GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_1));\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_1 != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n dst_tmp[i] = src[x + i * nblocks];\n }\n *dst++ = make_block_q4_1x16(dst_tmp, interleave_block);\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nstatic inline void get_scale_min_k4(int j,\n const uint8_t * GGML_RESTRICT q,\n uint8_t * GGML_RESTRICT d,\n uint8_t * GGML_RESTRICT m) {\n if (j < 4) {\n *d = q[j] & 63;\n *m = q[j + 4] & 63;\n } else {\n *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);\n *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);\n }\n}\n\nstatic int repack_q4_k_to_q4_1_16_bl(struct ggml_tensor * t,\n int interleave_block,\n const void * GGML_RESTRICT data,\n size_t data_size) {\n GGML_ASSERT(t->type == GGML_TYPE_Q4_K);\n GGML_ASSERT(interleave_block == 16);\n GGML_ASSERT(QK_K / QK4_1 == 8);\n\n constexpr int nrows_interleaved = 16;\n\n block_q4_1x16 * dst = (block_q4_1x16 *) t->data;\n const block_q4_K * src = (const block_q4_K *) data;\n block_q4_1 dst_tmp[16];\n int nrow = ggml_nrows(t);\n int nblocks = t->ne[0] / QK_K;\n\n if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_K != 0) {\n return -1;\n }\n\n for (int b = 0; b < nrow; b += nrows_interleaved) {\n for (int64_t x = 0; x < nblocks; x++) {\n for (int j = 0; j < 8; j++) {\n for (int i = 0; i < nrows_interleaved; i++) {\n uint8_t sc, m;\n const float d = GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d);\n const float min =\n GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin);\n get_scale_min_k4(j, src[x + i * nblocks].scales, &sc, &m);\n const float d1 = d * sc;\n const float m1 = min * m;\n\n dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d = GGML_FP32_TO_FP16(d1);\n dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m = GGML_FP32_TO_FP16(-m1);\n // src -> [b0, b32] [b1, b33] ... [b31, b63]\n // dst -> [b0, b16] [b1, b17] ... [b15, b31] [b32, b48] [b33, b49] ... [b47, b63]\n const uint8_t * q = src[x + i * nblocks].qs + (j / 2) * QK4_1;\n if (j % 2 == 0) {\n for (int ii = 0; ii < 16; ii++) {\n dst_tmp[i].qs[ii] = (q[ii] & 0x0F) | ((q[ii + 16] & 0x0F) << 4);\n }\n } else {\n for (int ii = 0; ii < 16; ii++) {\n dst_tmp[i].qs[ii] = ((q[ii] & 0xF0) >> 4) | (q[ii + 16] & 0xF0);\n }\n }\n }\n *dst++ = make_block_q4_1x16(dst_tmp, interleave_block);\n }\n }\n src += nrows_interleaved * nblocks;\n }\n return 0;\n\n GGML_UNUSED(data_size);\n}\n\nnamespace ggml::cpu::riscv64_spacemit {\n\ntemplate \nint repack(struct ggml_tensor *, const void *, size_t);\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_0_to_q4_0_16_bl(t, 16, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_1_to_q4_1_16_bl(t, 16, data, data_size);\n}\n\ntemplate <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) {\n return repack_q4_k_to_q4_1_16_bl(t, 16, data, data_size);\n}\n\nclass tensor_traits_base : public ggml::cpu::tensor_traits {\n public:\n virtual int repack(struct ggml_tensor * t, const void * data, size_t data_size) = 0;\n};\n\ntemplate class tensor_traits : public tensor_traits_base {\n bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n switch (op->op) {\n case GGML_OP_MUL_MAT:\n size = ggml_row_size(GGML_TYPE_Q8_0, ggml_nelements(op->src[1])) * 4;\n size = ((size + QK4_0 - 1) / QK4_0) * (QK4_0 * sizeof(float) + sizeof(float));\n return true;\n default:\n // GGML_ABORT(\"fatal error\");\n break;\n }\n return false;\n }\n\n bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override {\n switch (op->op) {\n case GGML_OP_MUL_MAT:\n if (op->src[0]->type == GGML_TYPE_Q4_0 || //\n op->src[0]->type == GGML_TYPE_Q4_1 || //\n op->src[0]->type == GGML_TYPE_Q4_K) {\n forward_mul_mat_q4(params, op);\n return true;\n }\n default:\n // GGML_ABORT(\"fatal error\");\n break;\n }\n return false;\n }\n\n void forward_mul_mat_q4(ggml_compute_params * params, ggml_tensor * op) {\n const ggml_tensor * src0 = op->src[0];\n const ggml_tensor * src1 = op->src[1];\n ggml_tensor * dst = op;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n int ith = params->ith;\n int nth = params->nth;\n\n [[maybe_unused]] const enum ggml_type type = src0->type;\n\n void * w_data = (void *) src0->data;\n const float * feature = (const float *) src1->data;\n float * output = (float *) dst->data;\n\n const size_t batch_feature = ne12 * ne13;\n [[maybe_unused]] const size_t batch_weight = ne02 * ne03;\n const size_t gemm_m = ne11;\n const size_t gemm_k = ne10;\n const size_t gemm_n = ne01;\n\n GGML_ASSERT(batch_weight == 1);\n\n const size_t block_count_k = div_round_up(gemm_k, QK4_0);\n const size_t per_gemm_workspace_size = gemm_m * block_count_k * q8_blk_size(QK4_0);\n const size_t per_gemm_workspace_stride =\n div_round_up(per_gemm_workspace_size, alignof(uint64_t)) * alignof(uint64_t);\n const size_t gemm_workspace_size = batch_feature * per_gemm_workspace_stride;\n const size_t desired_wsize = gemm_workspace_size + alignof(uint64_t) - 1;\n\n if (ith == 0 && params->wsize < desired_wsize) {\n throw std::runtime_error(\"wsize less than desired_wsize\");\n }\n\n std::vector qnbitgemm_args(batch_feature);\n\n for (size_t i = 0; i < batch_feature; i++) {\n qnbitgemm_args[i].a_ptr = feature + gemm_m * gemm_k * i;\n qnbitgemm_args[i].lda = gemm_k;\n qnbitgemm_args[i].packed_quant_b_data = (const std::byte *) w_data;\n qnbitgemm_args[i].quant_b_scale = nullptr;\n\n if constexpr (std::is_same_v) {\n qnbitgemm_args[i].quant_b_zp = nullptr;\n } else {\n qnbitgemm_args[i].quant_b_zp = w_data;\n }\n\n qnbitgemm_args[i].bias = nullptr;\n qnbitgemm_args[i].c_ptr = output + gemm_m * gemm_n * i;\n qnbitgemm_args[i].ldc = gemm_n;\n }\n\n const uintptr_t ws_ptr = reinterpret_cast(params->wdata);\n void * ws = reinterpret_cast((ws_ptr + alignof(uint64_t) - 1) & (~(alignof(uint64_t) - 1)));\n const size_t quant_a_stride = block_count_k * q8_blk_size(QK4_0);\n\n {\n constexpr size_t block_size_m = 4;\n size_t per_gemm_block_count_m = div_round_up(gemm_m, block_size_m);\n int32_t task_count = batch_feature * per_gemm_block_count_m;\n int32_t task_per_thread = (task_count + nth - 1) / nth;\n int32_t start = ith * task_per_thread;\n int32_t end = std::min((ith + 1) * task_per_thread, task_count);\n for (int32_t compute_idx = start; compute_idx < end; compute_idx++) {\n int32_t gemm_idx = compute_idx / per_gemm_block_count_m;\n int32_t block_idx_in_gemm = compute_idx % per_gemm_block_count_m;\n int32_t m_idx = block_idx_in_gemm * block_size_m;\n const qnbitgemm_spacemit_ime_args & data = qnbitgemm_args[gemm_idx];\n int32_t rows_tobe_handled = (gemm_m - m_idx) > block_size_m ? block_size_m : (gemm_m - m_idx);\n\n if (rows_tobe_handled == block_size_m) {\n const float * a_row_ptr = data.a_ptr + m_idx * data.lda;\n std::byte * quant_a_row_ptr =\n static_cast(ws) + gemm_idx * per_gemm_workspace_stride + m_idx * quant_a_stride;\n sqnbitgemm_spacemit_ime::ime1::quantize_a_4row_i8(QK4_0, a_row_ptr, gemm_k, quant_a_row_ptr);\n } else {\n while (rows_tobe_handled) {\n const float * a_row_ptr = data.a_ptr + m_idx * data.lda;\n std::byte * quant_a_row_ptr = static_cast(ws) +\n gemm_idx * per_gemm_workspace_stride + m_idx * quant_a_stride;\n sqnbitgemm_spacemit_ime::ime1::quantize_a_row_i8(QK4_0, a_row_ptr, gemm_k, quant_a_row_ptr);\n rows_tobe_handled -= 1;\n m_idx += 1;\n }\n }\n }\n }\n\n ggml_barrier(params->threadpool);\n\n if (ith >= ggml::cpu::riscv64_spacemit::num_ai_cores) {\n return;\n }\n nth = std::min(nth, int{ ggml::cpu::riscv64_spacemit::num_ai_cores });\n\n size_t threads_per_gemm = nth / batch_feature;\n constexpr size_t gemm_m_stride = 128;\n size_t nc = gemm_n;\n const size_t gemm_m_blocked = div_round_up(gemm_m, gemm_m_stride);\n const size_t max_nc = div_round_up(gemm_n * gemm_m_blocked, threads_per_gemm);\n if (max_nc < nc) {\n nc = std::min(nc, div_round_up(max_nc, QGEMM_STRIDEN_THREAD_ALIGN) * QGEMM_STRIDEN_THREAD_ALIGN);\n }\n const size_t gemm_n_stride = nc;\n const size_t thread_count_m = div_round_up(gemm_m, gemm_m_stride);\n const size_t thread_count_n = div_round_up(gemm_n, gemm_n_stride);\n threads_per_gemm = thread_count_m * thread_count_n;\n\n {\n int task_count = batch_feature * threads_per_gemm;\n int task_per_thread = (task_count + nth - 1) / nth;\n int start = ith * task_per_thread;\n int end = std::min((ith + 1) * task_per_thread, task_count);\n for (int compute_idx = start; compute_idx < end; compute_idx++) {\n const auto gemm_i = compute_idx / threads_per_gemm;\n const auto blk_i = compute_idx % threads_per_gemm;\n const auto * data = &qnbitgemm_args[gemm_i];\n\n const auto tid_n = blk_i / thread_count_m;\n const auto tid_m = blk_i % thread_count_m;\n\n const size_t m_start = tid_m * gemm_m_stride;\n const size_t m_count = std::min(gemm_m - m_start, (size_t) gemm_m_stride);\n\n const size_t n_start = tid_n * gemm_n_stride;\n const size_t n_count = std::min(gemm_n - n_start, (size_t) gemm_n_stride);\n\n void * per_gemm_ws = reinterpret_cast(ws) + gemm_i * per_gemm_workspace_stride;\n\n sqnbitgemm_spacemit_ime_i8i4(QK4_0, gemm_k, data, per_gemm_ws, m_start, m_count, n_start, n_count);\n }\n }\n }\n\n int repack(struct ggml_tensor * t, const void * data, size_t data_size) override {\n GGML_LOG_DEBUG(\"%s: repack tensor %s with %s_%dx%d\\n\", __func__, t->name, ggml_type_name(t->type),\n (int) NB_COLS, (int) INTER_SIZE);\n return ggml::cpu::riscv64_spacemit::repack(t, data, data_size);\n }\n};\n\nclass tensor_traits_common : public tensor_traits_base {\n bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n switch (op->op) {\n case GGML_OP_NORM:\n case GGML_OP_RMS_NORM:\n size = 0;\n return true;\n default:\n // GGML_ABORT(\"fatal error\");\n break;\n }\n return false;\n }\n\n bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override {\n switch (op->op) {\n case GGML_OP_NORM:\n forward_norm_f32(params, op);\n return true;\n case GGML_OP_RMS_NORM:\n forward_rms_norm_f32(params, op);\n return true;\n default:\n // GGML_ABORT(\"fatal error\");\n break;\n }\n return false;\n }\n\n void forward_norm_f32(ggml_compute_params * params, ggml_tensor * op) {\n const ggml_tensor * src0 = op->src[0];\n ggml_tensor * dst = op;\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n float epsilon;\n memcpy(&epsilon, dst->op_params, sizeof(float));\n\n GGML_ASSERT(epsilon > 0.0f);\n\n auto * input = (float *) src0->data;\n auto * output = (float *) dst->data;\n\n const auto hidden_size = ne00;\n const auto task_count = ne01 * ne02 * ne03;\n const auto task_per_thread = (task_count + nth - 1) / nth;\n\n const auto task_begin = ith * task_per_thread;\n const auto task_end = std::min((ith + 1) * task_per_thread, task_count);\n\n for (auto task_idx = task_begin; task_idx < task_end; task_idx++) {\n auto offset = task_idx * hidden_size;\n auto * p_input = const_cast(input + offset);\n\n auto * p_output = output + offset;\n auto * p_temp_output = p_output;\n auto * p_gamma_data = (const float *) nullptr;\n auto * p_beta_data = (const float *) nullptr;\n size_t gvl = __riscv_vsetvlmax_e32m4();\n vfloat32m4_t sum = __riscv_vfmv_v_f_f32m4(0.f, gvl);\n vfloat32m4_t sum_sq = __riscv_vfmv_v_f_f32m4(0.f, gvl);\n int64_t length = hidden_size;\n while (length > 0) {\n gvl = __riscv_vsetvl_e32m4(length);\n // load data\n vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_input, gvl);\n\n sum = __riscv_vfadd_vv_f32m4(sum, src_data, gvl);\n sum_sq = __riscv_vfmacc_vv_f32m4(sum_sq, src_data, src_data, gvl);\n\n __riscv_vse32_v_f32m4(p_temp_output, src_data, gvl);\n\n p_input += gvl;\n p_temp_output += gvl;\n length -= gvl;\n }\n\n gvl = __riscv_vsetvlmax_e32m1();\n\n float mean = 0.f;\n vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.f, gvl);\n vfloat32m1_t mean_v =\n __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum, 0), __riscv_vget_v_f32m4_f32m1(sum, 1), gvl);\n mean_v = __riscv_vfadd_vv_f32m1(mean_v, __riscv_vget_v_f32m4_f32m1(sum, 2), gvl);\n mean_v = __riscv_vfadd_vv_f32m1(mean_v, __riscv_vget_v_f32m4_f32m1(sum, 3), gvl);\n mean_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_v, zero_v, gvl);\n mean = __riscv_vfmv_f_s_f32m1_f32(mean_v);\n mean /= hidden_size;\n\n vfloat32m1_t mean_square_v = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum_sq, 0),\n __riscv_vget_v_f32m4_f32m1(sum_sq, 1), gvl);\n mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 2), gvl);\n mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 3), gvl);\n mean_square_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_square_v, zero_v, gvl);\n\n float mean_square = __riscv_vfmv_f_s_f32m1_f32(mean_square_v);\n mean_square /= hidden_size;\n mean_square = sqrt(mean_square - mean * mean + epsilon);\n\n mean_square = 1.0f / mean_square;\n length = hidden_size;\n p_temp_output = p_output;\n\n if (p_gamma_data == nullptr && p_beta_data == nullptr) {\n while (length > 0) {\n gvl = __riscv_vsetvl_e32m4(length);\n vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n src_data = __riscv_vfsub_vf_f32m4(src_data, mean, gvl);\n src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n p_temp_output += gvl;\n p_output += gvl;\n length -= gvl;\n }\n } else if (p_beta_data == nullptr) {\n while (length > 0) {\n gvl = __riscv_vsetvl_e32m4(length);\n vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl);\n src_data = __riscv_vfsub_vf_f32m4(src_data, mean, gvl);\n src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n src_data = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl);\n __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n p_temp_output += gvl;\n p_output += gvl;\n p_gamma_data += gvl;\n length -= gvl;\n }\n } else if (p_gamma_data != nullptr) {\n while (length > 0) {\n gvl = __riscv_vsetvl_e32m4(length);\n vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl);\n src_data = __riscv_vfsub_vf_f32m4(src_data, mean, gvl);\n src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n src_data = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl);\n vfloat32m4_t beta_data_v = __riscv_vle32_v_f32m4(p_beta_data, gvl);\n src_data = __riscv_vfadd_vv_f32m4(src_data, beta_data_v, gvl);\n p_beta_data += gvl;\n __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n p_temp_output += gvl;\n p_output += gvl;\n p_gamma_data += gvl;\n length -= gvl;\n }\n }\n }\n }\n\n void forward_rms_norm_f32(ggml_compute_params * params, ggml_tensor * op) {\n const ggml_tensor * src0 = op->src[0];\n ggml_tensor * dst = op;\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n const int ith = params->ith;\n const int nth = params->nth;\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n float epsilon;\n memcpy(&epsilon, dst->op_params, sizeof(float));\n\n GGML_ASSERT(epsilon > 0.0f);\n\n auto * input = (float *) src0->data;\n auto * output = (float *) dst->data;\n\n const auto hidden_size = ne00;\n const auto task_count = ne01 * ne02 * ne03;\n const auto task_per_thread = (task_count + nth - 1) / nth;\n\n const auto task_begin = ith * task_per_thread;\n const auto task_end = std::min((ith + 1) * task_per_thread, task_count);\n\n for (auto task_idx = task_begin; task_idx < task_end; task_idx++) {\n auto offset = task_idx * hidden_size;\n auto * p_input = const_cast(input + offset);\n auto * p_output = output + offset;\n auto * p_temp_output = p_output;\n auto * p_gamma_data = (const float *) nullptr;\n auto * p_beta_data = (const float *) nullptr;\n\n size_t gvl = __riscv_vsetvlmax_e32m4();\n // vfloat32m4_t sum = __riscv_vfmv_v_f_f32m4(0.f, gvl);\n vfloat32m4_t sum_sq = __riscv_vfmv_v_f_f32m4(0.f, gvl);\n int64_t length = hidden_size;\n while (length > 0) {\n gvl = __riscv_vsetvl_e32m4(length);\n // load data\n vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_input, gvl);\n\n sum_sq = __riscv_vfmacc_vv_f32m4(sum_sq, src_data, src_data, gvl);\n\n __riscv_vse32_v_f32m4(p_temp_output, src_data, gvl);\n\n p_input += gvl;\n p_temp_output += gvl;\n length -= gvl;\n }\n\n gvl = __riscv_vsetvlmax_e32m1();\n\n // float mean = 0.f;\n vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.f, gvl);\n\n vfloat32m1_t mean_square_v = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum_sq, 0),\n __riscv_vget_v_f32m4_f32m1(sum_sq, 1), gvl);\n mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 2), gvl);\n mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 3), gvl);\n mean_square_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_square_v, zero_v, gvl);\n\n float mean_square = __riscv_vfmv_f_s_f32m1_f32(mean_square_v);\n mean_square /= hidden_size;\n\n mean_square = sqrt(mean_square + epsilon);\n\n mean_square = 1.0f / mean_square;\n length = hidden_size;\n p_temp_output = p_output;\n\n if (p_gamma_data == nullptr && p_beta_data == nullptr) {\n while (length > 0) {\n gvl = __riscv_vsetvl_e32m4(length);\n vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n p_temp_output += gvl;\n p_output += gvl;\n length -= gvl;\n }\n } else if (p_beta_data == nullptr) {\n while (length > 0) {\n gvl = __riscv_vsetvl_e32m4(length);\n vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl);\n src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n src_data = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl);\n __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n p_temp_output += gvl;\n p_output += gvl;\n p_gamma_data += gvl;\n length -= gvl;\n }\n } else if (p_gamma_data != nullptr) {\n while (length > 0) {\n gvl = __riscv_vsetvl_e32m4(length);\n vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl);\n src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n src_data = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl);\n vfloat32m4_t beta_data_v = __riscv_vle32_v_f32m4(p_beta_data, gvl);\n src_data = __riscv_vfadd_vv_f32m4(src_data, beta_data_v, gvl);\n p_beta_data += gvl;\n __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n p_temp_output += gvl;\n p_output += gvl;\n p_gamma_data += gvl;\n length -= gvl;\n }\n }\n }\n }\n\n int repack(struct ggml_tensor * t, const void * data, size_t data_size) override {\n memcpy(t->data, data, data_size);\n return 0;\n }\n};\n\nstatic const tensor_traits q4_0_16x8_q8_0;\nstatic const tensor_traits q4_1_16x8_q8_0;\nstatic const tensor_traits q4_k_16x8_q8_0;\nstatic const tensor_traits_common rvv_impl;\n\n} // namespace ggml::cpu::riscv64_spacemit\n\nstatic const ggml::cpu::tensor_traits * ggml_riscv64_spacemit_get_optimal_repack_type(const struct ggml_tensor * cur) {\n if (cur->type == GGML_TYPE_Q4_0) {\n if (cur->ne[1] % 16 == 0) {\n return &ggml::cpu::riscv64_spacemit::q4_0_16x8_q8_0;\n }\n } else if (cur->type == GGML_TYPE_Q4_1) {\n if (cur->ne[1] % 16 == 0) {\n return &ggml::cpu::riscv64_spacemit::q4_1_16x8_q8_0;\n }\n } else if (cur->type == GGML_TYPE_Q4_K) {\n if (cur->ne[1] % 16 == 0) {\n return &ggml::cpu::riscv64_spacemit::q4_k_16x8_q8_0;\n }\n } else if (cur->type == GGML_TYPE_F32) {\n return &ggml::cpu::riscv64_spacemit::rvv_impl;\n }\n\n return nullptr;\n}\n\nstatic enum ggml_status ggml_backend_riscv64_spacemit_buffer_init_tensor(ggml_backend_buffer_t buffer,\n struct ggml_tensor * tensor) {\n tensor->extra =\n (void *) const_cast(ggml_riscv64_spacemit_get_optimal_repack_type(tensor));\n\n GGML_UNUSED(buffer);\n\n return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_riscv64_spacemit_buffer_set_tensor(ggml_backend_buffer_t buffer,\n struct ggml_tensor * tensor,\n const void * data,\n size_t offset,\n size_t size) {\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(size == ggml_nbytes(tensor));\n\n auto tensor_traits = (ggml::cpu::riscv64_spacemit::tensor_traits_base *) tensor->extra;\n if (tensor_traits) {\n auto OK = tensor_traits->repack(tensor, data, size);\n GGML_ASSERT(OK == 0);\n }\n\n GGML_UNUSED(buffer);\n}\n\nstatic const char * ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n return \"CPU_RISCV64_SPACEMIT\";\n\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_riscv64_spacemit_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,\n size_t size) {\n ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n\n if (buffer == nullptr) {\n return nullptr;\n }\n\n buffer->buft = buft;\n buffer->iface.init_tensor = ggml_backend_riscv64_spacemit_buffer_init_tensor;\n buffer->iface.set_tensor = ggml_backend_riscv64_spacemit_buffer_set_tensor;\n buffer->iface.get_tensor = nullptr;\n buffer->iface.cpy_tensor = nullptr;\n return buffer;\n}\n\nstatic size_t ggml_backend_cpu_riscv64_spacemit_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n return 64;\n\n GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_cpu_riscv64_spacemit_nbytes(ggml_backend_buffer_type_t buft,\n const struct ggml_tensor * tensor) {\n for (int i = 0; i < GGML_MAX_DIMS; ++i) {\n if (tensor->ne[i] <= 0) {\n return 0;\n }\n }\n\n size_t nbytes;\n const size_t blck_size = ggml_blck_size(tensor->type);\n if (blck_size == 1) {\n nbytes = ggml_type_size(tensor->type);\n for (int i = 0; i < GGML_MAX_DIMS; ++i) {\n nbytes += (tensor->ne[i] - 1) * tensor->nb[i];\n }\n } else {\n nbytes = tensor->ne[0] * tensor->nb[0] / blck_size;\n if (tensor->type == GGML_TYPE_Q4_K) {\n GGML_ASSERT(nbytes % sizeof(block_q4_K) == 0);\n nbytes = (nbytes / sizeof(block_q4_K)) * sizeof(block_q4_1) * 8;\n for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n nbytes += (tensor->ne[i] - 1) * (tensor->nb[i] / sizeof(block_q4_K)) * sizeof(block_q4_1) * 8;\n }\n } else {\n for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n nbytes += (tensor->ne[i] - 1) * tensor->nb[i];\n }\n }\n }\n\n GGML_UNUSED(buft);\n return nbytes;\n}\n\nnamespace ggml::cpu::riscv64_spacemit {\n\nclass extra_buffer_type : ggml::cpu::extra_buffer_type {\n bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {\n switch (op->op) {\n case GGML_OP_MUL_MAT:\n if (op->src[0]->buffer && (ggml_n_dims(op->src[0]) == 2) &&\n op->src[0]->buffer->buft == ggml_backend_cpu_riscv64_spacemit_buffer_type() &&\n ggml_riscv64_spacemit_get_optimal_repack_type(op->src[0])) {\n if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {\n return false;\n }\n if (op->src[1]->type == GGML_TYPE_F32) {\n return true;\n }\n }\n break;\n case GGML_OP_NORM:\n case GGML_OP_RMS_NORM:\n if (op->src[0]->type == GGML_TYPE_F32) {\n return true;\n }\n break;\n default:\n // GGML_ABORT(\"fatal error\");\n break;\n }\n return false;\n }\n\n ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {\n switch (op->op) {\n case GGML_OP_MUL_MAT:\n if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_riscv64_spacemit_buffer_type()) {\n return (ggml::cpu::tensor_traits *) op->src[0]->extra;\n }\n break;\n case GGML_OP_NORM:\n case GGML_OP_RMS_NORM:\n return (ggml::cpu::tensor_traits *) (&ggml::cpu::riscv64_spacemit::rvv_impl);\n default:\n // GGML_ABORT(\"fatal error\");\n break;\n }\n\n return nullptr;\n }\n};\n\n} // namespace ggml::cpu::riscv64_spacemit\n\nggml_backend_buffer_type_t ggml_backend_cpu_riscv64_spacemit_buffer_type(void) {\n static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_riscv64_spacemit = {\n /* .iface = */\n {\n /* .get_name = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_get_alignment,\n /* .get_max_size = */ nullptr,\n /* .get_alloc_size = */ ggml_backend_cpu_riscv64_spacemit_nbytes,\n /* .is_host = */ nullptr,\n },\n /* .device = */\n ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n /* .context = */\n new ggml::cpu::riscv64_spacemit::extra_buffer_type(),\n };\n\n return &ggml_backend_cpu_buffer_type_riscv64_spacemit;\n}\n" }, { "path": "src/ggml-cpu/spacemit/ime.h", "content": "#pragma once\n\n#include \"ggml-alloc.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nggml_backend_buffer_type_t ggml_backend_cpu_riscv64_spacemit_buffer_type(void);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cpu/spacemit/ime1_kernels.cpp", "content": "#include \"ggml.h\"\n#include \"ime_kernels.h\"\n\n#include \n#include \n\n// clang-format off\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#pragma GCC diagnostic ignored \"-Wcast-qual\"\n#pragma GCC diagnostic ignored \"-Wunused-parameter\"\n#endif\n// clang-format on\nnamespace sqnbitgemm_spacemit_ime {\n\n#define QUANTIZEM4ROW_KERNEL \\\n \"vmv.s.x v16, zero \\n\\t\" \\\n \"vfabs.v v8, v0 \\n\\t\" \\\n \"vfredmax.vs v16, v8, v16 \\n\\t\" \\\n \"vfmv.f.s f10, v16 \\n\\t\" \\\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\" \\\n \"fsw f10, (a1) \\n\\t\" \\\n \"fdiv.s f11, %[FONE], f10 \\n\\t\" \\\n \"vfmul.vf v16, v0, f11 \\n\\t\" \\\n \"vfcvt.x.f.v v16, v16 \\n\\t\" \\\n \"vsetvli t0, zero, e16, mf2 \\n\\t\" \\\n \"vnclip.wx v16, v16, zero \\n\\t\" \\\n \"vnclip.wx v17, v17, zero \\n\\t\" \\\n \"vnclip.wx v18, v18, zero \\n\\t\" \\\n \"vnclip.wx v19, v19, zero \\n\\t\" \\\n \"vnclip.wx v20, v20, zero \\n\\t\" \\\n \"vnclip.wx v21, v21, zero \\n\\t\" \\\n \"vnclip.wx v22, v22, zero \\n\\t\" \\\n \"vnclip.wx v23, v23, zero \\n\\t\" \\\n \"vsetvli t0, zero, e8, mf4 \\n\\t\" \\\n \"vnclip.wx v24, v16, zero \\n\\t\" \\\n \"vnclip.wx v25, v17, zero \\n\\t\" \\\n \"vnclip.wx v26, v18, zero \\n\\t\" \\\n \"vnclip.wx v27, v19, zero \\n\\t\" \\\n \"vnclip.wx v28, v20, zero \\n\\t\" \\\n \"vnclip.wx v29, v21, zero \\n\\t\" \\\n \"vnclip.wx v30, v22, zero \\n\\t\" \\\n \"vnclip.wx v31, v23, zero \\n\\t\"\n\n#define QUANTIZEM4ROW_STORE \\\n \"addi t1, %[BlkLen], 0 \\n\\t\" \\\n \"vsetvli t0, t1, e8, mf4 \\n\\t\" \\\n \"vse8.v v24, (s1) \\n\\t\" \\\n \"addi s1, s1, 32 \\n\\t\" \\\n \"sub t1, t1, t0 \\n\\t\" \\\n \"vsetvli t0, t1, e8, mf4 \\n\\t\" \\\n \"vse8.v v25, (s1) \\n\\t\" \\\n \"addi s1, s1, 32 \\n\\t\" \\\n \"sub t1, t1, t0 \\n\\t\" \\\n \"vsetvli t0, t1, e8, mf4 \\n\\t\" \\\n \"vse8.v v26, (s1) \\n\\t\" \\\n \"addi s1, s1, 32 \\n\\t\" \\\n \"sub t1, t1, t0 \\n\\t\" \\\n \"vsetvli t0, t1, e8, mf4 \\n\\t\" \\\n \"vse8.v v27, (s1) \\n\\t\" \\\n \"addi s1, s1, 32 \\n\\t\" \\\n \"sub t1, t1, t0 \\n\\t\" \\\n \"vsetvli t0, t1, e8, mf4 \\n\\t\" \\\n \"vse8.v v28, (s1) \\n\\t\" \\\n \"addi s1, s1, 32 \\n\\t\" \\\n \"sub t1, t1, t0 \\n\\t\" \\\n \"vsetvli t0, t1, e8, mf4 \\n\\t\" \\\n \"vse8.v v29, (s1) \\n\\t\" \\\n \"addi s1, s1, 32 \\n\\t\" \\\n \"sub t1, t1, t0 \\n\\t\" \\\n \"vsetvli t0, t1, e8, mf4 \\n\\t\" \\\n \"vse8.v v30, (s1) \\n\\t\" \\\n \"addi s1, s1, 32 \\n\\t\" \\\n \"sub t1, t1, t0 \\n\\t\" \\\n \"vsetvli t0, t1, e8, mf4 \\n\\t\" \\\n \"vse8.v v31, (s1) \\n\\t\"\n\nnamespace ime1 {\nvoid quantize_a_4row_i8(size_t BlkLen, const float * A, size_t CountK, std::byte * QuantA) {\n constexpr float range_max_reciprocal = 1.0f / ((1 << 7) - 1);\n const float fone = 1.0f;\n\n if (BlkLen == 16 || BlkLen == 32 || BlkLen == 64) {\n for (size_t row_index = 0; row_index < 4; ++row_index) {\n const float * SRC = A + row_index * CountK;\n std::byte * DST = QuantA + row_index * sizeof(float);\n\n const size_t offset = (4 - row_index) * 4 + row_index * 8;\n const size_t stride = 4 * (sizeof(float) + BlkLen);\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"addi t2, %[CountK], 0 \\n\\t\"\n \"addi a1, %[DST], 0 \\n\\t\"\n \"blt t2, %[BlkLen], TAIL%= \\n\\t\"\n\n \"LOOP%=: \\n\\t\"\n \"vsetvli t0, %[BlkLen], e32, m8 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"sub t2, t2, t0 \\n\\t\"\n \"slli t1, t0, 2 \\n\\t\"\n \"add %[SRC], %[SRC], t1 \\n\\t\"\n \"add s1, a1, %[OFFSET] \\n\\t\"\n\n QUANTIZEM4ROW_KERNEL QUANTIZEM4ROW_STORE\n\n \"add a1, a1, %[STRIDE] \\n\\t\"\n \"bge t2, %[BlkLen], LOOP%= \\n\\t\"\n\n \"TAIL%=: \\n\\t\"\n \"blez t2, QUIT%= \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v24, v24, v24 \\n\\t\"\n \"vsetvli t0, t2, e32, m8 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"add s1, a1, %[OFFSET] \\n\\t\"\n\n QUANTIZEM4ROW_KERNEL\n\n \"addi t3, %[BlkLen], 0 \\n\\t\"\n \"addi s2, s1, 0 \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vxor.vv v8, v8, v8 \\n\\t\"\n \"SET_ZERO%=: \\n\\t\"\n \"vse8.v v8, (s2) \\n\\t\"\n \"addi s2, s2, 32 \\n\\t\"\n \"addi t3, t3, -8 \\n\\t\"\n \"bnez t3, SET_ZERO%= \\n\\t\"\n\n QUANTIZEM4ROW_STORE\n\n \"QUIT%=: \\n\\t\"\n : [SRC] \"+r\"(SRC)\n : [DST] \"r\"(DST), [BlkLen] \"r\"(BlkLen), [OFFSET] \"r\"(offset), [STRIDE] \"r\"(stride),\n [CountK] \"r\"(CountK), [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"s1\", \"s2\", \"f10\", \"f11\");\n }\n } else if (BlkLen == 128) {\n for (size_t row_index = 0; row_index < 4; ++row_index) {\n const float * SRC = A + row_index * CountK;\n std::byte * DST = QuantA + row_index * sizeof(float);\n\n const size_t offset = (4 - row_index) * 4 + row_index * 8;\n const size_t stride = 4 * (sizeof(float) + BlkLen);\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"li t6, 32 \\n\\t\"\n \"addi t2, %[CountK], 0 \\n\\t\"\n \"addi a1, %[DST], 0 \\n\\t\"\n \"add s1, a1, %[OFFSET] \\n\\t\"\n \"blt t2, %[BlkLen], TAIL%= \\n\\t\"\n\n \"LOOP%=: \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v8, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"addi t2, t2, -128 \\n\\t\"\n\n \"QUANTIZE%=: \\n\\t\"\n \"add s1, a1, %[OFFSET] \\n\\t\"\n \"vfabs.v v16, v0 \\n\\t\"\n \"vfabs.v v24, v8 \\n\\t\"\n \"vfmax.vv v16, v24, v16 \\n\\t\"\n \"vfredmax.vs v24, v16, v24 \\n\\t\"\n \"vfmv.f.s f10, v24 \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (a1) \\n\\t\"\n \"fdiv.s f11, %[FONE], f10 \\n\\t\"\n \"vfmul.vf v16, v0, f11 \\n\\t\"\n \"vfmul.vf v24, v8, f11 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vfcvt.x.f.v v24, v24 \\n\\t\"\n \"vsetvli t0, zero, e16, m4 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vnclip.wx v20, v24, zero \\n\\t\"\n \"vsetvli t0, zero, e8, m4 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vsetvli t0, zero, e64, m4 \\n\\t\"\n \"vsse64.v v16, (s1), t6 \\n\\t\"\n \"add a1, a1, %[STRIDE] \\n\\t\"\n \"bge t2, %[BlkLen], LOOP%= \\n\\t\"\n\n \"TAIL%=: \\n\\t\"\n \"blez t2, QUIT%= \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v0, v0, v0 \\n\\t\"\n \"vxor.vv v8, v8, v8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v24, v24, v24 \\n\\t\"\n \"vsetvli t0, t2, e32, m8 \\n\\t\"\n \"sub t2, t2, t0 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vsetvli t0, t2, e32, m8 \\n\\t\"\n \"vle32.v v8, (%[SRC]) \\n\\t\"\n \"sub t2, t2, t2 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"jal x0, QUANTIZE%= \\n\\t\"\n\n \"QUIT%=: \\n\\t\"\n : [SRC] \"+r\"(SRC)\n : [DST] \"r\"(DST), [BlkLen] \"r\"(BlkLen), [OFFSET] \"r\"(offset), [STRIDE] \"r\"(stride),\n [CountK] \"r\"(CountK), [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t6\", \"a1\", \"s1\", \"s2\", \"f10\", \"f11\");\n }\n } else if (BlkLen == 256) {\n for (size_t row_index = 0; row_index < 4; ++row_index) {\n const float * SRC = A + row_index * CountK;\n std::byte * DST = QuantA + row_index * sizeof(float);\n const size_t offset = (4 - row_index) * 4 + row_index * 8;\n const size_t stride = 4 * (sizeof(float) + BlkLen);\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"li t6, 32 \\n\\t\"\n \"addi t2, %[CountK], 0 \\n\\t\"\n \"addi a1, %[DST], 0 \\n\\t\"\n \"add s1, a1, %[OFFSET] \\n\\t\"\n \"blt t2, %[BlkLen], TAIL%= \\n\\t\"\n\n \"LOOP%=: \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v8, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v16, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v24, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], -768 \\n\\t\"\n \"addi t2, t2, -256 \\n\\t\"\n \"vfabs.v v0, v0 \\n\\t\"\n \"vfabs.v v8, v8 \\n\\t\"\n \"vfabs.v v16, v16 \\n\\t\"\n \"vfabs.v v24, v24 \\n\\t\"\n \"vfmax.vv v8, v0, v8 \\n\\t\"\n \"vfmax.vv v24, v24, v16 \\n\\t\"\n \"vfmax.vv v8, v8, v24 \\n\\t\"\n \"vfredmax.vs v24, v8, v24 \\n\\t\"\n \"vfmv.f.s f10, v24 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v8, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v16, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v24, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n\n \"QUANTIZE%=: \\n\\t\"\n \"add s1, a1, %[OFFSET] \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (a1) \\n\\t\"\n \"fdiv.s f11, %[FONE], f10 \\n\\t\"\n \"vfmul.vf v0, v0, f11 \\n\\t\"\n \"vfmul.vf v8, v8, f11 \\n\\t\"\n \"vfmul.vf v16, v16, f11 \\n\\t\"\n \"vfmul.vf v24, v24, f11 \\n\\t\"\n \"vfcvt.x.f.v v0, v0 \\n\\t\"\n \"vfcvt.x.f.v v8, v8 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vfcvt.x.f.v v24, v24 \\n\\t\"\n \"vsetvli t0, zero, e16, m4 \\n\\t\"\n \"vnclip.wx v0, v0, zero \\n\\t\"\n \"vnclip.wx v4, v8, zero \\n\\t\"\n \"vnclip.wx v8, v16, zero \\n\\t\"\n \"vnclip.wx v12, v24, zero \\n\\t\"\n \"vsetvli t0, zero, e8, m4 \\n\\t\"\n \"vnclip.wx v0, v0, zero \\n\\t\"\n \"vnclip.wx v4, v8, zero \\n\\t\"\n \"vsetvli t0, zero, e64, m8 \\n\\t\"\n \"vsse64.v v0, (s1), t6 \\n\\t\"\n \"add a1, a1, %[STRIDE] \\n\\t\"\n \"bge t2, %[BlkLen], LOOP%= \\n\\t\"\n\n \"TAIL%=: \\n\\t\"\n \"blez t2, QUIT%= \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v0, v0, v0 \\n\\t\"\n \"vxor.vv v8, v8, v8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v24, v24, v24 \\n\\t\"\n \"addi t1, t2, 0 \\n\\t\"\n \"vsetvli t0, t1, e32, m8 \\n\\t\"\n \"sub t1, t1, t0 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vsetvli t0, t1, e32, m8 \\n\\t\"\n \"sub t1, t1, t0 \\n\\t\"\n \"vle32.v v8, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vsetvli t0, t1, e32, m8 \\n\\t\"\n \"sub t1, t1, t0 \\n\\t\"\n \"vle32.v v16, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vsetvli t0, t1, e32, m8 \\n\\t\"\n \"vle32.v v24, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], -768 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfabs.v v0, v0 \\n\\t\"\n \"vfabs.v v8, v8 \\n\\t\"\n \"vfabs.v v16, v16 \\n\\t\"\n \"vfabs.v v24, v24 \\n\\t\"\n \"vfmax.vv v8, v0, v8 \\n\\t\"\n \"vfmax.vv v24, v16, v24 \\n\\t\"\n \"vfmax.vv v8, v8, v24 \\n\\t\"\n \"vfredmax.vs v24, v8, v24 \\n\\t\"\n \"vfmv.f.s f10, v24 \\n\\t\"\n \"add s1, a1, %[OFFSET] \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (a1) \\n\\t\"\n \"fdiv.s f11, %[FONE], f10 \\n\\t\"\n \"vsetvli t0, zero, e64, m8 \\n\\t\"\n \"vxor.vv v0, v0, v0 \\n\\t\"\n \"vsse64.v v0, (s1), t6 \\n\\t\"\n\n \"TAIL_LOOP%=: \\n\\t\"\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vxor.vv v0, v0, v0 \\n\\t\"\n \"vsetvli t0, t2, e32, m1 \\n\\t\"\n \"sub t2, t2, t0 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 32 \\n\\t\"\n \"vfmul.vf v1, v0, f11 \\n\\t\"\n \"vfcvt.x.f.v v2, v1 \\n\\t\"\n \"vsetvli t0, zero, e16, mf2 \\n\\t\"\n \"vnclip.wx v3, v2, zero \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vnclip.wx v3, v3, zero \\n\\t\"\n \"vse8.v v3, (s1) \\n\\t\"\n \"addi s1, s1, 32 \\n\\t\"\n \"bnez t2, TAIL_LOOP%= \\n\\t\"\n\n \"QUIT%=: \\n\\t\"\n : [SRC] \"+r\"(SRC)\n : [DST] \"r\"(DST), [BlkLen] \"r\"(BlkLen), [OFFSET] \"r\"(offset), [STRIDE] \"r\"(stride),\n [CountK] \"r\"(CountK), [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t6\", \"a1\", \"s1\", \"s2\", \"f10\", \"f11\");\n }\n }\n}\n\nvoid quantize_a_row_i8(size_t BlkLen, const float * A, size_t CountK, std::byte * QuantA) {\n const float * SRC = A;\n std::byte * DST = QuantA;\n constexpr float range_max_reciprocal = 1.0f / ((1 << 7) - 1);\n const float fone = 1.0f;\n std::byte * QuantA_offset = QuantA + CountK + 4 * ((CountK + BlkLen - 1) / BlkLen);\n size_t offset = (CountK + BlkLen - 1) / BlkLen * BlkLen - CountK;\n\n if (CountK <= BlkLen) {\n float max_abs_A = 0.0f;\n for (size_t k = 0; k < CountK; k++) {\n max_abs_A = std::max(max_abs_A, fabsf(A[k]));\n }\n float scale_A = max_abs_A * range_max_reciprocal;\n\n ((float *) QuantA)[0] = scale_A;\n\n auto * QuantAData_offset = (int8_t *) (QuantA + sizeof(float));\n\n for (size_t k = 0; k < CountK; k++) {\n QuantAData_offset[k] =\n (int8_t) std::clamp(roundf(A[k] / scale_A), (float) std::numeric_limits::lowest(),\n (float) std::numeric_limits::max());\n }\n for (size_t k = CountK; k < BlkLen; k++) {\n QuantAData_offset[k] = 0;\n }\n\n return;\n }\n\n if (BlkLen != 32 || BlkLen != 64 || BlkLen != 128) {\n __asm__ volatile(\n \"vsetvli t0, zero, e8, m8 \\n\\t\"\n \"vxor.vv v24, v24, v24 \\n\\t\"\n \"LOOP%=: \\n\\t\"\n \"vsetvli t0, %[CNT], e8, m8 \\n\\t\"\n \"vse8.v v24, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 128 \\n\\t\"\n \"sub %[CNT], %[CNT], t0 \\n\\t\"\n \"bnez %[CNT], LOOP%= \\n\\t\"\n : [DST] \"+r\"(QuantA_offset), [CNT] \"+r\"(offset)\n :\n : \"cc\", \"t0\");\n }\n if (BlkLen == 16) {\n float buffer[64] = { 0.0f };\n __asm__ volatile(\n \"addi t3, zero, 16*8 \\n\\t\"\n \"addi t2, zero, 16 \\n\\t\"\n \"blt %[K], t3, LOOP_K%= \\n\\t\"\n \"blt %[K], t2, TAIL%= \\n\\t\"\n \"LOOP_MAIN%=: \\n\\t\"\n \"vsetvli t1, zero, e32, m2 \\n\\t\"\n \"addi %[K], %[K], -128 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 64 \\n\\t\"\n \"vle32.v v2, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 64 \\n\\t\"\n \"vle32.v v4, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 64 \\n\\t\"\n \"vle32.v v6, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 64 \\n\\t\"\n \"vle32.v v8, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 64 \\n\\t\"\n \"vle32.v v10, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 64 \\n\\t\"\n \"vle32.v v12, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 64 \\n\\t\"\n \"vle32.v v14, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 64 \\n\\t\"\n \"addi a1, %[BUFFER], 0 \\n\\t\"\n \"vfabs.v v16, v0 \\n\\t\"\n \"vfabs.v v18, v2 \\n\\t\"\n \"vfabs.v v20, v4 \\n\\t\"\n \"vfabs.v v22, v6 \\n\\t\"\n \"vfabs.v v24, v8 \\n\\t\"\n \"vfabs.v v26, v10 \\n\\t\"\n \"vfabs.v v28, v12 \\n\\t\"\n \"vfabs.v v30, v14 \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vfmax.vv v16, v16, v17 \\n\\t\"\n \"vfmax.vv v18, v18, v19 \\n\\t\"\n \"vfmax.vv v20, v20, v21 \\n\\t\"\n \"vfmax.vv v22, v22, v23 \\n\\t\"\n \"vfmax.vv v24, v24, v25 \\n\\t\"\n \"vfmax.vv v26, v26, v27 \\n\\t\"\n \"vfmax.vv v28, v28, v29 \\n\\t\"\n \"vfmax.vv v30, v30, v31 \\n\\t\"\n \"vse32.v v16, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vse32.v v18, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vse32.v v20, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vse32.v v22, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vse32.v v24, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vse32.v v26, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vse32.v v28, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vse32.v v30, (a1) \\n\\t\"\n \"addi a1, %[BUFFER], 0 \\n\\t\"\n \"flw f0, (a1) \\n\\t\"\n \"flw f1, 4(a1) \\n\\t\"\n \"flw f2, 8(a1) \\n\\t\"\n \"flw f3, 12(a1) \\n\\t\"\n \"flw f4, 16(a1) \\n\\t\"\n \"flw f5, 20(a1) \\n\\t\"\n \"flw f6, 24(a1) \\n\\t\"\n \"flw f7, 28(a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f10, f3, f7 \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"fdiv.s f10, %[FONE], f10 \\n\\t\"\n \"flw f0, (a1) \\n\\t\"\n \"flw f1, 4(a1) \\n\\t\"\n \"flw f2, 8(a1) \\n\\t\"\n \"flw f3, 12(a1) \\n\\t\"\n \"flw f4, 16(a1) \\n\\t\"\n \"flw f5, 20(a1) \\n\\t\"\n \"flw f6, 24(a1) \\n\\t\"\n \"flw f7, 28(a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f11, f3, f7 \\n\\t\"\n \"fmul.s f11, f11, %[RMAXREC] \\n\\t\"\n \"fsw f11, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"fdiv.s f11, %[FONE], f11 \\n\\t\"\n \"flw f0, (a1) \\n\\t\"\n \"flw f1, 4(a1) \\n\\t\"\n \"flw f2, 8(a1) \\n\\t\"\n \"flw f3, 12(a1) \\n\\t\"\n \"flw f4, 16(a1) \\n\\t\"\n \"flw f5, 20(a1) \\n\\t\"\n \"flw f6, 24(a1) \\n\\t\"\n \"flw f7, 28(a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f12, f3, f7 \\n\\t\"\n \"fmul.s f12, f12, %[RMAXREC] \\n\\t\"\n \"fsw f12, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"fdiv.s f12, %[FONE], f12 \\n\\t\"\n \"flw f0, (a1) \\n\\t\"\n \"flw f1, 4(a1) \\n\\t\"\n \"flw f2, 8(a1) \\n\\t\"\n \"flw f3, 12(a1) \\n\\t\"\n \"flw f4, 16(a1) \\n\\t\"\n \"flw f5, 20(a1) \\n\\t\"\n \"flw f6, 24(a1) \\n\\t\"\n \"flw f7, 28(a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f13, f3, f7 \\n\\t\"\n \"fmul.s f13, f13, %[RMAXREC] \\n\\t\"\n \"fsw f13, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"fdiv.s f13, %[FONE], f13 \\n\\t\"\n \"flw f0, (a1) \\n\\t\"\n \"flw f1, 4(a1) \\n\\t\"\n \"flw f2, 8(a1) \\n\\t\"\n \"flw f3, 12(a1) \\n\\t\"\n \"flw f4, 16(a1) \\n\\t\"\n \"flw f5, 20(a1) \\n\\t\"\n \"flw f6, 24(a1) \\n\\t\"\n \"flw f7, 28(a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f14, f3, f7 \\n\\t\"\n \"fmul.s f14, f14, %[RMAXREC] \\n\\t\"\n \"fsw f14, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"fdiv.s f14, %[FONE], f14 \\n\\t\"\n \"flw f0, (a1) \\n\\t\"\n \"flw f1, 4(a1) \\n\\t\"\n \"flw f2, 8(a1) \\n\\t\"\n \"flw f3, 12(a1) \\n\\t\"\n \"flw f4, 16(a1) \\n\\t\"\n \"flw f5, 20(a1) \\n\\t\"\n \"flw f6, 24(a1) \\n\\t\"\n \"flw f7, 28(a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f15, f3, f7 \\n\\t\"\n \"fmul.s f15, f15, %[RMAXREC] \\n\\t\"\n \"fsw f15, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"fdiv.s f15, %[FONE], f15 \\n\\t\"\n \"flw f0, (a1) \\n\\t\"\n \"flw f1, 4(a1) \\n\\t\"\n \"flw f2, 8(a1) \\n\\t\"\n \"flw f3, 12(a1) \\n\\t\"\n \"flw f4, 16(a1) \\n\\t\"\n \"flw f5, 20(a1) \\n\\t\"\n \"flw f6, 24(a1) \\n\\t\"\n \"flw f7, 28(a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f16, f3, f7 \\n\\t\"\n \"fmul.s f16, f16, %[RMAXREC] \\n\\t\"\n \"fsw f16, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"fdiv.s f16, %[FONE], f16 \\n\\t\"\n \"flw f0, (a1) \\n\\t\"\n \"flw f1, 4(a1) \\n\\t\"\n \"flw f2, 8(a1) \\n\\t\"\n \"flw f3, 12(a1) \\n\\t\"\n \"flw f4, 16(a1) \\n\\t\"\n \"flw f5, 20(a1) \\n\\t\"\n \"flw f6, 24(a1) \\n\\t\"\n \"flw f7, 28(a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f17, f3, f7 \\n\\t\"\n \"fmul.s f17, f17, %[RMAXREC] \\n\\t\"\n \"fsw f17, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], -136 \\n\\t\"\n \"fdiv.s f17, %[FONE], f17 \\n\\t\"\n \"vsetvli t0, zero, e32, m2 \\n\\t\"\n \"vfmul.vf v16, v0, f10 \\n\\t\"\n \"vfmul.vf v18, v2, f11 \\n\\t\"\n \"vfmul.vf v20, v4, f12 \\n\\t\"\n \"vfmul.vf v22, v6, f13 \\n\\t\"\n \"vfmul.vf v24, v8, f14 \\n\\t\"\n \"vfmul.vf v26, v10, f15 \\n\\t\"\n \"vfmul.vf v28, v12, f16 \\n\\t\"\n \"vfmul.vf v30, v14, f17 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vfcvt.x.f.v v18, v18 \\n\\t\"\n \"vfcvt.x.f.v v20, v20 \\n\\t\"\n \"vfcvt.x.f.v v22, v22 \\n\\t\"\n \"vfcvt.x.f.v v24, v24 \\n\\t\"\n \"vfcvt.x.f.v v26, v26 \\n\\t\"\n \"vfcvt.x.f.v v28, v28 \\n\\t\"\n \"vfcvt.x.f.v v30, v30 \\n\\t\"\n \"vsetvli t0, zero, e16, m1 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vnclip.wx v18, v18, zero \\n\\t\"\n \"vnclip.wx v20, v20, zero \\n\\t\"\n \"vnclip.wx v22, v22, zero \\n\\t\"\n \"vnclip.wx v24, v24, zero \\n\\t\"\n \"vnclip.wx v26, v26, zero \\n\\t\"\n \"vnclip.wx v28, v28, zero \\n\\t\"\n \"vnclip.wx v30, v30, zero \\n\\t\"\n \"vsetvli t0, t1, e8, mf2 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vnclip.wx v18, v18, zero \\n\\t\"\n \"vnclip.wx v20, v20, zero \\n\\t\"\n \"vnclip.wx v22, v22, zero \\n\\t\"\n \"vnclip.wx v24, v24, zero \\n\\t\"\n \"vnclip.wx v26, v26, zero \\n\\t\"\n \"vnclip.wx v28, v28, zero \\n\\t\"\n \"vnclip.wx v30, v30, zero \\n\\t\"\n \"vse8.v v16, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"vse8.v v18, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"vse8.v v20, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"vse8.v v22, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"vse8.v v24, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"vse8.v v26, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"vse8.v v28, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 20 \\n\\t\"\n \"vse8.v v30, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 16 \\n\\t\"\n \"bge %[K], t3, LOOP_MAIN%= \\n\\t\"\n \"blt %[K], t2, TAIL%= \\n\\t\"\n \"LOOP_K%=: \\n\\t\"\n \"vsetvli t1, %[K], e32, m2 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 64 \\n\\t\"\n \"sub %[K], %[K], t1 \\n\\t\"\n \"vfabs.v v16, v0 \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vfmax.vv v16, v16, v17 \\n\\t\"\n \"vse32.v v16, (%[BUFFER]) \\n\\t\"\n \"flw f0, (%[BUFFER]) \\n\\t\"\n \"flw f1, 4(%[BUFFER]) \\n\\t\"\n \"flw f2, 8(%[BUFFER]) \\n\\t\"\n \"flw f3, 12(%[BUFFER]) \\n\\t\"\n \"flw f4, 16(%[BUFFER]) \\n\\t\"\n \"flw f5, 20(%[BUFFER]) \\n\\t\"\n \"flw f6, 24(%[BUFFER]) \\n\\t\"\n \"flw f7, 28(%[BUFFER]) \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f10, f3, f7 \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 4 \\n\\t\"\n \"fdiv.s f11, %[FONE], f10 \\n\\t\"\n \"vsetvli t0, zero, e32, m2 \\n\\t\"\n \"vfmul.vf v16, v0, f11 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vsetvli t0, zero, e16, m1 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vsetvli t0, t1, e8, mf2 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vse8.v v16, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 16 \\n\\t\"\n \"bge %[K], t2, LOOP_K%= \\n\\t\"\n \"TAIL%=: \\n\\t\"\n \"blez %[K], END%= \\n\\t\"\n \"vsetvli t0, t3, e32, m2 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"jal x0, LOOP_K%= \\n\\t\"\n \"END%=: \\n\\t\"\n : [SRC] \"+r\"(SRC), [DST] \"+r\"(DST), [K] \"+r\"(CountK)\n : [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal), [BUFFER] \"r\"(buffer)\n : \"cc\", \"t3\", \"t2\", \"t1\", \"t0\", \"a1\", \"f0\", \"f1\", \"f2\", \"f3\", \"f4\", \"f5\", \"f6\", \"f7\", \"f10\", \"f11\", \"f12\",\n \"f13\", \"f14\", \"f15\", \"f16\", \"f17\");\n } else if (BlkLen == 32) {\n __asm__ volatile(\n \"addi t3, zero, 32*4 \\n\\t\"\n \"addi t2, zero, 32 \\n\\t\"\n\n \"addi a1, %[SRC], 0 \\n\\t\"\n \"addi a2, %[SRC], 128 \\n\\t\"\n \"addi a3, %[SRC], 256 \\n\\t\"\n \"addi a4, %[SRC], 384 \\n\\t\"\n\n \"addi s1, %[DST], 0 \\n\\t\"\n \"addi s2, %[DST], 36 \\n\\t\"\n \"addi s3, %[DST], 72 \\n\\t\"\n \"addi s4, %[DST], 108 \\n\\t\"\n \"blt %[K], t3, LOOP_K%= \\n\\t\"\n \"blt %[K], t2, TAIL%= \\n\\t\"\n\n \"LOOP_MAIN%=: \\n\\t\"\n \"vsetvli t1, zero, e32, m4 \\n\\t\"\n \"addi %[K], %[K], -128 \\n\\t\"\n \"vle32.v v0, (a1) \\n\\t\"\n \"addi a1, a1, 512 \\n\\t\"\n \"vle32.v v4, (a2) \\n\\t\"\n \"addi a2, a2, 512 \\n\\t\"\n \"vle32.v v8, (a3) \\n\\t\"\n \"addi a3, a3, 512 \\n\\t\"\n \"vle32.v v12, (a4) \\n\\t\"\n \"addi a4, a4, 512 \\n\\t\"\n \"vfabs.v v16, v0 \\n\\t\"\n \"vfabs.v v20, v4 \\n\\t\"\n \"vfabs.v v24, v8 \\n\\t\"\n \"vfabs.v v28, v12 \\n\\t\"\n \"vsetvli t0, zero, e32, m2 \\n\\t\"\n \"vfmax.vv v16, v16, v18 \\n\\t\"\n \"vfmax.vv v20, v20, v22 \\n\\t\"\n \"vfmax.vv v24, v24, v26 \\n\\t\"\n \"vfmax.vv v28, v28, v30 \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vfmax.vv v16, v16, v17 \\n\\t\"\n \"vfmax.vv v20, v20, v21 \\n\\t\"\n \"vfmax.vv v24, v24, v25 \\n\\t\"\n \"vfmax.vv v28, v28, v29 \\n\\t\"\n\n \"vfredmax.vs v17, v16, v17 \\n\\t\"\n \"vfredmax.vs v21, v20, v21 \\n\\t\"\n \"vfredmax.vs v25, v24, v25 \\n\\t\"\n \"vfredmax.vs v29, v28, v29 \\n\\t\"\n \"vfmv.f.s f10, v17 \\n\\t\"\n \"vfmv.f.s f11, v21 \\n\\t\"\n \"vfmv.f.s f12, v25 \\n\\t\"\n \"vfmv.f.s f13, v29 \\n\\t\"\n\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fmul.s f11, f11, %[RMAXREC] \\n\\t\"\n \"fmul.s f12, f12, %[RMAXREC] \\n\\t\"\n \"fmul.s f13, f13, %[RMAXREC] \\n\\t\"\n \"fsw f10, (s1) \\n\\t\"\n \"addi s1, s1, 4 \\n\\t\"\n\n \"fsw f11, (s2) \\n\\t\"\n \"addi s2, s2, 4 \\n\\t\"\n \"fsw f12, (s3) \\n\\t\"\n \"addi s3, s3, 4 \\n\\t\"\n \"fsw f13, (s4) \\n\\t\"\n \"addi s4, s4, 4 \\n\\t\"\n \"fdiv.s f10, %[FONE], f10 \\n\\t\"\n \"fdiv.s f11, %[FONE], f11 \\n\\t\"\n \"fdiv.s f12, %[FONE], f12 \\n\\t\"\n \"fdiv.s f13, %[FONE], f13 \\n\\t\"\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vfmul.vf v16, v0, f10 \\n\\t\"\n \"vfmul.vf v20, v4, f11 \\n\\t\"\n \"vfmul.vf v24, v8, f12 \\n\\t\"\n \"vfmul.vf v28, v12, f13 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vfcvt.x.f.v v20, v20 \\n\\t\"\n \"vfcvt.x.f.v v24, v24 \\n\\t\"\n \"vfcvt.x.f.v v28, v28 \\n\\t\"\n \"vsetvli t0, zero, e16, m2 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vnclip.wx v20, v20, zero \\n\\t\"\n \"vnclip.wx v24, v24, zero \\n\\t\"\n \"vnclip.wx v28, v28, zero \\n\\t\"\n \"vsetvli t0, t1, e8, m1 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vnclip.wx v20, v20, zero \\n\\t\"\n \"vnclip.wx v24, v24, zero \\n\\t\"\n \"vnclip.wx v28, v28, zero \\n\\t\"\n \"vse8.v v16, (s1) \\n\\t\"\n \"addi s1, s1, 140 \\n\\t\"\n \"vse8.v v20, (s2) \\n\\t\"\n \"addi s2, s2, 140 \\n\\t\"\n \"vse8.v v24, (s3) \\n\\t\"\n \"addi s3, s3, 140 \\n\\t\"\n \"vse8.v v28, (s4) \\n\\t\"\n \"addi s4, s4, 140 \\n\\t\"\n \"bge %[K], t3, LOOP_MAIN%= \\n\\t\"\n \"blt %[K], t2, TAIL%= \\n\\t\"\n \"LOOP_K%=: \\n\\t\"\n \"vsetvli t1, %[K], e32, m4 \\n\\t\"\n \"vle32.v v0, (a1) \\n\\t\"\n \"addi a1, a1, 128 \\n\\t\"\n \"sub %[K], %[K], t1 \\n\\t\"\n \"vfabs.v v16, v0 \\n\\t\"\n \"vsetvli t0, zero, e32, m2 \\n\\t\"\n \"vfmax.vv v16, v16, v18 \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vfmax.vv v16, v16, v17 \\n\\t\"\n \"vfredmax.vs v17, v16, v17 \\n\\t\"\n \"vfmv.f.s f10, v17 \\n\\t\"\n\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (s1) \\n\\t\"\n \"addi s1, s1, 4 \\n\\t\"\n \"fdiv.s f11, %[FONE], f10 \\n\\t\"\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vfmul.vf v16, v0, f11 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vsetvli t0, zero, e16, m2 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vse8.v v16, (s1) \\n\\t\"\n \"addi s1, s1, 32 \\n\\t\"\n \"bge %[K], t2, LOOP_K%= \\n\\t\"\n \"TAIL%=: \\n\\t\"\n \"blez %[K], END%= \\n\\t\"\n \"vsetvli t0, t3, e32, m4 \\n\\t\"\n \"vxor.vv v0, v0, v0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"jal x0, LOOP_K%= \\n\\t\"\n \"END%=: \\n\\t\"\n : [K] \"+r\"(CountK)\n : [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal), [SRC] \"r\"(SRC), [DST] \"r\"(DST)\n : \"cc\", \"t3\", \"t2\", \"t1\", \"t0\", \"a1\", \"a2\", \"a3\", \"a4\", \"s1\", \"s2\", \"s3\", \"s4\", \"f10\", \"f11\", \"f12\", \"f13\");\n } else if (BlkLen == 64) {\n __asm__ volatile(\n \"addi t3, zero, 64*2 \\n\\t\"\n \"addi t2, zero, 64 \\n\\t\"\n \"addi a1, %[SRC], 0 \\n\\t\"\n \"addi a2, %[SRC], 256 \\n\\t\"\n \"addi s1, %[DST], 0 \\n\\t\"\n \"addi s2, %[DST], 68 \\n\\t\"\n \"blt %[K], t3, LOOP_K%= \\n\\t\"\n \"blt %[K], t2, TAIL%= \\n\\t\"\n \"LOOP_MAIN%=: \\n\\t\"\n \"vsetvli t1, zero, e32, m8 \\n\\t\"\n \"addi %[K], %[K], -128 \\n\\t\"\n \"vle32.v v0, (a1) \\n\\t\"\n \"addi a1, a1, 512 \\n\\t\"\n \"vle32.v v8, (a2) \\n\\t\"\n \"addi a2, a2, 512 \\n\\t\"\n \"vfabs.v v16, v0 \\n\\t\"\n \"vfabs.v v24, v8 \\n\\t\"\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vfmax.vv v16, v16, v20 \\n\\t\"\n \"vfmax.vv v24, v24, v28 \\n\\t\"\n \"vsetvli t0, zero, e32, m2 \\n\\t\"\n \"vfmax.vv v16, v16, v18 \\n\\t\"\n \"vfmax.vv v24, v24, v26 \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vfmax.vv v16, v16, v17 \\n\\t\"\n \"vfmax.vv v24, v24, v25 \\n\\t\"\n \"vfredmax.vs v17, v16, v17 \\n\\t\"\n \"vfredmax.vs v25, v24, v25 \\n\\t\"\n \"vfmv.f.s f10, v17 \\n\\t\"\n \"vfmv.f.s f11, v25 \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fmul.s f11, f11, %[RMAXREC] \\n\\t\"\n \"fsw f10, (s1) \\n\\t\"\n \"addi s1, s1, 4 \\n\\t\"\n \"fsw f11, (s2) \\n\\t\"\n \"addi s2, s2, 4 \\n\\t\"\n \"fdiv.s f10, %[FONE], f10 \\n\\t\"\n \"fdiv.s f11, %[FONE], f11 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfmul.vf v16, v0, f10 \\n\\t\"\n \"vfmul.vf v24, v8, f11 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vfcvt.x.f.v v24, v24 \\n\\t\"\n \"vsetvli t0, zero, e16, m4 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vnclip.wx v24, v24, zero \\n\\t\"\n \"vsetvli t0, t1, e8, m2 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vnclip.wx v24, v24, zero \\n\\t\"\n \"vse8.v v16, (s1) \\n\\t\"\n \"addi s1, s1, 132 \\n\\t\"\n \"vse8.v v24, (s2) \\n\\t\"\n \"addi s2, s2, 132 \\n\\t\"\n \"bge %[K], t3, LOOP_MAIN%= \\n\\t\"\n \"blt %[K], t2, TAIL%= \\n\\t\"\n \"LOOP_K%=: \\n\\t\"\n \"vsetvli t1, %[K], e32, m8 \\n\\t\"\n \"vle32.v v0, (a1) \\n\\t\"\n \"addi a1, a1, 256 \\n\\t\"\n \"sub %[K], %[K], t1 \\n\\t\"\n \"vfabs.v v16, v0 \\n\\t\"\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vfmax.vv v16, v16, v20 \\n\\t\"\n \"vsetvli t0, zero, e32, m2 \\n\\t\"\n \"vfmax.vv v16, v16, v18 \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vfmax.vv v16, v16, v17 \\n\\t\"\n \"vfredmax.vs v17, v16, v17 \\n\\t\"\n \"vfmv.f.s f10, v17 \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (s1) \\n\\t\"\n \"addi s1, s1, 4 \\n\\t\"\n \"fdiv.s f11, %[FONE], f10 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfmul.vf v16, v0, f11 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vsetvli t0, zero, e16, m4 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vsetvli t0, zero, e8, m2 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vse8.v v16, (s1) \\n\\t\"\n \"addi s1, s1, 64 \\n\\t\"\n \"bge %[K], t2, LOOP_K%= \\n\\t\"\n \"TAIL%=: \\n\\t\"\n \"blez %[K], END%= \\n\\t\"\n \"vsetvli t0, t3, e32, m8 \\n\\t\"\n \"vxor.vv v0, v0, v0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"jal x0, LOOP_K%= \\n\\t\"\n \"END%=: \\n\\t\"\n : [K] \"+r\"(CountK)\n : [SRC] \"r\"(SRC), [DST] \"r\"(DST), [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal)\n : \"cc\", \"t3\", \"t2\", \"t1\", \"t0\", \"a1\", \"a2\", \"s1\", \"s2\", \"f10\", \"f11\");\n } else if (BlkLen == 128) {\n __asm__ volatile(\n \"addi t2, zero, 128 \\n\\t\"\n \"addi a1, %[SRC], 0 \\n\\t\"\n \"addi a2, %[SRC], 256 \\n\\t\"\n \"blt %[K], t2, TAIL%= \\n\\t\"\n \"LOOP_K%=: \\n\\t\"\n \"vsetvli t1, zero, e32, m8 \\n\\t\"\n \"vle32.v v0, (a1) \\n\\t\"\n \"addi a1, a1, 512 \\n\\t\"\n \"vle32.v v8, (a2) \\n\\t\"\n \"addi a2, a2, 512 \\n\\t\"\n \"sub %[K], %[K], t2 \\n\\t\"\n \"QUANT%=: \\n\\t\"\n \"vfabs.v v16, v0 \\n\\t\"\n \"vfabs.v v24, v8 \\n\\t\"\n \"vfmax.vv v24, v16, v24 \\n\\t\"\n \"vsetvli t1, zero, e32, m4 \\n\\t\"\n \"vfmax.vv v28, v24, v28 \\n\\t\"\n \"vsetvli t0, zero, e32, m2 \\n\\t\"\n \"vfmax.vv v30, v28, v30 \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vfmax.vv v30, v30, v31 \\n\\t\"\n \"vfredmax.vs v31, v30, v31 \\n\\t\"\n \"vfmv.f.s f10, v31 \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 4 \\n\\t\"\n \"fdiv.s f11, %[FONE], f10 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfmul.vf v16, v0, f11 \\n\\t\"\n \"vfmul.vf v24, v8, f11 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vfcvt.x.f.v v24, v24 \\n\\t\"\n \"vsetvli t0, zero, e16, m4 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vnclip.wx v20, v24, zero \\n\\t\"\n \"vsetvli t0, zero, e8, m4 \\n\\t\"\n \"vnclip.wx v16, v16, zero \\n\\t\"\n \"vse8.v v16, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 128 \\n\\t\"\n \"bge %[K], t2, LOOP_K%= \\n\\t\"\n \"TAIL%=: \\n\\t\"\n \"blez %[K], END%= \\n\\t\"\n \"vsetvli t1, zero, e32, m8 \\n\\t\"\n \"vxor.vv v0, v0, v0 \\n\\t\"\n \"vxor.vv v8, v8, v8 \\n\\t\"\n \"vsetvli t0, %[K], e32, m8 \\n\\t\"\n \"vle32.v v0, (a1) \\n\\t\"\n \"sub %[K], %[K], t0 \\n\\t\"\n \"vsetvli t0, %[K], e32, m8 \\n\\t\"\n \"vle32.v v8, (a2) \\n\\t\"\n \"sub %[K], %[K], t0 \\n\\t\"\n \"vsetvli t1, zero, e32, m8 \\n\\t\"\n \"jal x0, QUANT%= \\n\\t\"\n \"END%=: \\n\\t\"\n\n : [DST] \"+r\"(DST), [K] \"+r\"(CountK)\n : [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal), [SRC] \"r\"(SRC)\n : \"cc\", \"t2\", \"t1\", \"t0\", \"a1\", \"a2\", \"f10\", \"f11\");\n } else {\n float buffer[8] = { 0.0f };\n size_t cnt = BlkLen / 256;\n\n __asm__ volatile(\n \"slli t3, %[BLK], 2 \\n\\t\"\n \"blt %[K], %[BLK], LOOP_TAIL%= \\n\\t\"\n \"LOOP_MAIN%=: \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vxor.vv v31, v31, v31 \\n\\t\"\n \"vse32.v v31, (%[BUFFER]) \\n\\t\"\n \"addi t6, %[CNT], 0 \\n\\t\"\n \"LOOP_CMP%=: \\n\\t\"\n \"addi t6, t6, -1 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v8, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v16, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v24, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vfabs.v v0, v0 \\n\\t\"\n \"vfabs.v v8, v8 \\n\\t\"\n \"vfabs.v v16, v16 \\n\\t\"\n \"vfabs.v v24, v24 \\n\\t\"\n \"vfmax.vv v8, v0, v8 \\n\\t\"\n \"vfmax.vv v16, v16, v24 \\n\\t\"\n \"vfmax.vv v0, v0, v16 \\n\\t\"\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vfmax.vv v0, v0, v4 \\n\\t\"\n \"vsetvli t0, zero, e32, m2 \\n\\t\"\n \"vfmax.vv v0, v0, v2 \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vfmax.vv v0, v0, v1 \\n\\t\"\n \"vle32.v v30, (%[BUFFER]) \\n\\t\"\n \"vfmax.vv v31, v30, v0 \\n\\t\"\n \"vse32.v v31, (%[BUFFER]) \\n\\t\"\n \"bnez t6, LOOP_CMP%= \\n\\t\"\n \"sub %[SRC], %[SRC], t3 \\n\\t\"\n \"addi t6, %[CNT], 0 \\n\\t\"\n \"flw f0, (%[BUFFER]) \\n\\t\"\n \"flw f1, 4(%[BUFFER]) \\n\\t\"\n \"flw f2, 8(%[BUFFER]) \\n\\t\"\n \"flw f3, 12(%[BUFFER]) \\n\\t\"\n \"flw f4, 16(%[BUFFER]) \\n\\t\"\n \"flw f5, 20(%[BUFFER]) \\n\\t\"\n \"flw f6, 24(%[BUFFER]) \\n\\t\"\n \"flw f7, 28(%[BUFFER]) \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f10, f3, f7 \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 4 \\n\\t\"\n \"fdiv.s f11, %[FONE], f10 \\n\\t\"\n \"addi t6, %[CNT], 0 \\n\\t\"\n \"LOOP_QUANT%=: \\n\\t\"\n \"addi t6, t6, -1 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v8, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v16, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vle32.v v24, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfmul.vf v0, v0, f11 \\n\\t\"\n \"vfmul.vf v8, v8, f11 \\n\\t\"\n \"vfmul.vf v16, v16, f11 \\n\\t\"\n \"vfmul.vf v24, v24, f11 \\n\\t\"\n \"vfcvt.x.f.v v0, v0 \\n\\t\"\n \"vfcvt.x.f.v v8, v8 \\n\\t\"\n \"vfcvt.x.f.v v16, v16 \\n\\t\"\n \"vfcvt.x.f.v v24, v24 \\n\\t\"\n \"vsetvli t0, zero, e16, m4 \\n\\t\"\n \"vnclip.wx v0, v0, zero \\n\\t\"\n \"vnclip.wx v4, v8, zero \\n\\t\"\n \"vnclip.wx v8, v16, zero \\n\\t\"\n \"vnclip.wx v12, v24, zero \\n\\t\"\n \"vsetvli t0, zero, e8, m4 \\n\\t\"\n \"vnclip.wx v0, v0, zero \\n\\t\"\n \"vnclip.wx v4, v8, zero \\n\\t\"\n \"vse8.v v0, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 128 \\n\\t\"\n \"vse8.v v4, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 128 \\n\\t\"\n \"bnez t6, LOOP_QUANT%= \\n\\t\"\n \"sub %[K], %[K], %[BLK] \\n\\t\"\n \"bge %[K], %[BLK], LOOP_MAIN%= \\n\\t\"\n \"blez %[K], END%= \\n\\t\"\n \"LOOP_TAIL%=: \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vxor.vv v31, v31, v31 \\n\\t\"\n \"vse32.v v31, (%[BUFFER]) \\n\\t\"\n \"addi t6, %[K], 0 \\n\\t\"\n \"addi s1, %[SRC], 0 \\n\\t\"\n \"TAIL_CMP%=: \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v0, v0, v0 \\n\\t\"\n \"vsetvli t0, t6, e32, m8 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi %[SRC], %[SRC], 256 \\n\\t\"\n \"sub t6, t6, t0 \\n\\t\"\n \"vfabs.v v0, v0 \\n\\t\"\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vfmax.vv v0, v0, v4 \\n\\t\"\n \"vsetvli t0, zero, e32, m2 \\n\\t\"\n \"vfmax.vv v0, v0, v2 \\n\\t\"\n \"vsetvli t0, zero, e32, m1 \\n\\t\"\n \"vfmax.vv v0, v0, v1 \\n\\t\"\n \"vle32.v v30, (%[BUFFER]) \\n\\t\"\n \"vfmax.vv v31, v30, v0 \\n\\t\"\n \"vse32.v v31, (%[BUFFER]) \\n\\t\"\n \"bnez t6, TAIL_CMP%= \\n\\t\"\n \"addi t6, %[K], 0 \\n\\t\"\n \"flw f0, (%[BUFFER]) \\n\\t\"\n \"flw f1, 4(%[BUFFER]) \\n\\t\"\n \"flw f2, 8(%[BUFFER]) \\n\\t\"\n \"flw f3, 12(%[BUFFER]) \\n\\t\"\n \"flw f4, 16(%[BUFFER]) \\n\\t\"\n \"flw f5, 20(%[BUFFER]) \\n\\t\"\n \"flw f6, 24(%[BUFFER]) \\n\\t\"\n \"flw f7, 28(%[BUFFER]) \\n\\t\"\n \"fmax.s f1, f0, f1 \\n\\t\"\n \"fmax.s f3, f2, f3 \\n\\t\"\n \"fmax.s f5, f4, f5 \\n\\t\"\n \"fmax.s f7, f6, f7 \\n\\t\"\n \"fmax.s f3, f1, f3 \\n\\t\"\n \"fmax.s f7, f5, f7 \\n\\t\"\n \"fmax.s f10, f3, f7 \\n\\t\"\n \"fmul.s f10, f10, %[RMAXREC] \\n\\t\"\n \"fsw f10, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 4 \\n\\t\"\n \"fdiv.s f11, %[FONE], f10 \\n\\t\"\n \"addi t6, %[K], 0 \\n\\t\"\n \"TAIL_QUANT%=: \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v0, v0, v0 \\n\\t\"\n \"vsetvli t1, t6, e32, m8 \\n\\t\"\n \"vle32.v v0, (s1) \\n\\t\"\n \"addi s1, s1, 256 \\n\\t\"\n \"sub t6, t6, t1 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfmul.vf v0, v0, f11 \\n\\t\"\n \"vfcvt.x.f.v v0, v0 \\n\\t\"\n \"vsetvli t0, zero, e16, m4 \\n\\t\"\n \"vnclip.wx v0, v0, zero \\n\\t\"\n \"vsetvli t0, t1, e8, m2 \\n\\t\"\n \"vnclip.wx v0, v0, zero \\n\\t\"\n \"vse8.v v0, (%[DST]) \\n\\t\"\n \"addi %[DST], %[DST], 64 \\n\\t\"\n \"bnez t6, TAIL_QUANT%= \\n\\t\"\n \"END%=: \\n\\t\"\n : [SRC] \"+r\"(SRC), [DST] \"+r\"(DST), [K] \"+r\"(CountK)\n : [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal), [BLK] \"r\"(BlkLen), [BUFFER] \"r\"(buffer),\n [CNT] \"r\"(cnt)\n : \"cc\", \"t1\", \"t0\", \"t6\", \"s1\", \"f0\", \"f1\", \"f2\", \"f3\", \"f4\", \"f5\", \"f6\");\n }\n}\n\n} // namespace ime1\n\nnamespace {\n#define SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 \\\n \"vmadot v16, v14, v0 \\n\\t\" \\\n \"vmadot v18, v14, v1 \\n\\t\" \\\n \"vmadot v20, v14, v2 \\n\\t\" \\\n \"vmadot v22, v14, v3 \\n\\t\" \\\n \"vmadot v16, v15, v4 \\n\\t\" \\\n \"vmadot v18, v15, v5 \\n\\t\" \\\n \"vmadot v20, v15, v6 \\n\\t\" \\\n \"vmadot v22, v15, v7 \\n\\t\"\n\n#define SQ4BIT_KERNEL_ACC_1X4X4 \\\n \"vfcvt.f.x.v v16, v16 \\n\\t\" \\\n \"vfcvt.f.x.v v18, v18 \\n\\t\" \\\n \"vfcvt.f.x.v v20, v20 \\n\\t\" \\\n \"vfcvt.f.x.v v22, v22 \\n\\t\" \\\n \"addi s2, s1, 16 \\n\\t\" \\\n \"addi s3, s1, 32 \\n\\t\" \\\n \"addi s4, s1, 48 \\n\\t\" \\\n \"addi s6, s5, 12 \\n\\t\" \\\n \"vfmacc.vv v28, v16, v24 \\n\\t\" \\\n \"vfmacc.vv v29, v18, v25 \\n\\t\" \\\n \"vfmacc.vv v30, v20, v26 \\n\\t\" \\\n \"vfmacc.vv v31, v22, v27 \\n\\t\"\n\n#define SQ4BIT_KERNEL_ACC_F16_1X4X4 \\\n \"vfcvt.f.x.v v16, v16 \\n\\t\" \\\n \"vfcvt.f.x.v v18, v18 \\n\\t\" \\\n \"vfcvt.f.x.v v20, v20 \\n\\t\" \\\n \"vfcvt.f.x.v v22, v22 \\n\\t\" \\\n \"addi s2, s1, 8 \\n\\t\" \\\n \"addi s3, s1, 16 \\n\\t\" \\\n \"addi s4, s1, 24 \\n\\t\" \\\n \"addi s6, s5, 12 \\n\\t\" \\\n \"vfmacc.vv v28, v16, v24 \\n\\t\" \\\n \"vfmacc.vv v29, v18, v25 \\n\\t\" \\\n \"vfmacc.vv v30, v20, v26 \\n\\t\" \\\n \"vfmacc.vv v31, v22, v27 \\n\\t\"\n\n#define SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 \\\n \"vle8.v v4, (s1) \\n\\t\" \\\n \"addi s1, s1, 128 \\n\\t\" \\\n \"vle8.v v5, (s2) \\n\\t\" \\\n \"addi s2, s2, 128 \\n\\t\" \\\n \"vle8.v v6, (s3) \\n\\t\" \\\n \"addi s3, s3, 128 \\n\\t\" \\\n \"vle8.v v7, (s4) \\n\\t\" \\\n \"addi s4, s4, 128 \\n\\t\" \\\n \"vsetvli t0, zero, e8, mf4 \\n\\t\" \\\n \"vle8.v v14, (s5) \\n\\t\" \\\n \"addi s5, s5, 16 \\n\\t\" \\\n \"vle8.v v15, (s6) \\n\\t\" \\\n \"addi s6, s6, 16 \\n\\t\" \\\n \"addi t5, t5, -1 \\n\\t\" \\\n \"vsetvli t0, zero, e8, m1 \\n\\t\" \\\n \"vand.vi v0, v4, 15 \\n\\t\" \\\n \"vand.vi v1, v5, 15 \\n\\t\" \\\n \"vand.vi v2, v6, 15 \\n\\t\" \\\n \"vand.vi v3, v7, 15 \\n\\t\" \\\n \"vsrl.vi v4, v4, 4 \\n\\t\" \\\n \"vsrl.vi v5, v5, 4 \\n\\t\" \\\n \"vsrl.vi v6, v6, 4 \\n\\t\" \\\n \"vsrl.vi v7, v7, 4 \\n\\t\"\n\n#define SQ4BIT_KERNEL_LOAD_ZP_16X1 \\\n \"vsetvli t0, zero, e8, mf2 \\n\\t\" \\\n \"vle8.v v1, (s7) \\n\\t\" \\\n \"vsetvli t0, zero, e8, m1 \\n\\t\" \\\n \"vrgather.vv v8, v1, v13 \\n\\t\" \\\n \"vadd.vi v13, v13, 4 \\n\\t\" \\\n \"vrgather.vv v9, v1, v13 \\n\\t\" \\\n \"vadd.vi v13, v13, 4 \\n\\t\" \\\n \"vrgather.vv v10, v1, v13 \\n\\t\" \\\n \"vadd.vi v13, v13, 4 \\n\\t\" \\\n \"vrgather.vv v11, v1, v13 \\n\\t\" \\\n \"vadd.vi v13, v13, -12 \\n\\t\"\n\n// using for M4Kernel\n#define LOAD_B_16x8x2 \\\n \"vsetvli t0, zero, e8, m1 \\n\\t\" \\\n \"vle8.v v6, (s1) \\n\\t\" \\\n \"addi s1, s1, 32*4 \\n\\t\" \\\n \"vle8.v v7, (s2) \\n\\t\" \\\n \"addi s2, s2, 32*4 \\n\\t\" \\\n \"vle8.v v8, (s3) \\n\\t\" \\\n \"addi s3, s3, 32*4 \\n\\t\" \\\n \"vle8.v v9, (s4) \\n\\t\" \\\n \"addi s4, s4, 32*4 \\n\\t\" \\\n \\\n \"vand.vi v2, v6, 15 \\n\\t\" \\\n \"vand.vi v3, v7, 15 \\n\\t\" \\\n \"vand.vi v4, v8, 15 \\n\\t\" \\\n \"vand.vi v5, v9, 15 \\n\\t\" \\\n \\\n \"vsrl.vi v6, v6, 4 \\n\\t\" \\\n \"vsrl.vi v7, v7, 4 \\n\\t\" \\\n \"vsrl.vi v8, v8, 4 \\n\\t\" \\\n \"vsrl.vi v9, v9, 4 \\n\\t\"\n\n// [s2|s5, s3, s4, s6]\n#define LOAD_SCALE_4x16_FP16 \\\n \"addi s2, s5, -8 \\n\\t\" \\\n \"addi s3, s5, 8 \\n\\t\" \\\n \"addi s4, s5, 16 \\n\\t\" \\\n \"addi s6, s5, 24 \\n\\t\" \\\n \"li t1, 0xf0 \\n\\t\" \\\n \"vmv.s.x v0, t1 \\n\\t\" \\\n \"vsetvli t0, zero, e16, mf4 \\n\\t\" \\\n \"vle16.v v9, (s5) \\n\\t\" \\\n \"vle16.v v11, (s3) \\n\\t\" \\\n \"vle16.v v13, (s4) \\n\\t\" \\\n \"vle16.v v15, (s6) \\n\\t\" \\\n \"vsetvli t0, zero, e16, mf2 \\n\\t\" \\\n \"vle16.v v9, (s2), v0.t \\n\\t\" \\\n \"vle16.v v11, (s5), v0.t \\n\\t\" \\\n \"vle16.v v13, (s3), v0.t \\n\\t\" \\\n \"vle16.v v15, (s4), v0.t \\n\\t\" \\\n \"vfwcvt.f.f.v v8, v9 \\n\\t\" \\\n \"vfwcvt.f.f.v v10, v11 \\n\\t\" \\\n \"vfwcvt.f.f.v v12, v13 \\n\\t\" \\\n \"vfwcvt.f.f.v v14, v15 \\n\\t\" \\\n \"vsetvli t0, zero, e32, m1 \\n\\t\" \\\n \"vmv.v.v v9, v8 \\n\\t\" \\\n \"vmv.v.v v11, v10 \\n\\t\" \\\n \"vmv.v.v v13, v12 \\n\\t\" \\\n \"vmv.v.v v15, v14 \\n\\t\" \\\n \"li t1, 0xf0 \\n\\t\" \\\n \"vmv.s.x v0, t1 \\n\\t\" \\\n \"vsetvli t0, zero, e32, mf2 \\n\\t\" \\\n \"vfmul.vf v8, v8, f1 \\n\\t\" \\\n \"vfmul.vf v10, v10, f1 \\n\\t\" \\\n \"vfmul.vf v12, v12, f1 \\n\\t\" \\\n \"vfmul.vf v14, v14, f1 \\n\\t\" \\\n \"vfmul.vf v9, v9, f3 \\n\\t\" \\\n \"vfmul.vf v11, v11, f3 \\n\\t\" \\\n \"vfmul.vf v13, v13, f3 \\n\\t\" \\\n \"vfmul.vf v15, v15, f3 \\n\\t\" \\\n \"vsetvli t0, zero, e32, m1 \\n\\t\" \\\n \"vfmul.vf v8, v8, f2, v0.t \\n\\t\" \\\n \"vfmul.vf v10, v10, f2, v0.t \\n\\t\" \\\n \"vfmul.vf v12, v12, f2, v0.t \\n\\t\" \\\n \"vfmul.vf v14, v14, f2, v0.t \\n\\t\" \\\n \"vfmul.vf v9, v9, f4, v0.t \\n\\t\" \\\n \"vfmul.vf v11, v11, f4, v0.t \\n\\t\" \\\n \"vfmul.vf v13, v13, f4, v0.t \\n\\t\" \\\n \"vfmul.vf v15, v15, f4, v0.t \\n\\t\"\n\n// [s2|s5, s3, s4, s6]\n#define LOAD_SCALE_4x16 \\\n \"addi s2, s5, -16 \\n\\t\" \\\n \"addi s3, s5, 16 \\n\\t\" \\\n \"addi s4, s5, 32 \\n\\t\" \\\n \"addi s6, s5, 48 \\n\\t\" \\\n \"li t1, 0xf0 \\n\\t\" \\\n \"vmv.s.x v0, t1 \\n\\t\" \\\n \"vsetvli t0, zero, e32, mf2 \\n\\t\" \\\n \"vle32.v v8, (s5) \\n\\t\" \\\n \"vle32.v v10, (s3) \\n\\t\" \\\n \"vle32.v v12, (s4) \\n\\t\" \\\n \"vle32.v v14, (s6) \\n\\t\" \\\n \"vsetvli t0, zero, e32, m1 \\n\\t\" \\\n \"vle32.v v8, (s2), v0.t \\n\\t\" \\\n \"vle32.v v10, (s5), v0.t \\n\\t\" \\\n \"vle32.v v12, (s3), v0.t \\n\\t\" \\\n \"vle32.v v14, (s4), v0.t \\n\\t\" \\\n \"vmv.v.v v9, v8 \\n\\t\" \\\n \"vmv.v.v v11, v10 \\n\\t\" \\\n \"vmv.v.v v13, v12 \\n\\t\" \\\n \"vmv.v.v v15, v14 \\n\\t\" \\\n \"vsetvli t0, zero, e32, mf2 \\n\\t\" \\\n \"vfmul.vf v8, v8, f1 \\n\\t\" \\\n \"vfmul.vf v10, v10, f1 \\n\\t\" \\\n \"vfmul.vf v12, v12, f1 \\n\\t\" \\\n \"vfmul.vf v14, v14, f1 \\n\\t\" \\\n \"vfmul.vf v9, v9, f3 \\n\\t\" \\\n \"vfmul.vf v11, v11, f3 \\n\\t\" \\\n \"vfmul.vf v13, v13, f3 \\n\\t\" \\\n \"vfmul.vf v15, v15, f3 \\n\\t\" \\\n \"vsetvli t0, zero, e32, m1 \\n\\t\" \\\n \"vfmul.vf v8, v8, f2, v0.t \\n\\t\" \\\n \"vfmul.vf v10, v10, f2, v0.t \\n\\t\" \\\n \"vfmul.vf v12, v12, f2, v0.t \\n\\t\" \\\n \"vfmul.vf v14, v14, f2, v0.t \\n\\t\" \\\n \"vfmul.vf v9, v9, f4, v0.t \\n\\t\" \\\n \"vfmul.vf v11, v11, f4, v0.t \\n\\t\" \\\n \"vfmul.vf v13, v13, f4, v0.t \\n\\t\" \\\n \"vfmul.vf v15, v15, f4, v0.t \\n\\t\"\n\n//[s1| BIAS, s2, s3, s4]\n#define LOAD_BIAS \\\n \"vsetvli t0, zero, e32, mf2 \\n\\t\" \\\n \"li t1, 0xf0 \\n\\t\" \\\n \"vmv.s.x v0, t1 \\n\\t\" \\\n \"addi s1, %[BIAS], -16 \\n\\t\" \\\n \"addi s2, %[BIAS], 16 \\n\\t\" \\\n \"addi s3, %[BIAS], 32 \\n\\t\" \\\n \"addi s4, %[BIAS], 48 \\n\\t\" \\\n \\\n \"vle32.v v24, (%[BIAS]) \\n\\t\" \\\n \"vle32.v v26, (s2) \\n\\t\" \\\n \"vle32.v v28, (s3) \\n\\t\" \\\n \"vle32.v v30, (s4) \\n\\t\" \\\n \"vsetvli t0, zero, e32, m1 \\n\\t\" \\\n \"vle32.v v24, (s1), v0.t \\n\\t\" \\\n \"vle32.v v26, (%[BIAS]), v0.t \\n\\t\" \\\n \"vle32.v v28, (s2), v0.t \\n\\t\" \\\n \"vle32.v v30, (s3), v0.t \\n\\t\" \\\n \"vmv.v.v v25, v24 \\n\\t\" \\\n \"vmv.v.v v27, v26 \\n\\t\" \\\n \"vmv.v.v v29, v28 \\n\\t\" \\\n \"vmv.v.v v31, v30 \\n\\t\"\n\n#define SQ4BIT_KERNEL_COMP_4x16x16 \\\n \"vmadot v16, v10, v2 \\n\\t\" \\\n \"vmadot v18, v10, v3 \\n\\t\" \\\n \"vmadot v20, v10, v4 \\n\\t\" \\\n \"vmadot v22, v10, v5 \\n\\t\" \\\n \"vmadot v16, v11, v6 \\n\\t\" \\\n \"vmadot v18, v11, v7 \\n\\t\" \\\n \"vmadot v20, v11, v8 \\n\\t\" \\\n \"vmadot v22, v11, v9 \\n\\t\"\n\n#define SAVE_RESULT_4x16 \\\n \"addi a1, %[C], 0 \\n\\t\" \\\n \"add a2, %[C], %[LDC] \\n\\t\" \\\n \"add a3, a2, %[LDC] \\n\\t\" \\\n \"add a4, a3, %[LDC] \\n\\t\" \\\n \"addi a2, a2, -16 \\n\\t\" \\\n \"addi a4, a4, -16 \\n\\t\" \\\n \"li t1, 0xf0 \\n\\t\" \\\n \"vmv.s.x v0, t1 \\n\\t\" \\\n \"vsetvli t0, zero, e32, mf2 \\n\\t\" \\\n \\\n \"vse32.v v24, (a1) \\n\\t\" \\\n \"addi a1, a1, 16 \\n\\t\" \\\n \"vse32.v v25, (a3) \\n\\t\" \\\n \"addi a3, a3, 16 \\n\\t\" \\\n \\\n \"vse32.v v26, (a1) \\n\\t\" \\\n \"addi a1, a1, 16 \\n\\t\" \\\n \"vse32.v v27, (a3) \\n\\t\" \\\n \"addi a3, a3, 16 \\n\\t\" \\\n \\\n \"vse32.v v28, (a1) \\n\\t\" \\\n \"addi a1, a1, 16 \\n\\t\" \\\n \"vse32.v v29, (a3) \\n\\t\" \\\n \"addi a3, a3, 16 \\n\\t\" \\\n \\\n \"vse32.v v30, (a1) \\n\\t\" \\\n \"vse32.v v31, (a3) \\n\\t\" \\\n \"vsetvli t0, zero, e32, m1 \\n\\t\" \\\n \\\n \"vse32.v v24, (a2), v0.t \\n\\t\" \\\n \"addi a2, a2, 16 \\n\\t\" \\\n \"vse32.v v25, (a4), v0.t \\n\\t\" \\\n \"addi a4, a4, 16 \\n\\t\" \\\n \\\n \"vse32.v v26, (a2), v0.t \\n\\t\" \\\n \"addi a2, a2, 16 \\n\\t\" \\\n \"vse32.v v27, (a4), v0.t \\n\\t\" \\\n \"addi a4, a4, 16 \\n\\t\" \\\n \\\n \"vse32.v v28, (a2), v0.t \\n\\t\" \\\n \"addi a2, a2, 16 \\n\\t\" \\\n \"vse32.v v29, (a4), v0.t \\n\\t\" \\\n \"addi a4, a4, 16 \\n\\t\" \\\n \\\n \"vse32.v v30, (a2), v0.t \\n\\t\" \\\n \"vse32.v v31, (a4), v0.t \\n\\t\"\n\n#define SQ4BIT_KERNEL_LOAD_ZP_16X1_v2 \\\n \"vsetvli t0, zero, e8, mf2 \\n\\t\" \\\n \"vle8.v v11, (s6) \\n\\t\" \\\n \"vsetvli t0, zero, e8, m1 \\n\\t\" \\\n \"vrgather.vv v12, v11, v1 \\n\\t\" \\\n \"vadd.vi v1, v1, 4 \\n\\t\" \\\n \"vrgather.vv v13, v11, v1 \\n\\t\" \\\n \"vadd.vi v1, v1, 4 \\n\\t\" \\\n \"vrgather.vv v14, v11, v1 \\n\\t\" \\\n \"vadd.vi v1, v1, 4 \\n\\t\" \\\n \"vrgather.vv v15, v11, v1 \\n\\t\" \\\n \"vadd.vi v1, v1, -12 \\n\\t\"\n\ntemplate \nvoid SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl(size_t BlkLen,\n const std::byte * QuantA,\n const std::byte * QuantBData,\n const float * QuantBScale,\n const std::byte * QuantBZeroPoint,\n float * C,\n size_t CountN,\n size_t BlockCountK,\n const float * Bias,\n const size_t ldc) {\n GGML_UNUSED(QuantBScale);\n GGML_UNUSED(QuantBZeroPoint);\n size_t LDC = ldc * sizeof(float);\n const size_t INNER = BlkLen / 16;\n float tmp[4 * 16];\n\n if constexpr (HasZeroPoint) {\n for (size_t n = 0; n < CountN; n += 16) {\n size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n;\n std::byte * QuantBDataPtr = (std::byte *) QuantBData + //\n n * BlockCountK * BlkLen / 2 + // b data\n n * BlockCountK * sizeof(uint8_t) + // zp\n n * BlockCountK * sizeof(_Float16); // scale\n float * CPtr = C + n;\n if (NBLKS < 16) {\n CPtr = tmp;\n LDC = 16 * sizeof(float);\n }\n if (Bias != nullptr) {\n const float * bias = Bias + n;\n if (NBLKS < 16) {\n __asm__ volatile(\n \"vsetvli t0, %[N], e32, m2 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"vse32.v v0, (%[DST]) \\n\\t\"\n :\n : [SRC] \"r\"(bias), [DST] \"r\"(tmp), [N] \"r\"(NBLKS)\n : \"cc\", \"t0\");\n bias = tmp;\n }\n __asm__ volatile(LOAD_BIAS\n\n \"addi t3, %[BlockCountK], 0 \\n\\t\"\n\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"li s1, 24 \\n\\t\"\n \"vmv.v.i v1, 3 \\n\\t\"\n \"vsetvli t0, s1, e8, m1 \\n\\t\"\n \"vmv.v.i v1, 2 \\n\\t\"\n \"vsetvli t0, zero, e8, mf2 \\n\\t\"\n \"vmv.v.i v1, 1 \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vmv.v.i v1, 0 \\n\\t\"\n\n \"addi a1, %[A], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n\n \"BLOCK_COUNTK_LOOP%=: \\n\\t\"\n // scale offset\n \"addi s5, s1, 0 \\n\\t\"\n // zp offset\n \"addi s6, s1, 32 \\n\\t\"\n \"addi s1, s6, 16 \\n\\t\"\n \"addi s2, s1, 32 \\n\\t\"\n \"addi s3, s1, 32*2 \\n\\t\"\n \"addi s4, s1, 32*3 \\n\\t\"\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n // load a scale\n \"flw f1, (a1) \\n\\t\"\n \"flw f2, 4(a1) \\n\\t\"\n \"flw f3, 8(a1) \\n\\t\"\n \"flw f4, 12(a1) \\n\\t\"\n \"addi a1, a1, 16 \\n\\t\"\n \"addi t2, %[INNER], 0 \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_ZP_16X1_v2\n\n \"BLOCK_INNER_LOOP%=: \\n\\t\"\n\n LOAD_B_16x8x2\n\n \"vle8.v v10, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vle8.v v11, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vsub.vv v2, v2, v12 \\n\\t\"\n \"vsub.vv v6, v6, v12 \\n\\t\"\n \"vsub.vv v3, v3, v13 \\n\\t\"\n \"vsub.vv v7, v7, v13 \\n\\t\"\n \"vsub.vv v4, v4, v14 \\n\\t\"\n \"vsub.vv v8, v8, v14 \\n\\t\"\n \"vsub.vv v5, v5, v15 \\n\\t\"\n \"vsub.vv v9, v9, v15 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_4x16x16\n\n \"addi t2, t2, -1 \\n\\t\"\n \"bnez t2, BLOCK_INNER_LOOP%= \\n\\t\"\n\n LOAD_SCALE_4x16_FP16\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfcvt.f.x.v v16, v16 \\n\\t\"\n \"vfmacc.vv v24, v16, v8 \\n\\t\"\n \"addi t3, t3, -1 \\n\\t\"\n \"bnez t3, BLOCK_COUNTK_LOOP%= \\n\\t\"\n\n \"RESULT_SAVE%=: \\n\\t\"\n\n SAVE_RESULT_4x16\n\n :\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr), [BIAS] \"r\"(bias)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\",\n \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n\n } else {\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v24, v24, v24 \\n\\t\"\n \"addi t3, %[BlockCountK], 0 \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"li s1, 24 \\n\\t\"\n \"vmv.v.i v1, 3 \\n\\t\"\n \"vsetvli t0, s1, e8, m1 \\n\\t\"\n \"vmv.v.i v1, 2 \\n\\t\"\n \"vsetvli t0, zero, e8, mf2 \\n\\t\"\n \"vmv.v.i v1, 1 \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vmv.v.i v1, 0 \\n\\t\"\n \"addi a1, %[A], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"BLOCK_COUNTK_LOOP%=: \\n\\t\"\n // scale offset\n \"addi s5, s1, 0 \\n\\t\"\n // zp offset\n \"addi s6, s1, 32 \\n\\t\"\n \"addi s1, s6, 16 \\n\\t\"\n \"addi s2, s1, 32 \\n\\t\"\n \"addi s3, s1, 32*2 \\n\\t\"\n \"addi s4, s1, 32*3 \\n\\t\"\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n // load a scale\n \"flw f1, (a1) \\n\\t\"\n \"flw f2, 4(a1) \\n\\t\"\n \"flw f3, 8(a1) \\n\\t\"\n \"flw f4, 12(a1) \\n\\t\"\n \"addi a1, a1, 16 \\n\\t\"\n \"addi t2, %[INNER], 0 \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_ZP_16X1_v2\n\n \"BLOCK_INNER_LOOP%=: \\n\\t\"\n\n LOAD_B_16x8x2\n\n \"vle8.v v10, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vle8.v v11, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vsub.vv v2, v2, v12 \\n\\t\"\n \"vsub.vv v6, v6, v12 \\n\\t\"\n \"vsub.vv v3, v3, v13 \\n\\t\"\n \"vsub.vv v7, v7, v13 \\n\\t\"\n \"vsub.vv v4, v4, v14 \\n\\t\"\n \"vsub.vv v8, v8, v14 \\n\\t\"\n \"vsub.vv v5, v5, v15 \\n\\t\"\n \"vsub.vv v9, v9, v15 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_4x16x16\n\n \"addi t2, t2, -1 \\n\\t\"\n \"bnez t2, BLOCK_INNER_LOOP%= \\n\\t\"\n\n LOAD_SCALE_4x16_FP16\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfcvt.f.x.v v16, v16 \\n\\t\"\n \"vfmacc.vv v24, v16, v8 \\n\\t\"\n \"addi t3, t3, -1 \\n\\t\"\n \"bnez t3, BLOCK_COUNTK_LOOP%= \\n\\t\"\n\n \"RESULT_SAVE%=: \\n\\t\"\n\n SAVE_RESULT_4x16\n\n :\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\", \"s2\", \"s3\",\n \"s4\", \"s5\", \"s6\");\n }\n }\n } else {\n for (size_t n = 0; n < CountN; n += 16) {\n size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n;\n std::byte * QuantBDataPtr = (std::byte *) QuantBData + //\n n * BlockCountK * BlkLen / 2 + // b data\n n * BlockCountK * sizeof(_Float16); // scale\n float * CPtr = C + n;\n if (NBLKS < 16) {\n CPtr = tmp;\n LDC = 16 * sizeof(float);\n }\n if (Bias != nullptr) {\n const float * bias = Bias + n;\n if (NBLKS < 16) {\n __asm__ volatile(\n \"vsetvli t0, %[N], e32, m2 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"vse32.v v0, (%[DST]) \\n\\t\"\n :\n : [SRC] \"r\"(bias), [DST] \"r\"(tmp), [N] \"r\"(NBLKS)\n : \"cc\", \"t0\");\n bias = tmp;\n }\n __asm__ volatile(LOAD_BIAS\n\n \"addi t3, %[BlockCountK], 0 \\n\\t\"\n \"addi a1, %[A], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"BLOCK_COUNTK_LOOP%=: \\n\\t\"\n \"addi s5, s1, 0 \\n\\t\"\n \"addi s1, s5, 32 \\n\\t\"\n \"addi s2, s1, 32 \\n\\t\"\n \"addi s3, s1, 32*2 \\n\\t\"\n \"addi s4, s1, 32*3 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n // load a scale\n \"flw f1, (a1) \\n\\t\"\n \"flw f2, 4(a1) \\n\\t\"\n \"flw f3, 8(a1) \\n\\t\"\n \"flw f4, 12(a1) \\n\\t\"\n \"addi a1, a1, 16 \\n\\t\"\n \"addi t2, %[INNER], 0 \\n\\t\"\n \"BLOCK_INNER_LOOP%=: \\n\\t\"\n\n LOAD_B_16x8x2\n\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"vle8.v v10, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vle8.v v11, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vadd.vi v2, v2, -8 \\n\\t\"\n \"vadd.vi v3, v3, -8 \\n\\t\"\n \"vadd.vi v4, v4, -8 \\n\\t\"\n \"vadd.vi v5, v5, -8 \\n\\t\"\n \"vadd.vi v6, v6, -8 \\n\\t\"\n \"vadd.vi v7, v7, -8 \\n\\t\"\n \"vadd.vi v8, v8, -8 \\n\\t\"\n \"vadd.vi v9, v9, -8 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_4x16x16\n\n \"addi t2, t2, -1 \\n\\t\"\n \"bnez t2, BLOCK_INNER_LOOP%= \\n\\t\"\n\n LOAD_SCALE_4x16_FP16\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfcvt.f.x.v v16, v16 \\n\\t\"\n \"vfmacc.vv v24, v16, v8 \\n\\t\"\n \"addi t3, t3, -1 \\n\\t\"\n \"bnez t3, BLOCK_COUNTK_LOOP%= \\n\\t\"\n \"RESULT_SAVE%=: \\n\\t\"\n\n SAVE_RESULT_4x16\n\n :\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr), [BIAS] \"r\"(bias)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\",\n \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n\n } else {\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v24, v24, v24 \\n\\t\"\n \"addi t3, %[BlockCountK], 0 \\n\\t\"\n \"addi a1, %[A], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"BLOCK_COUNTK_LOOP%=: \\n\\t\"\n \"addi s5, s1, 0 \\n\\t\"\n \"addi s1, s5, 32 \\n\\t\"\n \"addi s2, s1, 32 \\n\\t\"\n \"addi s3, s1, 32*2 \\n\\t\"\n \"addi s4, s1, 32*3 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n // load a scale\n \"flw f1, (a1) \\n\\t\"\n \"flw f2, 4(a1) \\n\\t\"\n \"flw f3, 8(a1) \\n\\t\"\n \"flw f4, 12(a1) \\n\\t\"\n \"addi a1, a1, 16 \\n\\t\"\n \"addi t2, %[INNER], 0 \\n\\t\"\n \"BLOCK_INNER_LOOP%=: \\n\\t\"\n\n LOAD_B_16x8x2\n\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"vle8.v v10, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vle8.v v11, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vadd.vi v2, v2, -8 \\n\\t\"\n \"vadd.vi v3, v3, -8 \\n\\t\"\n \"vadd.vi v4, v4, -8 \\n\\t\"\n \"vadd.vi v5, v5, -8 \\n\\t\"\n \"vadd.vi v6, v6, -8 \\n\\t\"\n \"vadd.vi v7, v7, -8 \\n\\t\"\n \"vadd.vi v8, v8, -8 \\n\\t\"\n \"vadd.vi v9, v9, -8 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_4x16x16\n\n \"addi t2, t2, -1 \\n\\t\"\n \"bnez t2, BLOCK_INNER_LOOP%= \\n\\t\"\n\n LOAD_SCALE_4x16_FP16\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfcvt.f.x.v v16, v16 \\n\\t\"\n \"vfmacc.vv v24, v16, v8 \\n\\t\"\n \"addi t3, t3, -1 \\n\\t\"\n \"bnez t3, BLOCK_COUNTK_LOOP%= \\n\\t\"\n \"RESULT_SAVE%=: \\n\\t\"\n\n SAVE_RESULT_4x16\n\n :\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\", \"s2\", \"s3\",\n \"s4\", \"s5\", \"s6\");\n }\n }\n }\n if (CountN % 16 != 0) {\n // stroe output from tmp to C when NBLKS less than 16.\n float * CPtr = C + CountN / 16 * 16;\n const size_t N = CountN % 16;\n LDC = ldc * sizeof(float);\n __asm__ volatile(\n \"vsetvli t0, %[N], e32, m2 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi s2, %[SRC], 64 \\n\\t\"\n \"addi s3, %[SRC], 64*2 \\n\\t\"\n \"addi s4, %[SRC], 64*3 \\n\\t\"\n \"vle32.v v2, (s2) \\n\\t\"\n \"vle32.v v4, (s3) \\n\\t\"\n \"vle32.v v6, (s4) \\n\\t\"\n \"add t2, %[DST], %[LDC] \\n\\t\"\n \"add t3, t2, %[LDC] \\n\\t\"\n \"add t4, t3, %[LDC] \\n\\t\"\n \"vse32.v v0, (%[DST]) \\n\\t\"\n \"vse32.v v2, (t2) \\n\\t\"\n \"vse32.v v4, (t3) \\n\\t\"\n \"vse32.v v6, (t4) \\n\\t\"\n :\n : [N] \"r\"(N), [SRC] \"r\"(tmp), [DST] \"r\"(CPtr), [LDC] \"r\"(LDC)\n : \"cc\", \"t0\", \"t2\", \"t3\", \"t4\", \"s2\", \"s3\", \"s4\");\n }\n}\n\ntemplate \nvoid SQ4BitGemmM4Kernel_CompInt8_Impl(size_t BlkLen,\n const std::byte * QuantA,\n const std::byte * QuantBData,\n const float * QuantBScale,\n const std::byte * QuantBZeroPoint,\n float * C,\n size_t CountN,\n size_t BlockCountK,\n const float * Bias,\n const size_t ldc) {\n GGML_UNUSED(QuantBScale);\n GGML_UNUSED(QuantBZeroPoint);\n size_t LDC = ldc * sizeof(float);\n const size_t INNER = BlkLen / 16;\n float tmp[4 * 16];\n\n if constexpr (HasZeroPoint) {\n for (size_t n = 0; n < CountN; n += 16) {\n size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n;\n std::byte * QuantBDataPtr = (std::byte *) QuantBData + //\n n * BlockCountK * BlkLen / 2 + // b data\n n * BlockCountK * sizeof(uint8_t) + // zp\n n * BlockCountK * sizeof(float); // scale\n float * CPtr = C + n;\n if (NBLKS < 16) {\n CPtr = tmp;\n LDC = 16 * sizeof(float);\n }\n if (Bias != nullptr) {\n const float * bias = Bias + n;\n if (NBLKS < 16) {\n __asm__ volatile(\n \"vsetvli t0, %[N], e32, m2 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"vse32.v v0, (%[DST]) \\n\\t\"\n :\n : [SRC] \"r\"(bias), [DST] \"r\"(tmp), [N] \"r\"(NBLKS)\n : \"cc\", \"t0\");\n bias = tmp;\n }\n\n __asm__ volatile(LOAD_BIAS\n \"addi t3, %[BlockCountK], 0 \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"li s1, 24 \\n\\t\"\n \"vmv.v.i v1, 3 \\n\\t\"\n \"vsetvli t0, s1, e8, m1 \\n\\t\"\n \"vmv.v.i v1, 2 \\n\\t\"\n \"vsetvli t0, zero, e8, mf2 \\n\\t\"\n \"vmv.v.i v1, 1 \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vmv.v.i v1, 0 \\n\\t\"\n \"addi a1, %[A], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"BLOCK_COUNTK_LOOP%=: \\n\\t\"\n // scale offset\n \"addi s5, s1, 0 \\n\\t\"\n // zp offset\n \"addi s6, s1, 64 \\n\\t\"\n \"addi s1, s6, 16 \\n\\t\"\n \"addi s2, s1, 32 \\n\\t\"\n \"addi s3, s1, 32*2 \\n\\t\"\n \"addi s4, s1, 32*3 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n // load a scale\n \"flw f1, (a1) \\n\\t\"\n \"flw f2, 4(a1) \\n\\t\"\n \"flw f3, 8(a1) \\n\\t\"\n \"flw f4, 12(a1) \\n\\t\"\n \"addi a1, a1, 16 \\n\\t\"\n \"addi t2, %[INNER], 0 \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_ZP_16X1_v2\n\n \"BLOCK_INNER_LOOP%=: \\n\\t\"\n\n LOAD_B_16x8x2\n\n \"vle8.v v10, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vle8.v v11, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vsub.vv v2, v2, v12 \\n\\t\"\n \"vsub.vv v6, v6, v12 \\n\\t\"\n \"vsub.vv v3, v3, v13 \\n\\t\"\n \"vsub.vv v7, v7, v13 \\n\\t\"\n \"vsub.vv v4, v4, v14 \\n\\t\"\n \"vsub.vv v8, v8, v14 \\n\\t\"\n \"vsub.vv v5, v5, v15 \\n\\t\"\n \"vsub.vv v9, v9, v15 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_4x16x16\n\n \"addi t2, t2, -1 \\n\\t\"\n \"bnez t2, BLOCK_INNER_LOOP%= \\n\\t\"\n\n LOAD_SCALE_4x16\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfcvt.f.x.v v16, v16 \\n\\t\"\n \"vfmacc.vv v24, v16, v8 \\n\\t\"\n \"addi t3, t3, -1 \\n\\t\"\n \"bnez t3, BLOCK_COUNTK_LOOP%= \\n\\t\"\n\n \"RESULT_SAVE%=: \\n\\t\"\n\n SAVE_RESULT_4x16\n\n :\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr), [BIAS] \"r\"(bias)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\",\n \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n\n } else {\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v24, v24, v24 \\n\\t\"\n \"addi t3, %[BlockCountK], 0 \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"li s1, 24 \\n\\t\"\n \"vmv.v.i v1, 3 \\n\\t\"\n \"vsetvli t0, s1, e8, m1 \\n\\t\"\n \"vmv.v.i v1, 2 \\n\\t\"\n \"vsetvli t0, zero, e8, mf2 \\n\\t\"\n \"vmv.v.i v1, 1 \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vmv.v.i v1, 0 \\n\\t\"\n \"addi a1, %[A], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"BLOCK_COUNTK_LOOP%=: \\n\\t\"\n // scale offset\n \"addi s5, s1, 0 \\n\\t\"\n // zp offset\n \"addi s6, s1, 64 \\n\\t\"\n \"addi s1, s6, 16 \\n\\t\"\n \"addi s2, s1, 32 \\n\\t\"\n \"addi s3, s1, 32*2 \\n\\t\"\n \"addi s4, s1, 32*3 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n // load a scale\n // load a scale\n \"flw f1, (a1) \\n\\t\"\n \"flw f2, 4(a1) \\n\\t\"\n \"flw f3, 8(a1) \\n\\t\"\n \"flw f4, 12(a1) \\n\\t\"\n \"addi a1, a1, 16 \\n\\t\"\n \"addi t2, %[INNER], 0 \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_ZP_16X1_v2\n\n \"BLOCK_INNER_LOOP%=: \\n\\t\"\n\n LOAD_B_16x8x2\n\n \"vle8.v v10, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vle8.v v11, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vsub.vv v2, v2, v12 \\n\\t\"\n \"vsub.vv v6, v6, v12 \\n\\t\"\n \"vsub.vv v3, v3, v13 \\n\\t\"\n \"vsub.vv v7, v7, v13 \\n\\t\"\n \"vsub.vv v4, v4, v14 \\n\\t\"\n \"vsub.vv v8, v8, v14 \\n\\t\"\n \"vsub.vv v5, v5, v15 \\n\\t\"\n \"vsub.vv v9, v9, v15 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_4x16x16\n\n \"addi t2, t2, -1 \\n\\t\"\n \"bnez t2, BLOCK_INNER_LOOP%= \\n\\t\"\n\n LOAD_SCALE_4x16\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfcvt.f.x.v v16, v16 \\n\\t\"\n \"vfmacc.vv v24, v16, v8 \\n\\t\"\n \"addi t3, t3, -1 \\n\\t\"\n \"bnez t3, BLOCK_COUNTK_LOOP%= \\n\\t\"\n\n \"RESULT_SAVE%=: \\n\\t\"\n\n SAVE_RESULT_4x16\n\n :\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\", \"s2\", \"s3\",\n \"s4\", \"s5\", \"s6\");\n }\n }\n } else {\n for (size_t n = 0; n < CountN; n += 16) {\n size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n;\n std::byte * QuantBDataPtr = (std::byte *) QuantBData + //\n n * BlockCountK * BlkLen / 2 + // b data\n n * BlockCountK * sizeof(float); // scale\n float * CPtr = C + n;\n if (NBLKS < 16) {\n CPtr = tmp;\n LDC = 16 * sizeof(float);\n }\n if (Bias != nullptr) {\n const float * bias = Bias + n;\n if (NBLKS < 16) {\n __asm__ volatile(\n \"vsetvli t0, %[N], e32, m2 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"vse32.v v0, (%[DST]) \\n\\t\"\n :\n : [SRC] \"r\"(bias), [DST] \"r\"(tmp), [N] \"r\"(NBLKS)\n : \"cc\", \"t0\");\n bias = tmp;\n }\n __asm__ volatile(LOAD_BIAS\n \"addi t3, %[BlockCountK], 0 \\n\\t\"\n \"addi a1, %[A], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"BLOCK_COUNTK_LOOP%=: \\n\\t\"\n \"addi s5, s1, 0 \\n\\t\"\n \"addi s1, s5, 64 \\n\\t\"\n \"addi s2, s1, 32 \\n\\t\"\n \"addi s3, s1, 32*2 \\n\\t\"\n \"addi s4, s1, 32*3 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n // load a scale\n \"flw f1, (a1) \\n\\t\"\n \"flw f2, 4(a1) \\n\\t\"\n \"flw f3, 8(a1) \\n\\t\"\n \"flw f4, 12(a1) \\n\\t\"\n \"addi a1, a1, 16 \\n\\t\"\n \"addi t2, %[INNER], 0 \\n\\t\"\n \"BLOCK_INNER_LOOP%=: \\n\\t\"\n\n LOAD_B_16x8x2\n\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"vle8.v v10, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vle8.v v11, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vadd.vi v2, v2, -8 \\n\\t\"\n \"vadd.vi v3, v3, -8 \\n\\t\"\n \"vadd.vi v4, v4, -8 \\n\\t\"\n \"vadd.vi v5, v5, -8 \\n\\t\"\n \"vadd.vi v6, v6, -8 \\n\\t\"\n \"vadd.vi v7, v7, -8 \\n\\t\"\n \"vadd.vi v8, v8, -8 \\n\\t\"\n \"vadd.vi v9, v9, -8 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_4x16x16\n\n \"addi t2, t2, -1 \\n\\t\"\n \"bnez t2, BLOCK_INNER_LOOP%= \\n\\t\"\n\n LOAD_SCALE_4x16\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfcvt.f.x.v v16, v16 \\n\\t\"\n \"vfmacc.vv v24, v16, v8 \\n\\t\"\n \"addi t3, t3, -1 \\n\\t\"\n \"bnez t3, BLOCK_COUNTK_LOOP%= \\n\\t\"\n\n \"RESULT_SAVE%=: \\n\\t\"\n\n SAVE_RESULT_4x16\n\n :\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr), [BIAS] \"r\"(bias)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\",\n \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n\n } else {\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v24, v24, v24 \\n\\t\"\n \"addi t3, %[BlockCountK], 0 \\n\\t\"\n \"addi a1, %[A], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"BLOCK_COUNTK_LOOP%=: \\n\\t\"\n \"addi s5, s1, 0 \\n\\t\"\n \"addi s1, s5, 64 \\n\\t\"\n \"addi s2, s1, 32 \\n\\t\"\n \"addi s3, s1, 32*2 \\n\\t\"\n \"addi s4, s1, 32*3 \\n\\t\"\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n // load a scale\n \"flw f1, (a1) \\n\\t\"\n \"flw f2, 4(a1) \\n\\t\"\n \"flw f3, 8(a1) \\n\\t\"\n \"flw f4, 12(a1) \\n\\t\"\n \"addi a1, a1, 16 \\n\\t\"\n \"addi t2, %[INNER], 0 \\n\\t\"\n \"BLOCK_INNER_LOOP%=: \\n\\t\"\n\n LOAD_B_16x8x2\n\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"vle8.v v10, (a1) \\n\\t\"\n\n \"addi a1, a1, 32 \\n\\t\"\n \"vle8.v v11, (a1) \\n\\t\"\n \"addi a1, a1, 32 \\n\\t\"\n \"vadd.vi v2, v2, -8 \\n\\t\"\n \"vadd.vi v3, v3, -8 \\n\\t\"\n \"vadd.vi v4, v4, -8 \\n\\t\"\n \"vadd.vi v5, v5, -8 \\n\\t\"\n \"vadd.vi v6, v6, -8 \\n\\t\"\n \"vadd.vi v7, v7, -8 \\n\\t\"\n \"vadd.vi v8, v8, -8 \\n\\t\"\n \"vadd.vi v9, v9, -8 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_4x16x16\n\n \"addi t2, t2, -1 \\n\\t\"\n \"bnez t2, BLOCK_INNER_LOOP%= \\n\\t\"\n\n LOAD_SCALE_4x16\n\n \"vsetvli t0, zero, e32, m8 \\n\\t\"\n \"vfcvt.f.x.v v16, v16 \\n\\t\"\n \"vfmacc.vv v24, v16, v8 \\n\\t\"\n \"addi t3, t3, -1 \\n\\t\"\n \"bnez t3, BLOCK_COUNTK_LOOP%= \\n\\t\"\n\n \"RESULT_SAVE%=: \\n\\t\"\n\n SAVE_RESULT_4x16\n\n :\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\", \"s2\", \"s3\",\n \"s4\", \"s5\", \"s6\");\n }\n }\n }\n if (CountN % 16 != 0) {\n // stroe output from tmp to C when NBLKS less than 16.\n float * CPtr = C + CountN / 16 * 16;\n const size_t N = CountN % 16;\n LDC = ldc * sizeof(float);\n __asm__ volatile(\n \"vsetvli t0, %[N], e32, m2 \\n\\t\"\n \"vle32.v v0, (%[SRC]) \\n\\t\"\n \"addi s2, %[SRC], 64 \\n\\t\"\n \"addi s3, %[SRC], 64*2 \\n\\t\"\n \"addi s4, %[SRC], 64*3 \\n\\t\"\n \"vle32.v v2, (s2) \\n\\t\"\n \"vle32.v v4, (s3) \\n\\t\"\n \"vle32.v v6, (s4) \\n\\t\"\n \"add t2, %[DST], %[LDC] \\n\\t\"\n \"add t3, t2, %[LDC] \\n\\t\"\n \"add t4, t3, %[LDC] \\n\\t\"\n \"vse32.v v0, (%[DST]) \\n\\t\"\n \"vse32.v v2, (t2) \\n\\t\"\n \"vse32.v v4, (t3) \\n\\t\"\n \"vse32.v v6, (t4) \\n\\t\"\n :\n : [N] \"r\"(N), [SRC] \"r\"(tmp), [DST] \"r\"(CPtr), [LDC] \"r\"(LDC)\n : \"cc\", \"t0\", \"t2\", \"t3\", \"t4\", \"s2\", \"s3\", \"s4\");\n }\n}\n\ntemplate \nvoid SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(size_t BlkLen,\n const std::byte * QuantA,\n const std::byte * QuantBData,\n const float * QuantBScale,\n const std::byte * QuantBZeroPoint,\n float * C,\n size_t CountN,\n size_t BlockCountK,\n const float * Bias) {\n GGML_UNUSED(QuantBScale);\n GGML_UNUSED(QuantBZeroPoint);\n size_t INNER = BlkLen / 16;\n\n if constexpr (HasZeroPoint) {\n for (size_t n = 0; n < CountN; n += 16) {\n size_t nblks = (CountN - n) > 16 ? 16 : CountN - n;\n std::byte * QuantBDataPtr = (std::byte *) QuantBData + //\n n * BlockCountK * BlkLen / 2 + // b data\n n * BlockCountK * sizeof(uint8_t) + // zp\n n * BlockCountK * sizeof(_Float16); // scale\n float * CPtr = C + n;\n size_t cnt = BlockCountK;\n if (Bias != nullptr) {\n const float * bias = Bias + n;\n __asm__ volatile(\n \"addi t3, %[NBLKS], 0 \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n\n \"vmv.v.i v13, 3 \\n\\t\"\n \"li s1, 24 \\n\\t\"\n \"vsetvli t0, s1, e8, m1 \\n\\t\"\n \"vmv.v.i v13, 2 \\n\\t\"\n \"vsetvli t0, zero, e8, mf2 \\n\\t\"\n \"vmv.v.i v13, 1 \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vmv.v.i v13, 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"addi s2, %[B], 8 \\n\\t\"\n \"addi s3, %[B], 16 \\n\\t\"\n \"addi s4, %[B], 24 \\n\\t\"\n // zp offset\n \"addi s7, %[B], 32 \\n\\t\"\n // a offset\n \"addi s5, %[A], 0 \\n\\t\"\n \"addi s6, %[A], 12 \\n\\t\"\n\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v28, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v29, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v30, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v31, (%[BIAS]) \\n\\t\"\n\n \"LOOP_K%=: \\n\\t\"\n \"vsetvli t0, zero, e16, mf4 \\n\\t\"\n\n \"vle16.v v4, (s1) \\n\\t\"\n \"addi s1, s1, 48 \\n\\t\"\n \"vle16.v v5, (s2) \\n\\t\"\n \"addi s2, s2, 72 \\n\\t\"\n \"vle16.v v6, (s3) \\n\\t\"\n \"addi s3, s3, 96 \\n\\t\"\n \"vle16.v v7, (s4) \\n\\t\"\n \"addi s4, s4, 120 \\n\\t\"\n \"flw f1, (s5) \\n\\t\"\n \"addi s5, s5, 4 \\n\\t\"\n \"vfwcvt.f.f.v v8, v4 \\n\\t\"\n \"vfwcvt.f.f.v v9, v5 \\n\\t\"\n \"vfwcvt.f.f.v v10, v6 \\n\\t\"\n \"vfwcvt.f.f.v v11, v7 \\n\\t\"\n\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n \"addi t5, %[INNER], 0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v18, v18, v18 \\n\\t\"\n \"vxor.vv v20, v20, v20 \\n\\t\"\n \"vxor.vv v22, v22, v22 \\n\\t\"\n \"vfmul.vf v24, v8, f1 \\n\\t\"\n \"vfmul.vf v25, v9, f1 \\n\\t\"\n \"vfmul.vf v26, v10, f1 \\n\\t\"\n \"vfmul.vf v27, v11, f1 \\n\\t\"\n \"addi %[CNT], %[CNT], -1 \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_ZP_16X1\n\n \"LOOP_INNER%=: \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n \"vsub.vv v0, v0, v8 \\n\\t\"\n \"vsub.vv v4, v4, v8 \\n\\t\"\n \"vsub.vv v1, v1, v9 \\n\\t\"\n \"vsub.vv v5, v5, v9 \\n\\t\"\n \"vsub.vv v2, v2, v10 \\n\\t\"\n \"vsub.vv v6, v6, v10 \\n\\t\"\n \"vsub.vv v3, v3, v11 \\n\\t\"\n \"vsub.vv v7, v7, v11 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n \"bnez t5, LOOP_INNER%= \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n SQ4BIT_KERNEL_ACC_F16_1X4X4\n \"addi s7, s1, 32 \\n\\t\"\n\n \"bnez %[CNT], LOOP_K%= \\n\\t\"\n \"addi t3, zero, 16 \\n\\t\"\n \"addi s1, %[C], 16 \\n\\t\"\n \"addi s2, %[C], 32 \\n\\t\"\n \"addi s3, %[C], 48 \\n\\t\"\n \"blt %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"jal x0, END%= \\n\\t\"\n\n \"ST_TAIL%=: \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"END%=: \\n\\t\"\n\n : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks), [BIAS] \"+r\"(bias)\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\", \"s7\");\n } else {\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vxor.vv v28, v28, v28 \\n\\t\"\n\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"vmv.v.i v13, 3 \\n\\t\"\n \"li s1, 24 \\n\\t\"\n \"vsetvli t0, s1, e8, m1 \\n\\t\"\n \"vmv.v.i v13, 2 \\n\\t\"\n \"vsetvli t0, zero, e8, mf2 \\n\\t\"\n \"vmv.v.i v13, 1 \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vmv.v.i v13, 0 \\n\\t\"\n\n \"addi s1, %[B], 0 \\n\\t\"\n \"addi s2, %[B], 8 \\n\\t\"\n \"addi s3, %[B], 16 \\n\\t\"\n \"addi s4, %[B], 24 \\n\\t\"\n\n \"addi s7, %[B], 32 \\n\\t\"\n\n \"addi s5, %[A], 0 \\n\\t\"\n \"addi s6, %[A], 12 \\n\\t\"\n \"LOOP_K%=: \\n\\t\"\n \"vsetvli t0, zero, e16, mf4 \\n\\t\"\n \"vle16.v v4, (s1) \\n\\t\"\n \"addi s1, s1, 48 \\n\\t\"\n \"vle16.v v5, (s2) \\n\\t\"\n \"addi s2, s2, 72 \\n\\t\"\n \"vle16.v v6, (s3) \\n\\t\"\n \"addi s3, s3, 96 \\n\\t\"\n \"vle16.v v7, (s4) \\n\\t\"\n \"addi s4, s4, 120 \\n\\t\"\n \"flw f1, (s5) \\n\\t\"\n \"addi s5, s5, 4 \\n\\t\"\n\n \"vfwcvt.f.f.v v8, v4 \\n\\t\"\n \"vfwcvt.f.f.v v9, v5 \\n\\t\"\n \"vfwcvt.f.f.v v10, v6 \\n\\t\"\n \"vfwcvt.f.f.v v11, v7 \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n \"addi t5, %[INNER], 0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v18, v18, v18 \\n\\t\"\n \"vxor.vv v20, v20, v20 \\n\\t\"\n \"vxor.vv v22, v22, v22 \\n\\t\"\n \"vfmul.vf v24, v8, f1 \\n\\t\"\n \"vfmul.vf v25, v9, f1 \\n\\t\"\n \"vfmul.vf v26, v10, f1 \\n\\t\"\n \"vfmul.vf v27, v11, f1 \\n\\t\"\n \"addi %[CNT], %[CNT], -1 \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_ZP_16X1\n\n \"LOOP_INNER%=: \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n \"vsub.vv v0, v0, v8 \\n\\t\"\n \"vsub.vv v4, v4, v8 \\n\\t\"\n \"vsub.vv v1, v1, v9 \\n\\t\"\n \"vsub.vv v5, v5, v9 \\n\\t\"\n \"vsub.vv v2, v2, v10 \\n\\t\"\n \"vsub.vv v6, v6, v10 \\n\\t\"\n \"vsub.vv v3, v3, v11 \\n\\t\"\n \"vsub.vv v7, v7, v11 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n \"bnez t5, LOOP_INNER%= \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n SQ4BIT_KERNEL_ACC_F16_1X4X4\n \"addi s7, s1, 32 \\n\\t\"\n\n \"bnez %[CNT], LOOP_K%= \\n\\t\"\n \"addi t3, zero, 16 \\n\\t\"\n \"addi s1, %[C], 16 \\n\\t\"\n \"addi s2, %[C], 32 \\n\\t\"\n \"addi s3, %[C], 48 \\n\\t\"\n \"blt %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"jal x0, END%= \\n\\t\"\n\n \"ST_TAIL%=: \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"END%=: \\n\\t\"\n\n : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks)\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\", \"s7\");\n }\n }\n } else {\n for (size_t n = 0; n < CountN; n += 16) {\n size_t nblks = (CountN - n) > 16 ? 16 : CountN - n;\n std::byte * QuantBDataPtr = (std::byte *) QuantBData + //\n n * BlockCountK * BlkLen / 2 + // b data\n n * BlockCountK * sizeof(_Float16); // scale\n float * CPtr = C + n;\n size_t cnt = BlockCountK;\n if (Bias != nullptr) {\n const float * bias = Bias + n;\n __asm__ volatile(\n \"addi t3, %[NBLKS], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"addi s2, %[B], 8 \\n\\t\"\n \"addi s3, %[B], 16 \\n\\t\"\n \"addi s4, %[B], 24 \\n\\t\"\n \"addi s5, %[A], 0 \\n\\t\"\n \"addi s6, %[A], 12 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v28, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v29, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v30, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v31, (%[BIAS]) \\n\\t\"\n\n \"LOOP_K%=: \\n\\t\"\n \"vsetvli t0, zero, e16, mf4 \\n\\t\"\n\n \"vle16.v v4, (s1) \\n\\t\"\n \"addi s1, s1, 32 \\n\\t\"\n \"vle16.v v5, (s2) \\n\\t\"\n \"addi s2, s2, 56 \\n\\t\"\n \"vle16.v v6, (s3) \\n\\t\"\n \"addi s3, s3, 80 \\n\\t\"\n \"vle16.v v7, (s4) \\n\\t\"\n \"addi s4, s4, 104 \\n\\t\"\n \"flw f1, (s5) \\n\\t\"\n \"addi s5, s5, 4 \\n\\t\"\n \"vfwcvt.f.f.v v8, v4 \\n\\t\"\n \"vfwcvt.f.f.v v9, v5 \\n\\t\"\n \"vfwcvt.f.f.v v10, v6 \\n\\t\"\n \"vfwcvt.f.f.v v11, v7 \\n\\t\"\n\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n \"addi t5, %[INNER], 0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v18, v18, v18 \\n\\t\"\n \"vxor.vv v20, v20, v20 \\n\\t\"\n \"vxor.vv v22, v22, v22 \\n\\t\"\n \"vfmul.vf v24, v8, f1 \\n\\t\"\n \"vfmul.vf v25, v9, f1 \\n\\t\"\n \"vfmul.vf v26, v10, f1 \\n\\t\"\n \"vfmul.vf v27, v11, f1 \\n\\t\"\n \"addi %[CNT], %[CNT], -1 \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"LOOP_INNER%=: \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n \"vadd.vi v0, v0, -8 \\n\\t\"\n \"vadd.vi v1, v1, -8 \\n\\t\"\n \"vadd.vi v2, v2, -8 \\n\\t\"\n \"vadd.vi v3, v3, -8 \\n\\t\"\n \"vadd.vi v4, v4, -8 \\n\\t\"\n \"vadd.vi v5, v5, -8 \\n\\t\"\n \"vadd.vi v6, v6, -8 \\n\\t\"\n \"vadd.vi v7, v7, -8 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n \"bnez t5, LOOP_INNER%= \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n SQ4BIT_KERNEL_ACC_F16_1X4X4\n\n \"bnez %[CNT], LOOP_K%= \\n\\t\"\n \"addi t3, zero, 16 \\n\\t\"\n \"addi s1, %[C], 16 \\n\\t\"\n \"addi s2, %[C], 32 \\n\\t\"\n \"addi s3, %[C], 48 \\n\\t\"\n \"blt %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"jal x0, END%= \\n\\t\"\n\n \"ST_TAIL%=: \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"END%=: \\n\\t\"\n\n : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks), [BIAS] \"+r\"(bias)\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n } else {\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vxor.vv v28, v28, v28 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"addi s2, %[B], 8 \\n\\t\"\n \"addi s3, %[B], 16 \\n\\t\"\n \"addi s4, %[B], 24 \\n\\t\"\n\n \"addi s5, %[A], 0 \\n\\t\"\n \"addi s6, %[A], 12 \\n\\t\"\n \"LOOP_K%=: \\n\\t\"\n \"vsetvli t0, zero, e16, mf4 \\n\\t\"\n \"vle16.v v4, (s1) \\n\\t\"\n \"addi s1, s1, 32 \\n\\t\"\n \"vle16.v v5, (s2) \\n\\t\"\n \"addi s2, s2, 56 \\n\\t\"\n \"vle16.v v6, (s3) \\n\\t\"\n \"addi s3, s3, 80 \\n\\t\"\n \"vle16.v v7, (s4) \\n\\t\"\n \"addi s4, s4, 104 \\n\\t\"\n \"flw f1, (s5) \\n\\t\"\n \"addi s5, s5, 4 \\n\\t\"\n\n \"vfwcvt.f.f.v v8, v4 \\n\\t\"\n \"vfwcvt.f.f.v v9, v5 \\n\\t\"\n \"vfwcvt.f.f.v v10, v6 \\n\\t\"\n \"vfwcvt.f.f.v v11, v7 \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n \"addi t5, %[INNER], 0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v18, v18, v18 \\n\\t\"\n \"vxor.vv v20, v20, v20 \\n\\t\"\n \"vxor.vv v22, v22, v22 \\n\\t\"\n \"vfmul.vf v24, v8, f1 \\n\\t\"\n \"vfmul.vf v25, v9, f1 \\n\\t\"\n \"vfmul.vf v26, v10, f1 \\n\\t\"\n \"vfmul.vf v27, v11, f1 \\n\\t\"\n \"addi %[CNT], %[CNT], -1 \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"LOOP_INNER%=: \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n \"vadd.vi v0, v0, -8 \\n\\t\"\n \"vadd.vi v1, v1, -8 \\n\\t\"\n \"vadd.vi v2, v2, -8 \\n\\t\"\n \"vadd.vi v3, v3, -8 \\n\\t\"\n \"vadd.vi v4, v4, -8 \\n\\t\"\n \"vadd.vi v5, v5, -8 \\n\\t\"\n \"vadd.vi v6, v6, -8 \\n\\t\"\n \"vadd.vi v7, v7, -8 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n \"bnez t5, LOOP_INNER%= \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n SQ4BIT_KERNEL_ACC_F16_1X4X4\n\n \"bnez %[CNT], LOOP_K%= \\n\\t\"\n \"addi t3, zero, 16 \\n\\t\"\n \"addi s1, %[C], 16 \\n\\t\"\n \"addi s2, %[C], 32 \\n\\t\"\n \"addi s3, %[C], 48 \\n\\t\"\n \"blt %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"jal x0, END%= \\n\\t\"\n\n \"ST_TAIL%=: \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"END%=: \\n\\t\"\n\n : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks)\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n }\n }\n }\n}\n\ntemplate \nvoid SQ4BitGemmM1Kernel_CompInt8_Impl(size_t BlkLen,\n const std::byte * QuantA,\n const std::byte * QuantBData,\n const float * QuantBScale,\n const std::byte * QuantBZeroPoint,\n float * C,\n size_t CountN,\n size_t BlockCountK,\n const float * Bias) {\n GGML_UNUSED(QuantBScale);\n GGML_UNUSED(QuantBZeroPoint);\n const size_t INNER = BlkLen / 16;\n if constexpr (HasZeroPoint) {\n for (size_t n = 0; n < CountN; n += 16) {\n size_t nblks = (CountN - n) > 16 ? 16 : CountN - n;\n std::byte * QuantBDataPtr = (std::byte *) QuantBData + //\n n * BlockCountK * BlkLen / 2 + // b data\n n * BlockCountK * sizeof(uint8_t) + // zp\n n * BlockCountK * sizeof(float); // scale\n float * CPtr = C + n;\n size_t cnt = BlockCountK;\n if (Bias != nullptr) {\n const float * bias = Bias + n;\n __asm__ volatile(\n \"addi t3, %[NBLKS], 0 \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"vmv.v.i v13, 3 \\n\\t\"\n \"li s1, 24 \\n\\t\"\n \"vsetvli t0, s1, e8, m1 \\n\\t\"\n \"vmv.v.i v13, 2 \\n\\t\"\n \"vsetvli t0, zero, e8, mf2 \\n\\t\"\n \"vmv.v.i v13, 1 \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vmv.v.i v13, 0 \\n\\t\"\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vxor.vv v28, v28, v28 \\n\\t\"\n\n // scale offset, scale0.0, scale1.0, scale2.0, scale3.0....scale15.0\n \"addi s1, %[B], 0 \\n\\t\"\n \"addi s2, %[B], 16 \\n\\t\"\n \"addi s3, %[B], 32 \\n\\t\"\n \"addi s4, %[B], 48 \\n\\t\"\n // zp offset\n \"addi s7, %[B], 64 \\n\\t\"\n // a offset\n \"addi s5, %[A], 0 \\n\\t\"\n \"addi s6, %[A], 12 \\n\\t\"\n\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v28, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v29, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v30, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v31, (%[BIAS]) \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n \"LOOP_K%=: \\n\\t\"\n\n // load scale\n \"vle32.v v8, (s1) \\n\\t\"\n \"addi s1, s1, 80 \\n\\t\"\n \"vle32.v v9, (s2) \\n\\t\"\n \"addi s2, s2, 96 \\n\\t\"\n \"vle32.v v10, (s3) \\n\\t\"\n \"addi s3, s3, 112 \\n\\t\"\n \"vle32.v v11, (s4) \\n\\t\"\n \"addi s4, s4, 128 \\n\\t\"\n\n // load a scale\n \"flw f1, (s5) \\n\\t\"\n \"addi s5, s5, 4 \\n\\t\"\n\n \"addi t5, %[INNER], 0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v18, v18, v18 \\n\\t\"\n \"vxor.vv v20, v20, v20 \\n\\t\"\n \"vxor.vv v22, v22, v22 \\n\\t\"\n\n // a scale * b scale\n \"vfmul.vf v24, v8, f1 \\n\\t\"\n \"vfmul.vf v25, v9, f1 \\n\\t\"\n \"vfmul.vf v26, v10, f1 \\n\\t\"\n \"vfmul.vf v27, v11, f1 \\n\\t\"\n \"addi %[CNT], %[CNT], -1 \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_ZP_16X1\n\n \"LOOP_INNER%=: \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n \"vsub.vv v0, v0, v8 \\n\\t\"\n \"vsub.vv v4, v4, v8 \\n\\t\"\n \"vsub.vv v1, v1, v9 \\n\\t\"\n \"vsub.vv v5, v5, v9 \\n\\t\"\n \"vsub.vv v2, v2, v10 \\n\\t\"\n \"vsub.vv v6, v6, v10 \\n\\t\"\n \"vsub.vv v3, v3, v11 \\n\\t\"\n \"vsub.vv v7, v7, v11 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n \"bnez t5, LOOP_INNER%= \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n SQ4BIT_KERNEL_ACC_1X4X4\n \"addi s7, s1, 64 \\n\\t\"\n\n \"bnez %[CNT], LOOP_K%= \\n\\t\"\n\n \"addi t3, zero, 16 \\n\\t\"\n \"addi s1, %[C], 16 \\n\\t\"\n \"addi s2, %[C], 32 \\n\\t\"\n \"addi s3, %[C], 48 \\n\\t\"\n \"blt %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"jal x0, END%= \\n\\t\"\n\n \"ST_TAIL%=: \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"END%=: \\n\\t\"\n\n : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks), [BIAS] \"+r\"(bias)\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\", \"s7\");\n } else {\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vxor.vv v28, v28, v28 \\n\\t\"\n\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"vmv.v.i v13, 3 \\n\\t\"\n \"li s1, 24 \\n\\t\"\n \"vsetvli t0, s1, e8, m1 \\n\\t\"\n \"vmv.v.i v13, 2 \\n\\t\"\n \"vsetvli t0, zero, e8, mf2 \\n\\t\"\n \"vmv.v.i v13, 1 \\n\\t\"\n \"vsetvli t0, zero, e8, mf4 \\n\\t\"\n \"vmv.v.i v13, 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"addi s2, %[B], 16 \\n\\t\"\n \"addi s3, %[B], 32 \\n\\t\"\n \"addi s4, %[B], 48 \\n\\t\"\n\n \"addi s7, %[B], 64 \\n\\t\"\n\n \"addi s5, %[A], 0 \\n\\t\"\n \"addi s6, %[A], 12 \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n \"LOOP_K%=: \\n\\t\"\n \"vle32.v v8, (s1) \\n\\t\"\n \"addi s1, s1, 80 \\n\\t\"\n \"vle32.v v9, (s2) \\n\\t\"\n \"addi s2, s2, 96 \\n\\t\"\n \"vle32.v v10, (s3) \\n\\t\"\n \"addi s3, s3, 112 \\n\\t\"\n \"vle32.v v11, (s4) \\n\\t\"\n \"addi s4, s4, 128 \\n\\t\"\n\n \"flw f1, (s5) \\n\\t\"\n \"addi s5, s5, 4 \\n\\t\"\n\n \"addi t5, %[INNER], 0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v18, v18, v18 \\n\\t\"\n \"vxor.vv v20, v20, v20 \\n\\t\"\n \"vxor.vv v22, v22, v22 \\n\\t\"\n\n \"vfmul.vf v24, v8, f1 \\n\\t\"\n \"vfmul.vf v25, v9, f1 \\n\\t\"\n \"vfmul.vf v26, v10, f1 \\n\\t\"\n \"vfmul.vf v27, v11, f1 \\n\\t\"\n \"addi %[CNT], %[CNT], -1 \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_ZP_16X1\n\n \"LOOP_INNER%=: \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n \"vsub.vv v0, v0, v8 \\n\\t\"\n \"vsub.vv v4, v4, v8 \\n\\t\"\n \"vsub.vv v1, v1, v9 \\n\\t\"\n \"vsub.vv v5, v5, v9 \\n\\t\"\n \"vsub.vv v2, v2, v10 \\n\\t\"\n \"vsub.vv v6, v6, v10 \\n\\t\"\n \"vsub.vv v3, v3, v11 \\n\\t\"\n \"vsub.vv v7, v7, v11 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n \"bnez t5, LOOP_INNER%= \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n SQ4BIT_KERNEL_ACC_1X4X4\n \"addi s7, s1, 64 \\n\\t\"\n\n \"bnez %[CNT], LOOP_K%= \\n\\t\"\n\n \"addi t3, zero, 16 \\n\\t\"\n \"addi s1, %[C], 16 \\n\\t\"\n \"addi s2, %[C], 32 \\n\\t\"\n \"addi s3, %[C], 48 \\n\\t\"\n \"blt %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"jal x0, END%= \\n\\t\"\n\n \"ST_TAIL%=: \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"END%=: \\n\\t\"\n\n : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks)\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\", \"s7\");\n }\n }\n } else {\n for (size_t n = 0; n < CountN; n += 16) {\n size_t nblks = (CountN - n) > 16 ? 16 : CountN - n;\n std::byte * QuantBDataPtr = (std::byte *) QuantBData + //\n n * BlockCountK * BlkLen / 2 + // b data\n n * BlockCountK * sizeof(float); // scale\n float * CPtr = C + n;\n size_t cnt = BlockCountK;\n if (Bias != nullptr) {\n const float * bias = Bias + n;\n __asm__ volatile(\n \"addi t3, %[NBLKS], 0 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"addi s2, %[B], 16 \\n\\t\"\n \"addi s3, %[B], 32 \\n\\t\"\n \"addi s4, %[B], 48 \\n\\t\"\n \"addi s5, %[A], 0 \\n\\t\"\n \"addi s6, %[A], 12 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v28, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v29, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v30, (%[BIAS]) \\n\\t\"\n \"sub t3, t3, t0 \\n\\t\"\n \"addi %[BIAS], %[BIAS], 16 \\n\\t\"\n \"vsetvli t0, t3, e32, mf2 \\n\\t\"\n \"vle32.v v31, (%[BIAS]) \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n \"LOOP_K%=: \\n\\t\"\n \"vle32.v v8, (s1) \\n\\t\"\n \"addi s1, s1, 64 \\n\\t\"\n \"vle32.v v9, (s2) \\n\\t\"\n \"addi s2, s2, 80 \\n\\t\"\n \"vle32.v v10, (s3) \\n\\t\"\n \"addi s3, s3, 96 \\n\\t\"\n \"vle32.v v11, (s4) \\n\\t\"\n \"addi s4, s4, 112 \\n\\t\"\n \"flw f1, (s5) \\n\\t\"\n \"addi s5, s5, 4 \\n\\t\"\n\n \"addi t5, %[INNER], 0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v18, v18, v18 \\n\\t\"\n \"vxor.vv v20, v20, v20 \\n\\t\"\n \"vxor.vv v22, v22, v22 \\n\\t\"\n \"vfmul.vf v24, v8, f1 \\n\\t\"\n \"vfmul.vf v25, v9, f1 \\n\\t\"\n \"vfmul.vf v26, v10, f1 \\n\\t\"\n \"vfmul.vf v27, v11, f1 \\n\\t\"\n \"addi %[CNT], %[CNT], -1 \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"LOOP_INNER%=: \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n \"vadd.vi v0, v0, -8 \\n\\t\"\n \"vadd.vi v1, v1, -8 \\n\\t\"\n \"vadd.vi v2, v2, -8 \\n\\t\"\n \"vadd.vi v3, v3, -8 \\n\\t\"\n \"vadd.vi v4, v4, -8 \\n\\t\"\n \"vadd.vi v5, v5, -8 \\n\\t\"\n \"vadd.vi v6, v6, -8 \\n\\t\"\n \"vadd.vi v7, v7, -8 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n \"bnez t5, LOOP_INNER%= \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n SQ4BIT_KERNEL_ACC_1X4X4\n\n \"bnez %[CNT], LOOP_K%= \\n\\t\"\n \"addi t3, zero, 16 \\n\\t\"\n \"addi s1, %[C], 16 \\n\\t\"\n \"addi s2, %[C], 32 \\n\\t\"\n \"addi s3, %[C], 48 \\n\\t\"\n \"blt %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"jal x0, END%= \\n\\t\"\n\n \"ST_TAIL%=: \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"END%=: \\n\\t\"\n\n : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks), [BIAS] \"+r\"(bias)\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n } else {\n __asm__ volatile(\n \"vsetvli t0, zero, e32, m4 \\n\\t\"\n \"vxor.vv v28, v28, v28 \\n\\t\"\n \"addi s1, %[B], 0 \\n\\t\"\n \"addi s2, %[B], 16 \\n\\t\"\n \"addi s3, %[B], 32 \\n\\t\"\n \"addi s4, %[B], 48 \\n\\t\"\n\n \"addi s5, %[A], 0 \\n\\t\"\n \"addi s6, %[A], 12 \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n \"LOOP_K%=: \\n\\t\"\n \"vle32.v v8, (s1) \\n\\t\"\n \"addi s1, s1, 64 \\n\\t\"\n \"vle32.v v9, (s2) \\n\\t\"\n \"addi s2, s2, 80 \\n\\t\"\n \"vle32.v v10, (s3) \\n\\t\"\n \"addi s3, s3, 96 \\n\\t\"\n \"vle32.v v11, (s4) \\n\\t\"\n \"addi s4, s4, 112 \\n\\t\"\n \"flw f1, (s5) \\n\\t\"\n \"addi s5, s5, 4 \\n\\t\"\n\n \"addi t5, %[INNER], 0 \\n\\t\"\n \"vxor.vv v16, v16, v16 \\n\\t\"\n \"vxor.vv v18, v18, v18 \\n\\t\"\n \"vxor.vv v20, v20, v20 \\n\\t\"\n \"vxor.vv v22, v22, v22 \\n\\t\"\n \"vfmul.vf v24, v8, f1 \\n\\t\"\n \"vfmul.vf v25, v9, f1 \\n\\t\"\n \"vfmul.vf v26, v10, f1 \\n\\t\"\n \"vfmul.vf v27, v11, f1 \\n\\t\"\n \"addi %[CNT], %[CNT], -1 \\n\\t\"\n \"vsetvli t0, zero, e8, m1 \\n\\t\"\n \"LOOP_INNER%=: \\n\\t\"\n\n SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n \"vadd.vi v0, v0, -8 \\n\\t\"\n \"vadd.vi v1, v1, -8 \\n\\t\"\n \"vadd.vi v2, v2, -8 \\n\\t\"\n \"vadd.vi v3, v3, -8 \\n\\t\"\n \"vadd.vi v4, v4, -8 \\n\\t\"\n \"vadd.vi v5, v5, -8 \\n\\t\"\n \"vadd.vi v6, v6, -8 \\n\\t\"\n \"vadd.vi v7, v7, -8 \\n\\t\"\n\n SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n \"bnez t5, LOOP_INNER%= \\n\\t\"\n \"vsetvli t0, zero, e32, mf2 \\n\\t\"\n\n SQ4BIT_KERNEL_ACC_1X4X4\n\n \"bnez %[CNT], LOOP_K%= \\n\\t\"\n \"addi t3, zero, 16 \\n\\t\"\n \"addi s1, %[C], 16 \\n\\t\"\n \"addi s2, %[C], 32 \\n\\t\"\n \"addi s3, %[C], 48 \\n\\t\"\n \"blt %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"jal x0, END%= \\n\\t\"\n\n \"ST_TAIL%=: \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v28, (%[C]) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v29, (s1) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v30, (s2) \\n\\t\"\n \"vsetvli t0, %[NBLKS], e32, mf2 \\n\\t\"\n \"sub %[NBLKS], %[NBLKS], t0 \\n\\t\"\n \"vse32.v v31, (s3) \\n\\t\"\n \"END%=: \\n\\t\"\n\n : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks)\n : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n }\n }\n }\n}\n\ntemplate \ninline void SQ4BitGemmM4Kernel_CompInt8_DispatchOnBlkLen(size_t BlkLen,\n const std::byte * QuantA,\n const std::byte * QuantBData,\n const float * QuantBScale,\n const std::byte * QuantBZeroPoint,\n float * C,\n size_t CountM,\n size_t CountN,\n size_t BlockStrideQuantB,\n const float * Bias,\n const size_t ldc,\n const size_t scalestride) {\n if (scalestride == 4) {\n SQ4BitGemmM4Kernel_CompInt8_Impl(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, C,\n CountN, BlockStrideQuantB, Bias, ldc);\n\n } else if (scalestride == 2) {\n SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl(\n BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, C, CountN, BlockStrideQuantB, Bias, ldc);\n }\n}\n\ntemplate \ninline void SQ4BitGemmM1Kernel_CompInt8_DispatchOnBlkLen(size_t BlkLen,\n const std::byte * QuantA,\n const std::byte * QuantBData,\n const float * QuantBScale,\n const std::byte * QuantBZeroPoint,\n float * C,\n size_t CountM,\n size_t CountN,\n size_t BlockStrideQuantB,\n const float * Bias,\n const size_t ldc,\n const size_t scalestride) {\n if (scalestride == 4) {\n SQ4BitGemmM1Kernel_CompInt8_Impl(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, C,\n CountN, BlockStrideQuantB, Bias);\n } else if (scalestride == 2) {\n SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(BlkLen, QuantA, QuantBData, QuantBScale,\n QuantBZeroPoint, C, CountN, BlockStrideQuantB, Bias);\n }\n}\n\n} // namespace\n\nnamespace ime1 {\nsize_t gemm_kernel_i8i4(size_t BlkLen,\n const std::byte * QuantA,\n const std::byte * QuantBData,\n const float * QuantBScale,\n const std::byte * QuantBZeroPoint,\n float * C,\n size_t CountM,\n size_t CountN,\n size_t CountK,\n size_t BlockCountK,\n size_t ldc,\n const float * Bias,\n const size_t ScaleStride) {\n GGML_UNUSED(CountM);\n GGML_UNUSED(CountK);\n GGML_UNUSED(ldc);\n if (CountM >= 4) {\n if (QuantBZeroPoint != nullptr) {\n SQ4BitGemmM4Kernel_CompInt8_DispatchOnBlkLen(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint,\n C, CountM, CountN, BlockCountK, Bias, ldc, ScaleStride);\n } else {\n SQ4BitGemmM4Kernel_CompInt8_DispatchOnBlkLen(BlkLen, QuantA, QuantBData, QuantBScale,\n QuantBZeroPoint, C, CountM, CountN, BlockCountK, Bias,\n ldc, ScaleStride);\n }\n return 4;\n } else {\n if (QuantBZeroPoint != nullptr) {\n SQ4BitGemmM1Kernel_CompInt8_DispatchOnBlkLen(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint,\n C, CountM, CountN, BlockCountK, Bias, ldc, ScaleStride);\n } else {\n SQ4BitGemmM1Kernel_CompInt8_DispatchOnBlkLen(BlkLen, QuantA, QuantBData, QuantBScale,\n QuantBZeroPoint, C, CountM, CountN, BlockCountK, Bias,\n ldc, ScaleStride);\n }\n return 1;\n }\n}\n} // namespace ime1\n} // namespace sqnbitgemm_spacemit_ime\n" }, { "path": "src/ggml-cpu/spacemit/ime_kernels.h", "content": "#pragma once\n\n#include \n\nnamespace sqnbitgemm_spacemit_ime {\nnamespace ime1 {\nsize_t gemm_kernel_i8i4(size_t blk_len,\n const std::byte * quant_a_ptr,\n const std::byte * quant_b_data,\n const float * quant_b_scale,\n const std::byte * quant_b_zp,\n float * c_ptr,\n size_t count_m,\n size_t count_n,\n size_t count_k,\n size_t block_count_k,\n size_t ldc,\n const float * bias,\n const size_t scale_stride);\n\nvoid quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, std::byte * quant_a_ptr);\n\nvoid quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, std::byte * quant_a_ptr);\n\n} // namespace ime1\n} // namespace sqnbitgemm_spacemit_ime\n" }, { "path": "src/ggml-cpu/traits.cpp", "content": "#include \"traits.h\"\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n\nnamespace ggml::cpu {\ntensor_traits::~tensor_traits() {}\n\nextra_buffer_type::~extra_buffer_type() {}\n} // namespace ggml::cpu\n\nbool ggml_cpu_extra_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) {\n for (auto extra : ggml_backend_cpu_get_extra_buffer_types()) {\n if (extra && extra->context) {\n auto buf_extra = (ggml::cpu::extra_buffer_type *) extra->context;\n auto tensor_traits = buf_extra->get_tensor_traits(op);\n if (tensor_traits && tensor_traits->compute_forward(params, op)) {\n return true;\n }\n }\n }\n return false;\n}\n\nbool ggml_cpu_extra_work_size(int n_threads, const struct ggml_tensor * op, size_t * size) {\n for (auto extra : ggml_backend_cpu_get_extra_buffer_types()) {\n if (extra && extra->context) {\n auto buf_extra = (ggml::cpu::extra_buffer_type *) extra->context;\n auto tensor_traits = buf_extra->get_tensor_traits(op);\n if (tensor_traits && tensor_traits->work_size(n_threads, op, *size)) {\n return true;\n }\n }\n }\n return false;\n}\n" }, { "path": "src/ggml-cpu/traits.h", "content": "#pragma once\n#include \"ggml-backend-impl.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"ggml.h\"\n\n#ifdef __cplusplus\n# include \nextern \"C\" {\n#endif\n\n// return true if op part of extra \"accelerator\"\nbool ggml_cpu_extra_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op);\nbool ggml_cpu_extra_work_size(int n_threads, const struct ggml_tensor * op, size_t * size);\n\n#ifdef __cplusplus\n}\n\nnamespace ggml::cpu {\n// register in tensor->extra\nclass tensor_traits {\n public:\n virtual ~tensor_traits();\n virtual bool work_size(int n_threads, const struct ggml_tensor * op, size_t & size) = 0;\n virtual bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) = 0;\n};\n\nclass extra_buffer_type {\n public:\n virtual ~extra_buffer_type();\n virtual bool supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) = 0;\n virtual tensor_traits * get_tensor_traits(const struct ggml_tensor * op) = 0;\n};\n} // namespace ggml::cpu\n\n// implemented in ggml-cpu.cpp.\nstd::vector & ggml_backend_cpu_get_extra_buffer_types();\n\n#endif\n" }, { "path": "src/ggml-cpu/unary-ops.cpp", "content": "#include \"unary-ops.h\"\n\nstatic inline float op_abs(float x) {\n return fabsf(x);\n}\n\nstatic inline float op_sgn(float x) {\n return (x > 0.f) ? 1.f : ((x < 0.f) ? -1.f : 0.f);\n}\n\nstatic inline float op_neg(float x) {\n return -x;\n}\n\nstatic inline float op_step(float x) {\n return (x > 0.f) ? 1.f : 0.f;\n}\n\nstatic inline float op_tanh(float x) {\n return tanhf(x);\n}\n\nstatic inline float op_elu(float x) {\n return (x > 0.f) ? x : expm1f(x);\n}\n\nstatic inline float op_relu(float x) {\n return (x > 0.f) ? x : 0.f;\n}\n\nstatic inline float op_sigmoid(float x) {\n return 1.f / (1.f + expf(-x));\n}\n\nstatic inline float op_hardsigmoid(float x) {\n return fminf(1.0f, fmaxf(0.0f, (x + 3.0f) / 6.0f));\n}\n\nstatic inline float op_exp(float x) {\n return expf(x);\n}\n\nstatic inline float op_hardswish(float x) {\n return x * fminf(1.0f, fmaxf(0.0f, (x + 3.0f) / 6.0f));\n}\n\nstatic inline float op_sqr(float x) {\n return x * x;\n}\n\nstatic inline float op_sqrt(float x) {\n return sqrtf(x);\n}\n\nstatic inline float op_xielu(float x, float alpha_n, float alpha_p, float beta, float eps) {\n if (x > 0.0f) {\n return alpha_p * x * x + beta * x;\n } else {\n const float min_x_eps = fminf(x, eps);\n return (expm1f(min_x_eps) - x) * alpha_n + beta * x;\n }\n}\n\nstatic inline float op_sin(float x) {\n return sinf(x);\n}\n\nstatic inline float op_cos(float x) {\n return cosf(x);\n}\n\nstatic inline float op_log(float x) {\n return logf(x);\n}\n\nstatic inline float op_expm1(float x) {\n return expf(x) - 1.0f;\n}\n\nstatic inline float op_softplus(float x) {\n return (x > 20.0f) ? x : logf(1.0f + expf(x));\n}\n\nstatic inline float op_floor(float x) {\n return floorf(x);\n}\n\nstatic inline float op_ceil(float x) {\n return ceilf(x);\n}\n\nstatic inline float op_round(float x) {\n return roundf(x);\n}\n\nstatic inline float op_trunc(float x) {\n return truncf(x);\n}\n\ntemplate \nstatic inline void vec_unary_op(int64_t n, dst_t * y, const src0_t * x) {\n constexpr auto src0_to_f32 = type_conversion_table::to_f32;\n constexpr auto f32_to_dst = type_conversion_table::from_f32;\n\n for (int i = 0; i < n; i++) {\n y[i] = f32_to_dst(op(src0_to_f32(x[i])));\n }\n}\n\ntemplate \nstatic void apply_unary_op(const ggml_compute_params * params, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst) && ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT( nb0 == sizeof(dst_t));\n GGML_ASSERT(nb00 == sizeof(src0_t));\n\n const auto [ir0, ir1] = get_thread_range(params, src0);\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir/(ne02*ne01);\n const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n dst_t * dst_ptr = (dst_t *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );\n const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);\n\n vec_unary_op(ne0, dst_ptr, src0_ptr);\n }\n}\n\n// TODO: Use the 'traits' lookup table (for type conversion fns), instead of a mass of 'if' conditions with long templates\ntemplate \nstatic void unary_op(const ggml_compute_params * params, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n /* */ if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { // all f32\n apply_unary_op(params, dst);\n } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { // all f16\n apply_unary_op(params, dst);\n } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16\n apply_unary_op(params, dst);\n } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_F32) {\n apply_unary_op(params, dst);\n } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {\n apply_unary_op(params, dst);\n } else {\n fprintf(stderr, \"%s: unsupported types: dst: %s, src0: %s\\n\", __func__,\n ggml_type_name(dst->type), ggml_type_name(src0->type));\n GGML_ABORT(\"fatal error\");\n }\n}\n\ntemplate \nstatic void unary_op_params(const ggml_compute_params * params, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n /* */ if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { // all f32\n apply_unary_op(params, dst);\n } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { // all f16\n apply_unary_op(params, dst);\n } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16\n apply_unary_op(params, dst);\n } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_F32) {\n apply_unary_op(params, dst);\n } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {\n apply_unary_op(params, dst);\n } else {\n fprintf(stderr, \"%s: unsupported types: dst: %s, src0: %s\\n\", __func__,\n ggml_type_name(dst->type), ggml_type_name(src0->type));\n GGML_ABORT(\"fatal error\");\n }\n}\n\n// Extend vec_unary_op to support functors\ntemplate \nstatic inline void vec_unary_op_functor(int64_t n, dst_t * y, const src0_t * x, Op op) {\n constexpr auto src0_to_f32 = type_conversion_table::to_f32;\n constexpr auto f32_to_dst = type_conversion_table::from_f32;\n\n for (int i = 0; i < n; i++) {\n y[i] = f32_to_dst(op(src0_to_f32(x[i])));\n }\n}\n\n// Extend apply_unary_op to support functors\ntemplate \nstatic void apply_unary_op_functor(const ggml_compute_params * params, ggml_tensor * dst, Op op) {\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(ggml_is_contiguous_1(src0) && ggml_is_contiguous_1(dst) && ggml_are_same_shape(src0, dst));\n\n GGML_TENSOR_UNARY_OP_LOCALS\n\n GGML_ASSERT( nb0 == sizeof(dst_t));\n GGML_ASSERT(nb00 == sizeof(src0_t));\n\n const auto [ir0, ir1] = get_thread_range(params, src0);\n\n for (int64_t ir = ir0; ir < ir1; ++ir) {\n const int64_t i03 = ir/(ne02*ne01);\n const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n dst_t * dst_ptr = (dst_t *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );\n const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);\n\n vec_unary_op_functor(ne0, dst_ptr, src0_ptr, op);\n }\n}\n\n// Generic dispatcher for functors\ntemplate \nstatic void unary_op_functor(const ggml_compute_params * params, ggml_tensor * dst, Op op) {\n const ggml_tensor * src0 = dst->src[0];\n\n /* */ if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { // all f32\n apply_unary_op_functor(params, dst, op);\n } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { // all f16\n apply_unary_op_functor(params, dst, op);\n } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16\n apply_unary_op_functor(params, dst, op);\n } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_F32) {\n apply_unary_op_functor(params, dst, op);\n } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {\n apply_unary_op_functor(params, dst, op);\n } else {\n fprintf(stderr, \"%s: unsupported types: dst: %s, src0: %s\\n\", __func__,\n ggml_type_name(dst->type), ggml_type_name(src0->type));\n GGML_ABORT(\"fatal error\");\n }\n}\n\nvoid ggml_compute_forward_abs(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_sgn(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_neg(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_step(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_tanh(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_elu(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_relu(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_sigmoid(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_hardsigmoid(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_exp(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_hardswish(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_sqr(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_sqrt(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_sin(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_cos(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_log(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_expm1(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_softplus(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_floor(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_ceil(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_round(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_trunc(const ggml_compute_params * params, ggml_tensor * dst) {\n unary_op(params, dst);\n}\n\nvoid ggml_compute_forward_xielu(const ggml_compute_params * params, ggml_tensor * dst) {\n const float alpha_n = ggml_get_op_params_f32(dst, 1);\n const float alpha_p = ggml_get_op_params_f32(dst, 2);\n const float beta = ggml_get_op_params_f32(dst, 3);\n const float eps = ggml_get_op_params_f32(dst, 4);\n\n const auto xielu_op_params = [alpha_n, alpha_p, beta, eps](float f) {\n return op_xielu(f, alpha_n, alpha_p, beta, eps);\n };\n\n unary_op_functor(params, dst, xielu_op_params);\n}\n\n" }, { "path": "src/ggml-cpu/unary-ops.h", "content": "#pragma once\n\n#include \"common.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nvoid ggml_compute_forward_abs(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sgn(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_neg(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_step(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_tanh(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_elu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sigmoid(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_hardsigmoid(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_exp(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_hardswish(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sqr(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sqrt(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sin(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cos(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_log(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_expm1(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_softplus(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_floor(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_ceil(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_round(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_trunc(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_xielu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cpu/vec.cpp", "content": "#include \"vec.h\"\n\n#include \n\n// precomputed gelu table for f16 (128 KB)\nggml_fp16_t ggml_table_gelu_f16[1 << 16];\n\n// precomputed quick gelu table for f16 (128 KB)\nggml_fp16_t ggml_table_gelu_quick_f16[1 << 16];\n\nvoid ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) {\n assert(nrc == 1);\n GGML_UNUSED(nrc);\n GGML_UNUSED(bx);\n GGML_UNUSED(by);\n GGML_UNUSED(bs);\n\n#if defined(GGML_SIMD)\n float sumf = 0.0f;\n\n #if defined(__ARM_FEATURE_SVE)\n const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;\n const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16\n const int ggml_f32_step = 8 * ggml_f32_epr; // choose 8 SVE registers\n\n const int np = (n & ~(ggml_f32_step - 1));\n svfloat32_t sum1 = svdup_n_f32(0.0f);\n svfloat32_t sum2 = svdup_n_f32(0.0f);\n svfloat32_t sum3 = svdup_n_f32(0.0f);\n svfloat32_t sum4 = svdup_n_f32(0.0f);\n svfloat32_t sum5 = svdup_n_f32(0.0f);\n svfloat32_t sum6 = svdup_n_f32(0.0f);\n svfloat32_t sum7 = svdup_n_f32(0.0f);\n svfloat32_t sum8 = svdup_n_f32(0.0f);\n svfloat32_t ax1,ax2,ax3,ax4,ax5,ax6,ax7,ax8;\n svfloat32_t ay1,ay2,ay3,ay4,ay5,ay6,ay7,ay8;\n for (int i = 0; i < np; i += ggml_f32_step) {\n ax1 = GGML_F32_VEC_LOAD(x + i);\n ay1 = GGML_F32_VEC_LOAD(y + i);\n sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1);\n\n ax2 = GGML_F32_VEC_LOAD(x + i + 1*ggml_f32_epr);\n ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);\n sum2 = GGML_F32_VEC_FMA(sum2, ax2, ay2);\n\n ax3 = GGML_F32_VEC_LOAD(x + i + 2*ggml_f32_epr);\n ay3 = GGML_F32_VEC_LOAD(y + i + 2*ggml_f32_epr);\n sum3 = GGML_F32_VEC_FMA(sum3, ax3, ay3);\n\n ax4 = GGML_F32_VEC_LOAD(x + i + 3*ggml_f32_epr);\n ay4 = GGML_F32_VEC_LOAD(y + i + 3*ggml_f32_epr);\n sum4 = GGML_F32_VEC_FMA(sum4, ax4, ay4);\n\n ax5 = GGML_F32_VEC_LOAD(x + i + 4*ggml_f32_epr);\n ay5 = GGML_F32_VEC_LOAD(y + i + 4*ggml_f32_epr);\n sum5 = GGML_F32_VEC_FMA(sum5, ax5, ay5);\n\n ax6 = GGML_F32_VEC_LOAD(x + i + 5*ggml_f32_epr);\n ay6 = GGML_F32_VEC_LOAD(y + i + 5*ggml_f32_epr);\n sum6 = GGML_F32_VEC_FMA(sum6, ax6, ay6);\n\n ax7 = GGML_F32_VEC_LOAD(x + i + 6*ggml_f32_epr);\n ay7 = GGML_F32_VEC_LOAD(y + i + 6*ggml_f32_epr);\n sum7 = GGML_F32_VEC_FMA(sum7, ax7, ay7);\n\n ax8 = GGML_F32_VEC_LOAD(x + i + 7*ggml_f32_epr);\n ay8 = GGML_F32_VEC_LOAD(y + i + 7*ggml_f32_epr);\n sum8 = GGML_F32_VEC_FMA(sum8, ax8, ay8);\n }\n // leftovers\n // Since 8 unrolls are done in above loop, leftovers lie in range [0, ggml_f32_step] which is handled in below loop\n const int np2 = (n & ~(ggml_f32_epr - 1));\n for (int i = np; i < np2; i += ggml_f32_epr) {\n ax1 = GGML_F32_VEC_LOAD(x + i);\n ay1 = GGML_F32_VEC_LOAD(y + i);\n sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1);\n }\n // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only\n if (np2 < n) {\n svbool_t pg = svwhilelt_b32(np2, n);\n ax1 = svld1_f32(pg, x + np2);\n ay1 = svld1_f32(pg, y + np2);\n sum1 = svmad_f32_m(pg, ax1, ay1, sum1);\n }\n // reduce sum1,sum2 to sum1\n GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8);\n #elif defined(__riscv_v_intrinsic)\n int vl = __riscv_vsetvlmax_e32m8();\n vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);\n vfloat32m8_t vsum;\n vfloat32m8_t ax;\n vfloat32m8_t ay;\n vsum = __riscv_vfmv_v_f_f32m8_tu(vsum, 0.0f, vl);\n for (int i = 0; i < n; i += vl) {\n vl = __riscv_vsetvl_e32m8(n - i);\n ax = __riscv_vle32_v_f32m8_tu(ax, &x[i], vl);\n ay = __riscv_vle32_v_f32m8_tu(ay, &y[i], vl);\n vsum = __riscv_vfmacc_vv_f32m8_tu(vsum, ax, ay, vl);\n }\n vl = __riscv_vsetvlmax_e32m8();\n vs = __riscv_vfredusum_vs_f32m8_f32m1(vsum, vs, vl);\n sumf += __riscv_vfmv_f_s_f32m1_f32(vs);\n #else\n const int np = (n & ~(GGML_F32_STEP - 1));\n\n GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };\n\n GGML_F32_VEC ax[GGML_F32_ARR];\n GGML_F32_VEC ay[GGML_F32_ARR];\n\n for (int i = 0; i < np; i += GGML_F32_STEP) {\n for (int j = 0; j < GGML_F32_ARR; j++) {\n ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);\n ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);\n\n sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);\n }\n }\n\n // reduce sum0..sum3 to sum0\n GGML_F32_VEC_REDUCE(sumf, sum);\n\n // leftovers\n for (int i = np; i < n; ++i) {\n sumf += x[i]*y[i];\n }\n #endif\n#else\n // scalar\n ggml_float sumf = 0.0;\n for (int i = 0; i < n; ++i) {\n sumf += (ggml_float)(x[i]*y[i]);\n }\n#endif\n\n *s = sumf;\n}\n\nvoid ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc) {\n assert(nrc == 1);\n GGML_UNUSED(nrc);\n GGML_UNUSED(bx);\n GGML_UNUSED(by);\n GGML_UNUSED(bs);\n int i = 0;\n ggml_float sumf = 0;\n\n#if defined(__AVX512BF16__)\n __m512 c1 = _mm512_setzero_ps();\n __m512 c2 = _mm512_setzero_ps();\n for (; i + 64 <= n; i += 64) {\n c1 = _mm512_dpbf16_ps(c1, m512bh(_mm512_loadu_si512((x + i))),\n m512bh(_mm512_loadu_si512((y + i))));\n c2 = _mm512_dpbf16_ps(c2, m512bh(_mm512_loadu_si512((x + i + 32))),\n m512bh(_mm512_loadu_si512((y + i + 32))));\n }\n sumf += (ggml_float)_mm512_reduce_add_ps(c1);\n sumf += (ggml_float)_mm512_reduce_add_ps(c2);\n\n#elif defined(__AVX512F__)\n#define LOAD(p) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)(p))), 16))\n __m512 c1 = _mm512_setzero_ps();\n __m512 c2 = _mm512_setzero_ps();\n for (; i + 32 <= n; i += 32) {\n c1 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i), LOAD(y + i)), c1);\n c2 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c2);\n }\n sumf += (ggml_float)_mm512_reduce_add_ps(c1);\n sumf += (ggml_float)_mm512_reduce_add_ps(c2);\n\n#undef LOAD\n#elif defined(__AVX2__) || defined(__AVX__)\n#if defined(__AVX2__)\n#define LOAD(p) _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16))\n#else\n#define LOAD(p) _mm256_castsi256_ps(_mm256_insertf128_si256(_mm256_castsi128_si256(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16)), (_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_bsrli_si128(_mm_loadu_si128((const __m128i *)(p)), 8)), 16)), 1))\n#endif\n __m256 c1 = _mm256_setzero_ps();\n __m256 c2 = _mm256_setzero_ps();\n __m256 c3 = _mm256_setzero_ps();\n __m256 c4 = _mm256_setzero_ps();\n for (; i + 32 <= n; i += 32) {\n c1 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i), LOAD(y + i)), c1);\n c2 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 8), LOAD(y + i + 8)), c2);\n c3 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c3);\n c4 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 24), LOAD(y + i + 24)), c4);\n }\n __m128 g;\n c1 = _mm256_add_ps(_mm256_add_ps(c1, c3),\n _mm256_add_ps(c2, c4));\n g = _mm_add_ps(_mm256_extractf128_ps(c1, 1),\n _mm256_castps256_ps128(c1));\n g = _mm_add_ps(g, _mm_movehl_ps(g, g));\n g = _mm_add_ss(g, _mm_movehdup_ps(g));\n sumf += (ggml_float)_mm_cvtss_f32(g);\n\n#undef LOAD\n#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfwma)\n size_t vl = __riscv_vsetvlmax_e32m4();\n\n // initialize accumulators to all zeroes\n vfloat32m4_t vsum0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n vfloat32m4_t vsum1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n\n // calculate step size\n const size_t epr = __riscv_vsetvlmax_e16m2();\n const size_t step = epr * 2;\n const int np = (n & ~(step - 1));\n\n // unroll by 2\n for (; i < np; i += step) {\n vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i], epr);\n vbfloat16m2_t ay0 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i], epr);\n vsum0 = __riscv_vfwmaccbf16_vv_f32m4(vsum0, ax0, ay0, epr);\n __asm__ __volatile__ (\"\" ::: \"memory\");\n\n vbfloat16m2_t ax1 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i + epr], epr);\n vbfloat16m2_t ay1 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i + epr], epr);\n vsum1 = __riscv_vfwmaccbf16_vv_f32m4(vsum1, ax1, ay1, epr);\n __asm__ __volatile__ (\"\" ::: \"memory\");\n }\n\n // accumulate in 1 register\n vsum0 = __riscv_vfadd_vv_f32m4(vsum0, vsum1, vl);\n\n // leftovers\n for (i = np; i < n; i += vl) {\n vl = __riscv_vsetvl_e16m2(n - i);\n vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i], vl);\n vbfloat16m2_t ay0 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i], vl);\n vsum0 = __riscv_vfwmaccbf16_vv_f32m4(vsum0, ax0, ay0, vl);\n }\n\n // reduce\n vl = __riscv_vsetvlmax_e32m4();\n vfloat32m1_t redsum = __riscv_vfredusum_vs_f32m4_f32m1(vsum0, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);\n sumf += __riscv_vfmv_f_s_f32m1_f32(redsum);\n\n#elif defined(__POWER9_VECTOR__) || defined(__VXE__) || defined(__VXE2__)\n const int np = (n & ~(GGML_BF16_STEP - 1));\n if (np > 0) {\n GGML_F32_VEC sum[4] = {GGML_F32_VEC_ZERO};\n for (; i < np; i += GGML_BF16_STEP) {\n GGML_BF16_VEC vx0 = GGML_BF16_VEC_LOAD(x + i);\n GGML_BF16_VEC vx1 = GGML_BF16_VEC_LOAD(x + i + 8);\n GGML_BF16_VEC vy0 = GGML_BF16_VEC_LOAD(y + i);\n GGML_BF16_VEC vy1 = GGML_BF16_VEC_LOAD(y + i + 8);\n GGML_BF16_FMA_LO(sum[0], vx0, vy0);\n GGML_BF16_FMA_HI(sum[1], vx0, vy0);\n GGML_BF16_FMA_LO(sum[2], vx1, vy1);\n GGML_BF16_FMA_HI(sum[3], vx1, vy1);\n }\n GGML_F32x4_REDUCE_4(sumf, sum[0], sum[1], sum[2], sum[3]);\n }\n#endif\n\n for (; i < n; ++i) {\n sumf += (ggml_float)(GGML_BF16_TO_FP32(x[i]) *\n GGML_BF16_TO_FP32(y[i]));\n }\n *s = sumf;\n}\n\nvoid ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc) {\n assert(nrc == 1);\n GGML_UNUSED(nrc);\n GGML_UNUSED(bx);\n GGML_UNUSED(by);\n GGML_UNUSED(bs);\n\n ggml_float sumf = 0.0;\n\n\n#if defined(GGML_SIMD)\n #if defined(__ARM_FEATURE_SVE)\n const int sve_register_length = svcntb() * 8; //get vector length\n const int ggml_f16_epr = sve_register_length / 16; // running when 16\n const int ggml_f16_step = 8 * ggml_f16_epr; // choose 8 SVE registers\n\n const int np= (n & ~(ggml_f16_step - 1));\n svfloat16_t sum1 = svdup_n_f16(0.0f);\n svfloat16_t sum2 = svdup_n_f16(0.0f);\n svfloat16_t sum3 = svdup_n_f16(0.0f);\n svfloat16_t sum4 = svdup_n_f16(0.0f);\n\n svfloat16_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;\n svfloat16_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;\n for (int i = 0; i < np; i += ggml_f16_step) {\n ax1 = GGML_F16x_VEC_LOAD(x + i + 0 * ggml_f16_epr, 0);\n ay1 = GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0);\n sum1 = GGML_F16x_VEC_FMA(sum1, ax1, ay1);\n\n ax2 = GGML_F16x_VEC_LOAD(x + i + 1 * ggml_f16_epr, 1);\n ay2 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1);\n sum2 = GGML_F16x_VEC_FMA(sum2, ax2, ay2);\n\n ax3 = GGML_F16x_VEC_LOAD(x + i + 2 * ggml_f16_epr, 2);\n ay3 = GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2);\n sum3 = GGML_F16x_VEC_FMA(sum3, ax3, ay3);\n\n ax4 = GGML_F16x_VEC_LOAD(x + i + 3 * ggml_f16_epr, 3);\n ay4 = GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3);\n sum4 = GGML_F16x_VEC_FMA(sum4, ax4, ay4);\n\n ax5 = GGML_F16x_VEC_LOAD(x + i + 4 * ggml_f16_epr, 4);\n ay5 = GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4);\n sum1 = GGML_F16x_VEC_FMA(sum1, ax5, ay5);\n\n ax6 = GGML_F16x_VEC_LOAD(x + i + 5 * ggml_f16_epr, 5);\n ay6 = GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5);\n sum2 = GGML_F16x_VEC_FMA(sum2, ax6, ay6);\n\n ax7 = GGML_F16x_VEC_LOAD(x + i + 6 * ggml_f16_epr, 6);\n ay7 = GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6);\n sum3 = GGML_F16x_VEC_FMA(sum3, ax7, ay7);\n\n ax8 = GGML_F16x_VEC_LOAD(x + i + 7 * ggml_f16_epr, 7);\n ay8 = GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7);\n sum4 = GGML_F16x_VEC_FMA(sum4, ax8, ay8);\n }\n\n const int np2 = (n & ~(ggml_f16_epr - 1)); // round down to multiple of 8\n for (int k = np; k < np2; k += ggml_f16_epr) {\n svfloat16_t rx = GGML_F16x_VEC_LOAD(x + k, 0);\n svfloat16_t ry = GGML_F16x_VEC_LOAD(y + k, 0);\n sum1 = GGML_F16x_VEC_FMA(sum1, rx, ry);\n }\n\n if (np2 < n) {\n svbool_t pg = svwhilelt_b16(np2, n);\n svfloat16_t hx = svld1_f16(pg, (const __fp16 *)(x + np2));\n svfloat16_t hy = svld1_f16(pg, (const __fp16 *)(y + np2));\n\n sum1 = svmad_f16_x(pg, hx, hy, sum1);\n }\n GGML_F16x_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4);\n #elif defined(__riscv_v_intrinsic)\n #if defined(__riscv_zvfh)\n int vl = __riscv_vsetvlmax_e32m2();\n vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);\n vfloat32m2_t vsum;\n vfloat16m1_t ax;\n vfloat16m1_t ay;\n vsum = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vmv_v_x_u32m2(0, vl));\n for (int i = 0; i < n; i += vl) {\n vl = __riscv_vsetvl_e16m1(n - i);\n ax = __riscv_vle16_v_f16m1_tu(ax, (const _Float16 *)&x[i], vl);\n ay = __riscv_vle16_v_f16m1_tu(ay, (const _Float16 *)&y[i], vl);\n vsum = __riscv_vfwmacc_vv_f32m2_tu(vsum, ax, ay, vl);\n }\n vl = __riscv_vsetvlmax_e32m1();\n vfloat32m1_t ac0 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(vsum, 0), __riscv_vget_v_f32m2_f32m1(vsum, 1), vl);\n vs = __riscv_vfredusum_vs_f32m1_f32m1(ac0, vs, vl);\n sumf += __riscv_vfmv_f_s_f32m1_f32(vs);\n #else\n for (int i = 0; i < n; ++i) {\n sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));\n }\n #endif // __riscv_zvfh\n #else\n const int np = (n & ~(GGML_F16_STEP - 1));\n\n GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };\n\n GGML_F16_VEC ax[GGML_F16_ARR];\n GGML_F16_VEC ay[GGML_F16_ARR];\n\n for (int i = 0; i < np; i += GGML_F16_STEP) {\n for (int j = 0; j < GGML_F16_ARR; j++) {\n ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);\n ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);\n\n sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);\n }\n }\n\n // reduce sum0..sum3 to sum0\n GGML_F16_VEC_REDUCE(sumf, sum);\n\n // leftovers\n for (int i = np; i < n; ++i) {\n sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));\n }\n // if you hit this, you are likely running outside the FP range\n assert(!isnan(sumf) && !isinf(sumf));\n #endif\n#else\n for (int i = 0; i < n; ++i) {\n sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));\n }\n#endif // GGML_SIMD\n\n *s = sumf;\n}\n\nvoid ggml_vec_silu_f32(const int n, float * y, const float * x) {\n int i = 0;\n#if defined(__AVX512F__) && defined(__AVX512DQ__)\n for (; i + 15 < n; i += 16) {\n _mm512_storeu_ps(y + i, ggml_v_silu(_mm512_loadu_ps(x + i)));\n }\n#elif defined(__AVX2__) && defined(__FMA__)\n for (; i + 7 < n; i += 8) {\n _mm256_storeu_ps(y + i, ggml_v_silu(_mm256_loadu_ps(x + i)));\n }\n#elif defined(__SSE2__)\n for (; i + 3 < n; i += 4) {\n _mm_storeu_ps(y + i, ggml_v_silu(_mm_loadu_ps(x + i)));\n }\n#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n const int vlen = svcntw();\n for (; i < n; i += vlen) {\n const svbool_t pg = svwhilelt_b32_s32(i, n);\n svst1_f32(pg, y + i, ggml_v_silu(pg, svld1_f32(pg, x + i)));\n }\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n for (; i + 3 < n; i += 4) {\n vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i)));\n }\n#elif defined(__riscv_v_intrinsic)\n for (int vl; i < n; i += vl) {\n vl = __riscv_vsetvl_e32m2(n - i);\n vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);\n vfloat32m2_t vy = ggml_v_silu_m2(vx, vl);\n __riscv_vse32_v_f32m2(&y[i], vy, vl);\n }\n#endif\n for (; i < n; ++i) {\n y[i] = ggml_silu_f32(x[i]);\n }\n}\n\nvoid ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * g) {\n int i = 0;\n#if defined(__AVX512F__) && defined(__AVX512DQ__)\n for (; i + 15 < n; i += 16) {\n _mm512_storeu_ps(y + i, _mm512_mul_ps(ggml_v_silu(_mm512_loadu_ps(x + i)), _mm512_loadu_ps(g + i)));\n }\n#elif defined(__AVX2__) && defined(__FMA__)\n for (; i + 7 < n; i += 8) {\n _mm256_storeu_ps(y + i, _mm256_mul_ps(ggml_v_silu(_mm256_loadu_ps(x + i)), _mm256_loadu_ps(g + i)));\n }\n#elif defined(__SSE2__)\n for (; i + 3 < n; i += 4) {\n _mm_storeu_ps(y + i, _mm_mul_ps(ggml_v_silu(_mm_loadu_ps(x + i)), _mm_loadu_ps(g + i)));\n }\n#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n const int vlen = svcntw();\n for (; i < n; i += vlen) {\n const svbool_t pg = svwhilelt_b32_s32(i, n);\n svst1_f32(pg, y + i, svmul_f32_x(pg, ggml_v_silu(pg, svld1_f32(pg, x + i)), svld1_f32(pg, g + i)));\n }\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n for (; i + 3 < n; i += 4) {\n vst1q_f32(y + i, vmulq_f32(ggml_v_silu(vld1q_f32(x + i)), vld1q_f32(g + i)));\n }\n#elif defined(__riscv_v_intrinsic)\n for (int vl; i < n; i += vl) {\n vl = __riscv_vsetvl_e32m2(n - i);\n vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);\n vfloat32m2_t vg = __riscv_vle32_v_f32m2(&g[i], vl);\n vfloat32m2_t vy = __riscv_vfmul_vv_f32m2(ggml_v_silu_m2(vx, vl), vg, vl);\n __riscv_vse32_v_f32m2(&y[i], vy, vl);\n }\n#endif\n for (; i < n; ++i) {\n y[i] = ggml_silu_f32(x[i]) * g[i];\n }\n}\n\nggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean) {\n int i = 0;\n ggml_float sum = 0;\n// TODO: optimize to process the remaining elements in groups using the smaller vector sizes from AVX2 and SSE\n// ref: https://github.com/ggml-org/llama.cpp/pull/15953#pullrequestreview-3310928344\n#if defined(__AVX512F__) && defined(__AVX512DQ__)\n for (; i + 15 < n; i += 16) {\n __m512 val = _mm512_sub_ps(_mm512_loadu_ps(x + i),\n _mm512_set1_ps(mean));\n _mm512_storeu_ps(y + i, val);\n sum += (ggml_float)_mm512_reduce_add_ps(_mm512_mul_ps(val, val));\n }\n#elif defined(__AVX2__) && defined(__FMA__)\n for (; i + 7 < n; i += 8) {\n __m256 val = _mm256_sub_ps(_mm256_loadu_ps(x + i),\n _mm256_set1_ps(mean));\n _mm256_storeu_ps(y + i, val);\n val = _mm256_mul_ps(val,val);\n __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),\n _mm256_castps256_ps128(val));\n val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));\n val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));\n sum += (ggml_float)_mm_cvtss_f32(val2);\n }\n#elif defined(__SSE2__)\n for (; i + 3 < n; i += 4) {\n __m128 val = _mm_sub_ps(_mm_loadu_ps(x + i),\n _mm_set1_ps(mean));\n _mm_storeu_ps(y + i, val);\n val = _mm_mul_ps(val, val);\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\n val = _mm_add_ps(val, _mm_movehl_ps(val, val));\n val = _mm_add_ss(val, _mm_movehdup_ps(val));\n#else\n __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));\n val = _mm_add_ps(val, tmp);\n tmp = _mm_movehl_ps(tmp, val);\n val = _mm_add_ss(val, tmp);\n#endif // __AVX__ || __AVX2__ || __AVX512F__\n sum += (ggml_float)_mm_cvtss_f32(val);\n }\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n for (; i + 3 < n; i += 4) {\n float32x4_t val = vsubq_f32(vld1q_f32(x + i),\n vdupq_n_f32(mean));\n vst1q_f32(y + i, val);\n val = vmulq_f32(val, val);\n sum += (ggml_float)vaddvq_f32(val);\n }\n#elif defined(__VXE__) || defined(__VXE2__)\n for (; i + 3 < n; i += 4) {\n float32x4_t val = vec_sub(vec_xl(0, x + i), vec_splats(mean));\n vec_xst(val, 0, y + i);\n val = vec_mul(val, val);\n sum += (ggml_float)vec_hsum_f32x4(val);\n }\n#elif defined(__riscv_v_intrinsic)\n vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);\n for (int vl; i < n; i += vl) {\n vl = __riscv_vsetvl_e32m2(n - i);\n vfloat32m2_t val = __riscv_vfsub_vf_f32m2(__riscv_vle32_v_f32m2(&x[i], vl), mean, vl);\n __riscv_vse32_v_f32m2(&y[i], val, vl);\n val = __riscv_vfmul_vv_f32m2(val, val, vl);\n vsum = __riscv_vfwredusum_vs_f32m2_f64m1(val, vsum, vl);\n }\n sum = (ggml_float)__riscv_vfmv_f_s_f64m1_f64(vsum);\n#endif\n for (; i < n; ++i) {\n float val = x[i] - mean;\n y[i] = val;\n val *= val;\n sum += (ggml_float)val;\n }\n return sum/n;\n}\n\nggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {\n int i = 0;\n ggml_float sum = 0;\n#if defined(__AVX512F__) && defined(__AVX512DQ__)\n for (; i + 15 < n; i += 16) {\n __m512 val = ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i),\n _mm512_set1_ps(max)));\n _mm512_storeu_ps(y + i, val);\n sum += (ggml_float)_mm512_reduce_add_ps(val);\n }\n#elif defined(__AVX2__) && defined(__FMA__)\n for (; i + 7 < n; i += 8) {\n __m256 val = ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i),\n _mm256_set1_ps(max)));\n _mm256_storeu_ps(y + i, val);\n __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),\n _mm256_castps256_ps128(val));\n val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));\n val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));\n sum += (ggml_float)_mm_cvtss_f32(val2);\n }\n#elif defined(__SSE2__)\n for (; i + 3 < n; i += 4) {\n __m128 val = ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i),\n _mm_set1_ps(max)));\n _mm_storeu_ps(y + i, val);\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\n val = _mm_add_ps(val, _mm_movehl_ps(val, val));\n val = _mm_add_ss(val, _mm_movehdup_ps(val));\n#else\n __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));\n val = _mm_add_ps(val, tmp);\n tmp = _mm_movehl_ps(tmp, val);\n val = _mm_add_ss(val, tmp);\n#endif\n sum += (ggml_float)_mm_cvtss_f32(val);\n }\n#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n const int vlen = svcntw();\n for (; i < n; i += vlen) {\n const svbool_t pg = svwhilelt_b32_s32(i, n);\n svfloat32_t val = ggml_v_expf(pg, svsub_f32_x(pg, svld1_f32(pg, x + i),\n svdup_n_f32_x(pg, max)));\n svst1_f32(pg, y + i, val);\n sum += (ggml_float)svaddv_f32(pg, val);\n }\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n for (; i + 3 < n; i += 4) {\n float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i),\n vdupq_n_f32(max)));\n vst1q_f32(y + i, val);\n sum += (ggml_float)vaddvq_f32(val);\n }\n#elif defined(__riscv_v_intrinsic)\n vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);\n for (int avl; i < n; i += avl) {\n avl = __riscv_vsetvl_e32m2(n - i);\n vfloat32m2_t val = ggml_v_expf_m2(__riscv_vfsub_vf_f32m2(__riscv_vle32_v_f32m2(&x[i], avl), max, avl), avl);\n __riscv_vse32_v_f32m2(&y[i], val, avl);\n vsum = __riscv_vfwredusum_vs_f32m2_f64m1(val, vsum, avl);\n }\n return (ggml_float)__riscv_vfmv_f_s_f64m1_f64(vsum);\n#endif\n for (; i < n; ++i) {\n float val = expf(x[i] - max);\n sum += (ggml_float)val;\n y[i] = val;\n }\n return sum;\n}\n\nggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max) {\n // log(soft_max) = log(soft_max_i / soft_max_sum) = log(soft_max_i) - log(soft_max_sum) = (logit_i - max) - log(soft_max_i)\n\n int i = 0;\n ggml_float sum = 0;\n for (; i < n; ++i) {\n float val = x[i] - max;\n y[i] = val;\n sum += (ggml_float)expf(val);\n }\n return sum = (ggml_float)logf(sum);\n}\n" }, { "path": "src/ggml-cpu/vec.h", "content": "// Vectorized functions for fundamental operations\n\n#pragma once\n\n#include \"ggml-impl.h\"\n#include \"simd-mappings.h\"\n#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n\n#if defined(GGML_USE_ACCELERATE)\n#include \n#endif\n\n// floating point type used to accumulate sums\ntypedef double ggml_float;\n\n#define GGML_GELU_FP16\n#define GGML_GELU_QUICK_FP16\n\n#define GGML_SOFT_MAX_UNROLL 4\n#define GGML_VEC_DOT_UNROLL 2\n#define GGML_VEC_MAD_UNROLL 32\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n//\n// global data\n//\n\n// precomputed gelu table for f16 (128 KB)\nextern ggml_fp16_t ggml_table_gelu_f16[1 << 16];\n\n// precomputed quick gelu table for f16 (128 KB)\nextern ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16];\n\n//\n// fundamental operations\n//\n\nvoid ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc);\nvoid ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc);\nvoid ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc);\n\nvoid ggml_vec_silu_f32(const int n, float * y, const float * x);\nggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean); //it will also center y ( y = y - mean )\nggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max);\nggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max);\n\ninline static void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) { for (int i = 0; i < n; ++i) x[i] = v; }\ninline static void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }\n\ninline static void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }\ninline static void ggml_vec_cpy_i32(const int n, int32_t * y, const int32_t * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }\n\ninline static void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const ggml_fp16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }\ninline static void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }\n\ninline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) {\n int i = 0;\n#if defined(__AVX2__)\n for (; i + 7 < n; i += 8) {\n __m256 vx = _mm256_loadu_ps(x + i);\n __m256 vy = _mm256_loadu_ps(y + i);\n __m256 vz = _mm256_add_ps(vx, vy);\n _mm256_storeu_ps(z + i, vz);\n }\n#endif\n for (; i < n; ++i) {\n z[i] = x[i] + y[i];\n }\n}\n\ninline static void ggml_vec_add_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {\n for (int i = 0; i < n; ++i) {\n z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) + GGML_CPU_FP16_TO_FP32(y[i]));\n }\n}\ninline static void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) { for (int i = 0; i < n; ++i) z[i] = x[i] + v; }\ninline static void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }\ninline static void ggml_vec_acc1_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] += v; }\ninline static void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] - y[i]; }\ninline static void ggml_vec_sub_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {\n for (int i = 0; i < n; ++i) {\n z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) - GGML_CPU_FP16_TO_FP32(y[i]));\n }\n}\ninline static void ggml_vec_set_f32 (const int n, float * x, const float v) { for (int i = 0; i < n; ++i) x[i] = v; }\ninline static void ggml_vec_cpy_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }\ninline static void ggml_vec_neg_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = -x[i]; }\ninline static void ggml_vec_neg_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(-GGML_CPU_FP16_TO_FP32(x[i]));\n }\n}\n\ninline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }\ninline static void ggml_vec_mul_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {\n for (int i = 0; i < n; ++i) {\n z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) * GGML_CPU_FP16_TO_FP32(y[i]));\n }\n}\ninline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }\ninline static void ggml_vec_div_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {\n for (int i = 0; i < n; ++i) {\n z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) / GGML_CPU_FP16_TO_FP32(y[i]));\n }\n}\n\n// compute GGML_VEC_DOT_UNROLL dot products at once\n// xs - x row stride in bytes\ninline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GGML_RESTRICT s, void * GGML_RESTRICT xv, ggml_fp16_t * GGML_RESTRICT y) {\n ggml_float sumf[GGML_VEC_DOT_UNROLL] = { 0.0 };\n\n ggml_fp16_t * GGML_RESTRICT x[GGML_VEC_DOT_UNROLL];\n\n for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {\n x[i] = (ggml_fp16_t *) ((char *) xv + i*xs);\n }\n\n#if defined(GGML_SIMD)\n #if defined(__ARM_FEATURE_SVE)\n\n const int sve_register_length = svcntb() * 8;\n const int ggml_f16_epr = sve_register_length / 16; // running when 16\n const int ggml_f16_step = 8 * ggml_f16_epr; // choose 8 SVE registers\n\n const int np = (n & ~(ggml_f16_step - 1));\n\n svfloat16_t sum_00 = svdup_n_f16(0.0f);\n svfloat16_t sum_01 = svdup_n_f16(0.0f);\n svfloat16_t sum_02 = svdup_n_f16(0.0f);\n svfloat16_t sum_03 = svdup_n_f16(0.0f);\n\n svfloat16_t sum_10 = svdup_n_f16(0.0f);\n svfloat16_t sum_11 = svdup_n_f16(0.0f);\n svfloat16_t sum_12 = svdup_n_f16(0.0f);\n svfloat16_t sum_13 = svdup_n_f16(0.0f);\n\n svfloat16_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;\n svfloat16_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;\n\n for (int i = 0; i < np; i += ggml_f16_step) {\n ay1 = GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0); // 8 elements\n\n ax1 = GGML_F16x_VEC_LOAD(x[0] + i + 0*ggml_f16_epr, 0); // 8 elements\n sum_00 = GGML_F16x_VEC_FMA(sum_00, ax1, ay1); // sum_00 = sum_00+ax1*ay1\n ax1 = GGML_F16x_VEC_LOAD(x[1] + i + 0*ggml_f16_epr, 0); // 8 elements\n sum_10 = GGML_F16x_VEC_FMA(sum_10, ax1, ay1);\n\n ay2 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1); // next 8 elements\n\n ax2 = GGML_F16x_VEC_LOAD(x[0] + i + 1*ggml_f16_epr, 1); // next 8 elements\n sum_01 = GGML_F16x_VEC_FMA(sum_01, ax2, ay2);\n ax2 = GGML_F16x_VEC_LOAD(x[1] + i + 1*ggml_f16_epr, 1);\n sum_11 = GGML_F16x_VEC_FMA(sum_11, ax2, ay2);\n\n ay3 = GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2);\n\n ax3 = GGML_F16x_VEC_LOAD(x[0] + i + 2*ggml_f16_epr, 2);\n sum_02 = GGML_F16x_VEC_FMA(sum_02, ax3, ay3);\n ax3 = GGML_F16x_VEC_LOAD(x[1] + i + 2*ggml_f16_epr, 2);\n sum_12 = GGML_F16x_VEC_FMA(sum_12, ax3, ay3);\n\n ay4 = GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3);\n\n ax4 = GGML_F16x_VEC_LOAD(x[0] + i + 3*ggml_f16_epr, 3);\n sum_03 = GGML_F16x_VEC_FMA(sum_03, ax4, ay4);\n ax4 = GGML_F16x_VEC_LOAD(x[1] + i + 3*ggml_f16_epr, 3);\n sum_13 = GGML_F16x_VEC_FMA(sum_13, ax4, ay4);\n\n ay5 = GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4);\n\n ax5 = GGML_F16x_VEC_LOAD(x[0] + i + 4*ggml_f16_epr, 4);\n\n sum_00 = GGML_F16x_VEC_FMA(sum_00, ax5, ay5);\n ax5 = GGML_F16x_VEC_LOAD(x[1] + i + 4*ggml_f16_epr, 4);\n sum_10 = GGML_F16x_VEC_FMA(sum_10, ax5, ay5);\n\n ay6 = GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5);\n\n ax6 = GGML_F16x_VEC_LOAD(x[0] + i + 5*ggml_f16_epr, 5);\n\n sum_01 = GGML_F16x_VEC_FMA(sum_01, ax6, ay6);\n ax6 = GGML_F16x_VEC_LOAD(x[1] + i + 5*ggml_f16_epr, 5);\n sum_11 = GGML_F16x_VEC_FMA(sum_11, ax6, ay6);\n\n ay7 = GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6);\n\n ax7 = GGML_F16x_VEC_LOAD(x[0] + i + 6*ggml_f16_epr, 6);\n\n sum_02 = GGML_F16x_VEC_FMA(sum_02, ax7, ay7);\n ax7 = GGML_F16x_VEC_LOAD(x[1] + i + 6*ggml_f16_epr, 6);\n sum_12 = GGML_F16x_VEC_FMA(sum_12, ax7, ay7);\n\n ay8 = GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7);\n\n ax8 = GGML_F16x_VEC_LOAD(x[0] + i + 7*ggml_f16_epr, 7);\n\n sum_03 = GGML_F16x_VEC_FMA(sum_03, ax8, ay8);\n ax8 = GGML_F16x_VEC_LOAD(x[1] + i + 7*ggml_f16_epr, 7);\n sum_13 = GGML_F16x_VEC_FMA(sum_13, ax8, ay8);\n }\n\n const int np2 = (n & ~(ggml_f16_epr - 1));\n for (int k = np; k < np2; k += ggml_f16_epr) {\n svfloat16_t ry = GGML_F16x_VEC_LOAD(y + k, 0);\n\n svfloat16_t rx = GGML_F16x_VEC_LOAD(x[0] + k, 0);\n sum_00 = GGML_F16x_VEC_FMA(sum_00, rx, ry);\n rx = GGML_F16x_VEC_LOAD(x[1] + k, 0);\n sum_10 = GGML_F16x_VEC_FMA(sum_10, rx, ry);\n }\n\n if (np2 < n) {\n svbool_t pg = svwhilelt_b16(np2, n);\n svfloat16_t hx_0 = svld1_f16(pg, (const __fp16 *)(x[0] + np2));\n svfloat16_t hx_1 = svld1_f16(pg, (const __fp16 *)(x[1] + np2));\n svfloat16_t hy = svld1_f16(pg, (const __fp16 *)(y + np2));\n\n sum_00 = svmad_f16_x(pg, hx_0, hy, sum_00);\n sum_10 = svmad_f16_x(pg, hx_1, hy, sum_10);\n }\n GGML_F16x_VEC_REDUCE(sumf[0], sum_00, sum_01, sum_02, sum_03);\n GGML_F16x_VEC_REDUCE(sumf[1], sum_10, sum_11, sum_12, sum_13);\n\n #elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfh)\n size_t vl = __riscv_vsetvlmax_e32m4();\n\n // initialize accumulators to all zeroes\n vfloat32m4_t vsum0_0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n vfloat32m4_t vsum0_1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n vfloat32m4_t vsum1_0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n vfloat32m4_t vsum1_1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n\n // calculate step size\n const size_t epr = __riscv_vsetvlmax_e16m2();\n const size_t step = epr * 2;\n const int np = (n & ~(step - 1));\n\n // unroll by 2 along the row dimension\n for (int i = 0; i < np; i += step) {\n vfloat16m2_t ay0 = __riscv_vle16_v_f16m2((const _Float16 *)(y + i), epr);\n vfloat16m2_t ax0_0 = __riscv_vle16_v_f16m2((const _Float16 *)(x[0] + i), epr);\n vfloat16m2_t ax1_0 = __riscv_vle16_v_f16m2((const _Float16 *)(x[1] + i), epr);\n vsum0_0 = __riscv_vfwmacc_vv_f32m4(vsum0_0, ax0_0, ay0, epr);\n vsum1_0 = __riscv_vfwmacc_vv_f32m4(vsum1_0, ax1_0, ay0, epr);\n\n vfloat16m2_t ay1 = __riscv_vle16_v_f16m2((const _Float16 *)(y + i + epr), epr);\n vfloat16m2_t ax0_1 = __riscv_vle16_v_f16m2((const _Float16 *)(x[0] + i + epr), epr);\n vfloat16m2_t ax1_1 = __riscv_vle16_v_f16m2((const _Float16 *)(x[1] + i + epr), epr);\n vsum0_1 = __riscv_vfwmacc_vv_f32m4(vsum0_1, ax0_1, ay1, epr);\n vsum1_1 = __riscv_vfwmacc_vv_f32m4(vsum1_1, ax1_1, ay1, epr);\n }\n\n vfloat32m4_t vsum0 = __riscv_vfadd_vv_f32m4(vsum0_0, vsum0_1, vl);\n vfloat32m4_t vsum1 = __riscv_vfadd_vv_f32m4(vsum1_0, vsum1_1, vl);\n\n // leftovers\n for (int i = np; i < n; i += vl) {\n vl = __riscv_vsetvl_e16m2(n - i);\n vfloat16m2_t ay = __riscv_vle16_v_f16m2((const _Float16 *)(y + i), vl);\n vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16 *)(x[0] + i), vl);\n vfloat16m2_t ax1 = __riscv_vle16_v_f16m2((const _Float16 *)(x[1] + i), vl);\n\n vsum0 = __riscv_vfwmacc_vv_f32m4(vsum0, ax0, ay, vl);\n vsum1 = __riscv_vfwmacc_vv_f32m4(vsum1, ax1, ay, vl);\n }\n\n // reduce\n vl = __riscv_vsetvlmax_e32m2();\n vfloat32m2_t acc0_0 = __riscv_vfadd_vv_f32m2(__riscv_vget_v_f32m4_f32m2(vsum0, 0),\n __riscv_vget_v_f32m4_f32m2(vsum0, 1), vl);\n vl = __riscv_vsetvlmax_e32m1();\n vfloat32m1_t acc0_1 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(acc0_0, 0),\n __riscv_vget_v_f32m2_f32m1(acc0_0, 1), vl);\n vfloat32m1_t redsum0 = __riscv_vfredusum_vs_f32m1_f32m1(\n acc0_1, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);\n\n vl = __riscv_vsetvlmax_e32m2();\n vfloat32m2_t acc1_0 = __riscv_vfadd_vv_f32m2(__riscv_vget_v_f32m4_f32m2(vsum1, 0),\n __riscv_vget_v_f32m4_f32m2(vsum1, 1), vl);\n vl = __riscv_vsetvlmax_e32m1();\n vfloat32m1_t acc1_1 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(acc1_0, 0),\n __riscv_vget_v_f32m2_f32m1(acc1_0, 1), vl);\n vfloat32m1_t redsum1 = __riscv_vfredusum_vs_f32m1_f32m1(\n acc1_1, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);\n sumf[0] = __riscv_vfmv_f_s_f32m1_f32(redsum0);\n sumf[1] = __riscv_vfmv_f_s_f32m1_f32(redsum1);\n\n #else\n const int np = (n & ~(GGML_F16_STEP - 1));\n\n GGML_F16_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } };\n\n GGML_F16_VEC ax[GGML_F16_ARR];\n GGML_F16_VEC ay[GGML_F16_ARR];\n\n for (int i = 0; i < np; i += GGML_F16_STEP) {\n for (int j = 0; j < GGML_F16_ARR; j++) {\n ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);\n\n for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {\n ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j);\n\n sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]);\n }\n }\n }\n\n // reduce sum0..sum3 to sum0\n for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {\n GGML_F16_VEC_REDUCE(sumf[k], sum[k]);\n }\n\n // leftovers\n for (int i = np; i < n; ++i) {\n for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {\n sumf[j] += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[j][i])*GGML_CPU_FP16_TO_FP32(y[i]));\n }\n }\n #endif\n#else\n for (int i = 0; i < n; ++i) {\n for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {\n sumf[j] += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[j][i])*GGML_CPU_FP16_TO_FP32(y[i]));\n }\n }\n#endif\n\n for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {\n s[i] = (float)sumf[i];\n }\n}\n\ninline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const float * GGML_RESTRICT x, const float v) {\n#if defined(GGML_SIMD)\n #if defined(__ARM_FEATURE_SVE)\n\n const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;\n const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16\n const int ggml_f32_step = 8 * ggml_f32_epr; // choose 8 SVE registers\n GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);\n\n const int np = (n & ~(ggml_f32_step - 1));\n svfloat32_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;\n svfloat32_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;\n for (int i = 0; i < np; i += ggml_f32_step) {\n\n ax1 = GGML_F32_VEC_LOAD(x + i);\n ay1 = GGML_F32_VEC_LOAD(y + i);\n ay1 = GGML_F32_VEC_FMA(ay1, ax1, vx);\n\n GGML_F32_VEC_STORE(y + i, ay1);\n\n ax2 = GGML_F32_VEC_LOAD(x + i + 1*ggml_f32_epr);\n ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);\n ay2 = GGML_F32_VEC_FMA(ay2, ax2, vx);\n\n GGML_F32_VEC_STORE(y + i + 1*ggml_f32_epr, ay2);\n\n ax3 = GGML_F32_VEC_LOAD(x + i + 2*ggml_f32_epr);\n ay3 = GGML_F32_VEC_LOAD(y + i + 2*ggml_f32_epr);\n ay3 = GGML_F32_VEC_FMA(ay3, ax3, vx);\n\n GGML_F32_VEC_STORE(y + i + 2*ggml_f32_epr, ay3);\n\n ax4 = GGML_F32_VEC_LOAD(x + i + 3*ggml_f32_epr);\n ay4 = GGML_F32_VEC_LOAD(y + i + 3*ggml_f32_epr);\n ay4 = GGML_F32_VEC_FMA(ay4, ax4, vx);\n\n GGML_F32_VEC_STORE(y + i + 3*ggml_f32_epr, ay4);\n\n ax5 = GGML_F32_VEC_LOAD(x + i + 4*ggml_f32_epr);\n ay5 = GGML_F32_VEC_LOAD(y + i + 4*ggml_f32_epr);\n ay5 = GGML_F32_VEC_FMA(ay5, ax5, vx);\n\n GGML_F32_VEC_STORE(y + i + 4*ggml_f32_epr, ay5);\n\n ax6 = GGML_F32_VEC_LOAD(x + i + 5*ggml_f32_epr);\n ay6 = GGML_F32_VEC_LOAD(y + i + 5*ggml_f32_epr);\n ay6 = GGML_F32_VEC_FMA(ay6, ax6, vx);\n\n GGML_F32_VEC_STORE(y + i + 5*ggml_f32_epr, ay6);\n\n ax7 = GGML_F32_VEC_LOAD(x + i + 6*ggml_f32_epr);\n ay7 = GGML_F32_VEC_LOAD(y + i + 6*ggml_f32_epr);\n ay7 = GGML_F32_VEC_FMA(ay7, ax7, vx);\n\n GGML_F32_VEC_STORE(y + i + 6*ggml_f32_epr, ay7);\n\n ax8 = GGML_F32_VEC_LOAD(x + i + 7*ggml_f32_epr);\n ay8 = GGML_F32_VEC_LOAD(y + i + 7*ggml_f32_epr);\n ay8 = GGML_F32_VEC_FMA(ay8, ax8, vx);\n\n GGML_F32_VEC_STORE(y + i + 7*ggml_f32_epr, ay8);\n }\n // leftovers\n // Since 8 unrolls are done in above loop, leftovers lie in range [0, ggml_f32_step] which is handled in below loop\n const int np2 = (n & ~(ggml_f32_epr - 1));\n for (int i = np; i < np2; i += ggml_f32_epr) {\n ax1 = GGML_F32_VEC_LOAD(x + i);\n ay1 = GGML_F32_VEC_LOAD(y + i);\n ay1 = GGML_F32_VEC_FMA(ay1, ax1, vx);\n\n GGML_F32_VEC_STORE(y + i, ay1);\n }\n // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only\n if (np2 < n) {\n svbool_t pg =svwhilelt_b32(np2, n);\n ax1 = svld1_f32(pg, x + np2);\n ay1 = svld1_f32(pg, y + np2);\n ay1 = svmad_f32_m(pg, ax1, vx, ay1);\n\n svst1_f32(pg, y + np2, ay1);\n }\n #elif defined(__riscv_v_intrinsic)\n for (int i = 0, avl; i < n; i += avl) {\n avl = __riscv_vsetvl_e32m8(n - i);\n vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl);\n vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);\n vfloat32m8_t ny = __riscv_vfmadd_vf_f32m8(ax, v, ay, avl);\n __riscv_vse32_v_f32m8(&y[i], ny, avl);\n }\n #else\n const int np = (n & ~(GGML_F32_STEP - 1));\n\n GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);\n\n GGML_F32_VEC ax[GGML_F32_ARR];\n GGML_F32_VEC ay[GGML_F32_ARR];\n\n for (int i = 0; i < np; i += GGML_F32_STEP) {\n for (int j = 0; j < GGML_F32_ARR; j++) {\n ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);\n ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);\n ay[j] = GGML_F32_VEC_FMA(ay[j], ax[j], vx);\n\n GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);\n }\n }\n\n // leftovers\n for (int i = np; i < n; ++i) {\n y[i] += x[i]*v;\n }\n #endif\n#else\n // scalar\n for (int i = 0; i < n; ++i) {\n y[i] += x[i]*v;\n }\n#endif\n}\n\ninline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y, const ggml_fp16_t * GGML_RESTRICT x, const float v) {\n#if defined(GGML_SIMD) && defined(__ARM_FEATURE_SVE)\n const int sve_register_length = svcntb() * 8;\n const int ggml_f16_epr = sve_register_length / 16;\n const int ggml_f16_step = 8 * ggml_f16_epr;\n\n GGML_F16x_VEC vx = GGML_F16x_VEC_SET1(v);\n\n int np = (n & ~(ggml_f16_step - 1));\n\n svfloat16_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;\n svfloat16_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;\n for (int i = 0; i < np; i += ggml_f16_step) {\n ax1 = GGML_F16x_VEC_LOAD(x + i + 0 * ggml_f16_epr, 0);\n ay1 = GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0);\n ay1 = GGML_F16x_VEC_FMA(ay1, ax1, vx);\n\n GGML_F16x_VEC_STORE(y + i + 0 * ggml_f16_epr, ay1, 0);\n\n ax2 = GGML_F16x_VEC_LOAD(x + i + 1 * ggml_f16_epr, 1);\n ay2 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1);\n ay2 = GGML_F16x_VEC_FMA(ay2, ax2, vx);\n\n GGML_F16x_VEC_STORE(y + i + 1 * ggml_f16_epr, ay2, 1);\n\n ax3 = GGML_F16x_VEC_LOAD(x + i + 2 * ggml_f16_epr, 2);\n ay3 = GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2);\n ay3 = GGML_F16x_VEC_FMA(ay3, ax3, vx);\n\n GGML_F16x_VEC_STORE(y + i + 2 * ggml_f16_epr, ay3, 2);\n\n ax4 = GGML_F16x_VEC_LOAD(x + i + 3 * ggml_f16_epr, 3);\n ay4 = GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3);\n ay4 = GGML_F16x_VEC_FMA(ay4, ax4, vx);\n\n GGML_F16x_VEC_STORE(y + i + 3 * ggml_f16_epr, ay4, 3);\n\n ax5 = GGML_F16x_VEC_LOAD(x + i + 4 * ggml_f16_epr, 4);\n ay5 = GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4);\n ay5 = GGML_F16x_VEC_FMA(ay5, ax5, vx);\n\n GGML_F16x_VEC_STORE(y + i + 4 * ggml_f16_epr, ay5, 4);\n\n ax6 = GGML_F16x_VEC_LOAD(x + i + 5 * ggml_f16_epr, 5);\n ay6 = GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5);\n ay6 = GGML_F16x_VEC_FMA(ay6, ax6, vx);\n\n GGML_F16x_VEC_STORE(y + i + 5 * ggml_f16_epr, ay6, 5);\n\n ax7 = GGML_F16x_VEC_LOAD(x + i + 6 * ggml_f16_epr, 6);\n ay7 = GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6);\n ay7 = GGML_F16x_VEC_FMA(ay7, ax7, vx);\n\n GGML_F16x_VEC_STORE(y + i + 6 * ggml_f16_epr, ay7, 6);\n\n ax8 = GGML_F16x_VEC_LOAD(x + i + 7 * ggml_f16_epr, 7);\n ay8 = GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7);\n ay8 = GGML_F16x_VEC_FMA(ay8, ax8, vx);\n\n GGML_F16x_VEC_STORE(y + i + 7 * ggml_f16_epr, ay8, 7);\n }\n const int np2 = (n & ~(ggml_f16_epr - 1));\n for (int k = np; k < np2; k += ggml_f16_epr) {\n svfloat16_t rx = GGML_F16x_VEC_LOAD(x + k, 0);\n svfloat16_t ry = GGML_F16x_VEC_LOAD(y + k, 0);\n ry = GGML_F16x_VEC_FMA(ry, rx, vx);\n\n GGML_F16x_VEC_STORE(y + k, ry, 0);\n }\n\n if (np2 < n) {\n svbool_t pg = svwhilelt_b16(np2, n);\n svfloat16_t hx = svld1_f16(pg, (const __fp16 *)(x + np2));\n svfloat16_t hy = svld1_f16(pg, (const __fp16 *)(y + np2));\n hy = svmad_f16_x(pg, hx, vx, hy);\n svst1_f16(pg, (__fp16 *)(y + np2), hy);\n }\n np = n;\n#elif defined(__riscv_zvfh) // implies __riscv_v_intrinsic\n const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);\n const _Float16 scale = *(const _Float16*)(&s);\n\n // calculate step size\n const int epr = __riscv_vsetvlmax_e16m4();\n const int step = epr * 2;\n int np = (n & ~(step - 1));\n\n // unroll by 2\n for (int i = 0; i < np; i += step) {\n vfloat16m4_t ax0 = __riscv_vle16_v_f16m4((const _Float16*)x + i, epr);\n vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, epr);\n ay0 = __riscv_vfmacc_vf_f16m4(ay0, scale, ax0, epr);\n __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, epr);\n __asm__ __volatile__ (\"\" ::: \"memory\");\n\n vfloat16m4_t ax1 = __riscv_vle16_v_f16m4((const _Float16*)x + i + epr, epr);\n vfloat16m4_t ay1 = __riscv_vle16_v_f16m4((const _Float16*)y + i + epr, epr);\n ay1 = __riscv_vfmacc_vf_f16m4(ay1, scale, ax1, epr);\n __riscv_vse16_v_f16m4((_Float16*)y + i + epr, ay1, epr);\n __asm__ __volatile__ (\"\" ::: \"memory\");\n }\n\n // leftovers\n int vl;\n for (int i = np; i < n; i += vl) {\n vl = __riscv_vsetvl_e16m4(n - i);\n vfloat16m4_t ax0 = __riscv_vle16_v_f16m4((const _Float16*)x + i, vl);\n vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, vl);\n ay0 = __riscv_vfmacc_vf_f16m4(ay0, scale, ax0, vl);\n __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, vl);\n }\n np = n;\n#elif defined(GGML_SIMD)\n const int np = (n & ~(GGML_F16_STEP - 1));\n\n GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);\n\n GGML_F16_VEC ax[GGML_F16_ARR];\n GGML_F16_VEC ay[GGML_F16_ARR];\n\n for (int i = 0; i < np; i += GGML_F16_STEP) {\n for (int j = 0; j < GGML_F16_ARR; j++) {\n ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);\n ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);\n ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);\n\n GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);\n }\n }\n#else\n const int np = 0;\n#endif\n\n // leftovers\n for (int i = np; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(y[i]) + GGML_CPU_FP16_TO_FP32(x[i])*v);\n }\n}\n\n// xs and vs are byte strides of x and v\ninline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * GGML_RESTRICT y, const float * GGML_RESTRICT xv, const float * GGML_RESTRICT vv) {\n\n const float * GGML_RESTRICT x[GGML_VEC_MAD_UNROLL];\n const float * GGML_RESTRICT v[GGML_VEC_MAD_UNROLL];\n\n for (int i = 0; i < GGML_VEC_MAD_UNROLL; ++i) {\n x[i] = (const float *) ((const char *) xv + i*xs);\n v[i] = (const float *) ((const char *) vv + i*vs);\n }\n\n#if defined(GGML_SIMD)\n #if defined(__ARM_FEATURE_SVE)\n // scalar Route to scalar implementation //TODO: Write SVE code\n for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n for (int i = 0; i < n; ++i) {\n y[i] += x[k][i]*v[k][0];\n }\n }\n #elif defined(__riscv_v_intrinsic)\n for (int i = 0, avl; i < n; i += avl) {\n avl = __riscv_vsetvl_e32m8(n - i);\n vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);\n for (int k = 0; k < GGML_VEC_MAD_UNROLL; k++) {\n vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[k][i], avl);\n ay = __riscv_vfmadd_vf_f32m8(ax, v[k][0], ay, avl);\n }\n __riscv_vse32_v_f32m8(&y[i], ay, avl);\n }\n #else\n const int np = (n & ~(GGML_F32_STEP - 1));\n\n GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL];\n\n for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n vx[k] = GGML_F32_VEC_SET1(v[k][0]);\n }\n\n GGML_F32_VEC ax[GGML_VEC_MAD_UNROLL][GGML_F32_ARR];\n GGML_F32_VEC ay[GGML_F32_ARR];\n\n for (int i = 0; i < np; i += GGML_F32_STEP) {\n for (int j = 0; j < GGML_F32_ARR; j++) {\n ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);\n\n for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n ax[k][j] = GGML_F32_VEC_LOAD(x[k] + i + j*GGML_F32_EPR);\n ay[j] = GGML_F32_VEC_FMA(ay[j], ax[k][j], vx[k]);\n }\n\n GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);\n }\n }\n\n // leftovers\n for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n for (int i = np; i < n; ++i) {\n y[i] += x[k][i]*v[k][0];\n }\n }\n #endif\n#else\n // scalar\n for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n for (int i = 0; i < n; ++i) {\n y[i] += x[k][i]*v[k][0];\n }\n }\n#endif\n}\n\ninline static void ggml_vec_mad1_f32(const int n, float * y, const float * x, const float s, const float b) {\n#if defined(GGML_USE_ACCELERATE)\n vDSP_vsmsa(x, 1, &s, &b, y, 1, n);\n#elif defined(GGML_SIMD)\n #if defined(__ARM_FEATURE_SVE)\n // scalar ; TODO: Write SVE code\n for (int i = 0; i < n; ++i) {\n y[i] = x[i]*s + b;\n }\n #elif defined(__riscv_v_intrinsic)\n for (int i = 0, avl; i < n; i += avl) {\n avl = __riscv_vsetvl_e32m8(n - i);\n vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl);\n vfloat32m8_t vb = __riscv_vfmv_v_f_f32m8(b, avl);\n vfloat32m8_t ny = __riscv_vfmadd_vf_f32m8(ax, s, vb, avl);\n __riscv_vse32_v_f32m8(&y[i], ny, avl);\n }\n #else\n const int np = (n & ~(GGML_F32_STEP - 1));\n\n GGML_F32_VEC vs = GGML_F32_VEC_SET1(s);\n GGML_F32_VEC vb = GGML_F32_VEC_SET1(b);\n\n GGML_F32_VEC ay[GGML_F32_ARR];\n\n for (int i = 0; i < np; i += GGML_F32_STEP) {\n for (int j = 0; j < GGML_F32_ARR; j++) {\n ay[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);\n ay[j] = GGML_F32_VEC_FMA(vb, ay[j], vs);\n\n GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);\n }\n }\n\n // leftovers\n for (int i = np; i < n; ++i) {\n y[i] = x[i]*s + b;\n }\n #endif\n#else\n // scalar\n for (int i = 0; i < n; ++i) {\n y[i] = x[i]*s + b;\n }\n#endif\n}\n\n//inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] *= v; }\ninline static void ggml_vec_scale_f32(const int n, float * y, const float v) {\n#if defined(GGML_USE_ACCELERATE)\n vDSP_vsmul(y, 1, &v, y, 1, n);\n#elif defined(GGML_SIMD)\n #if defined(__ARM_FEATURE_SVE)\n const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;\n const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16\n const int ggml_f32_step = 2 * ggml_f32_epr;\n\n GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);\n const int np = (n & ~(ggml_f32_step - 1));\n svfloat32_t ay1;\n svfloat32_t ay2;\n for (int i = 0; i < np; i += ggml_f32_step) {\n ay1 = GGML_F32_VEC_LOAD(y + i);\n ay1 = GGML_F32_VEC_MUL(ay1, vx);\n GGML_F32_VEC_STORE(y + i, ay1);\n\n ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);\n ay2 = GGML_F32_VEC_MUL(ay2, vx);\n GGML_F32_VEC_STORE(y + i + 1*ggml_f32_epr, ay2);\n }\n // leftovers\n // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only\n for (int i = np; i < n; i += ggml_f32_epr) {\n svbool_t pg = svwhilelt_b32(i, n);\n ay1 = svld1_f32(pg, y + i);\n ay1 = svmul_f32_m(pg, ay1, vx);\n svst1_f32(pg, y + i, ay1);\n }\n #elif defined(__riscv_v_intrinsic)\n for (int i = 0, avl; i < n; i += avl) {\n avl = __riscv_vsetvl_e32m8(n - i);\n vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);\n vfloat32m8_t ny = __riscv_vfmul_vf_f32m8(ay, v, avl);\n __riscv_vse32_v_f32m8(&y[i], ny, avl);\n }\n #else\n const int np = (n & ~(GGML_F32_STEP - 1));\n\n GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);\n\n GGML_F32_VEC ay[GGML_F32_ARR];\n\n for (int i = 0; i < np; i += GGML_F32_STEP) {\n for (int j = 0; j < GGML_F32_ARR; j++) {\n ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);\n ay[j] = GGML_F32_VEC_MUL(ay[j], vx);\n\n GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);\n }\n }\n\n // leftovers\n for (int i = np; i < n; ++i) {\n y[i] *= v;\n }\n #endif\n#else\n // scalar\n for (int i = 0; i < n; ++i) {\n y[i] *= v;\n }\n#endif\n}\n\ninline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {\n#if defined(GGML_SIMD) && defined(__ARM_FEATURE_SVE)\n const int sve_register_length = svcntb() * 8;\n const int ggml_f16_epr = sve_register_length / 16;\n const int ggml_f16_step = 2 * ggml_f16_epr;\n\n GGML_F16x_VEC vx = GGML_F16x_VEC_SET1(v);\n const int np = (n & ~(ggml_f16_step - 1));\n svfloat16_t ay1, ay2;\n\n for (int i = 0; i < np; i += ggml_f16_step) {\n ay1 = GGML_F16x_VEC_LOAD(y + i + 0*ggml_f16_epr, 0);\n ay1 = GGML_F16x_VEC_MUL(ay1, vx);\n GGML_F16x_VEC_STORE(y + i + 0*ggml_f16_epr, ay1, 0);\n\n ay2 = GGML_F16x_VEC_LOAD(y + i + 1*ggml_f16_epr, 1);\n ay2 = GGML_F16x_VEC_MUL(ay2, vx);\n GGML_F16x_VEC_STORE(y + i + 1*ggml_f16_epr, ay2, 1);\n }\n // leftovers\n // maximum number of leftover elements will be less that ggmlF_16x_epr. Apply predicated svmad on available elements only\n if (np < n) {\n svbool_t pg = svwhilelt_b16(np, n);\n svfloat16_t hy = svld1_f16(pg, (__fp16 *)(y + np));\n svfloat16_t out = svmul_f16_m(pg, hy, vx);\n svst1_f16(pg, (__fp16 *)(y + np), out);\n }\n#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfh)\n const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);\n const _Float16 scale = *(const _Float16*)(&s);\n\n // calculate step size\n const int epr = __riscv_vsetvlmax_e16m4();\n const int step = epr * 2;\n const int np = (n & ~(step - 1));\n\n // unroll by 2\n for (int i = 0; i < np; i += step) {\n vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, epr);\n ay0 = __riscv_vfmul_vf_f16m4(ay0, scale, epr);\n __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, epr);\n __asm__ __volatile__ (\"\" ::: \"memory\");\n\n vfloat16m4_t ay1 = __riscv_vle16_v_f16m4((const _Float16*)y + i + epr, epr);\n ay1 = __riscv_vfmul_vf_f16m4(ay1, scale, epr);\n __riscv_vse16_v_f16m4((_Float16*)y + i + epr, ay1, epr);\n __asm__ __volatile__ (\"\" ::: \"memory\");\n }\n\n // leftovers\n int vl;\n for (int i = np; i < n; i += vl) {\n vl = __riscv_vsetvl_e16m4(n - i);\n vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, vl);\n ay0 = __riscv_vfmul_vf_f16m4(ay0, scale, vl);\n __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, vl);\n }\n#elif defined(GGML_SIMD)\n const int np = (n & ~(GGML_F16_STEP - 1));\n\n GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);\n\n GGML_F16_VEC ay[GGML_F16_ARR];\n\n for (int i = 0; i < np; i += GGML_F16_STEP) {\n for (int j = 0; j < GGML_F16_ARR; j++) {\n ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);\n ay[j] = GGML_F16_VEC_MUL(ay[j], vx);\n\n GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);\n }\n }\n\n // leftovers\n for (int i = np; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(y[i])*v);\n }\n#else\n // scalar\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(y[i])*v);\n }\n#endif\n}\n\ninline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }\ninline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }\ninline static void ggml_vec_sqr_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n float v = GGML_CPU_FP16_TO_FP32(x[i]);\n y[i] = GGML_CPU_FP32_TO_FP16(v*v);\n }\n}\ninline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }\ninline static void ggml_vec_sqrt_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(sqrtf(GGML_CPU_FP16_TO_FP32(x[i])));\n }\n}\ninline static void ggml_vec_log_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]); }\ninline static void ggml_vec_log_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(logf(GGML_CPU_FP16_TO_FP32(x[i])));\n }\n}\ninline static void ggml_vec_sin_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sinf(x[i]); }\ninline static void ggml_vec_sin_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(sinf(GGML_CPU_FP16_TO_FP32(x[i])));\n }\n}\ninline static void ggml_vec_cos_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = cosf(x[i]); }\ninline static void ggml_vec_cos_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(cosf(GGML_CPU_FP16_TO_FP32(x[i])));\n }\n}\ninline static void ggml_vec_abs_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }\ninline static void ggml_vec_abs_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(fabsf(GGML_CPU_FP16_TO_FP32(x[i])));\n }\n}\ninline static void ggml_vec_sgn_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }\ninline static void ggml_vec_sgn_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n float v = GGML_CPU_FP16_TO_FP32(x[i]);\n y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? 1.f : ((v < 0.f) ? -1.f : 0.f));\n }\n}\ninline static void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }\ninline static void ggml_vec_step_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16((GGML_CPU_FP16_TO_FP32(x[i]) > 0.f) ? 1.f : 0.f);\n }\n}\ninline static void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = tanhf(x[i]); }\ninline static void ggml_vec_tanh_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(tanhf(GGML_CPU_FP16_TO_FP32(x[i])));\n }\n}\ninline static void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expm1f(x[i]); }\ninline static void ggml_vec_elu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n const float v = GGML_CPU_FP16_TO_FP32(x[i]);\n y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? v : expm1f(v));\n }\n}\ninline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }\ninline static void ggml_vec_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n float v = GGML_CPU_FP16_TO_FP32(x[i]);\n y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? v : 0.f);\n }\n}\ninline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }\ninline static void ggml_vec_leaky_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float ns) {\n for (int i = 0; i < n; ++i) {\n float v = GGML_CPU_FP16_TO_FP32(x[i]);\n y[i] = GGML_CPU_FP32_TO_FP16(((v > 0.f) ? v : 0.f) + ns * ((v < 0.0f) ? v : 0.f));\n }\n}\ninline static void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); }\ninline static void ggml_vec_sigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(1.f / (1.f + expf(-GGML_CPU_FP16_TO_FP32(x[i]))));\n }\n}\n// TODO: optimize performance\ninline static void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }\ninline static void ggml_vec_hardswish_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n float v = GGML_CPU_FP16_TO_FP32(x[i]);\n y[i] = GGML_CPU_FP32_TO_FP16(v * fminf(1.0f, fmaxf(0.0f, (v + 3.0f) / 6.0f)));\n }\n}\ninline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }\ninline static void ggml_vec_hardsigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(fminf(1.0f, fmaxf(0.0f, (GGML_CPU_FP16_TO_FP32(x[i]) + 3.0f) / 6.0f)));\n }\n}\ninline static void ggml_vec_exp_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = expf(x[i]); }\ninline static void ggml_vec_exp_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = GGML_CPU_FP32_TO_FP16(expf(GGML_CPU_FP16_TO_FP32(x[i])));\n }\n}\n\nstatic const float GELU_COEF_A = 0.044715f;\nstatic const float GELU_QUICK_COEF = -1.702f;\nstatic const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;\nstatic const float SQRT_2_INV = 0.70710678118654752440084436210484f;\n\ninline static float ggml_gelu_f32(float x) {\n return 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));\n}\n\ninline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n const uint16_t * i16 = (const uint16_t *) x;\n for (int i = 0; i < n; ++i) {\n y[i] = ggml_table_gelu_f16[i16[i]];\n }\n}\n\ninline static void ggml_vec_gelu_erf_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n float xi = GGML_CPU_FP16_TO_FP32(x[i]);\n float res = 0.5f*xi*(1.0f + erff(xi*SQRT_2_INV));\n y[i] = GGML_CPU_FP32_TO_FP16(res);\n }\n}\n\n#ifdef GGML_GELU_FP16\ninline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {\n uint16_t t;\n for (int i = 0; i < n; ++i) {\n if (x[i] <= -10.0f) {\n y[i] = 0.0f;\n } else if (x[i] >= 10.0f) {\n y[i] = x[i];\n } else {\n ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);\n memcpy(&t, &fp16, sizeof(uint16_t));\n y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_f16[t]);\n }\n }\n}\n#else\ninline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = ggml_gelu_f32(x[i]);\n }\n}\n#endif\n\ninline static void ggml_vec_gelu_erf_f32(const int n, float * y, const float * x) {\n for (int i = 0; i < n; ++i) {\n float xi = x[i];\n y[i] = 0.5f*xi*(1.0f + erff(xi*SQRT_2_INV));\n }\n}\n\ninline static float ggml_gelu_quick_f32(float x) {\n return x*(1.0f/(1.0f+expf(GELU_QUICK_COEF*x)));\n}\n\n//inline static void ggml_vec_gelu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n// const uint16_t * i16 = (const uint16_t *) x;\n// for (int i = 0; i < n; ++i) {\n// y[i] = ggml_table_gelu_quick_f16[i16[i]];\n// }\n//}\n\n#ifdef GGML_GELU_QUICK_FP16\ninline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {\n uint16_t t;\n for (int i = 0; i < n; ++i) {\n ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);\n memcpy(&t, &fp16, sizeof(uint16_t));\n y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_quick_f16[t]);\n }\n}\n#else\ninline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = ggml_gelu_quick_f32(x[i]);\n }\n}\n#endif\n\ninline static void ggml_vec_gelu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n float v = GGML_CPU_FP16_TO_FP32(x[i]);\n y[i] = GGML_CPU_FP32_TO_FP16(v*(1.0f/(1.0f+expf(GELU_QUICK_COEF*v))));\n }\n}\n\n// Sigmoid Linear Unit (SiLU) function\ninline static float ggml_silu_f32(float x) {\n return x/(1.0f + expf(-x));\n}\ninline static ggml_fp16_t ggml_silu_f16(ggml_fp16_t x) {\n float v = GGML_CPU_FP16_TO_FP32(x);\n return GGML_CPU_FP32_TO_FP16(v/(1.0f + expf(-v)));\n}\n\n#if __FINITE_MATH_ONLY__\n#error \"some routines in ggml.c require non-finite math arithmetics -- pass -fno-finite-math-only to the compiler to fix\"\n#error \"ref: https://github.com/ggml-org/llama.cpp/pull/7154#issuecomment-2143844461\"\n#endif\n\n/* Below function was borrowed from the GitHub repository:\nhttps://github.com/openvinotoolkit/openvino/blob/master/src/plugins/intel_cpu/src/nodes/kernels/scaled_attn/common.hpp */\n#if defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n inline static svfloat32_t exp_ps_sve(svbool_t pg, svfloat32_t src) {\n // Constants\n const svfloat32_t log2_e = svdup_n_f32(1.4426950409f);\n const svfloat32_t ln2 = svdup_n_f32(0.6931473921f);\n const svfloat32_t half_ln2_sq = svdup_n_f32(0.2413862043f);\n const svuint32_t not_mask17 = svdup_n_u32(~((1u << 17) - 1));\n const svfloat32_t one = svdup_n_f32(1.0f);\n const svfloat32_t inactive1 = svdup_n_f32(0.0f);\n const svint32_t inactive2 = svdup_n_s32(0);\n\n // Algorithm starts here\n svfloat32_t t0 = svmul_f32_m(pg, src, log2_e); // y = x * log2(e)\n svfloat32_t t1 = svrintm_f32_m(inactive1, pg, t0); // rount to int (float)\n svint32_t t2 = svcvt_s32_f32_m(inactive2, pg, t1); // n\n\n t1 = svsub_f32_m(pg, t0, t1); // a = y - floor(y)\n t1 = svadd_f32_m(pg, t1, one); // b = a + 1\n\n svuint32_t t3 = svlsr_n_u32_m(pg, svreinterpret_u32_f32(t1), 17); // v = b >> 17 (u32)\n svfloat32_t t4 = svexpa_f32(t3); // c = fexpa(v)\n t4 = svscale_f32_m(pg, t4, t2); // fexpa(v) * 2^(n)\n\n // and_(t2.d, t1.d, not_mask17.d)\n svfloat32_t t5 = svreinterpret_f32_u32(svand_u32_m(pg, svreinterpret_u32_f32(t1), not_mask17));\n t5 = svsub_f32_m(pg, t1, t5); // z\n t0 = svmla_f32_m(pg, ln2, t5, half_ln2_sq); // ln2 + half_ln2_sq * z\n t0 = svmla_f32_m(pg, one, t5, t0); // 1 + (ln2 * z) + (half_ln2_sq * z * z)\n t0 = svmul_f32_m(pg, t0, t4); // Final result\n\n return t0;\n }\n#endif\n\n#if defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n\ninline static svfloat32_t ggml_v_expf(svbool_t pg, svfloat32_t x) {\n const svfloat32_t r = svdup_n_f32_x(pg, 0x1.8p23f);\n const svfloat32_t z = svmla_n_f32_x(pg, r, x, 0x1.715476p+0f);\n const svfloat32_t n = svsub_f32_x(pg, z, r);\n const svfloat32_t b = svmls_n_f32_x(pg, svmls_n_f32_x(pg, x, n, 0x1.62e4p-1f), n, 0x1.7f7d1cp-20f);\n const svuint32_t e = svlsl_n_u32_x(pg, svreinterpret_u32_f32(z), 23);\n const svfloat32_t k = svreinterpret_f32_u32(svadd_u32_x(pg, e, svreinterpret_u32_f32(svdup_n_f32_x(pg, 1))));\n const svbool_t c = svacgt_n_f32(pg, n, 126);\n const svfloat32_t u = svmul_f32_x(pg, b, b);\n const svfloat32_t j = svmla_f32_x(pg,\n svmul_n_f32_x(pg, b, 0x1.ffffecp-1f),\n svmla_f32_x(pg, svmla_f32_x(pg, svdup_n_f32_x(pg, 0x1.fffdb6p-2f), svdup_n_f32_x(pg, 0x1.555e66p-3f), b),\n svmla_f32_x(pg, svdup_n_f32_x(pg, 0x1.573e2ep-5f), svdup_n_f32_x(pg, 0x1.0e4020p-7f), b), u), u);\n const svuint32_t d = svdup_n_u32_z(svcmple_n_f32(pg, n, 0.0), 0x82000000);\n const svfloat32_t s1 = svreinterpret_f32_u32(svadd_n_u32_x(pg, d, 0x7f000000));\n const svfloat32_t s2 = svreinterpret_f32_u32(svsub_u32_x(pg, e, d));\n return svsel_f32(svacgt_f32(pg, n, svdup_n_f32_x(pg, 192)), svmul_f32_x(pg, s1, s1),\n svsel_f32(c, svmul_f32_x(pg, svmla_f32_x(pg, s2, s2, j), s1), svmla_f32_x(pg, k, k, j)));\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static svfloat32_t ggml_v_silu(svbool_t pg, svfloat32_t x) {\n const svfloat32_t one = svdup_n_f32_x(pg, 1.0f);\n const svfloat32_t zero = svdup_n_f32_x(pg, 0.0f);\n const svfloat32_t neg_x = svsub_f32_x(pg, zero, x);\n const svfloat32_t exp_neg_x = ggml_v_expf(pg, neg_x);\n const svfloat32_t one_plus_exp_neg_x = svadd_f32_x(pg, one, exp_neg_x);\n return svdiv_f32_x(pg, x, one_plus_exp_neg_x);\n}\n\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static float32x4_t ggml_v_expf(float32x4_t x) {\n const float32x4_t r = vdupq_n_f32(0x1.8p23f);\n const float32x4_t z = vfmaq_f32(r, x, vdupq_n_f32(0x1.715476p+0f));\n const float32x4_t n = vsubq_f32(z, r);\n const float32x4_t b = vfmsq_f32(vfmsq_f32(x, n, vdupq_n_f32(0x1.62e4p-1f)), n,\n vdupq_n_f32(0x1.7f7d1cp-20f));\n const uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23);\n const float32x4_t k = vreinterpretq_f32_u32(vaddq_u32(e, vreinterpretq_u32_f32(vdupq_n_f32(1))));\n const uint32x4_t c = vcagtq_f32(n, vdupq_n_f32(126));\n const float32x4_t u = vmulq_f32(b, b);\n const float32x4_t j = vfmaq_f32(\n vmulq_f32(vdupq_n_f32(0x1.ffffecp-1f), b),\n vfmaq_f32(vfmaq_f32(vdupq_n_f32(0x1.fffdb6p-2f), vdupq_n_f32(0x1.555e66p-3f), b),\n vfmaq_f32(vdupq_n_f32(0x1.573e2ep-5f), vdupq_n_f32(0x1.0e4020p-7f), b), u), u);\n if (!vpaddd_u64(vreinterpretq_u64_u32(c)))\n return vfmaq_f32(k, j, k);\n const uint32x4_t d = vandq_u32(vclezq_f32(n), vdupq_n_u32(0x82000000));\n const float32x4_t s1 = vreinterpretq_f32_u32(vaddq_u32(d, vdupq_n_u32(0x7f000000)));\n const float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, d));\n return vbslq_f32(vcagtq_f32(n, vdupq_n_f32(192)), vmulq_f32(s1, s1),\n vbslq_f32(c, vmulq_f32(vfmaq_f32(s2, s2, j), s1), vfmaq_f32(k, k, j)));\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static float32x4_t ggml_v_silu(float32x4_t x) {\n const float32x4_t one = vdupq_n_f32(1.0f);\n const float32x4_t zero = vdupq_n_f32(0.0f);\n const float32x4_t neg_x = vsubq_f32(zero, x);\n const float32x4_t exp_neg_x = ggml_v_expf(neg_x);\n const float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x);\n return vdivq_f32(x, one_plus_exp_neg_x);\n}\n\n#elif defined(__AVX512F__) && defined(__AVX512DQ__)\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static __m512 ggml_v_expf(__m512 x) {\n const __m512 r = _mm512_set1_ps(0x1.8p23f);\n const __m512 z = _mm512_fmadd_ps(x, _mm512_set1_ps(0x1.715476p+0f), r);\n const __m512 n = _mm512_sub_ps(z, r);\n const __m512 b =\n _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.7f7d1cp-20f),\n _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.62e4p-1f), x));\n const __mmask16 d =\n _mm512_cmp_ps_mask(_mm512_abs_ps(n), _mm512_set1_ps(192), _CMP_GT_OQ);\n const __m512 u = _mm512_mul_ps(b, b);\n const __m512 j = _mm512_fmadd_ps(\n _mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_set1_ps(0x1.0e4020p-7f), b,\n _mm512_set1_ps(0x1.573e2ep-5f)),\n u,\n _mm512_fmadd_ps(_mm512_set1_ps(0x1.555e66p-3f), b,\n _mm512_set1_ps(0x1.fffdb6p-2f))),\n u,\n _mm512_fmadd_ps(_mm512_set1_ps(0x1.ffffecp-1f), b, _mm512_set1_ps(1.0F)));\n const __m512 res = _mm512_scalef_ps(j, n);\n if (_mm512_kortestz(d, d))\n return res;\n const __m512 zero = _mm512_setzero_ps();\n const __m512 alt = _mm512_mask_blend_ps(\n _mm512_cmp_ps_mask(n, zero, _CMP_LE_OQ), _mm512_set1_ps(INFINITY), zero);\n return _mm512_mask_blend_ps(d, res, alt);\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static __m512 ggml_v_silu(__m512 x) {\n const __m512 one = _mm512_set1_ps(1);\n const __m512 zero = _mm512_setzero_ps();\n const __m512 neg_x = _mm512_sub_ps(zero, x);\n const __m512 exp_neg_x = ggml_v_expf(neg_x);\n const __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x);\n return _mm512_div_ps(x, one_plus_exp_neg_x);\n}\n\n#elif defined(__AVX2__) && defined(__FMA__)\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static __m256 ggml_v_expf(__m256 x) {\n const __m256 r = _mm256_set1_ps(0x1.8p23f);\n const __m256 z = _mm256_fmadd_ps(x, _mm256_set1_ps(0x1.715476p+0f), r);\n const __m256 n = _mm256_sub_ps(z, r);\n const __m256 b = _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.7f7d1cp-20f),\n _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.62e4p-1f), x));\n const __m256i e = _mm256_slli_epi32(_mm256_castps_si256(z), 23);\n const __m256 k = _mm256_castsi256_ps(\n _mm256_add_epi32(e, _mm256_castps_si256(_mm256_set1_ps(1))));\n const __m256i c = _mm256_castps_si256(\n _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n),\n _mm256_set1_ps(126), _CMP_GT_OQ));\n const __m256 u = _mm256_mul_ps(b, b);\n const __m256 j = _mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_set1_ps(0x1.0e4020p-7f), b,\n _mm256_set1_ps(0x1.573e2ep-5f)), u,\n _mm256_fmadd_ps(_mm256_set1_ps(0x1.555e66p-3f), b,\n _mm256_set1_ps(0x1.fffdb6p-2f))),\n u, _mm256_mul_ps(_mm256_set1_ps(0x1.ffffecp-1f), b));\n if (!_mm256_movemask_ps(_mm256_castsi256_ps(c)))\n return _mm256_fmadd_ps(j, k, k);\n const __m256i g = _mm256_and_si256(\n _mm256_castps_si256(_mm256_cmp_ps(n, _mm256_setzero_ps(), _CMP_LE_OQ)),\n _mm256_set1_epi32(0x82000000u));\n const __m256 s1 =\n _mm256_castsi256_ps(_mm256_add_epi32(g, _mm256_set1_epi32(0x7f000000u)));\n const __m256 s2 = _mm256_castsi256_ps(_mm256_sub_epi32(e, g));\n const __m256i d = _mm256_castps_si256(\n _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n),\n _mm256_set1_ps(192), _CMP_GT_OQ));\n return _mm256_or_ps(\n _mm256_and_ps(_mm256_castsi256_ps(d), _mm256_mul_ps(s1, s1)),\n _mm256_andnot_ps(\n _mm256_castsi256_ps(d),\n _mm256_or_ps(\n _mm256_and_ps(_mm256_castsi256_ps(c),\n _mm256_mul_ps(_mm256_fmadd_ps(s2, j, s2), s1)),\n _mm256_andnot_ps(_mm256_castsi256_ps(c), _mm256_fmadd_ps(k, j, k)))));\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static __m256 ggml_v_silu(__m256 x) {\n const __m256 one = _mm256_set1_ps(1);\n const __m256 zero = _mm256_setzero_ps();\n const __m256 neg_x = _mm256_sub_ps(zero, x);\n const __m256 exp_neg_x = ggml_v_expf(neg_x);\n const __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x);\n return _mm256_div_ps(x, one_plus_exp_neg_x);\n}\n\n#elif defined(__SSE2__) // __AVX2__ / __ARM_NEON\n\n#if defined(__FMA__)\n#define MADD128(x, y, z) _mm_fmadd_ps(x, y, z)\n#define NMADD128(x, y, z) _mm_fnmadd_ps(x, y, z)\n#else\n#define MADD128(x, y, z) _mm_add_ps(_mm_mul_ps(x, y), z)\n#define NMADD128(x, y, z) _mm_sub_ps(z, _mm_mul_ps(x, y))\n#endif\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static __m128 ggml_v_expf(__m128 x) {\n const __m128 r = _mm_set1_ps(0x1.8p23f);\n const __m128 z = MADD128(x, _mm_set1_ps(0x1.715476p+0f), r);\n const __m128 n = _mm_sub_ps(z, r);\n const __m128 b =\n NMADD128(n, _mm_set1_ps(0x1.7f7d1cp-20f), NMADD128(n, _mm_set1_ps(0x1.62e4p-1f), x));\n const __m128i e = _mm_slli_epi32(_mm_castps_si128(z), 23);\n const __m128 k = _mm_castsi128_ps(_mm_add_epi32(e, _mm_castps_si128(_mm_set1_ps(1))));\n const __m128i c =\n _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(126)));\n const __m128 u = _mm_mul_ps(b, b);\n const __m128 j =\n MADD128(MADD128(MADD128(_mm_set1_ps(0x1.0e4020p-7f), b, _mm_set1_ps(0x1.573e2ep-5f)), u,\n MADD128(_mm_set1_ps(0x1.555e66p-3f), b, _mm_set1_ps(0x1.fffdb6p-2f))),\n u, _mm_mul_ps(_mm_set1_ps(0x1.ffffecp-1f), b));\n if (!_mm_movemask_epi8(c))\n return MADD128(j, k, k);\n const __m128i g = _mm_and_si128(_mm_castps_si128(_mm_cmple_ps(n, _mm_setzero_ps())),\n _mm_set1_epi32(0x82000000u));\n const __m128 s1 = _mm_castsi128_ps(_mm_add_epi32(g, _mm_set1_epi32(0x7f000000u)));\n const __m128 s2 = _mm_castsi128_ps(_mm_sub_epi32(e, g));\n const __m128i d =\n _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(192)));\n return _mm_or_ps(\n _mm_and_ps(_mm_castsi128_ps(d), _mm_mul_ps(s1, s1)),\n _mm_andnot_ps(_mm_castsi128_ps(d),\n _mm_or_ps(_mm_and_ps(_mm_castsi128_ps(c), _mm_mul_ps(MADD128(s2, j, s2), s1)),\n _mm_andnot_ps(_mm_castsi128_ps(c), MADD128(k, j, k)))));\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static __m128 ggml_v_silu(__m128 x) {\n const __m128 one = _mm_set1_ps(1);\n const __m128 zero = _mm_setzero_ps();\n const __m128 neg_x = _mm_sub_ps(zero, x);\n const __m128 exp_neg_x = ggml_v_expf(neg_x);\n const __m128 one_plus_exp_neg_x = _mm_add_ps(one, exp_neg_x);\n return _mm_div_ps(x, one_plus_exp_neg_x);\n}\n\n#elif defined(__riscv_v_intrinsic)\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static vfloat32m2_t ggml_v_expf_m2(vfloat32m2_t x, int vl) {\n const vfloat32m2_t r = __riscv_vfmv_v_f_f32m2(0x1.8p23f, vl);\n#ifdef __riscv_xtheadvector\n // workaround for compiler bug (gcc 14.3.0: Error: unrecognized opcode `th.vmv1r.v v2,v4')\n vfloat32m2_t z = __riscv_vfadd_vf_f32m2(r, 0.0f, vl);\n z = __riscv_vfmacc_vf_f32m2(z, 0x1.715476p+0f, x, vl);\n#else\n const vfloat32m2_t z = __riscv_vfmacc_vf_f32m2(r, 0x1.715476p+0f, x, vl);\n#endif\n const vfloat32m2_t n = __riscv_vfsub_vv_f32m2(z, r, vl);\n const vfloat32m2_t b = __riscv_vfnmsac_vf_f32m2(__riscv_vfnmsac_vf_f32m2(x, 0x1.62e4p-1f, n, vl),\n 0x1.7f7d1cp-20f, n, vl);\n const vuint32m2_t e = __riscv_vsll_vx_u32m2(__riscv_vreinterpret_v_f32m2_u32m2(z), 23, vl);\n const vfloat32m2_t k = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vadd_vx_u32m2(e, 0x3f800000, vl)); // 1.0f\n const vbool16_t c = __riscv_vmfgt_vf_f32m2_b16(__riscv_vfabs_v_f32m2(n, vl), 126.0f, vl);\n const vfloat32m2_t u = __riscv_vfmul_vv_f32m2(b, b, vl);\n const vfloat32m2_t j = __riscv_vfmacc_vv_f32m2(\n __riscv_vfmul_vf_f32m2(b, 0x1.ffffecp-1f, vl),\n __riscv_vfmacc_vv_f32m2(\n __riscv_vfmacc_vf_f32m2(__riscv_vfmv_v_f_f32m2(0x1.fffdb6p-2f, vl), 0x1.555e66p-3f, b, vl),\n __riscv_vfmacc_vf_f32m2(__riscv_vfmv_v_f_f32m2(0x1.573e2ep-5f, vl), 0x1.0e4020p-7f, b, vl),\n u, vl), u, vl);\n if (!__riscv_vcpop_m_b16(c, vl))\n return __riscv_vfmacc_vv_f32m2(k, j, k, vl);\n const vbool16_t dm = __riscv_vmfle_vf_f32m2_b16(n, 0.0f, vl);\n const vuint32m2_t d = __riscv_vmerge_vxm_u32m2(__riscv_vmv_v_x_u32m2(0, vl), 0x82000000, dm, vl);\n const vfloat32m2_t s1 = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vadd_vx_u32m2(d, 0x7f000000, vl));\n const vfloat32m2_t s2 = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vsub_vv_u32m2(e, d, vl));\n const vfloat32m2_t r1 = __riscv_vmerge_vvm_f32m2(\n __riscv_vfmacc_vv_f32m2(k, k, j, vl),\n __riscv_vfmul_vv_f32m2(__riscv_vfmacc_vv_f32m2(s2, s2, j, vl), s1, vl),\n c, vl);\n return __riscv_vmerge_vvm_f32m2(\n r1, __riscv_vfmul_vv_f32m2(s1, s1, vl),\n __riscv_vmfgt_vf_f32m2_b16(__riscv_vfabs_v_f32m2(n, vl), 192.0f, vl),\n vl);\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static vfloat32m2_t ggml_v_silu_m2(vfloat32m2_t x, int vl) {\n const vfloat32m2_t neg_x = __riscv_vfneg_v_f32m2(x, vl);\n const vfloat32m2_t exp_neg_x = ggml_v_expf_m2(neg_x, vl);\n const vfloat32m2_t one_plus_exp_neg_x = __riscv_vfadd_vf_f32m2(exp_neg_x, 1.0f, vl);\n return __riscv_vfdiv_vv_f32m2(x, one_plus_exp_neg_x, vl);\n}\n\n#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __riscv_v_intrinsic\n\ninline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n for (int i = 0; i < n; ++i) {\n y[i] = ggml_silu_f16(x[i]);\n }\n}\n\ninline static float ggml_silu_backward_f32(float x, float dy) {\n const float s = 1.0f/(1.0f + expf(-x));\n return dy*s*(1.0f + x*(1.0f - s));\n}\n\ninline static ggml_fp16_t ggml_silu_backward_f16(ggml_fp16_t x, ggml_fp16_t dy) {\n const float v = GGML_CPU_FP16_TO_FP32(x);\n const float s = 1.0f/(1.0f + expf(-v));\n return GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(dy)*s*(1.0f + v*(1.0f - s)));\n}\n\ninline static void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {\n for (int i = 0; i < n; ++i) {\n dx[i] = ggml_silu_backward_f32(x[i], dy[i]);\n }\n}\n\ninline static void ggml_vec_silu_backward_f16(const int n, ggml_fp16_t * dx, const ggml_fp16_t * x, const ggml_fp16_t * dy) {\n for (int i = 0; i < n; ++i) {\n dx[i] = ggml_silu_backward_f16(x[i], dy[i]);\n }\n}\n\ninline static void ggml_vec_reglu_f32 (const int n, float * y, const float * x, const float * g) {\n for (int i = 0; i < n; ++i) {\n y[i] = (x[i] > 0.f) ? x[i] * g[i] : 0.f;\n }\n}\n\ninline static void ggml_vec_reglu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n for (int i = 0; i < n; ++i) {\n float v = GGML_CPU_FP16_TO_FP32(x[i]);\n y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? v * GGML_CPU_FP16_TO_FP32(g[i]) : 0.f);\n }\n}\n\n#ifdef GGML_GELU_FP16\ninline static void ggml_vec_geglu_f32(const int n, float * y, const float * x, const float * g) {\n uint16_t t;\n for (int i = 0; i < n; ++i) {\n if (x[i] <= -10.0f) {\n y[i] = 0.0f;\n } else if (x[i] >= 10.0f) {\n y[i] = x[i] * g[i];\n } else {\n ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);\n memcpy(&t, &fp16, sizeof(uint16_t));\n y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_f16[t]) * g[i];\n }\n }\n}\n#else\ninline static void ggml_vec_geglu_f32(const int n, float * y, const float * x, const float * g) {\n for (int i = 0; i < n; ++i) {\n y[i] = ggml_gelu_f32(x[i]) * g[i];\n }\n}\n#endif\n\ninline static void ggml_vec_geglu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n const uint16_t * i16 = (const uint16_t *) x;\n for (int i = 0; i < n; ++i) {\n float v = GGML_CPU_FP16_TO_FP32(g[i]);\n y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(ggml_table_gelu_f16[i16[i]]) * v);\n }\n}\n\nvoid ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * g);\n\ninline static void ggml_vec_swiglu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n for (int i = 0; i < n; ++i) {\n float xi = GGML_CPU_FP16_TO_FP32(x[i]);\n float gi = GGML_CPU_FP16_TO_FP32(g[i]);\n y[i] = GGML_CPU_FP32_TO_FP16((xi/(1.0f + expf(-xi))) * gi);\n }\n}\n\ninline static void ggml_vec_geglu_erf_f32(const int n, float * y, const float * x, const float * g) {\n for (int i = 0; i < n; ++i) {\n float xi = x[i];\n y[i] = 0.5f * xi * (1.0f + erff(xi*SQRT_2_INV)) * g[i];\n }\n}\n\ninline static void ggml_vec_geglu_erf_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n for (int i = 0; i < n; ++i) {\n float xi = GGML_CPU_FP16_TO_FP32(x[i]);\n float gi = GGML_CPU_FP16_TO_FP32(g[i]);\n y[i] = GGML_CPU_FP32_TO_FP16(0.5f * xi * (1.0f + erff(xi*SQRT_2_INV)) * gi);\n }\n}\n\n#ifdef GGML_GELU_QUICK_FP16\ninline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) {\n uint16_t t;\n for (int i = 0; i < n; ++i) {\n ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);\n memcpy(&t, &fp16, sizeof(uint16_t));\n y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_quick_f16[t]) * g[i];\n }\n}\n#else\ninline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) {\n for (int i = 0; i < n; ++i) {\n y[i] = ggml_gelu_quick_f32(x[i]) * g[i];\n }\n}\n#endif\n\ninline static void ggml_vec_geglu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n const uint16_t * i16 = (const uint16_t *) x;\n for (int i = 0; i < n; ++i) {\n float v = GGML_CPU_FP16_TO_FP32(g[i]);\n y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(ggml_table_gelu_quick_f16[i16[i]]) * v);\n }\n}\n\ninline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {\n#ifndef GGML_USE_ACCELERATE\n ggml_float sum = 0.0;\n for (int i = 0; i < n; ++i) {\n sum += (ggml_float)x[i];\n }\n *s = (float)sum;\n#else\n vDSP_sve(x, 1, s, n);\n#endif\n}\n\ninline static void ggml_vec_cumsum_f32(const int n, float * y, const float * x) {\n for (int i = 0; i < n; ++i) {\n if (i == 0) {\n y[i] = x[i];\n } else {\n y[i] = y[i - 1] + x[i];\n }\n }\n}\n\ninline static void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {\n ggml_float sum = 0.0;\n for (int i = 0; i < n; ++i) {\n sum += (ggml_float)x[i];\n }\n *s = sum;\n}\n\ninline static void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {\n float sum = 0.0f;\n for (int i = 0; i < n; ++i) {\n sum += GGML_CPU_FP16_TO_FP32(x[i]);\n }\n *s = sum;\n}\n\ninline static void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16_t * x) {\n float sum = 0.0f;\n for (int i = 0; i < n; ++i) {\n sum += GGML_BF16_TO_FP32(x[i]);\n }\n *s = sum;\n}\n\ninline static void ggml_vec_max_f32(const int n, float * s, const float * x) {\n#ifndef GGML_USE_ACCELERATE\n float max = -INFINITY;\n for (int i = 0; i < n; ++i) {\n max = MAX(max, x[i]);\n }\n *s = max;\n#else\n vDSP_maxv(x, 1, s, n);\n#endif\n}\n\ninline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {\n ggml_vec_norm_f32(n, s, x);\n *s = 1.f/(*s);\n}\n\ninline static void ggml_vec_argmax_f32(const int n, int * s, const float * x) {\n float max = -INFINITY;\n int idx = 0;\n for (int i = 0; i < n; ++i) {\n max = MAX(max, x[i]);\n if (max == x[i]) { idx = i; }\n }\n *s = idx;\n}\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-cuda/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.18) # for CMAKE_CUDA_ARCHITECTURES\n\nfind_package(CUDAToolkit)\n\nif (CUDAToolkit_FOUND)\n message(STATUS \"CUDA Toolkit found\")\n\n if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)\n # native == GPUs available at build time\n # 50 == Maxwell, lowest CUDA 12 standard\n # 60 == P100, FP16 CUDA intrinsics\n # 61 == Pascal, __dp4a instruction (per-byte integer dot product)\n # 70 == V100, FP16 tensor cores\n # 75 == Turing, int8 tensor cores\n # 80 == Ampere, asynchronous data loading, faster tensor core instructions\n # 86 == RTX 3000, needs CUDA v11.1\n # 89 == RTX 4000, needs CUDA v11.8\n # 120 == Blackwell, needs CUDA v12.8, FP4 tensor cores\n #\n # XX-virtual == compile CUDA code as PTX, do JIT compilation to binary code on first run\n # XX-real == compile CUDA code as device code for this specific architecture\n # no suffix == compile as both PTX and device code\n #\n # The default behavior for a non-native is to build virtual architectures as needed to cover all features needed\n # for best performance and to also build real architectures for the most commonly used GPUs.\n if (GGML_NATIVE AND CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"11.6\" AND CMAKE_VERSION VERSION_GREATER_EQUAL \"3.24\")\n set(CMAKE_CUDA_ARCHITECTURES \"native\")\n else()\n if (CUDAToolkit_VERSION VERSION_LESS \"13\")\n list(APPEND CMAKE_CUDA_ARCHITECTURES 50-virtual 61-virtual 70-virtual)\n endif ()\n\n list(APPEND CMAKE_CUDA_ARCHITECTURES 75-virtual 80-virtual 86-real)\n\n if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"11.8\")\n list(APPEND CMAKE_CUDA_ARCHITECTURES 89-real)\n endif()\n\n if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"12.8\")\n # The CUDA architecture 120f-virtual would in principle work for Blackwell support\n # but the newly added \"f\" suffix conflicted with a preexising regex for validating CUDA architectures in CMake.\n # So either a recent CMake version or one with the backported fix is needed.\n # The following versions should work:\n # - CMake >= v3.31.8 && CMake < v4.0.0\n # - CMake >= v4.0.2\n # This is NOT documented in the CMake release notes,\n # check Modules/Internal/CMakeCUDAArchitecturesValidate.cmake in the CMake git repository instead.\n # However, the architectures 120a-real and 121a-real should work with basically any CMake version and\n # until the release of e.g. Rubin there is no benefit to shipping virtual architectures for Blackwell.\n list(APPEND CMAKE_CUDA_ARCHITECTURES 120a-real)\n endif()\n if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"12.9\")\n list(APPEND CMAKE_CUDA_ARCHITECTURES 121a-real)\n endif()\n endif()\n endif()\n\n enable_language(CUDA)\n\n # TODO: Remove once CCCL 3.2 has been released and bundled with CUDA Toolkit\n if (GGML_CUDA_CUB_3DOT2)\n include(FetchContent)\n\n FetchContent_Declare(\n CCCL\n GIT_REPOSITORY https://github.com/nvidia/cccl.git\n GIT_TAG v3.2.0\n GIT_SHALLOW TRUE\n )\n\n FetchContent_MakeAvailable(CCCL)\n endif()\n\n # Replace any plain 12X CUDA architectures with their \"architecture-specific\" equivalents 12Xa.\n # 12X is forwards-compatible, 12Xa is not.\n # Notably the Blackwell FP4 tensor core instructions are not forwards compatible and therefore need 12Xa.\n # But while 12X vs. 12Xa can be checked in device code there is (to my knowledge) no easy way to do the same check in host code.\n # So for now just replace all instances of 12X with 12Xa, this should be fine until Rubin is released.\n foreach(ARCHS IN ITEMS CMAKE_CUDA_ARCHITECTURES CMAKE_CUDA_ARCHITECTURES_NATIVE)\n set(FIXED_ARCHS \"\")\n foreach(ARCH IN LISTS ${ARCHS})\n if (ARCH MATCHES \"^12[0-9](-real|-virtual)?$\")\n string(REGEX REPLACE \"^(12[0-9])((-real|-virtual)?)$\" \"\\\\1a\\\\2\" FIXED_ARCH ${ARCH})\n message(STATUS \"Replacing ${ARCH} in ${ARCHS} with ${FIXED_ARCH}\")\n list(APPEND FIXED_ARCHS \"${FIXED_ARCH}\")\n else()\n list(APPEND FIXED_ARCHS \"${ARCH}\")\n endif()\n endforeach()\n set(${ARCHS} ${FIXED_ARCHS})\n endforeach()\n\n # If we try to compile a \"native\" build it will use the 12X architectures and fail.\n # So we should instead use the native architectures as determined by CMake after replacing 12X with 12Xa.\n # But if at the time of the build no GPUs are connected at all CMAKE_CUDA_ARCHITECTURES will contain garbage that we should not use.\n if (CMAKE_CUDA_ARCHITECTURES STREQUAL \"native\" AND CMAKE_CUDA_ARCHITECTURES_NATIVE MATCHES \"^[0-9]+(a|f)?(-real|-virtual)?(;[0-9]+(a|f)?(-real|-virtual)?|;)*$\")\n set(CMAKE_CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES_NATIVE})\n endif()\n message(STATUS \"Using CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} CMAKE_CUDA_ARCHITECTURES_NATIVE=${CMAKE_CUDA_ARCHITECTURES_NATIVE}\")\n\n file(GLOB GGML_HEADERS_CUDA \"*.cuh\")\n list(APPEND GGML_HEADERS_CUDA \"../../include/ggml-cuda.h\")\n\n file(GLOB GGML_SOURCES_CUDA \"*.cu\")\n file(GLOB SRCS \"template-instances/fattn-tile*.cu\")\n list(APPEND GGML_SOURCES_CUDA ${SRCS})\n file(GLOB SRCS \"template-instances/fattn-mma*.cu\")\n list(APPEND GGML_SOURCES_CUDA ${SRCS})\n file(GLOB SRCS \"template-instances/mmq*.cu\")\n list(APPEND GGML_SOURCES_CUDA ${SRCS})\n file(GLOB SRCS \"template-instances/mmf*.cu\")\n list(APPEND GGML_SOURCES_CUDA ${SRCS})\n\n if (GGML_CUDA_FA_ALL_QUANTS)\n file(GLOB SRCS \"template-instances/fattn-vec*.cu\")\n list(APPEND GGML_SOURCES_CUDA ${SRCS})\n add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS)\n else()\n file(GLOB SRCS \"template-instances/fattn-vec*q4_0-q4_0.cu\")\n list(APPEND GGML_SOURCES_CUDA ${SRCS})\n file(GLOB SRCS \"template-instances/fattn-vec*q8_0-q8_0.cu\")\n list(APPEND GGML_SOURCES_CUDA ${SRCS})\n file(GLOB SRCS \"template-instances/fattn-vec*f16-f16.cu\")\n list(APPEND GGML_SOURCES_CUDA ${SRCS})\n endif()\n\n ggml_add_backend_library(ggml-cuda\n ${GGML_HEADERS_CUDA}\n ${GGML_SOURCES_CUDA}\n )\n\n add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})\n\n if (GGML_CUDA_GRAPHS)\n add_compile_definitions(GGML_CUDA_USE_GRAPHS)\n endif()\n\n if (GGML_CUDA_FORCE_MMQ)\n add_compile_definitions(GGML_CUDA_FORCE_MMQ)\n endif()\n\n if (GGML_CUDA_FORCE_CUBLAS)\n add_compile_definitions(GGML_CUDA_FORCE_CUBLAS)\n endif()\n\n if (GGML_CUDA_NO_VMM)\n add_compile_definitions(GGML_CUDA_NO_VMM)\n endif()\n\n if (NOT GGML_CUDA_FA)\n add_compile_definitions(GGML_CUDA_NO_FA)\n endif()\n\n if (GGML_CUDA_NO_PEER_COPY)\n add_compile_definitions(GGML_CUDA_NO_PEER_COPY)\n endif()\n\n if (GGML_STATIC)\n if (WIN32)\n # As of 12.3.1 CUDA Toolkit for Windows does not offer a static cublas library\n target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas)\n else ()\n if (GGML_CUDA_CUB_3DOT2)\n target_link_libraries(ggml-cuda PRIVATE CCCL::CCCL)\n endif()\n if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"10.1\")\n target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)\n else()\n target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas_static)\n endif()\n endif()\n else()\n if (GGML_CUDA_CUB_3DOT2)\n target_link_libraries(ggml-cuda PRIVATE CCCL::CCCL)\n endif()\n target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas)\n endif()\n\n if (GGML_CUDA_NO_VMM)\n # No VMM requested, no need to link directly with the cuda driver lib (libcuda.so)\n else()\n target_link_libraries(ggml-cuda PRIVATE CUDA::cuda_driver)\n endif()\n\n set(CUDA_CXX_FLAGS \"\")\n\n set(CUDA_FLAGS -use_fast_math -extended-lambda)\n\n if (GGML_CUDA_DEBUG)\n list(APPEND CUDA_FLAGS -lineinfo)\n add_compile_definitions(GGML_CUDA_DEBUG)\n endif()\n\n if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"12.8\")\n # Options are:\n # - none (not recommended)\n # - speed (nvcc's default)\n # - balance\n # - size\n list(APPEND CUDA_FLAGS -compress-mode=${GGML_CUDA_COMPRESSION_MODE})\n endif()\n\n if (GGML_FATAL_WARNINGS)\n list(APPEND CUDA_FLAGS -Werror all-warnings)\n endif()\n\n if (GGML_ALL_WARNINGS AND NOT MSVC)\n set(NVCC_CMD ${CMAKE_CUDA_COMPILER} .c)\n if (NOT CMAKE_CUDA_HOST_COMPILER STREQUAL \"\")\n list(APPEND NVCC_CMD -ccbin ${CMAKE_CUDA_HOST_COMPILER})\n endif()\n\n execute_process(\n COMMAND ${NVCC_CMD} -Xcompiler --version\n OUTPUT_VARIABLE CUDA_CCFULLVER\n ERROR_QUIET\n )\n\n if (NOT CUDA_CCFULLVER MATCHES clang)\n set(CUDA_CCID \"GNU\")\n execute_process(\n COMMAND ${NVCC_CMD} -Xcompiler \"-dumpfullversion -dumpversion\"\n OUTPUT_VARIABLE CUDA_CCVER\n ERROR_QUIET\n OUTPUT_STRIP_TRAILING_WHITESPACE\n )\n else()\n if (CUDA_CCFULLVER MATCHES Apple)\n set(CUDA_CCID \"AppleClang\")\n else()\n set(CUDA_CCID \"Clang\")\n endif()\n string(REGEX REPLACE \"^.* version ([0-9.]*).*$\" \"\\\\1\" CUDA_CCVER ${CUDA_CCFULLVER})\n endif()\n\n message(STATUS \"CUDA host compiler is ${CUDA_CCID} ${CUDA_CCVER}\")\n\n ggml_get_flags(${CUDA_CCID} ${CUDA_CCVER})\n list(APPEND CUDA_CXX_FLAGS ${CXX_FLAGS} ${GF_CXX_FLAGS}) # This is passed to -Xcompiler later\n endif()\n\n if (NOT MSVC)\n list(APPEND CUDA_CXX_FLAGS -Wno-pedantic)\n else()\n # CCCL 3.2 onwards will require a cpp-standard-compliant preprocessor for MSVC\n # https://github.com/NVIDIA/cccl/pull/6827\n list(APPEND CUDA_CXX_FLAGS /Zc:preprocessor)\n endif()\n\n list(JOIN CUDA_CXX_FLAGS \" \" CUDA_CXX_FLAGS_JOINED) # pass host compiler flags as a single argument\n\n if (NOT CUDA_CXX_FLAGS_JOINED STREQUAL \"\")\n list(APPEND CUDA_FLAGS -Xcompiler ${CUDA_CXX_FLAGS_JOINED})\n endif()\n\n target_compile_options(ggml-cuda PRIVATE \"$<$:${CUDA_FLAGS}>\")\nelse()\n message(FATAL_ERROR \"CUDA Toolkit not found\")\nendif()\n" }, { "path": "src/ggml-cuda/acc.cu", "content": "#include \"acc.cuh\"\n\nstatic __global__ void acc_f32(const float * x, const float * y, float * dst, const int64_t ne,\n const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,\n const int64_t s11, const int64_t s12, const int64_t s13, const int64_t offset) {\n const int64_t i = blockDim.x * blockIdx.x + threadIdx.x;\n\n if (i >= ne) {\n return;\n }\n\n int64_t src1_idx = i - offset;\n\n int64_t tmp = src1_idx;\n const int64_t i13 = tmp / s13;\n tmp -= i13 * s13;\n const int64_t i12 = tmp / s12;\n tmp -= i12 * s12;\n const int64_t i11 = tmp / s11;\n tmp -= i11 * s11;\n const int64_t i10 = tmp;\n\n float val = x[i];\n if (src1_idx >= 0 && i10 < ne10 && i11 < ne11 && i12 < ne12 && i13 < ne13) {\n val += y[((i13*ne12 + i12) * ne11 + i11) * ne10 + i10];\n }\n dst[i] = val;\n}\n\nstatic void acc_f32_cuda(const float * x, const float * y, float * dst, const int64_t n_elements,\n const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,\n const int64_t s1, const int64_t s2, const int64_t s3, const int64_t offset, cudaStream_t stream) {\n const int num_blocks = (n_elements + CUDA_ACC_BLOCK_SIZE - 1) / CUDA_ACC_BLOCK_SIZE;\n acc_f32<<>>(x, y, dst, n_elements, ne10, ne11, ne12, ne13, s1, s2, s3, offset);\n}\n\nvoid ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n const float * src0_d = (const float *) src0->data;\n const float * src1_d = (const float *) src1->data;\n float * dst_d = (float *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous(src1));\n GGML_ASSERT(dst->nb[0] == ggml_element_size(dst));\n GGML_ASSERT(ggml_is_contiguously_allocated(dst));\n\n const int64_t s1 = dst->op_params[0] / sizeof(float);\n const int64_t s2 = dst->op_params[1] / sizeof(float);\n const int64_t s3 = dst->op_params[2] / sizeof(float);\n const int64_t offset = dst->op_params[3] / sizeof(float);\n\n acc_f32_cuda(src0_d, src1_d, dst_d, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], s1, s2, s3, offset, stream);\n}\n" }, { "path": "src/ggml-cuda/acc.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_ACC_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/add-id.cu", "content": "#include \"add-id.cuh\"\n\nstatic __global__ void add_id_kernel(\n const float * src0, const float * src1, const int32_t * src2, float * dst,\n int64_t ne0, int64_t ne1,\n size_t nb01, size_t nb02,\n size_t nb11,\n size_t nb21\n ) {\n\n const int64_t i1 = blockIdx.x;\n const int64_t i2 = blockIdx.y;\n\n const int i11 = *(const int32_t *) ((const char *) src2 + i1*sizeof(int32_t) + i2*nb21);\n\n const size_t nb1 = ne0 * sizeof(float);\n const size_t nb2 = ne1 * nb1;\n\n float * dst_row = (float *)((char *)dst + i1*nb1 + i2*nb2);\n const float * src0_row = (const float *)((const char *)src0 + i1*nb01 + i2*nb02);\n const float * src1_row = (const float *)((const char *)src1 + i11*nb11);\n\n for (int64_t i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) {\n dst_row[i0] = src0_row[i0] + src1_row[i0];\n }\n}\n\nvoid ggml_cuda_op_add_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const ggml_tensor * src2 = dst->src[2];\n\n GGML_TENSOR_TERNARY_OP_LOCALS\n\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(src2->type == GGML_TYPE_I32);\n\n GGML_ASSERT(nb00 == sizeof(float));\n GGML_ASSERT(nb10 == sizeof(float));\n GGML_ASSERT(nb20 == sizeof(int32_t));\n\n const float * src0_d = (const float *)src0->data;\n const float * src1_d = (const float *)src1->data;\n const int32_t * src2_d = (const int32_t *)src2->data;\n float * dst_d = (float *)dst->data;\n\n int threads = std::min((int)ne00, 768); // cols\n dim3 blocks(ne01, ne02); // n_experts_used, n_tokens\n add_id_kernel<<>>(\n src0_d, src1_d, src2_d, dst_d,\n ne0, ne1,\n nb01, nb02,\n nb11,\n nb21\n );\n}\n" }, { "path": "src/ggml-cuda/add-id.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_add_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/arange.cu", "content": "#include \"arange.cuh\"\n\nstatic __global__ void arange_f32(float * dst, const int ne0, const float start, const float step) {\n // blockIDx.x: idx of ne0 / BLOCK_SIZE\n int nidx = threadIdx.x + blockIdx.x * blockDim.x;\n if (nidx >= ne0) {\n return;\n }\n dst[nidx] = start + step * nidx;\n}\n\nstatic void arange_f32_cuda(float * dst, const int ne0, const float start, const float step, cudaStream_t stream) {\n int num_blocks = (ne0 + CUDA_ARANGE_BLOCK_SIZE - 1) / CUDA_ARANGE_BLOCK_SIZE;\n arange_f32<<>>(dst, ne0, start, step);\n}\n\nvoid ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n float start;\n float stop;\n float step;\n memcpy(&start, (float *)dst->op_params + 0, sizeof(float));\n memcpy(&stop, (float *)dst->op_params + 1, sizeof(float));\n memcpy(&step, (float *)dst->op_params + 2, sizeof(float));\n\n int64_t steps = (int64_t)ceil((stop - start) / step);\n GGML_ASSERT(ggml_nelements(dst) == steps);\n\n arange_f32_cuda(dst_d, dst->ne[0], start, step, stream);\n}\n" }, { "path": "src/ggml-cuda/arange.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_ARANGE_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/argmax.cu", "content": "#include \n#include \n\n#include \"argmax.cuh\"\n#include \"common.cuh\"\n#include \"sum.cuh\"\n\nstatic __global__ void argmax_f32(const float * __restrict__ x, int32_t * __restrict__ dst, const int64_t ncols) {\n const int64_t row = blockIdx.x;\n\n float maxval = -FLT_MAX;\n int argmax = -1;\n const float * rowx = x + row * ncols;\n\n for (int32_t col = threadIdx.x; col < ncols; col += blockDim.x) {\n const float val = rowx[col];\n if (val > maxval) {\n maxval = val;\n argmax = col;\n }\n }\n\n#pragma unroll\n for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) {\n const float val = __shfl_xor_sync(0xFFFFFFFF, maxval, offset, WARP_SIZE);\n const int col = __shfl_xor_sync(0xFFFFFFFF, argmax, offset, WARP_SIZE);\n if (val > maxval) {\n maxval = val;\n argmax = col;\n }\n }\n\n const int n_warps = blockDim.x / WARP_SIZE;\n const int lane_id = threadIdx.x % WARP_SIZE;\n const int warp_id = threadIdx.x / WARP_SIZE;\n if (n_warps > 1) {\n constexpr int max_warps = 1024 / WARP_SIZE;\n __shared__ float shared_maxval[max_warps];\n __shared__ int shared_argmax[max_warps];\n if (lane_id == 0) {\n shared_maxval[warp_id] = maxval;\n shared_argmax[warp_id] = argmax;\n }\n\n __syncthreads();\n\n if (warp_id == 0) {\n if (lane_id < n_warps) {\n maxval = shared_maxval[lane_id];\n argmax = shared_argmax[lane_id];\n }\n#pragma unroll\n for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) {\n const float val = __shfl_xor_sync(0xFFFFFFFF, maxval, offset, WARP_SIZE);\n const int col = __shfl_xor_sync(0xFFFFFFFF, argmax, offset, WARP_SIZE);\n if (val > maxval) {\n maxval = val;\n argmax = col;\n }\n }\n }\n }\n\n if (warp_id == 0 && lane_id == 0) {\n dst[row] = argmax;\n }\n}\n\nvoid ggml_cuda_argmax(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_I32);\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n const int64_t ne00 = src0->ne[0];\n const int64_t nrows = ggml_nrows(src0);\n\n const float * src0_d = (const float *) src0->data;\n int32_t * dst_d = (int32_t *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n const int64_t num_blocks = nrows;\n const int64_t num_threads = std::min(1024, (ne00 + WARP_SIZE - 1) / WARP_SIZE * WARP_SIZE);\n const dim3 blocks_dim(num_threads, 1, 1);\n const dim3 blocks_num(num_blocks, 1, 1);\n\n argmax_f32<<>>(src0_d, dst_d, ne00);\n}\n" }, { "path": "src/ggml-cuda/argmax.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_argmax(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/argsort.cu", "content": "#include \"argsort.cuh\"\n\n#ifdef GGML_CUDA_USE_CUB\n# include \n# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1)\n# define STRIDED_ITERATOR_AVAILABLE\n# endif\nusing namespace cub;\n#endif // GGML_CUDA_USE_CUB\n\nstatic __global__ void init_indices(int * indices, const int ncols, const int nrows) {\n const int col = blockIdx.x * blockDim.x + threadIdx.x;\n const int row = blockIdx.y;\n\n if (col < ncols && row < nrows) {\n indices[row * ncols + col] = col;\n }\n}\n\n#ifndef STRIDED_ITERATOR_AVAILABLE\nstatic __global__ void init_offsets(int * offsets, const int ncols, const int nrows) {\n const int idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (idx <= nrows) {\n offsets[idx] = idx * ncols;\n }\n}\n#endif // STRIDED_ITERATOR_AVAILABLE\n\n#ifdef GGML_CUDA_USE_CUB\nvoid argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool,\n const float * x,\n int * dst,\n const int ncols,\n const int nrows,\n ggml_sort_order order,\n cudaStream_t stream) {\n ggml_cuda_pool_alloc temp_indices_alloc(pool, ncols * nrows);\n ggml_cuda_pool_alloc temp_keys_alloc(pool, ncols * nrows);\n\n int * temp_indices = temp_indices_alloc.get();\n float * temp_keys = temp_keys_alloc.get();\n\n static const int block_size = 256;\n const dim3 grid_size((ncols + block_size - 1) / block_size, nrows);\n init_indices<<>>(temp_indices, ncols, nrows);\n\n#ifdef STRIDED_ITERATOR_AVAILABLE\n auto offset_iterator = cuda::make_strided_iterator(cuda::make_counting_iterator(0), ncols);\n#else\n ggml_cuda_pool_alloc offsets_alloc(pool, nrows + 1);\n int * offset_iterator = offsets_alloc.get();\n const dim3 offset_grid((nrows + block_size - 1) / block_size);\n init_offsets<<>>(offset_iterator, ncols, nrows);\n#endif\n CUDA_CHECK(cudaMemcpyAsync(temp_keys, x, ncols * nrows * sizeof(float), cudaMemcpyDeviceToDevice, stream));\n\n size_t temp_storage_bytes = 0;\n\n if (order == GGML_SORT_ORDER_ASC) {\n if (nrows == 1) {\n DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, temp_keys, // keys (in-place)\n temp_indices, dst, // values (indices)\n ncols, 0, sizeof(float) * 8, stream);\n } else {\n DeviceSegmentedSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, temp_keys, // keys (in-place)\n temp_indices, dst, // values (indices)\n ncols * nrows, nrows, // num items, num segments\n offset_iterator, offset_iterator + 1, stream);\n }\n } else {\n if (nrows == 1) {\n DeviceRadixSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, temp_keys, // keys (in-place)\n temp_indices, dst, // values (indices)\n ncols, 0, sizeof(float) * 8, stream);\n } else {\n DeviceSegmentedSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, temp_keys, temp_indices,\n dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1,\n stream);\n }\n }\n\n ggml_cuda_pool_alloc temp_storage_alloc(pool, temp_storage_bytes);\n void * d_temp_storage = temp_storage_alloc.get();\n\n if (order == GGML_SORT_ORDER_ASC) {\n if (nrows == 1) {\n DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, // keys (in-place)\n temp_indices, dst, // values (indices)\n ncols, 0, sizeof(float) * 8, stream);\n } else {\n DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, temp_indices, dst,\n ncols * nrows, nrows, offset_iterator, offset_iterator + 1, stream);\n }\n } else {\n if (nrows == 1) {\n DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, // keys (in-place)\n temp_indices, dst, // values (indices)\n ncols, 0, sizeof(float) * 8, stream);\n } else {\n DeviceSegmentedSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys,\n temp_indices, dst, ncols * nrows, nrows, offset_iterator,\n offset_iterator + 1, stream);\n }\n }\n}\n#endif // GGML_CUDA_USE_CUB\n\n// Bitonic sort implementation\ntemplate\nstatic inline __device__ void ggml_cuda_swap(T & a, T & b) {\n T tmp = a;\n a = b;\n b = tmp;\n}\n\ntemplate\nstatic __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ncols_pad) {\n // bitonic sort\n int col = threadIdx.x;\n int row = blockIdx.x;\n\n if (col >= ncols_pad) {\n return;\n }\n\n const float * x_row = x + row * ncols;\n extern __shared__ int dst_row[];\n\n // initialize indices\n dst_row[col] = col;\n\n __syncthreads();\n\n for (int k = 2; k <= ncols_pad; k *= 2) {\n for (int j = k / 2; j > 0; j /= 2) {\n int ixj = col ^ j;\n if (ixj > col) {\n if ((col & k) == 0) {\n if (dst_row[col] >= ncols ||\n (dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ?\n x_row[dst_row[col]] > x_row[dst_row[ixj]] :\n x_row[dst_row[col]] < x_row[dst_row[ixj]]))\n ) {\n ggml_cuda_swap(dst_row[col], dst_row[ixj]);\n }\n } else {\n if (dst_row[ixj] >= ncols ||\n (dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ?\n x_row[dst_row[col]] < x_row[dst_row[ixj]] :\n x_row[dst_row[col]] > x_row[dst_row[ixj]]))\n ) {\n ggml_cuda_swap(dst_row[col], dst_row[ixj]);\n }\n }\n }\n __syncthreads();\n }\n }\n\n // copy the result to dst without the padding\n if (col < ncols) {\n dst[row * ncols + col] = dst_row[col];\n }\n}\n\nstatic int next_power_of_2(int x) {\n int n = 1;\n while (n < x) {\n n *= 2;\n }\n return n;\n}\n\nvoid argsort_f32_i32_cuda_bitonic(const float * x,\n int * dst,\n const int ncols,\n const int nrows,\n ggml_sort_order order,\n cudaStream_t stream) {\n // bitonic sort requires ncols to be power of 2\n const int ncols_pad = next_power_of_2(ncols);\n\n const dim3 block_dims(ncols_pad, 1, 1);\n const dim3 block_nums(nrows, 1, 1);\n const size_t shared_mem = ncols_pad * sizeof(int);\n\n // FIXME: this limit could be raised by ~2-4x on Ampere or newer\n GGML_ASSERT(shared_mem <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);\n\n if (order == GGML_SORT_ORDER_ASC) {\n k_argsort_f32_i32\n <<>>(x, dst, ncols, ncols_pad);\n } else if (order == GGML_SORT_ORDER_DESC) {\n k_argsort_f32_i32\n <<>>(x, dst, ncols, ncols_pad);\n } else {\n GGML_ABORT(\"fatal error\");\n }\n}\n\nvoid ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *)src0->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_I32);\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n const int64_t ncols = src0->ne[0];\n const int64_t nrows = ggml_nrows(src0);\n\n enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];\n\n#ifdef GGML_CUDA_USE_CUB\n const int ncols_pad = next_power_of_2(ncols);\n const size_t shared_mem = ncols_pad * sizeof(int);\n const size_t max_shared_mem = ggml_cuda_info().devices[ggml_cuda_get_device()].smpb;\n\n if (shared_mem > max_shared_mem || ncols > 1024) {\n ggml_cuda_pool & pool = ctx.pool();\n argsort_f32_i32_cuda_cub(pool, src0_d, (int *) dst_d, ncols, nrows, order, stream);\n } else {\n argsort_f32_i32_cuda_bitonic(src0_d, (int *) dst_d, ncols, nrows, order, stream);\n }\n#else\n argsort_f32_i32_cuda_bitonic(src0_d, (int *) dst_d, ncols, nrows, order, stream);\n#endif\n}\n" }, { "path": "src/ggml-cuda/argsort.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\n#ifdef GGML_CUDA_USE_CUB\nvoid argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool,\n const float * x,\n int * dst,\n const int ncols,\n const int nrows,\n ggml_sort_order order,\n cudaStream_t stream);\n#endif // GGML_CUDA_USE_CUB\nvoid argsort_f32_i32_cuda_bitonic(const float * x,\n int * dst,\n const int ncols,\n const int nrows,\n ggml_sort_order order,\n cudaStream_t stream);\n" }, { "path": "src/ggml-cuda/binbcast.cu", "content": "#include \"binbcast.cuh\"\n#include \n#include \n\nstatic __device__ __forceinline__ float op_repeat(const float a, const float b) {\n return b;\n GGML_UNUSED(a);\n}\n\nstatic __device__ __forceinline__ float op_add(const float a, const float b) {\n return a + b;\n}\n\nstatic __device__ __forceinline__ float op_sub(const float a, const float b) {\n return a - b;\n}\n\nstatic __device__ __forceinline__ float op_mul(const float a, const float b) {\n return a * b;\n}\n\nstatic __device__ __forceinline__ float op_div(const float a, const float b) {\n return a / b;\n}\n\ntemplate \nstatic __global__ void k_bin_bcast(const src0_t * src0,\n const src1_t * src1,\n dst_t * dst,\n const int ne0,\n const int ne1,\n const int ne2,\n const uint3 ne3,\n const uint3 ne10,\n const uint3 ne11,\n const uint3 ne12,\n const uint3 ne13,\n /*const int s0,*/\n const int s1,\n const int s2,\n const int s3,\n const int s00,\n const int s01,\n const int s02,\n const int s03,\n const int s10,\n const int s11,\n const int s12,\n const int s13,\n src1_ptrs... src1s) {\n const uint32_t i0s = blockDim.x * blockIdx.x + threadIdx.x;\n const uint32_t i1 = (blockDim.y * blockIdx.y + threadIdx.y);\n const uint32_t i2 = fastdiv((blockDim.z * blockIdx.z + threadIdx.z), ne3);\n const uint32_t i3 = (blockDim.z * blockIdx.z + threadIdx.z) - (i2 * ne3.z);\n\n if (i0s >= (uint32_t)ne0 || i1 >= (uint32_t)ne1 || i2 >= (uint32_t)ne2 || i3 >= ne3.z) {\n return;\n }\n\n const uint32_t i11 = fastmodulo(i1, ne11);\n const uint32_t i12 = fastmodulo(i2, ne12);\n const uint32_t i13 = fastmodulo(i3, ne13);\n\n const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;\n const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;\n const size_t i_dst = i3*s3 + i2*s2 + i1*s1;\n\n const src0_t * src0_row = src0 ? (src0 + i_src0) : nullptr;\n dst_t * dst_row = dst + i_dst;\n\n for (int i0 = i0s; i0 < ne0; i0 += blockDim.x * gridDim.x) {\n const uint32_t i10 = fastmodulo(i0, ne10);\n\n float result = src0_row ? (float) src0_row[i0*s00] : 0.0f;\n if constexpr (sizeof...(src1_ptrs) > 0) {\n result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10*s10])));\n } else {\n result = bin_op(result, (float)src1[i_src1 + i10*s10]);\n }\n\n dst_row[i0] = (dst_t) result;\n }\n}\n\ntemplate \nstatic __global__ void k_bin_bcast_unravel(const src0_t * src0,\n const src1_t * src1,\n dst_t * dst,\n const uint3 ne0,\n const uint3 ne1,\n const uint3 ne2,\n const uint32_t ne3,\n const uint3 prod_012,\n const uint3 prod_01,\n const uint3 ne10,\n const uint3 ne11,\n const uint3 ne12,\n const uint3 ne13,\n /*const int s0,*/\n const int s1,\n const int s2,\n const int s3,\n const int s00,\n const int s01,\n const int s02,\n const int s03,\n const int s10,\n const int s11,\n const int s12,\n const int s13,\n src1_ptrs... src1s) {\n const int i = blockDim.x*blockIdx.x + threadIdx.x;\n\n const uint32_t i3 = fastdiv(i, prod_012);\n const uint32_t i2 = fastdiv(i - i3 * prod_012.z, prod_01);\n const uint32_t i1 = fastdiv(i - i3 * prod_012.z - i2 * prod_01.z, ne0);\n const uint32_t i0 = i - i3 * prod_012.z - i2 * prod_01.z - i1 * ne0.z;\n\n if (i0 >= ne0.z || i1 >= ne1.z || i2 >= ne2.z || i3 >= ne3) {\n return;\n }\n\n const int i11 = fastmodulo(i1, ne11);\n const int i12 = fastmodulo(i2, ne12);\n const int i13 = fastmodulo(i3, ne13);\n\n const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;\n const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;\n const size_t i_dst = i3*s3 + i2*s2 + i1*s1;\n\n const src0_t * src0_row = src0 ? (src0 + i_src0) : nullptr;\n dst_t * dst_row = dst + i_dst;\n\n const int i10 = fastmodulo(i0, ne10);\n\n float result = src0_row ? (float) src0_row[i0*s00] : 0.0f;\n if constexpr (sizeof...(src1_ptrs) > 0) {\n result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10*s10])));\n } else {\n result = bin_op(result, (float)src1[i_src1 + i10*s10]);\n }\n\n dst_row[i0] = (dst_t) result;\n}\n\ntemplate \nstatic void launch_bin_bcast_pack(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,\n const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,\n cudaStream_t stream, std::index_sequence) {\n GGML_TENSOR_BINARY_OP_LOCALS\n\n int nr0 = ne10 / ne0;\n int nr1 = ne11 / ne1;\n int nr2 = ne12 / ne2;\n int nr3 = ne13 / ne3;\n\n int nr[4] = { nr0, nr1, nr2, nr3 };\n\n int64_t cne[] = { ne0, ne1, ne2, ne3 };\n int64_t cne0[] = { ne00, ne01, ne02, ne03 };\n int64_t cne1[] = { ne10, ne11, ne12, ne13 };\n\n size_t cnb[] = { nb0, nb1, nb2, nb3 };\n size_t cnb0[] = { nb00, nb01, nb02, nb03 };\n size_t cnb1[] = { nb10, nb11, nb12, nb13 };\n\n auto collapse = [](int64_t cne[]) {\n cne[0] *= cne[1];\n cne[1] = cne[2];\n cne[2] = cne[3];\n cne[3] = 1;\n };\n\n auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {\n cnb[1] *= cne[1];\n cnb[2] *= cne[2];\n cnb[3] *= cne[3];\n };\n\n if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && !ggml_is_permuted(src0) && !ggml_is_permuted(src1)) {\n for (int i = 0; i < 4; i++) {\n if (nr[i] != 1) {\n break;\n }\n if (i > 0) {\n collapse_nb(cnb, cne);\n collapse_nb(cnb0, cne0);\n collapse_nb(cnb1, cne1);\n collapse(cne);\n collapse(cne0);\n collapse(cne1);\n }\n }\n }\n\n {\n int64_t ne0 = cne[0];\n int64_t ne1 = cne[1];\n int64_t ne2 = cne[2];\n int64_t ne3 = cne[3];\n\n //int64_t ne00 = cne0[0]; GGML_UNUSED(ne00);\n //int64_t ne01 = cne0[1]; GGML_UNUSED(ne01);\n //int64_t ne02 = cne0[2]; GGML_UNUSED(ne02);\n //int64_t ne03 = cne0[3]; GGML_UNUSED(ne03);\n\n size_t nb0 = cnb[0];\n size_t nb1 = cnb[1];\n size_t nb2 = cnb[2];\n size_t nb3 = cnb[3];\n\n size_t nb00 = cnb0[0];\n size_t nb01 = cnb0[1];\n size_t nb02 = cnb0[2];\n size_t nb03 = cnb0[3];\n\n size_t nb10 = cnb1[0];\n size_t nb11 = cnb1[1];\n size_t nb12 = cnb1[2];\n size_t nb13 = cnb1[3];\n\n //size_t s0 = nb0 / sizeof(dst_t);\n size_t s1 = nb1 / sizeof(dst_t);\n size_t s2 = nb2 / sizeof(dst_t);\n size_t s3 = nb3 / sizeof(dst_t);\n\n size_t s10 = nb10 / sizeof(src1_t);\n size_t s11 = nb11 / sizeof(src1_t);\n size_t s12 = nb12 / sizeof(src1_t);\n size_t s13 = nb13 / sizeof(src1_t);\n\n size_t s00 = nb00 / sizeof(src0_t);\n size_t s01 = nb01 / sizeof(src0_t);\n size_t s02 = nb02 / sizeof(src0_t);\n size_t s03 = nb03 / sizeof(src0_t);\n\n GGML_ASSERT(nb0 % sizeof(dst_t) == 0);\n GGML_ASSERT(nb1 % sizeof(dst_t) == 0);\n GGML_ASSERT(nb2 % sizeof(dst_t) == 0);\n GGML_ASSERT(nb3 % sizeof(dst_t) == 0);\n\n GGML_ASSERT(nb00 % sizeof(src0_t) == 0);\n GGML_ASSERT(nb01 % sizeof(src0_t) == 0);\n GGML_ASSERT(nb02 % sizeof(src0_t) == 0);\n GGML_ASSERT(nb03 % sizeof(src0_t) == 0);\n\n GGML_ASSERT(nb10 % sizeof(src1_t) == 0);\n GGML_ASSERT(nb11 % sizeof(src1_t) == 0);\n GGML_ASSERT(nb12 % sizeof(src1_t) == 0);\n GGML_ASSERT(nb13 % sizeof(src1_t) == 0);\n\n const int block_size = 128;\n\n int64_t hne0 = std::max(ne0 / 2LL, 1LL);\n\n dim3 block_dims;\n block_dims.x = std::min(hne0, block_size);\n block_dims.y = std::min(ne1, block_size / block_dims.x);\n block_dims.z = std::min(std::min(ne2 * ne3, block_size / block_dims.x / block_dims.y), 64U);\n\n dim3 block_nums((hne0 + block_dims.x - 1) / block_dims.x, (ne1 + block_dims.y - 1) / block_dims.y,\n (ne2 * ne3 + block_dims.z - 1) / block_dims.z);\n\n const uint3 ne10 = init_fastdiv_values((uint32_t) cne1[0]);\n const uint3 ne11 = init_fastdiv_values((uint32_t) cne1[1]);\n const uint3 ne12 = init_fastdiv_values((uint32_t) cne1[2]);\n const uint3 ne13 = init_fastdiv_values((uint32_t) cne1[3]);\n\n if (block_nums.z > 65535 || block_nums.y > 65535) {\n int block_num = (ne0 * ne1 * ne2 * ne3 + block_size - 1) / block_size;\n const uint3 prod_012 = init_fastdiv_values((uint32_t) (ne0 * ne1 * ne2));\n const uint3 prod_01 = init_fastdiv_values((uint32_t) (ne0 * ne1));\n const uint3 ne0_fastdiv = init_fastdiv_values((uint32_t) ne0);\n const uint3 ne1_fastdiv = init_fastdiv_values((uint32_t) ne1);\n const uint3 ne2_fastdiv = init_fastdiv_values((uint32_t) ne2);\n\n if constexpr (sizeof...(I) > 0) {\n k_bin_bcast_unravel<<>>(\n src0_dd, src1_dd, dst_dd, ne0_fastdiv, ne1_fastdiv, ne2_fastdiv, ne3, prod_012, prod_01, ne10, ne11,\n ne12, ne13,\n /*s0,*/ s1, s2, s3,\n s00, s01, s02, s03,\n s10, s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...);\n } else {\n k_bin_bcast_unravel\n <<>>(src0_dd, src1_dd, dst_dd, ne0_fastdiv, ne1_fastdiv,\n ne2_fastdiv, ne3, prod_012, prod_01, ne10, ne11, ne12, ne13,\n /*s0,*/ s1, s2, s3,\n s00, s01, s02, s03,\n s10, s11, s12, s13);\n }\n } else {\n const uint3 ne3_fastdiv = init_fastdiv_values((uint32_t) ne3);\n if constexpr (sizeof...(I) > 0) {\n k_bin_bcast<<>>(\n src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3_fastdiv, ne10, ne11, ne12, ne13,\n /*s0,*/ s1, s2, s3,\n s00 ,s01, s02, s03,\n s10, s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...);\n } else {\n k_bin_bcast<<>>(\n src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3_fastdiv, ne10, ne11, ne12, ne13,\n /*s0,*/ s1, s2, s3,\n s00, s01, s02, s03,\n s10, s11, s12, s13);\n }\n }\n }\n}\n\ntemplate \nstatic __global__ void k_repeat_back(\n const T * __restrict__ src, T * __restrict__ dst, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const size_t s00, const size_t s01, const size_t s02, const size_t s03,\n const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3) {\n\n const int64_t tid0 = int64_t(blockIdx.x)*blockDim.x + threadIdx.x;\n const int64_t tid1 = int64_t(blockIdx.y)*blockDim.y + threadIdx.y;\n const int64_t tid23 = int64_t(blockIdx.z)*blockDim.z + threadIdx.z;\n const int64_t tid2 = tid23 % ne2;\n const int64_t tid3 = tid23 / ne2;\n\n if (tid0 >= ne0) {\n return;\n }\n\n T sum = 0;\n for (int64_t i3 = tid3; i3 < ne03; i3 += ne3) {\n for (int64_t i2 = tid2; i2 < ne02; i2 += ne2) {\n for (int64_t i1 = tid1; i1 < ne01; i1 += ne1) {\n for (int64_t i0 = tid0; i0 < ne00; i0 += ne0) {\n sum += src[i3*s03 + i2*s02 + i1*s01 + i0*s00];\n }\n }\n }\n }\n dst[tid3*ne2*ne1*ne0 + tid2*ne1*ne0 + tid1*ne0 + tid0] = sum;\n}\n\ntemplate \nstruct bin_bcast_cuda {\n template\n void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,\n const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,\n cudaStream_t stream) {\n launch_bin_bcast_pack(\n src0, src1, dst, src0_dd, src1_dd, dst_dd, stream, std::make_index_sequence{});\n }\n};\n\ntemplate \nstatic void repeat_back_cuda(\n const T * src, T * dst, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const size_t s00, const size_t s01, const size_t s02, const size_t s03,\n const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {\n\n const dim3 block_dims(WARP_SIZE, 1, 1);\n const dim3 block_nums((ne0 + WARP_SIZE - 1) / WARP_SIZE, ne1, ne2*ne3);\n k_repeat_back<<>>\n (src, dst, ne00, ne01, ne02, ne03, s00, s01, s02, s03, ne0, ne1, ne2, ne3);\n}\n\ntemplate\nstatic void ggml_cuda_op_bin_bcast(\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,\n const void * src0_dd, const void * src1_dd, void * dst_dd, cudaStream_t stream) {\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16);\n\n if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n op()(src0, src1, dst, (const float *)src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {\n op()(src0, src1, dst, (const half *) src0_dd, (const half *)src1_dd, (half *) dst_dd, stream);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) {\n op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (half *) dst_dd, stream);\n } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {\n op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);\n } else {\n fprintf(stderr, \"%s: unsupported types: dst: %s, src0: %s, src1: %s\\n\", __func__,\n ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));\n GGML_ABORT(\"fatal error\");\n }\n}\n\nvoid ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_bin_bcast>(dst, dst->src[0], dst, nullptr, dst->src[0]->data, dst->data, ctx.stream());\n}\n\nvoid ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_bin_bcast>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());\n}\n\nvoid ggml_cuda_op_sub(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_bin_bcast>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());\n}\n\nvoid ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_bin_bcast>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());\n}\n\nvoid ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_bin_bcast>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());\n}\n\ntemplate \nstatic void ggml_cuda_op_fused_binbcast_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n cudaStream_t stream = ctx.stream();\n\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n launch_bin_bcast_pack(src0, src1, dst,\n (const float *) src0->data, (const float *) src1->data, (float *) dst->data,\n stream, std::make_index_sequence{});\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {\n launch_bin_bcast_pack(src0, src1, dst,\n (const half *) src0->data, (const half *) src1->data, (half *) dst->data,\n stream, std::make_index_sequence{});\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) {\n launch_bin_bcast_pack(src0, src1, dst,\n (const half *) src0->data, (const float *) src1->data, (half *) dst->data,\n stream, std::make_index_sequence{});\n } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {\n launch_bin_bcast_pack(src0, src1, dst,\n (const half *) src0->data, (const float *) src1->data, (float *) dst->data,\n stream, std::make_index_sequence{});\n } else {\n fprintf(stderr,\n \"%s: unsupported types for fusion: dst: %s, src0: %s, src1: %s\\n\",\n __func__, ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));\n GGML_ABORT(\"fatal error\");\n }\n}\n\n\nvoid ggml_cuda_op_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, int n_fuse) {\n GGML_ASSERT(2 <= n_fuse && n_fuse <= 8);\n\n switch (n_fuse) {\n case 2:\n ggml_cuda_op_fused_binbcast_impl(ctx, dst);\n break;\n case 3:\n ggml_cuda_op_fused_binbcast_impl(ctx, dst);\n break;\n case 4:\n ggml_cuda_op_fused_binbcast_impl(ctx, dst);\n break;\n case 5:\n ggml_cuda_op_fused_binbcast_impl(ctx, dst);\n break;\n case 6:\n ggml_cuda_op_fused_binbcast_impl(ctx, dst);\n break;\n case 7:\n ggml_cuda_op_fused_binbcast_impl(ctx, dst);\n break;\n case 8:\n ggml_cuda_op_fused_binbcast_impl(ctx, dst);\n break;\n default:\n GGML_ASSERT(false && \"Unsupported n_fuse value\");\n }\n}\n\nvoid ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(src0->type == dst->type);\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_can_repeat(dst, src0));\n\n cudaStream_t stream = ctx.stream();\n\n GGML_TENSOR_UNARY_OP_LOCALS;\n\n GGML_ASSERT(ne2*ne3 <= (1 << 15));\n\n const size_t ts = ggml_type_size(src0->type);\n const size_t s00 = nb00 / ts;\n const size_t s01 = nb01 / ts;\n const size_t s02 = nb02 / ts;\n const size_t s03 = nb03 / ts;\n\n switch (dst->type) {\n case GGML_TYPE_F32: {\n const float * src0_d = (const float *) src0->data;\n float * dst_d = (float *) dst->data;\n repeat_back_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s00, s01, s02, s03, ne0, ne1, ne2, ne3, stream);\n } break;\n default: {\n GGML_ASSERT(false);\n } break;\n }\n}\n" }, { "path": "src/ggml-cuda/binbcast.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\nvoid ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\nvoid ggml_cuda_op_sub(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\nvoid ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\nvoid ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, int n_fuse);\n" }, { "path": "src/ggml-cuda/clamp.cu", "content": "#include \"clamp.cuh\"\n\nstatic __device__ __forceinline__ float op_clamp(float x, float min, float max) {\n return fminf(fmaxf(x, min), max);\n}\n\ntemplate \nstatic __global__ void op_clamp_kernel(const T * x, T * dst, const T min, const T max, const int k) {\n const int i = blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n\n dst[i] = (T)op_clamp((float)x[i], (float)min, (float)max);\n}\n\ntemplate \nstatic void clamp_cuda(const T * x, T * dst, const T min, const T max, const int k, cudaStream_t stream) {\n const int num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE;\n op_clamp_kernel<<>>(x, dst, min, max, k);\n}\n\n\nvoid ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const void * src0_d = src0->data;\n void * dst_d = dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n GGML_ASSERT(src0->type == dst->type);\n\n float min;\n float max;\n memcpy(&min, dst->op_params, sizeof(float));\n memcpy(&max, (float *) dst->op_params + 1, sizeof(float));\n\n if (src0->type == GGML_TYPE_F16) {\n clamp_cuda((const half *)src0_d, (half *)dst_d, (half)min, (half)max, ggml_nelements(src0), stream);\n } else {\n clamp_cuda((const float *)src0_d, (float *)dst_d, (float)min, (float)max, ggml_nelements(src0), stream);\n }\n}\n" }, { "path": "src/ggml-cuda/clamp.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_CLAMP_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/common.cuh", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cuda.h\"\n\n#include \n#include \n\n#if defined(GGML_USE_HIP)\n#define GGML_COMMON_DECL_HIP\n#define GGML_COMMON_IMPL_HIP\n#else\n#define GGML_COMMON_DECL_CUDA\n#define GGML_COMMON_IMPL_CUDA\n#if defined(GGML_USE_MUSA)\n#define GGML_COMMON_DECL_MUSA\n#define GGML_COMMON_IMPL_MUSA\n#endif\n#endif\n#include \"ggml-common.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(GGML_USE_HIP)\n#include \"vendors/hip.h\"\n#elif defined(GGML_USE_MUSA)\n#include \"vendors/musa.h\"\n#else\n#include \"vendors/cuda.h\"\n#endif // defined(GGML_USE_HIP)\n\n#define STRINGIZE_IMPL(...) #__VA_ARGS__\n#define STRINGIZE(...) STRINGIZE_IMPL(__VA_ARGS__)\n\n#define WARP_SIZE 32\n#define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed)\n#define CUDART_HMASK 12000 // CUDA 12.0, min. ver. for half2 -> uint mask comparisons\n\n#define GGML_CUDA_CC_PASCAL 600\n#define GGML_CUDA_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products\n#define GGML_CUDA_CC_VOLTA 700\n#define GGML_CUDA_CC_TURING 750\n#define GGML_CUDA_CC_AMPERE 800\n#define GGML_CUDA_CC_ADA_LOVELACE 890\n// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see\n// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms\n#define GGML_CUDA_CC_BLACKWELL 1200\n#define GGML_CUDA_CC_DGX_SPARK 1210\n#define GGML_CUDA_CC_RUBIN 1300\n#define GGML_CUDA_CC_OFFSET_AMD 0x1000000\n#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000\n#define GGML_CUDA_CC_IS_NVIDIA(cc) (cc < GGML_CUDA_CC_OFFSET_MTHREADS)\n\n// AMD\n// GCN/CDNA, wave size is 64\n#define GGML_CUDA_CC_GCN4 (GGML_CUDA_CC_OFFSET_AMD + 0x803) // Tonga, Fiji, Polaris, minimum for fast fp16\n#define GGML_CUDA_CC_VEGA (GGML_CUDA_CC_OFFSET_AMD + 0x900) // Vega56/64, minimum for fp16 dual issue\n#define GGML_CUDA_CC_VEGA20 (GGML_CUDA_CC_OFFSET_AMD + 0x906) // MI50/Radeon VII, minimum for dp4a\n#define GGML_CUDA_CC_CDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x908) // MI100, minimum for MFMA, acc registers\n#define GGML_CUDA_CC_CDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x910) // MI210, minimum acc register renameing\n#define GGML_CUDA_CC_CDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x942) // MI300\n\n// RDNA removes MFMA, dp4a, xnack, acc registers, wave size is 32\n#define GGML_CUDA_CC_RDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x1010) // RX 5000\n#define GGML_CUDA_CC_RDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a\n#define GGML_CUDA_CC_RDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA\n#define GGML_CUDA_CC_RDNA3_5 (GGML_CUDA_CC_OFFSET_AMD + 0x1150) // AI 370, AI Max 395 laptops.\n#define GGML_CUDA_CC_RDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000\n\n#define GGML_CUDA_CC_IS_AMD(cc) (cc >= GGML_CUDA_CC_OFFSET_AMD)\n#define GGML_CUDA_CC_IS_RDNA(cc) (cc >= GGML_CUDA_CC_RDNA1)\n#define GGML_CUDA_CC_IS_RDNA1(cc) (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2)\n#define GGML_CUDA_CC_IS_RDNA2(cc) (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3)\n#define GGML_CUDA_CC_IS_RDNA3_0(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA3_5)\n#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4)\n#define GGML_CUDA_CC_IS_RDNA3(cc) (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc))\n#define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4)\n#define GGML_CUDA_CC_IS_GCN(cc) (cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1)\n#define GGML_CUDA_CC_IS_CDNA(cc) (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1)\n#define GGML_CUDA_CC_IS_CDNA1(cc) (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2)\n#define GGML_CUDA_CC_IS_CDNA2(cc) (cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3)\n#define GGML_CUDA_CC_IS_CDNA3(cc) (cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_RDNA1)\n\n// Moore Threads\n#define MUSART_HMASK 40300 // MUSA rc4.3, min. ver. for half2 -> uint mask comparisons\n\n#define GGML_CUDA_CC_QY1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x210) // MTT S80, MTT S3000\n#define GGML_CUDA_CC_QY2 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x220) // MTT S4000\n#define GGML_CUDA_CC_PH1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x310) // MTT S5000\n\n#define GGML_CUDA_CC_IS_MTHREADS(cc) (cc >= GGML_CUDA_CC_OFFSET_MTHREADS && cc < GGML_CUDA_CC_OFFSET_AMD)\n#define GGML_CUDA_CC_IS_QY1(cc) (cc >= GGML_CUDA_CC_QY1 && cc < GGML_CUDA_CC_QY2)\n#define GGML_CUDA_CC_IS_QY2(cc) (cc >= GGML_CUDA_CC_QY2 && cc < GGML_CUDA_CC_PH1)\n#define GGML_CUDA_CC_IS_PH1(cc) (cc >= GGML_CUDA_CC_PH1)\n\n#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070\n# define GGML_CUDA_USE_CUB\n#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070\n\n#ifdef __CUDA_ARCH_LIST__\nconstexpr bool ggml_cuda_has_arch_impl(int) {\n return false;\n}\n\ntemplate\nconstexpr bool ggml_cuda_has_arch_impl(const int arch, const int first, Archs... rest) {\n return arch == first || ggml_cuda_has_arch_impl(arch, rest...);\n}\n\nconstexpr bool ggml_cuda_has_arch(const int arch) {\n return ggml_cuda_has_arch_impl(arch, __CUDA_ARCH_LIST__);\n}\n\nconstexpr int ggml_cuda_highest_compiled_arch_impl(const int /*arch*/, const int cur) {\n if (cur == 0) {\n return -1;\n }\n return cur;\n}\n\ntemplate\nconstexpr int ggml_cuda_highest_compiled_arch_impl(const int arch, const int cur, const int first, Archs... rest) {\n if (first <= arch && first > cur) {\n return ggml_cuda_highest_compiled_arch_impl(arch, first, rest...);\n } else {\n return ggml_cuda_highest_compiled_arch_impl(arch, cur, rest...);\n }\n}\n\nconstexpr int ggml_cuda_highest_compiled_arch(const int arch) {\n return ggml_cuda_highest_compiled_arch_impl(arch, 0, __CUDA_ARCH_LIST__);\n}\n#else\nstatic int ggml_cuda_highest_compiled_arch(const int arch) {\n return arch;\n}\n#endif // __CUDA_ARCH_LIST__\n\n// ---------------------------------------------------------------------------------------------------------\n\n#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses\n\n#define GGML_CUDA_MAX_STREAMS 8\n\n[[noreturn]]\nvoid ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg);\n\n#define CUDA_CHECK_GEN(err, success, error_fn) \\\n do { \\\n auto err_ = (err); \\\n if (err_ != (success)) { \\\n ggml_cuda_error(#err, __func__, __FILE__, __LINE__, error_fn(err_)); \\\n } \\\n } while (0)\n\n#define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString)\n\n#if CUDART_VERSION >= 12000 || defined(GGML_USE_MUSA)\n static const char * cublas_get_error_str(const cublasStatus_t err) {\n return cublasGetStatusString(err);\n }\n#else\n static const char * cublas_get_error_str(const cublasStatus_t err) {\n switch (err) {\n case CUBLAS_STATUS_SUCCESS: return \"CUBLAS_STATUS_SUCCESS\";\n case CUBLAS_STATUS_NOT_INITIALIZED: return \"CUBLAS_STATUS_NOT_INITIALIZED\";\n case CUBLAS_STATUS_ALLOC_FAILED: return \"CUBLAS_STATUS_ALLOC_FAILED\";\n case CUBLAS_STATUS_INVALID_VALUE: return \"CUBLAS_STATUS_INVALID_VALUE\";\n case CUBLAS_STATUS_ARCH_MISMATCH: return \"CUBLAS_STATUS_ARCH_MISMATCH\";\n case CUBLAS_STATUS_MAPPING_ERROR: return \"CUBLAS_STATUS_MAPPING_ERROR\";\n case CUBLAS_STATUS_EXECUTION_FAILED: return \"CUBLAS_STATUS_EXECUTION_FAILED\";\n case CUBLAS_STATUS_INTERNAL_ERROR: return \"CUBLAS_STATUS_INTERNAL_ERROR\";\n case CUBLAS_STATUS_NOT_SUPPORTED: return \"CUBLAS_STATUS_NOT_SUPPORTED\";\n default: return \"unknown error\";\n }\n }\n#endif // CUDART_VERSION >= 12000\n\n#define CUBLAS_CHECK(err) CUDA_CHECK_GEN(err, CUBLAS_STATUS_SUCCESS, cublas_get_error_str)\n\n#if !defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)\nstatic const char * cu_get_error_str(CUresult err) {\n const char * err_str;\n cuGetErrorString(err, &err_str);\n return err_str;\n}\n#define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str)\n#endif\n\n#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)\n# define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \\\n do { \\\n static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = { false }; \\\n const int id = ggml_cuda_get_device(); \\\n if (!shared_memory_limit_raised[id]) { \\\n CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes)); \\\n shared_memory_limit_raised[id] = true; \\\n } \\\n } while (0)\n#else\n# define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \\\n do { \\\n GGML_UNUSED(nbytes); \\\n } while (0)\n#endif // !(defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)\n\n#if CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)\n#define GGML_CUDA_ASSUME(x) __builtin_assume(x)\n#else\n#define GGML_CUDA_ASSUME(x)\n#endif // CUDART_VERSION >= 11010\n\n#if (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM))\n#define GGML_USE_VMM\n#endif // (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM))\n\n#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL\n#define FP16_AVAILABLE\n#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL\n\n#if defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610\n#define FAST_FP16_AVAILABLE\n#endif // defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610\n\n#if defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)\n#define AMD_MFMA_AVAILABLE\n#endif // defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)\n\n#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA3))\n#define AMD_WMMA_AVAILABLE\n#endif // defined(GGML_USE_HIP) && defined(RDNA4)\n\n// The Volta instructions are in principle available on Turing or newer but they are effectively unusable:\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n#define VOLTA_MMA_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING\n#define TURING_MMA_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING\n\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#define AMPERE_MMA_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN\n# define BLACKWELL_MMA_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL\n\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#define CP_ASYNC_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n\n#if !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220)\n#define FLASH_ATTN_AVAILABLE\n#endif // !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220)\n\n#if defined(TURING_MMA_AVAILABLE)\n#define LDMATRIX_TRANS_AVAILABLE\n#endif // defined(TURING_MMA_AVAILABLE)\n\nstatic bool fp16_available(const int cc) {\n return ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_PASCAL ||\n (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1);\n}\n\nstatic bool fast_fp16_available(const int cc) {\n return GGML_CUDA_CC_IS_AMD(cc) ||\n (GGML_CUDA_CC_IS_NVIDIA(cc) && fp16_available(cc) && ggml_cuda_highest_compiled_arch(cc) != 610) ||\n (GGML_CUDA_CC_IS_MTHREADS(cc) && fp16_available(cc));\n}\n\n// To be used for feature selection of external libraries, e.g. cuBLAS.\nstatic bool fast_fp16_hardware_available(const int cc) {\n return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_PASCAL && cc != 610) || GGML_CUDA_CC_IS_AMD(cc) ||\n (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);\n}\n\n// To be used for feature selection of external libraries, e.g. cuBLAS.\nstatic bool fp16_mma_hardware_available(const int cc) {\n return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_VOLTA) ||\n GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc) ||\n (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);\n}\n\nstatic bool bf16_mma_hardware_available(const int cc) {\n return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE) ||\n GGML_CUDA_CC_IS_CDNA(cc) || cc >= GGML_CUDA_CC_RDNA3 ||\n (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1);\n}\n\nstatic bool fp32_mma_hardware_available(const int cc) {\n return GGML_CUDA_CC_IS_CDNA(cc);\n}\n\nstatic bool amd_mfma_available(const int cc) {\n#if !defined(GGML_HIP_NO_MMQ_MFMA)\n return GGML_CUDA_CC_IS_CDNA(cc);\n#else\n return false;\n#endif //!defined(GGML_HIP_NO_MMQ_MFMA)\n}\n\nstatic bool amd_wmma_available(const int cc) {\n return (GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_RDNA3(cc));\n}\n\nstatic bool volta_mma_available(const int cc) {\n return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_VOLTA;\n}\n\nstatic bool turing_mma_available(const int cc) {\n return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_TURING;\n}\n\nstatic bool ampere_mma_available(const int cc) {\n return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE;\n}\n\nstatic bool cp_async_available(const int cc) {\n return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE;\n}\n\nstatic bool blackwell_mma_available(const int cc) {\n return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_BLACKWELL &&\n ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN;\n}\n\nstatic constexpr __device__ int ggml_cuda_get_physical_warp_size() {\n#if defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__))\n return 64;\n#else\n return 32;\n#endif // defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__))\n}\n\n// Maximum number of bytes that can be copied in a single instruction.\nstatic constexpr __device__ int ggml_cuda_get_max_cpy_bytes() {\n#ifdef GGML_USE_HIP\n return 16;\n#else\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n return 16;\n#else\n return 8;\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n#endif // GGML_USE_HIP\n}\n\n\n[[noreturn]]\nstatic __device__ void no_device_code(\n const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) {\n\n#if defined(GGML_USE_HIP)\n printf(\"%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\\n\",\n file_name, line, function_name, arch);\n GGML_UNUSED(arch_list);\n#else\n printf(\"%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\\n\",\n file_name, line, function_name, arch, arch_list);\n#endif // defined(GGML_USE_HIP)\n __trap();\n\n GGML_UNUSED(no_device_code); // suppress unused function warning\n\n#if defined(GGML_USE_MUSA)\n __builtin_unreachable();\n#endif // defined(GGML_USE_MUSA)\n}\n\n#ifdef __CUDA_ARCH__\n#define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__))\n#else\n#define NO_DEVICE_CODE //GGML_ABORT(\"NO_DEVICE_CODE not valid in host code.\")\n#endif // __CUDA_ARCH__\n\n// The compiler is always able to unroll loops if they contain continue expressions.\n// In such cases loop unrolling can still be achieved via recursion:\ntemplate \nstruct ggml_cuda_unroll {\n template \n __device__ void operator()(const Func & f, Args... args) const {\n f(n - 1, args...);\n ggml_cuda_unroll{}(f, args...);\n }\n};\n\ntemplate <>\nstruct ggml_cuda_unroll<1> {\n template \n __device__ void operator()(const Func & f, Args... args) const {\n f(0, args...);\n }\n};\n\ntemplate\nstatic __device__ __forceinline__ int warp_reduce_sum(int x) {\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n return __reduce_add_sync(0xffffffff, x);\n#else\n#pragma unroll\n for (int offset = width/2; offset > 0; offset >>= 1) {\n x += __shfl_xor_sync(0xffffffff, x, offset, width);\n }\n return x;\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n}\n\ntemplate\nstatic __device__ __forceinline__ float warp_reduce_sum(float x) {\n#pragma unroll\n for (int offset = width/2; offset > 0; offset >>= 1) {\n x += __shfl_xor_sync(0xffffffff, x, offset, width);\n }\n return x;\n}\n\ntemplate\nstatic __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {\n#pragma unroll\n for (int offset = width/2; offset > 0; offset >>= 1) {\n a.x += __shfl_xor_sync(0xffffffff, a.x, offset, width);\n a.y += __shfl_xor_sync(0xffffffff, a.y, offset, width);\n }\n return a;\n}\n\ntemplate\nstatic __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {\n#ifdef FP16_AVAILABLE\n#pragma unroll\n for (int offset = width/2; offset > 0; offset >>= 1) {\n a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, offset, width));\n }\n return a;\n\n#else\n NO_DEVICE_CODE;\n return a;\n#endif // FP16_AVAILABLE\n}\n\ntemplate\nstatic __device__ __forceinline__ int warp_reduce_all(int x) {\n if (width == ggml_cuda_get_physical_warp_size()) {\n return __all_sync(0xffffffff, x);\n } else {\n#pragma unroll\n for (int offset = width/2; offset > 0; offset >>= 1) {\n x = __shfl_xor_sync(0xffffffff, x, offset, width) && x;\n }\n return x;\n }\n}\n\ntemplate\nstatic __device__ __forceinline__ int warp_reduce_any(int x) {\n if (width == ggml_cuda_get_physical_warp_size()) {\n return __any_sync(0xffffffff, x);\n } else {\n#pragma unroll\n for (int offset = width/2; offset > 0; offset >>= 1) {\n x = __shfl_xor_sync(0xffffffff, x, offset, width) || x;\n }\n return x;\n }\n}\n\ntemplate\nstatic __device__ __forceinline__ float warp_reduce_max(float x) {\n#pragma unroll\n for (int offset = width/2; offset > 0; offset >>= 1) {\n x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, offset, width));\n }\n return x;\n}\n\ntemplate\nstatic __device__ __forceinline__ T warp_prefix_inclusive_sum(T x) {\n const int lane_id = threadIdx.x % width;\n#pragma unroll\n for (int offset = 1; offset < width; offset <<= 1) {\n const T t = __shfl_up_sync(0xffffffff, x, offset, width);\n if (lane_id >= offset) {\n x += t;\n }\n }\n return x;\n}\n\ntemplate\nstatic __device__ __forceinline__ float2 warp_prefix_inclusive_sum(float2 a) {\n const int lane_id = threadIdx.x % width;\n#pragma unroll\n for (int offset = 1; offset < width; offset <<= 1) {\n const float t_x = __shfl_up_sync(0xffffffff, a.x, offset, width);\n const float t_y = __shfl_up_sync(0xffffffff, a.y, offset, width);\n if (lane_id >= offset) {\n a.x += t_x;\n a.y += t_y;\n }\n }\n return a;\n}\n\ntemplate\nstatic __device__ __forceinline__ half2 warp_prefix_inclusive_sum(half2 a) {\n#ifdef FP16_AVAILABLE\n const int lane_id = threadIdx.x % width;\n#pragma unroll\n for (int offset = 1; offset < width; offset <<= 1) {\n const half2 t = __shfl_up_sync(0xffffffff, a, offset, width);\n if (lane_id >= offset) {\n a = __hadd2(a, t);\n }\n }\n return a;\n\n#else\n NO_DEVICE_CODE;\n return a;\n#endif // FP16_AVAILABLE\n}\n\nenum class block_reduce_method {\n MAX,\n SUM,\n};\n\ntemplate\nstruct block_reduce_policy;\n\ntemplate \ninline constexpr bool is_any = (std::is_same_v || ...);\n\ntemplate\ninline constexpr bool ggml_cuda_dependent_false_v = false;\n\ntemplate struct block_reduce_policy {\n static __device__ T reduce(T val) {\n if constexpr(is_any) {\n return warp_reduce_sum(val);\n } else {\n static_assert(ggml_cuda_dependent_false_v, \"Unsupported type for block reduce sum\");\n }\n }\n\n static __device__ T sentinel() {\n if constexpr (std::is_same_v) {\n return 0.0f;\n } else if constexpr (std::is_same_v) {\n return make_float2(0.0f, 0.0f);\n } else if constexpr (std::is_same_v) {\n return make_half2(0.0f, 0.0f);\n } else if constexpr (std::is_same_v) {\n return 0;\n } else {\n static_assert(ggml_cuda_dependent_false_v, \"Unsupported type for block reduce sum\");\n }\n }\n};\n\ntemplate struct block_reduce_policy {\n static __device__ T reduce(T val) {\n if constexpr (is_any) {\n return warp_reduce_max(val);\n } else {\n static_assert(ggml_cuda_dependent_false_v, \"Unsupported type for block reduce max\");\n }\n }\n\n static __device__ T sentinel() {\n if constexpr (std::is_same_v) {\n return -INFINITY;\n } else if constexpr (std::is_same_v) {\n return make_half2(-INFINITY, -INFINITY);\n } else {\n static_assert(ggml_cuda_dependent_false_v, \"Unsupported type for block reduce max\");\n }\n }\n};\n\ntemplate \nstatic __device__ T block_reduce(T val, T * shared_vals) {\n val = block_reduce_policy::reduce(val);\n const unsigned int block_size = block_size_template == 0 ? blockDim.x : block_size_template;\n if (block_size > WARP_SIZE) {\n assert((block_size <= 1024) && (block_size % WARP_SIZE) == 0);\n const int warp_id = threadIdx.x / WARP_SIZE;\n const int lane_id = threadIdx.x % WARP_SIZE;\n if (lane_id == 0) {\n shared_vals[warp_id] = val;\n }\n __syncthreads();\n val = block_reduce_policy::sentinel();\n if (lane_id < (static_cast(block_size) / WARP_SIZE)) {\n val = shared_vals[lane_id];\n }\n return block_reduce_policy::reduce(val);\n }\n\n return val;\n}\n\nstatic __device__ __forceinline__ half ggml_cuda_hmax(const half a, const half b) {\n#ifdef FP16_AVAILABLE\n\n#if !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX\n return __float2half(fmaxf(__half2float(a), __half2float(b)));\n#else\n return __hmax(a, b);\n#endif // !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX\n\n#else\n NO_DEVICE_CODE;\n GGML_UNUSED(b);\n return a;\n#endif // FP16_AVAILABLE\n}\n\nstatic __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const half2 b) {\n#if defined(GGML_USE_HIP)\n return half2(__hmax(a.x, b.x), __hmax(a.y, b.y));\n#elif CUDART_VERSION >= CUDART_HMAX\n return __hmax2(a, b);\n#else\n half2 ret;\n reinterpret_cast(ret.x) = __float2half(fmaxf( __low2float(a), __low2float(b)));\n reinterpret_cast(ret.y) = __float2half(fmaxf(__high2float(a), __high2float(b)));\n return ret;\n#endif\n}\n\ntemplate\nstatic __device__ __forceinline__ half2 warp_reduce_max(half2 x) {\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL || defined(GGML_USE_HIP)\n#pragma unroll\n for (int offset = width/2; offset > 0; offset >>= 1) {\n x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, offset, width));\n }\n return x;\n#else\n GGML_UNUSED(x);\n NO_DEVICE_CODE;\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL || defined(GGML_USE_HIP)\n}\n\n#if (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || \\\n (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK)\nstatic __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half2 b) {\n const uint32_t mask_low = 0x0000FFFF * (float( __low2half(a)) > float( __low2half(b)));\n const uint32_t mask_high = 0xFFFF0000 * (float(__high2half(a)) > float(__high2half(b)));\n return mask_low | mask_high;\n}\n#endif // (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK)\n\nstatic __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, int c) {\n#if defined(GGML_USE_HIP)\n#if defined(CDNA) || defined(RDNA2) || defined(__gfx906__)\n c = __builtin_amdgcn_sdot4(a, b, c, false);\n#elif defined(RDNA3) || defined(RDNA4)\n c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);\n#elif defined(RDNA1) || defined(__gfx900__)\n int tmp1;\n int tmp2;\n asm(\"\\n \\\n v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \\n \\\n v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \\n \\\n v_add3_u32 %0, %1, %2, %0 \\n \\\n v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \\n \\\n v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \\n \\\n v_add3_u32 %0, %1, %2, %0 \\n \\\n \"\n : \"+v\"(c), \"=&v\"(tmp1), \"=&v\"(tmp2)\n : \"v\"(a), \"v\"(b)\n );\n#else\n const int8x4_t va = reinterpret_cast(a);\n const int8x4_t vb = reinterpret_cast(b);\n c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];\n#endif\n return c;\n\n#else // defined(GGML_USE_HIP)\n\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)\n return __dp4a(a, b, c);\n#else // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)\n const int8_t * a8 = (const int8_t *) &a;\n const int8_t * b8 = (const int8_t *) &b;\n return c + a8[0]*b8[0] + a8[1]*b8[1] + a8[2]*b8[2] + a8[3]*b8[3];\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)\n\n#endif // defined(GGML_USE_HIP)\n}\n\nstatic __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float v, const float u) {\n acc += v*u;\n}\n\nstatic __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float2 v, const float2 u) {\n acc += v.x*u.x;\n acc += v.y*u.y;\n}\n\n#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))\n#define V_DOT2_F32_F16_AVAILABLE\n#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))\n\nstatic __device__ __forceinline__ void ggml_cuda_mad(float & acc, const half2 v, const half2 u) {\n#ifdef V_DOT2_F32_F16_AVAILABLE\n asm volatile(\"v_dot2_f32_f16 %0, %1, %2, %0\" : \"+v\"(acc) : \"v\"(v), \"v\"(u));\n#else\n#ifdef FAST_FP16_AVAILABLE\n const float2 tmp = __half22float2(v*u);\n acc += tmp.x + tmp.y;\n#else\n const float2 tmpv = __half22float2(v);\n const float2 tmpu = __half22float2(u);\n acc += tmpv.x * tmpu.x;\n acc += tmpv.y * tmpu.y;\n#endif // FAST_FP16_AVAILABLE\n#endif // V_DOT2_F32_F16_AVAILABLE\n}\n\nstatic __device__ __forceinline__ void ggml_cuda_mad(half2 & acc, const half2 v, const half2 u) {\n#ifdef FAST_FP16_AVAILABLE\n acc += v*u;\n#else\n const float2 tmpv = __half22float2(v);\n const float2 tmpu = __half22float2(u);\n float2 tmpacc = __half22float2(acc);\n tmpacc.x += tmpv.x * tmpu.x;\n tmpacc.y += tmpv.y * tmpu.y;\n acc = make_half2(tmpacc.x, tmpacc.y);\n#endif // FAST_FP16_AVAILABLE\n}\n\n// Aligned memory transfers of 8/16 bytes can be faster than 2 transfers with 4 bytes, especially on AMD.\n// Important: do not use this function if dst and src both point at registers.\n// Due to the strict aliasing rule the compiler can do incorrect optimizations if src and dst have different types.\n// The function is intended for copies between registers and SRAM/VRAM to make the compiler emit the right instructions.\n// If dst and src point at different address spaces then they are guaranteed to not be aliased.\ntemplate \nstatic __device__ __forceinline__ void ggml_cuda_memcpy_1(void * __restrict__ dst, const void * __restrict__ src) {\n static_assert(\n nbytes <= ggml_cuda_get_max_cpy_bytes() || alignment == 0,\n \"You are misusing the alignment parameter for ggml_cuda_memcpy_1. \"\n \"The intent is for the parameter is only as a workaround if either one of the pointers is not properly aligned. \"\n \"If you use it to do more bytes per copy than ggml_cuda_max_cpy_bytes() the reads and writes may not be coalesced. \"\n \"Call ggml_cuda_memcpy_1 in a loop instead.\");\n if constexpr (alignment != 0) {\n static_assert(nbytes % alignment == 0, \"bad alignment\");\n }\n constexpr int nb_per_cpy = alignment == 0 ? nbytes : alignment;\n\n#pragma unroll\n for (int i = 0; i < nbytes/nb_per_cpy; ++i) {\n if constexpr (nb_per_cpy == 1) {\n ((char *) dst)[i] = ((const char *) src)[i];\n } else if constexpr (nb_per_cpy == 2) {\n ((short *) dst)[i] = ((const short *) src)[i];\n } else if constexpr (nb_per_cpy == 4) {\n ((int *) dst)[i] = ((const int *) src)[i];\n } else if constexpr (nb_per_cpy == 8) {\n ((int2 *) dst)[i] = ((const int2 *) src)[i];\n } else if constexpr (nb_per_cpy == 16) {\n ((int4 *) dst)[i] = ((const int4 *) src)[i];\n } else {\n static_assert(nbytes == 0 && nbytes == -1, \"bad nbytes\");\n }\n }\n}\n\nstatic __device__ __forceinline__ float ggml_cuda_e8m0_to_fp32(uint8_t x) {\n#if CUDART_VERSION >= 12080\n const nv_bfloat16 e = __nv_cvt_e8m0_to_bf16raw(x);\n return (float) e;\n#else\n uint32_t bits;\n if (x == 0) {\n bits = 0x00400000;\n } else {\n bits = (uint32_t) x << 23;\n }\n\n float result;\n memcpy(&result, &bits, sizeof(float));\n return result;\n#endif // CUDART_VERSION >= 12050\n}\n\n__device__ __forceinline__ uint8_t ggml_cuda_float_to_fp4_e2m1(float x, float e) {\n const uint8_t sign_bit = (x < 0.0f) << 3;\n float ax = fabsf(x) * e;\n\n // Positive LUT\n static constexpr float pos_lut[8] = { 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f };\n\n int best_i = 0;\n float best_err = fabsf(ax - pos_lut[0]);\n\n#pragma unroll\n for (int i = 1; i < 8; ++i) {\n const float err = fabsf(ax - pos_lut[i]);\n if (err < best_err) {\n best_err = err;\n best_i = i;\n }\n }\n\n return static_cast(best_i | sign_bit);\n}\n\n// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.\n// Precompute mp (m' in the paper) and L such that division\n// can be computed using a multiply (high 32b of 64b result)\n// and a shift:\n//\n// n/d = (mulhi(n, mp) + n) >> L;\nstatic const uint3 init_fastdiv_values(uint64_t d_64) {\n GGML_ASSERT(d_64 != 0);\n GGML_ASSERT(d_64 <= std::numeric_limits::max());\n\n uint32_t d = (uint32_t)d_64;\n\n // compute L = ceil(log2(d));\n uint32_t L = 0;\n while (L < 32 && (uint32_t{ 1 } << L) < d) {\n L++;\n }\n\n uint32_t mp = (uint32_t) ((uint64_t{ 1 } << 32) * ((uint64_t{ 1 } << L) - d) / d + 1);\n // pack divisor as well to reduce error surface\n return make_uint3(mp, L, d);\n}\n\nstatic __device__ __forceinline__ uint32_t fastdiv(uint32_t n, const uint3 fastdiv_values) {\n // expects fastdiv_values to contain in \n // fastdiv_values.z is unused and optimized away by the compiler.\n // Compute high 32 bits of n * mp\n const uint32_t hi = __umulhi(n, fastdiv_values.x);\n // add n, apply bit shift\n return (hi + n) >> fastdiv_values.y;\n}\n\nstatic __device__ __forceinline__ uint32_t fastmodulo(uint32_t n, const uint3 fastdiv_values) {\n // expects fastdiv_values to contain in (see init_fastdiv_values)\n return n - fastdiv(n, fastdiv_values) * fastdiv_values.z;\n}\n\n// Calculate both division and modulo at once, returns \nstatic __device__ __forceinline__ uint2 fast_div_modulo(uint32_t n, const uint3 fastdiv_values) {\n // expects fastdiv_values to contain in (see init_fastdiv_values)\n const uint32_t div_val = fastdiv(n, fastdiv_values);\n const uint32_t mod_val = n - div_val * fastdiv_values.z;\n return make_uint2(div_val, mod_val);\n}\n\ntypedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, float2 & v);\n\nstatic __device__ __forceinline__ float get_alibi_slope(\n const float max_bias, const uint32_t h, const uint32_t n_head_log2, const float m0, const float m1\n) {\n if (max_bias <= 0.0f) {\n return 1.0f;\n }\n const float base = h < n_head_log2 ? m0 : m1;\n const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n return powf(base, exph);\n}\n\ntemplate \nstruct ggml_cuda_type_traits;\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = 1;\n static constexpr int qr = 1;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK4_0;\n static constexpr int qr = QR4_0;\n static constexpr int qi = QI4_0;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK4_1;\n static constexpr int qr = QR4_1;\n static constexpr int qi = QI4_1;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK5_0;\n static constexpr int qr = QR5_0;\n static constexpr int qi = QI5_0;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK5_1;\n static constexpr int qr = QR5_1;\n static constexpr int qi = QI5_1;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK8_0;\n static constexpr int qr = QR8_0;\n static constexpr int qi = QI8_0;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_MXFP4;\n static constexpr int qr = QR_MXFP4;\n static constexpr int qi = QI_MXFP4;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR2_K;\n static constexpr int qi = QI2_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR3_K;\n static constexpr int qi = QI3_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR4_K;\n static constexpr int qi = QI4_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR5_K;\n static constexpr int qi = QI5_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR6_K;\n static constexpr int qi = QI6_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR2_XXS;\n static constexpr int qi = QI2_XXS;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR2_XS;\n static constexpr int qi = QI2_XS;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR2_S;\n static constexpr int qi = QI2_S;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR3_XXS;\n static constexpr int qi = QI3_XXS;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR1_S;\n static constexpr int qi = QI1_S;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR1_M;\n static constexpr int qi = QI1_M;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK4_NL;\n static constexpr int qr = QR4_NL;\n static constexpr int qi = QI4_NL;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR4_XS;\n static constexpr int qi = QI4_XS;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits {\n static constexpr int qk = QK_K;\n static constexpr int qr = QR3_S;\n static constexpr int qi = QI3_S;\n};\n\n//////////////////////\n\nstruct ggml_cuda_device_info {\n int device_count;\n\n struct cuda_device_info {\n int cc; // compute capability\n int nsm; // number of streaming multiprocessors\n size_t smpb; // max. shared memory per block\n size_t smpbo; // max. shared memory per block (with opt-in)\n bool integrated; // Device is integrated as opposed to discrete\n bool vmm; // virtual memory support\n size_t vmm_granularity; // granularity of virtual memory\n size_t total_vram;\n int warp_size; // Number of threads in a dispatch\n bool supports_cooperative_launch; // whether cooperative launch is supported\n };\n\n cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {};\n\n std::array default_tensor_split = {};\n};\n\nconst ggml_cuda_device_info & ggml_cuda_info();\n\nvoid ggml_cuda_set_device(int device);\nint ggml_cuda_get_device();\n\nstruct ggml_cuda_pool {\n virtual ~ggml_cuda_pool() = default;\n\n virtual void * alloc(size_t size, size_t * actual_size) = 0;\n virtual void free(void * ptr, size_t size) = 0;\n};\n\ntemplate\nstruct ggml_cuda_pool_alloc {\n ggml_cuda_pool * pool = nullptr;\n T * ptr = nullptr;\n size_t actual_size = 0;\n\n ggml_cuda_pool_alloc() = default;\n\n explicit ggml_cuda_pool_alloc(ggml_cuda_pool & pool) : pool(&pool) {\n }\n\n ggml_cuda_pool_alloc(ggml_cuda_pool & pool, size_t size) : pool(&pool) {\n alloc(size);\n }\n\n ~ggml_cuda_pool_alloc() {\n if (ptr != nullptr) {\n pool->free(ptr, actual_size);\n }\n }\n\n // size is in number of elements\n T * alloc(size_t size) {\n GGML_ASSERT(pool != nullptr);\n GGML_ASSERT(ptr == nullptr);\n ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size);\n return ptr;\n }\n\n T * alloc(ggml_cuda_pool & pool, size_t size) {\n this->pool = &pool;\n return alloc(size);\n }\n\n T * get() {\n return ptr;\n }\n\n ggml_cuda_pool_alloc(const ggml_cuda_pool_alloc &) = delete;\n ggml_cuda_pool_alloc(ggml_cuda_pool_alloc &&) = delete;\n ggml_cuda_pool_alloc& operator=(const ggml_cuda_pool_alloc &) = delete;\n ggml_cuda_pool_alloc& operator=(ggml_cuda_pool_alloc &&) = delete;\n};\n\n\n// backend interface\n\nstruct ggml_tensor_extra_gpu {\n void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors\n cudaEvent_t events[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; // events for synchronizing multiple GPUs\n};\n\n\n#if (defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)) || defined(GGML_MUSA_GRAPHS)\n#define USE_CUDA_GRAPH\n#endif\n\nstruct ggml_cuda_graph_node_properties {\n void * node_data;\n ggml_op node_op;\n enum ggml_type node_type;\n int32_t flags;\n int64_t ne[GGML_MAX_DIMS];\n size_t nb[GGML_MAX_DIMS];\n void * src_data[GGML_MAX_SRC];\n int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];\n};\n\nstatic_assert(std::is_trivial::value, \"ggml_cuda_graph_node_properties must be trivial\");\n\nstruct ggml_cuda_graph {\n#ifdef USE_CUDA_GRAPH\n ~ggml_cuda_graph() {\n if (instance != nullptr) {\n CUDA_CHECK(cudaGraphExecDestroy(instance));\n }\n if (graph != nullptr) {\n CUDA_CHECK(cudaGraphDestroy(graph));\n }\n }\n cudaGraph_t graph = nullptr;\n cudaGraphExec_t instance = nullptr;\n size_t num_nodes = 0;\n std::vector nodes;\n bool disable_due_to_gpu_arch = false;\n bool warmup_complete = false;\n std::vector props;\n\n // these are extra tensors (inputs) that participate in the ggml graph but are not nodes\n // they properties also have to match in order to be able to safely reuse a CUDA graph\n // ref: https://github.com/ggml-org/llama.cpp/pull/18583\n // ref: https://github.com/ggml-org/llama.cpp/pull/19165\n std::vector extra;\n\n bool is_enabled() const {\n static const bool disable_cuda_graphs_due_to_env = (getenv(\"GGML_CUDA_DISABLE_GRAPHS\") != nullptr);\n return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env);\n }\n#endif\n};\n\nstruct ggml_cuda_concurrent_event {\n std::vector join_events;\n cudaEvent_t fork_event = nullptr;\n\n int n_streams = 0;\n std::unordered_map stream_mapping;\n\n // Original order of nodes in this concurrent region (before interleaving)\n // Used to restore grouping for fusion within streams\n std::vector original_order;\n\n const ggml_tensor * join_node;\n\n ggml_cuda_concurrent_event() = default;\n\n ggml_cuda_concurrent_event(const ggml_cuda_concurrent_event &) = delete;\n ggml_cuda_concurrent_event & operator=(const ggml_cuda_concurrent_event &) = delete;\n\n explicit ggml_cuda_concurrent_event(int n_streams) : n_streams(n_streams) {\n join_events.resize(n_streams);\n\n for (size_t i = 0; i < join_events.size(); ++i) {\n CUDA_CHECK(cudaEventCreateWithFlags(&join_events[i], cudaEventDisableTiming));\n }\n\n CUDA_CHECK(cudaEventCreateWithFlags(&fork_event, cudaEventDisableTiming));\n }\n\n ggml_cuda_concurrent_event(ggml_cuda_concurrent_event && other) noexcept\n : join_events(std::move(other.join_events))\n , fork_event(other.fork_event)\n , n_streams(other.n_streams)\n , stream_mapping(std::move(other.stream_mapping))\n , original_order(std::move(other.original_order))\n , join_node(other.join_node) {\n other.fork_event = nullptr;\n }\n\n // 1. check if any branches write to overlapping memory ranges (except the join node)\n // 2. check whether all srcs are either within the branch or outside the nodes covered by ggml_cuda_concurrent_event\n // we assume all nodes have the same buffer\n bool is_valid() const {\n std::vector>> write_ranges;\n write_ranges.resize(n_streams);\n\n // get join_node's memory range to exclude from overlap checking.\n // multiple nodes can use join_node's buffer; we synchronize on the join node.\n const ggml_tensor * join_t = join_node->view_src ? join_node->view_src : join_node;\n const int64_t join_start = (int64_t) join_t->data;\n const int64_t join_end = join_start + ggml_nbytes(join_t);\n\n for (const auto & [tensor, stream] : stream_mapping) {\n const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor;\n const int64_t t_start = (int64_t) t->data;\n const int64_t t_end = t_start + ggml_nbytes(t);\n\n // skip tensors that overlap with join_node's buffer.\n if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) {\n continue;\n }\n\n // concurrent streams begin from 1\n write_ranges[stream - 1].emplace_back(t_start, t_end);\n }\n\n for (int i = 0; i < n_streams; ++i) {\n // sorts first by start then by end of write range\n std::sort(write_ranges[i].begin(), write_ranges[i].end());\n }\n\n bool writes_overlap = false;\n bool dependent_srcs = false;\n for (const auto & [tensor, stream] : stream_mapping) {\n const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor;\n const int64_t t_start = (int64_t) t->data;\n const int64_t t_end = t_start + ggml_nbytes(t);\n\n // skip tensors that overlap with join_node's buffer\n if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) {\n continue;\n }\n\n // check if this buffer's write data overlaps with another stream's\n std::pair data_range = std::make_pair(t_start, t_end);\n for (int i = 0; i < n_streams; ++i) {\n if (i == stream - 1) {\n continue;\n }\n auto it = std::lower_bound(write_ranges[i].begin(), write_ranges[i].end(), data_range);\n\n if (it != write_ranges[i].end()) {\n const std::pair & other = *it;\n\n // std::lower_bound returns the first element where other >= data_range (lexicographically).\n // This guarantees other.first >= data_range.first.\n // Therefore, overlap occurs iff other.first < data_range.second\n // (i.e., the other range starts before this range ends).\n if (other.first < data_range.second) {\n GGML_LOG_DEBUG(\"Writes overlap for %s\", tensor->name);\n writes_overlap = true;\n break;\n }\n }\n }\n\n //check if all srcs are either in branch or don't have a branch\n for (int i = 0; i < GGML_MAX_SRC; ++i) {\n if (!tensor->src[i]) {\n continue;\n }\n\n auto it = stream_mapping.find(tensor->src[i]);\n\n if (it == stream_mapping.end()) {\n continue;\n }\n\n if (it->second != stream) {\n dependent_srcs = true;\n break;\n }\n }\n\n if (dependent_srcs || writes_overlap) {\n break;\n }\n }\n\n return !writes_overlap && !dependent_srcs;\n }\n\n ~ggml_cuda_concurrent_event() {\n if (fork_event != nullptr) {\n CUDA_CHECK(cudaEventDestroy(fork_event));\n }\n for (cudaEvent_t e : join_events) {\n if (e != nullptr) {\n CUDA_CHECK(cudaEventDestroy(e));\n }\n }\n }\n};\n\nstruct ggml_cuda_stream_context {\n std::unordered_map concurrent_events;\n\n void reset() {\n concurrent_events.clear();\n }\n};\n\nstruct ggml_backend_cuda_context {\n int device;\n std::string name;\n cudaEvent_t copy_event = nullptr;\n\n cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } };\n cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};\n\n int curr_stream_no = 0;\n\n#ifdef USE_CUDA_GRAPH\n // Map from first_node_ptr to cuda_graph - allows multiple graphs per context\n // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe)\n std::unordered_map> cuda_graphs;\n\n ggml_cuda_graph * cuda_graph(const void * first_node_ptr) {\n auto it = cuda_graphs.find(first_node_ptr);\n if (it == cuda_graphs.end()) {\n cuda_graphs[first_node_ptr] = std::make_unique();\n return cuda_graphs[first_node_ptr].get();\n }\n return it->second.get();\n }\n\n // Check if any CUDA graph is enabled for this context (used by kernels that need to know\n // if graphs are in use without having access to the specific graph key)\n bool any_cuda_graph_enabled() const {\n for (const auto & [key, graph] : cuda_graphs) {\n if (graph && graph->is_enabled()) {\n return true;\n }\n }\n return false;\n }\n\n // Check if any CUDA graph has an instance for this context\n bool any_cuda_graph_has_instance() const {\n for (const auto & [key, graph] : cuda_graphs) {\n if (graph && graph->instance != nullptr) {\n return true;\n }\n }\n return false;\n }\n#endif // USE_CUDA_GRAPH\n\n explicit ggml_backend_cuda_context(int device) :\n device(device),\n name(GGML_CUDA_NAME + std::to_string(device)) {\n }\n\n ggml_cuda_stream_context concurrent_stream_context;\n\n ~ggml_backend_cuda_context();\n\n cudaStream_t stream(int device, int stream) {\n if (streams[device][stream] == nullptr) {\n ggml_cuda_set_device(device);\n CUDA_CHECK(cudaStreamCreateWithFlags(&streams[device][stream], cudaStreamNonBlocking));\n }\n return streams[device][stream];\n }\n\n cudaStream_t stream() { return stream(device, curr_stream_no); }\n\n ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; }\n\n cublasHandle_t cublas_handle(int device) {\n if (cublas_handles[device] == nullptr) {\n ggml_cuda_set_device(device);\n CUBLAS_CHECK(cublasCreate(&cublas_handles[device]));\n CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH));\n }\n return cublas_handles[device];\n }\n\n cublasHandle_t cublas_handle() {\n return cublas_handle(device);\n }\n\n // pool\n std::unique_ptr pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS];\n\n static std::unique_ptr new_pool_for_device(int device, int stream_no);\n\n ggml_cuda_pool & pool(int device) {\n if (pools[device][curr_stream_no] == nullptr) {\n pools[device][curr_stream_no] = new_pool_for_device(device, curr_stream_no);\n }\n return *pools[device][curr_stream_no];\n }\n\n ggml_cuda_pool & pool() {\n return pool(device);\n }\n};\n\nstruct ggml_cuda_mm_fusion_args_host {\n const ggml_tensor * x_bias = nullptr;\n const ggml_tensor * gate = nullptr;\n const ggml_tensor * gate_bias = nullptr;\n ggml_glu_op glu_op;\n};\nstruct ggml_cuda_mm_fusion_args_device {\n const void * x_bias = nullptr;\n const void * gate = nullptr;\n const void * gate_bias = nullptr;\n ggml_glu_op glu_op;\n};\n" }, { "path": "src/ggml-cuda/concat.cu", "content": "#include \"concat.cuh\"\n\n// contiguous kernels\nstatic __global__ void concat_f32_dim0(const float * x, const float * y, float * dst, const int ne0, const int ne00) {\n int nidx = threadIdx.x + blockIdx.x * blockDim.x;\n if (nidx >= ne0) {\n return;\n }\n\n int offset_dst =\n nidx +\n blockIdx.y * ne0 +\n blockIdx.z * ne0 * gridDim.y;\n\n if (nidx < ne00) { // src0\n int offset_src =\n nidx +\n blockIdx.y * ne00 +\n blockIdx.z * ne00 * gridDim.y;\n dst[offset_dst] = x[offset_src];\n } else {\n int offset_src =\n (nidx - ne00) +\n blockIdx.y * (ne0 - ne00) +\n blockIdx.z * (ne0 - ne00) * gridDim.y;\n dst[offset_dst] = y[offset_src];\n }\n}\n\nstatic __global__ void concat_f32_dim1(const float * x, const float * y, float * dst, const int ne0, const int ne01) {\n int nidx = threadIdx.x + blockIdx.x * blockDim.x;\n if (nidx >= ne0) {\n return;\n }\n\n int offset_dst =\n nidx +\n blockIdx.y * ne0 +\n blockIdx.z * ne0 * gridDim.y;\n\n if (blockIdx.y < (unsigned)ne01) { // src0\n int offset_src =\n nidx +\n blockIdx.y * ne0 +\n blockIdx.z * ne0 * ne01;\n dst[offset_dst] = x[offset_src];\n } else {\n int offset_src =\n nidx +\n (blockIdx.y - ne01) * ne0 +\n blockIdx.z * ne0 * (gridDim.y - ne01);\n dst[offset_dst] = y[offset_src];\n }\n}\n\nstatic __global__ void concat_f32_dim2(const float * x, const float * y, float * dst, const int ne0, const int ne02) {\n int nidx = threadIdx.x + blockIdx.x * blockDim.x;\n if (nidx >= ne0) {\n return;\n }\n\n int offset_dst =\n nidx +\n blockIdx.y * ne0 +\n blockIdx.z * ne0 * gridDim.y;\n\n if (blockIdx.z < (unsigned)ne02) { // src0\n int offset_src =\n nidx +\n blockIdx.y * ne0 +\n blockIdx.z * ne0 * gridDim.y;\n dst[offset_dst] = x[offset_src];\n } else {\n int offset_src =\n nidx +\n blockIdx.y * ne0 +\n (blockIdx.z - ne02) * ne0 * gridDim.y;\n dst[offset_dst] = y[offset_src];\n }\n}\n\nstatic void concat_f32_cuda(const float * x, const float * y, float * dst, int ne00, int ne01, int ne02, int ne0, int ne1, int ne2, int dim, cudaStream_t stream) {\n int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE;\n dim3 gridDim(num_blocks, ne1, ne2);\n if (dim == 0) {\n concat_f32_dim0<<>>(x, y, dst, ne0, ne00);\n return;\n }\n if (dim == 1) {\n concat_f32_dim1<<>>(x, y, dst, ne0, ne01);\n return;\n }\n concat_f32_dim2<<>>(x, y, dst, ne0, ne02);\n}\n\n// non-contiguous kernel (slow)\ntemplate \nstatic __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE)\n concat_f32_non_cont(\n const char * src0,\n const char * src1,\n char * dst,\n int64_t ne00,\n int64_t ne01,\n int64_t ne02,\n int64_t ne03,\n uint64_t nb00,\n uint64_t nb01,\n uint64_t nb02,\n uint64_t nb03,\n int64_t /*ne10*/,\n int64_t /*ne11*/,\n int64_t /*ne12*/,\n int64_t /*ne13*/,\n uint64_t nb10,\n uint64_t nb11,\n uint64_t nb12,\n uint64_t nb13,\n int64_t ne0,\n int64_t /*ne1*/,\n int64_t /*ne2*/,\n int64_t /*ne3*/,\n uint64_t nb0,\n uint64_t nb1,\n uint64_t nb2,\n uint64_t nb3){\n static_assert(dim >= 0 && dim <= 3, \"dim must be in [0, 3]\");\n\n const int64_t i3 = blockIdx.z;\n const int64_t i2 = blockIdx.y;\n const int64_t i1 = blockIdx.x;\n\n const float * x;\n\n for (int64_t i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) {\n if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n x = (const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);\n } else {\n if constexpr (dim == 0) {\n x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + i1 * nb11 + (i0 - ne00) * nb10);\n } else if constexpr (dim == 1) {\n x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + (i1 - ne01) * nb11 + i0 * nb10);\n } else if constexpr (dim == 2) {\n x = (const float *) (src1 + i3 * nb13 + (i2 - ne02) * nb12 + i1 * nb11 + i0 * nb10);\n } else if constexpr (dim == 3) {\n x = (const float *) (src1 + (i3 - ne03) * nb13 + i2 * nb12 + i1 * nb11 + i0 * nb10);\n }\n }\n\n float * y = (float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = *x;\n }\n}\n\n\nvoid ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n cudaStream_t stream = ctx.stream();\n\n const int32_t dim = ((int32_t *) dst->op_params)[0];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {\n const float * src0_d = (const float *)src0->data;\n const float * src1_d = (const float *)src1->data;\n\n float * dst_d = (float *)dst->data;\n\n if (dim != 3) {\n for (int i3 = 0; i3 < dst->ne[3]; i3++) {\n concat_f32_cuda(\n src0_d + i3 * (src0->nb[3] / 4),\n src1_d + i3 * (src1->nb[3] / 4),\n dst_d + i3 * ( dst->nb[3] / 4),\n src0->ne[0], src0->ne[1], src0->ne[2],\n dst->ne[0], dst->ne[1], dst->ne[2], dim, stream);\n }\n } else {\n const size_t size0 = ggml_nbytes(src0);\n const size_t size1 = ggml_nbytes(src1);\n\n CUDA_CHECK(cudaMemcpyAsync(dst_d, src0_d, size0, cudaMemcpyDeviceToDevice, stream));\n CUDA_CHECK(cudaMemcpyAsync(dst_d + size0/4, src1_d, size1, cudaMemcpyDeviceToDevice, stream));\n }\n } else {\n dim3 grid_dim(dst->ne[1], dst->ne[2], dst->ne[3]);\n auto launch_kernel = [&](auto dim) {\n concat_f32_non_cont<<>>(\n (const char *) src0->data, (const char *) src1->data, (char *) dst->data,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],\n src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],\n dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3]);\n };\n switch (dim) {\n case 0:\n launch_kernel(std::integral_constant{});\n break;\n case 1:\n launch_kernel(std::integral_constant{});\n break;\n case 2:\n launch_kernel(std::integral_constant{});\n break;\n case 3:\n launch_kernel(std::integral_constant{});\n break;\n default:\n GGML_ABORT(\"Invalid dim: %d\", dim);\n break;\n }\n }\n}\n" }, { "path": "src/ggml-cuda/concat.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_CONCAT_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/conv-transpose-1d.cu", "content": "#include \"conv-transpose-1d.cuh\"\n\nstatic __global__ void conv_transpose_1d_kernel(\n const int s0, const int p0, const int d0, const int output_size,\n const int src0_ne0, const int src0_ne1, const int src0_ne2, const int src0_ne3,\n const int src1_ne0, const int src1_ne1, const int src1_ne2, const int src1_ne3,\n const int dst_ne0, const int dst_ne1, const int dst_ne2, const int dst_ne3,\n const float * src0, const float * src1, float * dst) {\n int global_index = threadIdx.x + blockIdx.x * blockDim.x;\n if (global_index >= output_size) {\n return;\n }\n\n int out_index = global_index / dst_ne0;\n\n float accumulator = 0;\n\n for (int c = 0; c < src0_ne2; c++) {\n int idx = global_index % dst_ne0;\n\n int kernel_offset = (src0_ne0 * src0_ne1 * c) + (out_index * src0_ne0);\n int input_offset = src1_ne0 * c;\n\n for (int i = 0; i < src1_ne0; i++) {\n if (!(idx >= i*s0 && idx < i*s0 + src0_ne0)) {\n continue;\n }\n int weight_idx = idx - i*s0;\n\n float kernel_weight = src0[kernel_offset + weight_idx];\n float input_value = src1[input_offset+i];\n\n accumulator += kernel_weight * input_value;\n }\n }\n dst[global_index] = accumulator;\n GGML_UNUSED_VARS(p0, d0, src0_ne3, src1_ne3, dst_ne3, src1_ne1, dst_ne1, src1_ne2, dst_ne2);\n}\n\nstatic void conv_transpose_1d_f32_f32_cuda(\n const int s0, const int p0, const int d0, const int output_size,\n const int src0_ne0, const int src0_ne1, const int src0_ne2, const int src0_ne3,\n const int src1_ne0, const int src1_ne1, const int src1_ne2, const int src1_ne3,\n const int dst_ne0, const int dst_ne1, const int dst_ne2, const int dst_ne3,\n const float * src0, const float * src1, float * dst,\n cudaStream_t stream) {\n\n const int num_blocks = (output_size + CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE - 1) / CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE;\n conv_transpose_1d_kernel<<>>(\n s0,p0,d0,output_size,\n src0_ne0, src0_ne1, src0_ne2, src0_ne3,\n src1_ne0, src1_ne1, src1_ne2, src1_ne3,\n dst_ne0, dst_ne1, dst_ne2, dst_ne3,\n src0,src1, dst);\n}\n\nvoid ggml_cuda_op_conv_transpose_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *)src0->data;\n\n const ggml_tensor * src1 = dst->src[1];\n const float * src1_d = (const float *)src1->data;\n\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src1));\n\n const int32_t * opts = (const int32_t *)dst->op_params;\n\n const int s0 = opts[0];\n const int p0 = 0;//opts[3];\n const int d0 = 1;//opts[4];\n\n const int64_t output_size = ggml_nelements(dst);\n\n conv_transpose_1d_f32_f32_cuda(s0, p0, d0, output_size,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],\n dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n src0_d, src1_d, dst_d, stream);\n}\n" }, { "path": "src/ggml-cuda/conv-transpose-1d.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_conv_transpose_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/conv2d-dw.cu", "content": "#include \"conv2d-dw.cuh\"\n\nstruct conv_params {\n int in_w, in_h;\n int out_w, out_h;\n int kernel_w, kernel_h;\n int stride_x, stride_y;\n int padding_x, padding_y;\n int dilation_x, dilation_y;\n int channels, batches;\n};\n\nstruct kernel_bounds {\n int y_min, y_max;\n int x_min, x_max;\n};\n\n__device__ __forceinline__ kernel_bounds calculate_kernel_bounds(int out_x, int out_y, const conv_params & params) {\n kernel_bounds bounds;\n bounds.y_min = max(0, (params.padding_y - out_y * params.stride_y + params.dilation_y - 1) / params.dilation_y);\n bounds.y_max =\n min(params.kernel_h,\n (params.in_h + params.padding_y - out_y * params.stride_y + params.dilation_y - 1) / params.dilation_y);\n bounds.x_min = max(0, (params.padding_x - out_x * params.stride_x + params.dilation_x - 1) / params.dilation_x);\n bounds.x_max =\n min(params.kernel_w,\n (params.in_w + params.padding_x - out_x * params.stride_x + params.dilation_x - 1) / params.dilation_x);\n return bounds;\n}\n\n__device__ __forceinline__ int calculate_input_coord(int out_coord, int kern_coord, int stride, int dilation, int padding) {\n return out_coord * stride + kern_coord * dilation - padding;\n}\n\nstruct whcn_layout {\n __device__ static int input_index(int n, int c, int y, int x, const conv_params & params) {\n return n * (params.channels * params.in_w * params.in_h) + c * params.in_w * params.in_h + y * params.in_w + x;\n }\n\n __device__ static int kernel_index(int c, int ky, int kx, const conv_params & params) {\n return c * params.kernel_h * params.kernel_w + ky * params.kernel_w + kx;\n }\n\n __device__ static int output_index(int n, int c, int y, int x, const conv_params & params) {\n return n * (params.channels * params.out_w * params.out_h) + c * params.out_w * params.out_h +\n y * params.out_w + x;\n }\n\n __device__ static void unpack_indices(int global_idx, const conv_params & params, int & n, int & c, int & out_y,\n int & out_x) {\n out_x = global_idx % params.out_w;\n out_y = (global_idx / params.out_w) % params.out_h;\n c = (global_idx / (params.out_w * params.out_h)) % params.channels;\n n = global_idx / (params.out_w * params.out_h * params.channels);\n }\n};\n\nstruct cwhn_layout {\n __device__ static int input_index(int n, int c, int y, int x, const conv_params & params) {\n return n * (params.channels * params.in_w * params.in_h) + (y * params.in_w + x) * params.channels + c;\n }\n\n __device__ static int kernel_index(int c, int ky, int kx, const conv_params & params) {\n return (ky * params.kernel_w + kx) * params.channels + c;\n }\n\n __device__ static int output_index(int n, int c, int y, int x, const conv_params & params) {\n return n * (params.channels * params.out_w * params.out_h) + y * (params.out_w * params.channels) +\n x * params.channels + c;\n }\n\n __device__ static void unpack_indices(int global_idx, const conv_params & params, int & n, int & c, int & out_y,\n int & out_x) {\n c = global_idx % params.channels;\n out_x = (global_idx / params.channels) % params.out_w;\n out_y = (global_idx / (params.channels * params.out_w)) % params.out_h;\n n = global_idx / (params.channels * params.out_w * params.out_h);\n }\n};\n\ntemplate \n__global__ void conv2d_dw_kernel(const T * __restrict__ input, const T * __restrict__ kernel, T * __restrict__ output,\n const int in_w, const int in_h, const int out_w, const int out_h,\n const int kernel_w, const int kernel_h, const int stride_x, const int stride_y,\n const int padding_x, const int padding_y, const int dilation_x, const int dilation_y,\n const int channels, const int batches) {\n const int global_idx = blockIdx.x * blockDim.x + threadIdx.x;\n const int total_elements = batches * channels * out_h * out_w;\n\n if (global_idx >= total_elements) {\n return;\n }\n\n conv_params params = { in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x,\n stride_y, padding_x, padding_y, dilation_x, dilation_y, channels, batches };\n\n int batch_idx, channel_idx, out_y_idx, out_x_idx;\n Layout::unpack_indices(global_idx, params, batch_idx, channel_idx, out_y_idx, out_x_idx);\n\n T accumulator = 0;\n kernel_bounds bounds = calculate_kernel_bounds(out_x_idx, out_y_idx, params);\n\n for (int kern_y = bounds.y_min; kern_y < bounds.y_max; ++kern_y) {\n int in_y_idx = calculate_input_coord(out_y_idx, kern_y, params.stride_y, params.dilation_y, params.padding_y);\n\n for (int kern_x = bounds.x_min; kern_x < bounds.x_max; ++kern_x) {\n int in_x_idx = calculate_input_coord(out_x_idx, kern_x, params.stride_x, params.dilation_x, params.padding_x);\n\n const T input_val = input[Layout::input_index(batch_idx, channel_idx, in_y_idx, in_x_idx, params)];\n const T kernel_val = kernel[Layout::kernel_index(channel_idx, kern_y, kern_x, params)];\n\n accumulator += input_val * kernel_val;\n }\n }\n\n output[Layout::output_index(batch_idx, channel_idx, out_y_idx, out_x_idx, params)] = accumulator;\n}\n\nvoid ggml_cuda_op_conv2d_dw(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * kernel = dst->src[0];\n const ggml_tensor * input = dst->src[1];\n\n GGML_ASSERT(kernel->type == GGML_TYPE_F32 && input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32);\n const float * w_d = (const float *) kernel->data;\n const float * x_d = (const float *) input->data;\n float * y_d = (float *) dst->data;\n\n const int32_t * p = (const int32_t *) dst->op_params;\n const int stride_x = p[0];\n const int stride_y = p[1];\n const int padding_x = p[2];\n const int padding_y = p[3];\n const int dilation_x = p[4];\n const int dilation_y = p[5];\n\n const int in_w = input->ne[0];\n const int in_h = input->ne[1];\n const int kernel_w = kernel->ne[0];\n const int kernel_h = kernel->ne[1];\n const int out_w = dst->ne[0];\n const int out_h = dst->ne[1];\n const int channels = dst->ne[2];\n const int batches = dst->ne[3];\n\n cudaStream_t st = ctx.stream();\n\n const int total = batches * channels * out_h * out_w;\n const int blocks = (total + CUDA_CONV2D_DW_BLOCK_SIZE - 1) / CUDA_CONV2D_DW_BLOCK_SIZE;\n\n if (ggml_is_contiguous(input)) {\n conv2d_dw_kernel<<>>(\n x_d, w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, padding_x, padding_y,\n dilation_x, dilation_y, channels, batches);\n } else if (ggml_is_contiguous_channels(input)) {\n conv2d_dw_kernel<<>>(\n x_d, w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, padding_x, padding_y,\n dilation_x, dilation_y, channels, batches);\n } else {\n GGML_ABORT(\"Unsupported memory layout for conv_2d_dw\");\n }\n}\n" }, { "path": "src/ggml-cuda/conv2d-dw.cuh", "content": "#pragma once\n#include \"common.cuh\"\n\n#define CUDA_CONV2D_DW_BLOCK_SIZE 256\nvoid ggml_cuda_op_conv2d_dw(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/conv2d-transpose.cu", "content": "#include \n\n#include \"conv2d-transpose.cuh\"\n#include \"ggml.h\"\n\n__global__ void conv2d_transpose_kernel(const float * __restrict__ input, const half * __restrict__ kernel,\n float * __restrict__ output, const int in_w, const int in_h, const int out_w,\n const int out_h, const int kernel_w, const int kernel_h, const int stride,\n const int c_in, const int c_out, const int batches) {\n const int global_idx = blockIdx.x * blockDim.x + threadIdx.x;\n\n const int total_elements = out_w * out_h * c_out * batches;\n\n if (global_idx >= total_elements) {\n return;\n }\n\n const int out_x_idx = global_idx % out_w;\n const int out_y_idx = (global_idx / out_w) % out_h;\n const int c_idx = (global_idx / (out_w * out_h)) % c_out;\n const int n_idx = global_idx / (out_w * out_h * c_out);\n\n float accumulator = 0;\n // For each output idx, find the inputs that contribute to it by checking stride alignment and bounds\n\n for (int c_in_idx = 0; c_in_idx < c_in; c_in_idx++) {\n for (int kh = 0; kh < kernel_h; ++kh) {\n int in_y = out_y_idx - kh;\n if (in_y < 0 || in_y % stride) continue;\n in_y /= stride;\n if (in_y >= in_h) continue;\n\n for (int kw = 0; kw < kernel_w; ++kw) {\n int in_x = out_x_idx - kw;\n if (in_x < 0 || in_x % stride) continue;\n in_x /= stride;\n if (in_x >= in_w) continue;\n\n const int input_idx = (in_w * in_h * c_in) * n_idx + (in_w * in_h) * c_in_idx + (in_w) *in_y + in_x;\n const int kernel_idx =\n (kernel_h * kernel_w * c_out) * c_in_idx + (kernel_h * kernel_w) * c_idx + (kernel_w) *kh + kw;\n\n float input_val = input[input_idx];\n half kern_val = kernel[kernel_idx];\n\n accumulator += input_val * (float) kern_val;\n }\n }\n }\n\n output[(out_w * out_h * c_out) * n_idx + (out_w * out_h) * c_idx + (out_w) *out_y_idx + out_x_idx] = accumulator;\n}\n\n//input is (W, H, C_in, N), Kernel is (W, H, C_out, C_in)\nvoid ggml_cuda_conv_2d_transpose_p0(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * kernel = dst->src[0];\n const ggml_tensor * input = dst->src[1];\n\n GGML_ASSERT(kernel->type == GGML_TYPE_F16 && input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32);\n\n const float * input_data = (const float *) input->data;\n float * output_data = (float *) dst->data;\n const half * kernel_data = (const half *) kernel->data;\n\n const int input_w = input->ne[0];\n const int input_h = input->ne[1];\n const int output_w = dst->ne[0];\n const int output_h = dst->ne[1];\n const int channels_in = input->ne[2];\n const int channels_out = kernel->ne[2];\n const int kernel_w = kernel->ne[0];\n const int kernel_h = kernel->ne[1];\n const int stride = dst->op_params[0];\n const int batches = input->ne[3];\n\n GGML_ASSERT(channels_in == kernel->ne[3]);\n GGML_ASSERT(stride > 0);\n\n cudaStream_t st = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous(input));\n GGML_ASSERT(ggml_is_contiguous(kernel));\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n const int total = (output_w * output_h * channels_out * batches);\n const int blocks = (total + CUDA_CONV2D_TRANSPOSE_BLOCK_SIZE - 1) / CUDA_CONV2D_TRANSPOSE_BLOCK_SIZE;\n\n conv2d_transpose_kernel<<>>(\n input_data, kernel_data, output_data, input_w, input_h, output_w, output_h, kernel_w, kernel_h, stride,\n channels_in, channels_out, batches);\n}\n" }, { "path": "src/ggml-cuda/conv2d-transpose.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_CONV2D_TRANSPOSE_BLOCK_SIZE 256\nvoid ggml_cuda_conv_2d_transpose_p0(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/conv2d.cu", "content": "#include \"conv2d.cuh\"\n#include \"convert.cuh\"\n\nstruct conv_params {\n const int64_t IW, IH;\n const int64_t OW, OH;\n const int64_t KW, KH;\n const int64_t ST_X, ST_Y;\n const int64_t PD_X, PD_Y;\n const int64_t DL_X, DL_Y;\n const int64_t IC, OC;\n const int64_t B;\n const int64_t TOTAL;\n};\n\nstruct kernel_bounds {\n int64_t y_min, y_max;\n int64_t x_min, x_max;\n};\n\n__device__ __forceinline__ int64_t max64(int64_t a, int64_t b) {\n return (a > b) ? a : b;\n}\n\n__device__ __forceinline__ int64_t min64(int64_t a, int64_t b) {\n return (a < b) ? a : b;\n}\n\n__device__ __forceinline__ kernel_bounds calculate_kernel_bounds(int64_t out_x, int64_t out_y, const conv_params & P) {\n kernel_bounds bounds;\n bounds.y_min = max64(0, (P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y);\n bounds.y_max = min64(P.KH, (P.IH + P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y);\n bounds.x_min = max64(0, (P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X);\n bounds.x_max = min64(P.KW, (P.IW + P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X);\n return bounds;\n}\n\n__device__ __forceinline__ int calculate_input_coord(int64_t out_coord,\n int64_t kern_coord,\n int64_t stride,\n int64_t dilation,\n int64_t padding) {\n return out_coord * stride + kern_coord * dilation - padding;\n}\n\nstruct whcn_layout {\n __device__ static int64_t input_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv_params & P) {\n return n * (P.IC * P.IW * P.IH) + c * P.IW * P.IH + y * P.IW + x;\n }\n\n __device__ static int64_t kernel_index(int64_t c_out, int64_t c_in, int64_t ky, int64_t kx, const conv_params & P) {\n return c_out * (P.IC * P.KH * P.KW) + c_in * (P.KH * P.KW) + ky * P.KW + kx;\n }\n\n __device__ static int64_t output_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv_params & P) {\n return n * (P.OC * P.OW * P.OH) + c * P.OW * P.OH + y * P.OW + x;\n }\n\n __device__ static void unpack_indices(int64_t global_idx,\n const conv_params & P,\n int64_t & n,\n int64_t & c,\n int64_t & out_y,\n int64_t & out_x) {\n out_x = global_idx % P.OW;\n out_y = (global_idx / P.OW) % P.OH;\n c = (global_idx / (P.OW * P.OH)) % P.OC;\n n = global_idx / (P.OW * P.OH * P.OC);\n }\n};\n\ntemplate \nstatic __global__ void conv2d_kernel(const float * __restrict__ input,\n const T * __restrict__ kernel,\n float * __restrict__ output,\n const conv_params P) {\n const int64_t global_idx = blockIdx.x * blockDim.x + threadIdx.x;\n\n if (global_idx >= P.TOTAL) {\n return;\n }\n\n int64_t n, c_out, out_y, out_x;\n Layout::unpack_indices(global_idx, P, n, c_out, out_y, out_x);\n\n float acc = 0.0f;\n\n for (int64_t c_in = 0; c_in < P.IC; ++c_in) {\n kernel_bounds bounds = calculate_kernel_bounds(out_x, out_y, P);\n\n for (int64_t ky = bounds.y_min; ky < bounds.y_max; ++ky) {\n const int64_t in_y = calculate_input_coord(out_y, ky, P.ST_Y, P.DL_Y, P.PD_Y);\n\n for (int64_t kx = bounds.x_min; kx < bounds.x_max; ++kx) {\n const int64_t in_x = calculate_input_coord(out_x, kx, P.ST_X, P.DL_X, P.PD_X);\n\n const float input_val = input[Layout::input_index(n, c_in, in_y, in_x, P)];\n const T kernel_val = kernel[Layout::kernel_index(c_out, c_in, ky, kx, P)];\n acc += (input_val * ggml_cuda_cast(kernel_val));\n }\n }\n }\n\n // [N, OC, OH, OW]\n output[Layout::output_index(n, c_out, out_y, out_x, P)] = acc;\n}\n\ntemplate \nstatic void conv2d_cuda(const float * X_D, const T * K_D, float * Y_D, const conv_params P, cudaStream_t st) {\n const int blocks = (P.TOTAL + CUDA_CONV2D_BLOCK_SIZE - 1) / CUDA_CONV2D_BLOCK_SIZE;\n conv2d_kernel<<>>(X_D, K_D, Y_D, P);\n}\n\nstatic void conv2d_cuda_f16(const float * X_D, const half * K_D, float * Y_D, const conv_params P, cudaStream_t st) {\n conv2d_cuda(X_D, K_D, Y_D, P, st);\n}\n\nstatic void conv2d_cuda_f32(const float * X_D, const float * K_D, float * Y_D, const conv_params P, cudaStream_t st) {\n conv2d_cuda(X_D, K_D, Y_D, P, st);\n}\n\nvoid ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * kernel = dst->src[0];\n const ggml_tensor * input = dst->src[1];\n float * K_D = (float *) kernel->data;\n const float * X_D = (const float *) input->data;\n float * Y_D = (float *) dst->data;\n\n GGML_ASSERT(ggml_is_contiguous(kernel));\n GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32);\n\n // same number of input channels\n GGML_ASSERT(input->ne[2] == kernel->ne[2]);\n\n cudaStream_t st = ctx.stream();\n\n const int32_t * p = (const int32_t *) dst->op_params;\n const int ST_X = p[0]; // stride_x\n const int ST_Y = p[1]; // stride_y\n const int PD_X = p[2]; // padding_x\n const int PD_Y = p[3]; // padding_y\n const int DL_X = p[4]; // dilation_x\n const int DL_Y = p[5]; // dilation_y\n\n // No cwhn\n GGML_ASSERT(p[6] == false);\n\n const int IW = input->ne[0]; // input_w\n const int IH = input->ne[1]; // input_h\n const int OW = dst->ne[0]; // output_w\n const int OH = dst->ne[1]; // output_h\n const int KW = kernel->ne[0]; // kernel_w\n const int KH = kernel->ne[1]; // kernel_h\n const int IC = input->ne[2]; // input_channels\n const int OC = kernel->ne[3]; // ouptut_chanles\n const int B = input->ne[3]; // n_batches\n\n const int64_t total = B * OC * OH * OW;\n conv_params params = { IW, IH, OW, OH, KW, KH, ST_X, ST_Y, PD_X, PD_Y, DL_X, DL_Y, IC, OC, B, total };\n\n if (kernel->type == GGML_TYPE_F16) {\n conv2d_cuda_f16(X_D, (half *) K_D, Y_D, params, st);\n } else {\n conv2d_cuda_f32(X_D, K_D, Y_D, params, st);\n }\n}\n" }, { "path": "src/ggml-cuda/conv2d.cuh", "content": "#pragma once\n#include \"common.cuh\"\n\n#define CUDA_CONV2D_BLOCK_SIZE 256\nvoid ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/convert.cu", "content": "#include \"convert.cuh\"\n#include \"dequantize.cuh\"\n\n#include \n\n#define CUDA_Q8_0_NE_ALIGN 2048\n\ntemplate \nstatic __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y,\n const int64_t ne00, const int64_t ne01,\n const int64_t ne0203, const uint3 ne02,\n const int64_t s01, const int64_t s02, const int64_t s03) {\n const int64_t i00 = 2 * (int64_t(blockDim.x)*blockIdx.x + threadIdx.x);\n\n if (i00 >= ne00) {\n return;\n }\n\n for (int64_t i01 = blockIdx.y; i01 < ne01; i01 += gridDim.y) {\n for (int64_t i0203 = blockIdx.z; i0203 < ne0203; i0203 += gridDim.z) {\n const uint2 dm = fast_div_modulo((uint32_t)i0203, ne02);\n const int64_t i02 = dm.y;\n const int64_t i03 = dm.x;\n\n const int64_t ibx0 = i03*s03 + i02*s02 + i01*s01;\n\n const int64_t ib = ibx0 + i00/qk; // block index\n const int64_t iqs = (i00%qk)/qr; // quant index\n const int64_t iybs = i00 - i00%qk; // y block start index\n const int64_t y_offset = qr == 1 ? 1 : qk/2;\n\n // dequantize\n float2 v;\n dequantize_kernel(vx, ib, iqs, v);\n\n const int64_t iy0 = (i0203*ne01 + i01)*ne00 + iybs + iqs;\n y[iy0 + 0] = ggml_cuda_cast(v.x);\n y[iy0 + y_offset] = ggml_cuda_cast(v.y);\n }\n }\n}\n\ntemplate \nstatic __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int64_t k) {\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL\n constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;\n\n const int64_t i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;\n const int * x0 = ((int *) vx) + blockIdx.x * nint;\n half2 * y2 = (half2 *) (y + i0);\n\n __shared__ int vals[nint];\n\n#pragma unroll\n for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {\n if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {\n break;\n }\n\n const int ix = ix0 + threadIdx.x;\n vals[ix] = x0[ix];\n }\n\n __syncthreads();\n\n#pragma unroll\n for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {\n if (need_check && i0 + iy + 2*threadIdx.x >= k) {\n return;\n }\n\n const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);\n const half d = *b0;\n const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];\n\n y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));\n }\n#else\n GGML_UNUSED_VARS(vx, y, k);\n NO_DEVICE_CODE;\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL\n}\n\ntemplate\nstatic __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {\n\n const int64_t i = blockIdx.x;\n\n // assume 32 threads\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8;\n const int64_t ir = tid%8;\n const int64_t ib = 8*i + ir;\n if (ib >= nb32) {\n return;\n }\n\n dst_t * y = yy + 256*i + 32*ir + 4*il;\n\n const block_q4_0 * x = (const block_q4_0 *)vx + ib;\n const float d = __half2float(x->d);\n const float dm = -8*d;\n\n const uint8_t * q = x->qs + 4*il;\n\n for (int l = 0; l < 4; ++l) {\n y[l+ 0] = d * (q[l] & 0xF) + dm;\n y[l+16] = d * (q[l] >> 4) + dm;\n }\n}\n\ntemplate\nstatic __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {\n\n const int64_t i = blockIdx.x;\n\n // assume 32 threads\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8;\n const int64_t ir = tid%8;\n const int64_t ib = 8*i + ir;\n if (ib >= nb32) {\n return;\n }\n\n dst_t * y = yy + 256*i + 32*ir + 4*il;\n\n const block_q4_1 * x = (const block_q4_1 *)vx + ib;\n const float2 d = __half22float2(x->dm);\n\n const uint8_t * q = x->qs + 4*il;\n\n for (int l = 0; l < 4; ++l) {\n y[l+ 0] = d.x * (q[l] & 0xF) + d.y;\n y[l+16] = d.x * (q[l] >> 4) + d.y;\n }\n}\n\n//================================== k-quants\n\ntemplate\nstatic __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_q2_K * x = (const block_q2_K *) vx;\n\n const int64_t tid = threadIdx.x;\n const int64_t n = tid/32;\n const int64_t l = tid - 32*n;\n const int64_t is = 8*n + l/16;\n\n const uint8_t q = x[i].qs[32*n + l];\n dst_t * y = yy + i*QK_K + 128*n;\n\n float dall = __low2half(x[i].dm);\n float dmin = __high2half(x[i].dm);\n y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);\n y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4);\n y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);\n y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);\n}\n\ntemplate\nstatic __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_q3_K * x = (const block_q3_K *) vx;\n\n const int64_t r = threadIdx.x/4;\n const int64_t tid = r/2;\n const int64_t is0 = r%2;\n const int64_t l0 = 16*is0 + 4*(threadIdx.x%4);\n const int64_t n = tid / 4;\n const int64_t j = tid - 4*n;\n\n uint8_t m = 1 << (4*n + j);\n int64_t is = 8*n + 2*j + is0;\n int shift = 2*j;\n\n int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :\n is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :\n is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :\n (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);\n float d_all = x[i].d;\n float dl = d_all * (us - 32);\n\n dst_t * y = yy + i*QK_K + 128*n + 32*j;\n const uint8_t * q = x[i].qs + 32*n;\n const uint8_t * hm = x[i].hmask;\n\n for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));\n}\n\nstatic inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {\n if (j < 4) {\n d = q[j] & 63; m = q[j + 4] & 63;\n } else {\n d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);\n m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);\n }\n}\n\ntemplate\nstatic __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n const block_q4_K * x = (const block_q4_K *) vx;\n\n const int64_t i = blockIdx.x;\n\n // assume 32 threads\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8;\n const int64_t ir = tid%8;\n const int64_t is = 2*il;\n const int64_t n = 4;\n\n dst_t * y = yy + i*QK_K + 64*il + n*ir;\n\n const float dall = __low2half(x[i].dm);\n const float dmin = __high2half(x[i].dm);\n\n const uint8_t * q = x[i].qs + 32*il + n*ir;\n\n uint8_t sc, m;\n get_scale_min_k4(is + 0, x[i].scales, sc, m);\n const float d1 = dall * sc; const float m1 = dmin * m;\n get_scale_min_k4(is + 1, x[i].scales, sc, m);\n const float d2 = dall * sc; const float m2 = dmin * m;\n for (int l = 0; l < n; ++l) {\n y[l + 0] = d1 * (q[l] & 0xF) - m1;\n y[l +32] = d2 * (q[l] >> 4) - m2;\n }\n}\n\ntemplate\nstatic __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n const block_q5_K * x = (const block_q5_K *) vx;\n\n const int64_t i = blockIdx.x;\n\n // assume 64 threads - this is very slightly better than the one below\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/16; // il is in 0...3\n const int64_t ir = tid%16; // ir is in 0...15\n const int64_t is = 2*il; // is is in 0...6\n\n dst_t * y = yy + i*QK_K + 64*il + 2*ir;\n\n const float dall = __low2half(x[i].dm);\n const float dmin = __high2half(x[i].dm);\n\n const uint8_t * ql = x[i].qs + 32*il + 2*ir;\n const uint8_t * qh = x[i].qh + 2*ir;\n\n uint8_t sc, m;\n get_scale_min_k4(is + 0, x[i].scales, sc, m);\n const float d1 = dall * sc; const float m1 = dmin * m;\n get_scale_min_k4(is + 1, x[i].scales, sc, m);\n const float d2 = dall * sc; const float m2 = dmin * m;\n\n uint8_t hm = 1 << (2*il);\n y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;\n y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;\n hm <<= 1;\n y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;\n y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;\n}\n\ntemplate\nstatic __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n const block_q6_K * x = (const block_q6_K *) vx;\n\n const int64_t i = blockIdx.x;\n\n // assume 64 threads - this is very slightly better than the one below\n const int64_t tid = threadIdx.x;\n const int64_t ip = tid/32; // ip is 0 or 1\n const int64_t il = tid - 32*ip; // 0...32\n const int64_t is = 8*ip + il/16;\n\n dst_t * y = yy + i*QK_K + 128*ip + il;\n\n const float d = x[i].d;\n\n const uint8_t * ql = x[i].ql + 64*ip + il;\n const uint8_t qh = x[i].qh[32*ip + il];\n const int8_t * sc = x[i].scales + is;\n\n y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);\n y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);\n y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);\n y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);\n}\n\ntemplate\nstatic __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_iq2_xxs * x = (const block_iq2_xxs *) vx;\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint16_t * q2 = x[i].qs + 4*ib;\n const uint8_t * aux8 = (const uint8_t *)q2;\n const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);\n const uint32_t aux32 = q2[2] | (q2[3] << 16);\n const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f;\n const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];\n for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n}\n\ntemplate\nstatic __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_iq2_xs * x = (const block_iq2_xs *) vx;\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint16_t * q2 = x[i].qs + 4*ib;\n const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));\n const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;\n const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];\n for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n}\n\ntemplate\nstatic __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_iq2_s * x = (const block_iq2_s *) vx;\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));\n const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;\n const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];\n for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n}\n\ntemplate\nstatic __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_iq3_xxs * x = (const block_iq3_xxs *) vx;\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint8_t * q3 = x[i].qs + 8*ib;\n const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;\n const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);\n const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);\n const uint32_t aux32 = gas[0] | (gas[1] << 16);\n const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f;\n const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];\n for (int j = 0; j < 4; ++j) {\n y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);\n y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);\n }\n}\n\ntemplate\nstatic __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_iq3_s * x = (const block_iq3_s *) vx;\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint8_t * qs = x[i].qs + 8*ib;\n const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));\n const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));\n const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf));\n const uint8_t signs = x[i].signs[4*ib + il];\n for (int j = 0; j < 4; ++j) {\n y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);\n y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);\n }\n}\n\ntemplate\nstatic __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_iq1_s * x = (const block_iq1_s *) vx;\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;\n const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1);\n uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;\n grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)];\n grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;\n grid32[0] &= 0x0f0f0f0f;\n for (int j = 0; j < 8; ++j) {\n y[j] = d * (q[j] + delta);\n }\n}\n\ntemplate\nstatic __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_iq1_m * x = (const block_iq1_m *) vx;\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint16_t * sc = (const uint16_t *)x[i].scales;\n iq1m_scale_t scale;\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);\n const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);\n const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;\n uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;\n grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];\n grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;\n grid32[0] &= 0x0f0f0f0f;\n for (int j = 0; j < 8; ++j) {\n y[j] = d * (q[j] + delta);\n }\n}\n\ntemplate\nstatic __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 4*il;\n const uint8_t * q4 = x[ib].qs + 4*il;\n const float d = (float)x[ib].d;\n for (int j = 0; j < 4; ++j) {\n y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];\n y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];\n }\n}\n\ntemplate\nstatic __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n const int64_t i = blockIdx.x;\n const block_iq4_xs * x = (const block_iq4_xs *)vx;\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 4*il;\n const uint8_t * q4 = x[i].qs + 16*ib + 4*il;\n const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);\n for (int j = 0; j < 4; ++j) {\n y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];\n y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];\n }\n}\n\ntemplate\nstatic __global__ void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n const int64_t i = blockIdx.x;\n const block_mxfp4 * x = (const block_mxfp4 *) vx + i*(QK_K/QK_MXFP4);\n\n const int64_t tid = threadIdx.x;\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 4*il;\n const uint8_t * q4 = x[ib].qs + 4*il;\n const float d = ggml_cuda_e8m0_to_fp32(x[ib].e);\n for (int j = 0; j < 4; ++j) {\n y[j+ 0] = d * kvalues_mxfp4[q4[j] & 0xf]*0.5f;\n y[j+16] = d * kvalues_mxfp4[q4[j] >> 4]*0.5f;\n }\n}\n\ntemplate \nstatic void dequantize_block_cuda(const void * vx, dst_t * y,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t s01, const int64_t s02, const int64_t s03, cudaStream_t stream) {\n const int64_t ne0203 = ne02*ne03;\n const uint3 ne02_fdv = init_fastdiv_values(ne02);\n const dim3 num_blocks((ne00 + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE), (int)std::min(ne01, (int64_t)65535), (int)std::min(ne0203, (int64_t)65535));\n dequantize_block<<>>\n (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03);\n}\n\ntemplate \nstatic void dequantize_block_cont_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {\n dequantize_block_cuda(vx, y, k, 1, 1, 1, k/qk, k/qk, k/qk, stream);\n}\n\nstatic void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int64_t k, cudaStream_t stream) {\n const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;\n if (k % CUDA_Q8_0_NE_ALIGN == 0) {\n const bool need_check = false;\n dequantize_block_q8_0_f16<<>>(vx, y, k);\n } else {\n const bool need_check = true;\n dequantize_block_q8_0_f16<<>>(vx, y, k);\n }\n}\n\ntemplate\nstatic void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_q2_K<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_q3_K<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb32 = k / 32;\n const int nb = (k + 255) / 256;\n dequantize_block_q4_0<<>>(vx, y, nb32);\n}\n\ntemplate\nstatic void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb32 = k / 32;\n const int nb = (k + 255) / 256;\n dequantize_block_q4_1<<>>(vx, y, nb32);\n}\n\ntemplate\nstatic void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_q4_K<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_q5_K<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_q6_K<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_iq2_xxs<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_iq2_xs<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_iq2_s<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_iq3_xxs<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_iq3_s<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_iq1_s<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = (k + QK_K - 1) / QK_K;\n dequantize_block_iq4_nl<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = k / QK_K;\n dequantize_block_iq1_m<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = (k + QK_K - 1) / QK_K;\n dequantize_block_iq4_xs<<>>(vx, y);\n}\n\ntemplate\nstatic void dequantize_row_mxfp4_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n const int nb = (k + QK_K - 1) / QK_K;\n dequantize_block_mxfp4<<>>(vx, y);\n}\n\ntemplate \nstatic __global__ void convert_unary(\n const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t ne00, const int64_t ne01,\n const int64_t ne0203, const uint3 ne02,\n const int64_t s01, const int64_t s02, const int64_t s03) {\n const int64_t i00 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i00 >= ne00) {\n return;\n }\n\n const src_t * x = (const src_t *) vx;\n\n for (int64_t i01 = blockIdx.y; i01 < ne01; i01 += gridDim.y) {\n for (int64_t i0203 = blockIdx.z; i0203 < ne0203; i0203 += gridDim.z) {\n const uint2 dm = fast_div_modulo((uint32_t)i0203, ne02);\n const int64_t i02 = dm.y;\n const int64_t i03 = dm.x;\n\n const int64_t ix = i03*s03 + i02*s02 + i01*s01 + i00;\n const int64_t iy = (i0203*ne01 + i01)*ne00 + i00;\n y[iy] = ggml_cuda_cast(x[ix]);\n }\n }\n}\n\ntemplate \nstatic void convert_unary_cuda(const void * vx, dst_t * y,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t s01, const int64_t s02, const int64_t s03, cudaStream_t stream) {\n const int64_t ne0203 = ne02*ne03;\n const uint3 ne02_fdv = init_fastdiv_values(ne02);\n const dim3 num_blocks((ne00 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE, (int)std::min(ne01, (int64_t)65535), (int)std::min(ne0203, (int64_t)65535));\n convert_unary<<>>\n (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03);\n}\n\ntemplate \nstatic void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n convert_unary_cuda(vx, y, k, 1, 1, 1, k, k, k, stream);\n}\n\nto_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {\n switch (type) {\n case GGML_TYPE_F32:\n return convert_unary_cont_cuda;\n case GGML_TYPE_F16:\n return convert_unary_cont_cuda;\n default:\n return nullptr;\n }\n}\n\nto_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {\n switch (type) {\n case GGML_TYPE_Q4_0:\n return dequantize_row_q4_0_cuda;\n case GGML_TYPE_Q4_1:\n return dequantize_row_q4_1_cuda;\n case GGML_TYPE_Q5_0:\n return dequantize_block_cont_cuda;\n case GGML_TYPE_Q5_1:\n return dequantize_block_cont_cuda;\n case GGML_TYPE_Q8_0:\n if (fp16_available(ggml_cuda_info().devices[ggml_cuda_get_device()].cc)) {\n return dequantize_block_q8_0_f16_cuda;\n }\n return dequantize_block_cont_cuda;\n case GGML_TYPE_Q2_K:\n return dequantize_row_q2_K_cuda;\n case GGML_TYPE_Q3_K:\n return dequantize_row_q3_K_cuda;\n case GGML_TYPE_Q4_K:\n return dequantize_row_q4_K_cuda;\n case GGML_TYPE_Q5_K:\n return dequantize_row_q5_K_cuda;\n case GGML_TYPE_Q6_K:\n return dequantize_row_q6_K_cuda;\n case GGML_TYPE_IQ2_XXS:\n return dequantize_row_iq2_xxs_cuda;\n case GGML_TYPE_IQ2_XS:\n return dequantize_row_iq2_xs_cuda;\n case GGML_TYPE_IQ2_S:\n return dequantize_row_iq2_s_cuda;\n case GGML_TYPE_IQ3_XXS:\n return dequantize_row_iq3_xxs_cuda;\n case GGML_TYPE_IQ1_S:\n return dequantize_row_iq1_s_cuda;\n case GGML_TYPE_IQ1_M:\n return dequantize_row_iq1_m_cuda;\n case GGML_TYPE_IQ4_NL:\n return dequantize_row_iq4_nl_cuda;\n case GGML_TYPE_IQ4_XS:\n return dequantize_row_iq4_xs_cuda;\n case GGML_TYPE_IQ3_S:\n return dequantize_row_iq3_s_cuda;\n case GGML_TYPE_MXFP4:\n return dequantize_row_mxfp4_cuda;\n case GGML_TYPE_F32:\n return convert_unary_cont_cuda;\n case GGML_TYPE_BF16:\n return convert_unary_cont_cuda;\n default:\n return nullptr;\n }\n}\n\nto_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {\n switch (type) {\n case GGML_TYPE_Q4_0:\n return dequantize_row_q4_0_cuda;\n case GGML_TYPE_Q4_1:\n return dequantize_row_q4_1_cuda;\n case GGML_TYPE_Q5_0:\n return dequantize_block_cont_cuda;\n case GGML_TYPE_Q5_1:\n return dequantize_block_cont_cuda;\n case GGML_TYPE_Q8_0:\n return dequantize_block_cont_cuda;\n case GGML_TYPE_Q2_K:\n return dequantize_row_q2_K_cuda;\n case GGML_TYPE_Q3_K:\n return dequantize_row_q3_K_cuda;\n case GGML_TYPE_Q4_K:\n return dequantize_row_q4_K_cuda;\n case GGML_TYPE_Q5_K:\n return dequantize_row_q5_K_cuda;\n case GGML_TYPE_Q6_K:\n return dequantize_row_q6_K_cuda;\n case GGML_TYPE_IQ2_XXS:\n return dequantize_row_iq2_xxs_cuda;\n case GGML_TYPE_IQ2_XS:\n return dequantize_row_iq2_xs_cuda;\n case GGML_TYPE_IQ2_S:\n return dequantize_row_iq2_s_cuda;\n case GGML_TYPE_IQ3_XXS:\n return dequantize_row_iq3_xxs_cuda;\n case GGML_TYPE_IQ1_S:\n return dequantize_row_iq1_s_cuda;\n case GGML_TYPE_IQ1_M:\n return dequantize_row_iq1_m_cuda;\n case GGML_TYPE_IQ4_NL:\n return dequantize_row_iq4_nl_cuda;\n case GGML_TYPE_IQ4_XS:\n return dequantize_row_iq4_xs_cuda;\n case GGML_TYPE_IQ3_S:\n return dequantize_row_iq3_s_cuda;\n case GGML_TYPE_MXFP4:\n return dequantize_row_mxfp4_cuda;\n case GGML_TYPE_F16:\n return convert_unary_cont_cuda;\n case GGML_TYPE_BF16:\n return convert_unary_cont_cuda;\n default:\n return nullptr;\n }\n}\n\nto_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) {\n switch (type) {\n case GGML_TYPE_F32:\n return convert_unary_cuda;\n case GGML_TYPE_Q4_0:\n return dequantize_block_cuda;\n case GGML_TYPE_Q4_1:\n return dequantize_block_cuda;\n case GGML_TYPE_Q5_0:\n return dequantize_block_cuda;\n case GGML_TYPE_Q5_1:\n return dequantize_block_cuda;\n case GGML_TYPE_Q8_0:\n return dequantize_block_cuda;\n case GGML_TYPE_BF16:\n return convert_unary_cuda;\n default:\n return nullptr;\n }\n}\n\nto_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) {\n switch (type) {\n case GGML_TYPE_F32:\n return convert_unary_cuda;\n case GGML_TYPE_Q4_0:\n return dequantize_block_cuda;\n case GGML_TYPE_Q4_1:\n return dequantize_block_cuda;\n case GGML_TYPE_Q5_0:\n return dequantize_block_cuda;\n case GGML_TYPE_Q5_1:\n return dequantize_block_cuda;\n case GGML_TYPE_Q8_0:\n return dequantize_block_cuda;\n case GGML_TYPE_F16:\n return convert_unary_cuda;\n default:\n return nullptr;\n }\n}\n\nto_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) {\n switch (type) {\n case GGML_TYPE_F16:\n return convert_unary_cuda;\n case GGML_TYPE_Q4_0:\n return dequantize_block_cuda;\n case GGML_TYPE_Q4_1:\n return dequantize_block_cuda;\n case GGML_TYPE_Q5_0:\n return dequantize_block_cuda;\n case GGML_TYPE_Q5_1:\n return dequantize_block_cuda;\n case GGML_TYPE_Q8_0:\n return dequantize_block_cuda;\n case GGML_TYPE_BF16:\n return convert_unary_cuda;\n default:\n return nullptr;\n }\n}\n" }, { "path": "src/ggml-cuda/convert.cuh", "content": "#pragma once\n#include \"common.cuh\"\n\n#define CUDA_DEQUANTIZE_BLOCK_SIZE 256\n\ntemplate\nusing to_t_cuda_t = void (*)(const void * x, T * y, int64_t k, cudaStream_t stream);\n\ntypedef to_t_cuda_t to_fp32_cuda_t;\ntypedef to_t_cuda_t to_fp16_cuda_t;\ntypedef to_t_cuda_t to_bf16_cuda_t;\n\nto_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type);\n\nto_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type);\n\nto_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type);\n\n// TODO more general support for non-contiguous inputs\n\ntemplate\nusing to_t_nc_cuda_t = void (*)(const void * x, T * y,\n int64_t ne00, int64_t ne01, int64_t ne02, int64_t ne03,\n int64_t s01, int64_t s02, int64_t s03, cudaStream_t stream);\n\ntypedef to_t_nc_cuda_t to_fp32_nc_cuda_t;\ntypedef to_t_nc_cuda_t to_fp16_nc_cuda_t;\ntypedef to_t_nc_cuda_t to_bf16_nc_cuda_t;\n\nto_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type);\nto_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type);\nto_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type);\n\ntemplate\n __host__ __device__ inline dst_t ggml_cuda_cast(src_t x) {\n if constexpr (std::is_same_v) {\n return x;\n } else if constexpr(std::is_same_v) {\n return __float2bfloat16(float(x));\n } else if constexpr(std::is_same_v) {\n return __bfloat162float(x);\n } else if constexpr(std::is_same_v && std::is_same_v) {\n return __float22half2_rn(x);\n } else if constexpr(std::is_same_v && std::is_same_v) {\n // bypass compile error on cuda 12.0.1\n#ifdef GGML_USE_HIP\n return __float22bfloat162_rn(x);\n#else\n return {x.x, x.y};\n#endif // GGML_USE_HIP\n } else if constexpr(std::is_same_v) {\n return int32_t(x);\n } else {\n return float(x);\n }\n}\n" }, { "path": "src/ggml-cuda/count-equal.cu", "content": "#include \"common.cuh\"\n#include \"count-equal.cuh\"\n\n#include \n\ntemplate \nstatic __global__ void count_equal(const T * __restrict__ x, const T * __restrict__ y, int64_t * __restrict__ dst, const int64_t dk, const int64_t k) {\n const int64_t i0 = (int64_t) blockIdx.x*dk;\n const int64_t i1 = min(i0 + dk, k);\n\n int nequal = 0;\n\n for (int64_t i = i0 + threadIdx.x; i < i1; i += WARP_SIZE) {\n const T xi = x[i];\n const T yi = y[i];\n nequal += xi == yi;\n }\n\n nequal = warp_reduce_sum(nequal);\n\n if (threadIdx.x != 0) {\n return;\n }\n\n atomicAdd((int *) dst, nequal);\n}\n\nvoid ggml_cuda_count_equal(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == src1->type);\n GGML_ASSERT( dst->type == GGML_TYPE_I64);\n\n GGML_ASSERT(ggml_are_same_shape(src0, src1));\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src1));\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n int64_t * dst_d = (int64_t *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;\n\n const int64_t ne = ggml_nelements(src0);\n GGML_ASSERT(ne < (1 << 30) && \"atomicAdd implementation only supports int\");\n const int64_t dne = GGML_PAD((ne + 4*nsm - 1) / (4*nsm), CUDA_COUNT_EQUAL_CHUNK_SIZE);\n\n CUDA_CHECK(cudaMemsetAsync(dst_d, 0, ggml_nbytes(dst), stream));\n\n const dim3 blocks_dim(WARP_SIZE, 1, 1);\n const dim3 blocks_num(std::min((int64_t)4*nsm, (ne + CUDA_COUNT_EQUAL_CHUNK_SIZE - 1)/CUDA_COUNT_EQUAL_CHUNK_SIZE), 1, 1);\n\n switch (src0->type) {\n case GGML_TYPE_I32: {\n const int * src0_d = (const int *) src0->data;\n const int * src1_d = (const int *) src1->data;\n count_equal<<>>(src0_d, src1_d, dst_d, dne, ne);\n } break;\n default:\n GGML_ASSERT(false);\n break;\n }\n}\n" }, { "path": "src/ggml-cuda/count-equal.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_COUNT_EQUAL_CHUNK_SIZE 128\n\nvoid ggml_cuda_count_equal(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/cp-async.cuh", "content": "// Simplified API for asynchronous data loading.\n\n#include \"common.cuh\"\n\n\nstatic __device__ __forceinline__ unsigned int ggml_cuda_cvta_generic_to_shared(void * generic_ptr) {\n#ifdef CP_ASYNC_AVAILABLE\n return __cvta_generic_to_shared(generic_ptr);\n#else\n GGML_UNUSED(generic_ptr);\n NO_DEVICE_CODE;\n return 0;\n#endif // CP_ASYNC_AVAILABLE\n}\n\n// Copies data from global to shared memory, cg == cache global.\n// Both the src and dst pointers must be aligned to 16 bit.\n// Shared memory uses 32 bit addressing, the pointer is passed as unsigned int.\n// Generic pointers can be converted to 32 bit shared memory pointers using __cvta_generic_to_shared.\n// Only the 16 bit copy is exposed because 4 and 8 bit copies did not yield performance improvements.\ntemplate \nstatic __device__ __forceinline__ void cp_async_cg_16(const unsigned int dst, const void * src) {\n static_assert(preload == 0 || preload == 64 || preload == 128 || preload == 256, \"bad preload\");\n#ifdef CP_ASYNC_AVAILABLE\n#if CUDART_VERSION >= 11040\n if (preload == 256) {\n asm volatile(\"cp.async.cg.shared.global.L2::256B [%0], [%1], 16;\"\n : : \"r\"(dst), \"l\"(src));\n } else if (preload == 128) {\n asm volatile(\"cp.async.cg.shared.global.L2::128B [%0], [%1], 16;\"\n : : \"r\"(dst), \"l\"(src));\n } else if (preload == 64) {\n asm volatile(\"cp.async.cg.shared.global.L2::64B [%0], [%1], 16;\"\n : : \"r\"(dst), \"l\"(src));\n } else\n#endif // CUDART_VERSION >= 11040\n {\n asm volatile(\"cp.async.cg.shared.global [%0], [%1], 16;\"\n : : \"r\"(dst), \"l\"(src));\n }\n#else\n GGML_UNUSED(dst);\n GGML_UNUSED(src);\n NO_DEVICE_CODE;\n#endif // CP_ASYNC_AVAILABLE\n}\n\n// Makes each thread wait until its asynchronous data copies are done.\n// This does NOT provide any additional synchronization.\n// In particular, when copying data with multiple warps a call to __syncthreads will be needed.\nstatic __device__ __forceinline__ void cp_async_wait_all() {\n#ifdef CP_ASYNC_AVAILABLE\n asm volatile(\"cp.async.wait_all;\");\n#else\n NO_DEVICE_CODE;\n#endif // CP_ASYNC_AVAILABLE\n}\n" }, { "path": "src/ggml-cuda/cpy-utils.cuh", "content": "#pragma once\n\n#include \"ggml-common.h\"\n#include \"convert.cuh\"\n\nstatic __device__ __forceinline__ int best_index_int8(int n, const int8_t * val, float x) {\n if (x <= val[0]) return 0;\n if (x >= val[n-1]) return n-1;\n int ml = 0, mu = n-1;\n while (mu-ml > 1) {\n int mav = (ml+mu)/2;\n if (x < val[mav]) mu = mav; else ml = mav;\n }\n return x - val[mu-1] < val[mu] - x ? mu-1 : mu;\n}\n\nstatic __device__ void quantize_f32_q4_0_block(const float * __restrict__ x, block_q4_0 * __restrict__ y) {\n float amax = 0.0f;\n float vmax = 0.0f;\n\n for (int j = 0; j < QK4_0; ++j) {\n const float v = x[j];\n if (amax < fabsf(v)) {\n amax = fabsf(v);\n vmax = v;\n }\n }\n\n const float d = vmax / -8;\n const float id = d ? 1.0f/d : 0.0f;\n\n y->d = d;\n\n for (int j = 0; j < QK4_0/2; ++j) {\n const float x0 = x[0 + j]*id;\n const float x1 = x[QK4_0/2 + j]*id;\n\n const uint8_t xi0 = min(15, (int8_t)(x0 + 8.5f));\n const uint8_t xi1 = min(15, (int8_t)(x1 + 8.5f));\n\n y->qs[j] = xi0;\n y->qs[j] |= xi1 << 4;\n }\n}\n\nstatic __device__ void quantize_f32_q4_1_block(const float * __restrict__ x, block_q4_1 * __restrict__ y) {\n float vmin = FLT_MAX;\n float vmax = -FLT_MAX;\n\n for (int j = 0; j < QK4_1; ++j) {\n const float v = x[j];\n if (v < vmin) vmin = v;\n if (v > vmax) vmax = v;\n }\n\n const float d = (vmax - vmin) / ((1 << 4) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y->dm.x = d;\n y->dm.y = vmin;\n\n for (int j = 0; j < QK4_1/2; ++j) {\n const float x0 = (x[0 + j] - vmin)*id;\n const float x1 = (x[QK4_1/2 + j] - vmin)*id;\n\n const uint8_t xi0 = min(15, (int8_t)(x0 + 0.5f));\n const uint8_t xi1 = min(15, (int8_t)(x1 + 0.5f));\n\n y->qs[j] = xi0;\n y->qs[j] |= xi1 << 4;\n }\n}\n\nstatic __device__ void quantize_f32_q5_0_block(const float * __restrict__ x, block_q5_0 * __restrict__ y) {\n float amax = 0.0f;\n float vmax = 0.0f;\n\n for (int j = 0; j < QK5_0; ++j) {\n const float v = x[j];\n if (amax < fabsf(v)) {\n amax = fabsf(v);\n vmax = v;\n }\n }\n\n const float d = vmax / -16;\n const float id = d ? 1.0f/d : 0.0f;\n\n y->d = d;\n\n uint32_t qh = 0;\n for (int j = 0; j < QK5_0/2; ++j) {\n const float x0 = x[0 + j]*id;\n const float x1 = x[QK5_0/2 + j]*id;\n\n const uint8_t xi0 = min(31, (int8_t)(x0 + 16.5f));\n const uint8_t xi1 = min(31, (int8_t)(x1 + 16.5f));\n\n y->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);\n }\n memcpy(y->qh, &qh, sizeof(qh));\n}\n\nstatic __device__ void quantize_f32_q5_1_block(const float * __restrict__ x, block_q5_1 * __restrict__ y) {\n float min = x[0];\n float max = x[0];\n\n for (int j = 1; j < QK5_1; ++j) {\n const float v = x[j];\n min = v < min ? v : min;\n max = v > max ? v : max;\n }\n\n const float d = (max - min) / 31;\n const float id = d ? 1.0f/d : 0.0f;\n\n y->dm.x = d;\n y->dm.y = min;\n\n uint32_t qh = 0;\n for (int j = 0; j < QK5_1/2; ++j) {\n const float x0 = (x[0 + j] - min)*id;\n const float x1 = (x[QK5_1/2 + j] - min)*id;\n\n const uint8_t xi0 = (uint8_t)(x0 + 0.5f);\n const uint8_t xi1 = (uint8_t)(x1 + 0.5f);\n\n y->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2);\n }\n memcpy(y->qh, &qh, sizeof(qh));\n}\n\nstatic __device__ void quantize_f32_q8_0_block(const float * __restrict__ x, block_q8_0 * __restrict__ y) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK8_0; j++) {\n const float v = x[j];\n amax = fmaxf(amax, fabsf(v));\n }\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y->d = d;\n\n for (int j = 0; j < QK8_0; ++j) {\n const float x0 = x[j]*id;\n y->qs[j] = roundf(x0);\n }\n}\n\nstatic __device__ void quantize_f32_iq4_nl_block(const float * __restrict__ x, block_iq4_nl * __restrict__ y) {\n float amax = 0.0f;\n float vmax = 0.0f;\n\n for (int j = 0; j < QK4_NL; ++j) {\n const float v = x[j];\n if (amax < fabsf(v)) {\n amax = fabsf(v);\n vmax = v;\n }\n }\n\n float d = vmax / kvalues_iq4nl[0];\n const float id = d ? 1.0f/d : 0.0f;\n\n float sumqx = 0, sumq2 = 0;\n for (int j = 0; j < QK4_NL/2; ++j) {\n const float x0 = x[0 + j]*id;\n const float x1 = x[QK4_NL/2 + j]*id;\n const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl, x0);\n const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl, x1);\n y->qs[j] = xi0 | (xi1 << 4);\n const float v0 = kvalues_iq4nl[xi0];\n const float v1 = kvalues_iq4nl[xi1];\n const float w0 = x[0 + j]*x[0 + j];\n const float w1 = x[QK4_NL/2 + j]*x[QK4_NL/2 + j];\n sumqx += w0*v0*x[j] + w1*v1*x[QK4_NL/2 + j];\n sumq2 += w0*v0*v0 + w1*v1*v1;\n }\n\n y->d = sumq2 > 0 ? sumqx/sumq2 : d;\n}\n\n// Wrapper functions for cpy.cu compatibility\nstatic __device__ void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {\n quantize_f32_q4_0_block((const float *)cxi, (block_q4_0 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {\n quantize_f32_q4_1_block((const float *)cxi, (block_q4_1 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {\n quantize_f32_q5_0_block((const float *)cxi, (block_q5_0 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) {\n quantize_f32_q5_1_block((const float *)cxi, (block_q5_1 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) {\n quantize_f32_q8_0_block((const float *)cxi, (block_q8_0 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_iq4_nl(const char * cxi, char * cdsti) {\n quantize_f32_iq4_nl_block((const float *)cxi, (block_iq4_nl *)cdsti);\n}\n\ntemplate\nstatic __device__ void cpy_1_scalar(const char * cxi, char * cdsti) {\n *(dst_t *) cdsti = ggml_cuda_cast(*(const src_t *) cxi);\n}\n" }, { "path": "src/ggml-cuda/cpy.cu", "content": "#include \"cpy.cuh\"\n#include \"dequantize.cuh\"\n#include \"cpy-utils.cuh\"\n#if defined(GGML_USE_MUSA) && defined(GGML_MUSA_MUDNN_COPY)\n#include \"ggml-musa/mudnn.cuh\"\n#endif // GGML_USE_MUSA && GGML_MUSA_MUDNN_COPY\n\ntypedef void (*cpy_kernel_t)(const char * cx, char * cdst);\n\nconst int CUDA_CPY_TILE_DIM_2D = 32; // 2D tile dimension for transposed blocks\nconst int CUDA_CPY_BLOCK_NM = 8; // block size of 3rd dimension if available\nconst int CUDA_CPY_BLOCK_ROWS = 8; // block dimension for marching through rows\n\ntemplate \nstatic __global__ void cpy_scalar(const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11,\n const int64_t nb12, const int64_t nb13) {\n const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i >= ne) {\n return;\n }\n\n // determine indices i03/i13, i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor\n // then combine those indices with the corresponding byte offsets to get the total offsets\n const int64_t i03 = i/(ne00 * ne01 * ne02);\n const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);\n const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;\n const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;\n const int64_t x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;\n\n const int64_t i13 = i/(ne10 * ne11 * ne12);\n const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);\n const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;\n const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;\n const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13;\n\n cpy_1(cx + x_offset, cdst + dst_offset);\n}\n\ntemplate \nstatic __global__ void cpy_scalar_transpose(const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11,\n const int64_t nb12, const int64_t nb13) {\n\n const T* src = reinterpret_cast(cx);\n T* dst = reinterpret_cast(cdst);\n\n const int64_t nmat = ne / (ne00 * ne01);\n const int64_t n = ne00 * ne01;\n\n const int x = blockIdx.x * CUDA_CPY_TILE_DIM_2D + threadIdx.x;\n const int y = blockIdx.y * CUDA_CPY_TILE_DIM_2D + threadIdx.y;\n const int tx = blockIdx.y * CUDA_CPY_TILE_DIM_2D + threadIdx.x; // transpose block offset\n const int ty = blockIdx.x * CUDA_CPY_TILE_DIM_2D + threadIdx.y;\n\n __shared__ float tile[2][CUDA_CPY_TILE_DIM_2D][CUDA_CPY_TILE_DIM_2D+1];\n int cur_tile_buf = 0;\n\n#pragma unroll\n for (int i = 0; i < CUDA_CPY_BLOCK_NM; ++i) {\n\n const unsigned int imat = blockIdx.z * CUDA_CPY_BLOCK_NM + i;\n if (imat >= nmat)\n break;\n\n#pragma unroll\n for (int j = 0; j < CUDA_CPY_TILE_DIM_2D; j += CUDA_CPY_BLOCK_ROWS) {\n if(x < ne01 && y + j < ne00){\n const int row = threadIdx.y+j;\n const int col = threadIdx.x * sizeof(float)/sizeof(T);\n T *tile2 = reinterpret_cast(tile[cur_tile_buf][row]);\n tile2[col] = src[imat*n + (y+j)*ne01 + x];\n }\n }\n\n __syncthreads();\n\n#pragma unroll\n for (int j = 0; j < CUDA_CPY_TILE_DIM_2D; j += CUDA_CPY_BLOCK_ROWS) {\n if (ty + j < ne01 && tx < ne00) {\n const int col = (threadIdx.y+j)*sizeof(float)/sizeof(T);\n const T *tile2 = reinterpret_cast(tile[cur_tile_buf][threadIdx.x]);\n dst[imat*n + (ty+j)*ne00 + tx] = tile2[col];\n }\n }\n\n cur_tile_buf = (cur_tile_buf + 1) % 2;\n }\n\n GGML_UNUSED_VARS(ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11,\n nb12, nb13);\n}\n\nstatic __device__ void cpy_blck_q8_0_f32(const char * cxi, char * cdsti) {\n float * cdstf = (float *)(cdsti);\n\n#pragma unroll\n for (int j = 0; j < QK8_0; j += 2) {\n float2 dq;\n dequantize_q8_0(cxi, 0, j, dq);\n *(cdstf + j) = dq.x;\n *(cdstf + j + 1) = dq.y;\n }\n}\n\ntemplate\nstatic __device__ void cpy_blck_q_f32(const char * cxi, char * cdsti) {\n float * cdstf = (float *)(cdsti);\n\n#pragma unroll\n for (int j = 0; j < qk/2; j++) {\n float2 dq;\n dequant(cxi, 0, j, dq);\n *(cdstf + j) = dq.x;\n *(cdstf + j + qk/2) = dq.y;\n }\n}\n\ntemplate \nstatic __global__ void cpy_f32_q(const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11,\n const int64_t nb12, const int64_t nb13) {\n const int64_t i = ((int64_t)blockDim.x*blockIdx.x + threadIdx.x)*qk;\n\n if (i >= ne) {\n return;\n }\n\n const int64_t i03 = i/(ne00 * ne01 * ne02);\n const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);\n const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;\n const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;\n const int64_t x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;\n\n const int64_t i13 = i/(ne10 * ne11 * ne12);\n const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);\n const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;\n const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;\n const int64_t dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13;\n\n cpy_blck(cx + x_offset, cdst + dst_offset);\n}\n\ntemplate \nstatic __global__ void cpy_q_f32(const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11,\n const int64_t nb12, const int64_t nb13) {\n const int64_t i = ((int64_t)blockDim.x*blockIdx.x + threadIdx.x)*qk;\n\n if (i >= ne) {\n return;\n }\n\n const int64_t i03 = i/(ne00 * ne01 * ne02);\n const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);\n const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;\n const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;\n const int64_t x_offset = (i00/qk)*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;\n\n const int64_t i13 = i/(ne10 * ne11 * ne12);\n const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);\n const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;\n const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;\n const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13;\n\n cpy_blck(cx + x_offset, cdst + dst_offset);\n}\n\ntemplate\nstatic __global__ void cpy_scalar_contiguous(const char * cx, char * cdst, const int64_t ne) {\n const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i >= ne) {\n return;\n }\n\n const src_t * x = (const src_t *) cx;\n dst_t * dst = (dst_t *) cdst;\n\n dst[i] = ggml_cuda_cast(x[i]);\n}\n\ntemplate\nstatic void ggml_cpy_scalar_contiguous_cuda(\n const char * cx, char * cdst, const int64_t ne,\ncudaStream_t stream) {\n\n const int64_t num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_scalar_contiguous<<>>\n (cx, cdst, ne);\n}\n\ntemplate\nstatic void ggml_cpy_scalar_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n if (transposed) {\n GGML_ASSERT(ne == ne00*ne01*ne02); // ne[3] is 1 assumed\n int64_t ne00n, ne01n, ne02n;\n if (nb00 <= nb02) { // most likely safe to handle nb00 = nb02 case here\n ne00n = ne00;\n ne01n = ne01;\n ne02n = ne02;\n } else {\n ne00n = ne00;\n ne01n = ne01*ne02;\n ne02n = 1;\n }\n\n int64_t grid_x = (ne01n + CUDA_CPY_TILE_DIM_2D - 1) / CUDA_CPY_TILE_DIM_2D;\n int64_t grid_y = (ne00n + CUDA_CPY_TILE_DIM_2D - 1) / CUDA_CPY_TILE_DIM_2D;\n int64_t grid_z = (ne/(ne01n*ne00n) + CUDA_CPY_BLOCK_NM - 1) / CUDA_CPY_BLOCK_NM;\n GGML_ASSERT(grid_x < UINT_MAX);\n GGML_ASSERT(grid_y < USHRT_MAX);\n GGML_ASSERT(grid_z < USHRT_MAX);\n dim3 dimGrid(grid_x, grid_y, grid_z);\n dim3 dimBlock(CUDA_CPY_TILE_DIM_2D, CUDA_CPY_BLOCK_ROWS, 1);\n cpy_scalar_transpose<<>>\n (cx, cdst, ne, ne00n, ne01n, ne02n, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n } else {\n const int64_t num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_scalar><<>>\n (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n }\n}\n\nstatic void ggml_cpy_f32_q8_0_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n GGML_ASSERT(ne % QK8_0 == 0);\n const int64_t num_blocks = ne / QK8_0;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_f32_q<<>>\n (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q8_0_f32_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n const int64_t num_blocks = ne;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_q_f32<<>>\n (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_q4_0_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n GGML_ASSERT(ne % QK4_0 == 0);\n const int64_t num_blocks = ne / QK4_0;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_f32_q<<>>\n (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q4_0_f32_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02,\n const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,\n const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,\n cudaStream_t stream) {\n const int64_t num_blocks = ne;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_q_f32, QK4_0><<>>(\n cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_q4_1_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n GGML_ASSERT(ne % QK4_1 == 0);\n const int64_t num_blocks = ne / QK4_1;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_f32_q<<>>\n (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q4_1_f32_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02,\n const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,\n const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,\n cudaStream_t stream) {\n const int64_t num_blocks = ne;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_q_f32, QK4_1><<>>(\n cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_q5_0_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n GGML_ASSERT(ne % QK5_0 == 0);\n const int64_t num_blocks = ne / QK5_0;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_f32_q<<>>\n (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q5_0_f32_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02,\n const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,\n const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,\n cudaStream_t stream) {\n const int64_t num_blocks = ne;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_q_f32, QK5_0><<>>(\n cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_q5_1_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n GGML_ASSERT(ne % QK5_1 == 0);\n const int64_t num_blocks = ne / QK5_1;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_f32_q<<>>\n (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q5_1_f32_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02,\n const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,\n const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,\n cudaStream_t stream) {\n const int64_t num_blocks = ne;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_q_f32, QK5_1><<>>(\n cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_iq4_nl_cuda(\n const char * cx, char * cdst, const int64_t ne,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n GGML_ASSERT(ne % QK4_NL == 0);\n const int64_t num_blocks = ne / QK4_NL;\n GGML_ASSERT(num_blocks < UINT_MAX);\n cpy_f32_q<<>>\n (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nvoid ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) {\n const int64_t ne = ggml_nelements(src0);\n GGML_ASSERT(ne == ggml_nelements(src1));\n\n const int64_t ne00 = src0->ne[0];\n const int64_t ne01 = src0->ne[1];\n const int64_t ne02 = src0->ne[2];\n\n //GGML_ASSERT(src0->ne[3] == 1);\n\n const int64_t nb00 = src0->nb[0];\n const int64_t nb01 = src0->nb[1];\n const int64_t nb02 = src0->nb[2];\n const int64_t nb03 = src0->nb[3];\n\n const int64_t ne10 = src1->ne[0];\n const int64_t ne11 = src1->ne[1];\n const int64_t ne12 = src1->ne[2];\n\n //GGML_ASSERT(src1->ne[3] == 1);\n\n const int64_t nb10 = src1->nb[0];\n const int64_t nb11 = src1->nb[1];\n const int64_t nb12 = src1->nb[2];\n const int64_t nb13 = src1->nb[3];\n\n cudaStream_t main_stream = ctx.stream();\n\n char * src0_ddc = (char *) src0->data;\n char * src1_ddc = (char *) src1->data;\n\n const bool contiguous_srcs = ggml_is_contiguous(src0) && ggml_is_contiguous(src1);\n const bool can_be_transposed = nb01 == (int64_t)ggml_element_size(src0) &&\n src0->ne[3] == 1 && nb02 == ne00 * ne01 * (int64_t)ggml_element_size(src0);\n\n if (src0->type == src1->type && contiguous_srcs) {\n GGML_ASSERT(ggml_nbytes(src0) == ggml_nbytes(src1));\n#if defined(GGML_USE_MUSA) && defined(GGML_MUSA_MUDNN_COPY)\n if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) {\n CUDA_CHECK(mudnnMemcpyAsync(ctx, src1, src0));\n } else\n#endif // GGML_USE_MUSA && GGML_MUSA_MUDNN_COPY\n {\n CUDA_CHECK(cudaMemcpyAsync(src1_ddc, src0_ddc, ggml_nbytes(src0), cudaMemcpyDeviceToDevice, main_stream));\n }\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {\n if (can_be_transposed) {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_BF16) {\n if (contiguous_srcs) {\n ggml_cpy_scalar_contiguous_cuda\n (src0_ddc, src1_ddc, ne, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {\n if (contiguous_srcs) {\n ggml_cpy_scalar_contiguous_cuda\n (src0_ddc, src1_ddc, ne, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {\n ggml_cpy_f32_q8_0_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q8_0 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q8_0_f32_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {\n ggml_cpy_f32_q4_0_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q4_0 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q4_0_f32_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {\n ggml_cpy_f32_q4_1_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q4_1 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q4_1_f32_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) {\n ggml_cpy_f32_q5_0_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q5_0_f32_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {\n ggml_cpy_f32_iq4_nl_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {\n ggml_cpy_f32_q5_1_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q5_1 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q5_1_f32_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {\n if (can_be_transposed) {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_BF16) {\n if (contiguous_srcs) {\n ggml_cpy_scalar_contiguous_cuda\n (src0_ddc, src1_ddc, ne, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {\n if (contiguous_srcs) {\n ggml_cpy_scalar_contiguous_cuda\n (src0_ddc, src1_ddc, ne, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16) {\n if (can_be_transposed) {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F16) {\n if (contiguous_srcs) {\n ggml_cpy_scalar_contiguous_cuda\n (src0_ddc, src1_ddc, ne, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32) {\n if (contiguous_srcs) {\n ggml_cpy_scalar_contiguous_cuda\n (src0_ddc, src1_ddc, ne, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32) {\n if (can_be_transposed) {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32) {\n if (contiguous_srcs) {\n ggml_cpy_scalar_contiguous_cuda\n (src0_ddc, src1_ddc, ne, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_F32) {\n if (contiguous_srcs) {\n ggml_cpy_scalar_contiguous_cuda\n (src0_ddc, src1_ddc, ne, main_stream);\n } else {\n ggml_cpy_scalar_cuda\n (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n }\n } else {\n GGML_ABORT(\"%s: unsupported type combination (%s to %s)\\n\", __func__,\n ggml_type_name(src0->type), ggml_type_name(src1->type));\n }\n}\n\nvoid ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n ggml_cuda_cpy(ctx, src0, dst);\n}\n" }, { "path": "src/ggml-cuda/cpy.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_CPY_BLOCK_SIZE 64\n\nvoid ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1);\n\nvoid ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/cross-entropy-loss.cu", "content": "#include \"common.cuh\"\n#include \"cross-entropy-loss.cuh\"\n#include \"sum.cuh\"\n\n#include \n#include \n\ntemplate \nstatic __global__ void cross_entropy_loss_f32(\n const float * __restrict__ logits, const float * __restrict__ labels, float * __restrict__ dst, const int nclasses, const int k) {\n extern __shared__ float tmp[];\n\n logits += int64_t(blockIdx.x)*nclasses;\n labels += int64_t(blockIdx.x)*nclasses;\n\n // Find maximum for softmax:\n float max_logit = -INFINITY;\n for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n const float val = logits[i];\n max_logit = fmaxf(max_logit, val);\n\n if (use_shared) {\n tmp[i] = val;\n }\n }\n max_logit = warp_reduce_max(max_logit);\n\n // Calculate log(softmax(logits)) which is just logits - max:\n float sum = 0.0f;\n for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n const float logit_i = use_shared ? tmp[i] : logits[i];\n sum += expf(logit_i - max_logit);\n }\n sum = warp_reduce_sum(sum);\n sum = logf(sum);\n\n // log(exp(logits - max) / sum) = (logits - max) - log(sum)\n float loss = 0.0f;\n for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n const float logit_i = use_shared ? tmp[i] : logits[i];\n loss += (logit_i - max_logit - sum) * labels[i];\n }\n loss = -warp_reduce_sum(loss) / (float)k;\n\n if (threadIdx.x != 0) {\n return;\n }\n\n dst[blockIdx.x] = loss;\n}\n\ntemplate \nstatic __global__ void cross_entropy_loss_back_f32(\n const float * __restrict__ grad, const float * __restrict__ logits, const float * __restrict__ labels,\n float * __restrict__ dst, const int nclasses) {\n extern __shared__ float tmp[];\n\n logits += int64_t(blockIdx.x)*nclasses;\n labels += int64_t(blockIdx.x)*nclasses;\n dst += int64_t(blockIdx.x)*nclasses;\n\n float maxval = -INFINITY;\n for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n const float val = logits[i];\n maxval = fmaxf(maxval, val);\n\n if (use_shared) {\n tmp[i] = val;\n }\n }\n maxval = warp_reduce_max(maxval);\n\n float sum = 0.0f;\n for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n const float val = expf((use_shared ? tmp[i] : logits[i]) - maxval);\n sum += val;\n\n if (use_shared) {\n tmp[i] = val;\n } else {\n dst[i] = val;\n }\n }\n sum = warp_reduce_sum(sum);\n const float sm_scale = 1.0f/sum;\n\n const float d_by_nrows = *grad/gridDim.x;\n for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n const float val = use_shared ? tmp[i] : dst[i];\n dst[i] = (val*sm_scale - labels[i])*d_by_nrows;\n }\n}\n\nvoid ggml_cuda_cross_entropy_loss(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src1));\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n const int64_t ne00 = src0->ne[0];\n const int64_t nrows = ggml_nrows(src0);\n\n const float * src0_d = (const float *) src0->data;\n const float * src1_d = (const float *) src1->data;\n float * dst_d = (float *) dst->data;\n\n ggml_cuda_pool & pool = ctx.pool();\n cudaStream_t stream = ctx.stream();\n\n const dim3 blocks_dim(WARP_SIZE, 1, 1);\n const dim3 blocks_num(nrows, 1, 1);\n const size_t nbytes_shared = ne00*sizeof(float);\n\n const int id = ggml_cuda_get_device();\n const size_t smpbo = ggml_cuda_info().devices[id].smpbo;\n\n ggml_cuda_pool_alloc dst_tmp(pool, blocks_num.x);\n\n if (nbytes_shared <= smpbo) {\n CUDA_SET_SHARED_MEMORY_LIMIT((cross_entropy_loss_f32), smpbo);\n cross_entropy_loss_f32<<>>(src0_d, src1_d, dst_tmp.ptr, ne00, nrows);\n } else {\n cross_entropy_loss_f32<<>>(src0_d, src1_d, dst_tmp.ptr, ne00, nrows);\n }\n CUDA_CHECK(cudaGetLastError());\n\n // Combine results from individual blocks:\n sum_f32_cuda(pool, dst_tmp.ptr, dst_d, blocks_num.x, stream);\n}\n\nvoid ggml_cuda_cross_entropy_loss_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * grad = dst->src[0];\n const ggml_tensor * src0f = dst->src[1];\n const ggml_tensor * src1f = dst->src[2];\n\n GGML_ASSERT(src0f->type == GGML_TYPE_F32);\n GGML_ASSERT(src1f->type == GGML_TYPE_F32);\n GGML_ASSERT( grad->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_scalar(grad));\n GGML_ASSERT(ggml_is_contiguous(src0f));\n GGML_ASSERT(ggml_is_contiguous(src1f));\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_are_same_shape(src0f, src1f));\n GGML_ASSERT(ggml_are_same_shape(src0f, dst));\n\n const int64_t ne00 = src0f->ne[0];\n const int64_t nrows = ggml_nrows(src0f);\n\n const float * grad_d = (const float *) grad->data;\n const float * src0f_d = (const float *) src0f->data;\n const float * src1f_d = (const float *) src1f->data;\n float * dst_d = (float *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n const dim3 blocks_dim(WARP_SIZE, 1, 1);\n const dim3 blocks_num(nrows, 1, 1);\n const size_t nbytes_shared = ne00*sizeof(float);\n\n const int id = ggml_cuda_get_device();\n const size_t smpbo = ggml_cuda_info().devices[id].smpbo;\n\n if (nbytes_shared <= smpbo) {\n CUDA_SET_SHARED_MEMORY_LIMIT((cross_entropy_loss_back_f32), smpbo);\n cross_entropy_loss_back_f32<<>>(grad_d, src0f_d, src1f_d, dst_d, ne00);\n } else {\n cross_entropy_loss_back_f32<<>>(grad_d, src0f_d, src1f_d, dst_d, ne00);\n }\n}\n" }, { "path": "src/ggml-cuda/cross-entropy-loss.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_CROSS_ENTROPY_LOSS_BLOCK_SIZE 256\n\nvoid ggml_cuda_cross_entropy_loss(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_cross_entropy_loss_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/cumsum.cu", "content": "#include \n#include \"cumsum.cuh\"\n#include \"convert.cuh\"\n#include \"ggml-cuda/common.cuh\"\n#include \"ggml.h\"\n\n#ifdef GGML_CUDA_USE_CUB\n# include \n#endif // GGML_CUDA_USE_CUB\n\ntemplate\nstatic __global__ void cumsum_cub_kernel(\n const T * __restrict__ src,\n T * __restrict__ dst,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t s01, const int64_t s02, const int64_t s03,\n const int64_t s1, const int64_t s2, const int64_t s3) {\n#ifdef GGML_CUDA_USE_CUB\n using BlockScanT = cub::BlockScan;\n\n __shared__ typename BlockScanT::TempStorage temp_storage;\n __shared__ T block_carry;\n\n const int tid = threadIdx.x;\n constexpr int UNROLL_FACTOR = 4;\n constexpr int TILE_SIZE = BLOCK_SIZE * UNROLL_FACTOR;\n\n const int64_t i1 = blockIdx.x;\n const int64_t i2 = blockIdx.y;\n const int64_t i3 = blockIdx.z;\n\n if (i1 >= ne01 || i2 >= ne02 || i3 >= ne03) {\n return;\n }\n\n const T * src_row = src + i1 * s01 + i2 * s02 + i3 * s03;\n T * dst_row = dst + i1 * s1 + i2 * s2 + i3 * s3;\n\n if (tid == 0) {\n block_carry = 0;\n }\n __syncthreads();\n\n for (int64_t start = 0; start < ne00; start += TILE_SIZE) {\n T items[UNROLL_FACTOR];\n T thread_sum = T(0);\n\n#pragma unroll\n for (int i = 0; i < UNROLL_FACTOR; i++) {\n int64_t idx = start + tid * UNROLL_FACTOR + i;\n T val = (idx < ne00) ? src_row[idx] : T(0);\n thread_sum += val;\n items[i] = thread_sum;\n }\n\n // Block-wide scan on thread sums\n T thread_prefix;\n T block_total;\n BlockScanT(temp_storage).InclusiveSum(thread_sum, thread_prefix, block_total);\n __syncthreads();\n\n // Add offset to each item and store\n T thread_offset = thread_prefix - thread_sum + block_carry;\n#pragma unroll\n for (int i = 0; i < UNROLL_FACTOR; i++) {\n int64_t idx = start + tid * UNROLL_FACTOR + i;\n if (idx < ne00) {\n dst_row[idx] = items[i] + thread_offset;\n }\n }\n\n __syncthreads();\n\n // Update carry for next tile\n if (tid == 0) {\n block_carry += block_total;\n }\n }\n#else\n NO_DEVICE_CODE;\n#endif // GGML_CUDA_USE_CUB\n}\n\n// Fallback kernel implementation\ntemplate\nstatic __global__ void cumsum_kernel(\n const T * src, T * dst,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t s00, const int64_t s01, const int64_t s02, const int64_t s03,\n const int64_t s0, const int64_t s1, const int64_t s2, const int64_t s3) {\n\n GGML_UNUSED_VARS(s00, s0);\n\n const int tid = threadIdx.x;\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n const int lane = tid % warp_size;\n const int warp = tid / warp_size;\n const int warps_per_block = blockDim.x / warp_size;\n\n extern __shared__ float smem[];\n float * s_vals = smem;\n float * s_warp_sums = smem + blockDim.x;\n float * s_carry = smem + blockDim.x + warps_per_block;\n float * s_chunk_total = s_carry + 1;\n\n // Initialize carry\n if (tid == 0) {\n *s_carry = 0.0f;\n }\n __syncthreads();\n\n const int64_t i3 = blockIdx.z;\n const int64_t i2 = blockIdx.y;\n const int64_t i1 = blockIdx.x;\n if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {\n return;\n }\n\n const T * src_row = src + i1 * s01 + i2 * s02 + i3 * s03;\n T * dst_row = dst + i1 * s1 + i2 * s2 + i3 * s3;\n\n // register blocking: process 4 elements per thread to hide latency\n // and reduce synchronization overhead\n constexpr int num_unroll = 4;\n T temp[num_unroll];\n\n for (int64_t i = 0; i < ne00; i += num_unroll * blockDim.x) {\n int64_t idx = i + tid * num_unroll;\n\n // thread local sequential scan\n temp[0] = (idx < ne00 ? src_row[idx] : T(0));\n#pragma unroll\n for (int64_t j = 1; j < num_unroll; j++) {\n temp[j] = temp[j - 1];\n if (idx + j < ne00) {\n temp[j] += src_row[idx + j];\n } else {\n temp[j] += 0;\n }\n }\n\n // last emenent is sum of all values assigned to thread\n float val = (idx < ne00) ? ggml_cuda_cast(temp[num_unroll - 1]) : 0.0f;\n\n // Warp inclusive scan\n val = warp_prefix_inclusive_sum(val);\n s_vals[tid] = val;\n\n if (lane == warp_size - 1) {\n s_warp_sums[warp] = val;\n }\n __syncthreads();\n\n // Exclusive scan of warp sums (warp 0 only)\n if (warp == 0) {\n float w = (tid < warps_per_block) ? s_warp_sums[tid] : 0.0f;\n float inc = warp_prefix_inclusive_sum(w);\n if (tid < warps_per_block) {\n s_warp_sums[tid] = inc - w; // exclusive sum\n }\n if (tid == warps_per_block - 1) {\n *s_chunk_total = inc; // total sum of this chunk\n }\n }\n __syncthreads();\n\n // write back results\n float carry = *s_carry;\n // calculate sum offset for this thread\n float final_val_offset = s_vals[tid] + s_warp_sums[warp] + carry - temp[num_unroll - 1];\n\n#pragma unroll\n for (int32_t j = 0; j < num_unroll; j++) {\n if (idx + j < ne00) {\n dst_row[idx + j] = temp[j] + ggml_cuda_cast(final_val_offset);\n }\n }\n\n __syncthreads();\n\n // Update carry for next chunk\n if (tid == 0) {\n *s_carry += *s_chunk_total;\n }\n }\n}\n\n#ifdef GGML_CUDA_USE_CUB\ntemplate \nstatic void cumsum_cub(ggml_cuda_pool & pool,\n const T * src,\n T * dst,\n int64_t ne,\n cudaStream_t stream) {\n size_t tmp_size = 0;\n\n // Query how much temp storage CUDA UnBound (CUB) needs\n cub::DeviceScan::InclusiveSum(nullptr, // d_temp_storage (null = just query size)\n tmp_size, // reference to size (will be set by CUB)\n src, // input pointer\n dst, // output pointer\n ne, // number of elements\n stream // CUDA stream to use\n );\n\n ggml_cuda_pool_alloc tmp_alloc(pool, tmp_size);\n\n // Perform the inclusive scan\n cub::DeviceScan::InclusiveSum((void *) tmp_alloc.get(), tmp_size, src, dst, ne, stream);\n}\n#endif // GGML_CUDA_USE_CUB\n\ntemplate\nstatic void cumsum_cuda(\n [[maybe_unused]] ggml_backend_cuda_context & ctx, const T * src, T * dst,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03,\n const int64_t nb0, const int64_t nb1, const int64_t nb2, const int64_t nb3,\n cudaStream_t stream) {\n\n const size_t type_size = sizeof(T);\n bool use_cub = false;\n#ifdef GGML_CUDA_USE_CUB\n // Check if we can use CUB (data must be contiguous along innermost dimension)\n const bool is_contiguous = (nb00 == type_size) && (nb0 == type_size);\n\n if (is_contiguous) {\n use_cub = true;\n const int64_t nrows = ne01 * ne02 * ne03;\n // TODO: Compare with DeviceSegmentedScan::InclusiveSegmentedSum for nrows > 1 once InclusiveSegmentedSum is released\n // Heuristics were determined as part of https://github.com/ggml-org/llama.cpp/pull/17004\n if (((nrows == 1) && (ne00 > 1024)) || (ne00 / nrows > 4096)) {\n for (int i=0; i= 1024) {\n cumsum_cub_kernel<<>>(\n src, dst,\n ne00, ne01, ne02, ne03,\n nb01 / type_size, nb02 / type_size, nb03 / type_size,\n nb1 / type_size, nb2 / type_size, nb3 / type_size\n );\n } else {\n cumsum_kernel<<>>(\n src, dst,\n ne00, ne01, ne02, ne03,\n nb00 / type_size, nb01 / type_size, nb02 / type_size, nb03 / type_size,\n nb0 / type_size, nb1 / type_size, nb2 / type_size, nb3 / type_size\n );\n }\n}\n\nvoid ggml_cuda_op_cumsum(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == dst->type);\n switch(src0->type) {\n case GGML_TYPE_F32:\n {\n cumsum_cuda(\n ctx, (const float *)src0->data, (float *)dst->data,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3],\n stream\n );\n } break;\n // We do not support those on CPU for now anyway, so comment them out because they cause errors on some CI platforms\n /*case GGML_TYPE_F16:\n {\n cumsum_cuda(\n (const half *)src0->data, (half *)dst->data,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3],\n stream\n );\n } break;\n case GGML_TYPE_BF16:\n {\n cumsum_cuda(\n (const nv_bfloat16 *)src0->data, (nv_bfloat16 *)dst->data,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3],\n stream\n );\n } break;*/\n default:\n GGML_ABORT(\"fatal error\");\n }\n}\n" }, { "path": "src/ggml-cuda/cumsum.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_CUMSUM_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_cumsum(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/dequantize.cuh", "content": "#include \"common.cuh\"\n\nstatic __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){\n const block_q4_0 * x = (const block_q4_0 *) vx;\n\n const float d = x[ib].d;\n\n const int vui = x[ib].qs[iqs];\n\n v.x = vui & 0xF;\n v.y = vui >> 4;\n\n v.x = (v.x - 8.0f) * d;\n v.y = (v.y - 8.0f) * d;\n}\n\nstatic __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int64_t ib, const int iqs, float2 & v){\n const block_q4_1 * x = (const block_q4_1 *) vx;\n\n const float2 dm = __half22float2(x[ib].dm);\n\n const int vui = x[ib].qs[iqs];\n\n v.x = vui & 0xF;\n v.y = vui >> 4;\n\n v.x = (v.x * dm.x) + dm.y;\n v.y = (v.y * dm.x) + dm.y;\n}\n\nstatic __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int64_t ib, const int iqs, float2 & v){\n const block_q5_0 * x = (const block_q5_0 *) vx;\n\n const float d = x[ib].d;\n\n uint32_t qh;\n memcpy(&qh, x[ib].qh, sizeof(qh));\n\n const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;\n const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;\n\n v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);\n v.y = ((x[ib].qs[iqs] >> 4) | xh_1);\n\n v.x = (v.x - 16.0f) * d;\n v.y = (v.y - 16.0f) * d;\n}\n\nstatic __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, float2 & v){\n const block_q5_1 * x = (const block_q5_1 *) vx;\n\n const float2 dm = __half22float2(x[ib].dm);\n\n uint32_t qh;\n memcpy(&qh, x[ib].qh, sizeof(qh));\n\n const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;\n const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;\n\n v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);\n v.y = ((x[ib].qs[iqs] >> 4) | xh_1);\n\n v.x = (v.x * dm.x) + dm.y;\n v.y = (v.y * dm.x) + dm.y;\n}\n\nstatic __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int64_t ib, const int iqs, float2 & v){\n const block_q8_0 * x = (const block_q8_0 *) vx;\n\n const float d = x[ib].d;\n\n v.x = x[ib].qs[iqs + 0];\n v.y = x[ib].qs[iqs + 1];\n\n v.x *= d;\n v.y *= d;\n}\n" }, { "path": "src/ggml-cuda/diag.cu", "content": "#include \"convert.cuh\"\n#include \"diag.cuh\"\n#include \"ggml.h\"\n\ntemplate \nstatic __global__ void diag_kernel(T * __restrict__ dst,\n const T * __restrict__ src,\n const int64_t ne0,\n const int64_t ne1,\n const int64_t ne2,\n const int64_t ne3,\n const int64_t total_elements) {\n const int64_t global_idx = blockIdx.x * blockDim.x + threadIdx.x;\n\n if (global_idx >= total_elements) {\n return;\n }\n\n const int64_t i0 = global_idx % ne0;\n const int64_t i1 = (global_idx / ne0) % ne1;\n const int64_t i2 = (global_idx / (ne0 * ne1)) % ne2;\n const int64_t i3 = global_idx / (ne0 * ne1 * ne2);\n\n const int64_t dst_idx = ((i3 * ne2 + i2) * ne1 + i1) * ne0 + i0;\n\n if (i0 == i1) {\n const int64_t batch_idx = i3 * ne2 + i2;\n const int64_t src_idx = batch_idx * ne0 + i0;\n dst[dst_idx] = src[src_idx];\n } else {\n dst[dst_idx] = ggml_cuda_cast(0);\n }\n GGML_UNUSED_VARS(ne3);\n}\n\nvoid ggml_cuda_op_diag(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n void * dst_d = dst->data;\n const void * src0_d = src0->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n const int64_t ne00 = src0->ne[0];\n const int64_t ne01 = src0->ne[1];\n const int64_t ne02 = src0->ne[2];\n const int64_t ne03 = src0->ne[3];\n\n const int64_t ne0 = dst->ne[0];\n const int64_t ne1 = dst->ne[1];\n const int64_t ne2 = dst->ne[2];\n const int64_t ne3 = dst->ne[3];\n\n GGML_ASSERT(ne00 == ne0);\n GGML_ASSERT(ne01 == 1);\n GGML_ASSERT(ne02 == ne2);\n GGML_ASSERT(ne03 == ne3);\n\n const int64_t n_elems = ggml_nelements(dst);\n const int64_t num_blocks = (n_elems + CUDA_DIAG_BLOCK_SIZE - 1) / CUDA_DIAG_BLOCK_SIZE;\n\n switch (dst->type) {\n case GGML_TYPE_F32:\n diag_kernel<<>>((float *) dst_d, (const float *) src0_d, ne0,\n ne1, ne2, ne3, n_elems);\n break;\n case GGML_TYPE_F16:\n diag_kernel<<>>((half *) dst_d, (const half *) src0_d, ne0,\n ne1, ne2, ne3, n_elems);\n break;\n default:\n GGML_ABORT(\"unsupported type\");\n }\n}\n" }, { "path": "src/ggml-cuda/diag.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_DIAG_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_diag(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/diagmask.cu", "content": "#include \"diagmask.cuh\"\n\nstatic __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) {\n const int col = blockDim.y*blockIdx.y + threadIdx.y;\n const int row = blockDim.x*blockIdx.x + threadIdx.x;\n\n if (col >= ncols) {\n return;\n }\n\n const int i = row*ncols + col;\n //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];\n //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU\n dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;\n}\n\nstatic void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) {\n const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1);\n const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE;\n const dim3 block_nums(nrows_x, block_num_x, 1);\n diag_mask_inf_f32<<>>(x, dst, ncols_x, rows_per_channel, n_past);\n}\n\nvoid ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *)src0->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n const int64_t ne00 = src0->ne[0];\n const int64_t ne01 = src0->ne[1];\n const int nrows0 = ggml_nrows(src0);\n\n const int n_past = ((int32_t *) dst->op_params)[0];\n\n diag_mask_inf_f32_cuda(src0_d, dst_d, ne00, nrows0, ne01, n_past, stream);\n}\n" }, { "path": "src/ggml-cuda/diagmask.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32\n\nvoid ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/fattn-common.cuh", "content": "#pragma once\n\n#include \"common.cuh\"\n#include \"convert.cuh\"\n#include \"vecdotq.cuh\"\n\n#include \n\n#define FATTN_KQ_STRIDE 256\n#define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction.\n#define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs.\n\n// log(2) = 0.6931, by adding this to the KQ maximum used for the softmax the numerical range representable\n// by the VKQ accumulators is effectively being shifted up by a factor of 2.\n// This reduces issues with numerical overflow but also causes larger values to be flushed to zero.\n// However, as the output from FlashAttention will usually be used as an input for a matrix multiplication this should be negligible.\n// Still, the value range should be shifted as much as necessary but as little as possible.\n// The macro on the following line shifts it by a factor of 2**3=8, as was needed to fix https://github.com/ggml-org/llama.cpp/issues/18606 .\n#define FATTN_KQ_MAX_OFFSET (3.0f*0.6931f)\n\ntypedef void (* fattn_kernel_t)(\n const char * __restrict__ Q,\n const char * __restrict__ K,\n const char * __restrict__ V,\n const char * __restrict__ mask,\n const char * __restrict__ sinks,\n const int * __restrict__ KV_max,\n float * __restrict__ dst,\n float2 * __restrict__ dst_meta,\n const float scale,\n const float max_bias,\n const float m0,\n const float m1,\n const uint32_t n_head_log2,\n const float logit_softcap,\n const int32_t ne00, const uint3 ne01, const int32_t ne02, const int32_t ne03,\n const int32_t nb01, const int32_t nb02, const int32_t nb03,\n const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13,\n const int32_t nb11, const int32_t nb12, const int64_t nb13,\n const int32_t nb21, const int32_t nb22, const int64_t nb23,\n const int32_t ne31, const int32_t ne32, const int32_t ne33,\n const int32_t nb31, const int32_t nb32, const int64_t nb33);\n\ntypedef float (*vec_dot_KQ_t)(\n const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds);\n\ntemplate \nstatic __device__ __forceinline__ float vec_dot_fattn_vec_KQ_f16(\n const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds_v) {\n\n const half2 * K_h2 = (const half2 *) K_c;\n GGML_UNUSED(Q_q8);\n GGML_UNUSED(Q_ds_v);\n\n constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += nthreads*cpy_ne) {\n __align__(16) half2 tmp[cpy_ne];\n ggml_cuda_memcpy_1(tmp, K_h2 + k_KQ_0 + (threadIdx.x % nthreads)*cpy_ne);\n#pragma unroll\n for (int k_KQ_1 = 0; k_KQ_1 < cpy_ne; ++k_KQ_1) {\n#ifdef V_DOT2_F32_F16_AVAILABLE\n ggml_cuda_mad(sum, tmp[k_KQ_1] , ((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]);\n#else\n ggml_cuda_mad(sum, __half22float2(tmp[k_KQ_1]), ((const float2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]);\n#endif // V_DOT2_F32_F16_AVAILABLE\n }\n }\n\n return sum;\n}\n\ntemplate\nstatic __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q4_0(\n const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {\n\n const block_q4_0 * K_q4_0 = (const block_q4_0 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ = k_KQ_0 + (nthreads == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads);\n\n const int ib = k_KQ / QI8_1;\n const int iqs4 = k_KQ % QI4_0;\n const int shift = k_KQ & (QI8_1/2);\n\n int v;\n ggml_cuda_memcpy_1(&v, K_q4_0[ib].qs + sizeof(int)*iqs4);\n v = (v >> shift) & 0x0F0F0F0F;\n const int u = Q_q8[k_KQ_0/nthreads];\n\n const int sumi = ggml_cuda_dp4a(v, u, 0);\n\n const float2 Q_ds = ((const float2 *) Q_ds_v)[k_KQ_0/nthreads];\n sum += __half2float(K_q4_0[ib].d) * (sumi*Q_ds.x - (8/QI8_1)*Q_ds.y);\n }\n\n return sum;\n}\n\ntemplate\nstatic __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q4_1(\n const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {\n\n const block_q4_1 * K_q4_1 = (const block_q4_1 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ = k_KQ_0 + (nthreads == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads);\n\n const int ib = k_KQ / QI8_1;\n const int iqs4 = k_KQ % QI4_1;\n const int shift = k_KQ & (QI8_1/2);\n\n int v;\n ggml_cuda_memcpy_1(&v, K_q4_1[ib].qs + sizeof(int)*iqs4);\n v = (v >> shift) & 0x0F0F0F0F;\n const int u = Q_q8[k_KQ_0/nthreads];\n\n const int sumi = ggml_cuda_dp4a(v, u, 0);\n\n const float2 K_dm = __half22float2(K_q4_1[ib].dm);\n const float2 Q_ds = ((const float2 *) Q_ds_v)[k_KQ_0/nthreads];\n\n sum += K_dm.x*Q_ds.x*sumi + K_dm.y*Q_ds.y/QI8_1;\n }\n\n return sum;\n}\n\ntemplate\nstatic __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q5_0(\n const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {\n\n const block_q5_0 * K_q5_0 = (const block_q5_0 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ = k_KQ_0 + (nthreads == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads);\n\n const int ib = k_KQ / QI8_1;\n const int iqs4 = k_KQ % QI5_0;\n const int iqs8 = k_KQ % QI8_1;\n const int shift = k_KQ & (QI8_1/2);\n\n int v;\n ggml_cuda_memcpy_1(&v, K_q5_0[ib].qs + sizeof(int)*iqs4);\n v = (v >> shift) & 0x0F0F0F0F;\n\n {\n int vh;\n ggml_cuda_memcpy_1(&vh, K_q5_0[ib].qh);\n vh >>= iqs8 * QI5_0;\n\n v |= (vh << 4) & 0x00000010; // 0 -> 4\n v |= (vh << 11) & 0x00001000; // 1 -> 12\n v |= (vh << 18) & 0x00100000; // 2 -> 20\n v |= (vh << 25) & 0x10000000; // 3 -> 28\n }\n\n const int u = Q_q8[k_KQ_0/nthreads];\n\n const int sumi = ggml_cuda_dp4a(v, u, 0);\n\n const float2 Q_ds = ((const float2 *) Q_ds_v)[k_KQ_0/nthreads];\n\n sum += __half2float(K_q5_0[ib].d) * (sumi*Q_ds.x - (16/QI8_1)*Q_ds.y);\n }\n\n return sum;\n}\n\ntemplate\nstatic __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q5_1(\n const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {\n\n const block_q5_1 * K_q5_1 = (const block_q5_1 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ = k_KQ_0 + (nthreads == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads);\n\n const int ib = k_KQ / QI8_1;\n const int iqs4 = k_KQ % QI5_1;\n const int iqs8 = k_KQ % QI8_1;\n const int shift = k_KQ & (QI8_1/2);\n\n int v;\n ggml_cuda_memcpy_1(&v, K_q5_1[ib].qs + sizeof(int)*iqs4);\n v = (v >> shift) & 0x0F0F0F0F;\n\n {\n int vh;\n ggml_cuda_memcpy_1(&vh, K_q5_1[ib].qh);\n vh >>= iqs8 * QI5_0;\n\n v |= (vh << 4) & 0x00000010; // 0 -> 4\n v |= (vh << 11) & 0x00001000; // 1 -> 12\n v |= (vh << 18) & 0x00100000; // 2 -> 20\n v |= (vh << 25) & 0x10000000; // 3 -> 28\n }\n\n const int u = Q_q8[k_KQ_0/nthreads];\n\n const int sumi = ggml_cuda_dp4a(v, u, 0);\n\n const float2 K_dm = __half22float2(K_q5_1[ib].dm);\n const float2 Q_ds = ((const float2 *) Q_ds_v)[k_KQ_0/nthreads];\n\n sum += K_dm.x*Q_ds.x*sumi + K_dm.y*Q_ds.y/QI8_1;\n }\n\n return sum;\n}\n\ntemplate \nstatic __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q8_0(\n const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {\n\n const block_q8_0 * K_q8_0 = (const block_q8_0 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ = k_KQ_0 + (nthreads == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads);\n\n const int ib = k_KQ / QI8_0;\n const int iqs = k_KQ % QI8_0;\n\n int v;\n ggml_cuda_memcpy_1(&v, K_q8_0[ib].qs + 4*iqs);\n\n const float2 * Q_ds = (const float2 *) Q_ds_v;\n const float Q_d = Q_ds[k_KQ_0/nthreads].x;\n\n sum += vec_dot_q8_0_q8_1_impl(&v, &Q_q8[k_KQ_0/nthreads], K_q8_0[ib].d, Q_d);\n }\n\n return sum;\n}\n\ntemplate \nstatic __device__ __forceinline__ void quantize_q8_1_to_shared(\n const float * __restrict__ x, const float scale, int * __restrict__ yq32, void * __restrict__ yds) {\n\n float vals[sizeof(int)] = {0.0f};\n#pragma unroll\n for (int l = 0; l < int(sizeof(int)); ++l) {\n vals[l] = (ni == WARP_SIZE || threadIdx.x < ni) ? scale * x[4*threadIdx.x + l] : 0.0f;\n }\n\n float amax = fabsf(vals[0]);\n float sum = vals[0];\n#pragma unroll\n for (int l = 1; l < int(sizeof(int)); ++l) {\n amax = fmaxf(amax, fabsf(vals[l]));\n sum += vals[l];\n }\n#pragma unroll\n for (int mask = QI8_1/2; mask > 0; mask >>= 1) {\n amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, mask, 32));\n sum += __shfl_xor_sync(0xFFFFFFFF, sum, mask, 32);\n }\n\n const float d = amax / 127;\n int q32 = 0;\n int8_t * q8 = (int8_t *) &q32;\n\n if (d != 0.0f) {\n#pragma unroll\n for (int l = 0; l < int(sizeof(int)); ++l) {\n q8[l] = roundf(vals[l] / d);\n }\n }\n\n yq32[threadIdx.x] = q32;\n if (threadIdx.x % QI8_1 == 0 && (ni == WARP_SIZE || threadIdx.x < ni)) {\n if (std::is_same::value) {\n ((half2 *) yds)[threadIdx.x/QI8_1] = make_half2(d, sum);\n } else {\n ((float2 *) yds)[threadIdx.x/QI8_1] = make_float2(d, sum);\n }\n }\n}\n\ntypedef void (*dequantize_V_t)(const void *, void *, const int64_t);\n\ntemplate \nstatic __device__ __forceinline__ void dequantize_V_f16(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n if constexpr (std::is_same_v) {\n ggml_cuda_memcpy_1(dst, (const half *) vx + i0);\n } else if constexpr (std::is_same_v) {\n static_assert(ne % 2 == 0, \"bad ne\");\n __align__(16) half2 tmp[ne/2];\n ggml_cuda_memcpy_1(tmp, (const half *) vx + i0);\n float2 * dst_f2 = (float2 *) dst;\n#pragma unroll\n for (int l = 0; l < ne/2; ++l) {\n dst_f2[l] = __half22float2(tmp[l]);\n }\n } else {\n static_assert(std::is_same_v, \"unsupported type\");\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void dequantize_V_q4_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q4_0 * x = (const block_q4_0 *) vx;\n\n const int64_t ib = i0 / QK4_0;\n const int iqs = i0 % (QK4_0/2);\n const int shift = (i0 % QK4_0) / (QK4_0/2);\n\n int q;\n static_assert(ne == 2 || ne == 4, \"bad ne\");\n ggml_cuda_memcpy_1(&q, x[ib].qs + iqs);\n q >>= 4*shift;\n q &= 0x0F0F0F0F;\n q = __vsubss4(q, 0x08080808);\n\n const int8_t * q8 = (const int8_t *) &q;\n\n#ifdef FP16_AVAILABLE\n if constexpr (std::is_same_v) {\n const half2 d = __half2half2(x[ib].d);\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]);\n }\n } else\n#endif // FP16_AVAILABLE\n if constexpr (std::is_same_v) {\n const float d = x[ib].d;\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = d * q8[l];\n }\n } else {\n static_assert(std::is_same_v, \"bad type\");\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void dequantize_V_q4_1(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q4_1 * x = (const block_q4_1 *) vx;\n\n const int64_t ib = i0 / QK4_1;\n const int iqs = i0 % (QK4_1/2);\n const int shift = (i0 % QK4_1) / (QK4_1/2);\n\n int q;\n static_assert(ne == 2 || ne == 4, \"bad ne\");\n ggml_cuda_memcpy_1(&q, x[ib].qs + iqs);\n q >>= 4*shift;\n q &= 0x0F0F0F0F;\n\n const int8_t * q8 = (const int8_t *) &q;\n\n#ifdef FP16_AVAILABLE\n if constexpr (std::is_same_v) {\n const half2 dm = x[ib].dm;\n const half2 d = __half2half2( __low2half(dm));\n const half2 m = __half2half2(__high2half(dm));\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m;\n }\n } else\n#endif // FP16_AVAILABLE\n if constexpr (std::is_same_v) {\n const float2 dm = __half22float2(x[ib].dm);\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = dm.x * q8[l] + dm.y;\n }\n } else {\n static_assert(std::is_same_v, \"bad type\");\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void dequantize_V_q5_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q5_0 * x = (const block_q5_0 *) vx;\n\n const int64_t ib = i0 / QK5_0;\n const int idq = i0 % QK5_0;\n const int iqs = i0 % (QK5_0/2);\n const int shift = (i0 % QK5_0) / (QK5_0/2);\n\n int q;\n static_assert(ne == 2 || ne == 4, \"bad ne\");\n ggml_cuda_memcpy_1(&q, x[ib].qs + iqs);\n q >>= 4*shift;\n q &= 0x0F0F0F0F;\n\n {\n int qh;\n ggml_cuda_memcpy_1(&qh, x[ib].qh);\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n q |= ((qh >> (idq + l)) & 0x00000001) << (8*l + 4);\n }\n }\n\n q = __vsubss4(q, 0x10101010);\n\n const int8_t * q8 = (const int8_t *) &q;\n\n#ifdef FP16_AVAILABLE\n if constexpr (std::is_same_v) {\n const half2 d = __half2half2(x[ib].d);\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]);\n }\n } else\n#endif // FP16_AVAILABLE\n if constexpr (std::is_same_v) {\n const float d = x[ib].d;\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = d * q8[l];\n }\n } else {\n static_assert(std::is_same_v, \"bad type\");\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void dequantize_V_q5_1(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q5_1 * x = (const block_q5_1 *) vx;\n\n const int64_t ib = i0 / QK5_1;\n const int idq = i0 % QK5_1;\n const int iqs = i0 % (QK5_1/2);\n const int shift = (i0 % QK5_1) / (QK5_1/2);\n\n int q;\n static_assert(ne == 2 || ne == 4, \"bad ne\");\n ggml_cuda_memcpy_1(&q, x[ib].qs + iqs);\n q >>= 4*shift;\n q &= 0x0F0F0F0F;\n\n {\n int qh;\n ggml_cuda_memcpy_1(&qh, x[ib].qh);\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n q |= ((qh >> (idq + l)) & 0x00000001) << (8*l + 4);\n }\n }\n\n const int8_t * q8 = (const int8_t *) &q;\n\n#ifdef FP16_AVAILABLE\n if constexpr (std::is_same_v) {\n const half2 dm = x[ib].dm;\n const half2 d = __half2half2( __low2half(dm));\n const half2 m = __half2half2(__high2half(dm));\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m;\n }\n } else\n#endif // FP16_AVAILABLE\n if constexpr (std::is_same_v) {\n const float2 dm = __half22float2(x[ib].dm);\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = dm.x * q8[l] + dm.y;\n }\n } else {\n static_assert(std::is_same_v, \"bad type\");\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void dequantize_V_q8_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q8_0 * x = (const block_q8_0 *) vx;\n\n const int64_t ib = i0 / QK8_0;\n const int iqs = i0 % QK8_0;\n\n static_assert(ne % 2 == 0, \"bad ne\");\n int8_t qs[ne];\n ggml_cuda_memcpy_1(qs, x[ib].qs + iqs);\n\n#ifdef FP16_AVAILABLE\n if constexpr (std::is_same::value) {\n const half2 d = __half2half2(x[ib].d);\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((half2 *) dst)[l0/2] = d * make_half2(qs[l0 + 0], qs[l0 + 1]);\n }\n } else\n#endif // FP16_AVAILABLE\n if constexpr (std::is_same::value) {\n const float d = x[ib].d;\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = d * qs[l];\n }\n } else {\n static_assert(std::is_same_v, \"unsupported type\");\n }\n}\n\ntemplate \nconstexpr __device__ vec_dot_KQ_t get_vec_dot_KQ() {\n if constexpr (type_K == GGML_TYPE_F16) {\n return vec_dot_fattn_vec_KQ_f16;\n } else if constexpr (type_K == GGML_TYPE_Q4_0) {\n return vec_dot_fattn_vec_KQ_q4_0;\n } else if constexpr (type_K == GGML_TYPE_Q4_1) {\n return vec_dot_fattn_vec_KQ_q4_1;\n } else if constexpr (type_K == GGML_TYPE_Q5_0) {\n return vec_dot_fattn_vec_KQ_q5_0;\n } else if constexpr (type_K == GGML_TYPE_Q5_1) {\n return vec_dot_fattn_vec_KQ_q5_1;\n } else if constexpr (type_K == GGML_TYPE_Q8_0) {\n return vec_dot_fattn_vec_KQ_q8_0;\n } else {\n static_assert(type_K == -1, \"bad type\");\n return nullptr;\n }\n}\n\ntemplate \nconstexpr __device__ dequantize_V_t get_dequantize_V() {\n if constexpr (type_V == GGML_TYPE_F16) {\n return dequantize_V_f16;\n } else if constexpr (type_V == GGML_TYPE_Q4_0) {\n return dequantize_V_q4_0;\n } else if constexpr (type_V == GGML_TYPE_Q4_1) {\n return dequantize_V_q4_1;\n } else if constexpr (type_V == GGML_TYPE_Q5_0) {\n return dequantize_V_q5_0;\n } else if constexpr (type_V == GGML_TYPE_Q5_1) {\n return dequantize_V_q5_1;\n } else if constexpr (type_V == GGML_TYPE_Q8_0) {\n return dequantize_V_q8_0;\n } else {\n static_assert(type_V == -1, \"bad type\");\n return nullptr;\n }\n}\n\ntemplate \n__launch_bounds__(FATTN_KQ_STRIDE/2, 1)\nstatic __global__ void flash_attn_mask_to_KV_max(\n const half2 * __restrict__ mask, int * __restrict__ KV_max, const int ne30, const int s31, const int s33) {\n const int ne31 = gridDim.x;\n const int tid = threadIdx.x;\n const int sequence = blockIdx.y;\n const int jt = blockIdx.x;\n\n mask += sequence*s33 + jt*ncols1*s31;\n\n __shared__ int buf_iw[WARP_SIZE];\n if (tid < WARP_SIZE) {\n buf_iw[tid] = 1;\n }\n __syncthreads();\n\n int KV_max_sj = (ne30 - 1) * FATTN_KQ_STRIDE;\n for (; KV_max_sj >= 0; KV_max_sj -= FATTN_KQ_STRIDE) {\n int all_inf = 1;\n\n#pragma unroll\n for (int j = 0; j < ncols1; ++j) {\n const float2 tmp = __half22float2(mask[j*s31 + KV_max_sj/2 + tid]);\n all_inf = all_inf && int(isinf(tmp.x)) && int(isinf(tmp.y));\n }\n\n all_inf = warp_reduce_all(all_inf);\n if (tid % WARP_SIZE == 0) {\n buf_iw[tid / WARP_SIZE] = all_inf;\n }\n __syncthreads();\n all_inf = buf_iw[tid % WARP_SIZE];\n __syncthreads();\n all_inf = warp_reduce_all(all_inf);\n\n if (!all_inf) {\n break;\n }\n }\n\n // If the break in the loop was not triggered, KV_max_sj is now -FATTN_KQ_STRIDE.\n // If the break was triggered it's the lower edge of the tile with the first non-masked values.\n // In either case, walk back the decrementation by FATTN_KQ_STRIDE.\n KV_max_sj += FATTN_KQ_STRIDE;\n\n if (threadIdx.x != 0) {\n return;\n }\n\n KV_max[sequence*ne31 + jt] = KV_max_sj;\n}\n\ntemplate // D == head size\n__launch_bounds__(D, 1)\nstatic __global__ void flash_attn_stream_k_fixup(\n float * __restrict__ dst, const float2 * __restrict__ dst_fixup, const int ne01, const int ne02, const int ne03,\n const int ne11, const int ne12, const int nbatch_fa) {\n constexpr int ncols = ncols1*ncols2;\n\n const int bidx0 = blockIdx.x;\n const int j = blockIdx.y;\n const int c = blockIdx.z;\n const int jc = j*ncols2 + c;\n const int tid = threadIdx.x;\n\n const float * dst_fixup_data = ((const float *) dst_fixup) + gridDim.x*(2*2*ncols);\n\n const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n\n const int iter_k = (ne11 + (nbatch_fa - 1)) / nbatch_fa;\n const int iter_j = (ne01 + (ncols1 - 1)) / ncols1;\n const int iter_z_gqa = (gqa_ratio + (ncols2 - 1)) / ncols2;\n\n const int kbc0 = int64_t(bidx0 + 0)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x;\n const int kbc0_stop = int64_t(bidx0 + 1)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x;\n\n const bool did_not_have_any_data = kbc0 == kbc0_stop;\n const bool wrote_beginning_of_tile = kbc0 % iter_k == 0;\n const bool did_not_write_last = kbc0/iter_k == kbc0_stop/iter_k && kbc0_stop % iter_k != 0;\n if (did_not_have_any_data || wrote_beginning_of_tile || did_not_write_last) {\n return;\n }\n\n // z_KV == K/V head index, zt_gqa = Q head start index per K/V head, jt = token position start index\n const int sequence = kbc0 /(iter_k*iter_j*iter_z_gqa*ne12);\n const int z_KV = (kbc0 - iter_k*iter_j*iter_z_gqa*ne12 * sequence)/(iter_k*iter_j*iter_z_gqa);\n const int zt_gqa = (kbc0 - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV)/(iter_k*iter_j);\n const int jt = (kbc0 - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV - iter_k*iter_j * zt_gqa) / iter_k;\n\n const int zt_Q = z_KV*gqa_ratio + zt_gqa*ncols2; // Global Q head start index.\n\n if (jt*ncols1 + j >= ne01 || zt_gqa*ncols2 + c >= gqa_ratio) {\n return;\n }\n\n dst += sequence*ne02*ne01*D + jt*ne02*(ncols1*D) + zt_Q*D + (j*ne02 + c)*D + tid;\n\n // Load the partial result that needs a fixup:\n float dst_val = 0.0f;\n float max_val = 0.0f;\n float rowsum = 0.0f;\n {\n dst_val = *dst;\n\n const float2 tmp = dst_fixup[bidx0*ncols + jc];\n max_val = tmp.x;\n rowsum = tmp.y;\n }\n\n // Iterate over previous blocks and compute the combined results.\n // All CUDA blocks that get here must have a previous block that needs a fixup.\n int bidx = bidx0 - 1;\n int kbc_stop = kbc0;\n while(true) {\n const int kbc = int64_t(bidx)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x;\n if (kbc == kbc_stop) { // Did not have any data.\n bidx--;\n kbc_stop = kbc;\n continue;\n }\n\n const float dst_add = dst_fixup_data[bidx*ncols*D + jc*D + tid];\n\n const float2 tmp = dst_fixup[(gridDim.x + bidx)*ncols + jc];\n\n // Scale the current and new value accumulators depending on the max. values.\n const float max_val_new = fmaxf(max_val, tmp.x);\n\n const float diff_val = max_val - max_val_new;\n const float diff_add = tmp.x - max_val_new;\n\n const float scale_val = diff_val >= SOFTMAX_FTZ_THRESHOLD ? expf(diff_val) : 0.0f;\n const float scale_add = diff_add >= SOFTMAX_FTZ_THRESHOLD ? expf(diff_add) : 0.0f;\n\n dst_val = scale_val*dst_val + scale_add*dst_add;\n rowsum = scale_val*rowsum + scale_add*tmp.y;\n\n max_val = max_val_new;\n\n // If this block started in a previous tile we are done and don't need to combine additional partial results.\n if (kbc % iter_k == 0 || kbc/iter_k < kbc0/iter_k) {\n break;\n }\n bidx--;\n kbc_stop = kbc;\n }\n\n // Write back final result:\n *dst = dst_val / rowsum;\n}\n\ntemplate // D == head size\n__launch_bounds__(D, 1)\nstatic __global__ void flash_attn_combine_results(\n const float * __restrict__ VKQ_parts,\n const float2 * __restrict__ VKQ_meta,\n float * __restrict__ dst,\n const int parallel_blocks) {\n // Dimension 0: threadIdx.x\n // Dimension 1: blockIdx.x\n // Dimension 2: blockIdx.y\n // Dimension 3: blockIdx.z\n // Memory layout is permuted with [0, 2, 1, 3]\n\n const int ne01 = gridDim.x;\n const int ne02 = gridDim.y;\n\n const int col = blockIdx.x;\n const int head = blockIdx.y;\n const int sequence = blockIdx.z;\n\n const int j_dst_unrolled = (sequence*ne01 + col)*ne02 + head;\n\n VKQ_parts += j_dst_unrolled * parallel_blocks*D;\n VKQ_meta += j_dst_unrolled * parallel_blocks;\n dst += j_dst_unrolled * D;\n\n const int tid = threadIdx.x;\n __builtin_assume(tid < D);\n\n extern __shared__ float2 meta[];\n for (int i = tid; i < 2*parallel_blocks; i += D) {\n ((float *) meta)[i] = ((const float *)VKQ_meta) [i];\n }\n\n __syncthreads();\n\n float kqmax = meta[0].x;\n for (int l = 1; l < parallel_blocks; ++l) {\n kqmax = max(kqmax, meta[l].x);\n }\n\n float VKQ_numerator = 0.0f;\n float VKQ_denominator = 0.0f;\n for (int l = 0; l < parallel_blocks; ++l) {\n const float KQ_max_scale = expf(meta[l].x - kqmax);\n\n VKQ_numerator += KQ_max_scale * VKQ_parts[l*D + tid];\n VKQ_denominator += KQ_max_scale * meta[l].y;\n }\n\n dst[tid] = VKQ_numerator / VKQ_denominator;\n}\n\ntemplate \nvoid launch_fattn(\n ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared,\n const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const int warp_size = WARP_SIZE\n) {\n constexpr int ncols = ncols1 * ncols2;\n\n const ggml_tensor * Q = dst->src[0];\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * V = dst->src[2];\n\n const bool V_is_K_view = V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs));\n\n const ggml_tensor * mask = dst->src[3];\n const ggml_tensor * sinks = dst->src[4];\n\n ggml_tensor * KQV = dst;\n\n GGML_ASSERT(Q->type == GGML_TYPE_F32);\n GGML_ASSERT(KQV->type == GGML_TYPE_F32);\n\n GGML_ASSERT(Q->nb[0] == ggml_element_size(Q));\n GGML_ASSERT(K->nb[0] == ggml_element_size(K));\n GGML_ASSERT(V->nb[0] == ggml_element_size(V));\n\n GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16);\n\n ggml_cuda_pool & pool = ctx.pool();\n cudaStream_t main_stream = ctx.stream();\n const int id = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[id].cc;\n const int nsm = ggml_cuda_info().devices[id].nsm;\n\n ggml_cuda_pool_alloc K_f16(pool);\n ggml_cuda_pool_alloc V_f16(pool);\n ggml_cuda_pool_alloc KV_max(pool);\n ggml_cuda_pool_alloc dst_tmp(pool);\n ggml_cuda_pool_alloc dst_tmp_meta(pool);\n\n const char * K_data = (const char *) K->data;\n size_t nb11 = K->nb[1];\n size_t nb12 = K->nb[2];\n size_t nb13 = K->nb[3];\n\n const char * V_data = (const char *) V->data;\n size_t nb21 = V->nb[1];\n size_t nb22 = V->nb[2];\n size_t nb23 = V->nb[3];\n\n if (need_f16_K && K->type != GGML_TYPE_F16) {\n const size_t bs = ggml_blck_size(K->type);\n const size_t ts = ggml_type_size(K->type);\n\n K_f16.alloc(ggml_nelements(K));\n if (ggml_is_contiguously_allocated(K)) {\n to_fp16_cuda_t to_fp16 = ggml_get_to_fp16_cuda(K->type);\n to_fp16(K_data, K_f16.ptr, ggml_nelements(K), main_stream);\n\n nb11 = nb11*bs*sizeof(half)/ts;\n nb12 = nb12*bs*sizeof(half)/ts;\n nb13 = nb13*bs*sizeof(half)/ts;\n } else {\n GGML_ASSERT(K->nb[0] == ts);\n to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(K->type);\n const int64_t s01 = nb11 / ts;\n const int64_t s02 = nb12 / ts;\n const int64_t s03 = nb13 / ts;\n to_fp16(K_data, K_f16.ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], s01, s02, s03, main_stream);\n\n nb11 = K->ne[0] * sizeof(half);\n nb12 = K->ne[1] * nb11;\n nb13 = K->ne[2] * nb12;\n }\n K_data = (char *) K_f16.ptr;\n }\n\n if (need_f16_V && V->type != GGML_TYPE_F16) {\n if (V_is_K_view) {\n V_data = K_data;\n nb21 = nb11;\n nb22 = nb12;\n nb23 = nb13;\n } else {\n const size_t bs = ggml_blck_size(V->type);\n const size_t ts = ggml_type_size(V->type);\n\n V_f16.alloc(ggml_nelements(V));\n if (ggml_is_contiguously_allocated(V)) {\n to_fp16_cuda_t to_fp16 = ggml_get_to_fp16_cuda(V->type);\n to_fp16(V_data, V_f16.ptr, ggml_nelements(V), main_stream);\n V_data = (char *) V_f16.ptr;\n\n nb21 = nb21*bs*sizeof(half)/ts;\n nb22 = nb22*bs*sizeof(half)/ts;\n nb23 = nb23*bs*sizeof(half)/ts;\n } else {\n GGML_ASSERT(V->nb[0] == ts);\n to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(V->type);\n const int64_t s01 = nb21 / ts;\n const int64_t s02 = nb22 / ts;\n const int64_t s03 = nb23 / ts;\n to_fp16(V_data, V_f16.ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], s01, s02, s03, main_stream);\n\n nb21 = V->ne[0] * sizeof(half);\n nb22 = V->ne[1] * nb21;\n nb23 = V->ne[2] * nb22;\n }\n V_data = (char *) V_f16.ptr;\n }\n }\n\n const int ntiles_x = ((Q->ne[1] + ncols1 - 1) / ncols1);\n const int gqa_ratio = Q->ne[2] / K->ne[2];\n const int ntiles_z_gqa = ((gqa_ratio + ncols2 - 1) / ncols2);\n const int ntiles_dst = ntiles_x * ntiles_z_gqa * K->ne[2] * Q->ne[3];\n\n // Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.\n // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or\n // multiple sequences of possibly different lengths.\n if (mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) {\n const int s31 = mask->nb[1] / sizeof(half2);\n const int s33 = mask->nb[3] / sizeof(half2);\n\n const dim3 blocks_num_KV_max(ntiles_x, Q->ne[3], 1);\n const dim3 block_dim_KV_max(FATTN_KQ_STRIDE/2, 1, 1);\n\n const int ne_KV_max = blocks_num_KV_max.x*blocks_num_KV_max.y;\n const int iter_k = K->ne[1] / FATTN_KQ_STRIDE;\n\n KV_max.alloc(ne_KV_max);\n flash_attn_mask_to_KV_max<<>>\n ((const half2 *) mask->data, KV_max.ptr, iter_k, s31, s33);\n CUDA_CHECK(cudaGetLastError());\n }\n\n const dim3 block_dim(warp_size, nwarps, 1);\n int max_blocks_per_sm = 1; // Max. number of active blocks limited by occupancy.\n CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, fattn_kernel, block_dim.x * block_dim.y * block_dim.z, nbytes_shared));\n GGML_ASSERT(max_blocks_per_sm > 0);\n int parallel_blocks = max_blocks_per_sm;\n\n const int ntiles_KV = (K->ne[1] + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length.\n\n dim3 blocks_num;\n if (stream_k) {\n // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup.\n const int max_blocks = max_blocks_per_sm*nsm;\n const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;\n const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves);\n\n const int nblocks_stream_k = std::min(max_blocks, ntiles_KV*ntiles_dst);\n\n const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || amd_wmma_available(cc) || tiles_efficiency_percent < 75;\n\n blocks_num.x = use_stream_k ? nblocks_stream_k : ntiles_dst;\n blocks_num.y = 1;\n blocks_num.z = 1;\n\n if (ntiles_dst % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles.\n dst_tmp_meta.alloc((size_t(blocks_num.x) * ncols * (2 + DV/2)));\n }\n } else {\n // parallel_blocks must not be larger than what the tensor size allows:\n parallel_blocks = std::min(parallel_blocks, ntiles_KV);\n\n // If ntiles_total % blocks_per_wave != 0 then some efficiency is lost due to tail effects.\n // Test whether parallel_blocks can be set to a higher value for better efficiency.\n const int blocks_per_wave = nsm * max_blocks_per_sm;\n int nwaves_best = 0;\n int efficiency_percent_best = 0;\n for (int parallel_blocks_test = parallel_blocks; parallel_blocks_test <= ntiles_KV; ++parallel_blocks_test) {\n const int nblocks_total = ntiles_dst * parallel_blocks_test;\n const int nwaves = (nblocks_total + blocks_per_wave - 1) / blocks_per_wave;\n const int efficiency_percent = 100 * nblocks_total / (nwaves*blocks_per_wave);\n\n // Stop trying configurations with more waves if we already have good efficiency to avoid excessive overhead.\n if (efficiency_percent_best >= 95 && nwaves > nwaves_best) {\n break;\n }\n\n if (efficiency_percent > efficiency_percent_best) {\n nwaves_best = nwaves;\n efficiency_percent_best = efficiency_percent;\n parallel_blocks = parallel_blocks_test;\n }\n }\n\n blocks_num.x = ntiles_x;\n blocks_num.y = parallel_blocks;\n blocks_num.z = ntiles_z_gqa*K->ne[2]*Q->ne[3];\n\n if (parallel_blocks > 1) {\n dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));\n dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));\n }\n }\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n float logit_softcap = 0.0f;\n\n memcpy(&scale, (const float *) KQV->op_params + 0, sizeof(float));\n memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));\n memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n if (logit_softcap != 0.0f) {\n scale /= logit_softcap;\n }\n\n const uint32_t n_head = Q->ne[2];\n const uint32_t n_head_log2 = 1u << uint32_t(floorf(log2f(float(n_head))));\n\n const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n // TODO other tensor dimensions after removal of WMMA kernel:\n const uint3 ne01 = init_fastdiv_values(Q->ne[1]);\n\n GGML_ASSERT(block_dim.x % warp_size == 0);\n fattn_kernel<<>>(\n (const char *) Q->data,\n K_data,\n V_data,\n mask ? ((const char *) mask->data) : nullptr,\n sinks ? ((const char *) sinks->data) : nullptr,\n KV_max.ptr,\n !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr,\n scale, max_bias, m0, m1, n_head_log2, logit_softcap,\n Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3],\n K->ne[0], K->ne[1], K->ne[2], K->ne[3], nb11, nb12, nb13,\n nb21, nb22, nb23,\n mask ? mask->ne[1] : 0, mask ? mask->ne[2] : 0, mask ? mask->ne[3] : 0,\n mask ? mask->nb[1] : 0, mask ? mask->nb[2] : 0, mask ? mask->nb[3] : 0\n );\n CUDA_CHECK(cudaGetLastError());\n\n if (stream_k) {\n if (ntiles_dst % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles.\n const dim3 block_dim_combine(DV, 1, 1);\n const dim3 blocks_num_combine = {blocks_num.x, ncols1, ncols2};\n\n flash_attn_stream_k_fixup\n <<>>\n ((float *) KQV->data, dst_tmp_meta.ptr, Q->ne[1], Q->ne[2], Q->ne[3], K->ne[1], K->ne[2], nbatch_fa);\n }\n } else if (parallel_blocks > 1) {\n const dim3 block_dim_combine(DV, 1, 1);\n const dim3 blocks_num_combine(Q->ne[1], Q->ne[2], Q->ne[3]);\n const size_t nbytes_shared_combine = parallel_blocks*sizeof(float2);\n\n flash_attn_combine_results\n <<>>\n (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data, parallel_blocks);\n }\n CUDA_CHECK(cudaGetLastError());\n}\n" }, { "path": "src/ggml-cuda/fattn-mma-f16.cuh", "content": "#include \"common.cuh\"\n#include \"cp-async.cuh\"\n#include \"mma.cuh\"\n#include \"fattn-common.cuh\"\n\nusing namespace ggml_cuda_mma;\n\n// Config options for the MMA kernel.\n// Should not affect results, only speed/register pressure/shared memory use.\nstruct fattn_mma_config {\n int nthreads; // Number of threads per CUDA block.\n int occupancy; // Targeted occupancy for the MMA kernel.\n int nbatch_fa; // Number of KV rows per softmax rescaling of KQ rowsums and VKQ accumulators.\n int nbatch_K2; // Number of K half2 values in direction of DKQ to load in parallel.\n int nbatch_V2; // Number of V half2 values in direction of DV to load in parallel.\n int nbatch_combine; // Number of VKQ half2 values in direction of DV to combine in parallel.\n int nstages_target; // Number of pipeline stages to use ideally, 1 == always load data synchronously, 2 == preload data if there is hardware support.\n bool Q_in_reg; // Whether the Q values should be kept permanently in registers.\n\n constexpr __host__ __device__ fattn_mma_config(\n int nthreads, int occupancy, int nbatch_fa, int nbatch_K2, int nbatch_V2, int nbatch_combine, int nstages_target, bool Q_in_reg) :\n nthreads(nthreads), occupancy(occupancy), nbatch_fa(nbatch_fa), nbatch_K2(nbatch_K2), nbatch_V2(nbatch_V2), nbatch_combine(nbatch_combine),\n nstages_target(nstages_target), Q_in_reg(Q_in_reg) {}\n};\n\n#define GGML_CUDA_FATTN_MMA_CONFIG_CASE(DKQ_, DV_, ncols_, nthreads_, occupancy_, nbatch_fa_, nbatch_K2_, nbatch_V2_, nbatch_combine_, nstages_target_, Q_in_reg_) \\\n if (DKQ == (DKQ_) && DV == (DV_) && ncols == (ncols_)) { \\\n static_assert((nthreads_) % 32 == 0 && (nthreads_) <= 512, \"bad nthreads\"); \\\n static_assert( (occupancy_) <= 8, \"bad occupancy\"); \\\n static_assert((nbatch_fa_) % 32 == 0 && (nbatch_fa_) <= 256, \"bad nbatch_fa\"); \\\n static_assert((nbatch_K2_) % 4 == 0 && (nbatch_K2_) <= 512, \"bad nbatch_K2\"); \\\n static_assert((nbatch_V2_) % 4 == 0 && (nbatch_V2_) <= 256, \"bad nbatch_V2\"); \\\n static_assert((nbatch_combine_) % 4 == 0 && (nbatch_combine_) <= 128, \"bad nbatch_combine\"); \\\n static_assert((nstages_target_) >= 1 && (nstages_target_) <= 2, \"bad nstages_target\"); \\\n return fattn_mma_config{(nthreads_), (occupancy_), (nbatch_fa_), (nbatch_K2_), (nbatch_V2_), (nbatch_combine_), (nstages_target_), (Q_in_reg_)}; \\\n } \\\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_ampere(const int DKQ, const int DV, const int ncols) {\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 2, 128, 32, 32, 32, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 128, 2, 64, 32, 32, 32, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 128, 2, 64, 32, 32, 32, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 128, 2, 64, 32, 32, 32, 2, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 128, 2, 128, 40, 40, 40, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 128, 2, 64, 40, 40, 40, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 128, 2, 64, 40, 40, 40, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 64, 128, 2, 64, 40, 40, 40, 2, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 8, 128, 2, 128, 48, 48, 48, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 16, 128, 2, 64, 48, 48, 48, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 32, 128, 2, 64, 48, 48, 48, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 64, 128, 2, 64, 48, 48, 48, 2, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 8, 128, 2, 128, 56, 56, 56, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16, 128, 2, 64, 56, 56, 56, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 128, 2, 64, 56, 56, 56, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 64, 128, 2, 64, 56, 56, 56, 2, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 8, 128, 2, 128, 64, 64, 64, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16, 128, 2, 64, 64, 64, 64, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2, 64, 64, 64, 64, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 128, 2, 64, 64, 64, 64, 2, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 4, 64, 128, 128, 128, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 4, 32, 128, 128, 128, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 32, 128, 128, 128, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 32, 128, 128, 128, 2, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 8, 64, 4, 32, 288, 256, 128, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16, 64, 4, 32, 288, 256, 128, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 128, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 128, 1, false);\n\n return fattn_mma_config(32, 1, 0, 0, 0, 0, 0, false);\n}\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_turing(const int DKQ, const int DV, const int ncols) {\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 128, 2, 64, 128, 128, 128, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 128, 2, 64, 128, 128, 128, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 2, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 8, 64, 4, 32, 96, 64, 128, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16, 64, 4, 32, 96, 64, 128, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 128, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 128, 1, false);\n\n return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n}\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_volta(const int DKQ, const int DV, const int ncols) {\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 8, 64, 4, 32, 288, 256, 64, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16, 64, 4, 32, 288, 256, 64, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 64, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 64, 1, false);\n\n // TODO tune specifically for Volta\n return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n}\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_rdna(const int DKQ, const int DV, const int ncols) {\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 128, 2, 64, 128, 128, 128, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 2, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 2, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16, 64, 4, 32, 96, 64, 128, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 128, 1, false);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 128, 1, false);\n\n // TODO tune specifically for RDNA\n return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n}\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_cdna(const int DKQ, const int DV, const int ncols) {\n // Conservative configs for CDNA (MI100+): 64KB LDS, wavefront64, nstages=1 (no cp.async).\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 2, 128, 32, 32, 32, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 128, 2, 64, 32, 32, 32, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 128, 2, 64, 32, 32, 32, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 2, 64, 32, 32, 32, 1, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 128, 2, 128, 40, 40, 40, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 128, 2, 64, 40, 40, 40, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 128, 2, 64, 40, 40, 40, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 64, 256, 2, 64, 40, 40, 40, 1, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 8, 128, 2, 128, 48, 48, 48, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 16, 128, 2, 64, 48, 48, 48, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 32, 128, 2, 64, 48, 48, 48, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 64, 256, 2, 64, 48, 48, 48, 1, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 8, 128, 2, 128, 56, 56, 56, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16, 128, 2, 64, 56, 56, 56, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 128, 2, 64, 56, 56, 56, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 64, 256, 2, 64, 56, 56, 56, 1, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 8, 128, 2, 128, 64, 64, 64, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16, 128, 2, 64, 64, 64, 64, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2, 64, 64, 64, 64, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 256, 2, 64, 64, 64, 64, 1, true);\n\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 4, 64, 128, 128, 128, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 4, 32, 128, 128, 128, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 32, 128, 128, 128, 1, true);\n GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2, 32, 128, 128, 128, 1, true);\n\n // Fallback for unsupported DKQ values (e.g. 576). Must return non-zero values to satisfy\n // compile-time static_asserts even though the kernel guard prevents runtime execution.\n // nthreads=256 gives nwarps=4 (warp_size=64) or 8 (warp_size=32), nbatch_fa=128 satisfies np*16 divisibility.\n return fattn_mma_config(256, 1, 128, 4, 4, 4, 1, false);\n}\n\nstatic __host__ fattn_mma_config ggml_cuda_fattn_mma_get_config(const int DKQ, const int DV, const int ncols, const int cc) {\n if (ampere_mma_available(cc)) {\n return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n }\n if (turing_mma_available(cc)) {\n return ggml_cuda_fattn_mma_get_config_turing(DKQ, DV, ncols);\n }\n if (amd_mfma_available(cc)) {\n return ggml_cuda_fattn_mma_get_config_cdna(DKQ, DV, ncols);\n }\n if (amd_wmma_available(cc)) {\n return ggml_cuda_fattn_mma_get_config_rdna(DKQ, DV, ncols);\n }\n GGML_ASSERT(volta_mma_available(cc));\n return ggml_cuda_fattn_mma_get_config_volta(DKQ, DV, ncols);\n}\n\nstatic constexpr __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config(const int DKQ, const int DV, const int ncols) {\n#if defined(AMPERE_MMA_AVAILABLE)\n return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n#elif defined(TURING_MMA_AVAILABLE)\n return ggml_cuda_fattn_mma_get_config_turing(DKQ, DV, ncols);\n#elif defined(AMD_MFMA_AVAILABLE)\n return ggml_cuda_fattn_mma_get_config_cdna(DKQ, DV, ncols);\n#elif defined(VOLTA_MMA_AVAILABLE)\n return ggml_cuda_fattn_mma_get_config_volta(DKQ, DV, ncols);\n#elif defined(AMD_WMMA_AVAILABLE)\n return ggml_cuda_fattn_mma_get_config_rdna(DKQ, DV, ncols);\n#else\n GGML_UNUSED_VARS(DKQ, DV, ncols);\n return fattn_mma_config(32, 1, 0, 0, 0, 0, 0, false);\n#endif // defined(AMPERE_MMA_AVAILABLE)\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nthreads(const int DKQ, const int DV, const int ncols, const int cc) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nthreads;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nthreads(const int DKQ, const int DV, const int ncols) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nthreads;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_occupancy(const int DKQ, const int DV, const int ncols, const int cc) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).occupancy;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_occupancy(const int DKQ, const int DV, const int ncols) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).occupancy;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nbatch_fa(const int DKQ, const int DV, const int ncols, const int cc) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nbatch_fa;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nbatch_fa(const int DKQ, const int DV, const int ncols) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nbatch_fa;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nbatch_K2(const int DKQ, const int DV, const int ncols, const int cc) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nbatch_K2;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nbatch_K2(const int DKQ, const int DV, const int ncols) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nbatch_K2;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nbatch_V2(const int DKQ, const int DV, const int ncols, const int cc) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nbatch_V2;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nbatch_V2(const int DKQ, const int DV, const int ncols) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nbatch_V2;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nbatch_combine(const int DKQ, const int DV, const int ncols, const int cc) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nbatch_combine;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nbatch_combine(const int DKQ, const int DV, const int ncols) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nbatch_combine;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nstages_target(const int DKQ, const int DV, const int ncols, const int cc) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nstages_target;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nstages_target(const int DKQ, const int DV, const int ncols) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nstages_target;\n}\n\nstatic __host__ bool ggml_cuda_fattn_mma_get_Q_in_reg(const int DKQ, const int DV, const int ncols, const int cc) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).Q_in_reg;\n}\n\nstatic constexpr __device__ bool ggml_cuda_fattn_mma_get_Q_in_reg(const int DKQ, const int DV, const int ncols) {\n return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).Q_in_reg;\n}\n\nstatic constexpr __device__ int get_cols_per_thread() {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n return 1; // AMD has a single column per thread.\n#else\n return 2; // This is specifically KQ columns, Volta only has a single VKQ column.\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n}\n\nstatic __host__ int get_cols_per_warp(const int cc) {\n if (turing_mma_available(cc) || amd_wmma_available(cc) || amd_mfma_available(cc)) {\n return 16;\n } else {\n // Volta\n return 32;\n }\n}\n\n// ------------------------------------------------------------------------------------------------------------------\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV, const int ncols1, const int ncols2, const int cc) {\n return cp_async_available(cc) && ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2, cc) : 0;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV, const int ncols1, const int ncols2) {\n#ifdef CP_ASYNC_AVAILABLE\n return ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2) : 0;\n#else\n GGML_UNUSED_VARS(DKQ, DV, ncols1, ncols2);\n return 0;\n#endif // CP_ASYNC_AVAILABLE\n}\n\n// ------------------------------------------------------------------------------------------------------------------\n\ntemplate\nstatic __device__ __forceinline__ void flash_attn_ext_f16_load_tile(\n const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int D2, const int stride_KV, const int i_sup) {\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n // K/V data is loaded with decreasing granularity for D for better memory bandwidth.\n // The minimum granularity with cp.async is 16 bytes, with synchronous data loading it's 4 bytes.\n if constexpr (use_cp_async) {\n static_assert(!oob_check, \"OOB check not compatible with cp_async\");\n constexpr int preload = 64;\n constexpr int h2_per_chunk = 16/sizeof(half2);\n const int chunks_per_row = D2 / h2_per_chunk;\n\n const unsigned int tile_KV_32 = ggml_cuda_cvta_generic_to_shared(tile_KV);\n\n auto load = [&] __device__ (auto n) {\n const int stride_k = warp_size >> n;\n const int k0_start = stride_k == warp_size ? 0 : chunks_per_row - chunks_per_row % (2*stride_k);\n const int k0_stop = chunks_per_row - chunks_per_row % (1*stride_k);\n const int stride_i = warp_size / stride_k;\n\n if (k0_start == k0_stop) {\n return;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < nbatch_fa; i0 += nwarps*stride_i) {\n const int i = i0 + threadIdx.y*stride_i + (stride_k == warp_size ? 0 : threadIdx.x / stride_k);\n\n if (i0 + nwarps*stride_i > nbatch_fa && i >= nbatch_fa) {\n break;\n }\n\n#pragma unroll\n for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n cp_async_cg_16(tile_KV_32 + i*(stride_tile*sizeof(half2)) + k*16, KV + i*stride_KV + k*h2_per_chunk);\n }\n }\n };\n // 1: max 32*16=512 bytes, 256 half\n // 2: max 16*16=256 bytes, 128 half\n // 3: max 8*16=128 bytes, 64 half\n // 4: max 4*16= 64 bytes, 32 half\n // 5: max 2*16= 32 bytes, 16 half\n // 6: max 1*16= 16 bytes, 8 half\n ggml_cuda_unroll<6>{}(load);\n } else {\n // TODO use ggml_cuda_memcpy_1\n auto load = [&] __device__ (const int n) {\n const int stride_k = warp_size >> n;\n const int k0_start = stride_k == warp_size ? 0 : D2 - D2 % (2*stride_k);\n const int k0_stop = D2 - D2 % (1*stride_k);\n const int stride_i = warp_size / stride_k;\n\n if (k0_start == k0_stop) {\n return;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < nbatch_fa; i0 += nwarps*stride_i) {\n const int i = i0 + threadIdx.y*stride_i + (stride_k == warp_size ? 0 : threadIdx.x / stride_k);\n\n if (i0 + nwarps*stride_i > nbatch_fa && i >= nbatch_fa) {\n break;\n }\n\n#pragma unroll\n for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n tile_KV[i*stride_tile + k] = !oob_check || i < i_sup ? KV[i*stride_KV + k] : make_half2(0.0f, 0.0f);\n }\n }\n };\n // 1: max 32* 4=128 bytes, 64 half\n // 2: max 16* 4= 64 bytes, 32 half\n // 3: max 8* 4= 32 bytes, 16 half\n // 4: max 4* 4= 16 bytes, 8 half\n ggml_cuda_unroll<4>{}(load);\n }\n}\n\ntemplate\nstatic __device__ __forceinline__ void flash_attn_ext_f16_load_mask(\n const half * const __restrict__ mask_h, half * const __restrict__ tile_mask,\n const int stride_mask, const int i_sup, const int j0, const uint3 ne01) {\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n if constexpr (use_cp_async) {\n static_assert(nbatch_fa <= 8*warp_size && nbatch_fa % 8 == 0, \"bad nbatch_fa\");\n static_assert(!oob_check, \"OOB check incompatible with cp_async\");\n constexpr int preload = nbatch_fa >= 32 ? nbatch_fa * sizeof(half) : 64;\n constexpr int cols_per_warp = 8*warp_size/nbatch_fa;\n constexpr int stride_j = nwarps * cols_per_warp;\n\n const unsigned int tile_mask_32 = ggml_cuda_cvta_generic_to_shared(tile_mask);\n\n#pragma unroll\n for (int j1 = 0; j1 < ncols1; j1 += stride_j) {\n const int j_sram = j1 + threadIdx.y*cols_per_warp + threadIdx.x / (warp_size/cols_per_warp);\n const int j_vram = fastmodulo(j0 + j_sram, ne01);\n\n if (j1 + stride_j > ncols1 && j_sram >= ncols1) {\n break;\n }\n\n const int i = 8 * (threadIdx.x % (nbatch_fa/8));\n\n cp_async_cg_16(tile_mask_32 + j_sram*(nbatch_fa*sizeof(half) + 16) + i*sizeof(half), mask_h + j_vram*stride_mask + i);\n }\n } else if constexpr (oob_check) {\n#pragma unroll\n for (int j1 = 0; j1 < ncols1; j1 += nwarps) {\n const int j_sram = j1 + threadIdx.y;\n const int j_vram = fastmodulo(j0 + j_sram, ne01);\n\n if (j1 + nwarps > ncols1 && j_sram >= ncols1) {\n break;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < nbatch_fa; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n tile_mask[j_sram*(nbatch_fa + 8) + i] = i < i_sup ? mask_h[j_vram*stride_mask + i] : half(0.0f);\n }\n }\n } else if constexpr (nbatch_fa < 2*warp_size) {\n constexpr int cols_per_warp = 2*warp_size/nbatch_fa;\n constexpr int stride_j = nwarps * cols_per_warp;\n#pragma unroll\n for (int j1 = 0; j1 < ncols1; j1 += stride_j) {\n const int j_sram = j1 + threadIdx.y*cols_per_warp + threadIdx.x / (warp_size/cols_per_warp);\n const int j_vram = fastmodulo(j0 + j_sram, ne01);\n\n if (j1 + stride_j > ncols1 && j_sram >= ncols1) {\n break;\n }\n\n const int i = threadIdx.x % (warp_size/cols_per_warp);\n\n ggml_cuda_memcpy_1(tile_mask + j_sram*(nbatch_fa + 8) + 2*i, mask_h + j_vram*stride_mask + 2*i);\n }\n } else {\n#pragma unroll\n for (int j1 = 0; j1 < ncols1; j1 += nwarps) {\n const int j_sram = j1 + threadIdx.y;\n const int j_vram = fastmodulo(j0 + j_sram, ne01);\n\n if (j1 + nwarps > ncols1 && j_sram >= ncols1) {\n break;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < nbatch_fa; i0 += 2*warp_size) {\n const int i = i0 + 2*threadIdx.x;\n\n ggml_cuda_memcpy_1(tile_mask + j_sram*(nbatch_fa + 8) + i, mask_h + j_vram*stride_mask + i);\n }\n }\n }\n}\n\ntemplate\nstatic __device__ __forceinline__ void flash_attn_ext_f16_iter(\n const float2 * const __restrict__ Q_f2,\n const half2 * const __restrict__ K_h2,\n const half2 * const __restrict__ V_h2,\n const half * const __restrict__ mask_h,\n float2 * const __restrict__ dstk,\n float2 * const __restrict__ dstk_fixup,\n const float scale,\n const float slope,\n const float logit_softcap,\n const uint3 ne01,\n const int ne02,\n const int stride_K,\n const int stride_V,\n const int stride_mask,\n half2 * const __restrict__ tile_Q,\n half2 * const __restrict__ tile_K,\n half2 * const __restrict__ tile_V,\n half * const __restrict__ tile_mask,\n T_B_KQ * const __restrict__ Q_B,\n T_C_VKQ * const __restrict__ VKQ_C,\n float * const __restrict__ KQ_max,\n float * const __restrict__ KQ_rowsum,\n const int jt,\n const int kb0,\n const int k_VKQ_sup) {\n#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n constexpr int ncols = ncols1 * ncols2;\n constexpr int cols_per_warp = T_B_KQ::I;\n constexpr int cols_per_thread = get_cols_per_thread();\n constexpr int np = cols_per_warp > ncols ? nwarps : nwarps * cols_per_warp/ncols; // Number of parallel CUDA warps per Q column.\n constexpr int nbatch_fa = ggml_cuda_fattn_mma_get_nbatch_fa(DKQ, DV, ncols);\n constexpr int nbatch_K2 = ggml_cuda_fattn_mma_get_nbatch_K2(DKQ, DV, ncols);\n constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2(DKQ, DV, ncols);\n constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols);\n constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2);\n\n constexpr int stride_tile_Q = DKQ/2 + 4;\n constexpr int stride_tile_K = nbatch_K2 + 4;\n\n constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4;\n\n const int k_VKQ_0 = kb0 * nbatch_fa;\n#if defined(TURING_MMA_AVAILABLE)\n T_C_KQ KQ_C[nbatch_fa/(np*(cols_per_warp == 8 ? T_C_KQ::I : T_C_KQ::J))];\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n T_C_KQ KQ_C[nbatch_fa/(np*T_C_KQ::J)];\n#else // Volta\n T_C_KQ KQ_C[nbatch_fa/(np*T_C_KQ::J)];\n#endif // defined(TURING_MMA_AVAILABLE)\n\n if constexpr (nstages > 1) {\n static_assert(!oob_check, \"OOB check incompatible with multi-stage pipeline\");\n static_assert(!V_is_K_view, \"K data reuse not implemented multi-stage loading\");\n static_assert(nbatch_K2 == DKQ/2, \"batching not implemented for multi stage loading\");\n constexpr bool use_cp_async = true;\n cp_async_wait_all();\n __syncthreads();\n flash_attn_ext_f16_load_tile\n (V_h2 + int64_t(k_VKQ_0)*stride_V, tile_V, nbatch_V2, stride_V, k_VKQ_sup);\n } else {\n constexpr bool use_cp_async = nstages == 1;\n if (ncols2 > 1 || mask_h) {\n flash_attn_ext_f16_load_mask\n (mask_h + k_VKQ_0, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);\n }\n }\n\n // For MLA K and V have the same data.\n // Therefore, iterate over K in reverse and later re-use the data if possible.\n#pragma unroll\n for (int k0_start = (DKQ/2-1) - (DKQ/2-1) % nbatch_K2; k0_start >= 0; k0_start -= nbatch_K2) {\n const int k0_stop = k0_start + nbatch_K2 < DKQ/2 ? k0_start + nbatch_K2 : DKQ/2;\n const int k0_diff = k0_stop - k0_start;\n\n if constexpr (nstages <= 1) {\n constexpr bool use_cp_async = nstages == 1;\n flash_attn_ext_f16_load_tile\n (K_h2 + int64_t(k_VKQ_0)*stride_K + k0_start, tile_K, k0_diff, stride_K, k_VKQ_sup);\n if (use_cp_async) {\n cp_async_wait_all();\n }\n __syncthreads();\n }\n\n // Calculate tile of KQ:\n if constexpr (Q_in_reg) {\n#pragma unroll\n for (int i_KQ_00 = 0; i_KQ_00 < nbatch_fa; i_KQ_00 += np*T_A_KQ::I) {\n const int i_KQ_0 = i_KQ_00 + (threadIdx.y % np)*T_A_KQ::I;\n#pragma unroll\n for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) {\n T_A_KQ K_A;\n load_ldmatrix(K_A, tile_K + i_KQ_0*stride_tile_K + (k_KQ_0 - k0_start), stride_tile_K);\n if constexpr (cols_per_warp == 8) {\n mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[k_KQ_0/T_A_KQ::J]);\n } else {\n // Wide version of KQ_C is column-major\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n // AMD matrix C is column-major.\n mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[k_KQ_0/T_A_KQ::J]);\n#else\n // swap A and B for CUDA.\n mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], Q_B[k_KQ_0/T_A_KQ::J], K_A);\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n }\n }\n }\n } else {\n#pragma unroll\n for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) {\n load_ldmatrix(Q_B[0], tile_Q + (threadIdx.y / np)*(T_B_KQ::I*stride_tile_Q) + k_KQ_0, stride_tile_Q);\n\n#pragma unroll\n for (int i_KQ_00 = 0; i_KQ_00 < nbatch_fa; i_KQ_00 += np*T_A_KQ::I) {\n const int i_KQ_0 = i_KQ_00 + (threadIdx.y % np)*T_A_KQ::I;\n\n T_A_KQ K_A;\n load_ldmatrix(K_A, tile_K + i_KQ_0*stride_tile_K + (k_KQ_0 - k0_start), stride_tile_K);\n\n if constexpr (cols_per_warp == 8) {\n mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[0]);\n } else {\n // Wide version of KQ_C is column-major\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n // AMD matrix C is column-major.\n mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[0]);\n#else\n // swap A and B for CUDA.\n mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], Q_B[0], K_A);\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n }\n }\n }\n }\n\n if constexpr (nstages <= 1) {\n __syncthreads(); // Only needed if tile_K == tile_V.\n }\n }\n\n if (use_logit_softcap) {\n constexpr int stride = cols_per_warp == 8 ? np*T_C_KQ::I : np*T_C_KQ::J;\n static_assert(nbatch_fa % stride == 0, \"bad loop size\");\n#pragma unroll\n for (int i = 0; i < nbatch_fa/stride; ++i) {\n#pragma unroll\n for (int l = 0; l < T_C_KQ::ne; ++l) {\n KQ_C[i].x[l] = logit_softcap*tanhf(KQ_C[i].x[l]);\n }\n }\n }\n\n float KQ_max_new[cols_per_thread];\n#pragma unroll\n for (int col = 0; col < cols_per_thread; ++col) {\n KQ_max_new[col] = KQ_max[col];\n }\n float KQ_rowsum_add[cols_per_thread] = {0.0f};\n\n if constexpr (cols_per_warp == 8) {\n if (ncols2 > 1 || mask_h) {\n#pragma unroll\n for (int i00 = 0; i00 < nbatch_fa; i00 += np*T_C_KQ::I) {\n const int i0 = i00 + (threadIdx.y % np)*T_C_KQ::I;\n#pragma unroll\n for (int l = 0; l < T_C_KQ::ne; ++l) {\n const int i = i0 + T_C_KQ::get_i(l);\n const int j = ((threadIdx.y / np)*T_C_KQ::J + T_C_KQ::get_j(l)) / ncols2;\n\n KQ_C[i00/(np*T_C_KQ::I)].x[l] += slope * __half2float(tile_mask[j*(nbatch_fa + 8) + i]);\n }\n }\n }\n\n // Calculate softmax for each KQ column using the current max. value.\n // The divisor is stored in KQ_rowsum and will be applied at the end.\n static_assert(nbatch_fa % (np*T_C_KQ::I) == 0, \"bad loop size\");\n#pragma unroll\n for (int k0 = 0; k0 < nbatch_fa; k0 += np*T_C_KQ::I) {\n#pragma unroll\n for (int l = 0; l < T_C_KQ::ne; ++l) {\n if (!oob_check || k0 + (threadIdx.y % np)*T_C_KQ::I + T_C_KQ::get_i(l) < k_VKQ_sup) {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n constexpr int KQ_idx = 0;\n#else\n // Turing + Volta:\n const int KQ_idx = l % 2;\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n KQ_max_new[KQ_idx] = fmaxf(KQ_max_new[KQ_idx], KQ_C[k0/(np*T_C_KQ::I)].x[l] + FATTN_KQ_MAX_OFFSET);\n }\n }\n }\n\n // Values per KQ column are spread across 8 threads:\n#pragma unroll\n for (int col = 0; col < cols_per_thread; ++col) {\n#pragma unroll\n for (int offset = 16; offset >= 4; offset >>= 1) {\n KQ_max_new[col] = fmaxf(KQ_max_new[col], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[col], offset, warp_size));\n }\n }\n\n static_assert(nbatch_fa % (np*T_C_KQ::I) == 0, \"bad loop size\");\n#pragma unroll\n for (int k0 = 0; k0 < nbatch_fa; k0 += np*T_C_KQ::I) {\n#pragma unroll\n for (int l = 0; l < T_C_KQ::ne; ++l) {\n if (!oob_check || k0 + (threadIdx.y % np)*T_C_KQ::I + T_C_KQ::get_i(l) < k_VKQ_sup) {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n constexpr int KQ_idx = 0;\n#else\n // Turing + Volta:\n const int KQ_idx = l % 2;\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n KQ_C[k0/(np*T_C_KQ::I)].x[l] = expf(KQ_C[k0/(np*T_C_KQ::I)].x[l] - KQ_max_new[KQ_idx]);\n KQ_rowsum_add[KQ_idx] += KQ_C[k0/(np*T_C_KQ::I)].x[l];\n } else {\n KQ_C[k0/(np*T_C_KQ::I)].x[l] = 0.0f;\n }\n }\n }\n } else { // not Turing mma or T_B_KQ::I > 8\n if (ncols2 > 1 || mask_h) {\n#pragma unroll\n for (int i00 = 0; i00 < nbatch_fa; i00 += np*T_C_KQ::J) {\n const int i0 = i00 + (threadIdx.y % np)*T_C_KQ::J;\n#pragma unroll\n for (int l0 = 0; l0 < T_C_KQ::ne; l0 += 2) {\n const int i = (i0 + T_C_KQ::get_j(l0)) / 2;\n const int j = ((threadIdx.y / np)*cols_per_warp + T_C_KQ::get_i(l0)) / ncols2;\n\n const float2 tmp = __half22float2(((const half2 *)tile_mask)[j*(nbatch_fa/2 + 4) + i]);\n KQ_C[i00/(np*T_C_KQ::J)].x[l0 + 0] += slope*tmp.x;\n KQ_C[i00/(np*T_C_KQ::J)].x[l0 + 1] += slope*tmp.y;\n }\n }\n }\n\n // Calculate softmax for each KQ column using the current max. value.\n // The divisor is stored in KQ_rowsum and will be applied at the end.\n static_assert(nbatch_fa % (np*T_C_KQ::J) == 0, \"bad loop size\");\n#pragma unroll\n for (int k0 = 0; k0 < nbatch_fa; k0 += np*T_C_KQ::J) {\n#pragma unroll\n for (int l = 0; l < T_C_KQ::ne; ++l) {\n if (!oob_check || k0 + (threadIdx.y % np)*T_C_KQ::J + T_C_KQ::get_j(l) < k_VKQ_sup) {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n constexpr int KQ_idx = 0;\n#else\n // Turing + Volta:\n const int KQ_idx = (l/2) % 2;\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n KQ_max_new[KQ_idx] = fmaxf(KQ_max_new[KQ_idx], KQ_C[(k0/(np*T_C_KQ::J))].x[l] + FATTN_KQ_MAX_OFFSET);\n }\n }\n }\n\n#pragma unroll\n for (int col = 0; col < cols_per_thread; ++col) {\n#if defined(TURING_MMA_AVAILABLE)\n // Values per KQ column are spread across 4 threads:\n constexpr int offset_first = 2;\n constexpr int offset_last = 1;\n#elif defined(AMD_MFMA_AVAILABLE)\n // MFMA: 4 threads per Q column (threadIdx.x % 16 == col, spaced by 16).\n constexpr int offset_first = 32;\n constexpr int offset_last = 16;\n#elif defined(AMD_WMMA_AVAILABLE)\n // Values per KQ column are spread across 2 threads:\n constexpr int offset_first = 16;\n constexpr int offset_last = 16;\n#else // Volta\n // Values per KQ column are spread across 2 threads:\n constexpr int offset_first = 2;\n constexpr int offset_last = 2;\n#endif // defined(TURING_MMA_AVAILABLE)\n#pragma unroll\n for (int offset = offset_first; offset >= offset_last; offset >>= 1) {\n KQ_max_new[col] = fmaxf(KQ_max_new[col], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[col], offset, warp_size));\n }\n }\n\n static_assert(nbatch_fa % (np*T_C_KQ::J) == 0, \"bad loop size\");\n#pragma unroll\n for (int k0 = 0; k0 < nbatch_fa; k0 += np*T_C_KQ::J) {\n#pragma unroll\n for (int l = 0; l < T_C_KQ::ne; ++l) {\n if (!oob_check || k0 + (threadIdx.y % np)*T_C_KQ::J + T_C_KQ::get_j(l) < k_VKQ_sup) {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n constexpr int KQ_idx = 0;\n#else\n // Turing + Volta:\n const int KQ_idx = (l/2) % 2;\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n KQ_C[(k0/(np*T_C_KQ::J))].x[l] = expf(KQ_C[(k0/(np*T_C_KQ::J))].x[l] - KQ_max_new[KQ_idx]);\n KQ_rowsum_add[KQ_idx] += KQ_C[(k0/(np*T_C_KQ::J))].x[l];\n } else {\n KQ_C[(k0/(np*T_C_KQ::J))].x[l] = 0.0f;\n }\n }\n }\n }\n\n {\n float KQ_max_scale[cols_per_thread];\n#pragma unroll\n for (int col = 0; col < cols_per_thread; ++col) {\n const float KQ_max_diff = KQ_max[col] - KQ_max_new[col];\n KQ_max_scale[col] = expf(KQ_max_diff);\n KQ_max[col] = KQ_max_new[col];\n\n *((uint32_t *) &KQ_max_scale[col]) *= KQ_max_diff >= SOFTMAX_FTZ_THRESHOLD;\n\n // Scale previous KQ_rowsum to account for a potential increase in KQ_max:\n KQ_rowsum[col] = KQ_max_scale[col]*KQ_rowsum[col] + KQ_rowsum_add[col];\n }\n\n#if defined(TURING_MMA_AVAILABLE)\n if constexpr (cols_per_warp == 8) {\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[cols_per_thread - 1]);\n#pragma unroll\n for (int i = 0; i < DV/T_C_VKQ::I; ++i) {\n#pragma unroll\n for (int l = 0; l < T_C_VKQ::ne; ++l) {\n VKQ_C[i].x[l] *= KQ_max_scale_h2;\n }\n }\n } else {\n#pragma unroll\n for (int col = 0; col < cols_per_thread; ++col) {\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]);\n#pragma unroll\n for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n for (int l0 = 0; l0 < T_C_VKQ::ne; l0 += 2) {\n VKQ_C[i].x[l0 + col] *= KQ_max_scale_h2;\n }\n }\n }\n }\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n const half2 KQ_max_scale_h2 = make_half2(\n KQ_max_scale[0], KQ_max_scale[0]);\n#pragma unroll\n for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n for (int l = 0; l < T_C_VKQ::ne; ++l) {\n VKQ_C[i].x[l] *= KQ_max_scale_h2;\n }\n }\n#else // Volta\n const half2 KQ_max_scale_h2 = make_half2(\n KQ_max_scale[(threadIdx.x / 2) % 2], KQ_max_scale[(threadIdx.x / 2) % 2]);\n#pragma unroll\n for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n for (int l = 0; l < T_C_VKQ::ne; ++l) {\n VKQ_C[i].x[l] *= KQ_max_scale_h2;\n }\n }\n#endif // defined(TURING_MMA_AVAILABLE)\n }\n\n // Convert KQ C tiles into B tiles for VKQ calculation:\n T_B_VKQ B[nbatch_fa/(np*2*T_B_VKQ::J)];\n static_assert(nbatch_fa % (np*2*T_B_VKQ::J) == 0, \"bad loop size\");\n if constexpr (cols_per_warp == 8) {\n#pragma unroll\n for (int k = 0; k < nbatch_fa/(np*2*T_B_VKQ::J); ++k) {\n B[k] = get_transposed(get_half2(KQ_C[k]));\n }\n } else {\n for (int k = 0; k < nbatch_fa/(np*2*T_B_VKQ::J); ++k) {\n B[k] = get_half2(KQ_C[k]);\n }\n }\n\n if constexpr (nstages > 1) {\n static_assert(!V_is_K_view, \"K data reuse not implemented multi-stage loading\");\n // Preload K tile for next iteration:\n constexpr bool use_cp_async = true;\n cp_async_wait_all();\n __syncthreads();\n if (!last_iter) {\n if (ncols2 > 1 || mask_h) {\n flash_attn_ext_f16_load_mask\n (mask_h + k_VKQ_0 + nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);\n }\n flash_attn_ext_f16_load_tile\n (K_h2 + int64_t(k_VKQ_0 + nbatch_fa)*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup);\n }\n }\n\n\n#if defined(AMD_WMMA_AVAILABLE) && !defined(LDMATRIX_TRANS_AVAILABLE)\n T_A_VKQ A_identity;\n make_identity_mat(A_identity);\n#endif // defined(AMD_WMMA_AVAILABLE) && !defined(LDMATRIX_TRANS_AVAILABLE)\n\n // Calculate VKQ tile, need to use logical rather than physical elements for i0 due to transposition of V:\n#pragma unroll\n for (int i0_start = 0; i0_start < DV; i0_start += 2*nbatch_V2) {\n static_assert(DV % (2*nbatch_V2) == 0, \"bad loop size\");\n const int i0_stop = i0_start + 2*nbatch_V2;\n const int i0_diff = i0_stop - i0_start;\n\n if constexpr (nstages <= 1) {\n if (!V_is_K_view || i0_stop > 2*nbatch_K2) {\n constexpr bool use_cp_async = nstages == 1;\n flash_attn_ext_f16_load_tile\n (V_h2 + int64_t(k_VKQ_0)*stride_V + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_sup);\n if (use_cp_async) {\n cp_async_wait_all();\n }\n __syncthreads();\n }\n }\n const half2 * tile_V_i = !V_is_K_view || i0_stop > 2*nbatch_K2 ? tile_V : tile_V + i0_start/2;\n\n#if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n constexpr int i0_stride = cols_per_warp == 8 ? T_C_VKQ::I : 2*T_C_VKQ::J;\n#pragma unroll\n for (int i_VKQ_0 = i0_start; i_VKQ_0 < i0_stop; i_VKQ_0 += i0_stride) {\n static_assert((nbatch_fa/2) % (np*T_A_VKQ::J) == 0, \"bad loop size\");\n#pragma unroll\n for (int k00 = 0; k00 < nbatch_fa/2; k00 += np*T_A_VKQ::J) {\n const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::J;\n\n T_A_VKQ A; // Transposed in SRAM but not in registers, gets transposed on load.\n#if defined(LDMATRIX_TRANS_AVAILABLE)\n load_ldmatrix_trans(A, tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V);\n#elif defined(AMD_MFMA_AVAILABLE)\n // MFMA A register layout: A_mat[i=lane%16][k=4*(lane/16)+reg].\n // Normal load gives A_mat[seq][dv] but we need A_mat[dv][seq] = V^T.\n // Load with transposed addressing: 4 strided half loads.\n {\n const half2 * xs0 = tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2;\n const half * xs0_h = (const half *) xs0;\n const int stride_h = stride_tile_V * 2; // stride in half units\n half * A_h = (half *) A.x;\n#pragma unroll\n for (int l = 0; l < 4; ++l) {\n A_h[l] = xs0_h[(4*(threadIdx.x / 16) + l) * stride_h + threadIdx.x % 16];\n }\n }\n#else\n // TODO: Try to transpose tile_V when loading gmem to smem.\n // Use mma to transpose T_A_VKQ for RDNA.\n T_A_VKQ A_trans;\n load_ldmatrix(A_trans, tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V);\n mma(A, A_trans, A_identity);\n#endif // defined(LDMATRIX_TRANS_AVAILABLE)\n if constexpr (T_B_KQ::I == 8) {\n mma(VKQ_C[i_VKQ_0/i0_stride], A, B[k00/(np*T_A_VKQ::J)]);\n } else {\n // Wide version of VKQ_C is column-major.\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n // AMD matrix C is column-major.\n mma(VKQ_C[i_VKQ_0/i0_stride], A, B[k00/(np*T_A_VKQ::J)]);\n#else\n // swap A and B for CUDA.\n mma(VKQ_C[i_VKQ_0/i0_stride], B[k00/(np*T_A_VKQ::J)], A);\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n }\n }\n }\n#else // Volta\n constexpr int i0_stride = 2*T_C_VKQ::J;\n#pragma unroll\n for (int i_VKQ_0 = i0_start; i_VKQ_0 < i0_stop; i_VKQ_0 += i0_stride) {\n static_assert(nbatch_fa % (np*T_A_VKQ::I) == 0, \"bad loop size\");\n static_assert(2*T_B_VKQ::J == T_A_VKQ::I, \"bad tile sizes\");\n#pragma unroll\n for (int k00 = 0; k00 < nbatch_fa; k00 += np*T_A_VKQ::I) {\n const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::I;\n\n T_A_VKQ A; // Transposed in both SRAM and registers, load normally.\n load_ldmatrix(A, tile_V_i + k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V);\n mma(VKQ_C[i_VKQ_0/i0_stride], B[k00/(np*T_A_VKQ::I)], A);\n }\n }\n#endif // defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n\n if constexpr (nstages <= 1) {\n __syncthreads(); // Only needed if tile_K == tile_V.\n }\n }\n#else\n GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup,\n scale, slope, logit_softcap, ne01, ne02,\n stride_K, stride_V, stride_mask,\n tile_Q, tile_K, tile_V, tile_mask,\n Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0);\n NO_DEVICE_CODE;\n#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)\n}\n\n#if defined(TURING_MMA_AVAILABLE)\ntemplate struct mma_tile_sizes {\n using T_A_KQ = tile<16, 8, half2>; // row-major\n using T_B_KQ = tile<16, 8, half2>; // column-major\n using T_C_KQ = tile<16, 16, float>; // column-major\n using T_A_VKQ = tile<16, 8, half2>; // row-major\n using T_B_VKQ = tile<16, 8, half2>; // column-major\n using T_C_VKQ = tile<16, 8, half2>; // column-major\n};\ntemplate<> struct mma_tile_sizes<8> {\n using T_A_KQ = tile<16, 8, half2>; // row-major\n using T_B_KQ = tile< 8, 8, half2>; // column-major\n using T_C_KQ = tile<16, 8, float>; // row-major\n using T_A_VKQ = tile<16, 8, half2>; // row-major\n using T_B_VKQ = tile< 8, 8, half2>; // column-major\n using T_C_VKQ = tile<16, 4, half2>; // row-major\n};\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\ntemplate struct mma_tile_sizes {\n using T_A_KQ = tile<16, 8, half2>; // row-major\n using T_B_KQ = tile<16, 8, half2>; // column-major\n using T_C_KQ = tile<16, 16, float>; // column-major\n using T_A_VKQ = tile<16, 8, half2>; // row-major\n using T_B_VKQ = tile<16, 8, half2>; // column-major\n using T_C_VKQ = tile<16, 8, half2>; // column-major\n};\n#else // Volta\ntemplate struct mma_tile_sizes {\n using T_A_KQ = tile< 8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major\n using T_B_KQ = tile<32, 4, half2, DATA_LAYOUT_I_MAJOR>; // column-major\n using T_C_KQ = tile<32, 8, float, DATA_LAYOUT_I_MAJOR>; // column-major\n using T_A_VKQ = tile< 8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED>; // column-major\n using T_B_VKQ = tile<32, 4, half2, DATA_LAYOUT_I_MAJOR>; // column-major\n using T_C_VKQ = tile<32, 4, half2, DATA_LAYOUT_I_MAJOR>; // column-major\n};\n#endif // defined(TURING_MMA_AVAILABLE)\n\ntemplate\nstatic __device__ __forceinline__ void flash_attn_ext_f16_process_tile(\n const float2 * const __restrict__ Q_f2,\n const half2 * const __restrict__ K_h2,\n const half2 * const __restrict__ V_h2,\n const half * const __restrict__ mask_h,\n const float * const __restrict__ sinks_f,\n float2 * const __restrict__ dstk,\n float2 * const __restrict__ dstk_fixup,\n const float scale,\n const float slope,\n const float logit_softcap,\n const uint3 ne01,\n const int ne02,\n const int gqa_ratio,\n const int ne11,\n const int stride_Q1,\n const int stride_Q2,\n const int stride_K,\n const int stride_V,\n const int stride_mask,\n const int jt,\n const int zt_gqa,\n const int kb0_start,\n const int kb0_stop) {\n#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)\n //In this kernel Q, K, V are matrices while i, j, k are matrix indices.\n\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n constexpr int ncols = ncols1 * ncols2;\n using T_A_KQ = typename mma_tile_sizes::T_A_KQ;\n using T_B_KQ = typename mma_tile_sizes::T_B_KQ;\n using T_C_KQ = typename mma_tile_sizes::T_C_KQ;\n using T_A_VKQ = typename mma_tile_sizes::T_A_VKQ;\n using T_B_VKQ = typename mma_tile_sizes::T_B_VKQ;\n using T_C_VKQ = typename mma_tile_sizes::T_C_VKQ;\n\n constexpr int cols_per_warp = T_B_KQ::I;\n constexpr int cols_per_thread = get_cols_per_thread();\n constexpr int np = cols_per_warp > ncols ? nwarps : nwarps * cols_per_warp/ncols; // Number of parallel CUDA warps per Q column.\n constexpr int nbatch_fa = ggml_cuda_fattn_mma_get_nbatch_fa (DKQ, DV, ncols);\n constexpr int nbatch_K2 = ggml_cuda_fattn_mma_get_nbatch_K2 (DKQ, DV, ncols);\n constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2 (DKQ, DV, ncols);\n constexpr int nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols);\n constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols);\n constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2);\n\n if (cols_per_warp > ncols) {\n NO_DEVICE_CODE;\n return;\n }\n\n static_assert(nwarps * (cols_per_warp/ncols2) % ncols1 == 0, \"bad nwarps\");\n\n constexpr int stride_tile_Q = DKQ/2 + 4;\n constexpr int stride_tile_K = nbatch_K2 + 4;\n\n constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4;\n constexpr int stride_tile_KV_max = stride_tile_K > stride_tile_V ? stride_tile_K : stride_tile_V;\n\n extern __shared__ half2 tile_Q[];\n half2 * tile_K = Q_in_reg ? tile_Q : tile_Q + ncols * stride_tile_Q;\n half2 * tile_V = nstages > 1 ? tile_K + nbatch_fa * stride_tile_K : tile_K;\n half * tile_mask = (half *) (nstages > 1 ? tile_V + nbatch_fa * stride_tile_V : tile_V + nbatch_fa * stride_tile_KV_max);\n\n T_B_KQ Q_B[(Q_in_reg ? DKQ/(2*T_B_KQ::J) : 1)];\n#if defined(TURING_MMA_AVAILABLE)\n T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)];\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)];\n#else // Volta\n T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)];\n#endif // defined(TURING_MMA_AVAILABLE)\n\n float KQ_rowsum[cols_per_thread] = {0.0f};\n float KQ_max[cols_per_thread];\n#pragma unroll\n for (int col = 0; col < cols_per_thread; ++col) {\n KQ_max[col] = -FLT_MAX/2.0f;\n }\n\n // Load Q data into tile_Q, either temporarily or permanently.\n // Q in registers is faster, but register pressure is the biggest bottleneck.\n // The loading is done with decreasing granularity for D for better memory bandwidth.\n const half2 scale_h2 = make_half2(scale, scale);\n#pragma unroll\n for (int stride_k : {warp_size, warp_size/2, warp_size/4, warp_size/8}) {\n const int k0_start = stride_k == warp_size ? 0 : DKQ/2 - (DKQ/2) % (2*stride_k);\n const int k0_stop = DKQ/2 - (DKQ/2) % (1*stride_k);\n const int stride_jc = warp_size / stride_k;\n\n if (k0_start == k0_stop) {\n continue;\n }\n\n#pragma unroll\n for (int jc0 = 0; jc0 < ncols; jc0 += nwarps*stride_jc) {\n const int jc = jc0 + threadIdx.y*stride_jc + (stride_k == warp_size ? 0 : threadIdx.x / stride_k);\n\n if (jc0 + nwarps*stride_jc > ncols && jc >= ncols) {\n break;\n }\n\n const int j = jc / ncols2;\n const int c = jc % ncols2;\n\n if ((ncols1 == 1 || jt*ncols1 + j < int(ne01.z)) && (ncols2 == 1 || zt_gqa*ncols2 + c < gqa_ratio)) {\n#pragma unroll\n for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n const float2 tmp = Q_f2[(jt*ncols1 + j)*stride_Q1 + c*stride_Q2 + k];\n tile_Q[jc*stride_tile_Q + k] = scale_h2 * make_half2(tmp.x, tmp.y);\n }\n } else {\n#pragma unroll\n for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n tile_Q[jc*stride_tile_Q + k] = make_half2(0.0f, 0.0f);\n }\n }\n }\n }\n\n __syncthreads();\n\n if (Q_in_reg) {\n const int j0 = (threadIdx.y / np) * cols_per_warp;\n\n#pragma unroll\n for (int k0 = 0; k0 < DKQ/2; k0 += T_B_KQ::J) {\n load_ldmatrix(Q_B[k0/T_B_KQ::J], tile_Q + j0*stride_tile_Q + k0, stride_tile_Q);\n }\n }\n\n __syncthreads();\n\n int kb0 = kb0_start;\n\n // Preload mask and K data for first iteration when using cp_async with multiple stages:\n if constexpr (nstages > 1) {\n static_assert(nbatch_K2 == DKQ/2, \"batching not implemented for multi-stage pipeline\");\n constexpr bool use_cp_async = true;\n constexpr bool oob_check = false;\n constexpr int k_VKQ_sup = nbatch_fa;\n if (ncols2 > 1 || mask_h) {\n flash_attn_ext_f16_load_mask\n (mask_h + kb0*nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);\n }\n flash_attn_ext_f16_load_tile\n (K_h2 + int64_t(kb0)*nbatch_fa*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup);\n }\n\n // kb0_start is always < kb0_stop so the last iter can be executed unconditionally.\n if constexpr (ncols2 == 1) {\n constexpr bool oob_check = true;\n for (; kb0 < kb0_stop-1; ++kb0) {\n constexpr bool last_iter = false;\n constexpr int k_VKQ_sup = nbatch_fa;\n flash_attn_ext_f16_iter\n \n (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,\n ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,\n KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);\n }\n constexpr bool last_iter = true;\n const int k_VKQ_sup = ne11 - kb0*nbatch_fa;\n flash_attn_ext_f16_iter\n \n (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,\n ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,\n KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);\n } else {\n constexpr bool oob_check = false;\n for (; kb0 < kb0_stop-1; ++kb0) {\n constexpr bool last_iter = false;\n constexpr int k_VKQ_sup = nbatch_fa;\n flash_attn_ext_f16_iter\n \n (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,\n ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,\n KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);\n }\n constexpr bool last_iter = true;\n constexpr int k_VKQ_sup = nbatch_fa;\n flash_attn_ext_f16_iter\n \n (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,\n ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,\n KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);\n }\n\n // With multi-stage loading there is no __syncthreads at the end of the iter,\n // there can be a race condition on shared memory access for combining/writing back results.\n if constexpr (nstages > 1 && nwarps*cols_per_warp > nbatch_fa) {\n __syncthreads();\n }\n\n // Finally, sum up partial KQ rowsums.\n {\n#if defined(TURING_MMA_AVAILABLE)\n // The partial sums are spread across 8/4 threads.\n constexpr int offset_first = cols_per_warp == 8 ? 16 : 2;\n constexpr int offset_last = cols_per_warp == 8 ? 4 : 1;\n#elif defined(AMD_MFMA_AVAILABLE)\n // The partial sums are spread across 4 threads (wavefront64, 16 cols).\n constexpr int offset_first = 32;\n constexpr int offset_last = 16;\n#elif defined(AMD_WMMA_AVAILABLE)\n // The partial sums are spread across 2 threads.\n constexpr int offset_first = 16;\n constexpr int offset_last = 16;\n#else // Volta\n // The partial sums are spread across 2 threads.\n constexpr int offset_first = 2;\n constexpr int offset_last = 2;\n#endif // defined(TURING_MMA_AVAILABLE)\n#pragma unroll\n for (int col = 0; col < cols_per_thread; ++col) {\n#pragma unroll\n for (int offset = offset_first; offset >= offset_last; offset >>= 1) {\n KQ_rowsum[col] += __shfl_xor_sync(0xFFFFFFFF, KQ_rowsum[col], offset, warp_size);\n }\n }\n }\n\n // If attention sinks are used, potentially re-scale if KQ_max is small.\n // Also add the sink as a value to KQ_rowsum, this is done after synchronization of KQ_rowsum\n // so it's being done unconditionally for every thread.\n if (!is_fixup && (np == 1 || threadIdx.y % np == 0) && sinks_f) {\n float KQ_max_scale[cols_per_thread];\n#pragma unroll\n for (int col = 0; col < cols_per_thread; ++col) {\n const int jc = cols_per_warp == 8 ? T_C_KQ::get_j(col) : T_C_KQ::get_i(2*col);\n const float sink = sinks_f[jc % ncols2];\n\n const float KQ_max_new = fmaxf(KQ_max[col], sink);\n const float KQ_max_diff = KQ_max[col] - KQ_max_new;\n KQ_max_scale[col] = expf(KQ_max_diff);\n KQ_max[col] = KQ_max_new;\n\n *((uint32_t *) &KQ_max_scale[col]) *= KQ_max_diff >= SOFTMAX_FTZ_THRESHOLD;\n\n const float KQ_max_add = expf(sink - KQ_max_new);\n KQ_rowsum[col] = KQ_max_scale[col]*KQ_rowsum[col] + KQ_max_add;\n }\n\n#if defined(TURING_MMA_AVAILABLE)\n if constexpr (cols_per_warp == 8) {\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[cols_per_thread - 1]);\n#pragma unroll\n for (int i = 0; i < DV/T_C_VKQ::I; ++i) {\n#pragma unroll\n for (int l = 0; l < T_C_VKQ::ne; ++l) {\n VKQ_C[i].x[l] *= KQ_max_scale_h2;\n }\n }\n } else {\n#pragma unroll\n for (int col = 0; col < cols_per_thread; ++col) {\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]);\n#pragma unroll\n for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n for (int l0 = 0; l0 < T_C_VKQ::ne; l0 += 2) {\n VKQ_C[i].x[l0 + col] *= KQ_max_scale_h2;\n }\n }\n }\n }\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]);\n#pragma unroll\n for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n for (int l = 0; l < T_C_VKQ::ne; ++l) {\n VKQ_C[i].x[l] *= KQ_max_scale_h2;\n }\n }\n#else // Volta\n const int col = (threadIdx.x / 2) % 2;\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]);\n#pragma unroll\n for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n for (int l = 0; l < T_C_VKQ::ne; ++l) {\n VKQ_C[i].x[l] *= KQ_max_scale_h2;\n }\n }\n#endif // defined(TURING_MMA_AVAILABLE)\n }\n\n // Combine VKQ accumulator values if np > 1.\n // It's also faster to do small writes to shared memory, then large write to VRAM than to do small writes to VRAM.\n // So also write VKQ accumulators to shared memory in column-major format if np == 1.\n\n constexpr int tile_stride = nbatch_combine + 4;\n static_assert((DV/2) % nbatch_combine == 0, \"bad nbatch_combine\");\n\n if constexpr (cols_per_warp == 8) {\n const int jc_cwmo = (threadIdx.x % (2*T_C_VKQ::J)) / T_C_VKQ::J; // jc combine write meta offset\n const int jc_cwm = threadIdx.y*(2*T_C_VKQ::J) + 2*T_C_VKQ::get_j(-1) + jc_cwmo; // jc combine write meta\n const float2 KQ_cmr = make_float2(KQ_max[jc_cwmo], KQ_rowsum[jc_cwmo]); // KQ combine max rowsum\n\n if (((!needs_fixup && !is_fixup) || np > 1) && threadIdx.x < 2*T_C_VKQ::J) {\n // Use the 16 bytes of padding in each row to store the meta data: KQ max, KQ rowsum, KQ max scale.\n ((float2 *) tile_Q)[jc_cwm*(tile_stride/2) + nbatch_combine/2] = KQ_cmr;\n }\n\n __syncthreads();\n\n if (np == 1) {\n // No combination is needed, the meta data can be directly written from registers to VRAM.\n if (needs_fixup && threadIdx.x < T_B_KQ::I) {\n float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols;\n dstk_fixup_meta[jc_cwm] = KQ_cmr;\n }\n if (is_fixup && threadIdx.x < T_B_KQ::I) {\n float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols;\n dstk_fixup_meta[jc_cwm] = KQ_cmr;\n }\n }\n } else {\n // jc_cwm = jc combine write meta\n // KQ_cmr = KQ combine max rowsum\n // Use the 16 bytes of padding in each Q column to store the meta data: KQ max, KQ rowsum, KQ max scale.\n#if defined(TURING_MMA_AVAILABLE)\n const int jc_cwm = threadIdx.y*cols_per_warp + T_C_VKQ::get_i(threadIdx.x % 4);\n const float2 KQ_cmr = make_float2(KQ_max[threadIdx.x % cols_per_thread], KQ_rowsum[threadIdx.x % cols_per_thread]);\n const bool thread_should_write = threadIdx.x % 4 < cols_per_thread;\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n const int jc_cwm = threadIdx.y*cols_per_warp + T_C_VKQ::get_i(0);\n const float2 KQ_cmr = make_float2(KQ_max[0], KQ_rowsum[0]);\n const bool thread_should_write = threadIdx.x / 16 < cols_per_thread;\n#else // Volta\n const int jc_cwm = threadIdx.y*cols_per_warp + T_C_KQ::get_i(threadIdx.x & 2);\n const float2 KQ_cmr = make_float2(KQ_max[(threadIdx.x & 2) / 2], KQ_rowsum[(threadIdx.x & 2) / 2]);\n const bool thread_should_write = T_C_KQ::J == 8 || T_C_KQ::get_j(threadIdx.x & 2) < 8;\n#endif // defined(TURING_MMA_AVAILABLE)\n\n if (((!needs_fixup && !is_fixup) || np > 1) && thread_should_write) {\n ((float2 *) tile_Q)[jc_cwm*(tile_stride/2) + nbatch_combine/2] = KQ_cmr;\n }\n\n __syncthreads();\n\n if (np == 1) {\n // No combination is needed, the meta data can be directly written from registers to VRAM.\n if (needs_fixup && thread_should_write) {\n float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols;\n dstk_fixup_meta[jc_cwm] = KQ_cmr;\n }\n if (is_fixup && thread_should_write) {\n float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols;\n dstk_fixup_meta[jc_cwm] = KQ_cmr;\n }\n }\n }\n\n if (np > 1 && threadIdx.y % np == 0) {\n // Combine the meta data for parallel warps via shared memory.\n // Warps with threadIdx.y % np != 0 must NOT return early.\n // All threads must return simultaneously to avoid race conditions with work on the next tile.\n\n constexpr int nmeta = np*cols_per_warp >= warp_size ? np*cols_per_warp/warp_size : 1;\n\n const int jc_meta = threadIdx.y*cols_per_warp + (np*cols_per_warp < warp_size ? threadIdx.x % (np*cols_per_warp) : threadIdx.x);\n float2 * const meta_ptr = ((float2 *) tile_Q) + jc_meta*(tile_stride/2) + nbatch_combine/2;\n float2 meta[nmeta];\n#pragma unroll\n for (int imeta = 0; imeta < nmeta; ++imeta) {\n meta[imeta] = meta_ptr[imeta * warp_size * tile_stride/2];\n }\n\n float KQ_cmn = meta[0].x; // KQ combine max new, max between all parallel warps.\n#pragma unroll\n for (int imeta = 1; imeta < nmeta; ++imeta) {\n KQ_cmn = fmaxf(KQ_cmn, meta[imeta].x);\n }\n#pragma unroll\n for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {\n if (offset < warp_size) {\n KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, warp_size));\n }\n }\n\n float KQ_cms[nmeta]; // KQ combine max scale per warp.\n#pragma unroll\n for (int imeta = 0; imeta < nmeta; ++imeta) {\n KQ_cms[imeta] = expf(meta[imeta].x - KQ_cmn);\n }\n\n float KQ_crs = KQ_cms[0]*meta[0].y; // KQ combine rowsum, scaled sum of all parallel warps.\n#pragma unroll\n for (int imeta = 1; imeta < nmeta; ++imeta) {\n KQ_crs += KQ_cms[imeta]*meta[imeta].y;\n }\n#pragma unroll\n for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {\n if (offset < warp_size) {\n KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, warp_size);\n }\n }\n\n __syncthreads();\n\n // Write back combined meta data:\n#pragma unroll\n for (int imeta = 0; imeta < nmeta; ++imeta) {\n if (np*cols_per_warp >= warp_size || threadIdx.x < np*cols_per_warp) {\n // Combined KQ max scale + rowsum.\n meta_ptr[imeta * warp_size * tile_stride/2] = make_float2(KQ_cms[imeta], KQ_crs);\n }\n }\n\n // Combined KQ max + rowsum.\n static_assert(cols_per_warp <= warp_size);\n if (needs_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) {\n float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols;\n dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs);\n }\n if (is_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) {\n float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols;\n dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs);\n }\n } else if (np > 1) {\n // Warps with threadIdx.y % np == 0 execute a __syncthreads() in the if branch.\n // Therefore, all other warps also need to execute a __syncthreads().\n // Otherwise the points at which warps synchronize with each other would become misaligned.\n __syncthreads();\n }\n\n#pragma unroll\n for (int k00 = 0; k00 < DV/2; k00 += nbatch_combine) {\n if constexpr (cols_per_warp == 8) {\n const int jc_cwd = threadIdx.y*T_B_KQ::I + T_B_KQ::get_i(-1); // jc combine write data\n#pragma unroll\n for (int k1 = 0; k1 < nbatch_combine; k1 += T_B_KQ::J) {\n const T_B_KQ B = get_transposed(VKQ_C[(k00 + k1)/T_B_KQ::J]); // Conversion of C to B matrix puts it in column-major format.\n\n#pragma unroll\n for (int l = 0; l < T_B_KQ::ne; ++l) {\n const int k = k1 + T_B_KQ::get_j(l);\n\n tile_Q[jc_cwd*tile_stride + k] = B.x[l];\n }\n }\n } else {\n const int j0 = threadIdx.y*cols_per_warp;\n#pragma unroll\n for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J) {\n#pragma unroll\n for (int l = 0; l < T_C_VKQ::ne; ++l) {\n const int j = j0 + T_C_VKQ::get_i(l);\n const int k = k1 + T_C_VKQ::get_j(l);\n\n tile_Q[j*tile_stride + k] = VKQ_C[(k00 + k1)/T_C_VKQ::J].x[l];\n }\n }\n }\n\n __syncthreads();\n\n if (np == 1 || threadIdx.y % np == 0) {\n // The first 2*2*gridDim.x*ncols floats in dstk_fixup are for storing max. values and row sums.\n // The values after that are for the partial results of the individual blocks.\n float2 * dstk_fixup_data = dstk_fixup + gridDim.x*(2*ncols) + blockIdx.x*(ncols*(DV/2));\n\n#pragma unroll\n for (int stride_k : {warp_size, warp_size/2, warp_size/4, warp_size/8}) {\n const int k0_start = stride_k == warp_size ? 0 : nbatch_combine - nbatch_combine % (2*stride_k);\n const int k0_stop = nbatch_combine - nbatch_combine % (1*stride_k);\n const int stride_jc = warp_size / stride_k;\n\n if (k0_start == k0_stop) {\n continue;\n }\n\n#pragma unroll\n for (int jc0_dst = 0; jc0_dst < ncols; jc0_dst += (nwarps/np)*stride_jc) {\n const int jc_dst = jc0_dst + (threadIdx.y/np)*stride_jc + (stride_k == warp_size ? 0 : threadIdx.x / stride_k);\n\n if (jc0_dst + (nwarps/np)*stride_jc > ncols && jc_dst >= ncols) {\n break;\n }\n\n const int jc_tile_K = (jc_dst/cols_per_warp)*(np*cols_per_warp) + jc_dst % cols_per_warp;\n\n const int j_dst = jc_dst / ncols2;\n const int c_dst = jc_dst % ncols2;\n\n if (!is_fixup && ((ncols1 > 1 && jt*ncols1 + j_dst >= int(ne01.z)) || (ncols2 > 1 && zt_gqa*ncols2 + c_dst >= gqa_ratio))) {\n continue;\n }\n\n const float * meta_j = (const float *) tile_Q + jc_tile_K*tile_stride + nbatch_combine;\n#pragma unroll\n for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n float2 dstk_val = make_float2(0.0f, 0.0f);\n#pragma unroll\n for (int ip = 0; ip < np; ++ip) {\n const float KQ_crs = np == 1 ? 1.0f : meta_j[ip*cols_per_warp * tile_stride + 0];\n const float2 dstk_val_add = __half22float2(tile_Q[(jc_tile_K + ip*cols_per_warp) * tile_stride + k]);\n dstk_val.x += dstk_val_add.x*KQ_crs;\n dstk_val.y += dstk_val_add.y*KQ_crs;\n }\n\n if (!needs_fixup && !is_fixup) {\n const float KQ_rowsum_j = meta_j[1];\n dstk_val.x /= KQ_rowsum_j;\n dstk_val.y /= KQ_rowsum_j;\n }\n\n if (is_fixup) {\n dstk_fixup_data[jc_dst*(DV/2) + k00 + k] = dstk_val;\n } else {\n dstk[((jt*ncols1 + j_dst)*ne02 + c_dst)*(DV/2) + k00 + k] = dstk_val;\n }\n }\n }\n }\n }\n if (np > 1) {\n __syncthreads();\n }\n }\n#else\n GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dstk_fixup,\n scale, slope, logit_softcap, ne01, ne02, gqa_ratio,\n stride_Q1, stride_Q2, stride_K, stride_V, stride_mask,\n jt, kb0_start, kb0_stop);\n NO_DEVICE_CODE;\n#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)\n}\n\ntemplate\n__launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2))\nstatic __global__ void flash_attn_ext_f16(\n const char * __restrict__ Q,\n const char * __restrict__ K,\n const char * __restrict__ V,\n const char * __restrict__ mask,\n const char * __restrict__ sinks,\n const int * __restrict__ KV_max,\n float * __restrict__ dst,\n float2 * __restrict__ dst_meta,\n const float scale,\n const float max_bias,\n const float m0,\n const float m1,\n const uint32_t n_head_log2,\n const float logit_softcap,\n const int32_t ne00, const uint3 ne01, const int32_t ne02, const int32_t ne03,\n const int32_t nb01, const int32_t nb02, const int32_t nb03,\n const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13,\n const int32_t nb11, const int32_t nb12, const int64_t nb13,\n const int32_t nb21, const int32_t nb22, const int64_t nb23,\n const int32_t ne31, const int32_t ne32, const int32_t ne33,\n const int32_t nb31, const int32_t nb32, const int64_t nb33) {\n#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE))\n\n // Skip unused kernel variants for faster compilation:\n if (use_logit_softcap && !(DKQ == 128 || DKQ == 256)) {\n NO_DEVICE_CODE;\n return;\n }\n#ifdef VOLTA_MMA_AVAILABLE\n if (ncols1*ncols2 < 32) {\n NO_DEVICE_CODE;\n return;\n }\n#endif // VOLTA_MMA_AVAILABLE\n\n#if __CUDA_ARCH__ == GGML_CUDA_CC_TURING\n if (ncols1*ncols2 > 32) {\n NO_DEVICE_CODE;\n return;\n }\n#endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING\n\n#if defined(AMD_WMMA_AVAILABLE)\n if (ncols1*ncols2 > 32 || ncols1*ncols2 < 16 || DKQ > 128 || ncols2 == 1) {\n NO_DEVICE_CODE;\n return;\n }\n#endif // defined(AMD_WMMA_AVAILABLE)\n\n#if defined(AMD_MFMA_AVAILABLE)\n if (DKQ != 64 && DKQ != 80 && DKQ != 96 && DKQ != 112 && DKQ != 128) {\n NO_DEVICE_CODE;\n return;\n }\n#endif // defined(AMD_MFMA_AVAILABLE)\n\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n constexpr int ncols = ncols1 * ncols2;\n constexpr int nbatch_fa = ggml_cuda_fattn_mma_get_nbatch_fa(DKQ, DV, ncols);\n constexpr int nthreads = ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols);\n constexpr int nwarps = nthreads / warp_size;\n\n const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n\n const int stride_Q1 = nb01 / sizeof(float2);\n const int stride_Q2 = nb02 / sizeof(float2);\n const int stride_K = nb11 / sizeof(half2);\n const int stride_mask = nb31 / sizeof(half);\n\n const int stride_V = V_is_K_view ? stride_K : nb21 / sizeof(half2);\n\n const int iter_k = (ne11 + (nbatch_fa - 1)) / nbatch_fa;\n const int iter_j = (ne01.z + (ncols1 - 1)) / ncols1;\n const int iter_z_gqa = (gqa_ratio + (ncols2 - 1)) / ncols2;\n\n // kbc == k block continuous, current index in continuous ijk space.\n int kbc = int64_t(blockIdx.x + 0)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x;\n const int kbc_stop = int64_t(blockIdx.x + 1)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x;\n\n // If the seams of 2 CUDA blocks fall within an output tile their results need to be combined.\n // For this we need to track both the block that starts the tile (needs_fixup) and the block that finishes the tile (is_fixup).\n // In the most general case >2 seams can fall into the same tile.\n\n // kb0 == k start index when in the output tile.\n int kb0_start = kbc % iter_k;\n int kb0_stop = min(iter_k, kb0_start + kbc_stop - kbc);\n\n while (kbc < kbc_stop && kb0_stop == iter_k) {\n // z_KV == K/V head index, zt_gqa = Q head start index per K/V head, jt = token position start index\n const int sequence = kbc /(iter_k*iter_j*iter_z_gqa*ne12);\n const int z_KV = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence)/(iter_k*iter_j*iter_z_gqa);\n const int zt_gqa = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV)/(iter_k*iter_j);\n const int jt = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV - iter_k*iter_j * zt_gqa) / iter_k;\n\n const int zt_Q = z_KV*gqa_ratio + zt_gqa*ncols2; // Global Q head start index.\n\n const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q);\n const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV);\n const half * mask_h = ncols2 == 1 && !mask ? nullptr :\n (const half *) (mask + nb33*(sequence % ne33));\n float2 * dstk = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2);\n\n const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);\n const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;\n\n const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;\n\n if (KV_max) {\n kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);\n }\n constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.\n if (kb0_start == 0) {\n constexpr bool needs_fixup = false; // CUDA block is working on an entire tile.\n flash_attn_ext_f16_process_tile\n (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,\n ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);\n } else {\n constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile.\n flash_attn_ext_f16_process_tile\n (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,\n ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);\n }\n\n kbc += iter_k;\n kbc -= kbc % iter_k;\n\n kb0_start = 0;\n kb0_stop = min(iter_k, kbc_stop - kbc);\n }\n\n if (kbc >= kbc_stop) {\n return;\n }\n\n // z_KV == K/V head index, zt_gqa = Q head start index per K/V head, jt = token position start index.\n const int sequence = kbc /(iter_k*iter_j*iter_z_gqa*ne12);\n const int z_KV = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence)/(iter_k*iter_j*iter_z_gqa);\n const int zt_gqa = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV)/(iter_k*iter_j);\n const int jt = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV - iter_k*iter_j * zt_gqa) / iter_k;\n\n const int zt_Q = z_KV*gqa_ratio + zt_gqa*ncols2; // Global Q head start index.\n\n const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q);\n const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV);\n const half * mask_h = ncols2 == 1 && !mask ? nullptr :\n (const half *) (mask + nb33*(sequence % ne33));\n float2 * dstk = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2);\n\n const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);\n const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;\n\n const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;\n\n if (KV_max) {\n kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);\n }\n\n constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.\n constexpr bool needs_fixup = false;\n flash_attn_ext_f16_process_tile\n (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,\n ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);\n#else\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n NO_DEVICE_CODE;\n#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE))\n}\n\ntemplate \nvoid ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * KQV = dst;\n const int id = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[id].cc;\n\n constexpr int ncols = ncols1 * ncols2;\n\n const int nthreads = ggml_cuda_fattn_mma_get_nthreads (DKQ, DV, ncols, cc);\n const int nbatch_fa = ggml_cuda_fattn_mma_get_nbatch_fa (DKQ, DV, ncols, cc);\n const int nbatch_K2 = ggml_cuda_fattn_mma_get_nbatch_K2 (DKQ, DV, ncols, cc);\n const int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2 (DKQ, DV, ncols, cc);\n const int nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols, cc);\n const bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols, cc);\n const int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, cc);\n\n const int cols_per_warp = std::min(ncols, get_cols_per_warp(cc));\n const int warp_size_host = ggml_cuda_info().devices[ctx.device].warp_size;\n const int nwarps = nthreads / warp_size_host;\n\n constexpr bool V_is_K_view = DKQ == 576; // Guaranteed by the kernel selection logic in fattn.cu\n\n const size_t nbytes_shared_KV_1stage = nbatch_fa * std::max(nbatch_K2 + 4, nbatch_V2 + 4) * sizeof(half2);\n const size_t nbytes_shared_KV_2stage = nbatch_fa * (nbatch_K2 + 4 + nbatch_V2 + 4) * sizeof(half2);\n const size_t nbytes_shared_Q = ncols * (DKQ/2 + 4) * sizeof(half2);\n const size_t nbytes_shared_mask = ncols1 * (nbatch_fa/2 + 4) * sizeof(half2);\n const size_t nbytes_shared_combine = nwarps*cols_per_warp * (nbatch_combine + 4) * sizeof(half2);\n\n const size_t nbytes_shared_KV = nstages <= 1 ? nbytes_shared_KV_1stage : nbytes_shared_KV_2stage;\n\n const size_t nbytes_shared_total = std::max(nbytes_shared_combine, Q_in_reg ?\n std::max(nbytes_shared_Q, nbytes_shared_KV + nbytes_shared_mask) :\n nbytes_shared_Q + nbytes_shared_KV + nbytes_shared_mask);\n\n float logit_softcap;\n memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n#if defined(GGML_USE_HIP)\n using fattn_kernel_ptr_t = const void*;\n#else\n using fattn_kernel_ptr_t = fattn_kernel_t;\n#endif // defined(GGML_USE_HIP)\n fattn_kernel_t fattn_kernel;\n if (logit_softcap == 0.0f) {\n constexpr bool use_logit_softcap = false;\n fattn_kernel = flash_attn_ext_f16;\n\n#if !defined(GGML_USE_MUSA)\n static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};\n if (!shared_memory_limit_raised[id]) {\n CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));\n shared_memory_limit_raised[id] = true;\n }\n#endif // !defined(GGML_USE_MUSA)\n } else {\n constexpr bool use_logit_softcap = true;\n fattn_kernel = flash_attn_ext_f16;\n\n#if !defined(GGML_USE_MUSA)\n static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};\n if (!shared_memory_limit_raised[id]) {\n CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));\n shared_memory_limit_raised[id] = true;\n }\n#endif // !defined(GGML_USE_MUSA)\n }\n\n launch_fattn\n (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, warp_size_host);\n}\n\n\n#define DECL_FATTN_MMA_F16_CASE(DKQ, DV, ncols1, ncols2) \\\n template void ggml_cuda_flash_attn_ext_mma_f16_case \\\n (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \\\n\n#define DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(DKQ, DV, ncols) \\\n extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 1, 1); \\\n extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 2, 2); \\\n extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 4, 4); \\\n extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 8, 8); \\\n extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/16, 16); \\\n\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 8)\n\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 16)\n\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 32)\n\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64, 64, 64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80, 80, 64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96, 96, 64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 64)\n\n// The number of viable configurations for Deepseek is very limited:\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 1, 16);\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 2, 16);\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 4, 16);\n\n// For GLM 4.7 Flash\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 4, 4);\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 8, 4);\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 16, 4);\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 1, 32);\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 2, 32);\n" }, { "path": "src/ggml-cuda/fattn-tile.cu", "content": "#include \"common.cuh\"\n#include \"fattn-tile.cuh\"\n#include \"fattn-wmma-f16.cuh\"\n\nvoid ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * V = dst->src[2];\n switch (K->ne[0]) {\n case 40: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_cuda_flash_attn_ext_tile_case< 40, 40>(ctx, dst);\n } break;\n case 64: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_cuda_flash_attn_ext_tile_case< 64, 64>(ctx, dst);\n } break;\n case 72: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_cuda_flash_attn_ext_tile_case< 72, 72>(ctx, dst);\n } break;\n case 80: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_cuda_flash_attn_ext_tile_case< 80, 80>(ctx, dst);\n } break;\n case 96: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_cuda_flash_attn_ext_tile_case< 96, 96>(ctx, dst);\n } break;\n case 112: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_cuda_flash_attn_ext_tile_case<112, 112>(ctx, dst);\n } break;\n case 128: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_cuda_flash_attn_ext_tile_case<128, 128>(ctx, dst);\n } break;\n case 256: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_cuda_flash_attn_ext_tile_case<256, 256>(ctx, dst);\n } break;\n case 576: {\n GGML_ASSERT(V->ne[0] == 512);\n ggml_cuda_flash_attn_ext_tile_case<576, 512>(ctx, dst);\n } break;\n default: {\n GGML_ABORT(\"Unsupported head size\");\n } break;\n }\n}\n" }, { "path": "src/ggml-cuda/fattn-tile.cuh", "content": "#include \"common.cuh\"\n#include \"fattn-common.cuh\"\n#include \"fattn-wmma-f16.cuh\"\n\n// nbatch_fa == number of KQ rows to process per iteration\n// nbatch_K == number of K columns to load in parallel for KQ calculation\n\n// TODO optimize kernel parameters for FP16 NVIDIA (P100)\n// TODO optimize kernel parameters for head sizes 40, 72, 80, 96, 112\n\n// The ROCm compiler cannot handle templating in __launch_bounds__.\n// As a workaround, define a macro to package the kernel parameters as uint32_t:\n#define GGML_CUDA_FATTN_TILE_CONFIG_CASE(DKQ_, DV_, ncols_, nthreads, occupancy, nbatch_fa, nbatch_K) \\\n if (DKQ == (DKQ_) && DV == (DV_) && ncols == (ncols_)) { \\\n static_assert((nthreads) <= 512, \"bad nthreads\"); \\\n static_assert((occupancy) <= 8, \"bad occupancy\"); \\\n static_assert((nbatch_fa) <= 256, \"bad nbatch_fa\"); \\\n static_assert((nbatch_K) <= 256, \"bad nbatch_K\"); \\\n return ((nthreads) << 0) | ((occupancy) << 10) | ((nbatch_fa) << 14) | ((nbatch_K) << 23); \\\n } \\\n\nstatic constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nvidia_fp16(const int DKQ, const int DV, const int ncols) {\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 64, 2, 64, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 64, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 256, 2, 64, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 16, 256, 2, 64, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 32, 256, 2, 64, 40)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 2, 64, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 4, 128, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 64)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 64, 2, 64, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 2, 64, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 8, 256, 2, 64, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 16, 256, 2, 64, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 32, 256, 2, 64, 72)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 64, 2, 64, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 4, 128, 2, 64, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 64, 40)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 2, 64, 2, 64, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 4, 128, 2, 64, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 8, 256, 2, 64, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 16, 256, 2, 64, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 64, 48)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 2, 64, 2, 64, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 4, 128, 2, 64, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 2, 64, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 64, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 64, 56)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 2, 64, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 4, 128, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 8, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 64, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 64, 64)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 4, 128, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 2, 64, 64)\n\n return 0;\n}\n\nstatic constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nvidia_fp32(const int DKQ, const int DV, const int ncols) {\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 64, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 16, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 32, 256, 2, 32, 40)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 2, 128, 3, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 4, 128, 3, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 128, 3, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 16, 128, 3, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 64)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 64, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 8, 256, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 16, 256, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 32, 256, 2, 32, 72)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 64, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 4, 128, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 8, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 16, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 32, 40)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 2, 64, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 4, 128, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 8, 256, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 16, 256, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 32, 48)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 2, 64, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 4, 128, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 32, 56)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 2, 128, 3, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 4, 128, 3, 32, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 8, 128, 3, 64, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 128, 3, 32, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 128, 3, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 3, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 32, 256)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 32, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 32, 64)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 4, 128, 2, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 2, 32, 64)\n\n return 0;\n}\n\nstatic constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_amd(const int DKQ, const int DV, const int ncols) {\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 64, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 16, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 32, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 64, 256, 2, 32, 40)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 2, 64, 3, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 4, 128, 3, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 128, 2, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 16, 256, 2, 128, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 64, 256, 2, 64, 64)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 64, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 8, 256, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 16, 256, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 32, 256, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 64, 256, 2, 32, 72)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 64, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 4, 128, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 8, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 16, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 64, 256, 2, 32, 40)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 2, 64, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 4, 128, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 8, 256, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 16, 256, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 64, 256, 2, 32, 48)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 2, 64, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 4, 128, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 64, 256, 2, 32, 56)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 2, 256, 2, 128, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 4, 128, 2, 64, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 8, 256, 2, 64, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 256, 2, 64, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 2, 64, 32)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 256, 2, 128, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 256, 2, 64, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 64, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 32, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 32, 128)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 4, 128, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 32, 512, 1, 128, 64)\n\n return 0;\n}\n\nstatic constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_amd_rdna(const int DKQ, const int DV, const int ncols) {\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 64, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 16, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 32, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 64, 256, 2, 32, 40)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 2, 64, 8, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 4, 64, 8, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 128, 5, 128, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 16, 128, 5, 128, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 32, 128, 4, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 64, 128, 5, 64, 64)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 64, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 8, 256, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 16, 256, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 32, 256, 2, 32, 72)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 64, 256, 2, 32, 72)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 64, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 4, 128, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 8, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 16, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 32, 40)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 64, 256, 2, 32, 40)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 2, 64, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 4, 128, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 8, 256, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 16, 256, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 32, 48)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 64, 256, 2, 32, 48)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 2, 64, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 4, 128, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 32, 56)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 64, 256, 2, 32, 56)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 2, 64, 8, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 4, 128, 8, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 8, 128, 8, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 256, 3, 128, 128)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 3, 128, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 3, 64, 64)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 64, 8, 32, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 6, 32, 256)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 128, 6, 32, 256)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 5, 32, 256)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 3, 64, 128)\n\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 4, 128, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 4, 64, 64)\n GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 32, 256, 2, 128, 64)\n\n return 0;\n}\n\nstatic __host__ uint32_t ggml_cuda_fattn_tile_get_config(const int DKQ, const int DV, const int ncols, const int cc) {\n if (GGML_CUDA_CC_IS_AMD(cc)) {\n if (GGML_CUDA_CC_IS_RDNA(cc)) {\n return ggml_cuda_fattn_tile_get_config_amd_rdna(DKQ, DV, ncols);\n }\n return ggml_cuda_fattn_tile_get_config_amd(DKQ, DV, ncols);\n }\n if (fast_fp16_available(cc)) {\n return ggml_cuda_fattn_tile_get_config_nvidia_fp16(DKQ, DV, ncols);\n }\n return ggml_cuda_fattn_tile_get_config_nvidia_fp32(DKQ, DV, ncols);\n}\n\nstatic constexpr __device__ uint32_t ggml_cuda_fattn_tile_get_config(const int DKQ, const int DV, const int ncols) {\n#ifdef GGML_USE_HIP\n#ifdef RDNA\n return ggml_cuda_fattn_tile_get_config_amd_rdna(DKQ, DV, ncols);\n#else\n return ggml_cuda_fattn_tile_get_config_amd(DKQ, DV, ncols);\n#endif // RDNA\n#else\n#ifdef FAST_FP16_AVAILABLE\n return ggml_cuda_fattn_tile_get_config_nvidia_fp16(DKQ, DV, ncols);\n#else\n return ggml_cuda_fattn_tile_get_config_nvidia_fp32(DKQ, DV, ncols);\n#endif // FAST_FP16_AVAILABLE\n#endif // GGML_USE_HIP\n}\n\nstatic __host__ int ggml_cuda_fattn_tile_get_nthreads(const int DKQ, const int DV, const int ncols, const int cc) {\n return (ggml_cuda_fattn_tile_get_config(DKQ, DV, ncols, cc) >> 0) & ((1 << 10) - 1);\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_tile_get_nthreads(const int DKQ, const int DV, const int ncols) {\n return (ggml_cuda_fattn_tile_get_config(DKQ, DV, ncols) >> 0) & ((1 << 10) - 1);\n}\n\nstatic __host__ int ggml_cuda_fattn_tile_get_occupancy(const int DKQ, const int DV, const int ncols, const int cc) {\n return (ggml_cuda_fattn_tile_get_config(DKQ, DV, ncols, cc) >> 10) & ((1 << 4) - 1);\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_tile_get_occupancy(const int DKQ, const int DV, const int ncols) {\n return (ggml_cuda_fattn_tile_get_config(DKQ, DV, ncols) >> 10) & ((1 << 4) - 1);\n}\n\nstatic __host__ int ggml_cuda_fattn_tile_get_nbatch_fa(const int DKQ, const int DV, const int ncols, const int cc) {\n return (ggml_cuda_fattn_tile_get_config(DKQ, DV, ncols, cc) >> 14) & ((1 << 9) - 1);\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_tile_get_nbatch_fa(const int DKQ, const int DV, const int ncols) {\n return (ggml_cuda_fattn_tile_get_config(DKQ, DV, ncols) >> 14) & ((1 << 9) - 1);\n}\n\nstatic __host__ int ggml_cuda_fattn_tile_get_nbatch_K(const int DKQ, const int DV, const int ncols, const int cc) {\n return (ggml_cuda_fattn_tile_get_config(DKQ, DV, ncols, cc) >> 23) & ((1 << 9) - 1);\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_tile_get_nbatch_K(const int DKQ, const int DV, const int ncols) {\n return (ggml_cuda_fattn_tile_get_config(DKQ, DV, ncols) >> 23) & ((1 << 9) - 1);\n}\n\n// TODO: deduplicate with mma-f16\ntemplate\nstatic __device__ __forceinline__ void flash_attn_tile_load_tile(\n const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int stride_KV, const int i_sup) {\n constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n auto load = [&] __device__ (const int n) {\n const int stride_j = warp_size >> n;\n\n if (stride_j == 0) {\n return;\n }\n\n const int j0_start = stride_j == warp_size ? 0 : ((J/2)/cpy_ne) - ((J/2)/cpy_ne) % (2*stride_j);\n const int j0_stop = ((J/2)/cpy_ne) - ((J/2)/cpy_ne) % (1*stride_j);\n const int stride_i = warp_size / stride_j;\n\n if (j0_start == j0_stop) {\n return;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < I; i0 += nwarps*stride_i) {\n const int i = i0 + threadIdx.y*stride_i + (stride_j == warp_size ? 0 : threadIdx.x / stride_j);\n\n if (i0 + nwarps*stride_i <= I || i < I) {\n#pragma unroll\n for (int j0 = j0_start; j0 < j0_stop; j0 += stride_j) {\n const int j = j0*cpy_ne + (stride_j == warp_size ? threadIdx.x : threadIdx.x % stride_j)*cpy_ne;\n\n const __align__(16) half2 zero[cpy_ne] = {{0.0f, 0.0f}};\n ggml_cuda_memcpy_1(\n tile_KV + i*(J/2 + J_padding) + j,\n !oob_check || i < i_sup ? KV + i*stride_KV + j : zero);\n }\n }\n }\n };\n // 1: max 64*16=512 bytes, 512 half\n // 2: max 32*16=512 bytes, 256 half\n // 3: max 16*16=256 bytes, 128 half\n // 4: max 8*16=128 bytes, 64 half\n // 5: max 4*16= 64 bytes, 32 half\n // 6: max 2*16= 32 bytes, 16 half\n // 7: max 1*16= 16 bytes, 8 half\n static_assert(J % 8 == 0, \"bad J\");\n static_assert((J/2) % cpy_ne == 0, \"bad J\");\n ggml_cuda_unroll<7>{}(load);\n}\n\ntemplate\nstatic __device__ __forceinline__ void flash_attn_tile_load_tile(\n const half2 * const __restrict__ KV, float * const __restrict__ tile_KV, const int stride_KV, const int i_sup) {\n constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n auto load = [&] __device__ (const int n) {\n const int stride_j = warp_size >> n;\n\n if (stride_j == 0) {\n return;\n }\n\n const int j0_start = stride_j == warp_size ? 0 : (J/cpy_ne) - (J/cpy_ne) % (2*stride_j);\n const int j0_stop = (J/cpy_ne) - (J/cpy_ne) % (1*stride_j);\n const int stride_i = warp_size / stride_j;\n\n if (j0_start == j0_stop) {\n return;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < I; i0 += nwarps*stride_i) {\n const int i = i0 + threadIdx.y*stride_i + (stride_j == warp_size ? 0 : threadIdx.x / stride_j);\n\n if (i0 + nwarps*stride_i <= I || i < I) {\n#pragma unroll\n for (int j0 = j0_start; j0 < j0_stop; j0 += stride_j) {\n const int j = j0*(cpy_ne/2) + (stride_j == warp_size ? threadIdx.x : threadIdx.x % stride_j)*(cpy_ne/2);\n\n const half2 zero[cpy_ne/2] = {{0.0f, 0.0f}};\n __align__(16) half2 tmp_h2[cpy_ne/2];\n ggml_cuda_memcpy_1(\n tmp_h2, !oob_check || i < i_sup ? KV + i*stride_KV + j : zero);\n\n __align__(16) float2 tmp_f2[cpy_ne/2];\n#pragma unroll\n for (int l = 0; l < cpy_ne/2; ++l) {\n tmp_f2[l] = __half22float2(tmp_h2[l]);\n }\n ggml_cuda_memcpy_1(tile_KV + i*(J + J_padding) + 2*j, tmp_f2);\n }\n }\n }\n };\n // 1: max 32*16=512 bytes, 128 float\n // 2: max 16*16=256 bytes, 64 float\n // 3: max 8*16=128 bytes, 32 float\n // 4: max 4*16= 64 bytes, 16 float\n // 5: max 2*16= 32 bytes, 8 float\n static_assert(J % 8 == 0, \"bad J\");\n static_assert(J % cpy_ne == 0, \"bad J\");\n ggml_cuda_unroll<5>{}(load);\n}\n\n// Function that performs a single iteration in for the KQ matrix multiplication:\ntemplate \nstatic __device__ __forceinline__ void flash_attn_tile_iter_KQ(\n T_vec_dot * const Q_tmp,\n const half2 * const __restrict__ K_h2,\n T_vec_dot * const KV_tmp,\n const int stride_K2,\n const int k_VKQ_0,\n const int k_VKQ_sup,\n const int k_KQ_0,\n float * KQ_acc) {\n constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n constexpr int ncols = ncols1*ncols2;\n constexpr int cpw = ncols > nwarps ? ncols/nwarps : 1; // Q columns per warp\n constexpr int np = nwarps > ncols ? nwarps/ncols : 1; // number of parallel warps per Q column\n\n flash_attn_tile_load_tile\n (K_h2 + int64_t(k_VKQ_0)*stride_K2 + k_KQ_0/2, KV_tmp, stride_K2, k_VKQ_sup);\n __syncthreads();\n\n#ifdef FAST_FP16_AVAILABLE\n static_assert((nbatch_K/2) % cpy_ne == 0, \"bad nbatch_K\");\n#pragma unroll\n for (int k_KQ_1 = 0; k_KQ_1 < nbatch_K/2; k_KQ_1 += cpy_ne) {\n __align__(16) half2 K_k[nbatch_fa/(np*warp_size)][cpy_ne];\n __align__(16) half2 Q_k[cpw][cpy_ne];\n#else\n static_assert(nbatch_K % cpy_ne == 0, \"bad nbatch_K\");\n#pragma unroll\n for (int k_KQ_1 = 0; k_KQ_1 < nbatch_K; k_KQ_1 += cpy_ne) {\n __align__(16) float K_k[nbatch_fa/(np*warp_size)][cpy_ne];\n __align__(16) float Q_k[cpw][cpy_ne];\n#endif // FAST_FP16_AVAILABLE\n\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) {\n const int i_KQ = i_KQ_0 + (threadIdx.y % np)*warp_size + threadIdx.x;\n\n#ifdef FAST_FP16_AVAILABLE\n ggml_cuda_memcpy_1(&K_k[i_KQ_0/(np*warp_size)], &KV_tmp[i_KQ*(nbatch_K/2 + cpy_ne) + k_KQ_1]);\n#else\n ggml_cuda_memcpy_1(&K_k[i_KQ_0/(np*warp_size)], &KV_tmp[i_KQ*(nbatch_K + cpy_ne) + k_KQ_1]);\n#endif // FAST_FP16_AVAILABLE\n }\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int jc = jc0 + (threadIdx.y / np)*cpw;\n\n#ifdef FAST_FP16_AVAILABLE\n ggml_cuda_memcpy_1(&Q_k[jc0], &Q_tmp[jc*(DKQ/2) + k_KQ_0/2 + k_KQ_1]);\n#else\n ggml_cuda_memcpy_1(&Q_k[jc0], &Q_tmp[jc* DKQ + k_KQ_0 + k_KQ_1]);\n#endif // FAST_FP16_AVAILABLE\n }\n\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) {\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n#pragma unroll\n for (int k = 0; k < cpy_ne; ++k) {\n ggml_cuda_mad(KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0], K_k[i_KQ_0/(np*warp_size)][k], Q_k[jc0][k]);\n }\n }\n }\n }\n\n if (k_KQ_0 + nbatch_K < DKQ) {\n __syncthreads(); // Sync not needed on last iteration.\n }\n}\n\n// Function that performs a single iteration of the main loop over up to nbatch_fa tokens.\ntemplate \nstatic __device__ __forceinline__ void flash_attn_tile_iter(\n T_vec_dot * const Q_tmp,\n const half2 * const __restrict__ K_h2,\n const half2 * const __restrict__ V_h2,\n const half * const __restrict__ mask,\n const uint3 ne01,\n const float logit_softcap,\n const float slope,\n T_KQ * const KQ,\n T_vec_dot * const KV_tmp,\n const int stride_K2,\n const int stride_V2,\n const int stride_mask,\n float * const KQ_max,\n float * const KQ_sum,\n T_acc * const VKQ,\n const int k_VKQ_0,\n const int k_VKQ_max,\n const int col_Q_0) {\n constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n constexpr int ncols = ncols1*ncols2;\n constexpr int cpw = ncols > nwarps ? ncols/nwarps : 1; // Q columns per warp\n constexpr int np = nwarps > ncols ? nwarps/ncols : 1; // number of parallel warps per Q column\n\n constexpr int DVp = (DV + 2*warp_size - 1) & ~(2*warp_size - 1); // DV padded to multiple of 2*warp_size.\n\n // KQ_cs == KQ chunk size, number of KQ values in j direction to store as one contiguous chunk in memory.\n // KQ is originally 2D but uses a Z-shaped 3D memory pattern like KQ[ncols/KQ_cs][DVp][KQ_cs].\n#ifdef FAST_FP16_AVAILABLE\n constexpr int KQ_cs = cpw < 2*cpy_ne ? cpw : 2*cpy_ne;\n#else\n constexpr int KQ_cs = cpw < 1*cpy_ne ? cpw : 1*cpy_ne;\n#endif // FAST_FP16_AVAILABLE\n static_assert(cpw % KQ_cs == 0, \"bad KQ_cs\");\n const int k_VKQ_sup = k_VKQ_max - k_VKQ_0; // k supremum, only smaller k values have valid KV data\n\n float KQ_max_new[cpw];\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n KQ_max_new[jc0] = KQ_max[jc0];\n }\n\n float KQ_acc[nbatch_fa/(np*warp_size) * cpw] = {0.0f}; // Accumulators for KQ matrix multiplication.\n\n // KQ = K @ Q matrix multiplication:\n constexpr int nbatch_K_last = DKQ % nbatch_K;\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < DKQ - nbatch_K_last; k_KQ_0 += nbatch_K) {\n flash_attn_tile_iter_KQ(\n Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);\n }\n if (nbatch_K_last > 0) {\n constexpr int k_KQ_0 = DKQ - nbatch_K_last;\n flash_attn_tile_iter_KQ(\n Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);\n }\n\n // Apply logit softcap + mask, update KQ_max:\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int j = fastmodulo(col_Q_0 + (jc0 + (threadIdx.y / np)*cpw)/ncols2, ne01);\n\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) {\n const int i_KQ = i_KQ_0 + (threadIdx.y % np)*warp_size + threadIdx.x;\n\n#if defined(FAST_FP16_AVAILABLE) && !defined(V_DOT2_F32_F16_AVAILABLE)\n // Without the v_dot2_f32_f16 instruction there is a higher risk of numerical overflow in the KQ calculation.\n // Therefore, scale down Q values and apply the inverse scale the FP32 KQ values afterwards again.\n KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0] *= 4.0f;\n#endif // defined(FAST_FP16_AVAILABLE) && !defined(V_DOT2_F32_F16_AVAILABLE)\n\n if (use_logit_softcap) {\n KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] = logit_softcap * tanhf(KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0]);\n }\n\n if (!oob_check || i_KQ < k_VKQ_sup) {\n KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] += (ncols2 > 1 || mask) ?\n slope*__half2float(mask[j*stride_mask + k_VKQ_0 + i_KQ]) : 0.0f;\n\n KQ_max_new[jc0] = fmaxf(KQ_max_new[jc0], KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] + FATTN_KQ_MAX_OFFSET);\n }\n }\n\n KQ_max_new[jc0] = warp_reduce_max(KQ_max_new[jc0]);\n }\n\n if constexpr (np == 1) {\n __syncthreads();\n } else {\n static_assert(cpw == 1, \"bad cpw\");\n __shared__ float KQ_max_new_shared[nwarps];\n if (threadIdx.x == 0) {\n KQ_max_new_shared[threadIdx.y] = KQ_max_new[0];\n }\n __syncthreads();\n KQ_max_new[0] = KQ_max_new_shared[(threadIdx.y & ~(np-1)) + threadIdx.x % np];\n KQ_max_new[0] = warp_reduce_max(KQ_max_new[0]);\n }\n\n // Calculate KQ softmax, write to shared KQ buffer, re-scale VKQ accumulators:\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; jc0 += KQ_cs) {\n#ifdef FAST_FP16_AVAILABLE\n __align__(16) half tmp[nbatch_fa/(np*warp_size)][KQ_cs];\n#else\n __align__(16) float tmp[nbatch_fa/(np*warp_size)][KQ_cs];\n#endif // FAST_FP16_AVAILABLE\n\n#pragma unroll\n for (int jc1 = 0; jc1 < KQ_cs; ++jc1) {\n const int jc = jc0 + jc1;\n\n const float KQ_max_scale = expf(KQ_max[jc] - KQ_max_new[jc]);\n KQ_max[jc] = KQ_max_new[jc];\n\n float KQ_sum_add = 0.0f;\n#pragma unroll\n for (int i0 = 0; i0 < nbatch_fa; i0 += np*warp_size) {\n const float val = !oob_check || i0 + (threadIdx.y % np)*warp_size + threadIdx.x < static_cast(k_VKQ_sup) ?\n expf(KQ_acc[(i0/(np*warp_size))*cpw + jc] - KQ_max[jc]) : 0.0f;\n KQ_sum_add += val;\n tmp[i0/(np*warp_size)][jc1] = val;\n }\n KQ_sum[jc] = KQ_sum[jc]*KQ_max_scale + KQ_sum_add;\n\n#ifdef FAST_FP16_AVAILABLE\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n VKQ[jc*((DVp/2)/warp_size) + i0/warp_size] *= KQ_max_scale_h2;\n }\n#else\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n VKQ[jc*((DVp/2)/warp_size) + i0/warp_size].x *= KQ_max_scale;\n VKQ[jc*((DVp/2)/warp_size) + i0/warp_size].y *= KQ_max_scale;\n }\n#endif // FAST_FP16_AVAILABLE\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < nbatch_fa; i0 += np*warp_size) {\n const int i = i0 + (threadIdx.y % np)*warp_size + threadIdx.x;\n\n ggml_cuda_memcpy_1(\n KQ + (jc0/KQ_cs + (threadIdx.y / np)*(cpw/KQ_cs))*(nbatch_fa*KQ_cs) + i*KQ_cs,\n tmp[i0/(np*warp_size)]);\n }\n }\n\n // VKQ = V @ KQ matrix multiplication:\n static_assert(DV <= DKQ, \"bad DV\");\n static_assert(DV % nbatch_K == 0 || (nbatch_K % 3 == 0 && DV % (nbatch_K*2/3) == 0), \"bad nbatch_K\");\n constexpr int nbatch_V = (DV % nbatch_K == 0 ? nbatch_K : nbatch_K*2/3) * nbatch_fa / DV; // Number of V columns that fit in SRAM for K.\n static_assert(nbatch_fa % nbatch_V == 0, \"bad nbatch_V\");\n static_assert(nbatch_V % np == 0, \"bad nbatch_V\");\n#pragma unroll\n for (int k0 = 0; k0 < nbatch_fa; k0 += nbatch_V) {\n flash_attn_tile_load_tile\n (V_h2 + int64_t(k_VKQ_0 + k0)*stride_V2, KV_tmp, stride_V2, k_VKQ_sup - k0);\n __syncthreads();\n\n#ifdef FAST_FP16_AVAILABLE\n#pragma unroll\n for (int k1 = 0; k1 < nbatch_V; k1 += np) {\n __align__(16) half2 V_k[(DVp/2)/warp_size];\n __align__(16) half2 KQ_k[cpw];\n\n constexpr int cpy_ne_D = cpy_ne/2 < (DVp/2)/warp_size ? cpy_ne/2 : (DVp/2)/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size*cpy_ne_D) {\n ggml_cuda_memcpy_1(&V_k[i0/warp_size], &KV_tmp[(k1 + threadIdx.y % np)*(DV/2) + i0 + threadIdx.x*cpy_ne_D]);\n }\n#pragma unroll\n for (int jc_VKQ_0 = 0; jc_VKQ_0 < cpw; jc_VKQ_0 += KQ_cs) {\n const int jc_KQ = jc_VKQ_0/KQ_cs + (threadIdx.y / np)*(cpw/KQ_cs);\n\n __align__(16) half tmp[KQ_cs];\n ggml_cuda_memcpy_1(\n &tmp, KQ + jc_KQ*(nbatch_fa*KQ_cs) + (k0 + k1 + threadIdx.y % np)*KQ_cs);\n#pragma unroll\n for (int jc_VKQ_1 = 0; jc_VKQ_1 < KQ_cs; ++jc_VKQ_1) {\n KQ_k[jc_VKQ_0+jc_VKQ_1] = __half2half2(tmp[jc_VKQ_1]);\n }\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n#pragma unroll\n for (int jc_VKQ_0 = 0; jc_VKQ_0 < cpw; ++jc_VKQ_0) {\n VKQ[jc_VKQ_0*((DVp/2)/warp_size) + i0/warp_size] += V_k[i0/warp_size]*KQ_k[jc_VKQ_0];\n }\n }\n }\n#else\n#pragma unroll\n for (int k1 = 0; k1 < nbatch_V; k1 += np) {\n __align__(16) float2 V_k[(DVp/2)/warp_size];\n __align__(16) float KQ_k[cpw];\n\n constexpr int cpy_ne_D = cpy_ne < DVp/warp_size ? cpy_ne : DVp/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp; i0 += warp_size*cpy_ne_D) {\n ggml_cuda_memcpy_1(&V_k[i0/(2*warp_size)], &KV_tmp[(k1 + threadIdx.y % np)*DV + i0 + threadIdx.x*cpy_ne_D]);\n }\n#pragma unroll\n for (int jc_VKQ_0 = 0; jc_VKQ_0 < cpw; jc_VKQ_0 += KQ_cs) {\n const int jc_KQ = jc_VKQ_0/KQ_cs + (threadIdx.y / np)*(cpw/KQ_cs);\n\n ggml_cuda_memcpy_1(\n &KQ_k[jc_VKQ_0], KQ + jc_KQ*(nbatch_fa*KQ_cs) + (k0 + k1 + threadIdx.y % np)*KQ_cs);\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n#pragma unroll\n for (int jc_VKQ_0 = 0; jc_VKQ_0 < cpw; ++jc_VKQ_0) {\n VKQ[jc_VKQ_0*((DVp/2)/warp_size) + i0/warp_size].x += V_k[i0/warp_size].x*KQ_k[jc_VKQ_0];\n VKQ[jc_VKQ_0*((DVp/2)/warp_size) + i0/warp_size].y += V_k[i0/warp_size].y*KQ_k[jc_VKQ_0];\n }\n }\n }\n#endif // FAST_FP16_AVAILABLE\n\n __syncthreads();\n }\n}\n\ntemplate // D == head size\n__launch_bounds__(ggml_cuda_fattn_tile_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_tile_get_occupancy(DKQ, DV, ncols1*ncols2))\nstatic __global__ void flash_attn_tile(\n const char * __restrict__ Q,\n const char * __restrict__ K,\n const char * __restrict__ V,\n const char * __restrict__ mask,\n const char * __restrict__ sinks,\n const int * __restrict__ KV_max,\n float * __restrict__ dst,\n float2 * __restrict__ dst_meta,\n const float scale,\n const float max_bias,\n const float m0,\n const float m1,\n const uint32_t n_head_log2,\n const float logit_softcap,\n const int32_t ne00, const uint3 ne01, const int32_t ne02, const int32_t ne03,\n const int32_t nb01, const int32_t nb02, const int32_t nb03,\n const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13,\n const int32_t nb11, const int32_t nb12, const int64_t nb13,\n const int32_t nb21, const int32_t nb22, const int64_t nb23,\n const int32_t ne31, const int32_t ne32, const int32_t ne33,\n const int32_t nb31, const int32_t nb32, const int64_t nb33) {\n#ifdef FLASH_ATTN_AVAILABLE\n\n // Skip unused kernel variants for faster compilation:\n\n if (\n#ifdef GGML_USE_WMMA_FATTN\n (ncols2 != 1 && DV != 40 && DV != 72 && DV != 512) ||\n#endif // GGML_USE_WMMA_FATTN\n (use_logit_softcap && !(DV == 128 || DV == 256))\n ) {\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n NO_DEVICE_CODE;\n return;\n }\n\n static_assert(ggml_cuda_fattn_tile_get_config(DKQ, DV, ncols1*ncols2) != 0, \"kernel config not defined\");\n\n constexpr int ncols = ncols1*ncols2;\n constexpr int warp_size = 32;\n constexpr int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, ncols1*ncols2) / warp_size;\n constexpr int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, ncols1*ncols2);\n constexpr int nbatch_K = ggml_cuda_fattn_tile_get_nbatch_K (DKQ, DV, ncols1*ncols2);\n\n // In this kernel Q, K, V are matrices while i, j, k are matrix indices.\n\n const int col_Q_0 = blockIdx.x * ncols1; // Index of the first Q column for this CUDA block to work on.\n\n const int sequence = blockIdx.z / (ne02/ncols2);\n const int head0 = blockIdx.z*ncols2 - sequence*ne02; // == blockIdx.z % (ne02/ncols2)\n const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n const float * Q_f = (const float *) (Q + nb03*sequence + nb02* head0);\n const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*(head0 / gqa_ratio));\n const half2 * V_h2 = (const half2 *) (V + nb23*sequence + nb22*(head0 / gqa_ratio)); // K and V have same shape\n\n const half * maskh = mask ? (const half *) (mask + nb33*(sequence % ne33)) : nullptr;\n\n const int stride_K2 = nb11 / sizeof(half2);\n const int stride_V2 = nb21 / sizeof(half2);\n const int stride_mask = nb31 / sizeof(half);\n\n const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, head0, n_head_log2, m0, m1) : 1.0f;\n\n constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n constexpr int cpw = ncols > nwarps ? ncols/nwarps : 1; // Q columns per warp.\n constexpr int np = nwarps > ncols ? nwarps/ncols : 1; // Number of parallel warps per Q column.\n static_assert(cpw == 1 || np == 1, \"bad cpw / np\");\n static_assert(nbatch_fa % (np*warp_size) == 0, \"nbatch_fa % (np*warp_size) != 0\");\n\n constexpr int DKQp = (DKQ + 2*warp_size - 1) & ~(2*warp_size - 1); // DKQ padded to multiple of 2*warp_size.\n constexpr int DVp = (DV + 2*warp_size - 1) & ~(2*warp_size - 1); // DV padded to multiple of 2*warp_size.\n\n // Q_tmp == SRAM buffer to hold Q data for the entire lifetime of the kernel.\n // KV_tmp == SRAM buffer to hold fragments of K/V data while iterating over ne11.\n // KV_tmp is padded to avoid memory conflicts for K (cpy_ne) and OOB accesses for V (DVp-DV).\n // KQ == SRAM buffer to hold KQ fragments between KQ and VKQ matrix multiplications.\n // VKQ == Accumulators in registers for the final VKQ result.\n#ifdef FAST_FP16_AVAILABLE\n __shared__ half2 Q_tmp[ncols * DKQ/2];\n __shared__ half2 KV_tmp[nbatch_fa * (nbatch_K/2 + cpy_ne) + DVp-DV];\n __shared__ half KQ[ncols * nbatch_fa];\n __align__(16) half2 VKQ[cpw * ((DVp/2)/warp_size)] = {{0.0f, 0.0f}};\n#else\n __shared__ float Q_tmp[ncols * DKQ];\n __shared__ float KV_tmp[nbatch_fa * (nbatch_K + cpy_ne) + DVp-DV];\n __shared__ float KQ[ncols * nbatch_fa];\n __align__(16) float2 VKQ[cpw * ((DVp/2)/warp_size)] = {{0.0f, 0.0f}};\n#endif // FAST_FP16_AVAILABLE\n\n float KQ_max[cpw];\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n KQ_max[j0/nwarps] = -FLT_MAX/2.0f;\n }\n float KQ_sum[cpw] = {0.0f};\n\n // Load Q data, convert to FP16 if fast:\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int jc = jc0 + (threadIdx.y / np)*cpw;\n\n const int j = jc / ncols2;\n const int c = jc % ncols2;\n\n constexpr int cpy_ne_D = cpy_ne < DKQp/warp_size ? cpy_ne : DKQp/warp_size;\n\n#pragma unroll\n for (int i0 = 0; i0 < DKQp; i0 += np*warp_size*cpy_ne_D) {\n if (i0 + np*warp_size*cpy_ne_D <= DKQ || i0 + (threadIdx.y % np)*(warp_size*cpy_ne_D) + threadIdx.x*cpy_ne_D < DKQ) {\n __align__(16) float tmp_f[cpy_ne_D] = {0.0f};\n ggml_cuda_memcpy_1\n (tmp_f, &Q_f[c*(nb02/sizeof(float)) + fastmodulo(col_Q_0 + j, ne01)*(nb01/sizeof(float))\n + i0 + (threadIdx.y % np)*(warp_size*cpy_ne_D) + threadIdx.x*cpy_ne_D]);\n\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; ++i1) {\n tmp_f[i1] *= scale;\n }\n\n#ifdef FAST_FP16_AVAILABLE\n __align__(16) half2 tmp_h2[cpy_ne_D/2];\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; i1 += 2) {\n tmp_h2[i1/2] = make_half2(tmp_f[i1 + 0], tmp_f[i1 + 1]);\n#if defined(FAST_FP16_AVAILABLE) && !defined(V_DOT2_F32_F16_AVAILABLE)\n // Without the v_dot2_f32_f16 instruction there is a higher risk of numerical overflow in the KQ calculation.\n // Therefore, scale down Q values and apply the inverse scale the FP32 KQ values afterwards again.\n tmp_h2[i1/2] *= make_half2(0.25f, 0.25f);\n#endif // defined(FAST_FP16_AVAILABLE) && !defined(V_DOT2_F32_F16_AVAILABLE)\n }\n ggml_cuda_memcpy_1(\n &Q_tmp[jc*(DKQ/2) + i0/2 + (threadIdx.y % np)*(warp_size*cpy_ne_D/2) + threadIdx.x*(cpy_ne_D/2)],\n tmp_h2);\n#else\n ggml_cuda_memcpy_1(\n &Q_tmp[jc* DKQ + i0 + (threadIdx.y % np)*(warp_size*cpy_ne_D) + threadIdx.x* cpy_ne_D],\n tmp_f);\n#endif // FAST_FP16_AVAILABLE\n }\n }\n }\n\n __syncthreads();\n\n // Main loop over KV cache:\n const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11;\n if (ncols2 == 1) {\n // Branch with out-of-bounds checks.\n int k_VKQ_0 = blockIdx.y*nbatch_fa;\n while (k_VKQ_0 < k_VKQ_max - nbatch_fa) {\n constexpr bool oob_check = false;\n flash_attn_tile_iter\n (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,\n stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);\n k_VKQ_0 += gridDim.y*nbatch_fa;\n }\n if (k_VKQ_0 < k_VKQ_max) {\n constexpr bool oob_check = true;\n flash_attn_tile_iter\n (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,\n stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);\n }\n } else {\n // Branch without out-of-bounds checks.\n for (int k_VKQ_0 = blockIdx.y*nbatch_fa; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nbatch_fa) {\n constexpr bool oob_check = false;\n flash_attn_tile_iter\n (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,\n stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);\n }\n }\n\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n KQ_sum[jc0] = warp_reduce_sum(KQ_sum[jc0]);\n }\n\n if constexpr (np > 1) {\n static_assert(cpw == 1, \"bad cpw\");\n static_assert(nbatch_fa*nbatch_K >= nwarps*DVp, \"KV_tmp too small\");\n\n#ifdef FAST_FP16_AVAILABLE\n half2 * VKQ_combine = (half2 *) KV_tmp;\n#else\n float * VKQ_combine = (float *) KV_tmp;\n#endif // FAST_FP16_AVAILABLE\n float * KQ_sum_combine = (float *) Q_tmp;\n\n if (threadIdx.y % np != 0) {\n#ifdef FAST_FP16_AVAILABLE\n constexpr int cpy_ne_D = cpy_ne < (DVp/2)/warp_size ? cpy_ne : (DVp/2)/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size*cpy_ne_D) {\n ggml_cuda_memcpy_1(&VKQ_combine[threadIdx.y*(DVp/2) + i0 + threadIdx.x*cpy_ne_D], &VKQ[i0/warp_size]);\n }\n#else\n constexpr int cpy_ne_D = cpy_ne < DVp/warp_size ? cpy_ne : DVp/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp; i0 += warp_size*cpy_ne_D) {\n ggml_cuda_memcpy_1(\n &VKQ_combine[threadIdx.y*DVp + i0 + threadIdx.x*cpy_ne_D], ((const float *) VKQ) + i0/warp_size);\n }\n#endif // FAST_FP16_AVAILABLE\n\n if (threadIdx.x == 0) {\n KQ_sum_combine[threadIdx.y] = KQ_sum[0];\n }\n\n return;\n }\n\n __syncthreads();\n\n#pragma unroll\n for (int ip = 1; ip < np; ++ip) {\n#ifdef FAST_FP16_AVAILABLE\n constexpr int cpy_ne_D = cpy_ne < (DVp/2)/warp_size ? cpy_ne : (DVp/2)/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size*cpy_ne_D) {\n __align__(16) half2 tmp[cpy_ne_D];\n ggml_cuda_memcpy_1(tmp, &VKQ_combine[(threadIdx.y + ip)*(DVp/2) + i0 + threadIdx.x*cpy_ne_D]);\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; ++i1) {\n VKQ[i0/warp_size + i1] += tmp[i1];\n }\n }\n#else\n constexpr int cpy_ne_D = cpy_ne < DVp/warp_size ? cpy_ne : DVp/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp; i0 += warp_size*cpy_ne_D) {\n __align__(16) float tmp[cpy_ne_D];\n ggml_cuda_memcpy_1(tmp, &VKQ_combine[(threadIdx.y + ip)*DVp + i0 + threadIdx.x*cpy_ne_D]);\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; ++i1) {\n ((float *)VKQ)[i0/warp_size + i1] += tmp[i1];\n }\n }\n#endif // FAST_FP16_AVAILABLE\n\n KQ_sum[0] += KQ_sum_combine[threadIdx.y + ip];\n }\n }\n\n // Attention sink: adjust KQ max and sum only for the first of all parallel blocks:\n if (sinks && blockIdx.y == 0) {\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int jc = jc0 + (threadIdx.y/np)*cpw;\n const float sink = ((const float *) sinks)[head0 + jc % ncols2];\n\n float KQ_max_new_j = fmaxf(KQ_max[jc0], sink);\n const float KQ_max_scale = expf(KQ_max[jc0] - KQ_max_new_j);\n KQ_max[jc0] = KQ_max_new_j;\n\n const float val = expf(sink - KQ_max[jc0]);\n KQ_sum[jc0] = KQ_sum[jc0]*KQ_max_scale + val;\n\n#ifdef FAST_FP16_AVAILABLE\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n VKQ[jc0*((DVp/2)/warp_size) + i0/warp_size] *= KQ_max_scale_h2;\n }\n#else\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n VKQ[jc0*((DVp/2)/warp_size) + i0/warp_size].x *= KQ_max_scale;\n VKQ[jc0*((DVp/2)/warp_size) + i0/warp_size].y *= KQ_max_scale;\n }\n#endif // FAST_FP16_AVAILABLE\n }\n }\n\n // Write back results:\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int jc = jc0 + (threadIdx.y/np)*cpw;\n\n const int j = jc / ncols2;\n const int c = jc % ncols2;\n\n if (ncols1 > 1 && col_Q_0 + j >= int(ne01.z)) {\n return;\n }\n\n const float scale = gridDim.y == 1 ? 1.0f/KQ_sum[jc0] : 1.0f;\n\n const int j_dst_unrolled = ((sequence*int(ne01.z) + col_Q_0 + j)*ne02 + head0 + c)*gridDim.y + blockIdx.y;\n\n#ifdef FAST_FP16_AVAILABLE\n constexpr int cpy_ne_D = cpy_ne/2 < (DVp/2)/warp_size ? cpy_ne/2 : (DVp/2)/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size*cpy_ne_D) {\n __align__(16) float2 tmp[cpy_ne_D];\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; ++i1) {\n tmp[i1] = __half22float2(VKQ[jc0*((DVp/2)/warp_size) + i0/warp_size + i1]);\n tmp[i1].x *= scale;\n tmp[i1].y *= scale;\n }\n if (i0 + warp_size*cpy_ne_D <= DV/2 || i0 + threadIdx.x*cpy_ne_D < DV/2) {\n ggml_cuda_memcpy_1(&dst[j_dst_unrolled*DV + 2*i0 + threadIdx.x*(2*cpy_ne_D)], tmp);\n }\n }\n#else\n constexpr int cpy_ne_D = cpy_ne < DVp/warp_size ? cpy_ne : DVp/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp; i0 += warp_size*cpy_ne_D) {\n if (i0 + warp_size*cpy_ne_D <= DV || i0 + threadIdx.x*cpy_ne_D < DV) {\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D/2; ++i1) {\n VKQ[jc0*((DVp/2)/warp_size) + i0/(2*warp_size) + i1].x *= scale;\n VKQ[jc0*((DVp/2)/warp_size) + i0/(2*warp_size) + i1].y *= scale;\n }\n ggml_cuda_memcpy_1(\n &dst[j_dst_unrolled*DV + i0 + threadIdx.x*cpy_ne_D],\n &VKQ[jc0*((DVp/2)/warp_size) + i0/(2*warp_size)]);\n }\n }\n#endif // FAST_FP16_AVAILABLE\n\n if (gridDim.y != 1 && threadIdx.x == 0) {\n dst_meta[j_dst_unrolled] = make_float2(KQ_max[jc0], KQ_sum[jc0]);\n }\n }\n#else\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n NO_DEVICE_CODE;\n#endif // FLASH_ATTN_AVAILABLE\n}\n\ntemplate \nstatic void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * Q = dst->src[0];\n\n const int id = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[id].cc;\n const int warp_size = 32;\n\n constexpr size_t nbytes_shared = 0;\n\n#ifdef GGML_USE_HIP\n if constexpr (DV <= 128) {\n if (Q->ne[1] > 32/ncols2) {\n constexpr int cols_per_block = 64;\n const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n fattn_kernel_t fattn_kernel = flash_attn_tile;\n launch_fattn\n (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);\n return;\n }\n }\n#endif // GGML_USE_HIP\n\n#ifndef GGML_USE_HIP\n if constexpr (DV <= 256)\n#endif // GGML_USE_HIP\n {\n if (Q->ne[1] > 16/ncols2) {\n constexpr int cols_per_block = 32;\n const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n fattn_kernel_t fattn_kernel = flash_attn_tile;\n launch_fattn\n (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);\n return;\n }\n }\n\n if (Q->ne[1] > 8/ncols2) {\n constexpr int cols_per_block = 16;\n const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n fattn_kernel_t fattn_kernel = flash_attn_tile;\n launch_fattn\n (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);\n return;\n }\n\n if constexpr (ncols2 <= 8) {\n if (Q->ne[1] > 4/ncols2) {\n constexpr int cols_per_block = 8;\n const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n fattn_kernel_t fattn_kernel = flash_attn_tile;\n launch_fattn\n (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);\n return;\n }\n }\n\n if constexpr (ncols2 <= 4) {\n if (Q->ne[1] > 2/ncols2) {\n constexpr int cols_per_block = 4;\n const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n fattn_kernel_t fattn_kernel = flash_attn_tile;\n launch_fattn\n (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);\n return;\n }\n }\n\n if constexpr (ncols2 <= 2) {\n constexpr int cols_per_block = 2;\n const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n fattn_kernel_t fattn_kernel = flash_attn_tile;\n launch_fattn\n (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);\n return;\n }\n\n GGML_ABORT(\"fatal error\");\n}\n\ntemplate \nstatic void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * KQV = dst;\n const ggml_tensor * Q = dst->src[0];\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * mask = dst->src[3];\n\n float max_bias = 0.0f;\n memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));\n\n GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);\n const int gqa_ratio = Q->ne[2] / K->ne[2];\n\n // On NVIDIA (Pascal and older) the GQA optimizations seem to be detrimental in some cases.\n // However, for DKQ == 576, DV == 512 only the kernel variant with GQA optimizations is implemented.\n const bool nvidia = GGML_CUDA_CC_IS_NVIDIA(ggml_cuda_info().devices[ggml_cuda_get_device()].cc);\n const int gqa_limit = nvidia && gqa_ratio <= 4 && DV <= 256 ? 16 : INT_MAX;\n const bool use_gqa_opt = mask && max_bias == 0.0f && Q->ne[1] <= gqa_limit && K->ne[1] % FATTN_KQ_STRIDE == 0;\n\n if constexpr (DV == 512) {\n if (use_gqa_opt && gqa_ratio % 16 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n if (use_gqa_opt && gqa_ratio % 4 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n }\n\n if constexpr (DV <= 256) {\n if (use_gqa_opt && gqa_ratio % 8 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n\n if (use_gqa_opt && gqa_ratio % 4 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n\n if (use_gqa_opt && gqa_ratio % 2 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n GGML_ABORT(\"fatal error\");\n}\n\ntemplate \nvoid ggml_cuda_flash_attn_ext_tile_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * KQV = dst;\n\n float logit_softcap;\n memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n if (logit_softcap == 0.0f) {\n constexpr bool use_logit_softcap = false;\n launch_fattn_tile_switch_ncols2(ctx, dst);\n } else {\n constexpr bool use_logit_softcap = true;\n launch_fattn_tile_switch_ncols2(ctx, dst);\n }\n}\n\nvoid ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\n#define DECL_FATTN_TILE_CASE(DKQ, DV) \\\n template void ggml_cuda_flash_attn_ext_tile_case \\\n (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \\\n\nextern DECL_FATTN_TILE_CASE( 40, 40);\nextern DECL_FATTN_TILE_CASE( 64, 64);\nextern DECL_FATTN_TILE_CASE( 72, 72);\nextern DECL_FATTN_TILE_CASE( 80, 80);\nextern DECL_FATTN_TILE_CASE( 96, 96);\nextern DECL_FATTN_TILE_CASE(112, 112);\nextern DECL_FATTN_TILE_CASE(128, 128);\nextern DECL_FATTN_TILE_CASE(256, 256);\nextern DECL_FATTN_TILE_CASE(576, 512);\n" }, { "path": "src/ggml-cuda/fattn-vec.cuh", "content": "#include \"common.cuh\"\n#include \"fattn-common.cuh\"\n\nstatic int ggml_cuda_fattn_vec_get_nthreads_host(const int cc) {\n return 128;\n GGML_UNUSED(cc);\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_vec_get_nthreads_device() {\n return 128;\n}\n\n// Currently llvm with the amdgcn target does not support unrolling loops\n// that contain a break that can not be resolved at compile time.\n#ifdef __clang__\n#pragma clang diagnostic push\n#pragma clang diagnostic ignored \"-Wpass-failed\"\n#endif // __clang__\ntemplate // D == head size\n__launch_bounds__(ggml_cuda_fattn_vec_get_nthreads_device(), 1)\nstatic __global__ void flash_attn_ext_vec(\n const char * __restrict__ Q,\n const char * __restrict__ K,\n const char * __restrict__ V,\n const char * __restrict__ mask,\n const char * __restrict__ sinks,\n const int * __restrict__ KV_max,\n float * __restrict__ dst,\n float2 * __restrict__ dst_meta,\n const float scale,\n const float max_bias,\n const float m0,\n const float m1,\n const uint32_t n_head_log2,\n const float logit_softcap,\n const int32_t ne00, const uint3 ne01, const int32_t ne02, const int32_t ne03,\n const int32_t nb01, const int32_t nb02, const int32_t nb03,\n const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13,\n const int32_t nb11, const int32_t nb12, const int64_t nb13,\n const int32_t nb21, const int32_t nb22, const int64_t nb23,\n const int32_t ne31, const int32_t ne32, const int32_t ne33,\n const int32_t nb31, const int32_t nb32, const int64_t nb33) {\n#ifdef FLASH_ATTN_AVAILABLE\n\n // Skip unused kernel variants for faster compilation:\n if (use_logit_softcap && !(D == 128 || D == 256)) {\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n NO_DEVICE_CODE;\n return;\n }\n\n //In this kernel Q, K, V are matrices while i, j, k are matrix indices.\n\n constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n#ifdef GGML_USE_HIP\n#ifdef RDNA\n constexpr int nthreads_KQ_q = 2;\n#else\n constexpr int nthreads_KQ_q = 4;\n#endif // RDNA\n constexpr int nthreads_V_q = (D/4 < 32 ? D/4 : 32);\n#else\n constexpr int nthreads_KQ_q = (D/4 < 32 ? D/4 : 32);\n constexpr int nthreads_V_q = (D/4 < 32 ? D/4 : 32);\n#endif // GGML_USE_HIP\n\n constexpr int nthreads = ggml_cuda_fattn_vec_get_nthreads_device();\n constexpr int nthreads_KQ = type_K == GGML_TYPE_F16 ? 128 / cpy_nb : nthreads_KQ_q;\n constexpr int nthreads_V = type_V == GGML_TYPE_F16 ? 128 / cpy_nb : nthreads_V_q;\n\n static_assert(WARP_SIZE % nthreads_KQ == 0, \"bad nthreads_K\");\n static_assert(WARP_SIZE % nthreads_V == 0, \"bad nthreads_V\");\n\n constexpr int V_rows_per_thread = type_V == GGML_TYPE_F16 ? 2*cpy_ne : 4;\n constexpr int V_cols_per_iter = WARP_SIZE / nthreads_V;\n\n constexpr vec_dot_KQ_t vec_dot_KQ = get_vec_dot_KQ();\n constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16;\n#ifdef V_DOT2_F32_F16_AVAILABLE\n constexpr dequantize_V_t dequantize_V = get_dequantize_V();\n#else\n constexpr dequantize_V_t dequantize_V = get_dequantize_V();\n#endif // V_DOT2_F32_F16_AVAILABLE\n\n const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on.\n\n const int sequence = blockIdx.z / ne02;\n const int head = blockIdx.z - sequence*ne02;\n const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n Q += nb03*sequence + nb02* head + nb01*ic0;\n K += nb13*sequence + nb12*(head / gqa_ratio);\n V += nb23*sequence + nb22*(head / gqa_ratio);\n\n const half * maskh = (const half *) (mask + nb33*(sequence % ne33) + nb31*ic0);\n\n const float slope = get_alibi_slope(max_bias, head, n_head_log2, m0, m1);\n\n static_assert(D % (2*WARP_SIZE) == 0, \"D not divisible by 2*WARP_SIZE == 64.\");\n constexpr int nwarps = nthreads / WARP_SIZE;\n const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;\n __builtin_assume(tid < nthreads);\n\n constexpr int ne_KQ = ncols*D;\n constexpr int ne_combine = nwarps*V_cols_per_iter*D;\n#ifdef V_DOT2_F32_F16_AVAILABLE\n half2 VKQ[ncols][(D/2)/nthreads_V] = {{{0.0f, 0.0f}}};\n __shared__ half KQ[ne_KQ > ne_combine ? ne_KQ : ne_combine];\n#else\n float2 VKQ[ncols][(D/2)/nthreads_V] = {{{0.0f, 0.0f}}};\n __shared__ float KQ[ne_KQ > ne_combine ? ne_KQ : ne_combine];\n#endif // V_DOT2_F32_F16_AVAILABLE\n\n float KQ_max[ncols];\n float KQ_sum[ncols];\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n KQ_max[j] = -FLT_MAX/2.0f;\n KQ_sum[j] = 0.0f;\n }\n\n // Convert Q to float2 (f16 K) or q8_1 (quantized K) and store in registers:\n#ifdef V_DOT2_F32_F16_AVAILABLE\n half2 Q_reg[ncols][(D/2)/nthreads_KQ]; // Will be initialized completely.\n#else\n __align__(16) float2 Q_reg[ncols][(D/2)/nthreads_KQ] = {{{0.0f, 0.0f}}}; // May be only partially initialized.\n#endif // V_DOT2_F32_F16_AVAILABLE\n int Q_i32[ncols][1 > D/(sizeof(int)*nthreads_KQ) ? 1 : D/(sizeof(int)*nthreads_KQ)];\n float2 Q_ds[ncols][1 > D/(sizeof(int)*nthreads_KQ) ? 1 : D/(sizeof(int)*nthreads_KQ)];\n if constexpr (Q_q8_1) {\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (j0 + nwarps > ncols && j >= ncols) {\n break;\n }\n\n // Reuse KQ as temporary storage for converting Q to q8_1:\n int * tmp_q_i32 = (int *) &KQ[j*D];\n float2 * tmp_q_ds = (float2 *) (tmp_q_i32 + D/sizeof(int));\n\n // Set memory to zero if out of bounds:\n if (ncols > 1 && ic0 + j >= int(ne01.z)) {\n#pragma unroll\n for (int i0 = 0; i0 < int(D/sizeof(int)); i0 += WARP_SIZE) {\n const int i = i0 + threadIdx.x;\n\n if (i0 + WARP_SIZE <= int(D/sizeof(int)) || i < int(D/sizeof(int))) {\n tmp_q_i32[i] = 0;\n }\n }\n if (threadIdx.x < D/QK8_1) {\n tmp_q_ds[threadIdx.x] = make_float2(0.0f, 0.0f);\n }\n } else {\n const float * Q_f = (const float *) (Q + j*nb01);\n constexpr int nthreads_quantize = D/sizeof(int) < WARP_SIZE ? D/sizeof(int) : WARP_SIZE;\n#pragma unroll\n for (int i0 = 0; i0 < int(D/sizeof(int)); i0 += nthreads_quantize) {\n quantize_q8_1_to_shared\n (Q_f + i0*sizeof(int), scale, tmp_q_i32 + i0, tmp_q_ds + i0/QI8_1);\n }\n }\n }\n\n __syncthreads();\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n int * tmp_q_i32 = (int *) &KQ[j*D];\n float2 * tmp_q_ds = (float2 *) (tmp_q_i32 + D/sizeof(int));\n\n#pragma unroll\n for (int i0 = 0; i0 < int(D/sizeof(int)); i0 += nthreads_KQ) {\n const int i = i0 + (nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ);\n\n Q_i32[j][i0/nthreads_KQ] = tmp_q_i32[i];\n Q_ds[j][i0/nthreads_KQ] = tmp_q_ds[i/QI8_1];\n }\n }\n\n __syncthreads();\n } else {\n#ifdef V_DOT2_F32_F16_AVAILABLE\n const half2 scale_h2 = make_half2(scale, scale);\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n const float2 * Q_j = (const float2 *) (Q + j*nb01);\n#pragma unroll\n for (int i0 = 0; i0 < D/2; i0 += nthreads_KQ*cpy_ne) {\n const int i = i0 + (nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ)*cpy_ne;\n\n __align__(16) float2 tmp[cpy_ne] = {{0.0f, 0.0f}};\n if (ncols == 1 || ic0 + j < int(ne01.z)) {\n ggml_cuda_memcpy_1(tmp, &Q_j[i]);\n ggml_cuda_memcpy_1(tmp + cpy_ne/2, &Q_j[i + cpy_ne/2]);\n }\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne; ++i1) {\n Q_reg[j][i0/nthreads_KQ + i1] = make_half2(tmp[i1].x, tmp[i1].y);\n }\n }\n#pragma unroll\n for (int k = 0; k < (D/2)/nthreads_KQ; ++k) {\n Q_reg[j][k] *= scale_h2;\n }\n }\n#else\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n const float2 * Q_j = (const float2 *) (Q + j*nb01);\n#pragma unroll\n for (int i0 = 0; i0 < D/2; i0 += nthreads_KQ*cpy_ne) {\n const int i = i0 + (nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ)*cpy_ne;\n if (ncols == 1 || ic0 + j < int(ne01.z)) {\n ggml_cuda_memcpy_1(&Q_reg[j][i0/nthreads_KQ], &Q_j[i]);\n ggml_cuda_memcpy_1(&Q_reg[j][i0/nthreads_KQ + cpy_ne/2], &Q_j[i + cpy_ne/2]);\n }\n }\n#pragma unroll\n for (int k = 0; k < (D/2)/nthreads_KQ; ++k) {\n Q_reg[j][k].x *= scale;\n Q_reg[j][k].y *= scale;\n }\n }\n#endif // V_DOT2_F32_F16_AVAILABLE\n }\n\n const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11;\n K += blockIdx.y*nthreads * nb11;\n V += blockIdx.y*nthreads * nb21;\n maskh += blockIdx.y*nthreads;\n for (int k_VKQ_0 = blockIdx.y*nthreads; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nthreads,\n // Increment pointers after each loop:\n K += gridDim.y*nthreads*nb11, V += gridDim.y*nthreads*nb21, maskh += gridDim.y*nthreads) {\n\n // Calculate KQ tile and keep track of new maximum KQ values:\n float KQ_reg[ncols]; // KQ in registers.\n\n float KQ_max_new[ncols];\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n KQ_max_new[j] = KQ_max[j];\n }\n\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < nthreads_KQ; ++i_KQ_0) {\n const int i_KQ = threadIdx.y*WARP_SIZE + (nthreads_KQ == WARP_SIZE ? 0 : (threadIdx.x & ~(nthreads_KQ-1))) + i_KQ_0;\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n float sum = vec_dot_KQ(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j]);\n sum = warp_reduce_sum(sum);\n\n if (use_logit_softcap) {\n sum = logit_softcap*tanhf(sum);\n }\n\n if (mask && (ncols == 1 || ic0 + j < int(ne01.z))) {\n sum += slope*__half2float(maskh[j*ne11 + i_KQ]);\n }\n\n KQ_max_new[j] = fmaxf(KQ_max_new[j], sum + FATTN_KQ_MAX_OFFSET);\n\n if ((nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ) == uint32_t(i_KQ_0)) {\n KQ_reg[j] = sum;\n }\n }\n }\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n#pragma unroll\n for (int offset = nthreads_KQ; offset < WARP_SIZE; offset <<= 1) {\n KQ_max_new[j] = fmaxf(KQ_max_new[j], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[j], offset, WARP_SIZE));\n }\n const float KQ_max_scale = expf(KQ_max[j] - KQ_max_new[j]);\n KQ_max[j] = KQ_max_new[j];\n\n KQ_reg[j] = expf(KQ_reg[j] - KQ_max[j]);\n KQ_sum[j] = KQ_sum[j]*KQ_max_scale + KQ_reg[j];\n KQ[j*nthreads + tid] = KQ_reg[j];\n\n#ifdef V_DOT2_F32_F16_AVAILABLE\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j][i_VKQ_0/nthreads_V] *= KQ_max_scale_h2;\n }\n#else\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j][i_VKQ_0/nthreads_V].x *= KQ_max_scale;\n VKQ[j][i_VKQ_0/nthreads_V].y *= KQ_max_scale;\n }\n#endif // V_DOT2_F32_F16_AVAILABLE\n }\n\n#ifndef GGML_USE_HIP\n __syncwarp();\n#endif // GGML_USE_HIP\n\n#pragma unroll\n for (int k0 = 0; k0 < WARP_SIZE; k0 += V_cols_per_iter) {\n const int k = threadIdx.y*WARP_SIZE + k0 + (nthreads_V == WARP_SIZE ? 0 : threadIdx.x / nthreads_V);\n\n#ifdef V_DOT2_F32_F16_AVAILABLE\n half2 KQ_k[ncols];\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n KQ_k[j] = __half2half2(KQ[j*nthreads + k]);\n }\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n half2 tmp[V_rows_per_thread/2];\n dequantize_V(V + k*nb21, tmp,\n 2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread);\n#pragma unroll\n for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) {\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n VKQ[j][i_VKQ_0/nthreads_V + i_VKQ_1] += tmp[i_VKQ_1]*KQ_k[j];\n }\n }\n }\n#else\n float KQ_k[ncols];\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n KQ_k[j] = KQ[j*nthreads + k];\n }\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n float2 tmp[V_rows_per_thread/2];\n dequantize_V(V + k*nb21, tmp,\n 2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread);\n#pragma unroll\n for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) {\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n VKQ[j][i_VKQ_0/nthreads_V + i_VKQ_1].x += tmp[i_VKQ_1].x*KQ_k[j];\n VKQ[j][i_VKQ_0/nthreads_V + i_VKQ_1].y += tmp[i_VKQ_1].y*KQ_k[j];\n }\n }\n }\n#endif // V_DOT2_F32_F16_AVAILABLE\n }\n }\n\n if (sinks && blockIdx.y == 0) {\n const float sink = ((const float *) sinks)[head];\n\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (j0 + nwarps > ncols && j >= ncols) {\n break;\n }\n\n const float kqmax_new_j = fmaxf(sink, KQ_max[j]);\n const float KQ_max_scale = expf(KQ_max[j] - kqmax_new_j);\n KQ_max[j] = kqmax_new_j;\n\n KQ_sum[j] = KQ_sum[j]*KQ_max_scale + (threadIdx.x == 0 ? expf(sink - KQ_max[j]) : 0.0f);\n\n#ifdef V_DOT2_F32_F16_AVAILABLE\n const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j][i_VKQ_0/nthreads_V] *= KQ_max_scale_h2;\n }\n#else\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j][i_VKQ_0/nthreads_V].x *= KQ_max_scale;\n VKQ[j][i_VKQ_0/nthreads_V].y *= KQ_max_scale;\n }\n#endif // V_DOT2_F32_F16_AVAILABLE\n }\n }\n\n __shared__ float KQ_max_shared[ncols][WARP_SIZE];\n __shared__ float KQ_sum_shared[ncols][WARP_SIZE];\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n if (threadIdx.y == 0) {\n KQ_max_shared[j][threadIdx.x] = -FLT_MAX/2.0f;\n KQ_sum_shared[j][threadIdx.x] = 0.0f;\n }\n }\n\n __syncthreads();\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n if (threadIdx.x == 0) {\n KQ_max_shared[j][threadIdx.y] = KQ_max[j];\n }\n }\n __syncthreads();\n\n#pragma unroll\n for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) {\n if (ncols > 1 && ic0 + j_VKQ >= int(ne01.z)) {\n break;\n }\n\n float kqmax_new = KQ_max_shared[j_VKQ][threadIdx.x];\n kqmax_new = warp_reduce_max(kqmax_new);\n const float kqmax_scale = expf(KQ_max[j_VKQ] - kqmax_new);\n KQ_max[j_VKQ] = kqmax_new;\n\n#ifdef V_DOT2_F32_F16_AVAILABLE\n half2 * VKQ_tmp = (half2 *) KQ + threadIdx.y*(V_cols_per_iter*D/2)\n + (nthreads_V == WARP_SIZE ? 0 : threadIdx.x / nthreads_V)*(D/2);\n\n const half2 kqmax_scale_h2 = make_half2(kqmax_scale, kqmax_scale);\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j_VKQ][i_VKQ_0/nthreads_V] *= kqmax_scale_h2;\n }\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n const int i_VKQ = i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*(V_rows_per_thread/2);\n\n ggml_cuda_memcpy_1(VKQ_tmp + i_VKQ, &VKQ[j_VKQ][i_VKQ_0/nthreads_V]);\n }\n#else\n float2 * VKQ_tmp = (float2 *) KQ + threadIdx.y*(V_cols_per_iter*D/2)\n + (nthreads_V == WARP_SIZE ? 0 : threadIdx.x / nthreads_V)*(D/2);\n\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j_VKQ][i_VKQ_0/nthreads_V].x *= kqmax_scale;\n VKQ[j_VKQ][i_VKQ_0/nthreads_V].y *= kqmax_scale;\n }\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n const int i_VKQ = i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*(V_rows_per_thread/2);\n\n ggml_cuda_memcpy_1(VKQ_tmp + i_VKQ, &VKQ[j_VKQ][i_VKQ_0/nthreads_V]);\n ggml_cuda_memcpy_1(VKQ_tmp + i_VKQ + V_rows_per_thread/4, &VKQ[j_VKQ][i_VKQ_0/nthreads_V + V_rows_per_thread/4]);\n }\n#endif // V_DOT2_F32_F16_AVAILABLE\n\n KQ_sum[j_VKQ] *= kqmax_scale;\n KQ_sum[j_VKQ] = warp_reduce_sum(KQ_sum[j_VKQ]);\n if (threadIdx.x == 0) {\n KQ_sum_shared[j_VKQ][threadIdx.y] = KQ_sum[j_VKQ];\n }\n\n __syncthreads();\n\n if (nthreads <= D || tid < D) {\n KQ_sum[j_VKQ] = KQ_sum_shared[j_VKQ][threadIdx.x];\n KQ_sum[j_VKQ] = warp_reduce_sum(KQ_sum[j_VKQ]);\n\n#pragma unroll\n for (int i0 = 0; i0 < D; i0 += nthreads) {\n float dst_val = 0;\n#pragma unroll\n for (int w = 0; w < nwarps; ++w) {\n#pragma unroll\n for (int v = 0; v < V_cols_per_iter; ++v) {\n dst_val += float(KQ[w*V_cols_per_iter*D + v*D + i0 + tid]);\n }\n }\n if (gridDim.y == 1) {\n dst_val /= KQ_sum[j_VKQ];\n }\n dst[(((sequence*int(ne01.z) + ic0 + j_VKQ)*ne02 + head)*gridDim.y + blockIdx.y)*D + i0 + tid] = dst_val;\n }\n }\n\n if (j_VKQ < ncols-1) {\n __syncthreads();\n }\n\n }\n\n if (gridDim.y != 1 && tid < ncols && (ncols == 1 || ic0 + tid < int(ne01.z))) {\n dst_meta[((sequence*int(ne01.z) + ic0 + tid)*ne02 + head)*gridDim.y + blockIdx.y] = make_float2(KQ_max[tid], KQ_sum[tid]);\n }\n#else\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n NO_DEVICE_CODE;\n#endif // FLASH_ATTN_AVAILABLE\n}\n#ifdef __clang__\n#pragma clang diagnostic pop\n#endif // __clang__\n\ntemplate \nvoid ggml_cuda_flash_attn_ext_vec_case_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n\n const int nthreads = ggml_cuda_fattn_vec_get_nthreads_host(cc);\n const int nwarps = nthreads / WARP_SIZE;\n fattn_kernel_t fattn_kernel = flash_attn_ext_vec;\n const bool need_f16_K = type_K == GGML_TYPE_F16;\n const bool need_f16_V = type_V == GGML_TYPE_F16;\n constexpr size_t nbytes_shared = 0;\n launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);\n}\n\ntemplate \nvoid ggml_cuda_flash_attn_ext_vec_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * KQV = dst;\n const ggml_tensor * Q = dst->src[0];\n\n float logit_softcap;\n memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n if (Q->ne[1] == 1) {\n constexpr int cols_per_block = 1;\n if (logit_softcap == 0.0f) {\n constexpr bool use_logit_softcap = false;\n ggml_cuda_flash_attn_ext_vec_case_impl(ctx, dst);\n } else {\n constexpr bool use_logit_softcap = true;\n ggml_cuda_flash_attn_ext_vec_case_impl(ctx, dst);\n }\n return;\n }\n\n constexpr int cols_per_block = 2;\n if (logit_softcap == 0.0f) {\n constexpr bool use_logit_softcap = false;\n ggml_cuda_flash_attn_ext_vec_case_impl(ctx, dst);\n } else {\n constexpr bool use_logit_softcap = true;\n ggml_cuda_flash_attn_ext_vec_case_impl(ctx, dst);\n }\n}\n\n#define DECL_FATTN_VEC_CASE(D, type_K, type_V) \\\n template void ggml_cuda_flash_attn_ext_vec_case \\\n (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \\\n\n#define EXTERN_DECL_FATTN_VEC_CASES(D, type_K) \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_F16); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q4_0); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q4_1); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_0); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_1); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q8_0); \\\n\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F16)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_0)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_1)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_0)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_1)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q8_0)\n\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F16)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_0)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_1)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_0)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_1)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q8_0)\n\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F16)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_0)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_1)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_0)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_1)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q8_0)\n" }, { "path": "src/ggml-cuda/fattn-wmma-f16.cu", "content": "// Old and deprecated WMMA FlashAttention implementation.\n// It is still needed for Volta since the memory layout of NVIDIA tensor cores changed with Turing.\n// Long-term the WMMA code should be replaced with a dedicated Volta implementation.\n\n#include \"common.cuh\"\n#include \"fattn-common.cuh\"\n#include \"fattn-wmma-f16.cuh\"\n\n#ifdef GGML_USE_WMMA_FATTN\n#if !defined(GGML_USE_HIP)\n#include \n#if defined(GGML_USE_MUSA)\nnamespace wmma = mtmusa::wmma;\n#else // GGML_USE_MUSA\nnamespace wmma = nvcuda::wmma;\n#endif // GGML_USE_MUSA\n#elif defined(GGML_USE_HIP)\n#include \nnamespace wmma = rocwmma;\n#endif // !defined(GGML_USE_HIP)\n#endif // GGML_USE_WMMA_FATTN\n\n// D == head size, VKQ_stride == num VKQ rows calculated in parallel:\ntemplate\n__launch_bounds__(nwarps*ggml_cuda_get_physical_warp_size(), 1)\nstatic __global__ void flash_attn_ext_f16(\n const char * __restrict__ Q,\n const char * __restrict__ K,\n const char * __restrict__ V,\n const char * __restrict__ mask,\n const char * __restrict__ sinks,\n const int * __restrict__ KV_max,\n float * __restrict__ dst,\n float2 * __restrict__ dst_meta,\n const float scale,\n const float max_bias,\n const float m0,\n const float m1,\n const uint32_t n_head_log2,\n const float logit_softcap,\n const int32_t ne00, const uint3 ne01, const int32_t ne02, const int32_t ne03,\n const int32_t nb01, const int32_t nb02, const int32_t nb03,\n const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13,\n const int32_t nb11, const int32_t nb12, const int64_t nb13,\n const int32_t nb21, const int32_t nb22, const int64_t nb23,\n const int32_t ne31, const int32_t ne32, const int32_t ne33,\n const int32_t nb31, const int32_t nb32, const int64_t nb33) {\n#if defined(FLASH_ATTN_AVAILABLE) && (defined(GGML_HIP_ROCWMMA_FATTN) && defined(GGML_USE_WMMA_FATTN))\n // Skip unused kernel variants for faster compilation:\n if (use_logit_softcap && !(D == 128 || D == 256)) {\n NO_DEVICE_CODE;\n return;\n }\n\n //In this kernel Q, K, V are matrices while i, j, k are matrix indices.\n\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n const int ic0 = ncols*blockIdx.x; // Index of the first Q/QKV column to work on.\n\n static_assert(D <= FATTN_KQ_STRIDE, \"D must be <= FATTN_KQ_STRIDE.\");\n static_assert(ncols == 8 || ncols % 16 == 0, \"ncols must be 8 or a multiple of 16.\");\n constexpr int frag_m = ncols == 8 ? 32 : 16;\n constexpr int frag_n = ncols == 8 ? 8 : 16;\n static_assert(D % frag_m == 0, \"If ncols == 8 then D % frag_m must be 0.\");\n#if defined(GGML_USE_HIP) && HIP_VERSION >= 60500000\n typedef wmma::fragment frag_a_K;\n typedef wmma::fragment frag_a_V;\n typedef wmma::fragment frag_b;\n typedef wmma::fragment frag_c_KQ;\n typedef wmma::fragment frag_c_VKQ;\n#else\n typedef wmma::fragment frag_a_K;\n typedef wmma::fragment frag_a_V;\n typedef wmma::fragment frag_b;\n typedef wmma::fragment frag_c_KQ;\n typedef wmma::fragment frag_c_VKQ;\n#endif\n\n constexpr int KQ_stride_tc = nwarps*frag_m; // Number of KQ rows calculated in parallel.\n constexpr int VKQ_ratio = KQ_stride_tc/VKQ_stride; // Number of parallel VKQ accumulators needed to keep all warps busy.\n static_assert(VKQ_ratio <= nwarps, \"VKQ_ratio must be <= nwarps.\");\n\n // Pad internal representation of KQ, KQV to reduce shared memory bank conflicts:\n constexpr int D_padded = D + 8;\n constexpr int kqs_padded = FATTN_KQ_STRIDE + 8;\n constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half);\n\n const int sequence = blockIdx.z / ne02;\n const int head = blockIdx.z - sequence*ne02;\n const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n const float * Q_f = (const float *) (Q + nb03* sequence + nb02* head + nb01*ic0);\n const half * K_h = (const half *) (K + nb13* sequence + nb12*(head / gqa_ratio));\n const half * V_h = (const half *) (V + nb13* sequence + nb12*(head / gqa_ratio)); // K and V have same shape\n const half * maskh = (const half *) (mask + nb33*(sequence % ne33) + nb31*ic0);\n const half2 * mask2 = (const half2 *) maskh;\n const float * sinksf = (const float *) sinks;\n\n const int stride_Q = nb01 / sizeof(float);\n const int stride_KV = nb11 / sizeof(half);\n\n const float slopef = get_alibi_slope(max_bias, head, n_head_log2, m0, m1);\n const half slopeh = __float2half(slopef);\n const half2 slope2 = make_half2(slopef, slopef);\n\n const half2 logit_softcap_2 = make_half2(logit_softcap, logit_softcap);\n\n frag_b Q_b[D/16][ncols/frag_n];\n\n // A single buffer for temporarily holding tiles of KQ and VKQ parts:\n constexpr int mem_KQ = ncols*kqs_padded*kqar;\n constexpr int mem_VKQ_parts = VKQ_ratio*ncols*D_padded;\n __shared__ half KQ[mem_KQ >= mem_VKQ_parts ? mem_KQ : mem_VKQ_parts];\n float * KQ_f = (float *) KQ;\n half2 * KQ2 = (half2 *) KQ;\n\n float KQ_rowsum_f[ncols/nwarps] = {0.0f};\n float KQ_max_f[ncols/nwarps];\n float KQ_max_scale_f[ncols/nwarps] = {0.0f};\n\n#pragma unroll\n for (int j = 0; j < ncols/nwarps; ++j) {\n KQ_max_f[j] = -FLT_MAX/2.0f;\n }\n\n half2 KQ_rowsum_h2[ncols/nwarps] = {{0.0f, 0.0f}};\n half2 KQ_max_h2[ncols/nwarps];\n half2 KQ_max_scale_h2[ncols/nwarps] = {{0.0f, 0.0f}};\n\n#pragma unroll\n for (int j = 0; j < ncols/nwarps; ++j) {\n KQ_max_h2[j] = make_half2(-HALF_MAX_HALF, -HALF_MAX_HALF);\n }\n\n __shared__ half VKQ[ncols*D_padded]; // Accumulator for final VKQ slice.\n half2 * VKQ2 = (half2 *) VKQ;\n\n#if defined(GGML_USE_HIP) && HIP_VERSION >= 60500000\n const _Float16 * K_h_f16 = reinterpret_cast(K_h);\n const _Float16 * V_h_f16 = reinterpret_cast(V_h);\n _Float16 * KQ_f16 = reinterpret_cast<_Float16 *>(KQ);\n _Float16 * VKQ_f16 = reinterpret_cast<_Float16 *>(VKQ);\n#else\n const half * K_h_f16 = K_h;\n const half * V_h_f16 = V_h;\n half * KQ_f16 = KQ;\n half * VKQ_f16 = VKQ;\n#endif\n\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n#pragma unroll\n for (int i0 = 0; i0 < D/2; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n if (i0 + warp_size > D/2 && i >= D/2) {\n break;\n }\n VKQ2[j*(D_padded/2) + i] = make_half2(0.0f, 0.0f);\n }\n }\n\n // Convert Q to half and apply scale, temporarily store in KQ:\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n#pragma unroll\n for (int i0 = 0; i0 < D; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n if (i0 + warp_size > D && i >= D) {\n break;\n }\n KQ[j*D_padded + i] = ic0 + j < int(ne01.z) ? Q_f[j*stride_Q + i] * scale : 0.0f;\n }\n }\n\n __syncthreads();\n\n // Load Q into tensor core fragments/registers since it will be used frequently:\n#pragma unroll\n for (int i0 = 0; i0 < D; i0 += 16) {\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += frag_n) {\n wmma::load_matrix_sync(Q_b[i0/16][j0/frag_n], KQ_f16 + j0*D_padded + i0, D_padded);\n }\n }\n\n __syncthreads();\n\n // Iterate over ne11 == previous tokens:\n const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11;\n for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE) {\n // Calculate tile of KQ:\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) {\n frag_c_KQ KQ_c[ncols/frag_n];\n#pragma unroll\n for (int j = 0; j < ncols/frag_n; ++j) {\n wmma::fill_fragment(KQ_c[j], static_cast(0.0f));\n }\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 16) {\n frag_a_K K_a;\n wmma::load_matrix_sync(K_a, K_h_f16 + int64_t(k_VKQ_0 + i_KQ_0 + frag_m*threadIdx.y)*stride_KV + k_KQ_0, stride_KV);\n#pragma unroll\n for (int j = 0; j < ncols/frag_n; ++j) {\n wmma::mma_sync(KQ_c[j], K_a, Q_b[k_KQ_0/16][j], KQ_c[j]);\n }\n }\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += frag_n) {\n wmma::store_matrix_sync((KQ_acc_t *) KQ + j0*kqs_padded + i_KQ_0 + frag_m*threadIdx.y, KQ_c[j0/frag_n], kqs_padded, wmma::mem_col_major);\n }\n }\n\n __syncthreads();\n\n // Calculate softmax for each KQ column using the current max. value.\n // The divisor is stored in KQ_rowsum and will be applied at the end.\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (std::is_same::value) {\n float KQ_f_tmp[FATTN_KQ_STRIDE / warp_size];\n#pragma unroll\n for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += warp_size) {\n const int k = k0 + threadIdx.x;\n\n KQ_f_tmp[k0/warp_size] = KQ_f[j*kqs_padded + k];\n\n if (use_logit_softcap) {\n KQ_f_tmp[k0/warp_size] = logit_softcap*tanhf(KQ_f_tmp[k0/warp_size]);\n }\n }\n\n float KQ_max_new = KQ_max_f[j0/nwarps];\n#pragma unroll\n for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += warp_size) {\n const int k = k0 + threadIdx.x;\n\n KQ_f_tmp[k0/warp_size] += mask && ic0 + j < int(ne01.z) ?\n __half2float(slopeh*maskh[j*(nb31/sizeof(half)) + k_VKQ_0 + k]) : 0.0f;\n KQ_max_new = max(KQ_max_new, KQ_f_tmp[k0/warp_size] + FATTN_KQ_MAX_OFFSET);\n }\n KQ_max_new = warp_reduce_max(KQ_max_new);\n\n const float diff = KQ_max_f[j0/nwarps] - KQ_max_new;\n KQ_max_scale_f[j0/nwarps] = expf(diff);\n if (diff <= SOFTMAX_FTZ_THRESHOLD) {\n KQ_max_scale_f[j0/nwarps] = 0.0f;\n }\n KQ_max_f[j0/nwarps] = KQ_max_new;\n\n float KQ_rowsum_add = 0.0f;\n#pragma unroll\n for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += warp_size) {\n const int k = k0 + threadIdx.x;\n\n const float diff = KQ_f_tmp[k0/warp_size] - KQ_max_f[j0/nwarps];\n KQ_f_tmp[k0/warp_size] = expf(diff);\n if (diff <= SOFTMAX_FTZ_THRESHOLD) {\n KQ_f_tmp[k0/warp_size] = 0.0f;\n }\n KQ_rowsum_add += KQ_f_tmp[k0/warp_size];\n KQ[j*(kqar*kqs_padded) + k] = KQ_f_tmp[k0/warp_size];\n }\n KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);\n\n // Scale previous KQ_rowsum to account for a potential increase in KQ_max:\n KQ_rowsum_f[j0/nwarps] = KQ_max_scale_f[j0/nwarps]*KQ_rowsum_f[j0/nwarps] + KQ_rowsum_add;\n } else {\n half2 KQ2_tmp[FATTN_KQ_STRIDE/(2*warp_size)];\n#pragma unroll\n for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += warp_size) {\n const int k = k0 + threadIdx.x;\n\n KQ2_tmp[k0/warp_size] = KQ2[j*(kqs_padded/2) + k];\n\n if (use_logit_softcap) {\n // There is no dedicated tangens hyperbolicus function for half2.\n KQ2_tmp[k0/warp_size] = h2exp(KQ2_tmp[k0/warp_size]*make_half2(2.0f, 2.0f));\n KQ2_tmp[k0/warp_size] = (KQ2_tmp[k0/warp_size] - make_half2(1.0f, 1.0f))\n /(KQ2_tmp[k0/warp_size] + make_half2(1.0f, 1.0f));\n\n KQ2_tmp[k0/warp_size] *= logit_softcap_2;\n }\n }\n\n half2 KQ_max_new = KQ_max_h2[j0/nwarps];\n#pragma unroll\n for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += warp_size) {\n const int k = k0 + threadIdx.x;\n\n KQ2_tmp[k0/warp_size] += mask && ic0 + j < int(ne01.z) ? slope2*mask2[(j*ne11 + k_VKQ_0)/2 + k] : make_half2(0.0f, 0.0f);\n KQ_max_new = ggml_cuda_hmax2(KQ_max_new, KQ2_tmp[k0/warp_size]);\n }\n KQ_max_new = __half2half2(warp_reduce_max(ggml_cuda_hmax(__low2half(KQ_max_new), __high2half(KQ_max_new))));\n const half2 diff = KQ_max_h2[j0/nwarps] - KQ_max_new;\n KQ_max_scale_h2[j0/nwarps] = h2exp(diff);\n const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));\n *((uint32_t *) &KQ_max_scale_h2[j0/nwarps]) &= ftz_mask;\n KQ_max_h2[j0/nwarps] = KQ_max_new;\n\n half2 KQ_rowsum_add = make_half2(0.0f, 0.0f);\n#pragma unroll\n for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += warp_size) {\n const int k = k0 + threadIdx.x;\n\n const half2 diff = KQ2_tmp[k0/warp_size] - KQ_max_h2[j0/nwarps];\n KQ2_tmp[k0/warp_size] = h2exp(diff);\n const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));\n *((uint32_t *) &KQ2_tmp[k0/warp_size]) &= ftz_mask;\n KQ_rowsum_add += KQ2_tmp[k0/warp_size];\n KQ2[j*(kqs_padded/2) + k] = KQ2_tmp[k0/warp_size];\n }\n KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);\n\n // Scale previous KQ_rowsum to account for a potential increase in KQ_max:\n KQ_rowsum_h2[j0/nwarps] = KQ_max_scale_h2[j0/nwarps]*KQ_rowsum_h2[j0/nwarps] + KQ_rowsum_add;\n }\n }\n\n __syncthreads();\n\n frag_b KQ_b[FATTN_KQ_STRIDE/(VKQ_ratio*16)][ncols/frag_n];\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += frag_n) {\n#pragma unroll\n for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {\n const int k = k0 + (threadIdx.y % VKQ_ratio)*16;\n wmma::load_matrix_sync(\n KQ_b[k0/(VKQ_ratio*16)][j0/frag_n],\n KQ_f16 + j0*(kqar*kqs_padded) + k,\n kqar*kqs_padded);\n }\n }\n\n frag_c_VKQ VKQ_c[D/VKQ_stride][ncols/frag_n];\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D; i_VKQ_0 += VKQ_stride) {\n#pragma unroll\n for (int j = 0; j < ncols/frag_n; ++j) {\n wmma::fill_fragment(VKQ_c[i_VKQ_0/VKQ_stride][j], static_cast(0.0f));\n }\n\n#pragma unroll\n for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {\n const int k = k0 + (threadIdx.y % VKQ_ratio)*16;\n\n frag_a_V v_a;\n wmma::load_matrix_sync(v_a, V_h_f16 + int64_t(k_VKQ_0 + k)*stride_KV + i_VKQ_0 + frag_m*(threadIdx.y/VKQ_ratio), stride_KV);\n#pragma unroll\n for (int j = 0; j < ncols/frag_n; ++j) {\n wmma::mma_sync(VKQ_c[i_VKQ_0/VKQ_stride][j], v_a, KQ_b[k0/(VKQ_ratio*16)][j], VKQ_c[i_VKQ_0/VKQ_stride][j]);\n }\n }\n }\n\n __syncthreads();\n\n const int offset_k = (threadIdx.y % VKQ_ratio) * (ncols*D_padded);\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += VKQ_stride) {\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += frag_n) {\n wmma::store_matrix_sync(\n KQ_f16 + offset_k + j0*D_padded + i_KQ_0 + frag_m*(threadIdx.y/VKQ_ratio),\n VKQ_c[i_KQ_0/VKQ_stride][j0/frag_n],\n D_padded, wmma::mem_col_major);\n }\n }\n\n __syncthreads();\n\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n half2 VKQ_scale;\n if (std::is_same::value) {\n VKQ_scale = make_half2(KQ_max_scale_f[j0/nwarps], KQ_max_scale_f[j0/nwarps]);\n } else {\n VKQ_scale = KQ_max_scale_h2[j0/nwarps];\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < D/2; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n if (i0 + warp_size > D/2 && i >= D/2) {\n break;\n }\n\n half2 VKQ_add = make_half2(0.0f, 0.0f);\n#pragma unroll\n for (int l = 0; l < VKQ_ratio; ++l) {\n VKQ_add += KQ2[l*(ncols*D_padded/2) + j*(D_padded/2) + i];\n }\n VKQ2[j*(D_padded/2) + i] = VKQ_scale*VKQ2[j*(D_padded/2) + i] + VKQ_add;\n }\n }\n\n __syncthreads();\n }\n\n // Apply attention sinks\n if (sinksf && blockIdx.y == 0) {\n const float sinkf = sinksf[head];\n const half sinkh = __float2half(sinkf);\n\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (std::is_same::value) {\n float kqmax_new = fmaxf(KQ_max_f[j0/nwarps], sinkf);\n\n const float KQ_max_scale = expf(KQ_max_f[j0/nwarps] - kqmax_new);\n KQ_max_f[j0/nwarps] = kqmax_new;\n\n KQ_rowsum_f[j0/nwarps] = KQ_rowsum_f[j0/nwarps] * KQ_max_scale + expf(sinkf - KQ_max_f[j0/nwarps]);\n\n const half2 scale_h2 = make_half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n for (int i0 = 0; i0 < D/2; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n if (i0 + warp_size > D/2 && i >= D/2) break;\n VKQ2[j*(D_padded/2) + i] *= scale_h2;\n }\n } else {\n half kqmax_old = __low2half(KQ_max_h2[j0/nwarps]);\n half kqmax_new = fmaxf(kqmax_old, sinkh);\n KQ_max_h2[j0/nwarps] = __half2half2(kqmax_new);\n\n const half KQ_max_scale_h = hexp(kqmax_old - kqmax_new);\n const half2 KQ_max_scale = __half2half2(KQ_max_scale_h);\n\n KQ_rowsum_h2[j0/nwarps] = KQ_rowsum_h2[j0/nwarps] * KQ_max_scale;\n const half val = hexp(sinkh - kqmax_new);\n KQ_rowsum_h2[j0/nwarps].x = __hadd(KQ_rowsum_h2[j0/nwarps].x, val);\n\n#pragma unroll\n for (int i0 = 0; i0 < D/2; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n if (i0 + warp_size > D/2 && i >= D/2) break;\n VKQ2[j*(D_padded/2) + i] *= KQ_max_scale;\n }\n }\n }\n\n __syncthreads();\n }\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j_VKQ = j0 + threadIdx.y;\n if (ic0 + j_VKQ >= int(ne01.z)) {\n return;\n }\n\n float KQ_rowsum_j;\n if (std::is_same::value) {\n KQ_rowsum_j = KQ_rowsum_f[j0/nwarps];\n } else {\n KQ_rowsum_j = __low2float(KQ_rowsum_h2[j0/nwarps]) + __high2float(KQ_rowsum_h2[j0/nwarps]);\n }\n\n const int j_dst_unrolled = ((sequence*int(ne01.z) + ic0 + j_VKQ)*ne02 + head)*gridDim.y + blockIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < D; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n if (i0 + warp_size > D && i >= D) {\n break;\n }\n float dst_val = VKQ[j_VKQ*D_padded + i];\n if (gridDim.y == 1) {\n dst_val /= KQ_rowsum_j;\n }\n dst[j_dst_unrolled*D + i] = dst_val;\n }\n\n if (gridDim.y == 1 || threadIdx.x != 0) {\n continue;\n }\n\n float2 dst_meta_val;\n if (std::is_same::value) {\n dst_meta_val.x = KQ_max_f[j0/nwarps];\n } else {\n dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]);\n }\n dst_meta_val.y = KQ_rowsum_j;\n dst_meta[j_dst_unrolled] = dst_meta_val;\n }\n#else\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n NO_DEVICE_CODE;\n#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(GGML_HIP_ROCWMMA_FATTN) && defined(GGML_USE_WMMA_FATTN))\n}\n\nconstexpr int get_max_power_of_2(int x) {\n return x % 2 == 0 ? 2*get_max_power_of_2(x/2) : 1;\n}\n\nstatic_assert(get_max_power_of_2(1) == 1, \"Test failed.\");\nstatic_assert(get_max_power_of_2(2) == 2, \"Test failed.\");\nstatic_assert(get_max_power_of_2(4) == 4, \"Test failed.\");\nstatic_assert(get_max_power_of_2(6) == 2, \"Test failed.\");\n\n// Number of VKQ rows calculated in parallel:\nconstexpr int get_VKQ_stride(int D, int nwarps, int frag_m) {\n return (get_max_power_of_2(D/frag_m) < nwarps ? get_max_power_of_2(D/frag_m) : nwarps)*frag_m;\n}\n\nstatic_assert(get_VKQ_stride(128, 1, 32) == 32, \"Test failed.\");\nstatic_assert(get_VKQ_stride(128, 2, 32) == 64, \"Test failed.\");\nstatic_assert(get_VKQ_stride(128, 4, 32) == 128, \"Test failed.\");\nstatic_assert(get_VKQ_stride( 64, 1, 32) == 32, \"Test failed.\");\nstatic_assert(get_VKQ_stride( 64, 2, 32) == 64, \"Test failed.\");\nstatic_assert(get_VKQ_stride( 64, 4, 32) == 64, \"Test failed.\");\nstatic_assert(get_VKQ_stride( 80, 1, 16) == 16, \"Test failed.\");\nstatic_assert(get_VKQ_stride( 80, 2, 16) == 16, \"Test failed.\");\nstatic_assert(get_VKQ_stride( 80, 4, 16) == 16, \"Test failed.\");\n\ntemplate \nvoid ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * KQV = dst;\n\n constexpr int nwarps = 4;\n\n constexpr int frag_m = cols_per_block == 8 && D % 32 == 0 ? 32 : 16;\n const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;\n\n float logit_softcap;\n memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n fattn_kernel_t fattn_kernel;\n if (logit_softcap == 0.0f) {\n constexpr bool use_logit_softcap = false;\n fattn_kernel = flash_attn_ext_f16<\n D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>;\n } else {\n constexpr bool use_logit_softcap = true;\n fattn_kernel = flash_attn_ext_f16<\n D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>;\n }\n launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, FATTN_KQ_STRIDE, true, true, false, warp_size);\n}\n\nvoid ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * KQV = dst;\n const ggml_tensor * Q = dst->src[0];\n\n const enum ggml_prec prec = ggml_flash_attn_ext_get_prec(KQV);\n const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size;\n\n if (prec != GGML_PREC_DEFAULT) {\n if (Q->ne[1] <= 32 || Q->ne[0] > 128) {\n constexpr int cols_per_block = 16;\n switch (Q->ne[0]) {\n case 64:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, float>(ctx, dst);\n break;\n case 80:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, float>(ctx, dst);\n break;\n case 96:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, float>(ctx, dst);\n break;\n case 112:\n ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, float>(ctx, dst);\n break;\n case 128:\n ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, float>(ctx, dst);\n break;\n case 256:\n ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, float>(ctx, dst);\n break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n } else {\n constexpr int cols_per_block = 32;\n switch (Q->ne[0]) {\n case 64:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, float>(ctx, dst);\n break;\n case 80:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, float>(ctx, dst);\n break;\n case 96:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, float>(ctx, dst);\n break;\n case 112:\n ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, float>(ctx, dst);\n break;\n case 128:\n ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, float>(ctx, dst);\n break;\n // case 256:\n // ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, float>(ctx, dst);\n // break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n }\n return;\n }\n\n#if !defined(GGML_USE_HIP)\n if (Q->ne[1] <= 8 && Q->ne[0] % warp_size == 0) {\n constexpr int cols_per_block = 8;\n switch (Q->ne[0]) {\n case 64:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst);\n break;\n case 96:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst);\n break;\n case 128:\n ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst);\n break;\n case 256:\n ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst);\n break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n return;\n }\n#endif // !defined(GGML_USE_HIP)\n\n if (Q->ne[1] <= 32) {\n constexpr int cols_per_block = 16;\n switch (Q->ne[0]) {\n case 64:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst);\n break;\n case 80:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, half>(ctx, dst);\n break;\n case 96:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst);\n break;\n case 112:\n ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, half>(ctx, dst);\n break;\n case 128:\n ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst);\n break;\n case 256:\n ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst);\n break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n return;\n }\n\n constexpr int cols_per_block = 32;\n switch (Q->ne[0]) {\n case 64:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst);\n break;\n case 80:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, half>(ctx, dst);\n break;\n case 96:\n ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst);\n break;\n case 112:\n ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, half>(ctx, dst);\n break;\n case 128:\n ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst);\n break;\n case 256:\n ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst);\n break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n" }, { "path": "src/ggml-cuda/fattn-wmma-f16.cuh", "content": "#pragma once\n\n#include \"common.cuh\"\n\n#if defined(GGML_USE_MUSA)\n#define GGML_USE_WMMA_FATTN\n#endif // defined(GGML_USE_MUSA)\n\n#if defined(GGML_HIP_ROCWMMA_FATTN)\n#if defined(CDNA) && (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0)\n#define GGML_USE_WMMA_FATTN\n#elif defined(CDNA)\n#warning \"rocwmma fattn on CDNA is broken on rocwmma v2.0.0, expect degraded performance\"\n#endif // defined(CDNA) && (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0)\n#if defined(RDNA3)\n#define GGML_USE_WMMA_FATTN\n#endif // defined(RDNA3)\n#if defined(RDNA4) && ROCWMMA_VERSION_MAJOR > 1\n#define GGML_USE_WMMA_FATTN\n#elif defined(RDNA4)\n#warning \"rocwmma fattn is not supported on RDNA4 on rocwmma < v2.0.0, expect degraded performance\"\n#endif // defined(RDNA4) && ROCWMMA_VERSION_MAJOR > 1\n#endif // defined(GGML_HIP_ROCWMMA_FATTN)\n\n// WMMA flash attention requires FP16 matrix instructions to be available for ggml code.\nstatic bool ggml_cuda_should_use_wmma_fattn(const int cc) {\n#if defined(GGML_USE_HIP) && !defined(GGML_HIP_ROCWMMA_FATTN)\n return false;\n#else\n if ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_VOLTA) ||\n GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_MTHREADS(cc)) {\n return true;\n } else if (GGML_CUDA_CC_IS_CDNA(cc)){\n#if defined(GGML_HIP_ROCWMMA_FATTN) && (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0)\n return true;\n#else\n return false;\n#endif // defined(GGML_HIP_ROCWMMA_FATTN) (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0)\n } else if (GGML_CUDA_CC_IS_RDNA4(cc)) {\n#if defined(GGML_HIP_ROCWMMA_FATTN) && ROCWMMA_VERSION_MAJOR > 1\n return true;\n#else\n return false;\n#endif // defined(GGML_HIP_ROCWMMA_FATTN) && ROCWMMA_VERSION_MAJOR > 1\n } else {\n return false;\n }\n#endif // defined(GGML_USE_HIP) && !defined(GGML_HIP_ROCWMMA_FATTN)\n}\n\nvoid ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/fattn.cu", "content": "#include \"common.cuh\"\n#include \"fattn-common.cuh\"\n#include \"fattn-mma-f16.cuh\"\n#include \"fattn-tile.cuh\"\n#include \"fattn-vec.cuh\"\n#include \"fattn-wmma-f16.cuh\"\n#include \"fattn.cuh\"\n\ntemplate \nstatic void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n const ggml_tensor * Q = dst->src[0];\n\n if constexpr (ncols2 <= 8) {\n if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) {\n ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst);\n return;\n }\n }\n\n if constexpr (ncols2 <= 16) {\n if ((turing_mma_available(cc) || amd_wmma_available(cc)) && Q->ne[1] <= 16/ncols2) {\n ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst);\n return;\n }\n }\n\n if (ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING || amd_wmma_available(cc) || Q->ne[1] <= 32/ncols2) {\n ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst);\n return;\n }\n\n ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst);\n}\n\ntemplate \nstatic void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n const ggml_tensor * KQV = dst;\n const ggml_tensor * Q = dst->src[0];\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * V = dst->src[2];\n const ggml_tensor * mask = dst->src[3];\n\n float max_bias = 0.0f;\n memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));\n\n // Edge cases like no mask, ALiBi, unpadded K/V, or misaligned addresses for large data transfers\n // are put into the template specialization without GQA optimizations.\n bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0;\n for (const ggml_tensor * t : {Q, K, V, mask}) {\n if (t == nullptr || ggml_is_quantized(t->type)) {\n continue;\n }\n for (size_t i = 1; i < GGML_MAX_DIMS; ++i) {\n if (t->nb[i] % 16 != 0) {\n use_gqa_opt = false;\n break;\n }\n }\n }\n\n GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);\n const int gqa_ratio = Q->ne[2] / K->ne[2];\n\n // On Volta the GQA optimizations aren't as impactful vs. minimizing wasted compute:\n if (cc == GGML_CUDA_CC_VOLTA) {\n if (use_gqa_opt && gqa_ratio % 8 == 0) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst);\n return;\n }\n\n if (use_gqa_opt && gqa_ratio % 4 == 0) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst);\n return;\n }\n\n if (use_gqa_opt && gqa_ratio % 2 == 0) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst);\n return;\n }\n\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst);\n return;\n }\n\n if (use_gqa_opt && gqa_ratio > 4) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst);\n return;\n }\n\n if (use_gqa_opt && gqa_ratio > 2) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst);\n return;\n }\n\n if (use_gqa_opt && gqa_ratio > 1) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst);\n return;\n }\n\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst);\n}\n\nstatic void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n const ggml_tensor * KQV = dst;\n const ggml_tensor * Q = dst->src[0];\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * V = dst->src[2];\n const ggml_tensor * mask = dst->src[3];\n\n switch (Q->ne[0]) {\n case 64:\n GGML_ASSERT(V->ne[0] == 64);\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 64, 64>(ctx, dst);\n break;\n case 80:\n GGML_ASSERT(V->ne[0] == 80);\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 80, 80>(ctx, dst);\n break;\n case 96:\n GGML_ASSERT(V->ne[0] == 96);\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2< 96, 96>(ctx, dst);\n break;\n case 112:\n GGML_ASSERT(V->ne[0] == 112);\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<112, 112>(ctx, dst);\n break;\n case 128:\n GGML_ASSERT(V->ne[0] == 128);\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<128, 128>(ctx, dst);\n break;\n case 256:\n GGML_ASSERT(V->ne[0] == 256);\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<256, 256>(ctx, dst);\n break;\n case 576: {\n // For Deepseek, go straight to the ncols1 switch to avoid compiling unnecessary kernels.\n GGML_ASSERT(V->ne[0] == 512);\n float max_bias = 0.0f;\n memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));\n\n const bool use_gqa_opt = mask && max_bias == 0.0f;\n GGML_ASSERT(use_gqa_opt);\n\n GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);\n const int gqa_ratio = Q->ne[2] / K->ne[2];\n if (gqa_ratio == 20) { // GLM 4.7 Flash\n if (cc >= GGML_CUDA_CC_DGX_SPARK) {\n if (Q->ne[1] <= 8) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);\n break;\n }\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 4>(ctx, dst);\n break;\n }\n if (cc >= GGML_CUDA_CC_BLACKWELL) {\n if (Q->ne[1] <= 4 && K->ne[1] >= 65536) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);\n break;\n }\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 4>(ctx, dst);\n break;\n }\n if (cc >= GGML_CUDA_CC_ADA_LOVELACE) {\n if (Q->ne[1] <= 4) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);\n break;\n }\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 4>(ctx, dst);\n break;\n }\n if (cc >= GGML_CUDA_CC_TURING) {\n if (Q->ne[1] <= 4) {\n if (K->ne[1] <= 16384) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);\n break;\n }\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 32>(ctx, dst);\n break;\n }\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 4>(ctx, dst);\n break;\n }\n // Volta:\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 4>(ctx, dst);\n } else if (gqa_ratio % 16 == 0) {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);\n } else {\n ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 4>(ctx, dst);\n }\n } break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\n#define FATTN_VEC_CASE(D, type_K, type_V) \\\n { \\\n const bool type_K_okay = K->type == (type_K) || (K->type == GGML_TYPE_F32 && (type_K) == GGML_TYPE_F16); \\\n const bool type_V_okay = V->type == (type_V) || (V->type == GGML_TYPE_F32 && (type_V) == GGML_TYPE_F16); \\\n if (Q->ne[0] == (D) && type_K_okay && type_V_okay) { \\\n ggml_cuda_flash_attn_ext_vec_case(ctx, dst); \\\n return; \\\n } \\\n } \\\n\n#define FATTN_VEC_CASES_ALL_D(type_K, type_V) \\\n FATTN_VEC_CASE( 64, type_K, type_V) \\\n FATTN_VEC_CASE(128, type_K, type_V) \\\n FATTN_VEC_CASE(256, type_K, type_V) \\\n\nstatic void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_tensor * Q = dst->src[0];\n ggml_tensor * K = dst->src[1];\n ggml_tensor * V = dst->src[2];\n\n#ifdef GGML_CUDA_FA_ALL_QUANTS\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_F16)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_1)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_0)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_1)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)\n#else\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)\n#endif // GGML_CUDA_FA_ALL_QUANTS\n\n GGML_ABORT(\"fatal error\");\n}\n\n// Best FlashAttention kernel for a specific GPU:\nenum best_fattn_kernel {\n BEST_FATTN_KERNEL_NONE = 0,\n BEST_FATTN_KERNEL_TILE = 200,\n BEST_FATTN_KERNEL_VEC = 100,\n BEST_FATTN_KERNEL_WMMA_F16 = 300,\n BEST_FATTN_KERNEL_MMA_F16 = 400,\n};\n\nstatic best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const ggml_tensor * dst) {\n#ifndef FLASH_ATTN_AVAILABLE\n GGML_UNUSED(device); GGML_UNUSED(dst);\n return BEST_FATTN_KERNEL_NONE;\n#endif// FLASH_ATTN_AVAILABLE\n\n const ggml_tensor * KQV = dst;\n const ggml_tensor * Q = dst->src[0];\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * V = dst->src[2];\n const ggml_tensor * mask = dst->src[3];\n\n const int gqa_ratio = Q->ne[2] / K->ne[2];\n GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);\n\n float max_bias = 0.0f;\n memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));\n\n // The effective batch size for the kernel can be increased by gqa_ratio.\n // The kernel versions without this optimization are also used for ALiBi, if there is no mask, or if the KV cache is not padded,\n bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0;\n for (const ggml_tensor * t : {Q, K, V, mask}) {\n if (t == nullptr || ggml_is_quantized(t->type)) {\n continue;\n }\n for (size_t i = 1; i < GGML_MAX_DIMS; ++i) {\n if (t->nb[i] % 16 != 0) {\n gqa_opt_applies = false;\n break;\n }\n }\n }\n\n const int cc = ggml_cuda_info().devices[device].cc;\n\n switch (K->ne[0]) {\n case 40:\n case 64:\n case 72:\n case 80:\n case 96:\n case 128:\n case 112:\n case 256:\n if (V->ne[0] != K->ne[0]) {\n return BEST_FATTN_KERNEL_NONE;\n }\n break;\n case 576:\n if (V->ne[0] != 512) {\n return BEST_FATTN_KERNEL_NONE;\n }\n if (!gqa_opt_applies) {\n return BEST_FATTN_KERNEL_NONE;\n }\n break;\n default:\n return BEST_FATTN_KERNEL_NONE;\n }\n\n#ifndef GGML_CUDA_FA_ALL_QUANTS\n if (K->type != V->type) {\n return BEST_FATTN_KERNEL_NONE;\n }\n#endif // GGML_CUDA_FA_ALL_QUANTS\n\n switch (K->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n break;\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n#ifndef GGML_CUDA_FA_ALL_QUANTS\n return BEST_FATTN_KERNEL_NONE;\n#endif // GGML_CUDA_FA_ALL_QUANTS\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q8_0:\n break;\n default:\n return BEST_FATTN_KERNEL_NONE;\n }\n\n if (mask && mask->ne[2] != 1) {\n return BEST_FATTN_KERNEL_NONE;\n }\n\n // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes:\n const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0;\n\n // If Turing tensor cores are available, use them:\n if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {\n if (can_use_vector_kernel) {\n if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) {\n if (cc >= GGML_CUDA_CC_ADA_LOVELACE && Q->ne[1] == 1 && Q->ne[3] == 1 && !(gqa_ratio > 4 && K->ne[1] >= 8192)) {\n return BEST_FATTN_KERNEL_VEC;\n }\n } else {\n if (cc >= GGML_CUDA_CC_ADA_LOVELACE) {\n if (Q->ne[1] <= 2) {\n return BEST_FATTN_KERNEL_VEC;\n }\n } else {\n if (Q->ne[1] == 1) {\n return BEST_FATTN_KERNEL_VEC;\n }\n }\n }\n if (!gqa_opt_applies && Q->ne[1] == 1) {\n return BEST_FATTN_KERNEL_VEC;\n }\n }\n return BEST_FATTN_KERNEL_MMA_F16;\n }\n\n if (volta_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {\n int gqa_ratio_eff = 1;\n const int ncols2_max = Q->ne[0] == 576 ? 16 : 8;\n while (gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {\n gqa_ratio_eff *= 2;\n }\n if (can_use_vector_kernel && Q->ne[1] * gqa_ratio_eff <= 2) {\n return BEST_FATTN_KERNEL_VEC;\n }\n if (Q->ne[1] * gqa_ratio_eff <= 16) {\n return BEST_FATTN_KERNEL_TILE; // On Volta tensor cores are only faster for sufficiently large matrices.\n }\n return BEST_FATTN_KERNEL_MMA_F16;\n }\n\n // Use the WMMA kernel if possible:\n if (ggml_cuda_should_use_wmma_fattn(cc) && K->ne[1] % FATTN_KQ_STRIDE == 0 && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 576) {\n if (can_use_vector_kernel && Q->ne[1] <= 2) {\n return BEST_FATTN_KERNEL_VEC;\n }\n return BEST_FATTN_KERNEL_WMMA_F16;\n }\n\n if (amd_wmma_available(cc) && GGML_CUDA_CC_IS_RDNA4(cc) && gqa_opt_applies && Q->ne[0] <= 128 && Q->ne[0] != 40 && Q->ne[0] != 72) {\n if (can_use_vector_kernel) {\n if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) {\n if (Q->ne[1] == 1) {\n if (!gqa_opt_applies) {\n return BEST_FATTN_KERNEL_VEC;\n }\n }\n } else {\n if (Q->ne[1] <= 2) {\n return BEST_FATTN_KERNEL_VEC;\n }\n }\n }\n int gqa_ratio_eff = 1;\n const int ncols2_max = Q->ne[0] == 576 ? 16 : 8;\n while (gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {\n gqa_ratio_eff *= 2;\n }\n if (Q->ne[1] * gqa_ratio_eff <= 8) {\n return BEST_FATTN_KERNEL_TILE; // AMD WMMA is only faster if the full tile width of 16 can be utilized.\n }\n return BEST_FATTN_KERNEL_MMA_F16;\n }\n\n // Use MFMA flash attention for CDNA (MI100+):\n if (amd_mfma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 256 && Q->ne[0] != 576) {\n const int64_t eff_nq = Q->ne[1] * (gqa_opt_applies ? gqa_ratio : 1);\n // MMA vs tile crossover benchmarked on MI300X @ d32768:\n // hsk=64 (gqa=4): MMA wins at eff >= 128 (+11%)\n // hsk=128 (gqa=4): MMA wins at eff >= 128 (+4%)\n if (eff_nq >= (GGML_CUDA_CC_IS_CDNA1(cc) && Q->ne[0] == 64 ? 64 : 128)) {\n return BEST_FATTN_KERNEL_MMA_F16;\n }\n // Fall through to tile kernel for small effective batch sizes.\n }\n\n // If there are no tensor cores available, use the generic tile kernel:\n if (can_use_vector_kernel) {\n if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) {\n if (Q->ne[1] == 1) {\n if (!gqa_opt_applies) {\n return BEST_FATTN_KERNEL_VEC;\n }\n }\n } else {\n if (Q->ne[1] <= 2) {\n return BEST_FATTN_KERNEL_VEC;\n }\n }\n }\n return BEST_FATTN_KERNEL_TILE;\n}\n\nvoid ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_set_device(ctx.device);\n switch (ggml_cuda_get_best_fattn_kernel(ggml_cuda_get_device(), dst)) {\n case BEST_FATTN_KERNEL_NONE:\n GGML_ABORT(\"fatal error\");\n case BEST_FATTN_KERNEL_TILE:\n ggml_cuda_flash_attn_ext_tile(ctx, dst);\n break;\n case BEST_FATTN_KERNEL_VEC:\n ggml_cuda_flash_attn_ext_vec(ctx, dst);\n break;\n case BEST_FATTN_KERNEL_WMMA_F16:\n ggml_cuda_flash_attn_ext_wmma_f16(ctx, dst);\n break;\n case BEST_FATTN_KERNEL_MMA_F16:\n ggml_cuda_flash_attn_ext_mma_f16(ctx, dst);\n break;\n }\n}\n\nbool ggml_cuda_flash_attn_ext_supported(int device, const ggml_tensor * dst) {\n return ggml_cuda_get_best_fattn_kernel(device, dst) != BEST_FATTN_KERNEL_NONE;\n}\n" }, { "path": "src/ggml-cuda/fattn.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nbool ggml_cuda_flash_attn_ext_supported(int device, const ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/fill.cu", "content": "#include \"fill.cuh\"\n#include \"convert.cuh\"\n\n#define CUDA_FILL_BLOCK_SIZE 256\n\ntemplate \nstatic __global__ void fill_kernel(T * dst, const int64_t k, const T value) {\n const int64_t i = (int64_t)blockDim.x * blockIdx.x + threadIdx.x;\n if (i >= k) {\n return;\n }\n dst[i] = value;\n}\n\nvoid ggml_cuda_op_fill(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n void * dst_d = dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n float value;\n memcpy(&value, dst->op_params, sizeof(float));\n\n const int64_t k = ggml_nelements(dst);\n const int64_t num_blocks = (k + CUDA_FILL_BLOCK_SIZE - 1) / CUDA_FILL_BLOCK_SIZE;\n\n switch (dst->type) {\n case GGML_TYPE_F32:\n fill_kernel<<>>((float *)dst_d, k, value);\n break;\n case GGML_TYPE_F16:\n fill_kernel<<>>((half *)dst_d, k, ggml_cuda_cast(value));\n break;\n default:\n GGML_ABORT(\"unsupported type\");\n }\n}\n" }, { "path": "src/ggml-cuda/fill.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_fill(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/gated_delta_net.cu", "content": "#include \"gated_delta_net.cuh\"\n\ntemplate \n__global__ void gated_delta_net_cuda(const float * q,\n const float * k,\n const float * v,\n const float * g,\n const float * beta,\n const float * curr_state,\n float * dst,\n int64_t H,\n int64_t n_tokens,\n int64_t n_seqs,\n int64_t sq1,\n int64_t sq2,\n int64_t sq3,\n int64_t sv1,\n int64_t sv2,\n int64_t sv3,\n int64_t sb1,\n int64_t sb2,\n int64_t sb3,\n const uint3 neqk1_magic,\n const uint3 rq3_magic,\n float scale) {\n const uint32_t h_idx = blockIdx.x;\n const uint32_t sequence = blockIdx.y;\n // each warp owns one column, using warp-level primitives to reduce across rows\n const int lane = threadIdx.x;\n const int col = blockIdx.z * blockDim.y + threadIdx.y;\n\n const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic);\n const uint32_t iq3 = fastdiv(sequence, rq3_magic);\n\n const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs;\n float * attn_data = dst;\n float * state = dst + attn_score_elems;\n\n const int64_t state_offset = (sequence * H + h_idx) * S_v * S_v;\n state += state_offset;\n curr_state += state_offset;\n attn_data += (sequence * n_tokens * H + h_idx) * S_v;\n\n constexpr int warp_size = ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v;\n static_assert(S_v % warp_size == 0, \"S_v must be a multiple of warp_size\");\n constexpr int rows_per_lane = (S_v + warp_size - 1) / warp_size;\n float s_shard[rows_per_lane];\n // state is stored transposed: M[col][i] = S[i][col], row col is contiguous\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n s_shard[r] = curr_state[col * S_v + i];\n }\n\n for (int t = 0; t < n_tokens; t++) {\n const float * q_t = q + iq3 * sq3 + t * sq2 + iq1 * sq1;\n const float * k_t = k + iq3 * sq3 + t * sq2 + iq1 * sq1;\n const float * v_t = v + sequence * sv3 + t * sv2 + h_idx * sv1;\n\n const int64_t gb_offset = sequence * sb3 + t * sb2 + h_idx * sb1;\n const float * beta_t = beta + gb_offset;\n const float * g_t = g + gb_offset * (KDA ? S_v : 1);\n\n const float beta_val = *beta_t;\n\n if constexpr (!KDA) {\n const float g_val = expf(*g_t);\n\n // kv[col] = (S^T @ k)[col] = sum_i S[i][col] * k[i]\n float kv_shard = 0.0f;\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n kv_shard += s_shard[r] * k_t[i];\n }\n float kv_col = warp_reduce_sum(kv_shard);\n\n // delta[col] = (v[col] - g * kv[col]) * beta\n float delta_col = (v_t[col] - g_val * kv_col) * beta_val;\n\n // fused: S[i][col] = g * S[i][col] + k[i] * delta[col]\n // attn[col] = (S^T @ q)[col] = sum_i S[i][col] * q[i]\n float attn_partial = 0.0f;\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n s_shard[r] = g_val * s_shard[r] + k_t[i] * delta_col;\n attn_partial += s_shard[r] * q_t[i];\n }\n\n float attn_col = warp_reduce_sum(attn_partial);\n\n if (lane == 0) {\n attn_data[col] = attn_col * scale;\n }\n } else {\n // kv[col] = sum_i g[i] * S[i][col] * k[i]\n float kv_shard = 0.0f;\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n kv_shard += expf(g_t[i]) * s_shard[r] * k_t[i];\n }\n\n float kv_col = warp_reduce_sum(kv_shard);\n\n // delta[col] = (v[col] - kv[col]) * beta\n float delta_col = (v_t[col] - kv_col) * beta_val;\n\n // fused: S[i][col] = g[i] * S[i][col] + k[i] * delta[col]\n // attn[col] = (S^T @ q)[col] = sum_i S[i][col] * q[i]\n float attn_partial = 0.0f;\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n s_shard[r] = expf(g_t[i]) * s_shard[r] + k_t[i] * delta_col;\n attn_partial += s_shard[r] * q_t[i];\n }\n\n float attn_col = warp_reduce_sum(attn_partial);\n\n if (lane == 0) {\n attn_data[col] = attn_col * scale;\n }\n }\n\n attn_data += S_v * H;\n }\n\n // Write state back to global memory (transposed layout)\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n state[col * S_v + i] = s_shard[r];\n }\n}\n\ntemplate \nstatic void launch_gated_delta_net(\n const float * q_d, const float * k_d, const float * v_d,\n const float * g_d, const float * b_d, const float * s_d,\n float * dst_d,\n int64_t S_v, int64_t H, int64_t n_tokens, int64_t n_seqs,\n int64_t sq1, int64_t sq2, int64_t sq3,\n int64_t sv1, int64_t sv2, int64_t sv3,\n int64_t sb1, int64_t sb2, int64_t sb3,\n int64_t neqk1, int64_t rq3,\n float scale, cudaStream_t stream) {\n //TODO: Add chunked kernel for even faster pre-fill\n const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;\n const int num_warps = 4;\n dim3 grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps);\n dim3 block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1);\n\n const uint3 neqk1_magic = init_fastdiv_values(neqk1);\n const uint3 rq3_magic = init_fastdiv_values(rq3);\n\n int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n\n switch (S_v) {\n case 16:\n gated_delta_net_cuda<16, KDA><<>>(\n q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H,\n n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,\n sb1, sb2, sb3, neqk1_magic, rq3_magic, scale);\n break;\n case 32:\n gated_delta_net_cuda<32, KDA><<>>(\n q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H,\n n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,\n sb1, sb2, sb3, neqk1_magic, rq3_magic, scale);\n break;\n case 64: {\n gated_delta_net_cuda<64, KDA><<>>(\n q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H,\n n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,\n sb1, sb2, sb3, neqk1_magic, rq3_magic, scale);\n break;\n }\n case 128: {\n gated_delta_net_cuda<128, KDA><<>>(\n q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H,\n n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,\n sb1, sb2, sb3, neqk1_magic, rq3_magic, scale);\n break;\n }\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\nvoid ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src_q = dst->src[0];\n ggml_tensor * src_k = dst->src[1];\n ggml_tensor * src_v = dst->src[2];\n ggml_tensor * src_g = dst->src[3];\n ggml_tensor * src_beta = dst->src[4];\n ggml_tensor * src_state = dst->src[5];\n\n GGML_TENSOR_LOCALS(int64_t, neq, src_q, ne);\n GGML_TENSOR_LOCALS(size_t , nbq, src_q, nb);\n GGML_TENSOR_LOCALS(int64_t, nek, src_k, ne);\n GGML_TENSOR_LOCALS(size_t , nbk, src_k, nb);\n GGML_TENSOR_LOCALS(int64_t, nev, src_v, ne);\n GGML_TENSOR_LOCALS(size_t, nbv, src_v, nb);\n GGML_TENSOR_LOCALS(size_t, nbb, src_beta, nb);\n\n const int64_t S_v = nev0;\n const int64_t H = nev1;\n const int64_t n_tokens = nev2;\n const int64_t n_seqs = nev3;\n\n const bool kda = (src_g->ne[0] == S_v);\n\n GGML_ASSERT(neq1 == nek1);\n const int64_t neqk1 = neq1;\n\n const int64_t rq3 = nev3 / neq3;\n\n const float * q_d = (const float *) src_q->data;\n const float * k_d = (const float *) src_k->data;\n const float * v_d = (const float *) src_v->data;\n const float * g_d = (const float *) src_g->data;\n const float * b_d = (const float *) src_beta->data;\n\n const float * s_d = (const float *) src_state->data;\n float * dst_d = (float *) dst->data;\n\n GGML_ASSERT(ggml_is_contiguous_rows(src_q));\n GGML_ASSERT(ggml_is_contiguous_rows(src_k));\n GGML_ASSERT(ggml_is_contiguous_rows(src_v));\n GGML_ASSERT(ggml_are_same_stride(src_q, src_k));\n GGML_ASSERT(src_g->ne[0] == 1 || kda);\n GGML_ASSERT(ggml_is_contiguous(src_g));\n GGML_ASSERT(ggml_is_contiguous(src_beta));\n GGML_ASSERT(ggml_is_contiguous(src_state));\n\n // strides in floats (beta strides used for both g and beta offset computation)\n const int64_t sq1 = nbq1 / sizeof(float);\n const int64_t sq2 = nbq2 / sizeof(float);\n const int64_t sq3 = nbq3 / sizeof(float);\n const int64_t sv1 = nbv1 / sizeof(float);\n const int64_t sv2 = nbv2 / sizeof(float);\n const int64_t sv3 = nbv3 / sizeof(float);\n const int64_t sb1 = nbb1 / sizeof(float);\n const int64_t sb2 = nbb2 / sizeof(float);\n const int64_t sb3 = nbb3 / sizeof(float);\n\n const float scale = 1.0f / sqrtf((float) S_v);\n\n cudaStream_t stream = ctx.stream();\n\n if (kda) {\n launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d,\n S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,\n sb1, sb2, sb3, neqk1, rq3, scale, stream);\n } else {\n launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d,\n S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,\n sb1, sb2, sb3, neqk1, rq3, scale, stream);\n }\n}\n" }, { "path": "src/ggml-cuda/gated_delta_net.cuh", "content": "#include \"common.cuh\"\n#include \"ggml.h\"\n\nvoid ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/getrows.cu", "content": "#include \"getrows.cuh\"\n#include \"dequantize.cuh\"\n#include \"convert.cuh\"\n\ntemplate\nstatic __global__ void k_get_rows(\n const void * __restrict__ src0, const int32_t * __restrict__ src1, dst_t * __restrict__ dst,\n const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/\n /*const int64_t ne10,*/ const int64_t ne11, const int64_t ne12, /*const int64_t ne13,*/\n /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3,\n /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03,\n const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) {\n\n for (int64_t z = blockIdx.z; z < ne11*ne12; z += gridDim.z) {\n for (int64_t i00 = 2*(blockIdx.y*blockDim.x + threadIdx.x); i00 < ne00; i00 += gridDim.y*blockDim.x) {\n // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher.\n const int i10 = blockIdx.x;\n const int i11 = z / ne12; // TODO fastdiv\n const int i12 = z % ne12;\n\n const int i01 = src1[i10*s10 + i11*s11 + i12*s12];\n\n dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;\n const void * src0_row = (const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03;\n\n const int ib = i00/qk; // block index\n const int iqs = (i00%qk)/qr; // quant index\n const int iybs = i00 - i00%qk; // dst block start index\n const int y_offset = qr == 1 ? 1 : qk/2;\n\n // dequantize\n float2 v;\n dequantize_kernel(src0_row, ib, iqs, v);\n\n dst_row[iybs + iqs + 0] = ggml_cuda_cast(v.x);\n dst_row[iybs + iqs + y_offset] = ggml_cuda_cast(v.y);\n }\n }\n}\n\ntemplate\nstatic __global__ void k_get_rows_float(\n const src0_t * __restrict__ src0, const int32_t * __restrict__ src1, dst_t * __restrict__ dst,\n const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/\n /*const int64_t ne10,*/ const int64_t ne11, const int64_t ne12, /*const int64_t ne13,*/\n /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3,\n /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03,\n const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) {\n\n for (int64_t z = blockIdx.z; z < ne11*ne12; z += gridDim.z) {\n for (int64_t i00 = blockIdx.y*blockDim.x + threadIdx.x; i00 < ne00; i00 += gridDim.y*blockDim.x) {\n // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher.\n const int i10 = blockIdx.x;\n const int i11 = z / ne12; // TODO fastdiv\n const int i12 = z % ne12;\n\n if (i00 >= ne00) {\n return;\n }\n\n const int i01 = src1[i10*s10 + i11*s11 + i12*s12];\n\n dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;\n const src0_t * src0_row = (const src0_t *)((const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03);\n\n dst_row[i00] = ggml_cuda_cast(src0_row[i00]);\n }\n }\n}\n\ntemplate\nstatic __global__ void k_get_rows_back_float(\n const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst, const int64_t ncols, const int64_t nrows_grad) {\n const int col = blockIdx.x*blockDim.x + threadIdx.x;\n\n if (col >= ncols) {\n return;\n }\n\n const int dst_row = blockIdx.y*blockDim.y + threadIdx.y;\n\n float sum = 0.0f;\n\n for (int64_t i = 0; i < nrows_grad; ++i) {\n if (rows[i] != dst_row) {\n continue;\n }\n sum += grad[i*ncols + col];\n }\n\n dst[dst_row*ncols + col] = sum;\n}\n\ntemplate\nstatic void get_rows_cuda_q(\n const void * src0_d, const int32_t * src1_d, dst_t * dst_d,\n const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,\n const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,\n const size_t nb1, const size_t nb2, const size_t nb3,\n cudaStream_t stream) {\n const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);\n const int block_num_y = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);\n const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));\n\n // strides in elements\n // const size_t s0 = nb0 / sizeof(dst_t);\n const size_t s1 = nb1 / sizeof(dst_t);\n const size_t s2 = nb2 / sizeof(dst_t);\n const size_t s3 = nb3 / sizeof(dst_t);\n\n const size_t s10 = nb10 / sizeof(int32_t);\n const size_t s11 = nb11 / sizeof(int32_t);\n const size_t s12 = nb12 / sizeof(int32_t);\n // const size_t s13 = nb13 / sizeof(int32_t);\n\n GGML_ASSERT(ne00 % 2 == 0);\n\n k_get_rows<<>>(\n src0_d, src1_d, dst_d,\n ne00, /*ne01, ne02, ne03,*/\n /*ne10,*/ ne11, ne12, /*ne13,*/\n /* s0,*/ s1, s2, s3,\n /* nb00,*/ nb01, nb02, nb03,\n s10, s11, s12/*, s13*/);\n}\n\ntemplate\nstatic void get_rows_cuda_float(\n const src0_t * src0_d, const int32_t * src1_d, dst_t * dst_d,\n const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,\n const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,\n const size_t nb1, const size_t nb2, const size_t nb3,\n cudaStream_t stream) {\n const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);\n const int block_num_y = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;\n const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));\n\n // strides in elements\n // const size_t s0 = nb0 / sizeof(dst_t);\n const size_t s1 = nb1 / sizeof(dst_t);\n const size_t s2 = nb2 / sizeof(dst_t);\n const size_t s3 = nb3 / sizeof(dst_t);\n\n const size_t s10 = nb10 / sizeof(int32_t);\n const size_t s11 = nb11 / sizeof(int32_t);\n const size_t s12 = nb12 / sizeof(int32_t);\n // const size_t s13 = nb13 / sizeof(int32_t);\n\n k_get_rows_float<<>>(\n src0_d, src1_d, dst_d,\n ne00, /*ne01, ne02, ne03,*/\n /*ne10,*/ ne11, ne12, /*ne13,*/\n /* s0,*/ s1, s2, s3,\n /* nb00,*/ nb01, nb02, nb03,\n s10, s11, s12/*, s13*/);\n}\n\ntemplate \nstatic void ggml_cuda_get_rows_switch_src0_type(\n const void * src0_d, const ggml_type src0_type, const int32_t * src1_d, dst_t * dst_d,\n const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,\n const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,\n const size_t nb1, const size_t nb2, const size_t nb3,\n cudaStream_t stream) {\n switch (src0_type) {\n case GGML_TYPE_F16:\n get_rows_cuda_float((const half *) src0_d, src1_d, dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_F32:\n get_rows_cuda_float((const float *) src0_d, src1_d, dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_I32:\n get_rows_cuda_float((const int32_t *) src0_d, src1_d, dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_BF16:\n get_rows_cuda_float((const nv_bfloat16 *) src0_d, src1_d, dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_Q4_0:\n get_rows_cuda_q(src0_d, src1_d, dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_Q4_1:\n get_rows_cuda_q(src0_d, src1_d, dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_Q5_0:\n get_rows_cuda_q(src0_d, src1_d, dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_Q5_1:\n get_rows_cuda_q(src0_d, src1_d, dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_Q8_0:\n get_rows_cuda_q(src0_d, src1_d, dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n default:\n // TODO: k-quants\n GGML_ABORT(\"%s: unsupported src0 type: %s\\n\", __func__, ggml_type_name(src0_type));\n break;\n }\n}\n\nvoid get_rows_cuda(\n const void * src0_d, ggml_type src0_type, const int32_t * src1_d, void * dst_d, ggml_type dst_type,\n int64_t ne00, size_t nb01, size_t nb02, size_t nb03,\n int64_t ne10, int64_t ne11, int64_t ne12, size_t nb10, size_t nb11, size_t nb12,\n size_t nb1, size_t nb2, size_t nb3,\n cudaStream_t stream) {\n switch (dst_type) {\n case GGML_TYPE_F32:\n ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (float *) dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_I32:\n ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (int32_t *) dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_F16:\n ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (half *) dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n case GGML_TYPE_BF16:\n ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (nv_bfloat16 *) dst_d,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n break;\n default:\n GGML_ABORT(\"%s: unsupported dst type: %s\\n\", __func__, ggml_type_name(dst_type));\n break;\n }\n}\n\nvoid ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n cudaStream_t stream = ctx.stream();\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n GGML_ASSERT(src1->type == GGML_TYPE_I32);\n GGML_ASSERT(ne13 == 1);\n\n GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));\n GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));\n GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));\n\n get_rows_cuda(src0->data, src0->type, (const int32_t *) src1->data, dst->data, dst->type,\n ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);\n}\n\nvoid ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0]; // gradients of forward pass output\n const ggml_tensor * src1 = dst->src[1]; // src1 in forward pass\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const float * src0_d = (const float *) src0->data;\n const int32_t * src1_d = (const int32_t *) src1->data;\n float * dst_d = (float *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_I32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src1));\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n GGML_ASSERT(ne02*ne03 == 1);\n GGML_ASSERT(ne12*ne13 == 1);\n GGML_ASSERT(ne2*ne3 == 1);\n\n const dim3 block_dims(CUDA_GET_ROWS_BACK_BLOCK_SIZE, 1, 1);\n const int block_num_x = (ne00 + CUDA_GET_ROWS_BACK_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BACK_BLOCK_SIZE;\n const dim3 block_nums(block_num_x, ne1, 1);\n\n k_get_rows_back_float<<>>(src0_d, src1_d, dst_d, ne00, ne10);\n}\n" }, { "path": "src/ggml-cuda/getrows.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_GET_ROWS_BLOCK_SIZE 256\n#define CUDA_GET_ROWS_BACK_BLOCK_SIZE 256\n\nvoid get_rows_cuda(\n const void * src0_d, ggml_type src0_type, const int32_t * src1_d, void * dst_d, ggml_type dst_type,\n int64_t ne00, size_t nb01, size_t nb02, size_t nb03,\n int64_t ne10, int64_t ne11, int64_t ne12, size_t nb10, size_t nb11, size_t nb12,\n size_t nb1, size_t nb2, size_t nb3,\n cudaStream_t stream);\n\nvoid ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/ggml-cuda.cu", "content": "#include \"ggml-cuda.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-cuda/common.cuh\"\n#include \"ggml-cuda/acc.cuh\"\n#include \"ggml-cuda/add-id.cuh\"\n#include \"ggml-cuda/arange.cuh\"\n#include \"ggml-cuda/argmax.cuh\"\n#include \"ggml-cuda/argsort.cuh\"\n#include \"ggml-cuda/binbcast.cuh\"\n#include \"ggml-cuda/clamp.cuh\"\n#include \"ggml-cuda/concat.cuh\"\n#include \"ggml-cuda/conv-transpose-1d.cuh\"\n#include \"ggml-cuda/conv2d.cuh\"\n#include \"ggml-cuda/conv2d-dw.cuh\"\n#include \"ggml-cuda/conv2d-transpose.cuh\"\n#include \"ggml-cuda/convert.cuh\"\n#include \"ggml-cuda/count-equal.cuh\"\n#include \"ggml-cuda/cpy.cuh\"\n#include \"ggml-cuda/cross-entropy-loss.cuh\"\n#include \"ggml-cuda/cumsum.cuh\"\n#include \"ggml-cuda/diagmask.cuh\"\n#include \"ggml-cuda/diag.cuh\"\n#include \"ggml-cuda/fattn.cuh\"\n#include \"ggml-cuda/getrows.cuh\"\n#include \"ggml-cuda/im2col.cuh\"\n#include \"ggml-cuda/mmf.cuh\"\n#include \"ggml-cuda/mmq.cuh\"\n#include \"ggml-cuda/mmvf.cuh\"\n#include \"ggml-cuda/mmvq.cuh\"\n#include \"ggml-cuda/norm.cuh\"\n#include \"ggml-cuda/opt-step-adamw.cuh\"\n#include \"ggml-cuda/opt-step-sgd.cuh\"\n#include \"ggml-cuda/out-prod.cuh\"\n#include \"ggml-cuda/pad.cuh\"\n#include \"ggml-cuda/pool2d.cuh\"\n#include \"ggml-cuda/quantize.cuh\"\n#include \"ggml-cuda/rope.cuh\"\n#include \"ggml-cuda/roll.cuh\"\n#include \"ggml-cuda/scale.cuh\"\n#include \"ggml-cuda/softcap.cuh\"\n#include \"ggml-cuda/softmax.cuh\"\n#include \"ggml-cuda/ssm-conv.cuh\"\n#include \"ggml-cuda/ssm-scan.cuh\"\n#include \"ggml-cuda/sum.cuh\"\n#include \"ggml-cuda/sumrows.cuh\"\n#include \"ggml-cuda/top-k.cuh\"\n#include \"ggml-cuda/mean.cuh\"\n#include \"ggml-cuda/tsembd.cuh\"\n#include \"ggml-cuda/topk-moe.cuh\"\n#include \"ggml-cuda/unary.cuh\"\n#include \"ggml-cuda/upscale.cuh\"\n#include \"ggml-cuda/wkv.cuh\"\n#include \"ggml-cuda/gla.cuh\"\n#include \"ggml-cuda/gated_delta_net.cuh\"\n#include \"ggml-cuda/set.cuh\"\n#include \"ggml-cuda/set-rows.cuh\"\n#include \"ggml-cuda/pad_reflect_1d.cuh\"\n#include \"ggml-cuda/solve_tri.cuh\"\n#include \"ggml-cuda/tri.cuh\"\n#include \"ggml-cuda/cumsum.cuh\"\n#include \"ggml-cuda/fill.cuh\"\n#include \"ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nstatic_assert(sizeof(half) == sizeof(ggml_fp16_t), \"wrong fp16 size\");\n\n[[noreturn]]\nvoid ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {\n int id = -1; // in case cudaGetDevice fails\n (void)cudaGetDevice(&id);\n\n GGML_LOG_ERROR(GGML_CUDA_NAME \" error: %s\\n\", msg);\n GGML_LOG_ERROR(\" current device: %d, in function %s at %s:%d\\n\", id, func, file, line);\n GGML_LOG_ERROR(\" %s\\n\", stmt);\n // abort with GGML_ABORT to get a stack trace\n GGML_ABORT(GGML_CUDA_NAME \" error\");\n}\n\n// this is faster on Windows\n// probably because the Windows CUDA libraries forget to make this check before invoking the drivers\nvoid ggml_cuda_set_device(int device) {\n int current_device;\n CUDA_CHECK(cudaGetDevice(¤t_device));\n\n if (device == current_device) {\n return;\n }\n\n CUDA_CHECK(cudaSetDevice(device));\n}\n\nint ggml_cuda_get_device() {\n int id;\n CUDA_CHECK(cudaGetDevice(&id));\n return id;\n}\n\nstatic cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) {\n ggml_cuda_set_device(device);\n cudaError_t err;\n if (getenv(\"GGML_CUDA_ENABLE_UNIFIED_MEMORY\") != nullptr) {\n err = cudaMallocManaged(ptr, size);\n#if defined(GGML_USE_HIP)\n if (err == hipSuccess) {\n // hipMemAdviseSetCoarseGrain is an optional performance hint;\n // ignore errors (e.g. hipErrorInvalidValue on some APU/iGPU configs).\n cudaMemAdvise(*ptr, size, hipMemAdviseSetCoarseGrain, device);\n (void)hipGetLastError(); // clear any error\n }\n\n // fall back to cudaMalloc if not supported (e.g. on Windows)\n if (err == hipErrorNotSupported) {\n static bool warned_unsupported = false;\n if (!warned_unsupported) {\n GGML_LOG_WARN(\"hipMallocManaged unsupported, falling back to hipMalloc.\\n\");\n warned_unsupported = true;\n }\n\n err = cudaMalloc(ptr, size);\n }\n#endif // defined(GGML_USE_HIP)\n } else {\n err = cudaMalloc(ptr, size);\n }\n return err;\n}\n\n#if defined(GGML_USE_HIP)\nstatic int ggml_cuda_parse_id(char devName[]) {\n // A list of possible Target IDs can be found under the rocclr/clr repo in device.cpp\n // these values are not stable so this is susceptible to breakage\n // https://github.com/ROCm/clr/blob/amd-staging/rocclr/device/device.cpp\n int archMajor = 0x0;\n int archMinor = 0x0;\n int archNum = GGML_CUDA_CC_OFFSET_AMD;\n int archLen = strlen(devName);\n char archName[archLen + 1];\n\n // strip leading 'gfx' while copying into our buffer\n if (archLen > 3) {\n strcpy(archName, &devName[3]);\n archLen -= 3;\n }\n\n // trim trailing :xnack- or :sramecc- statuses\n archLen = strcspn(archName, \":\");\n archName[archLen] = '\\0';\n\n // tease out the version information\n if (archLen > 8) {\n // versions labeled generic use '-' as delimiter\n // strip the trailing \"-generic\" then iterate through what remains\n if ((strstr(archName, \"-generic\"))) {\n archName[archLen - 8] = '\\0';\n char * pch;\n if ((pch = strtok(archName, \"-\"))) {\n archMajor = (int)strtoul(pch, 0, 16);\n if ((pch = strtok(NULL, \"-\"))) {\n archMinor = 0x10 * (int)strtoul(pch, 0, 16);\n }\n }\n }\n } else if (archLen >= 3) {\n // last two digits should be the minor * 0x10 + stepping\n archMinor = (int)strtoul(&archName[archLen - 2], 0, 16);\n archName[archLen - 2] = '\\0';\n\n // only the major version remains\n archMajor = (int)strtoul(archName, 0, 16);\n }\n archNum += archMajor * 0x100;\n archNum += archMinor;\n return archNum;\n}\n#endif // defined(GGML_USE_HIP)\n\nstatic ggml_cuda_device_info ggml_cuda_init() {\n ggml_cuda_device_info info = {};\n\n cudaError_t err = cudaGetDeviceCount(&info.device_count);\n if (err != cudaSuccess) {\n GGML_LOG_ERROR(\"%s: failed to initialize \" GGML_CUDA_NAME \": %s\\n\", __func__, cudaGetErrorString(err));\n return info;\n }\n\n GGML_ASSERT(info.device_count <= GGML_CUDA_MAX_DEVICES);\n\n int64_t total_vram = 0;\n for (int id = 0; id < info.device_count; ++id) {\n cudaDeviceProp prop;\n CUDA_CHECK(cudaGetDeviceProperties(&prop, id));\n total_vram += prop.totalGlobalMem;\n }\n GGML_LOG_INFO(\"%s: found %d \" GGML_CUDA_NAME \" devices (Total VRAM: %zu MiB):\\n\",\n __func__, info.device_count, (size_t)(total_vram / (1024 * 1024)));\n total_vram = 0;\n\n std::vector> turing_devices_without_mma;\n for (int id = 0; id < info.device_count; ++id) {\n int device_vmm = 0;\n\n#if defined(GGML_USE_VMM)\n CUdevice device;\n CU_CHECK(cuDeviceGet(&device, id));\n CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device));\n\n if (device_vmm) {\n CUmemAllocationProp alloc_prop = {};\n alloc_prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;\n alloc_prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;\n alloc_prop.location.id = id;\n CU_CHECK(cuMemGetAllocationGranularity(&info.devices[id].vmm_granularity, &alloc_prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));\n }\n#endif // defined(GGML_USE_VMM)\n info.devices[id].vmm = !!device_vmm;\n\n cudaDeviceProp prop;\n CUDA_CHECK(cudaGetDeviceProperties(&prop, id));\n\n info.default_tensor_split[id] = total_vram;\n total_vram += prop.totalGlobalMem;\n info.devices[id].integrated = false; // Temporarily disabled due to issues with corrupted output (e.g. #15034)\n info.devices[id].nsm = prop.multiProcessorCount;\n info.devices[id].smpb = prop.sharedMemPerBlock;\n info.devices[id].warp_size = prop.warpSize;\n\n#ifndef GGML_USE_MUSA\n int supports_coop_launch = 0;\n CUDA_CHECK(cudaDeviceGetAttribute(&supports_coop_launch, cudaDevAttrCooperativeLaunch, id));\n info.devices[id].supports_cooperative_launch = !!supports_coop_launch;\n#else\n info.devices[id].supports_cooperative_launch = false;\n#endif // !(GGML_USE_MUSA)\n\n#if defined(GGML_USE_HIP)\n info.devices[id].smpbo = prop.sharedMemPerBlock;\n\n info.devices[id].cc = ggml_cuda_parse_id(prop.gcnArchName);\n if ((info.devices[id].cc & 0xff00) == 0x0) {\n GGML_LOG_WARN(\"invalid architecture ID received for device %d %s: %s cc %d.%d\\n\",\n id, prop.name, prop.gcnArchName, prop.major, prop.minor);\n\n // Fallback to prop.major and prop.minor\n if (prop.major > 0) {\n info.devices[id].cc = GGML_CUDA_CC_OFFSET_AMD + prop.major * 0x100;\n info.devices[id].cc += prop.minor * 0x10;\n }\n }\n GGML_LOG_INFO(\" Device %d: %s, %s (0x%x), VMM: %s, Wave Size: %d, VRAM: %zu MiB\\n\",\n id, prop.name, prop.gcnArchName, info.devices[id].cc & 0xffff,\n device_vmm ? \"yes\" : \"no\", prop.warpSize,\n (size_t)(prop.totalGlobalMem / (1024 * 1024)));\n#elif defined(GGML_USE_MUSA)\n // FIXME: Ensure compatibility with varying warp sizes across different MUSA archs.\n info.devices[id].warp_size = 32;\n info.devices[id].smpbo = prop.sharedMemPerBlockOptin;\n info.devices[id].cc = GGML_CUDA_CC_OFFSET_MTHREADS + prop.major * 0x100;\n info.devices[id].cc += prop.minor * 0x10;\n GGML_LOG_INFO(\" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\\n\",\n id, prop.name, prop.major, prop.minor, device_vmm ? \"yes\" : \"no\",\n (size_t)(prop.totalGlobalMem / (1024 * 1024)));\n#else\n info.devices[id].smpbo = prop.sharedMemPerBlockOptin;\n info.devices[id].cc = 100*prop.major + 10*prop.minor;\n GGML_LOG_INFO(\" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\\n\",\n id, prop.name, prop.major, prop.minor, device_vmm ? \"yes\" : \"no\",\n (size_t)(prop.totalGlobalMem / (1024 * 1024)));\n std::string device_name(prop.name);\n if (device_name == \"NVIDIA GeForce MX450\") {\n turing_devices_without_mma.push_back({ id, device_name });\n } else if (device_name == \"NVIDIA GeForce MX550\") {\n turing_devices_without_mma.push_back({ id, device_name });\n } else if (device_name.substr(0, 21) == \"NVIDIA GeForce GTX 16\") {\n turing_devices_without_mma.push_back({ id, device_name });\n }\n\n // Temporary performance fix:\n // Setting device scheduling strategy for iGPUs with cc121 to \"spinning\" to avoid delays in cuda synchronize calls.\n // TODO: Check for future drivers the default scheduling strategy and\n // remove this call again when cudaDeviceScheduleSpin is default.\n if (prop.major == 12 && prop.minor == 1) {\n CUDA_CHECK(cudaSetDevice(id));\n CUDA_CHECK(cudaSetDeviceFlags(cudaDeviceScheduleSpin));\n }\n\n#endif // defined(GGML_USE_HIP)\n }\n\n if (ggml_cuda_highest_compiled_arch(GGML_CUDA_CC_TURING) >= GGML_CUDA_CC_TURING && !turing_devices_without_mma.empty()) {\n GGML_LOG_INFO(\"The following devices will have suboptimal performance due to a lack of tensor cores:\\n\");\n for (size_t device_pos = 0; device_pos < turing_devices_without_mma.size(); device_pos++) {\n GGML_LOG_INFO(\n \" Device %d: %s\\n\", turing_devices_without_mma[device_pos].first, turing_devices_without_mma[device_pos].second.c_str());\n }\n GGML_LOG_INFO(\n \"Consider compiling with CMAKE_CUDA_ARCHITECTURES=61-virtual;80-virtual and DGGML_CUDA_FORCE_MMQ to force the use of the Pascal code for Turing.\\n\");\n }\n\n for (int id = 0; id < info.device_count; ++id) {\n info.default_tensor_split[id] /= total_vram;\n }\n\n // configure logging to stdout\n // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));\n\n return info;\n}\n\nconst ggml_cuda_device_info & ggml_cuda_info() {\n static ggml_cuda_device_info info = ggml_cuda_init();\n return info;\n}\n\n// #define DEBUG_CUDA_MALLOC\n\n// buffer pool for cuda (legacy)\nstruct ggml_cuda_pool_leg : public ggml_cuda_pool {\n static const int MAX_BUFFERS = 256;\n\n int device;\n struct ggml_cuda_buffer {\n void * ptr = nullptr;\n size_t size = 0;\n };\n\n ggml_cuda_buffer buffer_pool[MAX_BUFFERS] = {};\n size_t pool_size = 0;\n\n explicit ggml_cuda_pool_leg(int device) :\n device(device) {\n }\n\n ~ggml_cuda_pool_leg() {\n ggml_cuda_set_device(device);\n for (int i = 0; i < MAX_BUFFERS; ++i) {\n ggml_cuda_buffer & b = buffer_pool[i];\n if (b.ptr != nullptr) {\n CUDA_CHECK(cudaFree(b.ptr));\n pool_size -= b.size;\n }\n }\n GGML_ASSERT(pool_size == 0);\n }\n\n void * alloc(size_t size, size_t * actual_size) override {\n#ifdef DEBUG_CUDA_MALLOC\n int nnz = 0;\n size_t max_size = 0;\n#endif\n size_t best_diff = 1ull << 36;\n int ibest = -1;\n for (int i = 0; i < MAX_BUFFERS; ++i) {\n ggml_cuda_buffer& b = buffer_pool[i];\n if (b.ptr != nullptr) {\n#ifdef DEBUG_CUDA_MALLOC\n ++nnz;\n if (b.size > max_size) max_size = b.size;\n#endif\n if (b.size >= size) {\n size_t diff = b.size - size;\n if (diff < best_diff) {\n best_diff = diff;\n ibest = i;\n if (!best_diff) {\n void * ptr = b.ptr;\n *actual_size = b.size;\n b.ptr = nullptr;\n b.size = 0;\n return ptr;\n }\n }\n }\n }\n }\n if (ibest >= 0) {\n ggml_cuda_buffer& b = buffer_pool[ibest];\n void * ptr = b.ptr;\n *actual_size = b.size;\n b.ptr = nullptr;\n b.size = 0;\n return ptr;\n }\n void * ptr;\n size_t look_ahead_size = (size_t) (1.05 * size);\n look_ahead_size = 256 * ((look_ahead_size + 255)/256);\n ggml_cuda_set_device(device);\n CUDA_CHECK(ggml_cuda_device_malloc(&ptr, look_ahead_size, device));\n *actual_size = look_ahead_size;\n pool_size += look_ahead_size;\n#ifdef DEBUG_CUDA_MALLOC\n GGML_LOG_INFO(\"%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\\n\", __func__, device, nnz,\n (uint32_t)(max_size / 1024 / 1024), (uint32_t)(pool_size / 1024 / 1024), (uint32_t)(size / 1024 / 1024));\n#endif\n return ptr;\n }\n\n void free(void * ptr, size_t size) override {\n for (int i = 0; i < MAX_BUFFERS; ++i) {\n ggml_cuda_buffer& b = buffer_pool[i];\n if (b.ptr == nullptr) {\n b.ptr = ptr;\n b.size = size;\n return;\n }\n }\n GGML_LOG_DEBUG(GGML_CUDA_NAME \" buffer pool full, increase MAX_CUDA_BUFFERS\\n\");\n ggml_cuda_set_device(device);\n CUDA_CHECK(cudaFree(ptr));\n pool_size -= size;\n }\n};\n\n// pool with virtual memory\n#if defined(GGML_USE_VMM)\nstruct ggml_cuda_pool_vmm : public ggml_cuda_pool {\n static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB\n\n int device;\n CUdeviceptr pool_addr = 0;\n size_t pool_used = 0;\n size_t pool_size = 0;\n size_t granularity;\n#if defined(GGML_USE_HIP)\n std::vector> mappings;\n#endif\n\n explicit ggml_cuda_pool_vmm(int device) :\n device(device),\n granularity(ggml_cuda_info().devices[device].vmm_granularity) {\n }\n\n ~ggml_cuda_pool_vmm() {\n if (pool_addr != 0) {\n#if defined(GGML_USE_HIP)\n // Workaround for https://github.com/ROCm/ROCR-Runtime/issues/285\n for (std::pair & mapping : mappings) {\n CU_CHECK(cuMemUnmap(mapping.first, mapping.second));\n }\n#else\n CU_CHECK(cuMemUnmap(pool_addr, pool_size));\n#endif\n CU_CHECK(cuMemAddressFree(pool_addr, CUDA_POOL_VMM_MAX_SIZE));\n }\n }\n\n void * alloc(size_t size, size_t * actual_size) override {\n // round up the allocation size to the alignment to ensure that all allocations are aligned for all data types\n const size_t alignment = 128;\n size = alignment * ((size + alignment - 1) / alignment);\n\n size_t avail = pool_size - pool_used;\n\n if (size > avail) {\n // round up to the next multiple of the granularity\n size_t reserve_size = size - avail;\n reserve_size = granularity * ((reserve_size + granularity - 1) / granularity);\n\n GGML_ASSERT(pool_size + reserve_size <= CUDA_POOL_VMM_MAX_SIZE);\n\n // allocate more physical memory\n CUmemAllocationProp prop = {};\n prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;\n prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;\n prop.location.id = device;\n CUmemGenericAllocationHandle handle;\n CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0));\n\n // reserve virtual address space (if not already reserved)\n if (pool_addr == 0) {\n CU_CHECK(cuMemAddressReserve(&pool_addr, CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0));\n }\n\n // map at the end of the pool\n CUdeviceptr start_ptr = (CUdeviceptr)((char *)(pool_addr) + pool_size);\n CU_CHECK(cuMemMap(start_ptr, reserve_size, 0, handle, 0));\n#if defined(GGML_USE_HIP)\n mappings.push_back({start_ptr, reserve_size});\n#endif\n\n // the memory allocation handle is no longer needed after mapping\n CU_CHECK(cuMemRelease(handle));\n\n // set access\n CUmemAccessDesc access = {};\n access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;\n access.location.id = device;\n access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;\n CU_CHECK(cuMemSetAccess((CUdeviceptr)((char *)(pool_addr) + pool_size), reserve_size, &access, 1));\n\n // add to the pool\n pool_size += reserve_size;\n\n //printf(\"cuda pool[%d]: size increased to %llu MB (reserved %llu MB)\\n\",\n // device, (unsigned long long) (pool_size/1024/1024),\n // (unsigned long long) (reserve_size/1024/1024));\n }\n\n GGML_ASSERT(pool_addr != 0);\n\n void * ptr = (void *) ((CUdeviceptr)((char *)(pool_addr) + pool_used));\n *actual_size = size;\n pool_used += size;\n\n#ifdef DEBUG_CUDA_MALLOC\n printf(\"cuda pool[%d]: allocated %llu bytes at %llx\\n\", device, (unsigned long long) size, ptr);\n#endif\n\n return ptr;\n }\n\n void free(void * ptr, size_t size) override {\n#ifdef DEBUG_CUDA_MALLOC\n printf(\"cuda pool[%d]: freed %llu bytes at %llx\\n\", device, (unsigned long long) size, ptr);\n#endif\n\n pool_used -= size;\n\n // all deallocations must be in reverse order of the allocations\n GGML_ASSERT(ptr == (void *) ((char *)(pool_addr) + pool_used));\n }\n};\n#endif // defined(GGML_USE_VMM)\n\nstd::unique_ptr ggml_backend_cuda_context::new_pool_for_device(int device,\n [[maybe_unused]] int stream_no) {\n#if defined(GGML_USE_VMM)\n if (ggml_cuda_info().devices[device].vmm) {\n return std::unique_ptr(new ggml_cuda_pool_vmm(device));\n }\n#endif // defined(GGML_USE_VMM)\n return std::unique_ptr(new ggml_cuda_pool_leg(device));\n}\n\n// destroying a cuBLAS handle while a graph is being captured in a different thread can result in a CUDA error\n// this lock is used to ensure that no cuBLAS handle is destroyed while a graph is being captured\n\nstatic std::mutex ggml_cuda_lock;\nstatic std::condition_variable ggml_cuda_lock_cv;\nstatic std::atomic ggml_cuda_lock_counter;\n\nggml_backend_cuda_context::~ggml_backend_cuda_context() {\n std::unique_lock lock(ggml_cuda_lock);\n ggml_cuda_lock_cv.wait(lock, []{ return ggml_cuda_lock_counter.load(std::memory_order_relaxed) == 0; });\n\n if (copy_event != nullptr) {\n CUDA_CHECK(cudaEventDestroy(copy_event));\n }\n for (int i = 0; i < GGML_CUDA_MAX_DEVICES; ++i) {\n for (int j = 0; j < GGML_CUDA_MAX_STREAMS; ++j) {\n if (streams[i][j] != nullptr) {\n CUDA_CHECK(cudaStreamDestroy(streams[i][j]));\n }\n }\n if (cublas_handles[i] != nullptr) {\n CUBLAS_CHECK(cublasDestroy(cublas_handles[i]));\n }\n }\n}\n\n\n// cuda buffer\n\nstruct ggml_backend_cuda_buffer_context {\n int device;\n void * dev_ptr = nullptr;\n std::string name;\n\n ggml_backend_cuda_buffer_context(int device, void * dev_ptr) :\n device(device), dev_ptr(dev_ptr),\n name(GGML_CUDA_NAME + std::to_string(device)) {\n }\n\n ~ggml_backend_cuda_buffer_context() {\n CUDA_CHECK(cudaFree(dev_ptr));\n }\n};\n\nstatic void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;\n delete ctx;\n}\n\nstatic bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {\n return buffer->iface.free_buffer == ggml_backend_cuda_buffer_free_buffer;\n}\n\nstatic void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {\n ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;\n return ctx->dev_ptr;\n}\n\nstatic enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {\n ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;\n\n if (tensor->view_src != NULL) {\n assert(tensor->view_src->buffer->buft == buffer->buft);\n return GGML_STATUS_SUCCESS;\n }\n\n if (ggml_is_quantized(tensor->type) && tensor->view_src == nullptr && ggml_backend_buffer_get_usage(buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) {\n // initialize padding to 0 to avoid possible NaN values\n const size_t original_size = ggml_nbytes(tensor);\n const size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);\n\n if (padded_size > original_size) {\n ggml_cuda_set_device(ctx->device);\n CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size));\n }\n }\n return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {\n ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;\n\n ggml_cuda_set_device(ctx->device);\n CUDA_CHECK(cudaMemsetAsync((char *)tensor->data + offset, value, size, cudaStreamPerThread));\n CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));\n}\n\nstatic void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;\n\n ggml_cuda_set_device(ctx->device);\n CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cudaStreamPerThread));\n CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));\n}\n\nstatic void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;\n\n ggml_cuda_set_device(ctx->device);\n CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cudaStreamPerThread));\n CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));\n}\n\nstatic bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {\n if (ggml_backend_buffer_is_cuda(src->buffer)) {\n ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;\n ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;\n if (src_ctx->device == dst_ctx->device) {\n CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice, cudaStreamPerThread));\n } else {\n#ifdef GGML_CUDA_NO_PEER_COPY\n return false;\n#else\n CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_ctx->device, src->data, src_ctx->device, ggml_nbytes(src), cudaStreamPerThread));\n#endif\n }\n CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));\n return true;\n }\n return false;\n\n GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;\n\n ggml_cuda_set_device(ctx->device);\n CUDA_CHECK(cudaMemsetAsync(ctx->dev_ptr, value, buffer->size, cudaStreamPerThread));\n CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));\n}\n\nstatic const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {\n /* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,\n /* .get_base = */ ggml_backend_cuda_buffer_get_base,\n /* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor,\n /* .memset_tensor = */ ggml_backend_cuda_buffer_memset_tensor,\n /* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor,\n /* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,\n /* .clear = */ ggml_backend_cuda_buffer_clear,\n /* .reset = */ NULL,\n};\n\n// cuda buffer type\nstruct ggml_backend_cuda_buffer_type_context {\n int device;\n std::string name;\n};\n\nstatic const char * ggml_backend_cuda_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;\n\n return ctx->name.c_str();\n}\n\nstatic bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {\n return buft->iface.get_name == ggml_backend_cuda_buffer_type_get_name;\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;\n\n ggml_cuda_set_device(buft_ctx->device);\n\n void * dev_ptr;\n cudaError_t err = ggml_cuda_device_malloc(&dev_ptr, size, buft_ctx->device);\n if (err != cudaSuccess) {\n // clear the error\n (void)cudaGetLastError();\n GGML_LOG_ERROR(\"%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\\n\", __func__, size / 1024.0 / 1024.0, buft_ctx->device, cudaGetErrorString(err));\n return nullptr;\n }\n\n ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr);\n\n return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);\n}\n\nstatic size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n return 128;\n\n GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n size_t size = ggml_nbytes(tensor);\n int64_t ne0 = tensor->ne[0];\n\n if (ggml_is_quantized(tensor->type)) {\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n GGML_ASSERT(tensor->nb[0] == ggml_element_size(tensor));\n size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n }\n\n return size;\n\n GGML_UNUSED(buft);\n}\n\nstatic const ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = {\n /* .get_name = */ ggml_backend_cuda_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,\n /* .get_max_size = */ NULL, // defaults to SIZE_MAX\n /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,\n /* .is_host = */ NULL,\n};\n\nggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n if (device >= ggml_backend_cuda_get_device_count()) {\n return nullptr;\n }\n\n static ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES];\n\n static bool ggml_backend_cuda_buffer_type_initialized = false;\n\n if (!ggml_backend_cuda_buffer_type_initialized) {\n for (int i = 0; i < ggml_backend_cuda_get_device_count(); i++) {\n ggml_backend_cuda_buffer_types[i] = {\n /* .iface = */ ggml_backend_cuda_buffer_type_interface,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cuda_reg(), i),\n /* .context = */ new ggml_backend_cuda_buffer_type_context{i, GGML_CUDA_NAME + std::to_string(i)},\n };\n }\n ggml_backend_cuda_buffer_type_initialized = true;\n }\n\n return &ggml_backend_cuda_buffer_types[device];\n}\n\n// cuda split buffer\n\nstatic int64_t get_row_rounding(const std::array & tensor_split) {\n int64_t row_rounding = 0;\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {\n continue;\n }\n\n const int cc = ggml_cuda_info().devices[id].cc;\n row_rounding = std::max(row_rounding, (int64_t)get_mmq_y_host(cc));\n }\n return row_rounding;\n}\n\nstatic void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array & tensor_split, int id) {\n const int64_t nrows = ggml_nrows(tensor);\n const int64_t rounding = get_row_rounding(tensor_split);\n\n *row_low = id == 0 ? 0 : nrows*tensor_split[id];\n *row_low -= *row_low % rounding;\n\n if (id == ggml_backend_cuda_get_device_count() - 1) {\n *row_high = nrows;\n } else {\n *row_high = nrows*tensor_split[id + 1];\n *row_high -= *row_high % rounding;\n }\n}\n\nstatic size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) {\n static_assert(GGML_MAX_DIMS == 4, \"GGML_MAX_DIMS is not 4 - update this function\");\n\n return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]);\n}\n\nstruct ggml_backend_cuda_split_buffer_type_context {\n int main_device;\n std::array tensor_split;\n std::string name;\n};\n\nstruct ggml_backend_cuda_split_buffer_context {\n ~ggml_backend_cuda_split_buffer_context() {\n for (ggml_tensor_extra_gpu * extra : tensor_extras) {\n for (int id = 0; id < GGML_CUDA_MAX_DEVICES; ++id) {\n for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) {\n if (extra->events[id][is] != nullptr) {\n CUDA_CHECK(cudaEventDestroy(extra->events[id][is]));\n }\n }\n if (extra->data_device[id] != nullptr) {\n CUDA_CHECK(cudaFree(extra->data_device[id]));\n }\n }\n delete extra;\n }\n }\n\n std::vector tensor_extras;\n};\n\n\nstatic void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;\n delete ctx;\n}\n\nstatic void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) {\n // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced\n return (void *)0x1000;\n\n GGML_UNUSED(buffer);\n}\n\nstatic enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {\n GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported\n GGML_ASSERT(ggml_is_contiguous(tensor) && \"split buffers only supported for contiguous tensors\");\n\n ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;\n ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;\n\n const int64_t ne0 = tensor->ne[0];\n\n ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};\n ctx->tensor_extras.push_back(extra);\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n int64_t row_low, row_high;\n get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);\n\n int64_t nrows_split = row_high - row_low;\n if (nrows_split == 0) {\n continue;\n }\n\n size_t size = ggml_nbytes_split(tensor, nrows_split);\n const size_t original_size = size;\n\n // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n\n // FIXME: do not crash if cudaMalloc fails\n // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first\n ggml_cuda_set_device(id);\n char * buf;\n CUDA_CHECK(ggml_cuda_device_malloc((void**)&buf, size, id));\n\n // set padding to 0 to avoid possible NaN values\n if (size > original_size) {\n CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size));\n }\n\n extra->data_device[id] = buf;\n\n for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) {\n CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming));\n }\n }\n tensor->extra = extra;\n return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n // split tensors must always be set in their entirety at once\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(size == ggml_nbytes(tensor));\n GGML_ASSERT(ggml_is_contiguous(tensor) && \"split buffers only supported for contiguous tensors\");\n\n ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;\n\n const int64_t ne0 = tensor->ne[0];\n const size_t nb1 = tensor->nb[1];\n ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n int64_t row_low, row_high;\n get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);\n\n int64_t nrows_split = row_high - row_low;\n if (nrows_split == 0) {\n continue;\n }\n\n const size_t offset_split = row_low*nb1;\n size_t size = ggml_nbytes_split(tensor, nrows_split);\n const size_t original_size = size;\n\n // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n\n const char * buf_host = (const char *)data + offset_split;\n CUDA_CHECK(cudaMemcpyAsync(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice, cudaStreamPerThread));\n }\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));\n }\n}\n\nstatic void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n // split tensors must always be set in their entirety at once\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(size == ggml_nbytes(tensor));\n GGML_ASSERT(ggml_is_contiguous(tensor) && \"split buffers only supported for contiguous tensors\");\n\n ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;\n\n const int64_t ne0 = tensor->ne[0];\n const size_t nb1 = tensor->nb[1];\n ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n int64_t row_low, row_high;\n get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);\n\n int64_t nrows_split = row_high - row_low;\n if (nrows_split == 0) {\n continue;\n }\n\n const size_t offset_split = row_low*nb1;\n size_t size = ggml_nbytes_split(tensor, nrows_split);\n const size_t original_size = size;\n\n // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n\n char * buf_host = (char *)data + offset_split;\n CUDA_CHECK(cudaMemcpyAsync(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost, cudaStreamPerThread));\n }\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));\n }\n}\n\nstatic void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n GGML_UNUSED(buffer);\n GGML_UNUSED(value);\n}\n\nstatic const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {\n /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer,\n /* .get_base = */ ggml_backend_cuda_split_buffer_get_base,\n /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor,\n /* .memset_tensor = */ NULL,\n /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor,\n /* .cpy_tensor = */ NULL,\n /* .clear = */ ggml_backend_cuda_split_buffer_clear,\n /* .reset = */ NULL,\n};\n\n// cuda split buffer type\n\nstatic const char * ggml_backend_cuda_split_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;\n\n return ctx->name.c_str();\n}\n\nstatic bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {\n return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_get_name;\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point\n // instead, we allocate them for each tensor separately in init_tensor\n // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,\n // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct.\n ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context();\n\n return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);\n}\n\nstatic size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n return 128;\n\n GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;\n GGML_ASSERT(ggml_is_contiguous(tensor) && \"split buffers only supported for contiguous tensors\");\n\n size_t total_size = 0;\n\n const int64_t ne0 = tensor->ne[0];\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n int64_t row_low, row_high;\n get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id);\n\n int64_t nrows_split = row_high - row_low;\n if (nrows_split == 0) {\n continue;\n }\n\n total_size += ggml_nbytes_split(tensor, nrows_split);\n\n // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n }\n\n return total_size;\n}\n\nstatic bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) {\n return false;\n\n GGML_UNUSED(buft);\n}\n\nstatic const ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = {\n /* .get_name = */ ggml_backend_cuda_split_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment,\n /* .get_max_size = */ NULL, // defaults to SIZE_MAX\n /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size,\n /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,\n};\n\nggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n static std::map>, struct ggml_backend_buffer_type> buft_map;\n\n std::array tensor_split_arr = {};\n\n bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; });\n if (all_zero) {\n tensor_split_arr = ggml_cuda_info().default_tensor_split;\n } else {\n float split_sum = 0.0f;\n for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {\n tensor_split_arr[i] = split_sum;\n split_sum += tensor_split[i];\n }\n for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {\n tensor_split_arr[i] /= split_sum;\n }\n }\n\n auto it = buft_map.find({main_device, tensor_split_arr});\n if (it != buft_map.end()) {\n return &it->second;\n }\n auto * ctx = new ggml_backend_cuda_split_buffer_type_context{\n main_device,\n tensor_split_arr,\n GGML_CUDA_NAME + std::to_string(main_device) + \"_Split\",\n };\n\n struct ggml_backend_buffer_type buft {\n /* .iface = */ ggml_backend_cuda_split_buffer_type_interface,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cuda_reg(), main_device),\n /* .context = */ ctx,\n };\n\n auto result = buft_map.emplace(std::make_pair(main_device, tensor_split_arr), buft);\n return &result.first->second;\n}\n\n// host buffer type\n\nstatic const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) {\n return GGML_CUDA_NAME \"_Host\";\n\n GGML_UNUSED(buft);\n}\n\nstatic bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {\n return buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;\n}\n\nstatic void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n CUDA_CHECK(cudaFreeHost(buffer->context));\n}\n\nstatic void * ggml_cuda_host_malloc(size_t size) {\n if (getenv(\"GGML_CUDA_NO_PINNED\") != nullptr) {\n return nullptr;\n }\n\n void * ptr = nullptr;\n cudaError_t err = cudaMallocHost((void **) &ptr, size);\n if (err != cudaSuccess) {\n // clear the error\n (void)cudaGetLastError();\n GGML_LOG_DEBUG(\"%s: failed to allocate %.2f MiB of pinned memory: %s\\n\", __func__,\n size / 1024.0 / 1024.0, cudaGetErrorString(err));\n return nullptr;\n }\n\n return ptr;\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n void * ptr = ggml_cuda_host_malloc(size);\n\n if (ptr == nullptr) {\n // fallback to cpu buffer\n return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n }\n\n ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);\n buffer->buft = buft;\n buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer;\n\n return buffer;\n}\n\nggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {\n static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_cuda_host_buffer_type_name,\n /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,\n /* .get_max_size = */ NULL, // defaults to SIZE_MAX\n /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,\n /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,\n },\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cuda_reg(), 0),\n /* .context = */ nullptr,\n };\n\n return &ggml_backend_cuda_buffer_type_host;\n}\n\n//static bool ggml_backend_buffer_is_cuda_host(ggml_backend_buffer_t buffer) {\n// return buffer->buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;\n//}\n\n/// kernels\n\ntypedef void (*ggml_cuda_op_mul_mat_t)(\n ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,\n const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,\n const int64_t src1_padded_row_size, cudaStream_t stream);\n\n#ifndef GGML_CUDA_PEER_MAX_BATCH_SIZE\n#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128\n#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE\n\n#define MUL_MAT_SRC1_COL_STRIDE 128\n\nstatic cudaError_t ggml_cuda_cpy_tensor_2d(\n void * dst, const struct ggml_tensor * src, int64_t i3, int64_t i2, int64_t i1_low, int64_t i1_high, cudaStream_t stream) {\n\n const char * src_ptr = (const char *) src->data;\n char * dst_ptr = (char *) dst;\n\n const int64_t ne0 = src->ne[0];\n const int64_t nb0 = src->nb[0];\n const int64_t nb1 = src->nb[1];\n const int64_t nb2 = src->nb[2];\n const int64_t nb3 = src->nb[3];\n const enum ggml_type type = src->type;\n const int64_t ts = ggml_type_size(type);\n const int64_t bs = ggml_blck_size(type);\n const int64_t i1_diff = i1_high - i1_low;\n\n const char * x = src_ptr + i1_low*nb1 + i2*nb2 + i3*nb3;\n if (nb0 == ts && nb1 == ts*ne0/bs) {\n return cudaMemcpyAsync(dst_ptr, x, i1_diff*nb1, cudaMemcpyDeviceToDevice, stream);\n } else if (nb0 == ts) {\n return cudaMemcpy2DAsync(dst_ptr, ts*ne0/bs, x, nb1, ts*ne0/bs, i1_diff, cudaMemcpyDeviceToDevice, stream);\n } else {\n for (int64_t i1 = 0; i1 < i1_diff; i1++) {\n const void * rx = (const void *) ((const char *) x + i1*nb1);\n void * rd = (void *) (dst_ptr + i1*ts*ne0/bs);\n // pretend the row is a matrix with cols=1\n cudaError_t r = cudaMemcpy2DAsync(rd, ts/bs, rx, nb0, ts/bs, ne0, cudaMemcpyDeviceToDevice, stream);\n if (r != cudaSuccess) {\n return r;\n }\n }\n return cudaSuccess;\n }\n}\n\nstruct cublas_force_compute_type {\n bool fp32 = false;\n bool fp16 = false;\n};\n\nstatic const cublas_force_compute_type & ggml_cuda_cublas_get_force_compute_type() {\n static const cublas_force_compute_type compute_type = [] {\n cublas_force_compute_type result;\n\n const bool ggml_cuda_force_cublas_compute_32f_env = getenv(\"GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F\") != nullptr;\n const bool ggml_cuda_force_cublas_compute_16f_env = getenv(\"GGML_CUDA_FORCE_CUBLAS_COMPUTE_16F\") != nullptr;\n\n GGML_ASSERT(ggml_cuda_force_cublas_compute_16f_env == false || ggml_cuda_force_cublas_compute_32f_env == false);\n\n if (ggml_cuda_force_cublas_compute_32f_env) {\n GGML_LOG_INFO(\"Detected GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F\\n\");\n result.fp32 = true;\n } else if (ggml_cuda_force_cublas_compute_16f_env) {\n GGML_LOG_INFO(\"Detected GGML_CUDA_FORCE_CUBLAS_COMPUTE_16F\\n\");\n result.fp16 = true;\n }\n\n return result;\n }();\n\n return compute_type;\n}\n\nstatic void ggml_cuda_op_mul_mat_cublas(\n ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,\n const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,\n const int64_t src1_padded_row_size, cudaStream_t stream) {\n\n GGML_ASSERT(src0_dd_i != nullptr);\n GGML_ASSERT(src1_ddf_i != nullptr);\n GGML_ASSERT(dst_dd_i != nullptr);\n\n const int64_t ne00 = src0->ne[0];\n const int64_t ne10 = src1->ne[0];\n\n const int64_t ne0 = dst->ne[0];\n\n const int64_t row_diff = row_high - row_low;\n\n int id = ggml_cuda_get_device();\n\n // the main device has a larger memory buffer to hold the results from all GPUs\n // ldc == nrows of the matrix that cuBLAS writes into\n int64_t ldc = id == ctx.device ? ne0 : row_diff;\n\n const int cc = ggml_cuda_info().devices[id].cc;\n\n const bool supports_bf16 = GGML_CUDA_CC_IS_NVIDIA(cc) || GGML_CUDA_CC_IS_AMD(cc) ||\n (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);\n\n const bool use_fp16 = (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT;\n\n if (supports_bf16 && src0->type == GGML_TYPE_BF16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1]) {\n ggml_cuda_pool_alloc src1_as_bf16(ctx.pool(id));\n if (src1->type != GGML_TYPE_BF16) {\n const to_bf16_cuda_t to_bf16_cuda = ggml_get_to_bf16_cuda(src1->type);\n GGML_ASSERT(to_bf16_cuda != nullptr);\n size_t ne = src1_ncols*ne10;\n src1_as_bf16.alloc(ne);\n to_bf16_cuda(src1_ddf_i, src1_as_bf16.get(), ne, stream);\n }\n const nv_bfloat16 * src1_ptr = src1->type == GGML_TYPE_BF16 ? (const nv_bfloat16 *) src1_ddf_i : src1_as_bf16.get();\n const nv_bfloat16 * src0_ptr = (const nv_bfloat16 *)src0_dd_i;\n ggml_cuda_pool_alloc dst_bf16(ctx.pool(id), row_diff*src1_ncols);\n\n const float alpha_f32 = 1.0f;\n const float beta_f32 = 0.0f;\n\n CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));\n CUBLAS_CHECK(\n cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,\n row_diff, src1_ncols, ne10,\n &alpha_f32, src0_ptr, CUDA_R_16BF, ne00,\n src1_ptr, CUDA_R_16BF, ne10,\n &beta_f32, dst_bf16.get(), CUDA_R_16BF, ldc,\n CUBLAS_COMPUTE_32F,\n CUBLAS_GEMM_DEFAULT_TENSOR_OP));\n\n const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_BF16);\n to_fp32_cuda(dst_bf16.get(), dst_dd_i, row_diff*src1_ncols, stream);\n } else if (fast_fp16_hardware_available(cc) && use_fp16) {\n // convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32\n ggml_cuda_pool_alloc src0_as_f16(ctx.pool(id));\n if (src0->type != GGML_TYPE_F16) {\n const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src0->type);\n GGML_ASSERT(to_fp16_cuda != nullptr);\n size_t ne = row_diff*ne00;\n src0_as_f16.alloc(ne);\n to_fp16_cuda(src0_dd_i, src0_as_f16.get(), ne, stream);\n }\n const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16.get();\n\n ggml_cuda_pool_alloc src1_as_f16(ctx.pool(id));\n if (src1->type != GGML_TYPE_F16) {\n const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);\n GGML_ASSERT(to_fp16_cuda != nullptr);\n size_t ne = src1_ncols*ne10;\n src1_as_f16.alloc(ne);\n to_fp16_cuda(src1_ddf_i, src1_as_f16.get(), ne, stream);\n }\n const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16.get();\n\n CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));\n\n const auto & force_compute_type = ggml_cuda_cublas_get_force_compute_type();\n\n if (!force_compute_type.fp16 && (GGML_CUDA_CC_IS_CDNA(cc)\n || GGML_CUDA_CC_IS_RDNA4(cc)\n || cc == GGML_CUDA_CC_VOLTA\n || force_compute_type.fp32))\n {\n const float alpha = 1.0f;\n const float beta = 0.0f;\n CUBLAS_CHECK(\n cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,\n row_diff, src1_ncols, ne10,\n &alpha, src0_ptr, CUDA_R_16F, ne00,\n src1_ptr, CUDA_R_16F, ne10,\n &beta, dst_dd_i, CUDA_R_32F, ldc,\n CUBLAS_COMPUTE_32F,\n CUBLAS_GEMM_DEFAULT_TENSOR_OP));\n } else {\n ggml_cuda_pool_alloc dst_f16(ctx.pool(id), row_diff*src1_ncols);\n\n const half alpha_f16 = 1.0f;\n const half beta_f16 = 0.0f;\n\n CUBLAS_CHECK(\n cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,\n row_diff, src1_ncols, ne10,\n &alpha_f16, src0_ptr, CUDA_R_16F, ne00,\n src1_ptr, CUDA_R_16F, ne10,\n &beta_f16, dst_f16.get(), CUDA_R_16F, ldc,\n CUBLAS_COMPUTE_16F,\n CUBLAS_GEMM_DEFAULT_TENSOR_OP));\n\n const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);\n to_fp32_cuda(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream);\n }\n } else {\n ggml_cuda_pool_alloc src0_ddq_as_f32(ctx.pool(id));\n ggml_cuda_pool_alloc src1_ddq_as_f32(ctx.pool(id));\n\n if (src0->type != GGML_TYPE_F32) {\n const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src0->type);\n GGML_ASSERT(to_fp32_cuda != nullptr);\n src0_ddq_as_f32.alloc(row_diff*ne00);\n to_fp32_cuda(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream);\n }\n if (src1->type != GGML_TYPE_F32) {\n const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src1->type);\n GGML_ASSERT(to_fp32_cuda != nullptr);\n src1_ddq_as_f32.alloc(src1_ncols*ne10);\n to_fp32_cuda(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream);\n }\n\n const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get();\n const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get();\n\n const float alpha = 1.0f;\n const float beta = 0.0f;\n\n CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));\n CUBLAS_CHECK(\n cublasSgemm(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,\n row_diff, src1_ncols, ne10,\n &alpha, src0_ddf_i, ne00,\n src1_ddf1_i, ne10,\n &beta, dst_dd_i, ldc));\n }\n\n GGML_UNUSED_VARS(dst, src1_ddq_i, src1_padded_row_size);\n}\n\nstatic void ggml_cuda_set_peer_access(const int n_tokens, int main_device) {\n static bool peer_access_enabled = false;\n\n const bool enable_peer_access = n_tokens <= GGML_CUDA_PEER_MAX_BATCH_SIZE;\n\n if (peer_access_enabled == enable_peer_access) {\n return;\n }\n\n#ifdef NDEBUG\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n ggml_cuda_set_device(id);\n CUDA_CHECK(cudaDeviceSynchronize());\n }\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n ggml_cuda_set_device(id);\n\n for (int id_other = 0; id_other < ggml_backend_cuda_get_device_count(); ++id_other) {\n if (id == id_other) {\n continue;\n }\n if (id != main_device && id_other != main_device) {\n continue;\n }\n\n int can_access_peer;\n CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id, id_other));\n if (can_access_peer) {\n if (enable_peer_access) {\n cudaError_t err = cudaDeviceEnablePeerAccess(id_other, 0);\n if (err != cudaErrorPeerAccessAlreadyEnabled) {\n CUDA_CHECK(err);\n } else {\n // reset the error\n (void)cudaGetLastError();\n }\n } else {\n cudaError_t err = cudaDeviceDisablePeerAccess(id_other);\n if (err != cudaErrorPeerAccessNotEnabled) {\n CUDA_CHECK(err);\n } else {\n // reset the error\n (void)cudaGetLastError();\n }\n }\n }\n }\n }\n\n ggml_cuda_set_device(main_device);\n#endif // NDEBUG\n\n peer_access_enabled = enable_peer_access;\n\n GGML_UNUSED(main_device);\n}\n\nstatic cudaError_t ggml_cuda_Memcpy2DPeerAsync(\n void * dst, int dstDevice, size_t dpitch, void * src, int srcDevice, size_t spitch, size_t width, size_t height, cudaStream_t stream) {\n\n#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)\n // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices\n cudaMemcpy3DPeerParms p = {};\n p.dstDevice = dstDevice;\n p.dstPtr = make_cudaPitchedPtr(dst, dpitch, dpitch, height);\n p.srcDevice = srcDevice;\n p.srcPtr = make_cudaPitchedPtr(src, spitch, spitch, height);\n p.extent = make_cudaExtent(width, height, 1);\n return cudaMemcpy3DPeerAsync(&p, stream);\n#else\n // HIP does not support cudaMemcpy3DPeerAsync or vmm pools\n GGML_UNUSED(dstDevice);\n GGML_UNUSED(srcDevice);\n return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, stream);\n#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)\n}\n\nstatic void ggml_cuda_op_mul_mat(\n ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op,\n quantize_cuda_t quantize_src1) {\n\n const int64_t ne00 = src0->ne[0];\n const int64_t ne01 = src0->ne[1];\n const int64_t ne02 = src0->ne[2];\n const int64_t ne03 = src0->ne[3];\n\n const int64_t ne10 = src1->ne[0];\n const int64_t ne11 = src1->ne[1];\n const int64_t ne12 = src1->ne[2];\n const int64_t ne13 = src1->ne[3];\n const int64_t nrows1 = ggml_nrows(src1);\n\n const int64_t ne0 = dst->ne[0];\n const int64_t ne1 = dst->ne[1];\n\n // const int64_t nb10 = src1->nb[0];\n const int64_t nb11 = src1->nb[1];\n const int64_t nb12 = src1->nb[2];\n const int64_t nb13 = src1->nb[3];\n\n const int64_t nb2 = dst->nb[2];\n const int64_t nb3 = dst->nb[3];\n\n ggml_backend_cuda_buffer_context * src1_ctx = (ggml_backend_cuda_buffer_context *) src1->buffer->context;\n ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *) dst->buffer->context;\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1));\n\n GGML_ASSERT(ne12 % ne02 == 0);\n GGML_ASSERT(ne13 % ne03 == 0);\n\n const int64_t i02_divisor = ne12 / ne02;\n const int64_t i03_divisor = ne13 / ne03;\n\n const size_t src0_ts = ggml_type_size(src0->type);\n const size_t src0_bs = ggml_blck_size(src0->type);\n const size_t q8_1_ts = sizeof(block_q8_1);\n const size_t q8_1_bs = QK8_1;\n\n const bool src0_is_contiguous = ggml_is_contiguous(src0);\n const bool src1_is_contiguous = ggml_is_contiguous(src1);\n\n const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING);\n\n const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft);\n GGML_ASSERT(!(split && ne02 > 1));\n GGML_ASSERT(!(split && ne03 > 1));\n GGML_ASSERT(!(split && ne02 < ne12));\n GGML_ASSERT(!(split && ne03 < ne13));\n\n ggml_tensor_extra_gpu * src0_extra = split ? (ggml_tensor_extra_gpu *) src0->extra : nullptr;\n\n\n std::array tensor_split;\n if (split) {\n ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;\n tensor_split = buft_ctx->tensor_split;\n }\n\n struct dev_data {\n int cc;\n\n ggml_cuda_pool_alloc src0_dd_alloc;\n ggml_cuda_pool_alloc src1_ddf_alloc;\n ggml_cuda_pool_alloc src1_ddq_alloc;\n ggml_cuda_pool_alloc dst_dd_alloc;\n\n char * src0_dd = nullptr;\n float * src1_ddf = nullptr; // float\n char * src1_ddq = nullptr; // q8_1\n float * dst_dd = nullptr;\n\n int64_t row_low;\n int64_t row_high;\n };\n\n dev_data dev[GGML_CUDA_MAX_DEVICES];\n\n int used_devices = 0;\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n dev[id].cc = ggml_cuda_info().devices[id].cc;\n\n // by default, use all rows\n dev[id].row_low = 0;\n dev[id].row_high = ne01;\n\n // for multi GPU, get the row boundaries from tensor split\n // and round to mul_mat_q tile sizes\n if (split) {\n const int64_t rounding = get_row_rounding(tensor_split);\n\n if (id != 0) {\n dev[id].row_low = ne01*tensor_split[id];\n if (dev[id].row_low < ne01) {\n dev[id].row_low -= dev[id].row_low % rounding;\n }\n }\n\n if (id != ggml_backend_cuda_get_device_count() - 1) {\n dev[id].row_high = ne01*tensor_split[id + 1];\n if (dev[id].row_high < ne01) {\n dev[id].row_high -= dev[id].row_high % rounding;\n }\n }\n }\n }\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) {\n continue;\n }\n\n used_devices++;\n\n const bool src1_on_device = id == src1_ctx->device;\n const bool dst_on_device = id == dst_ctx->device;\n\n ggml_cuda_set_device(id);\n cudaStream_t stream = ctx.stream(id, 0);\n\n if (src0_is_contiguous) {\n dev[id].src0_dd = split ? (char *) src0_extra->data_device[id] : (char *) src0->data;\n } else {\n // If src0 is not contiguous it will be copied to a temporary buffer.\n // This buffer needs to be cleared entirely because multiple regions will function as padding.\n const size_t nbytes_data = ggml_nbytes(src0);\n const size_t nbytes_padding = ggml_row_size(src0->type, MATRIX_ROW_PADDING - ne00 % MATRIX_ROW_PADDING);\n dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ctx.pool(id), nbytes_data + nbytes_padding);\n CUDA_CHECK(cudaMemsetAsync(dev[id].src0_dd, 0, nbytes_data + nbytes_padding, stream));\n }\n\n // If src0 is on a temporary compute buffer (partial offloading) there may be some padding that needs to be cleared:\n if (ne00 % MATRIX_ROW_PADDING != 0 && ggml_is_quantized(src0->type) && ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && src0->view_src == nullptr) {\n GGML_ASSERT(ggml_is_contiguously_allocated(src0));\n GGML_ASSERT(!src0->view_src);\n const size_t nbytes_data = ggml_row_size(src0->type, (dev[id].row_high - dev[id].row_low)*ne00);\n const size_t nbytes_padding = ggml_row_size(src0->type, MATRIX_ROW_PADDING - ne00 % MATRIX_ROW_PADDING);\n CUDA_CHECK(cudaMemsetAsync(dev[id].src0_dd + nbytes_data, 0, nbytes_padding, stream));\n }\n\n if (src1_on_device && src1_is_contiguous) {\n dev[id].src1_ddf = (float *) src1->data;\n } else {\n dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ctx.pool(id), ggml_nelements(src1));\n }\n\n if (quantize_src1) {\n size_t src_1_ddq_size = nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs;\n if (quantize_src1 == quantize_mmq_q8_1_cuda) {\n src_1_ddq_size += get_mmq_x_max_host(dev[id].cc)*sizeof(block_q8_1_mmq);\n }\n dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(ctx.pool(id), src_1_ddq_size);\n\n if (src1_on_device && src1_is_contiguous) {\n quantize_src1(\n dev[id].src1_ddf, nullptr, dev[id].src1_ddq, src0->type, ne10,\n nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float),\n src1_padded_col_size, ne11, ne12, ne13, stream);\n CUDA_CHECK(cudaGetLastError());\n }\n }\n\n if (dst_on_device) {\n dev[id].dst_dd = (float *) dst->data;\n } else {\n const size_t size_dst_ddf = split ? (dev[id].row_high - dev[id].row_low)*ne1 : ggml_nelements(dst);\n dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(ctx.pool(id), size_dst_ddf);\n }\n }\n\n // if multiple devices are used they need to wait for the main device\n // here an event is recorded that signals that the main device has finished calculating the input data\n if (split && used_devices > 1) {\n ggml_cuda_set_device(ctx.device);\n CUDA_CHECK(cudaEventRecord(src0_extra->events[ctx.device][0], ctx.stream()));\n }\n\n const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11;\n for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) {\n const int64_t is = split ? (src1_col_0/src1_col_stride) % GGML_CUDA_MAX_STREAMS : 0;\n const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride;\n\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) {\n continue;\n }\n\n const bool src1_on_device = id == src1_ctx->device;\n const bool dst_on_device = id == dst_ctx->device;\n const int64_t row_diff = dev[id].row_high - dev[id].row_low;\n\n ggml_cuda_set_device(id);\n cudaStream_t stream = ctx.stream(id, is);\n\n // wait for main GPU data if necessary\n if (split && (id != ctx.device || is != 0)) {\n CUDA_CHECK(cudaStreamWaitEvent(stream, src0_extra->events[ctx.device][0], 0));\n }\n\n for (int64_t i0 = 0; i0 < ne13*ne12; ++i0) {\n const int64_t i03 = i0 / ne12;\n const int64_t i02 = i0 % ne12;\n\n size_t src1_ddq_i_offset = i0*ne11 * src1_padded_col_size*q8_1_ts/q8_1_bs;\n if (quantize_src1 == quantize_mmq_q8_1_cuda) {\n src1_ddq_i_offset += src1_col_0 * sizeof(block_q8_1_mmq);\n } else {\n src1_ddq_i_offset += src1_col_0 * src1_padded_col_size*q8_1_ts/q8_1_bs;\n }\n\n // for split tensors the data begins at i0 == i0_offset_low\n const size_t nbytes_src0_matrix = ne01*ne00*src0_ts / src0_bs;\n char * src0_dd_i = dev[id].src0_dd + ((i03/i03_divisor)*ne02 + (i02/i02_divisor)) * nbytes_src0_matrix;\n float * src1_ddf_i = dev[id].src1_ddf + (i0*ne11 + src1_col_0) * ne10;\n char * src1_ddq_i = dev[id].src1_ddq + src1_ddq_i_offset;\n float * dst_dd_i = dev[id].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff);\n\n // the main device memory buffer can be on VRAM scratch, with space for all partial results\n // in that case an offset on dst_ddf_i is needed\n if (id == ctx.device) {\n dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split\n }\n\n // copy src0, src1 to device if necessary\n if (src1_is_contiguous) {\n if (id != ctx.device) {\n if (quantize_src1) {\n char * src1_ddq_i_source = dev[ctx.device].src1_ddq + src1_ddq_i_offset;\n if (quantize_src1 == quantize_mmq_q8_1_cuda) {\n const size_t pitch = ne11*sizeof(block_q8_1_mmq);\n const size_t width = src1_ncols*sizeof(block_q8_1_mmq);\n const size_t height = src1_padded_col_size/(4*QK8_1);\n CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(src1_ddq_i, id, pitch, src1_ddq_i_source, ctx.device, pitch, width, height, stream));\n } else {\n CUDA_CHECK(cudaMemcpyPeerAsync(\n src1_ddq_i, id, src1_ddq_i_source, ctx.device, src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream));\n }\n } else {\n float * src1_ddf_i_source = (float *) src1->data;\n src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10;\n CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, ctx.device,\n src1_ncols*ne10*sizeof(float), stream));\n }\n }\n } else if (src1_on_device && !src1_is_contiguous) {\n CUDA_CHECK(ggml_cuda_cpy_tensor_2d(\n src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream));\n } else {\n GGML_ABORT(\"fatal error\");\n }\n\n if (quantize_src1 && !src1_is_contiguous) {\n quantize_src1(\n src1_ddf_i, nullptr, src1_ddq_i, src0->type, ne10, ne10, ne11*ne10, ne12*ne11*ne10,\n src1_padded_col_size, src1_ncols, 1, 1, stream);\n CUDA_CHECK(cudaGetLastError());\n }\n\n if (src1_col_0 == 0 && !src0_is_contiguous && i03 % i03_divisor == 0 && i02 % i02_divisor == 0) {\n CUDA_CHECK(ggml_cuda_cpy_tensor_2d(\n src0_dd_i, src0, i03/i03_divisor, i02/i02_divisor, dev[id].row_low, dev[id].row_high, stream));\n }\n\n // do the computation\n op(ctx, src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i,\n dev[id].row_low, dev[id].row_high, src1_ncols, src1_padded_col_size, stream);\n CUDA_CHECK(cudaGetLastError());\n\n // copy dst to host or other device if necessary\n if (!dst_on_device) {\n void * dst_off_device = dst->data;\n if (split) {\n // src0 = weight matrix is saved as a transposed matrix for better memory layout.\n // dst is NOT transposed.\n // The outputs of matrix matrix multiplications can therefore NOT simply be concatenated for >1 GPU.\n // Instead they need to be copied to the correct slice in ne0 = dst row index.\n // If dst is a vector with ne0 == 1 then you don't have to do this but it still produces correct results.\n float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);\n GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));\n dhf_dst_i += src1_col_0*ne0 + dev[id].row_low;\n CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(\n dhf_dst_i, ctx.device, ne0*sizeof(float), dst_dd_i, id, row_diff*sizeof(float), row_diff*sizeof(float), src1_ncols, stream));\n } else {\n float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);\n GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));\n dhf_dst_i += src1_col_0*ne0;\n CUDA_CHECK(cudaMemcpyAsync(dhf_dst_i, dst_dd_i, src1_ncols*ne0*sizeof(float), cudaMemcpyDeviceToDevice, stream));\n }\n }\n\n // add event for the main device to wait on until other device is done\n if (split && (id != ctx.device || is != 0)) {\n CUDA_CHECK(cudaEventRecord(src0_extra->events[id][is], stream));\n }\n }\n }\n }\n\n // main device waits for all other devices to be finished\n if (split && ggml_backend_cuda_get_device_count() > 1) {\n int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE;\n is_max = is_max <= GGML_CUDA_MAX_STREAMS ? is_max : GGML_CUDA_MAX_STREAMS;\n\n ggml_cuda_set_device(ctx.device);\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n if (dev[id].row_low == dev[id].row_high) {\n continue;\n }\n for (int64_t is = 0; is < is_max; ++is) {\n CUDA_CHECK(cudaStreamWaitEvent(ctx.stream(), src0_extra->events[id][is], 0));\n }\n }\n }\n}\n\nstatic __global__ void k_compute_batched_ptrs(\n const void * src0_as_f16, const void * src1_as_f16, char * dst,\n const void ** ptrs_src, void ** ptrs_dst,\n int64_t ne12, int64_t ne13,\n int64_t ne23,\n size_t nb02, size_t nb03,\n size_t nb12, size_t nb13,\n size_t nbd2, size_t nbd3,\n int64_t r2, int64_t r3) {\n const int64_t i13 = blockIdx.x * blockDim.x + threadIdx.x;\n const int64_t i12 = blockIdx.y * blockDim.y + threadIdx.y;\n\n if (i13 >= ne13 || i12 >= ne12) {\n return;\n }\n\n const int64_t i03 = i13 / r3;\n const int64_t i02 = i12 / r2;\n\n ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_as_f16 + i02*nb02 + i03*nb03;\n ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_as_f16 + i12*nb12 + i13*nb13;\n ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3;\n}\n\n// Type traits for mapping ggml types to CUDA/cuBLAS types\ntemplate\nstruct batched_mul_mat_traits;\n\ntemplate<>\nstruct batched_mul_mat_traits {\n using cuda_type = float;\n static inline const cublasComputeType_t compute_type = CUBLAS_COMPUTE_32F;\n static inline const cudaDataType_t data_type = CUDA_R_32F;\n static inline const ggml_type ggml_type_val = GGML_TYPE_F32;\n static inline const float alpha = 1.0f;\n static inline const float beta = 0.0f;\n static inline const void* get_alpha() { static const float val = alpha; return &val; }\n static inline const void* get_beta() { static const float val = beta; return &val; }\n static inline auto get_nc_converter(ggml_type src_type) { return ggml_get_to_fp32_nc_cuda(src_type); }\n};\n\ntemplate<>\nstruct batched_mul_mat_traits {\n using cuda_type = nv_bfloat16;\n static inline const cublasComputeType_t compute_type = CUBLAS_COMPUTE_32F;\n static inline const cudaDataType_t data_type = CUDA_R_16BF;\n static inline const ggml_type ggml_type_val = GGML_TYPE_BF16;\n static inline const float alpha = 1.0f;\n static inline const float beta = 0.0f;\n static inline const void* get_alpha() { static const float val = alpha; return &val; }\n static inline const void* get_beta() { static const float val = beta; return &val; }\n static inline auto get_nc_converter(ggml_type src_type) { return ggml_get_to_bf16_nc_cuda(src_type); }\n};\n\ntemplate<>\nstruct batched_mul_mat_traits {\n using cuda_type = half;\n static inline const cublasComputeType_t compute_type = CUBLAS_COMPUTE_16F;\n static inline const cudaDataType_t data_type = CUDA_R_16F;\n static inline const ggml_type ggml_type_val = GGML_TYPE_F16;\n static inline const half alpha = 1.0;\n static inline const half beta = 0.0;\n static inline const void* get_alpha() { static const half val = alpha; return &val; }\n static inline const void* get_beta() { static const half val = beta; return &val; }\n static inline auto get_nc_converter(ggml_type src_type) { return ggml_get_to_fp16_nc_cuda(src_type); }\n};\n\ntemplate\nstatic void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n using traits = batched_mul_mat_traits;\n using cuda_t = typename traits::cuda_type;\n\n GGML_ASSERT(!ggml_is_transposed(src0));\n GGML_ASSERT(!ggml_is_transposed(src1));\n GGML_ASSERT(!ggml_backend_buft_is_cuda_split(src0->buffer->buft));\n GGML_ASSERT(src0->type == src0_type);\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n // Byte offsets and tensor dimensions are currently used in an inconsistent way for dst.\n // As long as dst is contiguous this does not matter though.\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int64_t ne_dst = ggml_nelements(dst);\n cudaStream_t main_stream = ctx.stream();\n CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream));\n\n float * dst_ddf = (float *) dst->data;\n const size_t ts_src1 = ggml_type_size(src1->type);\n GGML_ASSERT(nb10 == ts_src1);\n int64_t s11 = nb11 / ts_src1;\n int64_t s12 = nb12 / ts_src1;\n int64_t s13 = nb13 / ts_src1;\n\n const cuda_t * src0_ptr = nullptr;\n const cuda_t * src1_ptr = nullptr;\n\n ggml_cuda_pool_alloc src0_alloc(ctx.pool());\n ggml_cuda_pool_alloc src1_alloc(ctx.pool());\n\n bool is_src0_cont_2 = ggml_is_contiguous_2(src0);\n bool is_src1_cont_2 = ggml_is_contiguous_2(src1);\n\n // Handle src0\n src0_ptr = (const cuda_t *) src0->data;\n\n // Handle src1 - convert if necessary\n if (src1->type == src0_type) {\n src1_ptr = (const cuda_t *) src1->data;\n } else {\n // Convert src1 to target type using traits conversion functions\n const int64_t ne_src1 = ggml_nelements(src1);\n src1_alloc.alloc(ne_src1);\n\n const auto convert_func = traits::get_nc_converter(src1->type);\n GGML_ASSERT(convert_func != nullptr);\n convert_func(src1->data, src1_alloc.get(), ne10, ne11, ne12, ne13, s11, s12, s13, main_stream);\n src1_ptr = src1_alloc.get();\n s11 = ne10;\n s12 = ne11*s11;\n s13 = ne12*s12;\n\n is_src1_cont_2 = true;\n }\n\n // Setup destination buffer\n ggml_cuda_pool_alloc dst_temp(ctx.pool());\n char * dst_t;\n size_t nbd2 = dst->nb[2];\n size_t nbd3 = dst->nb[3];\n\n cublasComputeType_t cu_compute_type = traits::compute_type;\n cudaDataType_t cu_data_type = traits::data_type;\n cudaDataType_t cu_data_type_a = traits::data_type;\n cudaDataType_t cu_data_type_b = traits::data_type;\n const void * alpha = traits::get_alpha();\n const void * beta = traits::get_beta();\n\n const auto & force_compute_type = ggml_cuda_cublas_get_force_compute_type();\n\n int id = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[id].cc;\n static constexpr bool is_src0_type_f16 = src0_type == GGML_TYPE_F16;\n\n // bf16 and fp32 are already being computed in fp32 (ensure it using static_assert),\n // so checking necessity of forced fp32 only for fp16 src0_type\n static_assert(is_src0_type_f16 || traits::compute_type == CUBLAS_COMPUTE_32F);\n\n const bool need_compute_32f = is_src0_type_f16 && !force_compute_type.fp16 && (GGML_CUDA_CC_IS_CDNA(cc)\n || GGML_CUDA_CC_IS_RDNA4(cc)\n || cc == GGML_CUDA_CC_VOLTA\n || force_compute_type.fp32);\n\n if (dst->op_params[0] == GGML_PREC_DEFAULT && !need_compute_32f) {\n if constexpr (src0_type == GGML_TYPE_F32) {\n dst_t = (char *) dst_ddf; // Direct F32 output\n } else {\n dst_t = (char *) dst_temp.alloc(ne_dst);\n nbd2 /= sizeof(float) / sizeof(cuda_t);\n nbd3 /= sizeof(float) / sizeof(cuda_t);\n }\n } else {\n dst_t = (char *) dst_ddf;\n cu_compute_type = batched_mul_mat_traits::compute_type;\n cu_data_type = batched_mul_mat_traits::data_type;\n alpha = batched_mul_mat_traits::get_alpha();\n beta = batched_mul_mat_traits::get_beta();\n }\n\n GGML_ASSERT(ne12 % ne02 == 0);\n GGML_ASSERT(ne13 % ne03 == 0);\n\n // broadcast factors\n const int64_t r2 = ne12/ne02;\n const int64_t r3 = ne13/ne03;\n\n if (r2 == 1 && r3 == 1 && is_src0_cont_2 && is_src1_cont_2) {\n // with a [0, 2, 1, 3] perm. and ne02==1 the matrix strides need to be determined from dim 3:\n const int64_t sma = ne02 == 1 ? nb03/nb00 : nb02/nb00;\n const int64_t smb = ne12 == 1 ? s13 : s12;\n\n // there is no broadcast and src0, src1 are contiguous across dims 2, 3\n // use cublasGemmStridedBatchedEx\n CUBLAS_CHECK(\n cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,\n ne01, ne11, ne10,\n alpha, src0_ptr, cu_data_type_a, nb01/nb00, sma, // strideA\n src1_ptr, cu_data_type_b, s11, smb, // strideB\n beta, dst_t, cu_data_type, ne0, ne1*ne0, // strideC\n ne12*ne13,\n cu_compute_type,\n CUBLAS_GEMM_DEFAULT_TENSOR_OP));\n } else {\n // use cublasGemmBatchedEx\n const int64_t ne23 = ne12*ne13;\n\n ggml_cuda_pool_alloc ptrs_src(ctx.pool(), 2*ne23);\n ggml_cuda_pool_alloc< void *> ptrs_dst(ctx.pool(), 1*ne23);\n\n size_t src1_stride_size = sizeof(cuda_t);\n\n const int threads_x = 16;\n const int threads_y = 16;\n dim3 block_dims(threads_x, threads_y);\n\n dim3 grid_dims(\n (ne13 + threads_x - 1) / threads_x,\n (ne12 + threads_y - 1) / threads_y\n );\n k_compute_batched_ptrs<<>>(\n src0_ptr, src1_ptr, dst_t,\n ptrs_src.get(), ptrs_dst.get(),\n ne12, ne13,\n ne23,\n nb02, nb03,\n (src1->type == src0_type) ? nb12 : s12*src1_stride_size,\n (src1->type == src0_type) ? nb13 : s13*src1_stride_size,\n nbd2, nbd3,\n r2, r3);\n\n CUDA_CHECK(cudaGetLastError());\n\n CUBLAS_CHECK(\n cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,\n ne01, ne11, ne10,\n alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, nb01/nb00,\n (const void **) (ptrs_src.get() + 1*ne23), cu_data_type_b, s11,\n beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ne0,\n ne23,\n cu_compute_type,\n CUBLAS_GEMM_DEFAULT_TENSOR_OP));\n }\n\n // Convert output back to F32 if needed\n if (dst->op_params[0] == GGML_PREC_DEFAULT && cu_data_type != CUDA_R_32F) {\n const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(traits::ggml_type_val);\n to_fp32_cuda(dst_temp.get(), dst_ddf, ne_dst, main_stream);\n }\n}\n\nstatic void ggml_cuda_mul_mat_batched_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || src0->type == GGML_TYPE_F32);\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n ggml_cuda_mul_mat_batched_cublas_impl(ctx, src0, src1, dst);\n break;\n case GGML_TYPE_BF16:\n ggml_cuda_mul_mat_batched_cublas_impl(ctx, src0, src1, dst);\n break;\n case GGML_TYPE_F16:\n ggml_cuda_mul_mat_batched_cublas_impl(ctx, src0, src1, dst);\n break;\n default:\n GGML_ABORT(\"Unsupported type\");\n }\n}\n\nstatic bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up,\n const ggml_tensor * ffn_gate,\n const ggml_tensor * glu,\n const ggml_tensor * ffn_up_bias = nullptr,\n const ggml_tensor * ffn_gate_bias = nullptr) {\n const bool has_bias = ffn_up_bias != nullptr || ffn_gate_bias != nullptr;\n\n if (has_bias && (!ffn_up_bias || !ffn_gate_bias)) {\n return false;\n }\n\n const bool is_mul_mat = ffn_up->op == GGML_OP_MUL_MAT && ffn_gate->op == GGML_OP_MUL_MAT && glu->op == GGML_OP_GLU;\n const bool is_mul_mat_id = ffn_up->op == GGML_OP_MUL_MAT_ID && ffn_gate->op == GGML_OP_MUL_MAT_ID && glu->op == GGML_OP_GLU;\n\n GGML_ASSERT(ffn_up && ffn_gate && glu);\n\n if (!is_mul_mat && !is_mul_mat_id) {\n return false;\n }\n\n const ggml_op expected_bias_op = is_mul_mat ? GGML_OP_ADD : GGML_OP_ADD_ID;\n\n if (has_bias) {\n if (ffn_up_bias->op != expected_bias_op || ffn_gate_bias->op != expected_bias_op) {\n return false;\n }\n\n if (glu->src[0] != ffn_gate_bias || glu->src[1] != ffn_up_bias) {\n return false;\n }\n\n if (expected_bias_op == GGML_OP_ADD) {\n const bool up_has_mul = ffn_up_bias->src[0] == ffn_up || ffn_up_bias->src[1] == ffn_up;\n const bool gate_has_mul = ffn_gate_bias->src[0] == ffn_gate || ffn_gate_bias->src[1] == ffn_gate;\n if (!up_has_mul || !gate_has_mul) {\n return false;\n }\n } else { // GGML_OP_ADD_ID\n if (ffn_up_bias->src[0] != ffn_up || ffn_gate_bias->src[0] != ffn_gate) {\n return false;\n }\n if (ffn_up_bias->src[2] != ffn_up->src[2] || ffn_gate_bias->src[2] != ffn_gate->src[2]) {\n return false;\n }\n }\n } else {\n if (glu->src[0] != ffn_gate && glu->src[1] != ffn_up) {\n return false;\n }\n }\n\n if (ffn_up->src[0]->type != ffn_gate->src[0]->type || !ggml_are_same_shape(ffn_up->src[0], ffn_gate->src[0]) ||\n !ggml_are_same_stride(ffn_up->src[0], ffn_gate->src[0])) {\n return false;\n }\n\n if (ffn_up->src[1] != ffn_gate->src[1]) {\n return false;\n }\n\n if (ffn_up->src[2] && (ffn_up->src[2] != ffn_gate->src[2])) {\n return false;\n }\n\n static constexpr std::array valid_glu_ops = { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_OAI };\n\n if (std::find(valid_glu_ops.begin(), valid_glu_ops.end(), ggml_get_glu_op(glu)) == valid_glu_ops.end()) {\n return false;\n }\n\n if (const bool swapped = ggml_get_op_params_i32(glu, 1); swapped) {\n return false;\n }\n\n const bool split = ggml_backend_buft_is_cuda_split(ffn_up->src[0]->buffer->buft) ||\n ggml_backend_buft_is_cuda_split(ffn_gate->src[0]->buffer->buft);\n\n //TODO: add support for fusion for split buffers\n if (split) {\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) {\n ggml_tensor * src0 = tensor->src[0];\n ggml_tensor * src1 = tensor->src[1];\n const ggml_tensor * dst = tensor;\n\n const bool is_mul_mat_id = tensor->op == GGML_OP_MUL_MAT_ID;\n\n bool use_mul_mat_vec_f =\n (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) &&\n src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;\n\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]);\n\n const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft) ||\n ggml_backend_buft_is_cuda_split(src1->buffer->buft);\n\n //TODO: add support for fusion for split buffers\n if (split) {\n return false;\n }\n\n //we only support fusion for ncols_dst = 1\n if (tensor->op == GGML_OP_MUL_MAT && dst->ne[1] != 1) {\n return false;\n }\n\n if (tensor->op == GGML_OP_MUL_MAT_ID && dst->ne[2] != 1) {\n return false;\n }\n\n\n return use_mul_mat_vec_f;\n}\n\nstatic bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) {\n ggml_tensor * src0 = tensor->src[0];\n ggml_tensor * src1 = tensor->src[1];\n const ggml_tensor * dst = tensor;\n\n const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE &&\n ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) &&\n src0->view_src;\n\n bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && !bad_padding_clear && src1->type == GGML_TYPE_F32 &&\n dst->type == GGML_TYPE_F32 && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;\n\n // fusion is not universally faster on Pascal\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n if (cc <= GGML_CUDA_CC_PASCAL) {\n return false;\n }\n //we only support fusion for ncols_dst = 1\n if (tensor->op == GGML_OP_MUL_MAT && dst->ne[1] != 1) {\n return false;\n }\n\n if (tensor->op == GGML_OP_MUL_MAT_ID && dst->ne[2] != 1) {\n return false;\n }\n\n\n const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft) ||\n ggml_backend_buft_is_cuda_split(src1->buffer->buft);\n\n //TODO: add support for fusion for split buffers\n if (split) {\n return false;\n }\n\n return use_mul_mat_vec_q;\n}\n\nstatic void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft);\n\n // If src0 is a temporary compute buffer it may have some padding that needs to be cleared for mul_mat_vec_q or mul_mat_q.\n // But if src0 is also a view of another tensor then this cannot be done safely because it may overwrite valid tensor data.\n // Therefore, in such cases use cuBLAS.\n const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE\n && ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && src0->view_src;\n\n bool use_mul_mat_vec_f = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16)\n && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;\n bool use_mul_mat_f = !ggml_is_quantized(src0->type)\n && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;\n bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && !bad_padding_clear\n && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32\n && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;\n bool use_mul_mat_q = ggml_is_quantized(src0->type) && !bad_padding_clear\n && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;\n\n bool any_gpus_with_slow_fp16 = false;\n\n if (split) {\n ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;\n auto & tensor_split = buft_ctx->tensor_split;\n for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {\n // skip devices that are not going to do any work:\n if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {\n continue;\n }\n\n const int cc = ggml_cuda_info().devices[id].cc;\n const int warp_size = ggml_cuda_info().devices[id].warp_size;\n use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0);\n use_mul_mat_f = use_mul_mat_f && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false);\n use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]);\n any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available(cc);\n }\n } else {\n const int cc = ggml_cuda_info().devices[ctx.device].cc;\n const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size;\n use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0);\n use_mul_mat_f = use_mul_mat_f && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false);\n use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]);\n any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available(cc);\n }\n\n // debug helpers\n //printf(\"src0: %8d %8d %8d %8d\\n\", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]);\n //printf(\" %8d %8d %8d %8d\\n\", src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]);\n //printf(\"src1: %8d %8d %8d %8d\\n\", src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]);\n //printf(\" %8d %8d %8d %8d\\n\", src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]);\n //printf(\"src0 is contiguous %d, transposed %d, type = %s, name = %s\\n\", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name);\n //printf(\"src1 is contiguous %d, transposed %d, type = %s, name = %s\\n\", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name);\n\n //TODO update for generic tensor parallelism\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n bool use_batched_cublas_f16 = src0->type == GGML_TYPE_F16 && (src1->type == GGML_TYPE_F16 || !any_gpus_with_slow_fp16);\n bool use_batched_cublas_bf16 = src0->type == GGML_TYPE_BF16 && bf16_mma_hardware_available(cc);\n bool use_batched_cublas_f32 = src0->type == GGML_TYPE_F32;\n\n if (!split && use_mul_mat_vec_f) {\n // the custom F16 vector kernel can be used over batched cuBLAS GEMM\n // but this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention)\n ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst);\n } else if (!split && use_mul_mat_f) {\n ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst);\n } else if (!split && use_mul_mat_vec_q) {\n ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst);\n } else if (!split && use_mul_mat_q) {\n ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst);\n } else if (!split && (use_batched_cublas_f16 || use_batched_cublas_bf16 || use_batched_cublas_f32)\n && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) {\n // general KQ + KQV multi-batch without FlashAttention\n ggml_cuda_mul_mat_batched_cublas(ctx, src0, src1, dst);\n } else if (use_mul_mat_vec_f) {\n ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_f, nullptr);\n } else if (use_mul_mat_vec_q) {\n ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, quantize_row_q8_1_cuda);\n } else if (use_mul_mat_q) {\n ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, quantize_mmq_q8_1_cuda);\n } else {\n ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, nullptr);\n }\n}\n\nstatic void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const ggml_tensor * ids = dst->src[2];\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n GGML_ASSERT(!ggml_backend_buft_is_cuda_split(src0->buffer->buft) && \"mul_mat_id does not support split buffers\");\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n\n // [TAG_MUL_MAT_ID_CUDA_GRAPHS]\n if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n static_assert(MMVQ_MAX_BATCH_SIZE == MMVF_MAX_BATCH_SIZE);\n if (ne2 <= MMVQ_MAX_BATCH_SIZE) {\n if (ggml_is_quantized(src0->type)) {\n if (ne2 <= MMVQ_MMID_MAX_BATCH_SIZE) {\n ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst);\n return;\n }\n } else {\n if (GGML_CUDA_CC_IS_AMD(cc)) {\n ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);\n return;\n }\n }\n }\n\n if (ggml_cuda_should_use_mmq(src0->type, cc, ne12, /*n_experts=*/ne02)) {\n ggml_cuda_mul_mat_q(ctx, src0, src1, ids, dst);\n return;\n }\n\n if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) {\n ggml_cuda_mul_mat_f(ctx, src0, src1, ids, dst);\n return;\n }\n }\n\n // note: this path should not be reached when recording CUDA graphs, because it requires stream synchronization\n // TODO: add asserts to verify this. should work with CUDA, HIP, etc.\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(nb12 % nb11 == 0);\n GGML_ASSERT(nb2 % nb1 == 0);\n\n const ggml_type type_src1_sorted = (src0->type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc))\n || ggml_is_quantized(src0->type) ? GGML_TYPE_F32 : src0->type;\n const ggml_type type_dst_sorted = GGML_TYPE_F32;\n const size_t ts_src1_sorted = ggml_type_size(type_src1_sorted);\n const size_t ts_dst_sorted = ggml_type_size(type_dst_sorted);\n\n const int64_t n_expert_used = ids->ne[0];\n const int64_t ne_get_rows = ne12 * n_expert_used;\n\n std::vector ids_to_sorted_host;\n ids_to_sorted_host.reserve(2*ne_get_rows);\n std::vector ids_from_sorted_host(ne_get_rows);\n\n ggml_cuda_pool_alloc ids_buf_dev(ctx.pool(), 2*ne_get_rows);\n\n std::vector tokens_per_expert(ne02);\n\n ggml_cuda_pool_alloc src1_sorted(ctx.pool(), ne12*n_expert_used*ne10*ts_src1_sorted);\n ggml_cuda_pool_alloc dst_sorted(ctx.pool(), ne2 *n_expert_used* ne0*ts_dst_sorted);\n\n std::vector ids_host(ggml_nbytes(ids));\n CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids->data, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream));\n CUDA_CHECK(cudaStreamSynchronize(stream));\n\n for (int64_t i02 = 0; i02 < ne02; ++i02) { // expert matrices\n for (int64_t i12 = 0; i12 < ne12; ++i12) { // tokens\n for (int64_t iex = 0; iex < n_expert_used; ++iex) {\n const int32_t expert_to_use = *(const int32_t *)(ids_host.data() + i12*ids->nb[1] + iex*ids->nb[0]);\n assert(expert_to_use >= 0 && expert_to_use < ne02);\n if (expert_to_use == i02) {\n ids_from_sorted_host[i12*n_expert_used + iex] = ids_to_sorted_host.size();\n ids_to_sorted_host.push_back(i12*ne11 + iex % ne11);\n tokens_per_expert[i02]++;\n break;\n }\n }\n }\n }\n GGML_ASSERT(ids_to_sorted_host.size() == size_t(ne_get_rows));\n\n ids_to_sorted_host.insert(ids_to_sorted_host.end(), ids_from_sorted_host.begin(), ids_from_sorted_host.end());\n\n CUDA_CHECK(cudaMemcpyAsync(ids_buf_dev.ptr, ids_to_sorted_host.data(), 2*ne_get_rows*sizeof(int32_t), cudaMemcpyHostToDevice, stream));\n CUDA_CHECK(cudaStreamSynchronize(stream));\n\n const int32_t * ids_to_sorted = ids_buf_dev.ptr + 0*ne_get_rows;\n const int32_t * ids_from_sorted = ids_buf_dev.ptr + 1*ne_get_rows;\n\n get_rows_cuda(src1->data, src1->type, ids_to_sorted, src1_sorted.ptr, type_src1_sorted,\n ne10, nb11, nb12, nb13,\n ne_get_rows, 1, 1, sizeof(int32_t), ne_get_rows*sizeof(int32_t), ne_get_rows*sizeof(int32_t),\n ne10*ts_src1_sorted, ne_get_rows*ne10*ts_src1_sorted, ne_get_rows*ne10*ts_src1_sorted, stream);\n CUDA_CHECK(cudaGetLastError());\n\n char * src1_data_cur = (char *) src1_sorted.ptr;\n char * dst_data_cur = (char *) dst_sorted.ptr;\n for (int64_t i02 = 0; i02 < ne02; ++i02) {\n if (tokens_per_expert[i02] == 0) {\n continue;\n }\n\n ggml_tensor src0_slice = *src0;\n src0_slice.ne[2] = 1;\n src0_slice.nb[3] = src0_slice.nb[2];\n src0_slice.op = GGML_OP_VIEW;\n src0_slice.view_src = dst->src[0]; // non-const pointer to src0\n src0_slice.data = (char *) src0->data + i02*nb02;\n\n ggml_tensor src1_slice;\n memset(&src1_slice, 0, sizeof(src1_slice));\n src1_slice.buffer = src1->buffer;\n src1_slice.type = type_src1_sorted;\n src1_slice.ne[0] = ne10;\n src1_slice.ne[1] = tokens_per_expert[i02];\n src1_slice.ne[2] = 1;\n src1_slice.ne[3] = 1;\n src1_slice.nb[0] = ts_src1_sorted;\n src1_slice.nb[1] = src1_slice.ne[0] * src1_slice.nb[0];\n src1_slice.nb[2] = src1_slice.ne[1] * src1_slice.nb[1];\n src1_slice.nb[3] = src1_slice.ne[2] * src1_slice.nb[2];\n src1_slice.data = src1_data_cur;\n\n ggml_tensor dst_slice;\n memset(&dst_slice, 0, sizeof(dst_slice));\n dst_slice.buffer = dst->buffer;\n dst_slice.type = type_dst_sorted;\n dst_slice.ne[0] = ne0;\n dst_slice.ne[1] = tokens_per_expert[i02];\n dst_slice.ne[2] = 1;\n dst_slice.ne[3] = 1;\n dst_slice.nb[0] = ts_dst_sorted;\n dst_slice.nb[1] = dst_slice.ne[0] * dst_slice.nb[0];\n dst_slice.nb[2] = dst_slice.ne[1] * dst_slice.nb[1];\n dst_slice.nb[3] = dst_slice.ne[2] * dst_slice.nb[2];\n dst_slice.data = dst_data_cur;\n\n ggml_cuda_mul_mat(ctx, &src0_slice, &src1_slice, &dst_slice);\n CUDA_CHECK(cudaGetLastError());\n\n src1_data_cur += src1_slice.nb[2];\n dst_data_cur += dst_slice.nb[2];\n }\n\n get_rows_cuda(dst_sorted.ptr, type_dst_sorted, ids_from_sorted, dst->data, dst->type,\n ne0, ne0*ts_dst_sorted, ne_get_rows*ne0*ts_dst_sorted, ne_get_rows*ne0*ts_dst_sorted,\n ne_get_rows, 1, 1, sizeof(int32_t), ne_get_rows*sizeof(int32_t), ne_get_rows*sizeof(int32_t),\n nb1, nb2, nb3, stream);\n}\n\nstatic bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * dst) {\n // why is this here instead of mul_mat?\n if (dst->src[0] != nullptr && ggml_backend_buft_is_cuda_split(dst->src[0]->buffer->buft)) {\n ggml_cuda_set_peer_access(dst->src[1]->ne[1], ctx.device);\n }\n\n switch (dst->op) {\n case GGML_OP_ARGMAX:\n ggml_cuda_argmax(ctx, dst);\n break;\n case GGML_OP_COUNT_EQUAL:\n ggml_cuda_count_equal(ctx, dst);\n break;\n case GGML_OP_REPEAT:\n ggml_cuda_op_repeat(ctx, dst);\n break;\n case GGML_OP_REPEAT_BACK:\n ggml_cuda_op_repeat_back(ctx, dst);\n break;\n case GGML_OP_GET_ROWS:\n ggml_cuda_op_get_rows(ctx, dst);\n break;\n case GGML_OP_GET_ROWS_BACK:\n ggml_cuda_op_get_rows_back(ctx, dst);\n break;\n case GGML_OP_SET_ROWS:\n ggml_cuda_op_set_rows(ctx, dst);\n break;\n case GGML_OP_SET:\n ggml_cuda_op_set(ctx, dst);\n break;\n case GGML_OP_DUP:\n ggml_cuda_dup(ctx, dst);\n break;\n case GGML_OP_CPY:\n ggml_cuda_cpy(ctx, dst->src[0], dst->src[1]);\n break;\n case GGML_OP_CONT:\n ggml_cuda_dup(ctx, dst);\n break;\n case GGML_OP_ADD:\n case GGML_OP_ADD1: // TODO: more efficient implementation\n ggml_cuda_op_add(ctx, dst);\n break;\n case GGML_OP_ADD_ID:\n ggml_cuda_op_add_id(ctx, dst);\n break;\n case GGML_OP_SUB:\n ggml_cuda_op_sub(ctx, dst);\n break;\n case GGML_OP_ACC:\n ggml_cuda_op_acc(ctx, dst);\n break;\n case GGML_OP_MUL:\n ggml_cuda_op_mul(ctx, dst);\n break;\n case GGML_OP_DIV:\n ggml_cuda_op_div(ctx, dst);\n break;\n case GGML_OP_UNARY:\n switch (ggml_get_unary_op(dst)) {\n case GGML_UNARY_OP_ABS:\n ggml_cuda_op_abs(ctx, dst);\n break;\n case GGML_UNARY_OP_SGN:\n ggml_cuda_op_sgn(ctx, dst);\n break;\n case GGML_UNARY_OP_NEG:\n ggml_cuda_op_neg(ctx, dst);\n break;\n case GGML_UNARY_OP_STEP:\n ggml_cuda_op_step(ctx, dst);\n break;\n case GGML_UNARY_OP_GELU:\n ggml_cuda_op_gelu(ctx, dst);\n break;\n case GGML_UNARY_OP_SILU:\n ggml_cuda_op_silu(ctx, dst);\n break;\n case GGML_UNARY_OP_GELU_ERF:\n ggml_cuda_op_gelu_erf(ctx, dst);\n break;\n case GGML_UNARY_OP_GELU_QUICK:\n ggml_cuda_op_gelu_quick(ctx, dst);\n break;\n case GGML_UNARY_OP_TANH:\n ggml_cuda_op_tanh(ctx, dst);\n break;\n case GGML_UNARY_OP_RELU:\n ggml_cuda_op_relu(ctx, dst);\n break;\n case GGML_UNARY_OP_SIGMOID:\n ggml_cuda_op_sigmoid(ctx, dst);\n break;\n case GGML_UNARY_OP_HARDSIGMOID:\n ggml_cuda_op_hardsigmoid(ctx, dst);\n break;\n case GGML_UNARY_OP_HARDSWISH:\n ggml_cuda_op_hardswish(ctx, dst);\n break;\n case GGML_UNARY_OP_EXP:\n ggml_cuda_op_exp(ctx, dst);\n break;\n case GGML_UNARY_OP_ELU:\n ggml_cuda_op_elu(ctx, dst);\n break;\n case GGML_UNARY_OP_XIELU:\n ggml_cuda_op_xielu(ctx, dst);\n break;\n case GGML_UNARY_OP_FLOOR:\n ggml_cuda_op_floor(ctx, dst);\n break;\n case GGML_UNARY_OP_CEIL:\n ggml_cuda_op_ceil(ctx, dst);\n break;\n case GGML_UNARY_OP_ROUND:\n ggml_cuda_op_round(ctx, dst);\n break;\n case GGML_UNARY_OP_TRUNC:\n ggml_cuda_op_trunc(ctx, dst);\n break;\n case GGML_UNARY_OP_EXPM1:\n ggml_cuda_op_expm1(ctx, dst);\n break;\n case GGML_UNARY_OP_SOFTPLUS:\n ggml_cuda_op_softplus(ctx, dst);\n break;\n default:\n return false;\n }\n break;\n case GGML_OP_GLU:\n switch (ggml_get_glu_op(dst)) {\n case GGML_GLU_OP_REGLU:\n ggml_cuda_op_reglu(ctx, dst);\n break;\n case GGML_GLU_OP_GEGLU:\n ggml_cuda_op_geglu(ctx, dst);\n break;\n case GGML_GLU_OP_SWIGLU:\n ggml_cuda_op_swiglu(ctx, dst);\n break;\n case GGML_GLU_OP_SWIGLU_OAI:\n ggml_cuda_op_swiglu_oai(ctx, dst);\n break;\n case GGML_GLU_OP_GEGLU_ERF:\n ggml_cuda_op_geglu_erf(ctx, dst);\n break;\n case GGML_GLU_OP_GEGLU_QUICK:\n ggml_cuda_op_geglu_quick(ctx, dst);\n break;\n default:\n return false;\n }\n break;\n case GGML_OP_NORM:\n ggml_cuda_op_norm(ctx, dst);\n break;\n case GGML_OP_GROUP_NORM:\n ggml_cuda_op_group_norm(ctx, dst);\n break;\n case GGML_OP_L2_NORM:\n ggml_cuda_op_l2_norm(ctx, dst);\n break;\n case GGML_OP_CONCAT:\n ggml_cuda_op_concat(ctx, dst);\n break;\n case GGML_OP_UPSCALE:\n ggml_cuda_op_upscale(ctx, dst);\n break;\n case GGML_OP_PAD:\n ggml_cuda_op_pad(ctx, dst);\n break;\n case GGML_OP_PAD_REFLECT_1D:\n ggml_cuda_op_pad_reflect_1d(ctx, dst);\n break;\n case GGML_OP_ARANGE:\n ggml_cuda_op_arange(ctx, dst);\n break;\n case GGML_OP_TIMESTEP_EMBEDDING:\n ggml_cuda_op_timestep_embedding(ctx, dst);\n break;\n case GGML_OP_LEAKY_RELU:\n ggml_cuda_op_leaky_relu(ctx, dst);\n break;\n case GGML_OP_SILU_BACK:\n ggml_cuda_op_silu_back(ctx, dst);\n break;\n case GGML_OP_RMS_NORM:\n ggml_cuda_op_rms_norm(ctx, dst);\n break;\n case GGML_OP_RMS_NORM_BACK:\n ggml_cuda_op_rms_norm_back(ctx, dst);\n break;\n case GGML_OP_MUL_MAT:\n ggml_cuda_mul_mat(ctx, dst->src[0], dst->src[1], dst);\n break;\n case GGML_OP_MUL_MAT_ID:\n ggml_cuda_mul_mat_id(ctx, dst);\n break;\n case GGML_OP_OUT_PROD:\n ggml_cuda_out_prod(ctx, dst);\n break;\n case GGML_OP_SCALE:\n ggml_cuda_op_scale(ctx, dst);\n break;\n case GGML_OP_SQR:\n ggml_cuda_op_sqr(ctx, dst);\n break;\n case GGML_OP_SQRT:\n ggml_cuda_op_sqrt(ctx, dst);\n break;\n case GGML_OP_SIN:\n ggml_cuda_op_sin(ctx, dst);\n break;\n case GGML_OP_COS:\n ggml_cuda_op_cos(ctx, dst);\n break;\n case GGML_OP_CLAMP:\n ggml_cuda_op_clamp(ctx, dst);\n break;\n case GGML_OP_LOG:\n ggml_cuda_op_log(ctx, dst);\n break;\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n break;\n case GGML_OP_DIAG:\n ggml_cuda_op_diag(ctx, dst);\n break;\n case GGML_OP_DIAG_MASK_INF:\n ggml_cuda_op_diag_mask_inf(ctx, dst);\n break;\n case GGML_OP_SOFT_MAX:\n ggml_cuda_op_soft_max(ctx, dst);\n break;\n case GGML_OP_SOFT_MAX_BACK:\n ggml_cuda_op_soft_max_back(ctx, dst);\n break;\n case GGML_OP_ROPE:\n ggml_cuda_op_rope(ctx, dst);\n break;\n case GGML_OP_ROPE_BACK:\n ggml_cuda_op_rope_back(ctx, dst);\n break;\n case GGML_OP_ROLL:\n ggml_cuda_op_roll(ctx, dst);\n break;\n case GGML_OP_IM2COL:\n ggml_cuda_op_im2col(ctx, dst);\n break;\n case GGML_OP_IM2COL_3D:\n ggml_cuda_op_im2col_3d(ctx, dst);\n break;\n case GGML_OP_CONV_2D:\n ggml_cuda_op_conv2d(ctx, dst);\n break;\n case GGML_OP_CONV_2D_DW:\n ggml_cuda_op_conv2d_dw(ctx, dst);\n break;\n case GGML_OP_CONV_TRANSPOSE_2D:\n ggml_cuda_conv_2d_transpose_p0(ctx, dst);\n break;\n case GGML_OP_CONV_TRANSPOSE_1D:\n ggml_cuda_op_conv_transpose_1d(ctx,dst);\n break;\n case GGML_OP_POOL_2D:\n ggml_cuda_op_pool2d(ctx, dst);\n break;\n case GGML_OP_SUM:\n ggml_cuda_op_sum(ctx, dst);\n break;\n case GGML_OP_CUMSUM:\n ggml_cuda_op_cumsum(ctx, dst);\n break;\n case GGML_OP_SUM_ROWS:\n ggml_cuda_op_sum_rows(ctx, dst);\n break;\n case GGML_OP_MEAN:\n ggml_cuda_op_mean(ctx, dst);\n break;\n case GGML_OP_SSM_CONV:\n ggml_cuda_op_ssm_conv(ctx, dst);\n break;\n case GGML_OP_SSM_SCAN:\n ggml_cuda_op_ssm_scan(ctx, dst);\n break;\n case GGML_OP_TOP_K:\n ggml_cuda_op_top_k(ctx, dst);\n break;\n case GGML_OP_ARGSORT:\n ggml_cuda_op_argsort(ctx, dst);\n break;\n case GGML_OP_FLASH_ATTN_EXT:\n ggml_cuda_flash_attn_ext(ctx, dst);\n break;\n case GGML_OP_CROSS_ENTROPY_LOSS:\n ggml_cuda_cross_entropy_loss(ctx, dst);\n break;\n case GGML_OP_TRI:\n ggml_cuda_op_tri(ctx, dst);\n break;\n case GGML_OP_RWKV_WKV6:\n ggml_cuda_op_rwkv_wkv6(ctx, dst);\n break;\n case GGML_OP_GATED_LINEAR_ATTN:\n ggml_cuda_op_gated_linear_attn(ctx, dst);\n break;\n case GGML_OP_GATED_DELTA_NET:\n ggml_cuda_op_gated_delta_net(ctx, dst);\n break;\n case GGML_OP_RWKV_WKV7:\n ggml_cuda_op_rwkv_wkv7(ctx, dst);\n break;\n case GGML_OP_CROSS_ENTROPY_LOSS_BACK:\n ggml_cuda_cross_entropy_loss_back(ctx, dst);\n break;\n case GGML_OP_OPT_STEP_ADAMW:\n ggml_cuda_opt_step_adamw(ctx, dst);\n break;\n case GGML_OP_OPT_STEP_SGD:\n ggml_cuda_opt_step_sgd(ctx, dst);\n break;\n case GGML_OP_SOLVE_TRI:\n ggml_cuda_op_solve_tri(ctx, dst);\n break;\n case GGML_OP_FILL:\n ggml_cuda_op_fill(ctx, dst);\n break;\n default:\n return false;\n }\n\n cudaError_t err = cudaGetLastError();\n if (err != cudaSuccess) {\n GGML_LOG_ERROR(\"%s: %s failed\\n\", __func__, ggml_op_desc(dst));\n CUDA_CHECK(err);\n }\n\n return true;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n// backend\n\nstatic const char * ggml_backend_cuda_get_name(ggml_backend_t backend) {\n ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;\n\n return cuda_ctx->name.c_str();\n}\n\nstatic void ggml_backend_cuda_free(ggml_backend_t backend) {\n ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;\n\n delete cuda_ctx;\n delete backend;\n}\n\nstatic void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;\n ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && \"unsupported buffer type\");\n\n CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));\n}\n\nstatic void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;\n ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && \"unsupported buffer type\");\n\n CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));\n}\n\nstatic bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {\n ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;\n ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;\n\n if (!ggml_backend_is_cuda(backend_src) || !ggml_backend_is_cuda(backend_dst)) {\n return false;\n }\n\n if (!ggml_backend_buffer_is_cuda(src->buffer) || !ggml_backend_buffer_is_cuda(dst->buffer)) {\n return false;\n }\n\n // device -> device copy\n ggml_backend_cuda_context * cuda_ctx_src = (ggml_backend_cuda_context *)backend_src->context;\n ggml_backend_cuda_context * cuda_ctx_dst = (ggml_backend_cuda_context *)backend_dst->context;\n\n ggml_backend_cuda_buffer_context * buf_ctx_src = (ggml_backend_cuda_buffer_context *)buf_src->context;\n ggml_backend_cuda_buffer_context * buf_ctx_dst = (ggml_backend_cuda_buffer_context *)buf_dst->context;\n\n if (cuda_ctx_src->device != buf_ctx_src->device || cuda_ctx_dst->device != buf_ctx_dst->device) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: backend and buffer devices do not match\\n\", __func__);\n#endif\n return false;\n }\n\n if (backend_src != backend_dst) {\n // copy on src stream\n if (cuda_ctx_src->device == cuda_ctx_dst->device) {\n CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_src->stream()));\n } else {\n#ifdef GGML_CUDA_NO_PEER_COPY\n return false;\n#else\n CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), cuda_ctx_src->stream()));\n#endif\n }\n\n // record event on src stream after the copy\n if (!cuda_ctx_src->copy_event) {\n ggml_cuda_set_device(cuda_ctx_src->device);\n CUDA_CHECK(cudaEventCreateWithFlags(&cuda_ctx_src->copy_event, cudaEventDisableTiming));\n }\n\n CUDA_CHECK(cudaEventRecord(cuda_ctx_src->copy_event, cuda_ctx_src->stream()));\n\n // wait on dst stream for the copy to complete\n CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx_dst->stream(), cuda_ctx_src->copy_event, 0));\n } else {\n // src and dst are on the same backend\n CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_src->stream()));\n }\n return true;\n}\n\nstatic void ggml_backend_cuda_synchronize(ggml_backend_t backend) {\n ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;\n\n CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));\n\n GGML_UNUSED(backend);\n}\n\n#ifdef USE_CUDA_GRAPH\nstatic bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {\n\n bool use_cuda_graph = true;\n // Loop over nodes in GGML graph to obtain info needed for CUDA graph\n\n for (int i = 0; i < cgraph->n_nodes; i++) {\n ggml_tensor * node = cgraph->nodes[i];\n\n if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {\n continue;\n }\n\n if (node->src[0] && node->src[0]->buffer && ggml_backend_buft_is_cuda_split(node->src[0]->buffer->buft)) {\n use_cuda_graph = false; // Split buffers are not supported by CUDA graph capture\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: disabling CUDA graphs due to split buffer\\n\", __func__);\n#endif\n }\n\n // [TAG_MUL_MAT_ID_CUDA_GRAPHS]\n if (node->op == GGML_OP_MUL_MAT_ID && (!ggml_is_quantized(node->src[0]->type) || node->ne[2] > MMVQ_MMID_MAX_BATCH_SIZE)) {\n // under these conditions, the mul_mat_id operation will need to synchronize the stream, so we cannot use CUDA graphs\n // TODO: figure out a way to enable for larger batch sizes, without hurting performance\n // ref: https://github.com/ggml-org/llama.cpp/pull/18958\n use_cuda_graph = false;\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: disabling CUDA graphs due to unsupported node type\\n\", __func__);\n#endif\n }\n\n if (!use_cuda_graph) {\n break;\n }\n }\n\n return use_cuda_graph;\n}\n\nstatic void ggml_cuda_graph_node_set_properties(ggml_cuda_graph_node_properties * props, ggml_tensor * node) {\n memset(props, 0, sizeof(ggml_cuda_graph_node_properties));\n props->node_data = node->data;\n props->node_op = node->op;\n props->node_type = node->type;\n props->flags = node->flags;\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n props->ne[i] = node->ne[i];\n props->nb[i] = node->nb[i];\n }\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (!node->src[i]) {\n continue;\n }\n\n props->src_data[i] = node->src[i]->data;\n }\n memcpy(props->op_params, node->op_params, GGML_MAX_OP_PARAMS);\n}\n\nstatic bool ggml_cuda_graph_node_properties_match(ggml_tensor * node, ggml_cuda_graph_node_properties * props) {\n if (node->data != props->node_data && node->op != GGML_OP_VIEW) {\n return false;\n }\n\n if (node->op != props->node_op) {\n return false;\n }\n\n if (node->type != props->node_type) {\n return false;\n }\n\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n if (node->ne[i] != props->ne[i]) {\n return false;\n }\n if (node->nb[i] != props->nb[i]) {\n return false;\n }\n }\n\n if (node->op != GGML_OP_VIEW) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (!node->src[i]) {\n if (props->src_data[i] != nullptr) {\n return false;\n }\n continue;\n }\n\n if (node->src[i]->data != props->src_data[i]) {\n return false;\n }\n }\n }\n\n if (memcmp(props->op_params, node->op_params, GGML_MAX_OP_PARAMS) != 0) {\n return false;\n }\n\n if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) != (props->flags & GGML_TENSOR_FLAG_COMPUTE)) {\n return false;\n }\n\n return true;\n}\n\nstatic const void * ggml_cuda_graph_get_key(ggml_cgraph * cgraph) {\n return cgraph->nodes[0];\n}\n\nstatic bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph) {\n bool res = false;\n\n const void * graph_key = ggml_cuda_graph_get_key(cgraph);\n ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key);\n\n // Check if the graph size has changed\n if (graph->props.size() != (size_t)cgraph->n_nodes) {\n res = true;\n graph->props.resize(cgraph->n_nodes);\n }\n\n // Loop over nodes in GGML graph to determine if CUDA graph update is required\n // and store properties to allow this comparison for the next token\n std::unordered_set seen_node;\n std::vector srcs_extra;\n for (int i = 0; i < cgraph->n_nodes; i++) {\n bool props_match = true;\n\n seen_node.insert(cgraph->nodes[i]);\n\n if (!res) {\n props_match = ggml_cuda_graph_node_properties_match(cgraph->nodes[i], &graph->props[i]);\n }\n if (!props_match) {\n res = true;\n }\n ggml_cuda_graph_node_set_properties(&graph->props[i], cgraph->nodes[i]);\n\n for (int src_idx = 0; src_idx < GGML_MAX_SRC; ++src_idx) {\n ggml_tensor * src = cgraph->nodes[i]->src[src_idx];\n if (src && seen_node.find(src) == seen_node.end()) {\n srcs_extra.push_back(src);\n }\n }\n }\n\n if (graph->extra.size() != (size_t) srcs_extra.size()) {\n res = true;\n graph->extra.resize(srcs_extra.size());\n }\n\n for (size_t i = 0; i < srcs_extra.size(); ++i) {\n bool props_match = true;\n\n if (!res) {\n props_match = ggml_cuda_graph_node_properties_match(srcs_extra[i], &graph->extra[i]);\n }\n\n if (!props_match) {\n res = true;\n }\n ggml_cuda_graph_node_set_properties(&graph->extra[i], srcs_extra[i]);\n }\n\n return res;\n}\n\nstatic void ggml_cuda_graph_update_executable(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) {\n ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key);\n\n#if CUDART_VERSION >= 12000\n cudaGraphExecUpdateResultInfo result_info;\n cudaError_t stat = cudaGraphExecUpdate(graph->instance, graph->graph, &result_info);\n#else\n cudaGraphNode_t errorNode;\n cudaGraphExecUpdateResult result_info;\n cudaError_t stat = cudaGraphExecUpdate(graph->instance, graph->graph, &errorNode, &result_info);\n#endif // CUDART_VERSION >= 12000\n\n if (stat == cudaErrorGraphExecUpdateFailure) {\n#ifndef NDEBUG\n GGML_LOG_DEBUG(\"%s: CUDA graph update failed\\n\", __func__);\n#endif\n\n // The pre-existing graph exec cannot be updated due to violated constraints\n // so instead clear error and re-instantiate\n (void)cudaGetLastError();\n CUDA_CHECK(cudaGraphExecDestroy(graph->instance));\n graph->instance = nullptr;\n CUDA_CHECK(cudaGraphInstantiate(&graph->instance, graph->graph, NULL, NULL, 0));\n } else {\n GGML_ASSERT(stat == cudaSuccess);\n }\n}\n#endif // USE_CUDA_GRAPH\n\nstatic bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope,\n const ggml_tensor * view,\n const ggml_tensor * set_rows) {\n\n if (rope->op != GGML_OP_ROPE || view->op != GGML_OP_VIEW || set_rows->op != GGML_OP_SET_ROWS) {\n return false;\n }\n // ne3 not tested\n if (rope->src[0]->ne[3] != 1) {\n return false;\n }\n\n if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16) {\n return false;\n }\n\n if (set_rows->src[1]->type != GGML_TYPE_I64) {\n return false;\n }\n\n // The view should flatten two dims of rope into one dim\n if (!ggml_is_contiguous(view) || view->ne[0] != rope->ne[0] * rope->ne[1]) {\n return false;\n }\n\n // Only norm/neox shaders have the fusion code\n const int mode = ((const int32_t *) rope->op_params)[2];\n if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX) {\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) {\n args.sigmoid = false;\n args.softmax = false;\n args.delayed_softmax = false;\n args.prob_bias = false;\n args.norm = false;\n\n const int n_nodes = cgraph->n_nodes;\n ggml_tensor ** nodes = cgraph->nodes;\n\n if (nodes[node_idx]->op == GGML_OP_SOFT_MAX) {\n args.softmax = true;\n }\n\n if (nodes[node_idx]->op == GGML_OP_UNARY) {\n if (ggml_get_unary_op(nodes[node_idx]) != GGML_UNARY_OP_SIGMOID) {\n return false;\n }\n args.sigmoid = true;\n }\n\n if (nodes[node_idx]->op == GGML_OP_ARGSORT) {\n args.delayed_softmax = true;\n }\n\n node_idx++;\n\n if (args.sigmoid || args.softmax) {\n // SOFTMAX -> RESHAPE\n if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_RESHAPE ||\n nodes[node_idx]->src[0] != nodes[node_idx - 1]) {\n return false;\n }\n ggml_tensor * probs_reshaped = nodes[node_idx];\n node_idx++;\n\n if (node_idx >= n_nodes) {\n return false;\n }\n\n // src of bias add is the unreshaped probs (-2 instead of -1)\n if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == nodes[node_idx - 2]) {\n args.prob_bias = true;\n node_idx++;\n }\n // RESHAPE/ADD -> ARGSORT\n if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT) {\n return false;\n }\n\n if (args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 1]) {\n return false;\n } else if (!args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 2]) {\n return false;\n }\n\n node_idx++;\n\n // ARGSORT-> VIEW\n if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_VIEW ||\n nodes[node_idx]->src[0] != nodes[node_idx - 1]) {\n return false;\n }\n node_idx++;\n\n if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_GET_ROWS) {\n return false;\n }\n\n // GET_ROWS\n if (nodes[node_idx]->src[0] != probs_reshaped || nodes[node_idx]->src[1] != nodes[node_idx - 1]) {\n return false;\n }\n node_idx++;\n } else if (args.delayed_softmax) {\n if (node_idx - 2 < 0) {\n return false;\n }\n ggml_tensor * probs_reshaped = nodes[node_idx - 2];\n\n // VIEW->ARGSORT\n if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_VIEW ||\n nodes[node_idx]->src[0] != nodes[node_idx - 1]) {\n return false;\n }\n node_idx++;\n\n // GET_ROWS\n if (node_idx >= n_nodes || nodes[node_idx]->src[1] != nodes[node_idx - 1] ||\n nodes[node_idx]->src[0] != probs_reshaped) {\n return false;\n }\n node_idx++;\n\n static const std::vector remaining_ops = { GGML_OP_RESHAPE, GGML_OP_SOFT_MAX, GGML_OP_RESHAPE };\n\n for (const ggml_op op : remaining_ops) {\n if (node_idx >= n_nodes || nodes[node_idx]->op != op || nodes[node_idx]->src[0] != nodes[node_idx - 1]) {\n return false;\n }\n node_idx++;\n }\n }\n\n // At this point we can check for norm + scale. Everything is now at least valid till the norm\n if (node_idx >= n_nodes) {\n return true;\n }\n\n if (nodes[node_idx]->op == GGML_OP_RESHAPE) {\n //check RESHAPE->SUM_ROWS->CLAMP->DIV->RESHAPE\n static const std::vector norm_ops = { GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP };\n\n args.norm = true;\n for (const ggml_op op : norm_ops) {\n if (nodes[node_idx]->op == op && nodes[node_idx]->src[0] == nodes[node_idx - 1]) {\n node_idx++;\n } else {\n args.norm = false;\n return true;\n }\n }\n\n // DIV <- CLAMP, RESHAPE\n if (nodes[node_idx]->op != GGML_OP_DIV || nodes[node_idx]->src[1] != nodes[node_idx - 1] ||\n nodes[node_idx]->src[0] != nodes[node_idx - 3]) {\n args.norm = false;\n return true;\n }\n node_idx++;\n\n if (nodes[node_idx]->op != GGML_OP_RESHAPE || nodes[node_idx]->src[0] != nodes[node_idx - 1]) {\n args.norm = false;\n return true;\n }\n\n node_idx++;\n }\n\n if (nodes[node_idx]->op == GGML_OP_SCALE && nodes[node_idx]->src[0] == nodes[node_idx - 1]) {\n args.scale = true;\n }\n\n return true;\n}\n\nstatic bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,\n int node_idx,\n std::initializer_list ops,\n std::initializer_list unary_ops) {\n#ifndef NDEBUG\n const size_t num_unary = std::count(ops.begin(), ops.end(), GGML_OP_UNARY);\n GGML_ASSERT(unary_ops.size() == num_unary);\n#endif\n\n const auto is_equal = [](const std::initializer_list & list1,\n const std::initializer_list & list2) {\n return std::equal(list1.begin(), list1.end(), list2.begin(), list2.end());\n };\n\n std::initializer_list mul_mat_bias_glu_ops = { GGML_OP_MUL_MAT, GGML_OP_ADD, GGML_OP_MUL_MAT, GGML_OP_ADD, GGML_OP_GLU };\n std::initializer_list mul_mat_id_bias_glu_ops = { GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_GLU };\n\n std::initializer_list mul_mat_id_glu_ops = { GGML_OP_MUL_MAT_ID, GGML_OP_MUL_MAT_ID, GGML_OP_GLU };\n std::initializer_list mul_mat_glu_ops = { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU };\n\n if ((is_equal(mul_mat_bias_glu_ops, ops) || is_equal(mul_mat_id_bias_glu_ops, ops)) &&\n ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 4 })) {\n const ggml_tensor * ffn_gate = cgraph->nodes[node_idx];\n const ggml_tensor * ffn_gate_bias = cgraph->nodes[node_idx + 1];\n const ggml_tensor * ffn_up = cgraph->nodes[node_idx + 2];\n const ggml_tensor * ffn_up_bias = cgraph->nodes[node_idx + 3];\n const ggml_tensor * glu = cgraph->nodes[node_idx + 4];\n\n if (ggml_cuda_should_fuse_mul_mat(ffn_up, ffn_gate, glu, ffn_up_bias, ffn_gate_bias)) {\n return true;\n }\n }\n\n if ((is_equal(mul_mat_id_glu_ops, ops) || is_equal(mul_mat_glu_ops, ops)) &&\n ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 2 })) {\n const ggml_tensor * ffn_gate = cgraph->nodes[node_idx];\n const ggml_tensor * ffn_up = cgraph->nodes[node_idx + 1];\n const ggml_tensor * glu = cgraph->nodes[node_idx + 2];\n\n if (ggml_cuda_should_fuse_mul_mat(ffn_up, ffn_gate, glu)) {\n return true;\n }\n }\n\n std::initializer_list rope_set_rows_ops = { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };\n\n if (is_equal(rope_set_rows_ops, ops) && ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 2 })) {\n const ggml_tensor * rope = cgraph->nodes[node_idx];\n const ggml_tensor * view = cgraph->nodes[node_idx + 1];\n const ggml_tensor * set_rows = cgraph->nodes[node_idx + 2];\n\n if (ggml_cuda_should_fuse_rope_set_rows(rope, view, set_rows)) {\n return true;\n }\n }\n\n if (!ggml_can_fuse(cgraph, node_idx, ops)) {\n return false;\n }\n\n if ((ops.size() == 2 || ops.size() == 3) && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) {\n const ggml_tensor *rms_norm = cgraph->nodes[node_idx];\n const ggml_tensor *mul = cgraph->nodes[node_idx+1];\n const ggml_tensor *add = nullptr;\n\n if (ops.size() == 3 && ops.begin()[2] == GGML_OP_ADD) {\n add = cgraph->nodes[node_idx+2];\n }\n\n GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(rms_norm->type == GGML_TYPE_F32);\n\n //rms norm only supports F32\n if (mul->src[0]->type != GGML_TYPE_F32 ||\n mul->src[1]->type != GGML_TYPE_F32 ||\n mul->type != GGML_TYPE_F32) {\n return false;\n }\n\n if (add && (add->src[0]->type != GGML_TYPE_F32 ||\n add->src[1]->type != GGML_TYPE_F32 ||\n add->type != GGML_TYPE_F32) ) {\n return false;\n }\n\n //if rms norm is the B operand, then we don't handle broadcast\n if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) {\n return false;\n }\n\n //rms_norm kernel assumes contiguous rows\n if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) {\n return false;\n }\n\n if (add && (!ggml_is_contiguous(add->src[0]) || !ggml_is_contiguous_rows(add->src[1]))) {\n return false;\n }\n\n return true;\n }\n\n if (ops.size() == 2 && ops.begin()[0] == GGML_OP_SSM_CONV && ops.begin()[1] == GGML_OP_UNARY\n && unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_SILU) {\n const ggml_tensor * ssm_conv = cgraph->nodes[node_idx];\n const ggml_tensor * silu = cgraph->nodes[node_idx+1];\n\n if (ssm_conv->type != GGML_TYPE_F32 || silu->type != GGML_TYPE_F32) {\n return false;\n }\n\n return true;\n }\n\n if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL\n && unary_ops.size() == 1 && (unary_ops.begin()[0] == GGML_UNARY_OP_SILU || unary_ops.begin()[0] == GGML_UNARY_OP_SIGMOID || unary_ops.begin()[0] == GGML_UNARY_OP_SOFTPLUS)) {\n const ggml_tensor * unary = cgraph->nodes[node_idx];\n const ggml_tensor * mul = cgraph->nodes[node_idx+1];\n\n if (ggml_get_unary_op(unary) != unary_ops.begin()[0]) {\n return false;\n }\n\n if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) {\n return false;\n }\n\n if (unary->type != mul->type) {\n return false;\n }\n\n const ggml_tensor * other = (mul->src[0] == unary) ? mul->src[1] : mul->src[0];\n if (other->type != unary->type) {\n return false;\n }\n if (!ggml_is_contiguous_1(other) || !ggml_is_contiguous_1(unary->src[0]) || !ggml_are_same_shape(other, unary)) {\n return false;\n }\n\n return true;\n }\n\n if (ops.size() == 3 && ops.begin()[0] == GGML_OP_SCALE && ops.begin()[1] == GGML_OP_UNARY && ops.begin()[2] == GGML_OP_SCALE\n && unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_TANH) {\n const ggml_tensor *scale = cgraph->nodes[node_idx];\n const ggml_tensor *tanh = cgraph->nodes[node_idx+1];\n const ggml_tensor *scale2 = cgraph->nodes[node_idx+2];\n\n GGML_ASSERT(scale->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(scale->type == GGML_TYPE_F32);\n\n if (ggml_get_unary_op(tanh) != GGML_UNARY_OP_TANH) {\n return false;\n }\n\n // Check for bias\n if (ggml_get_op_params_f32(scale, 1) != 0.0f || ggml_get_op_params_f32(scale2, 1) != 0.0f) {\n return false;\n }\n\n return true;\n }\n\n return false;\n}\n\n// returns whether the write (out) nodes overwrite the read nodes in operation\nstatic bool ggml_cuda_check_fusion_memory_ranges(ggml_cgraph * cgraph,\n int node_idx,\n int node_count,\n int * out_nodes,\n int out_count) {\n auto nodes_overlap = [&](const ggml_tensor * a, const ggml_tensor * b) {\n const int64_t a_start = (int64_t) a->data;\n const int64_t a_end = a_start + ggml_nbytes(a);\n\n const int64_t b_start = (int64_t) b->data;\n const int64_t b_end = b_start + ggml_nbytes(b);\n\n if ((b_start <= a_start && a_start < b_end) || (a_start <= b_start && b_start < a_end)) {\n return true;\n }\n\n return false;\n };\n\n bool is_ok = true;\n // for nrows=1, all fusion operations correctly read the src before writing dst or do it elementwise, so we should be ok\n if (ggml_nrows(cgraph->nodes[node_idx]) == 1) {\n return true;\n }\n\n for (int i = 0; i < out_count; ++i) {\n const ggml_tensor * dst = cgraph->nodes[out_nodes[i]];\n\n for (int j = node_idx; j < node_idx + node_count; ++j) {\n // Loop over all srcs of all nodes in the fusion. If the src overlaps\n // the destination and the src is not an intermediate node that's being\n // elided, then disable fusion.\n\n for (int src_idx = 0; src_idx < GGML_MAX_SRC; ++src_idx) {\n const ggml_tensor * src = cgraph->nodes[j]->src[src_idx];\n\n if (!src || src->op == GGML_OP_NONE) {\n continue;\n }\n\n if (nodes_overlap(dst, src)) {\n bool found = false;\n\n for (int k = node_idx; k < j; ++k) {\n if (cgraph->nodes[k] == src) {\n found = true;\n break;\n }\n }\n\n if (!found) {\n is_ok = false;\n break;\n }\n }\n }\n }\n }\n\n return is_ok;\n}\n\nstatic void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, const bool use_cuda_graph, const bool cuda_graph_update_required, const void * graph_key) {\n bool graph_evaluated_or_captured = false;\n\n // flag used to determine whether it is an integrated_gpu\n const bool integrated = ggml_cuda_info().devices[cuda_ctx->device].integrated;\n\n ggml_cuda_stream_context & stream_ctx = cuda_ctx->stream_context();\n bool is_concurrent_event_active = false;\n ggml_cuda_concurrent_event * concurrent_event = nullptr;\n bool should_launch_concurrent_events = false;\n\n const auto try_launch_concurrent_event = [&](const ggml_tensor * node) {\n if (stream_ctx.concurrent_events.find(node) != stream_ctx.concurrent_events.end()) {\n concurrent_event = &stream_ctx.concurrent_events[node];\n\n is_concurrent_event_active = true;\n\n GGML_LOG_DEBUG(\"Launching %d streams at %s\\n\", concurrent_event->n_streams, node->name);\n\n cudaStream_t main_stream = cuda_ctx->stream(); // this should be stream 0\n GGML_ASSERT(cuda_ctx->curr_stream_no == 0);\n CUDA_CHECK(cudaEventRecord(concurrent_event->fork_event, main_stream));\n\n for (int i = 1; i <= concurrent_event->n_streams; ++i) {\n cudaStream_t stream = cuda_ctx->stream(cuda_ctx->device, i);\n CUDA_CHECK(cudaStreamWaitEvent(stream, concurrent_event->fork_event));\n }\n }\n };\n\n while (!graph_evaluated_or_captured) {\n // Only perform the graph execution if CUDA graphs are not enabled, or we are capturing the graph.\n // With the use of CUDA graphs, the execution will be performed by the graph launch.\n if (!use_cuda_graph || cuda_graph_update_required) {\n [[maybe_unused]] int prev_i = 0;\n\n if (stream_ctx.concurrent_events.size() > 0) {\n should_launch_concurrent_events = true;\n for (const auto & [tensor, event] : stream_ctx.concurrent_events) {\n should_launch_concurrent_events = should_launch_concurrent_events && event.is_valid();\n }\n }\n\n if (should_launch_concurrent_events) {\n // Restore original node order within each concurrent region to enable fusion within streams\n\n std::unordered_map node_to_idx;\n node_to_idx.reserve(cgraph->n_nodes);\n for (int i = 0; i < cgraph->n_nodes; ++i) {\n node_to_idx[cgraph->nodes[i]] = i;\n }\n\n for (auto & [fork_node, event] : stream_ctx.concurrent_events) {\n // Find positions of all nodes from this event in the current graph\n std::vector positions;\n positions.reserve(event.original_order.size());\n\n bool all_found = true;\n for (const ggml_tensor * orig_node : event.original_order) {\n auto it = node_to_idx.find(orig_node);\n if (it != node_to_idx.end()) {\n positions.push_back(it->second);\n } else {\n all_found = false;\n break;\n }\n }\n\n if (!all_found || positions.size() != event.original_order.size()) {\n continue;\n }\n\n // Sort positions to get contiguous range\n std::vector sorted_positions = positions;\n std::sort(sorted_positions.begin(), sorted_positions.end());\n\n bool is_contiguous = true;\n for (size_t i = 1; i < sorted_positions.size(); ++i) {\n if (sorted_positions[i] != sorted_positions[i-1] + 1) {\n is_contiguous = false;\n break;\n }\n }\n\n if (!is_contiguous) {\n continue;\n }\n\n // Restore original order at the sorted positions\n int start_pos = sorted_positions[0];\n for (size_t i = 0; i < event.original_order.size(); ++i) {\n cgraph->nodes[start_pos + i] = const_cast(event.original_order[i]);\n }\n }\n } else {\n stream_ctx.concurrent_events.clear();\n }\n\n for (int i = 0; i < cgraph->n_nodes; i++) {\n ggml_tensor * node = cgraph->nodes[i];\n if (is_concurrent_event_active) {\n GGML_ASSERT(concurrent_event);\n\n if (node == concurrent_event->join_node) {\n cuda_ctx->curr_stream_no = 0;\n for (int i = 1; i <= concurrent_event->n_streams; ++i) {\n // Wait on join events of forked streams in the main stream\n CUDA_CHECK(cudaEventRecord(concurrent_event->join_events[i - 1],\n cuda_ctx->stream(cuda_ctx->device, i)));\n CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx->stream(), concurrent_event->join_events[i - 1]));\n }\n\n is_concurrent_event_active = false;\n concurrent_event = nullptr;\n } else {\n GGML_ASSERT (concurrent_event->stream_mapping.find(node) != concurrent_event->stream_mapping.end());\n cuda_ctx->curr_stream_no = concurrent_event->stream_mapping[node];\n GGML_LOG_DEBUG(\"Setting stream no to %d for node %s\\n\", cuda_ctx->curr_stream_no, node->name);\n }\n } else if (i - prev_i > 1) {\n //the previous node was fused\n const ggml_tensor * prev_node = cgraph->nodes[i - 1];\n try_launch_concurrent_event(prev_node);\n\n if (is_concurrent_event_active) {\n cuda_ctx->curr_stream_no = concurrent_event->stream_mapping[node];\n GGML_LOG_DEBUG(\"Setting stream no to %d for node %s\\n\", cuda_ctx->curr_stream_no, node->name);\n }\n }\n\n#ifdef GGML_CUDA_DEBUG\n const int nodes_fused = i - prev_i - 1;\n if (nodes_fused > 0) {\n GGML_LOG_INFO(\"nodes_fused: %d\\n\", nodes_fused);\n }\n#endif\n prev_i = i;\n\n if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {\n continue;\n }\n\n if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n continue;\n }\n\n // start of fusion operations\n static bool disable_fusion = (getenv(\"GGML_CUDA_DISABLE_FUSION\") != nullptr);\n if (!disable_fusion) {\n ggml_cuda_topk_moe_args args;\n\n if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX ||\n cgraph->nodes[i]->op == GGML_OP_ARGSORT) {\n const bool can_fuse = ggml_cuda_topk_moe_fusion(cgraph, i, args);\n\n std::vector ops;\n\n if (can_fuse) {\n const ggml_tensor * logits = node->src[0];\n ggml_tensor * weights = nullptr;\n ggml_tensor * ids = nullptr;\n const ggml_tensor * bias = nullptr;\n const ggml_tensor * clamp = nullptr;\n const ggml_tensor * scale = nullptr;\n\n if (!args.delayed_softmax) {\n ggml_op gating_op = args.sigmoid ? GGML_OP_UNARY : GGML_OP_SOFT_MAX;\n int out_nodes[2]; // nodes which can't be elided\n\n if (args.prob_bias) {\n bias = cgraph->nodes[i + 2]->src[1];\n ops.insert(ops.end(), { gating_op, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT,\n GGML_OP_VIEW, GGML_OP_GET_ROWS });\n out_nodes[0] = i + 4;\n ids = cgraph->nodes[i + 4];\n } else {\n ops.insert(ops.end(), { gating_op, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW,\n GGML_OP_GET_ROWS });\n out_nodes[0] = i + 3;\n ids = cgraph->nodes[i + 3];\n }\n\n if (args.norm) {\n ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP,\n GGML_OP_DIV, GGML_OP_RESHAPE });\n clamp = cgraph->nodes[i + ops.size() - 3];\n }\n if (args.scale) {\n ops.insert(ops.end(), { GGML_OP_SCALE });\n scale = cgraph->nodes[i + ops.size() - 1];\n }\n\n weights = cgraph->nodes[i + ops.size() - 1];\n out_nodes[1] = i + ops.size() - 1;\n\n if (ggml_can_fuse_subgraph(cgraph, i, ops.size(), ops.data(), out_nodes, 2) &&\n ggml_cuda_should_use_topk_moe(node, logits, weights, ids) &&\n ggml_cuda_check_fusion_memory_ranges(cgraph, i, ops.size(), out_nodes, 2)) {\n ggml_cuda_op_topk_moe(*cuda_ctx, logits, weights, ids, clamp, scale, bias, args);\n i += ops.size() - 1;\n continue;\n }\n } else if (!args.norm && !args.prob_bias) {\n //special case gpt-oss, no norm, no bias.\n ops.insert(ops.end(), { GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS,\n GGML_OP_RESHAPE, GGML_OP_SOFT_MAX, GGML_OP_RESHAPE });\n weights = cgraph->nodes[i + 5];\n ids = cgraph->nodes[i + 1];\n const ggml_tensor * softmax = cgraph->nodes[i + 4];\n\n int out_nodes[2] = { i + 1, i + 5 };\n if (ggml_can_fuse_subgraph(cgraph, i, ops.size(), ops.data(), out_nodes, 2) &&\n ggml_cuda_should_use_topk_moe(softmax, logits, weights, ids) &&\n ggml_cuda_check_fusion_memory_ranges(cgraph, i, ops.size(), out_nodes, 2)) {\n ggml_cuda_op_topk_moe(*cuda_ctx, logits, weights, ids, clamp, scale, bias, args);\n i += ops.size() - 1;\n continue;\n }\n }\n }\n }\n\n if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }, {})) {\n ggml_tensor * rope = cgraph->nodes[i];\n ggml_tensor * set_rows = cgraph->nodes[i + 2];\n\n ggml_cuda_op_rope_fused(*cuda_ctx, rope, set_rows);\n i += 2;\n continue;\n }\n\n if (node->op == GGML_OP_ADD) {\n int n_fuse = 0;\n ggml_op ops[8];\n std::fill(ops, ops + 8, GGML_OP_ADD);\n\n for (; n_fuse <= 6; ++n_fuse){\n if (!ggml_can_fuse(cgraph, i + n_fuse, ops + n_fuse, 2)) {\n break;\n }\n if (cgraph->nodes[i + n_fuse] != cgraph->nodes[i + n_fuse + 1]->src[0]) {\n break;\n }\n if (!ggml_are_same_layout(cgraph->nodes[i + n_fuse]->src[1], cgraph->nodes[i + n_fuse + 1]->src[1])) {\n break;\n }\n }\n\n n_fuse++;\n\n if (n_fuse > 1) {\n ggml_tensor fused_add_node;\n memcpy(&fused_add_node, node, sizeof(ggml_tensor));\n for (int j = 0; j < n_fuse - 1; ++j) {\n fused_add_node.src[j + 2] = cgraph->nodes[i + j + 1]->src[1];\n }\n fused_add_node.data = cgraph->nodes[i + n_fuse - 1]->data;\n ggml_cuda_op_fused_add(*cuda_ctx, &fused_add_node, n_fuse);\n i += n_fuse - 1;\n\n continue;\n }\n }\n\n bool fused_mul_mat_vec = false;\n int fused_node_count = 0;\n\n for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) {\n const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID;\n\n if (ggml_cuda_can_fuse(cgraph, i, { op, bias_op, op, bias_op, GGML_OP_GLU }, {})) {\n ggml_tensor * glu = cgraph->nodes[i + 4];\n ggml_tensor * gate_bias_n = glu->src[0];\n ggml_tensor * up_bias_n = glu->src[1];\n\n //we don't assume the order for {gate, up}. Instead infer it from the bias tensor\n ggml_tensor * gate_n = nullptr;\n ggml_tensor * up_n = nullptr;\n\n if (gate_bias_n->src[0] == cgraph->nodes[i] || gate_bias_n->src[1] == cgraph->nodes[i]) {\n gate_n = cgraph->nodes[i];\n up_n = cgraph->nodes[i + 2];\n } else if (gate_bias_n->src[0] == cgraph->nodes[i + 2] || gate_bias_n->src[1] == cgraph->nodes[i + 2]) {\n gate_n = cgraph->nodes[i + 2];\n up_n = cgraph->nodes[i];\n } else {\n continue;\n }\n\n auto get_bias_tensor = [](const ggml_tensor * bias_node, const ggml_tensor * mul_node, ggml_op op_bias) {\n if (op_bias == GGML_OP_ADD) {\n if (bias_node->src[0] == mul_node) {\n return bias_node->src[1];\n }\n if (bias_node->src[1] == mul_node) {\n return bias_node->src[0];\n }\n return (ggml_tensor *) nullptr;\n }\n GGML_ASSERT(op_bias == GGML_OP_ADD_ID);\n GGML_ASSERT(bias_node->src[0] == mul_node);\n return bias_node->src[1];\n };\n\n ggml_tensor * up_bias_tensor = get_bias_tensor(up_bias_n, up_n, bias_op);\n ggml_tensor * gate_bias_tensor = get_bias_tensor(gate_bias_n, gate_n, bias_op);\n\n if (!up_bias_tensor || !gate_bias_tensor) {\n continue;\n }\n\n // we don't support repeating adds\n if (bias_op == GGML_OP_ADD &&\n (!ggml_are_same_shape(gate_bias_n->src[0], gate_bias_n->src[1]) ||\n !ggml_are_same_shape(up_bias_n->src[0], up_bias_n->src[1]))) {\n continue;\n }\n\n const ggml_tensor * src0 = up_n->src[0];\n const ggml_tensor * src1 = up_n->src[1];\n const ggml_tensor * ids = up_n->src[2];\n\n if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n)) {\n ggml_cuda_mm_fusion_args_host fusion_data{};\n fusion_data.gate = gate_n->src[0];\n fusion_data.x_bias = up_bias_tensor;\n fusion_data.gate_bias = gate_bias_tensor;\n fusion_data.glu_op = ggml_get_glu_op(glu);\n\n ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data);\n fused_mul_mat_vec = true;\n fused_node_count = 5;\n break;\n }\n\n if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) {\n ggml_cuda_mm_fusion_args_host fusion_data{};\n fusion_data.gate = gate_n->src[0];\n fusion_data.x_bias = up_bias_tensor;\n fusion_data.gate_bias = gate_bias_tensor;\n fusion_data.glu_op = ggml_get_glu_op(glu);\n\n ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data);\n fused_mul_mat_vec = true;\n fused_node_count = 5;\n break;\n }\n } else if (ggml_cuda_can_fuse(cgraph, i, { op, op, GGML_OP_GLU }, {})) {\n ggml_tensor * glu = cgraph->nodes[i + 2];\n ggml_tensor * gate = glu->src[0];\n ggml_tensor * up = glu->src[1];\n\n bool ok = (gate == cgraph->nodes[i] && up == cgraph->nodes[i + 1])\n || (gate == cgraph->nodes[i + 1] && up == cgraph->nodes[i]);\n\n if (!ok) continue;\n\n const ggml_tensor * src0 = up->src[0];\n const ggml_tensor * src1 = up->src[1];\n const ggml_tensor * ids = up->src[2];\n\n if (ggml_cuda_should_fuse_mul_mat_vec_f(up)) {\n ggml_cuda_mm_fusion_args_host fusion_data{};\n fusion_data.gate = gate->src[0];\n fusion_data.glu_op = ggml_get_glu_op(glu);\n\n ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data);\n fused_mul_mat_vec = true;\n fused_node_count = 3;\n break;\n }\n\n if (ggml_cuda_should_fuse_mul_mat_vec_q(up)) {\n ggml_cuda_mm_fusion_args_host fusion_data{};\n fusion_data.gate = gate->src[0];\n fusion_data.glu_op = ggml_get_glu_op(glu);\n\n ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data);\n fused_mul_mat_vec = true;\n fused_node_count = 3;\n break;\n }\n }\n }\n\n if (fused_mul_mat_vec) {\n i += fused_node_count - 1;\n continue;\n }\n\n fused_mul_mat_vec = false;\n fused_node_count = 0;\n\n for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) {\n const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID;\n\n if (!ggml_can_fuse(cgraph, i, { op, bias_op })) {\n continue;\n }\n\n ggml_tensor * mm_node = cgraph->nodes[i];\n ggml_tensor * bias_node = cgraph->nodes[i + 1];\n\n ggml_tensor * bias_tensor = nullptr;\n if (bias_op == GGML_OP_ADD) {\n if (bias_node->src[0] == mm_node) {\n bias_tensor = bias_node->src[1];\n } else if (bias_node->src[1] == mm_node) {\n bias_tensor = bias_node->src[0];\n } else {\n continue;\n }\n } else {\n if (bias_node->src[0] != mm_node) {\n continue;\n }\n bias_tensor = bias_node->src[1];\n }\n\n const ggml_tensor * src0 = mm_node->src[0];\n const ggml_tensor * src1 = mm_node->src[1];\n const ggml_tensor * ids = mm_node->src[2];\n\n if (bias_op == GGML_OP_ADD_ID && bias_node->src[2] != ids) {\n continue;\n }\n\n if (bias_op == GGML_OP_ADD && !ggml_are_same_shape(bias_node->src[0], bias_node->src[1])) {\n continue;\n }\n\n ggml_cuda_mm_fusion_args_host fusion_data{};\n fusion_data.x_bias = bias_tensor;\n\n if (ggml_cuda_should_fuse_mul_mat_vec_f(mm_node)) {\n ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, bias_node, &fusion_data);\n fused_mul_mat_vec = true;\n fused_node_count = 2;\n break;\n }\n\n if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) {\n ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, bias_node, &fusion_data);\n fused_mul_mat_vec = true;\n fused_node_count = 2;\n break;\n }\n }\n\n if (fused_mul_mat_vec) {\n i += fused_node_count - 1;\n continue;\n }\n\n if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD}, {})) {\n ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i+1], cgraph->nodes[i+2]);\n i += 2;\n continue;\n }\n\n if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL}, {})) {\n ggml_cuda_op_rms_norm_fused(*cuda_ctx, node, cgraph->nodes[i+1]);\n i++;\n continue;\n }\n\n if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SSM_CONV, GGML_OP_UNARY }, { GGML_UNARY_OP_SILU })) {\n ggml_cuda_op_ssm_conv(*cuda_ctx, node, cgraph->nodes[i+1]);\n i++;\n continue;\n }\n\n if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SILU }) ||\n ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SIGMOID }) ||\n ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SOFTPLUS })) {\n ggml_cuda_op_unary_mul(*cuda_ctx, node, cgraph->nodes[i+1]);\n i++;\n continue;\n }\n\n if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE }, { GGML_UNARY_OP_TANH })) {\n i += 2;\n ggml_cuda_op_softcap(*cuda_ctx, cgraph->nodes[i], node);\n continue;\n }\n }\n#ifndef NDEBUG\n assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n if (node->src[j] != nullptr) {\n assert(node->src[j]->buffer);\n assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) ||\n ggml_backend_buft_is_cuda_split(node->src[j]->buffer->buft) || (integrated && ggml_backend_buft_is_cuda_host(node->src[j]->buffer->buft)));\n }\n }\n#else\n GGML_UNUSED(integrated);\n#endif // NDEBUG\n\n bool ok = ggml_cuda_compute_forward(*cuda_ctx, node);\n if (!ok) {\n GGML_LOG_ERROR(\"%s: op not supported %s (%s)\\n\", __func__, node->name, ggml_op_name(node->op));\n }\n GGML_ASSERT(ok);\n\n if (!is_concurrent_event_active) {\n try_launch_concurrent_event(node);\n }\n }\n }\n\n#ifdef USE_CUDA_GRAPH\n ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key);\n if (use_cuda_graph && cuda_graph_update_required) { // End CUDA graph capture\n if (graph->graph != nullptr) {\n CUDA_CHECK(cudaGraphDestroy(graph->graph));\n graph->graph = nullptr;\n }\n\n CUDA_CHECK(cudaStreamEndCapture(cuda_ctx->stream(), &graph->graph));\n graph_evaluated_or_captured = true; // CUDA graph has been captured\n\n std::lock_guard lock(ggml_cuda_lock);\n if (ggml_cuda_lock_counter.fetch_sub(1, std::memory_order_relaxed) == 1) {\n ggml_cuda_lock_cv.notify_all();\n }\n } else {\n graph_evaluated_or_captured = true; // ggml graph has been directly evaluated\n }\n }\n\n if (use_cuda_graph) {\n ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key);\n if (graph->instance == nullptr) { // Create executable graph from captured graph.\n CUDA_CHECK(cudaGraphInstantiate(&graph->instance, graph->graph, NULL, NULL, 0));\n }\n if (cuda_graph_update_required) { // Update graph executable\n ggml_cuda_graph_update_executable(cuda_ctx, graph_key);\n }\n // Launch graph\n CUDA_CHECK(cudaGraphLaunch(graph->instance, cuda_ctx->stream()));\n#else\n GGML_UNUSED(graph_key);\n graph_evaluated_or_captured = true;\n#endif // USE_CUDA_GRAPH\n }\n}\n\n#ifdef USE_CUDA_GRAPH\nstatic bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) {\n ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key);\n\n if (graph->graph == nullptr) {\n if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_AMPERE) {\n if (!graph->disable_due_to_gpu_arch) {\n GGML_LOG_DEBUG(\"%s: disabling CUDA graphs due to GPU architecture\\n\", __func__);\n }\n graph->disable_due_to_gpu_arch = true;\n }\n }\n\n return graph->is_enabled();\n}\n#endif // USE_CUDA_GRAPH\n\nstatic enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {\n ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;\n\n ggml_cuda_set_device(cuda_ctx->device);\n\n bool use_cuda_graph = false;\n bool cuda_graph_update_required = false;\n const void * graph_key = nullptr;\n\n#ifdef USE_CUDA_GRAPH\n graph_key = ggml_cuda_graph_get_key(cgraph);\n\n ggml_cuda_graph_set_enabled(cuda_ctx, graph_key);\n\n ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key);\n if (graph->is_enabled()) {\n const bool graph_compatible = ggml_cuda_graph_check_compability(cgraph);\n if (graph_compatible) {\n const bool properties_changed = ggml_cuda_graph_update_required(cuda_ctx, cgraph);\n\n if (!graph->warmup_complete) {\n // Warmup: need at least 2 calls with no property change on the 2nd call\n if (!properties_changed) {\n graph->warmup_complete = true;\n GGML_LOG_DEBUG(\"%s: CUDA graph warmup complete\\n\", __func__);\n use_cuda_graph = true;\n cuda_graph_update_required = true;\n }\n // else: properties changed or first call - execute directly (use_cuda_graph stays false)\n } else {\n // Post-warmup: normal CUDA graph operation\n if (properties_changed) {\n // Properties changed - reset warmup, execute directly until stable again\n graph->warmup_complete = false;\n GGML_LOG_DEBUG(\"%s: CUDA graph warmup reset\\n\", __func__);\n } else {\n use_cuda_graph = true;\n cuda_graph_update_required = graph->instance == nullptr;\n }\n }\n }\n }\n#endif // USE_CUDA_GRAPH\n\n if (use_cuda_graph && cuda_graph_update_required) {\n // Start CUDA graph capture\n {\n std::lock_guard lock(ggml_cuda_lock);\n ggml_cuda_lock_counter.fetch_add(1, std::memory_order_relaxed);\n }\n\n CUDA_CHECK(cudaStreamBeginCapture(cuda_ctx->stream(), cudaStreamCaptureModeRelaxed));\n }\n\n ggml_cuda_graph_evaluate_and_capture(cuda_ctx, cgraph, use_cuda_graph, cuda_graph_update_required, graph_key);\n\n return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_cuda_event_record(ggml_backend_t backend, ggml_backend_event_t event) {\n ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;\n\n CUDA_CHECK(cudaEventRecord((cudaEvent_t)event->context, cuda_ctx->stream()));\n}\n\nstatic void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {\n ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;\n\n if (ggml_backend_is_cuda(backend)) {\n CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx->stream(), (cudaEvent_t)event->context, 0));\n } else {\n#if 0\n // untested\n auto wait_fn = [](void * user_data) {\n ggml_backend_event_t event = (ggml_backend_event_t)user_data;\n ggml_backend_event_synchronize(event);\n };\n\n CUDA_CHECK(cudaLaunchHostFunc(cuda_ctx->stream(), wait_fn, event));\n#endif\n GGML_ABORT(\"fatal error\");\n }\n}\n\nstatic void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) {\n ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;\n\n#ifdef USE_CUDA_GRAPH\n const void * graph_key = ggml_cuda_graph_get_key(cgraph);\n const bool use_cuda_graph = ggml_cuda_graph_set_enabled(cuda_ctx, graph_key);\n#else\n const bool use_cuda_graph = false;\n GGML_UNUSED(cuda_ctx);\n GGML_UNUSED(cgraph);\n#endif\n\n static bool enable_graph_optimization = [] {\n const char * env = getenv(\"GGML_CUDA_GRAPH_OPT\");\n return env != nullptr && atoi(env) == 1;\n }();\n\n if (!enable_graph_optimization) {\n return;\n }\n\n ggml_cuda_stream_context & stream_context = cuda_ctx->stream_context();\n stream_context.reset();\n\n if (!use_cuda_graph || ggml_backend_cuda_get_device_count() != 1) {\n return;\n }\n\n // number of out-degrees for a particular node\n std::unordered_map fan_out;\n // reverse mapping of node to index in the cgraph\n std::unordered_map node_indices;\n\n const auto & is_noop = [](const ggml_tensor * node) -> bool {\n return ggml_is_empty(node) || node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE ||\n node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE;\n };\n\n const auto & depends_on = [](const ggml_tensor * dst, const ggml_tensor * src) -> bool {\n for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) {\n if (dst->src[s] == src) {\n return true;\n }\n }\n // implicit dependency if they view the same tensor\n const ggml_tensor * dst2 = dst->view_src ? dst->view_src : dst;\n const ggml_tensor * src2 = src->view_src ? src->view_src : src;\n if (dst2 == src2) {\n return true;\n }\n return false;\n };\n\n for (int node_idx = 0; node_idx < cgraph->n_nodes; node_idx++) {\n const ggml_tensor * node = cgraph->nodes[node_idx];\n node_indices[node] = node_idx;\n\n if (is_noop(node)) {\n continue;\n }\n for (int src_idx = 0; src_idx < GGML_MAX_SRC; ++src_idx) {\n const ggml_tensor * src = cgraph->nodes[node_idx]->src[src_idx];\n //TODO: check why nrows > 1 fails\n if (node && !is_noop(node) && ggml_nrows(node) <= 1) {\n fan_out[src] += 1;\n }\n }\n }\n\n // Target Q, K, V for concurrency\n // this is a more general way to find nodes which can be candidates for concurrency (although it has not been tested for anything else):\n // 1. find fan-out (fork) nodes where the same input is used at least N times (in QKV, it would be \"attn-norm\")\n // 2. find the join node, where 2 or more of the outputs are required (in QKV, this would \"KQ\" or \"flash-attn\")\n // 3. account for all branches from the fork to the join\n // 4. To extend lifetimes of the tensors, we interleave the branches (see below for more details)\n // 5. save the original cgraph and restore it in graph_compute, to enable fusion within streams\n // See discussion: https://github.com/ggml-org/llama.cpp/pull/16991#issuecomment-3522620030\n\n const int min_fan_out = 3;\n const int max_fan_out = 3;\n\n // store {fork_idx, join_idx}\n std::vector> concurrent_node_ranges;\n\n for (const auto & [root_node, count] : fan_out) {\n if (count >= min_fan_out && count <= max_fan_out) {\n const int root_node_idx = node_indices[root_node];\n\n // only optimize for attn_norm\n // TODO: make this more generic\n if (!strstr(root_node->name, \"attn_norm\")) {\n continue;\n }\n\n bool is_part_of_event = false;\n for (const auto & [start, end] : concurrent_node_ranges) {\n if (root_node_idx >= start && root_node_idx <= end) {\n is_part_of_event = true;\n }\n }\n\n if (is_part_of_event) {\n continue;\n }\n\n std::vector> nodes_per_branch;\n for (int i = root_node_idx + 1; i < cgraph->n_nodes; ++i) {\n const ggml_tensor * node = cgraph->nodes[i];\n if (!is_noop(node) && depends_on(node, root_node)) {\n nodes_per_branch.push_back({ node });\n }\n }\n\n GGML_ASSERT(nodes_per_branch.size() == (size_t) count);\n\n //find the join point\n const ggml_tensor * join_node = nullptr;\n\n const auto & belongs_to_branch = [&](const ggml_tensor * node,\n const std::vector & branch) -> bool {\n for (const ggml_tensor * n : branch) {\n if (depends_on(node, n)) {\n return true;\n }\n }\n return false;\n };\n\n for (int i = root_node_idx + 1; i < cgraph->n_nodes; ++i) {\n const ggml_tensor * curr_node = cgraph->nodes[i];\n\n int num_joins = 0;\n for (size_t branch_idx = 0; branch_idx < nodes_per_branch.size(); branch_idx++) {\n if (belongs_to_branch(curr_node, nodes_per_branch[branch_idx])) {\n num_joins++;\n }\n }\n\n if (num_joins >= 2) {\n join_node = curr_node;\n break;\n }\n\n bool found_branch = false;\n for (size_t branch_idx = 0; branch_idx < nodes_per_branch.size(); branch_idx++) {\n std::vector & branch_vec = nodes_per_branch[branch_idx];\n if (belongs_to_branch(curr_node, branch_vec)) {\n //continue accumulating\n if (std::find(branch_vec.begin(), branch_vec.end(), curr_node) == branch_vec.end()) {\n branch_vec.push_back(curr_node);\n }\n found_branch = true;\n }\n }\n\n if (!found_branch && is_noop(curr_node)) {\n // we can put it in any branch because it will be ignored\n nodes_per_branch[0].push_back({ curr_node });\n }\n }\n\n if (join_node) {\n //Create ggml_cuda_concurrent_event\n ggml_cuda_concurrent_event concurrent_event(nodes_per_branch.size());\n concurrent_event.join_node = join_node;\n\n for (size_t branch_idx = 0; branch_idx < nodes_per_branch.size(); branch_idx++) {\n for (const ggml_tensor * n : nodes_per_branch[branch_idx]) {\n concurrent_event.stream_mapping[n] = branch_idx + 1;\n }\n }\n\n int fork_node_idx = node_indices[root_node];\n int join_node_idx = node_indices[join_node];\n\n int current_branch_idx = 0;\n int current_node_idx = fork_node_idx + 1;\n const int n_branches = nodes_per_branch.size();\n\n int total_branch_nodes = 0;\n for (std::vector branch_nodes : nodes_per_branch) {\n total_branch_nodes += branch_nodes.size();\n }\n\n // there are other nodes in the middle which are unaccounted for\n // usually (cpy) nodes, then ignore this fork\n if (join_node_idx - fork_node_idx - 1 != total_branch_nodes) {\n GGML_LOG_DEBUG(\n \"Skipping %s because the number of nodes in the middle is not equal to the total number of \"\n \"branch nodes %d != %d\\n\",\n root_node->name, join_node_idx - fork_node_idx - 1, total_branch_nodes);\n continue;\n }\n\n // Save the original order of nodes in this region before interleaving\n // This is used later to restore grouping for fusion within streams\n concurrent_event.original_order.reserve(total_branch_nodes);\n for (int i = fork_node_idx + 1; i < join_node_idx; ++i) {\n concurrent_event.original_order.push_back(cgraph->nodes[i]);\n }\n\n std::unordered_map & concurrent_events = cuda_ctx->stream_context().concurrent_events;\n GGML_ASSERT(concurrent_events.find(root_node) == concurrent_events.end());\n concurrent_events.emplace(root_node, std::move(concurrent_event));\n GGML_LOG_DEBUG(\"Adding stream at node %s %p\\n\", root_node->name, root_node);\n concurrent_node_ranges.emplace_back(fork_node_idx, join_node_idx);\n\n // interleave tensors to extend lifetimes so that ggml graph doesn't recycle them\n // example transformation:\n // [attn-norm, QMul, QNorm, QRope, KMul, KNorm, KRope, VMul, attn] ->\n // [attn-norm, QMul, KMul, VMul, QNorm, VNorm, QRope, KRope, attn]\n while (current_node_idx < join_node_idx) {\n std::vector & branch_nodes = nodes_per_branch[current_branch_idx];\n\n bool has_node = false;\n for (std::vector branch_node : nodes_per_branch) {\n has_node |= branch_node.size() > 0;\n }\n\n GGML_ASSERT(has_node);\n\n if (branch_nodes.empty()) {\n current_branch_idx = (current_branch_idx + 1) % n_branches;\n continue;\n }\n\n cgraph->nodes[current_node_idx] = const_cast(branch_nodes.front());\n current_node_idx++;\n branch_nodes.erase(branch_nodes.begin());\n\n // append all empty nodes\n while (!branch_nodes.empty() && is_noop(branch_nodes.front())) {\n cgraph->nodes[current_node_idx] = const_cast(branch_nodes.front());\n current_node_idx++;\n branch_nodes.erase(branch_nodes.begin());\n }\n\n current_branch_idx = (current_branch_idx + 1) % n_branches;\n }\n }\n }\n }\n}\n\nstatic const ggml_backend_i ggml_backend_cuda_interface = {\n /* .get_name = */ ggml_backend_cuda_get_name,\n /* .free = */ ggml_backend_cuda_free,\n /* .set_tensor_async = */ ggml_backend_cuda_set_tensor_async,\n /* .get_tensor_async = */ ggml_backend_cuda_get_tensor_async,\n /* .cpy_tensor_async = */ ggml_backend_cuda_cpy_tensor_async,\n /* .synchronize = */ ggml_backend_cuda_synchronize,\n /* .graph_plan_create = */ NULL,\n /* .graph_plan_free = */ NULL,\n /* .graph_plan_update = */ NULL,\n /* .graph_plan_compute = */ NULL,\n /* .graph_compute = */ ggml_backend_cuda_graph_compute,\n /* .event_record = */ ggml_backend_cuda_event_record,\n /* .event_wait = */ ggml_backend_cuda_event_wait,\n /* .graph_optimize = */ ggml_backend_cuda_graph_optimize,\n};\n\nstatic ggml_guid_t ggml_backend_cuda_guid() {\n static ggml_guid guid = { 0x2c, 0xdd, 0xe8, 0x1c, 0x65, 0xb3, 0x65, 0x73, 0x6a, 0x12, 0x88, 0x61, 0x1c, 0xc9, 0xdc, 0x25 };\n return &guid;\n}\n\nbool ggml_backend_is_cuda(ggml_backend_t backend) {\n return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cuda_guid());\n}\n\nint ggml_backend_cuda_get_device_count() {\n return ggml_cuda_info().device_count;\n}\n\nvoid ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) {\n cudaDeviceProp prop;\n CUDA_CHECK(cudaGetDeviceProperties(&prop, device));\n snprintf(description, description_size, \"%s\", prop.name);\n}\n\nvoid ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) {\n ggml_cuda_set_device(device);\n\n CUDA_CHECK(cudaMemGetInfo(free, total));\n}\n\nbool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {\n if (getenv(\"GGML_CUDA_REGISTER_HOST\") == nullptr) {\n return false;\n }\n\n#if CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA) || defined(GGML_USE_HIP)\n cudaError_t err = cudaHostRegister(buffer, size, cudaHostRegisterPortable | cudaHostRegisterReadOnly);\n if (err != cudaSuccess) {\n // clear the error\n (void)cudaGetLastError();\n\n GGML_LOG_DEBUG(\"%s: failed to register %.2f MiB of pinned memory: %s\\n\", __func__,\n size / 1024.0 / 1024.0, cudaGetErrorString(err));\n return false;\n }\n return true;\n#else\n GGML_UNUSED(buffer);\n GGML_UNUSED(size);\n return false;\n#endif // CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)\n}\n\nvoid ggml_backend_cuda_unregister_host_buffer(void * buffer) {\n if (getenv(\"GGML_CUDA_REGISTER_HOST\") == nullptr) {\n return;\n }\n\n cudaError_t err = cudaHostUnregister(buffer);\n if (err != cudaSuccess) {\n // clear the error\n (void)cudaGetLastError();\n }\n}\n\n\n// backend device\n\nstruct ggml_backend_cuda_device_context {\n int device;\n std::string name;\n std::string description;\n std::string pci_bus_id;\n int op_offload_min_batch_size;\n};\n\nstatic const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) {\n ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;\n return ctx->name.c_str();\n}\n\nstatic const char * ggml_backend_cuda_device_get_description(ggml_backend_dev_t dev) {\n ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;\n return ctx->description.c_str();\n}\n\n#if defined(__linux__)\n// Helper function to get available memory from /proc/meminfo for UMA systems\nstatic bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_kb, long * free_swap_kb) {\n FILE * meminfo_file = nullptr;\n // 2KB buffer for reading /proc/meminfo since it does not report size info, should be enough\n const size_t BUFFER_SIZE = 2048;\n auto file_buffer = std::make_unique(BUFFER_SIZE);\n size_t bytes_read = 0;\n long huge_tlb_total_pages = -1;\n long huge_tlb_free_pages = -1;\n long huge_tlb_page_size = -1;\n\n if (available_memory_kb == nullptr || free_swap_kb == nullptr) {\n return false;\n }\n\n meminfo_file = fopen(\"/proc/meminfo\", \"r\");\n if (meminfo_file == nullptr) {\n GGML_LOG_ERROR(\"%s: failed to open /proc/meminfo\\n\", __func__);\n return false;\n }\n\n // Read file into buffer\n bytes_read = fread(file_buffer.get(), 1, BUFFER_SIZE - 1, meminfo_file);\n fclose(meminfo_file);\n\n if (bytes_read == 0) {\n GGML_LOG_ERROR(\"%s: failed to read from /proc/meminfo\\n\", __func__);\n return false;\n }\n file_buffer[bytes_read] = '\\0';\n\n *available_memory_kb = -1;\n *free_swap_kb = -1;\n\n // Parse the file buffer line by line\n char * line = file_buffer.get();\n char * line_next;\n while (line < file_buffer.get() + bytes_read) {\n // Find the end of the current line\n line_next = strchr(line, '\\n');\n if (line_next != nullptr) {\n *line_next = '\\0';\n line_next++;\n } else {\n line_next = file_buffer.get() + bytes_read;\n }\n\n long value;\n if (sscanf(line, \"MemAvailable: %ld kB\", &value) == 1) {\n *available_memory_kb = value;\n } else if (sscanf(line, \"SwapFree: %ld kB\", &value) == 1) {\n *free_swap_kb = value;\n } else if (sscanf(line, \"HugePages_Total: %ld\", &value) == 1) {\n huge_tlb_total_pages = value;\n } else if (sscanf(line, \"HugePages_Free: %ld\", &value) == 1) {\n huge_tlb_free_pages = value;\n } else if (sscanf(line, \"Hugepagesize: %ld kB\", &value) == 1) {\n huge_tlb_page_size = value;\n }\n\n line = line_next;\n }\n\n if (huge_tlb_total_pages != 0 && huge_tlb_total_pages != -1) {\n *available_memory_kb = huge_tlb_free_pages * huge_tlb_page_size;\n\n // Hugetlbfs pages are not swappable.\n *free_swap_kb = 0;\n }\n\n GGML_LOG_DEBUG(\"%s: final available_memory_kb: %ld\\n\", __func__, *available_memory_kb);\n return true;\n}\n#endif // defined(__linux__)\n\nstatic void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;\n ggml_cuda_set_device(ctx->device);\n CUDA_CHECK(cudaMemGetInfo(free, total));\n\n// ref: https://github.com/ggml-org/llama.cpp/pull/17368\n#if defined(__linux__)\n // Check if this is a UMA (Unified Memory Architecture) system\n cudaDeviceProp prop;\n CUDA_CHECK(cudaGetDeviceProperties(&prop, ctx->device));\n\n // Check if UMA is explicitly enabled via environment variable\n bool uma_env = getenv(\"GGML_CUDA_ENABLE_UNIFIED_MEMORY\") != nullptr;\n bool is_uma = prop.integrated > 0 || uma_env;\n\n if (is_uma) {\n // For UMA systems (like DGX Spark), use system memory info\n long available_memory_kb = 0;\n long free_swap_kb = 0;\n\n if (ggml_backend_cuda_get_available_uma_memory(&available_memory_kb, &free_swap_kb) && available_memory_kb > 0) {\n *free = (size_t)available_memory_kb * 1024;\n } else {\n GGML_LOG_ERROR(\"%s: /proc/meminfo reading failed, using cudaMemGetInfo\\n\", __func__);\n }\n }\n#endif // defined(__linux__)\n\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend_dev_t dev) {\n GGML_UNUSED(dev);\n return GGML_BACKEND_DEVICE_TYPE_GPU;\n}\n\nstatic void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {\n ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;\n\n props->name = ggml_backend_cuda_device_get_name(dev);\n props->description = ggml_backend_cuda_device_get_description(dev);\n props->type = ggml_backend_cuda_device_get_type(dev);\n props->device_id = ctx->pci_bus_id.empty() ? nullptr : ctx->pci_bus_id.c_str();\n ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total);\n\n bool host_buffer = getenv(\"GGML_CUDA_NO_PINNED\") == nullptr;\n#ifdef GGML_CUDA_NO_PEER_COPY\n bool events = false;\n#else\n bool events = true;\n#endif\n\n props->caps = {\n /* .async = */ true,\n /* .host_buffer = */ host_buffer,\n /* .buffer_from_host_ptr = */ false,\n /* .events = */ events,\n };\n}\n\nstatic ggml_backend_t ggml_backend_cuda_device_init_backend(ggml_backend_dev_t dev, const char * params) {\n GGML_UNUSED(params);\n ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;\n return ggml_backend_cuda_init(ctx->device);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cuda_device_get_buffer_type(ggml_backend_dev_t dev) {\n ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;\n return ggml_backend_cuda_buffer_type(ctx->device);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cuda_device_get_host_buffer_type(ggml_backend_dev_t dev) {\n GGML_UNUSED(dev);\n return ggml_backend_cuda_host_buffer_type();\n}\n\n// TODO: move these functions here\nstatic bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {\n ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;\n\n // split buffers can only be used with GGML_OP_MUL_MAT\n if (op->op != GGML_OP_MUL_MAT) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (op->src[i] && op->src[i]->buffer && ggml_backend_buft_is_cuda_split(op->src[i]->buffer->buft)) {\n return false;\n }\n }\n }\n\n // check if all the sources are allocated on this device\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (op->src[i] && op->src[i]->buffer && ggml_backend_buft_is_cuda(op->src[i]->buffer->buft)) {\n ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)op->src[i]->buffer->buft->context;\n if (buft_ctx->device != dev_ctx->device) {\n return false;\n }\n }\n }\n\n switch (op->op) {\n case GGML_OP_UNARY:\n switch (ggml_get_unary_op(op)) {\n case GGML_UNARY_OP_ABS:\n case GGML_UNARY_OP_SGN:\n case GGML_UNARY_OP_NEG:\n case GGML_UNARY_OP_STEP:\n case GGML_UNARY_OP_GELU:\n case GGML_UNARY_OP_SILU:\n case GGML_UNARY_OP_RELU:\n case GGML_UNARY_OP_SIGMOID:\n case GGML_UNARY_OP_HARDSIGMOID:\n case GGML_UNARY_OP_HARDSWISH:\n case GGML_UNARY_OP_GELU_ERF:\n case GGML_UNARY_OP_GELU_QUICK:\n case GGML_UNARY_OP_TANH:\n case GGML_UNARY_OP_EXP:\n case GGML_UNARY_OP_EXPM1:\n case GGML_UNARY_OP_SOFTPLUS:\n case GGML_UNARY_OP_ELU:\n case GGML_UNARY_OP_XIELU:\n case GGML_UNARY_OP_FLOOR:\n case GGML_UNARY_OP_CEIL:\n case GGML_UNARY_OP_ROUND:\n case GGML_UNARY_OP_TRUNC:\n // TODO: should become:\n //return ggml_is_contiguous_rows(op->src[0]);\n return ggml_is_contiguous(op->src[0]);\n default:\n return false;\n }\n break;\n case GGML_OP_GLU:\n switch (ggml_get_glu_op(op)) {\n case GGML_GLU_OP_REGLU:\n case GGML_GLU_OP_GEGLU:\n case GGML_GLU_OP_SWIGLU:\n case GGML_GLU_OP_SWIGLU_OAI:\n case GGML_GLU_OP_GEGLU_ERF:\n case GGML_GLU_OP_GEGLU_QUICK:\n return ggml_is_contiguous_1(op->src[0]);\n default:\n return false;\n }\n break;\n case GGML_OP_MUL_MAT:\n case GGML_OP_MUL_MAT_ID:\n {\n struct ggml_tensor * a = op->src[0];\n struct ggml_tensor * b = op->src[1];\n if (a->buffer && ggml_backend_buft_is_cuda_split(a->buffer->buft)) {\n if (a->ne[2] > 1 || a->ne[3] > 1) {\n return false;\n }\n // for small weight matrices the active device can end up without any rows, don't use row split in those cases\n // this avoids some edge cases (and the performance would not be good anyways)\n ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) a->buffer->buft->context;\n int64_t row_low;\n int64_t row_high;\n get_row_split(&row_low, &row_high, a, buft_ctx->tensor_split, dev_ctx->device);\n if (row_low == row_high) {\n return false;\n }\n }\n if (b->type == GGML_TYPE_F16 && a->type != GGML_TYPE_F16) {\n return false;\n }\n#ifdef GGML_USE_MUSA\n const int cc = ggml_cuda_info().devices[dev_ctx->device].cc;\n if (b->ne[2]*b->ne[3] > 1 && !ggml_is_transposed(a) && !ggml_is_transposed(b)) {\n if (GGML_CUDA_CC_IS_QY1(cc) && op->op == GGML_OP_MUL_MAT &&\n a->type == GGML_TYPE_F16 && b->type == GGML_TYPE_F16) {\n return false;\n }\n if (GGML_CUDA_CC_IS_QY2(cc) && op->op == GGML_OP_MUL_MAT_ID &&\n a->type == GGML_TYPE_Q2_K && b->type == GGML_TYPE_F32) {\n return false;\n }\n }\n#endif // GGML_USE_MUSA\n switch (a->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_Q8_K:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ2_S:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_BF16:\n return true;\n default:\n return false;\n }\n } break;\n case GGML_OP_OUT_PROD:\n return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32;\n case GGML_OP_GET_ROWS:\n {\n switch (op->src[0]->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_F32:\n case GGML_TYPE_BF16:\n case GGML_TYPE_I32:\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n return true;\n default:\n return false;\n }\n } break;\n case GGML_OP_GET_ROWS_BACK:\n {\n return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->ne[2] == 1 && op->ne[3] == 1;\n } break;\n case GGML_OP_SET_ROWS:\n {\n return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 ||\n op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 ||\n op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) &&\n op->src[0]->type == GGML_TYPE_F32 &&\n (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32);\n } break;\n case GGML_OP_SET:\n {\n const ggml_type t = op->type;\n return (t == GGML_TYPE_F32 || t == GGML_TYPE_I32) &&\n t == op->src[0]->type &&\n t == op->src[1]->type;\n } break;\n case GGML_OP_CPY:\n {\n ggml_type src0_type = op->src[0]->type;\n ggml_type src1_type = op->src[1]->type;\n if ((src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16 || src0_type == GGML_TYPE_F16) &&\n (src1_type == GGML_TYPE_F32 || src1_type == GGML_TYPE_BF16 || src1_type == GGML_TYPE_F16)\n ) {\n return true;\n }\n if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q8_0) {\n return true;\n }\n if (src0_type == GGML_TYPE_Q8_0 && src1_type == GGML_TYPE_F32) {\n return true;\n }\n if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_0) {\n return true;\n }\n if (src0_type == GGML_TYPE_Q4_0 && src1_type == GGML_TYPE_F32) {\n return true;\n }\n if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_1) {\n return true;\n }\n if (src0_type == GGML_TYPE_Q4_1 && src1_type == GGML_TYPE_F32) {\n return true;\n }\n if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_0) {\n return true;\n }\n if (src0_type == GGML_TYPE_Q5_0 && src1_type == GGML_TYPE_F32) {\n return true;\n }\n if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_1) {\n return true;\n }\n if (src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_F32) {\n return true;\n }\n if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_IQ4_NL) {\n return true;\n }\n if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_I32) {\n return true;\n }\n if (src0_type == GGML_TYPE_I32 && src1_type == GGML_TYPE_F32) {\n return true;\n }\n if (src0_type == GGML_TYPE_I32 && src1_type == GGML_TYPE_I32) {\n return true;\n }\n if (src0_type == src1_type && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) {\n return true;\n }\n return false;\n } break;\n case GGML_OP_DUP:\n {\n ggml_type src0_type = op->src[0]->type;\n return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;\n } break;\n case GGML_OP_ARGMAX:\n case GGML_OP_COUNT_EQUAL:\n {\n return true;\n } break;\n case GGML_OP_REPEAT:\n {\n ggml_type src0_type = op->src[0]->type;\n return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;\n } break;\n case GGML_OP_REPEAT_BACK:\n return op->type == GGML_TYPE_F32 && (op->src[0]->ne[2]*op->src[0]->ne[3]) <= (1 << 15);\n case GGML_OP_CONCAT:\n {\n ggml_type src0_type = op->src[0]->type;\n return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;\n } break;\n case GGML_OP_CONV_TRANSPOSE_1D:\n {\n ggml_type src0_type = op->src[0]->type;\n ggml_type src1_type = op->src[1]->type;\n if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) {\n return true;\n }\n return false;\n } break;\n case GGML_OP_SILU_BACK:\n return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;\n break;\n case GGML_OP_NORM:\n case GGML_OP_RMS_NORM:\n case GGML_OP_L2_NORM:\n return true;\n case GGML_OP_RMS_NORM_BACK:\n return ggml_is_contiguous(op->src[0]);\n break;\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n case GGML_OP_ADD:\n case GGML_OP_ADD_ID:\n case GGML_OP_ADD1:\n case GGML_OP_SUB:\n case GGML_OP_MUL:\n case GGML_OP_DIV:\n case GGML_OP_SCALE:\n case GGML_OP_SQR:\n case GGML_OP_SQRT:\n case GGML_OP_SIN:\n case GGML_OP_COS:\n case GGML_OP_CLAMP:\n case GGML_OP_LOG:\n return true;\n case GGML_OP_SSM_SCAN: {\n if (op->src[3]->ne[0] == 1) {\n // Mamba2\n // (kernel only supports (d_state == 128 || d_state == 256) && d_head % 16 == 0)\n return (op->src[0]->ne[0] == 128 || op->src[0]->ne[0] == 256) && op->src[0]->ne[1] % 16 == 0;\n } else {\n // Mamba\n // (kernel only supports d_state == 16, d_head == 1, n_head % 128 == 0, n_group == 1)\n return op->src[0]->ne[0] == 16 && op->src[0]->ne[1] == 1 && op->src[0]->ne[2] % 128 == 0 && op->src[4]->ne[1] == 1;\n }\n }\n case GGML_OP_SSM_CONV: {\n // assumes d_inner % threads == 0\n return op->src[0]->ne[1] % 128 == 0;\n }\n case GGML_OP_CONT:\n return true;\n case GGML_OP_DIAG_MASK_INF:\n return true;\n case GGML_OP_SOFT_MAX:\n return true;\n case GGML_OP_SOFT_MAX_BACK: {\n float max_bias = 0.0f;\n memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float));\n return max_bias == 0.0f;\n }\n case GGML_OP_ROLL:\n if(op->src[0]->type == GGML_TYPE_F32) {\n return true;\n }\n return false;\n case GGML_OP_ROPE:\n case GGML_OP_ROPE_BACK: {\n return op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]);\n }\n case GGML_OP_IM2COL:\n case GGML_OP_IM2COL_3D:\n case GGML_OP_CONV_2D:\n case GGML_OP_CONV_2D_DW:\n case GGML_OP_CONV_TRANSPOSE_2D:\n case GGML_OP_POOL_2D:\n return true;\n case GGML_OP_ACC:\n // TODO: extend support like so:\n //return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]);\n return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);\n case GGML_OP_SUM:\n return ggml_is_contiguous_rows(op->src[0]);\n case GGML_OP_TOP_K:\n case GGML_OP_ARGSORT:\n#ifndef GGML_CUDA_USE_CUB\n return op->src[0]->ne[0] <= 1024;\n#else\n return true;\n#endif\n case GGML_OP_SUM_ROWS:\n case GGML_OP_MEAN:\n case GGML_OP_GROUP_NORM:\n return ggml_is_contiguous(op->src[0]);\n case GGML_OP_PAD:\n return true;\n case GGML_OP_UPSCALE:\n case GGML_OP_PAD_REFLECT_1D:\n case GGML_OP_ARANGE:\n case GGML_OP_TIMESTEP_EMBEDDING:\n case GGML_OP_LEAKY_RELU:\n case GGML_OP_RWKV_WKV6:\n case GGML_OP_GATED_LINEAR_ATTN:\n case GGML_OP_RWKV_WKV7:\n return true;\n case GGML_OP_GATED_DELTA_NET:\n //TODO: enable once MUSA compiler is solved https://github.com/ggml-org/llama.cpp/pull/19504#issuecomment-4018634327\n#ifdef GGML_USE_MUSA\n return false;\n#else\n return true;\n#endif // GGML_USE_MUSA\n case GGML_OP_FLASH_ATTN_EXT:\n return ggml_cuda_flash_attn_ext_supported(dev_ctx->device, op);\n case GGML_OP_CROSS_ENTROPY_LOSS:\n case GGML_OP_CROSS_ENTROPY_LOSS_BACK:\n case GGML_OP_OPT_STEP_ADAMW:\n case GGML_OP_OPT_STEP_SGD:\n case GGML_OP_FILL:\n case GGML_OP_CUMSUM:\n case GGML_OP_TRI:\n case GGML_OP_DIAG:\n case GGML_OP_SOLVE_TRI:\n return true;\n\n default:\n return false;\n }\n}\n\nstatic bool ggml_backend_cuda_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;\n const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated;\n return (((ggml_backend_buft_is_cuda(buft) || ggml_backend_buft_is_cuda_split(buft)) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft)));\n}\n\nstatic int64_t get_op_batch_size(const ggml_tensor * op) {\n switch (op->op) {\n case GGML_OP_GET_ROWS:\n return 0;\n case GGML_OP_MUL_MAT:\n return op->ne[1];\n case GGML_OP_MUL_MAT_ID:\n case GGML_OP_ROPE:\n case GGML_OP_ROPE_BACK:\n return op->ne[2];\n default:\n return ggml_nrows(op);\n }\n}\n\nstatic bool ggml_backend_cuda_device_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) {\n ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;\n\n return get_op_batch_size(op) >= dev_ctx->op_offload_min_batch_size;\n}\n\nstatic ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_t dev) {\n#ifdef GGML_CUDA_NO_PEER_COPY\n return nullptr;\n#else\n ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *)dev->context;\n\n ggml_cuda_set_device(dev_ctx->device);\n\n cudaEvent_t event;\n CUDA_CHECK(cudaEventCreateWithFlags(&event, cudaEventDisableTiming));\n\n return new ggml_backend_event {\n /* .device = */ dev,\n /* .context = */ event,\n };\n#endif\n}\n\nstatic void ggml_backend_cuda_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) {\n GGML_UNUSED(dev);\n\n CUDA_CHECK(cudaEventDestroy((cudaEvent_t)event->context));\n delete event;\n}\n\nstatic void ggml_backend_cuda_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) {\n GGML_UNUSED(dev);\n CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));\n}\n\nstatic const ggml_backend_device_i ggml_backend_cuda_device_interface = {\n /* .get_name = */ ggml_backend_cuda_device_get_name,\n /* .get_description = */ ggml_backend_cuda_device_get_description,\n /* .get_memory = */ ggml_backend_cuda_device_get_memory,\n /* .get_type = */ ggml_backend_cuda_device_get_type,\n /* .get_props = */ ggml_backend_cuda_device_get_props,\n /* .init_backend = */ ggml_backend_cuda_device_init_backend,\n /* .get_buffer_type = */ ggml_backend_cuda_device_get_buffer_type,\n /* .get_host_buffer_type = */ ggml_backend_cuda_device_get_host_buffer_type,\n /* .buffer_from_host_ptr = */ NULL,\n /* .supports_op = */ ggml_backend_cuda_device_supports_op,\n /* .supports_buft = */ ggml_backend_cuda_device_supports_buft,\n /* .offload_op = */ ggml_backend_cuda_device_offload_op,\n /* .event_new = */ ggml_backend_cuda_device_event_new,\n /* .event_free = */ ggml_backend_cuda_device_event_free,\n /* .event_synchronize = */ ggml_backend_cuda_device_event_synchronize,\n};\n\n// backend reg\n\nstruct ggml_backend_cuda_reg_context {\n std::vector devices;\n};\n\nstatic const char * ggml_backend_cuda_reg_get_name(ggml_backend_reg_t reg) {\n GGML_UNUSED(reg);\n return GGML_CUDA_NAME;\n}\n\nstatic size_t ggml_backend_cuda_reg_get_device_count(ggml_backend_reg_t reg) {\n ggml_backend_cuda_reg_context * ctx = (ggml_backend_cuda_reg_context *)reg->context;\n return ctx->devices.size();\n}\n\nstatic ggml_backend_dev_t ggml_backend_cuda_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n ggml_backend_cuda_reg_context * ctx = (ggml_backend_cuda_reg_context *)reg->context;\n GGML_ASSERT(index < ctx->devices.size());\n return ctx->devices[index];\n}\n\nstatic ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t reg) {\n static std::vector features = []() {\n std::vector features;\n #define _STRINGIFY(...) #__VA_ARGS__\n #define STRINGIFY(...) _STRINGIFY(__VA_ARGS__)\n\n #ifdef __CUDA_ARCH_LIST__\n features.push_back({ \"ARCHS\", STRINGIFY(__CUDA_ARCH_LIST__) });\n #endif\n\n #ifdef GGML_CUDA_FORCE_MMQ\n features.push_back({ \"FORCE_MMQ\", \"1\" });\n #endif\n\n #ifdef GGML_CUDA_FORCE_CUBLAS\n features.push_back({ \"FORCE_CUBLAS\", \"1\" });\n #endif\n\n #ifndef GGML_USE_VMM\n features.push_back({ \"NO_VMM\", \"1\" });\n #endif\n\n #ifdef GGML_CUDA_NO_PEER_COPY\n features.push_back({ \"NO_PEER_COPY\", \"1\" });\n #endif\n\n #ifdef GGML_CUDA_USE_GRAPHS\n features.push_back({ \"USE_GRAPHS\", \"1\" });\n #endif\n\n #ifdef GGML_CUDA_PEER_MAX_BATCH_SIZE\n features.push_back({ \"PEER_MAX_BATCH_SIZE\", STRINGIFY(GGML_CUDA_PEER_MAX_BATCH_SIZE) });\n #endif\n\n #ifdef GGML_CUDA_FA_ALL_QUANTS\n features.push_back({ \"FA_ALL_QUANTS\", \"1\" });\n #endif\n\n {\n const auto & info = ggml_cuda_info();\n for (int id = 0; id < info.device_count; ++id) {\n if (blackwell_mma_available(info.devices[id].cc)) {\n features.push_back({ \"BLACKWELL_NATIVE_FP4\", \"1\"});\n break;\n }\n }\n }\n\n #undef _STRINGIFY\n #undef STRINGIFY\n\n features.push_back({ nullptr, nullptr });\n\n return features;\n }();\n\n return features.data();\n\n GGML_UNUSED(reg);\n}\n\nstatic void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n GGML_UNUSED(reg);\n if (strcmp(name, \"ggml_backend_split_buffer_type\") == 0) {\n return (void *)ggml_backend_cuda_split_buffer_type;\n }\n if (strcmp(name, \"ggml_backend_register_host_buffer\") == 0) {\n return (void *)ggml_backend_cuda_register_host_buffer;\n }\n if (strcmp(name, \"ggml_backend_unregister_host_buffer\") == 0) {\n return (void *)ggml_backend_cuda_unregister_host_buffer;\n }\n if (strcmp(name, \"ggml_backend_get_features\") == 0) {\n return (void *)ggml_backend_cuda_get_features;\n }\n return nullptr;\n}\n\nstatic const ggml_backend_reg_i ggml_backend_cuda_reg_interface = {\n /* .get_name = */ ggml_backend_cuda_reg_get_name,\n /* .get_device_count = */ ggml_backend_cuda_reg_get_device_count,\n /* .get_device = */ ggml_backend_cuda_reg_get_device,\n /* .get_proc_address = */ ggml_backend_cuda_reg_get_proc_address,\n};\n\n// backend registry\nggml_backend_reg_t ggml_backend_cuda_reg() {\n static ggml_backend_reg reg;\n static bool initialized = false;\n\n {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n if (!initialized) {\n ggml_backend_cuda_reg_context * ctx = new ggml_backend_cuda_reg_context;\n const int min_batch_size = getenv(\"GGML_OP_OFFLOAD_MIN_BATCH\") ? atoi(getenv(\"GGML_OP_OFFLOAD_MIN_BATCH\")) : 32;\n\n for (int i = 0; i < ggml_cuda_info().device_count; i++) {\n ggml_backend_cuda_device_context * dev_ctx = new ggml_backend_cuda_device_context;\n dev_ctx->device = i;\n dev_ctx->name = GGML_CUDA_NAME + std::to_string(i);\n\n cudaDeviceProp prop;\n CUDA_CHECK(cudaGetDeviceProperties(&prop, i));\n dev_ctx->description = prop.name;\n\n char pci_bus_id[16] = {};\n snprintf(pci_bus_id, sizeof(pci_bus_id), \"%04x:%02x:%02x.0\", prop.pciDomainID, prop.pciBusID, prop.pciDeviceID);\n dev_ctx->pci_bus_id = pci_bus_id;\n dev_ctx->op_offload_min_batch_size = min_batch_size;\n\n ggml_backend_dev_t dev = new ggml_backend_device {\n /* .iface = */ ggml_backend_cuda_device_interface,\n /* .reg = */ ®,\n /* .context = */ dev_ctx\n };\n ctx->devices.push_back(dev);\n }\n\n reg = ggml_backend_reg {\n /* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_cuda_reg_interface,\n /* .context = */ ctx\n };\n }\n\n initialized = true;\n }\n\n return ®\n}\n\nggml_backend_t ggml_backend_cuda_init(int device) {\n if (device < 0 || device >= ggml_backend_cuda_get_device_count()) {\n GGML_LOG_ERROR(\"%s: invalid device %d\\n\", __func__, device);\n return nullptr;\n }\n\n ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context(device);\n if (ctx == nullptr) {\n GGML_LOG_ERROR(\"%s: failed to allocate context\\n\", __func__);\n return nullptr;\n }\n\n ggml_backend_t cuda_backend = new ggml_backend {\n /* .guid = */ ggml_backend_cuda_guid(),\n /* .iface = */ ggml_backend_cuda_interface,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cuda_reg(), device),\n /* .context = */ ctx,\n };\n\n return cuda_backend;\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_cuda_reg)\n" }, { "path": "src/ggml-cuda/gla.cu", "content": "#include \"common.cuh\"\n#include \"gla.cuh\"\n\ntemplate\nstatic __global__ void gated_linear_attn_f32(const int B, const int T, const int C, const int H, const float scale,\n const float * k, const float * v, const float * r, const float * td, const float * s, float * dst) {\n const int tid = threadIdx.x;\n const int bid = blockIdx.x;\n\n const int head_size = HEAD_SIZE;\n const int batch_i = bid / H;\n const int head_i = bid % H;\n const int state_size = C * head_size;\n const int n_seq_tokens = T / B;\n\n float state[head_size];\n __shared__ float _k[head_size], _r[head_size], _td[head_size];\n\n #pragma unroll\n for (int i = 0; i < head_size; i++) {\n state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid];\n }\n\n for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) {\n __syncthreads();\n _k[tid] = k[t];\n _r[tid] = r[t];\n _td[tid] = td[t];\n __syncthreads();\n\n const float _v = v[t];\n float y = 0;\n for (int j = 0; j < head_size; j += 4) {\n const float4 & k = (float4 &)(_k[j]);\n const float4 & r = (float4 &)(_r[j]);\n const float4 & td = (float4 &)(_td[j]);\n float4 & s = (float4 &)(state[j]);\n float4 kv;\n\n kv.x = k.x * _v;\n kv.y = k.y * _v;\n kv.z = k.z * _v;\n kv.w = k.w * _v;\n\n s.x = s.x * td.x + kv.x;\n s.y = s.y * td.y + kv.y;\n s.z = s.z * td.z + kv.z;\n s.w = s.w * td.w + kv.w;\n\n y += r.x * s.x;\n y += r.y * s.y;\n y += r.z * s.z;\n y += r.w * s.w;\n }\n dst[t] = y * scale;\n }\n\n #pragma unroll\n for (int i = 0; i < head_size; i++) {\n dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i];\n }\n}\n\nvoid ggml_cuda_op_gated_linear_attn(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const float * k_d = (const float *)dst->src[0]->data;\n const float * v_d = (const float *)dst->src[1]->data;\n const float * r_d = (const float *)dst->src[2]->data;\n const float * td_d = (const float *)dst->src[3]->data;\n const float * s_d = (const float *)dst->src[4]->data;\n\n const int64_t B = dst->src[4]->ne[1];\n const int64_t T = dst->src[0]->ne[2];\n const int64_t C = dst->ne[0];\n const int64_t H = dst->src[0]->ne[1];\n\n float scale;\n memcpy(&scale, (float*)dst->op_params, sizeof(float));\n\n float * dst_d = (float *)dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(dst->src[4]->type == GGML_TYPE_F32);\n GGML_ASSERT(C % H == 0);\n GGML_ASSERT(C / H == 64 || C / H == 128);\n\n\n if (C / H == 64) {\n gated_linear_attn_f32<64><<>>(B, T, C, H, scale, k_d, v_d, r_d, td_d, s_d, dst_d);\n } else {\n gated_linear_attn_f32<128><<>>(B, T, C, H, scale, k_d, v_d, r_d, td_d, s_d, dst_d);\n }\n}\n" }, { "path": "src/ggml-cuda/gla.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_gated_linear_attn(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/im2col.cu", "content": "#include \"im2col.cuh\"\n\n#define MAX_GRIDDIM_Z 65535\n\ntemplate \nstatic __global__ void im2col_kernel(\n const float * x, T * dst,\n int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH,\n int64_t IC_IH_IW, int64_t IH_IW, int64_t N_OH, int64_t KH_KW, int64_t IC_KH_KW,\n int s0, int s1, int p0, int p1, int d0, int d1) {\n const int64_t i = threadIdx.x + blockIdx.x * blockDim.x;\n if (i >= IC_KH_KW) {\n return;\n }\n\n const int64_t iic = i / (KH_KW);\n const int64_t rem = i - iic * KH_KW;\n const int64_t ikh = rem / KW;\n const int64_t ikw = rem - ikh * KW;\n\n const int64_t iow = blockIdx.y;\n for (int64_t iz = blockIdx.z; iz < N_OH; iz+=MAX_GRIDDIM_Z) {\n const int64_t in = iz / OH;\n const int64_t ioh = iz - in * OH;\n\n const int64_t iiw = iow * s0 + ikw * d0 - p0;\n const int64_t iih = ioh * s1 + ikh * d1 - p1;\n\n const int64_t offset_dst =\n ((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw;\n\n if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {\n dst[offset_dst] = 0.0f;\n } else {\n const int64_t offset_src = iic * IC_IH_IW + in * IH_IW;\n dst[offset_dst] = x[offset_src + iih * IW + iiw];\n }\n }\n\n GGML_UNUSED(IC);\n GGML_UNUSED(KH);\n}\n\n// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]\ntemplate \nstatic void im2col_cuda(const float * x, T* dst,\n int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,\n int64_t N, int64_t IC_IH_IW, int64_t IH_IW,\n int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {\n const int64_t IC_KH_KW = IC * KH * KW;\n const int64_t num_blocks = (IC_KH_KW + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;\n const int64_t N_OH = N * OH;\n const int64_t KH_KW = KW*KH;\n dim3 block_nums(num_blocks, OW, MIN(N_OH, MAX_GRIDDIM_Z));\n im2col_kernel<<>>(x, dst, IC, IW, IH, OH, OW, KW, KH,\n IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW,\n s0, s1, p0, p1, d0, d1);\n}\n\nstatic void im2col_cuda_f16(const float * x, half * dst,\n int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,\n int64_t N, int64_t IC_IH_IW, int64_t IH_IW,\n int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {\n\n im2col_cuda(x, dst, IW, IH, OW, OH, KW, KH, IC, N, IC_IH_IW, IH_IW, s0, s1, p0, p1, d0, d1, stream);\n}\n\nstatic void im2col_cuda_f32(const float * x, float * dst,\n int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,\n int64_t N, int64_t IC_IH_IW, int64_t IH_IW,\n int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {\n\n im2col_cuda(x, dst, IW, IH, OW, OH, KW, KH, IC, N, IC_IH_IW, IH_IW, s0, s1, p0, p1, d0, d1, stream);\n}\n\nvoid ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const float * src1_d = (const float *)src1->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);\n\n const int32_t s0 = ((const int32_t*)(dst->op_params))[0];\n const int32_t s1 = ((const int32_t*)(dst->op_params))[1];\n const int32_t p0 = ((const int32_t*)(dst->op_params))[2];\n const int32_t p1 = ((const int32_t*)(dst->op_params))[3];\n const int32_t d0 = ((const int32_t*)(dst->op_params))[4];\n const int32_t d1 = ((const int32_t*)(dst->op_params))[5];\n\n const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;\n\n const int64_t IC = src1->ne[is_2D ? 2 : 1];\n const int64_t IH = is_2D ? src1->ne[1] : 1;\n const int64_t IW = src1->ne[0];\n\n const int64_t KH = is_2D ? src0->ne[1] : 1;\n const int64_t KW = src0->ne[0];\n\n const int64_t OH = is_2D ? dst->ne[2] : 1;\n const int64_t OW = dst->ne[1];\n\n const int64_t IC_IH_IW = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32\n const int64_t N = src1->ne[is_2D ? 3 : 2];\n const int64_t IH_IW = src1->nb[is_2D ? 3 : 2] / 4; // nb is byte offset, src is type float32\n\n if(dst->type == GGML_TYPE_F16) {\n im2col_cuda_f16(src1_d, (half *) dst_d, IW, IH, OW, OH, KW, KH, IC, N, IC_IH_IW, IH_IW, s0, s1, p0, p1, d0, d1, stream);\n } else {\n im2col_cuda_f32(src1_d, (float *) dst_d, IW, IH, OW, OH, KW, KH, IC, N, IC_IH_IW, IH_IW, s0, s1, p0, p1, d0, d1, stream);\n }\n}\n\n// [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW]\ntemplate \nstatic __global__ void im2col_3d_kernel(\n const float * src, T * dst,\n int64_t N, int64_t IC, int64_t ID, int64_t IH, int64_t IW, int64_t OC,\n int64_t KD, int64_t KH, int64_t KW, int64_t OD, int64_t OH, int64_t OW,\n int64_t OH_OW, int64_t KD_KH_KW, int64_t ID_IH_IW, int64_t KH_KW, int64_t IH_IW, int64_t IC_ID_IH_IW,\n int64_t IC_KD_KH_KW, int64_t OW_KD_KH_KW, int64_t OD_OH_OW_IC_KD_KH_KW, int64_t OH_OW_IC_KD_KH_KW,\n int64_t OW_IC_KD_KH_KW, int64_t N_OD_OH, int64_t OD_OH,\n int64_t stride_q, int64_t stride_z, int64_t stride_y, int64_t stride_x,\n int s0, int s1, int s2, int p0, int p1, int p2, int d0, int d1, int d2) {\n const int64_t i = threadIdx.x + blockIdx.x * blockDim.x;\n if (i >= IC_KD_KH_KW) {\n return;\n }\n GGML_UNUSED(N); GGML_UNUSED(OC); GGML_UNUSED(OH_OW); GGML_UNUSED(OD); GGML_UNUSED(OW); GGML_UNUSED(KD); GGML_UNUSED(KH);\n GGML_UNUSED(ID_IH_IW); GGML_UNUSED(IH_IW); GGML_UNUSED(IC_ID_IH_IW); GGML_UNUSED(OW_KD_KH_KW);\n\n const int64_t iic = i / KD_KH_KW;\n const int64_t ikd = (i - iic * KD_KH_KW) / KH_KW;\n const int64_t ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / KW;\n const int64_t ikw = i % KW;\n\n const int64_t iow = blockIdx.y;\n for (int64_t iz = blockIdx.z; iz < N_OD_OH; iz+=MAX_GRIDDIM_Z) {\n const int64_t in = iz / OD_OH;\n const int64_t iod = (iz - in*OD_OH) / OH;\n const int64_t ioh = iz % OH;\n\n const int64_t iiw = iow * s0 + ikw * d0 - p0;\n const int64_t iih = ioh * s1 + ikh * d1 - p1;\n const int64_t iid = iod * s2 + ikd * d2 - p2;\n\n const int64_t offset_dst = in*OD_OH_OW_IC_KD_KH_KW + iod*OH_OW_IC_KD_KH_KW + ioh*OW_IC_KD_KH_KW + iow*IC_KD_KH_KW + iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw;\n\n if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) {\n dst[offset_dst] = 0.0f;\n } else {\n const int64_t offset_src = ((in * IC + iic) * stride_q) + (iid * stride_z) + (iih * stride_y) + (iiw * stride_x);\n dst[offset_dst] = src[offset_src];\n }\n }\n}\n\n// [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW]\ntemplate \nstatic void im2col_3d_cuda(const float * src, T* dst,\n int64_t N, int64_t IC, int64_t ID, int64_t IH, int64_t IW, int64_t OC,\n int64_t KD, int64_t KH, int64_t KW, int64_t OD, int64_t OH, int64_t OW,\n int64_t stride_q, int64_t stride_z, int64_t stride_y, int64_t stride_x,\n int s0, int s1, int s2, int p0, int p1, int p2, int d0, int d1, int d2, cudaStream_t stream) {\n const int64_t OH_OW = OH*OW;\n const int64_t KD_KH_KW = KD*KH*KW;\n const int64_t ID_IH_IW = ID*IH*IW;\n const int64_t KH_KW = KH*KW;\n const int64_t IH_IW = IH*IW;\n const int64_t IC_KD_KH_KW = IC*KD*KH*KW;\n const int64_t OW_KD_KH_KW = OW*KD*KH*KW;\n const int64_t N_OD_OH = N*OD*OH;\n const int64_t OD_OH = OD*OH;\n const int64_t IC_ID_IH_IW = IC*ID*IH*IW;\n const int64_t OD_OH_OW_IC_KD_KH_KW = OD*OH*OW*IC*KD*KH*KW;\n const int64_t OH_OW_IC_KD_KH_KW = OH*OW*IC*KD*KH*KW;\n const int64_t OW_IC_KD_KH_KW = OW*IC*KD*KH*KW;\n const int64_t num_blocks = (IC_KD_KH_KW + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;\n dim3 block_nums(num_blocks, OW, MIN(N_OD_OH, MAX_GRIDDIM_Z));\n im2col_3d_kernel<<>>(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW,\n OH_OW, KD_KH_KW, ID_IH_IW, KH_KW, IH_IW, IC_ID_IH_IW,\n IC_KD_KH_KW, OW_KD_KH_KW, OD_OH_OW_IC_KD_KH_KW,\n OH_OW_IC_KD_KH_KW, OW_IC_KD_KH_KW, N_OD_OH, OD_OH,\n stride_q, stride_z, stride_y, stride_x,\n s0, s1, s2, p0, p1, p2, d0, d1, d2);\n}\n\nstatic void im2col_3d_cuda_f16(const float * src, half * dst,\n int64_t N, int64_t IC, int64_t ID, int64_t IH, int64_t IW, int64_t OC,\n int64_t KD, int64_t KH, int64_t KW, int64_t OD, int64_t OH, int64_t OW,\n int64_t stride_q, int64_t stride_z, int64_t stride_y, int64_t stride_x,\n int s0, int s1, int s2, int p0, int p1, int p2, int d0, int d1, int d2, cudaStream_t stream) {\n\n im2col_3d_cuda(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW,\n stride_q, stride_z, stride_y, stride_x,\n s0, s1, s2, p0, p1, p2, d0, d1, d2, stream);\n}\n\nstatic void im2col_3d_cuda_f32(const float * src, float * dst,\n int64_t N, int64_t IC, int64_t ID, int64_t IH, int64_t IW, int64_t OC,\n int64_t KD, int64_t KH, int64_t KW, int64_t OD, int64_t OH, int64_t OW,\n int64_t stride_q, int64_t stride_z, int64_t stride_y, int64_t stride_x,\n int s0, int s1, int s2, int p0, int p1, int p2, int d0, int d1, int d2, cudaStream_t stream) {\n\n im2col_3d_cuda(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW,\n stride_q, stride_z, stride_y, stride_x,\n s0, s1, s2, p0, p1, p2, d0, d1, d2, stream);\n}\n\nvoid ggml_cuda_op_im2col_3d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const float * src1_d = (const float *)src1->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n const int32_t s2 = ((const int32_t *)(dst->op_params))[2];\n const int32_t p0 = ((const int32_t *)(dst->op_params))[3];\n const int32_t p1 = ((const int32_t *)(dst->op_params))[4];\n const int32_t p2 = ((const int32_t *)(dst->op_params))[5];\n const int32_t d0 = ((const int32_t *)(dst->op_params))[6];\n const int32_t d1 = ((const int32_t *)(dst->op_params))[7];\n const int32_t d2 = ((const int32_t *)(dst->op_params))[8];\n const int32_t IC = ((const int32_t *)(dst->op_params))[9];\n\n const int64_t N = ne13 / IC;\n const int64_t ID = ne12;\n const int64_t IH = ne11;\n const int64_t IW = ne10;\n\n const int64_t OC = ne03 / IC;\n const int64_t KD = ne02;\n const int64_t KH = ne01;\n const int64_t KW = ne00;\n\n const int64_t OD = ne3 / N;\n const int64_t OH = ne2;\n const int64_t OW = ne1;\n\n const size_t es = ggml_element_size(src1);\n const int64_t stride_x = src1->nb[0] / es;\n const int64_t stride_y = src1->nb[1] / es;\n const int64_t stride_z = src1->nb[2] / es;\n const int64_t stride_q = src1->nb[3] / es;\n\n if(dst->type == GGML_TYPE_F16) {\n im2col_3d_cuda_f16(src1_d, (half *) dst_d, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW,\n stride_q, stride_z, stride_y, stride_x,\n s0, s1, s2, p0, p1, p2, d0, d1, d2, stream);\n } else {\n im2col_3d_cuda_f32(src1_d, (float *) dst_d, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW,\n stride_q, stride_z, stride_y, stride_x,\n s0, s1, s2, p0, p1, p2, d0, d1, d2, stream);\n }\n}\n" }, { "path": "src/ggml-cuda/im2col.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_IM2COL_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\nvoid ggml_cuda_op_im2col_3d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/mean.cu", "content": "#include \"mean.cuh\"\n#include \"reduce_rows.cuh\"\n\n#ifdef GGML_CUDA_USE_CUB\n#include \nusing namespace cub;\n#endif // GGML_CUDA_USE_CUB\n\ntemplate __global__ void divide_by_count(T * result, size_t count) {\n *result /= static_cast(count);\n}\n\nvoid ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *) src0->data;\n float * dst_d = (float *) dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n const int64_t ncols = src0->ne[0];\n const int64_t nrows = ggml_nrows(src0);\n\n// Special case for reducing vectors\n#ifdef GGML_CUDA_USE_CUB\n#ifdef USE_CUDA_GRAPH\n cudaStreamCaptureStatus iscapturing;\n CUDA_CHECK(cudaStreamIsCapturing(stream, &iscapturing));\n#endif // USE_CUDA_GRAPH\n if ((nrows == 1) &&\n#ifdef USE_CUDA_GRAPH\n // Determine if CUDA graphs are effectively disabled for this context\n // (no graph instance exists and we're not capturing, OR graphs are explicitly enabled)\n (((ncols > 65536) &&\n (((!ctx.any_cuda_graph_has_instance()) && (iscapturing == cudaStreamCaptureStatusNone)) ||\n ctx.any_cuda_graph_enabled())) ||\n // CUDA graphs are enabled - use lower threshold\n ((ncols > 32768) &&\n !(((!ctx.any_cuda_graph_has_instance()) && (iscapturing == cudaStreamCaptureStatusNone)) ||\n ctx.any_cuda_graph_enabled())))) {\n#else\n (ncols > 65536)) {\n#endif // USE_CUDA_GRAPH\n // Single row - use device-wide reduction\n size_t tmp_size = 0;\n ggml_cuda_pool & pool = ctx.pool();\n\n DeviceReduce::Sum(nullptr, tmp_size, src0_d, dst_d, ncols, stream);\n\n ggml_cuda_pool_alloc tmp_alloc(pool, tmp_size);\n DeviceReduce::Sum(tmp_alloc.ptr, tmp_size, src0_d, dst_d, ncols, stream);\n\n // Divide by ncols\n divide_by_count<<<1, 1, 0, stream>>>(dst_d, ncols);\n return;\n }\n#endif // GGML_CUDA_USE_CUB\n\n const dim3 block_nums(nrows, 1, 1);\n\n const int id = ggml_cuda_get_device();\n const int nsm = ggml_cuda_info().devices[id].nsm;\n\n // Heuristic for block size selection to optimize occupancy.\n // See discussion in: https://github.com/ggml-org/llama.cpp/pull/15132\n if ((nrows / nsm) < 2) {\n const dim3 block_dims(512, 1, 1);\n reduce_rows_f32<<>>(src0_d, dst_d, ncols);\n } else {\n const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1);\n reduce_rows_f32<<>>(src0_d, dst_d, ncols);\n }\n}\n" }, { "path": "src/ggml-cuda/mean.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/mma.cuh", "content": "#pragma once\n// This file contains primitives that expose the tensor core PTX instructions for CUDA code.\n// The primitives can be used in a similar way as the nvcuda::wmma interface but with a well-defined memory layout.\n// The documentation for the PTX instructions can be found under:\n// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-multiply-accumulate-operation-using-mma-instruction\n//\n// Like with nvcuda::wmma there are three types of matrix tiles: A, B, and C with A @ B = C.\n// A is a row-major matrix with shape M x K.\n// B is a column-major matrix with shape K x N.\n// C is a column-major matrix with shape M x N.\n// A, B, and C are represented using the same fundamental data type: a row-major matrix with I rows and J columns.\n// Note that J is measured in physical 32 bit elements instead of logical elements.\n// The methods get_i and get_j can be used to get the physical 32 bit index of the lth element of a thread within a tile.\n// All matrix tiles have ne physical 32 bit elements per warp.\n//\n// As described in the PTX documentation, all pointers for load_ldmatrix must be to shared memory and aligned to 16 bytes.\n// The API in this file also assumes that the pointers for load_generic are aligned to 16 bytes, unaligned pointers are considered undefined behavior.\n\n#include \"common.cuh\"\n\n// On Volta each warp is doing 4 8x8 mma operations in parallel.\n// The basic memory layout for a 32x8 output tile is to stack 4 input tiles in I direction and to mirror the B tile.\n// However, the i indices in this file are by default permuted to simplify the index calculations.\n// #define GGML_CUDA_MMA_NO_VOLTA_PERM\n\n#if CUDART_VERSION >= 11080\n\nstatic __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {\n int ret = 0;\n\n#ifdef TURING_MMA_AVAILABLE\n asm(\"movmatrix.sync.aligned.m8n8.trans.b16 %0, %1;\"\n : \"=r\"(ret) : \"r\"(x));\n#else\n GGML_UNUSED(x);\n NO_DEVICE_CODE;\n#endif // defined(TURING_MMA_AVAILABLE)\n return ret;\n}\n\n#else\n\nstatic __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {\n // Imagine transposing row-major matrix to column-major matrix.\n const int src_i_low = 2 * (threadIdx.x % 4);\n const int src_i_high = src_i_low + 1;\n const int src_j = threadIdx.x / 4;\n\n const int src_laneid_low = src_i_low * 4 + src_j / 2;\n const int src_laneid_high = src_i_high * 4 + src_j / 2;\n\n const int shift_low = ((src_j + 0) % 2) * 16;\n const int shift_high = ((src_j + 1) % 2) * 16;\n\n const int ret_low = (__shfl_sync(0xFFFFFFFF, x, src_laneid_low, WARP_SIZE) >> shift_low) & 0x0000FFFF;\n const int ret_high = (__shfl_sync(0xFFFFFFFF, x, src_laneid_high, WARP_SIZE) << shift_high) & 0xFFFF0000;\n\n return ret_low | ret_high;\n}\n\n#endif // CUDART_VERSION >= 11080\n\nstatic __device__ __forceinline__ half2 ggml_cuda_movmatrix(const half2 x) {\n half2 ret;\n *((int *) &ret) = ggml_cuda_movmatrix(*((const int *) &x));\n return ret;\n}\n\nnamespace ggml_cuda_mma {\n\n // Some architectures like Volta or CDNA3 perform multiple matrix multiplications per warp in parallel,\n // effectively the warp is being split into subgroups of threads that each perform a single mma instruction.\n // In those cases the data can be split in different ways across the warp.\n enum data_layout {\n // By default the data uses the I direction as its major dimension and the J direction as its minor dimension.\n // For the A/C matrices this means I major == row major, J major == column major.\n // For the B matrix this means I major == column major, J major == row major.\n // MIRRORED == Each data value is held exactly once per thread subgroup.\n DATA_LAYOUT_I_MAJOR = 0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for RDNA4 and CDNA.\n DATA_LAYOUT_J_MAJOR = 10, // Matrix C for CDNA and RDNA4, int and float matrix C for RDNA3.\n DATA_LAYOUT_I_MAJOR_MIRRORED = 20, // Volta, matrix A&B for RDNA3.\n DATA_LAYOUT_J_MAJOR_MIRRORED = 30,\n };\n // Implemented mma combinations are:\n // - (I_MAJOR, I_MAJOR) -> I_MAJOR\n // - (I_MAJOR, I_MAJOR_MIRRORED) -> I_MAJOR\n // - (I_MAJOR, J_MAJOR_MIRRORED) -> I_MAJOR\n\n static constexpr bool is_i_major(const data_layout dl) {\n return dl == DATA_LAYOUT_I_MAJOR ||\n dl == DATA_LAYOUT_I_MAJOR_MIRRORED;\n }\n\n static constexpr __device__ data_layout get_input_data_layout() {\n#if defined(RDNA3) || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n return DATA_LAYOUT_I_MAJOR_MIRRORED;\n#else\n return DATA_LAYOUT_I_MAJOR;\n#endif // defined(RDNA3) || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n }\n\n template \n struct tile {};\n\n template \n struct tile {\n static constexpr int I = I_;\n static constexpr int J = J_;\n static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR;\n\n#if defined(AMD_MFMA_AVAILABLE)\n static constexpr int ne = I * J / 64;\n T x[ne] = {0};\n\n static constexpr __device__ bool supported() {\n if (I == 64 && J == 2) return true;\n if (I == 16 && J == 8) return true;\n if (I == 32 && J == 4) return true;\n if (I == 16 && J == 16) return true;\n if (I == 32 && J == 32) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (I == 64 && J == 2) { // Special tile size to load <16, 4> as <16, 8>\n return threadIdx.x % 16;\n } else if constexpr (I == 16 && J == 8) {\n return threadIdx.x % 16;\n } else if constexpr (I == 32 && J == 4) {\n return threadIdx.x % 32;\n } else if constexpr (I == 16 && J == 16) {\n return threadIdx.x % 16;\n } else if constexpr (I == 32 && J == 32) {\n return threadIdx.x % 32;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 64 && J == 2) { // Special tile size to load <16, 4> as <16, 8>\n return (2 * ((threadIdx.x / 16) % 2) + l);\n } else if constexpr (I == 16 && J == 8) {\n return 2 * (threadIdx.x / 16) + l;\n } else if constexpr (I == 32 && J == 4) {\n return 2 * (threadIdx.x / 32) + l;\n } else if constexpr (I == 16 && J == 16) {\n return 4 * (threadIdx.x / 16) + l;\n } else if constexpr (I == 32 && J == 32) {\n return 4 * (threadIdx.x / 32) + 8 * (l / 4) + (l % 4);\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#elif __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n static constexpr int ne = I * J / 32;\n T x[ne] = {0};\n\n static constexpr __device__ bool supported() {\n if (I == 32 && J == 8) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (I == 32 && J == 8) {\n#ifdef GGML_CUDA_MMA_NO_VOLTA_PERM\n return (((threadIdx.x % 16) / 4) * 8) + ((threadIdx.x / 16) * 4) + (l & 2) + (threadIdx.x % 2);\n#else\n return (l & 2) + (threadIdx.x & ~2);\n#endif // GGML_CUDA_MMA_NO_VOLTA_PERM\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 32 && J == 8) {\n return (threadIdx.x & 2) + (l & (4 + 1));\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#elif defined(AMD_WMMA_AVAILABLE)\n static constexpr int ne = I * J / 32;\n T x[ne] = {0};\n\n static constexpr __device__ bool supported() {\n if (I == 16 && J == 16) return true;\n if (I == 16 && J == 8) return true;\n if (I == 16 && J == 4) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (supported()) {\n return threadIdx.x % 16;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 16 && J == 16) {\n#if defined(RDNA3)\n if constexpr (std::is_same_v || std::is_same_v) {\n // matrix C\n return 2 * l + (threadIdx.x / 16);\n } else {\n // matrix A&B\n return l;\n }\n#else\n // matrix C is the transposed matrix A&B on RDNA4\n return ne * (threadIdx.x / 16) + l;\n#endif // defined(RDNA3)\n } else if constexpr (I == 16 && J == 8) {\n // mmq input for RDNA4\n return ne * (threadIdx.x / 16) + l;\n } else if constexpr (I == 16 && J == 4) {\n return ne * (threadIdx.x / 16) + l;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#else\n static constexpr int ne = I * J / 32;\n T x[ne] = {0};\n\n static constexpr __device__ bool supported() {\n if (I == 8 && J == 4) return true;\n if (I == 8 && J == 8) return true;\n if (I == 16 && J == 8) return true;\n if (I == 16 && J == 16) return true;\n if (I == 32 && J == 8) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (I == 8 && J == 4) {\n return threadIdx.x / 4;\n } else if constexpr (I == 8 && J == 8) {\n return threadIdx.x / 4;\n } else if constexpr (I == 16 && J == 8) {\n return ((l / 2) * 8) + (threadIdx.x / 4);\n } else if constexpr (I == 16 && J == 16) {\n return (((l / 2) % 2) * 8) + (threadIdx.x / 4);\n } else if constexpr (I == 32 && J == 8) {\n return tile<16, 8, T>::get_i(l); // Memory layout simply repeated with same pattern in i direction.\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 8 && J == 4) {\n return threadIdx.x % 4;\n } else if constexpr (I == 8 && J == 8) {\n return (l * 4) + (threadIdx.x % 4);\n } else if constexpr (I == 16 && J == 8) {\n return ((threadIdx.x % 4) * 2) + (l % 2);\n } else if constexpr (I == 16 && J == 16) {\n return ((l / 4) * 8) + ((threadIdx.x % 4) * 2) + (l % 2);\n } else if constexpr (I == 32 && J == 8) {\n return tile<16, 8, T>::get_j(l); // Memory layout simply repeated with same pattern in i direction.\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#endif // defined(GGML_USE_HIP)\n };\n\n template \n struct tile {\n static constexpr int I = I_;\n static constexpr int J = J_;\n static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR;\n\n#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n static constexpr int ne = I * J / WARP_SIZE;\n half2 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n if (I == 32 && J == 4) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (I == 32 && J == 4) {\n#ifdef GGML_CUDA_MMA_NO_VOLTA_PERM\n return (((threadIdx.x % 16) / 4) * 8) + ((threadIdx.x / 16) * 4) + (threadIdx.x % 4);\n#else\n return threadIdx.x;\n#endif // GGML_CUDA_MMA_NO_VOLTA_PERM\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 32 && J == 4) {\n return l;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#elif defined(AMD_WMMA_AVAILABLE)\n static constexpr int ne = I * J / 32;\n half2 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n if (I == 16 && J == 8) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (I == 16 && J == 8) {\n return threadIdx.x % 16;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 16 && J == 8) {\n return ne * (threadIdx.x / 16) + l;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#elif defined(AMD_MFMA_AVAILABLE)\n static constexpr int ne = I * J / 64;\n half2 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n if (I == 16 && J == 8) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (I == 16 && J == 8) {\n return threadIdx.x % 16;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 16 && J == 8) {\n return ne * (threadIdx.x / 16) + l;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#else\n static constexpr int ne = I * J / WARP_SIZE;\n half2 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n if (I == 8 && J == 4) return true;\n if (I == 8 && J == 8) return true;\n if (I == 16 && J == 8) return true;\n if (I == 16 && J == 16) return true;\n if (I == 32 && J == 8) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (I == 8 && J == 8) {\n return threadIdx.x / 4;\n } else if constexpr (I == 16 && J == 4) {\n return (l * 8) + (threadIdx.x / 4);\n } else if constexpr (I == 16 && J == 8) {\n return ((l % 2) * 8) + (threadIdx.x / 4);\n } else if constexpr (I == 32 && J == 8) {\n return ((l / 4) * 16) + ((l % 2) * 8) + (threadIdx.x / 4);\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 8 && J == 8) {\n return (l * 4) + (threadIdx.x % 4);\n } else if constexpr (I == 16 && J == 4) {\n return threadIdx.x % 4;\n } else if constexpr (I == 16 && J == 8) {\n return ((l / 2) * 4) + (threadIdx.x % 4);\n } else if constexpr (I == 32 && J == 8) {\n return ((l & 2) * 2) + (threadIdx.x % 4);\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#endif // __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n };\n\n template \n struct tile {\n static constexpr int I = I_;\n static constexpr int J = J_;\n static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR;\n\n#if defined(AMD_WMMA_AVAILABLE)\n static constexpr int ne = tile::ne;\n nv_bfloat162 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n return tile::supported();\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n return tile::get_i(l);\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n return tile::get_j(l);\n }\n#elif defined(AMD_MFMA_AVAILABLE)\n static constexpr int ne = tile::ne;\n nv_bfloat162 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n return tile::supported();\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n return tile::get_i(l);\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n return tile::get_j(l);\n }\n#else\n static constexpr int ne = I * J / WARP_SIZE;\n nv_bfloat162 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n if (I == 8 && J == 8) return true;\n if (I == 16 && J == 4) return true;\n if (I == 16 && J == 8) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (I == 8 && J == 8) {\n return threadIdx.x / 4;\n } else if constexpr (I == 16 && J == 4) {\n return (l * 8) + (threadIdx.x / 4);\n } else if constexpr (I == 16 && J == 8) {\n return ((l % 2) * 8) + (threadIdx.x / 4);\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 8 && J == 8) {\n return (l * 4) + (threadIdx.x % 4);\n } else if constexpr (I == 16 && J == 4) {\n return threadIdx.x % 4;\n } else if constexpr (I == 16 && J == 8) {\n return ((l / 2) * 4) + (threadIdx.x % 4);\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#endif // defined(AMD_WMMA_AVAILABLE)\n };\n\n template \n struct tile {\n static constexpr int I = I_;\n static constexpr int J = J_;\n static constexpr data_layout dl = DATA_LAYOUT_J_MAJOR;\n\n static constexpr int ne = tile::ne;\n T x[ne] = {0};\n\n static constexpr __device__ bool supported() {\n return tile::supported();\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n return tile::get_j(l);\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n return tile::get_i(l);\n }\n };\n\n template \n struct tile {\n static constexpr int I = I_;\n static constexpr int J = J_;\n static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;\n\n // RDNA3\n static constexpr int ne = I * J / 32 * 2;\n\n T x[ne] = {0};\n\n static constexpr __device__ bool supported() {\n if (I == 16 && J == 16) return true;\n if (I == 16 && J == 8) return true;\n if (I == 16 && J == 4) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int /*l*/) {\n if constexpr (supported()) {\n return threadIdx.x % 16;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (supported()) {\n return l;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n };\n\n template \n struct tile {\n static constexpr int I = I_;\n static constexpr int J = J_;\n static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;\n#if defined(RDNA3)\n static constexpr int ne = tile::ne;\n\n half2 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n return tile::supported();\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n return tile::get_i(l);\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n return tile::get_j(l);\n }\n#else // Volta\n static constexpr int ne = I * J / (WARP_SIZE/4);\n\n half2 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n if (I == 8 && J == 4) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int /*l*/) {\n if constexpr (I == 8 && J == 4) {\n return ((threadIdx.x / 16) * 4) + (threadIdx.x % 4);\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 8 && J == 4) {\n return l;\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n#endif // defined(RDNA3)\n };\n\n template \n struct tile {\n static constexpr int I = I_;\n static constexpr int J = J_;\n static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;\n static constexpr int ne = tile::ne;\n\n nv_bfloat162 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n return tile::supported();\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n return tile::get_i(l);\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n return tile::get_j(l);\n }\n };\n\n template \n struct tile {\n static constexpr int I = I_;\n static constexpr int J = J_;\n static constexpr data_layout dl = DATA_LAYOUT_J_MAJOR_MIRRORED;\n static constexpr int ne = I * J / (WARP_SIZE/4);\n\n half2 x[ne] = {{0.0f, 0.0f}};\n\n static constexpr __device__ bool supported() {\n if (I == 8 && J == 4) return true;\n return false;\n }\n\n static __device__ __forceinline__ int get_i(const int l) {\n if constexpr (I == 8 && J == 4) {\n return ((l / 2) * 4) + (threadIdx.x % 4);\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n\n static __device__ __forceinline__ int get_j(const int l) {\n if constexpr (I == 8 && J == 4) {\n return ((threadIdx.x / 16) * 2) + (l % 2);\n } else {\n NO_DEVICE_CODE;\n return -1;\n }\n }\n };\n\n#if defined(TURING_MMA_AVAILABLE)\n template \n static __device__ __forceinline__ tile get_half2(const tile & tile_float) {\n tile ret;\n#pragma unroll\n for (int l0 = 0; l0 < tile_float.ne; l0 += 2) {\n ret.x[l0/2] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]);\n }\n return ret;\n }\n\n static __device__ __forceinline__ tile<8, 8, half2> get_transposed(const tile<16, 4, half2> & t) {\n tile<8, 8, half2> ret;\n ret.x[0] = ggml_cuda_movmatrix(t.x[0]);\n ret.x[1] = ggml_cuda_movmatrix(t.x[1]);\n\n return ret;\n }\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n template \n static __device__ __forceinline__ tile get_half2(const tile & tile_float) {\n tile ret;\n#pragma unroll\n for (int l0 = 0; l0 < tile_float.ne; l0 += 2) {\n ret.x[l0/2] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]);\n }\n return ret;\n }\n\n static __device__ __forceinline__ tile<8, 8, half2> get_transposed(const tile<16, 4, half2> & t) {\n NO_DEVICE_CODE;\n return tile<8, 8, half2>{};\n }\n#else // Volta\n template \n static __device__ __forceinline__ tile get_half2(const tile & tile_float) {\n tile ret;\n#pragma unroll\n for (int l0 = 0; l0 < tile_float.ne; l0 += 4) {\n ret.x[l0/2 + 0] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]);\n ret.x[l0/2 + 1] = make_half2(tile_float.x[l0 + 2], tile_float.x[l0 + 3]);\n\n // On Volta FP16 and FP32 tiles have a different memory layout,\n // for the conversion threads with an offset of 2 need to exchange half their values:\n ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)] = __shfl_xor_sync(\n 0xFFFFFFFF, ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)], 2, WARP_SIZE);\n }\n return ret;\n }\n#endif // defined(TURING_MMA_AVAILABLE)\n\n static __device__ __forceinline__ void make_identity_mat(tile<16, 8, half2> & t) {\n#if defined(RDNA4)\n const int row = t.get_i(0);\n const int left_right = t.get_j(0) / 4;\n const int up_down = row / 8;\n const int idx = row % 8;\n reinterpret_cast(t.x)[idx] = left_right == up_down ? 1.0f : 0.0f;\n#else\n GGML_UNUSED_VARS(t);\n NO_DEVICE_CODE;\n#endif // defined(RDNA4)\n }\n\n template \n static __device__ __forceinline__ void load_generic(tile & t, const T * __restrict__ xs0, const int stride) {\n#if defined(AMD_MFMA_AVAILABLE)\n if constexpr (I == 64 && J == 2) { // Special tile size to load <16, 4> as <16, 8>\n#pragma unroll\n for (int l = 0; l < t.ne; ++l) {\n t.x[l] = xs0[t.get_i(l)*stride + t.get_j(l)];\n }\n } else {\n ggml_cuda_memcpy_1(t.x, xs0 + t.get_i(0) * stride + t.get_j(0));\n }\n#elif defined(AMD_WMMA_AVAILABLE)\n // All wmma layout has contiguous data when i-major.\n if constexpr (is_i_major(dl)) {\n // the data must be aligned to 16 bytes when bigger than ggml_cuda_get_max_cpy_bytes()\n constexpr int aligned_copy_bytes = ggml_cuda_get_max_cpy_bytes();\n if constexpr (sizeof(t.x) > aligned_copy_bytes) {\n static_assert(sizeof(t.x) % aligned_copy_bytes == 0, \"bad type size\");\n constexpr int aligned_copy_count = sizeof(t.x)/aligned_copy_bytes;\n#pragma unroll\n for (int i = 0; i < aligned_copy_count; ++i) {\n ggml_cuda_memcpy_1(t.x + t.ne/aligned_copy_count*i, xs0 + t.get_i(0) * stride + t.get_j(t.ne/aligned_copy_count*i));\n }\n } else {\n ggml_cuda_memcpy_1(t.x, xs0 + t.get_i(0) * stride + t.get_j(0));\n }\n } else {\n#pragma unroll\n for (int l = 0; l < t.ne; ++l) {\n t.x[l] = xs0[t.get_i(l)*stride + t.get_j(l)];\n }\n }\n#else\n#pragma unroll\n for (int l = 0; l < t.ne; ++l) {\n t.x[l] = xs0[t.get_i(l)*stride + t.get_j(l)];\n }\n#endif // defined(AMD_MFMA_AVAILABLE)\n }\n\n template \n static __device__ __forceinline__ void load_ldmatrix(\n tile<8, 8, T> & t, const T * __restrict__ xs0, const int stride) {\n#ifdef TURING_MMA_AVAILABLE\n int * xi = (int *) t.x;\n const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + ((threadIdx.x / t.I) * (t.J / 2)) % t.J;\n asm volatile(\"ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];\"\n : \"=r\"(xi[0]), \"=r\"(xi[1])\n : \"l\"(xs));\n#else\n load_generic(t, xs0, stride);\n#endif // TURING_MMA_AVAILABLE\n }\n\n template \n static __device__ __forceinline__ void load_ldmatrix(\n tile<16, 4, T> & t, const T * __restrict__ xs0, const int stride) {\n#ifdef TURING_MMA_AVAILABLE\n int * xi = (int *) t.x;\n const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride;\n asm volatile(\"ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];\"\n : \"=r\"(xi[0]), \"=r\"(xi[1])\n : \"l\"(xs));\n#else\n#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n GGML_UNUSED_VARS(t, xs0, stride);\n NO_DEVICE_CODE;\n#else\n load_generic(t, xs0, stride);\n#endif // __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n#endif // TURING_MMA_AVAILABLE\n }\n\n template \n static __device__ __forceinline__ void load_ldmatrix(\n tile<16, 8, T, dl> & t, const T * __restrict__ xs0, const int stride) {\n#if defined(TURING_MMA_AVAILABLE)\n int * xi = (int * ) t.x;\n const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2);\n asm volatile(\"ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];\"\n : \"=r\"(xi[0]), \"=r\"(xi[1]), \"=r\"(xi[2]), \"=r\"(xi[3])\n : \"l\"(xs));\n#else\n#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n#if 1\n // TODO: more generic handling\n static_assert(sizeof(T) == 4, \"bad type size\");\n ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 0, xs0 + t.get_i(0)*stride + 0);\n ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 4, xs0 + t.get_i(4)*stride + 4);\n#else\n load_generic(t, xs0, stride);\n#endif // 1\n#else\n load_generic(t, xs0, stride);\n#endif // __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n#endif // TURING_MMA_AVAILABLE\n }\n\n static __device__ __forceinline__ void load_ldmatrix(\n tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) {\n ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride);\n }\n\n static __device__ __forceinline__ void load_ldmatrix(\n tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) {\n#pragma unroll\n for (int l0 = 0; l0 < t.ne; l0 += 2) {\n ggml_cuda_memcpy_1<2*sizeof(half2)>(t.x + l0, xs0 + t.get_i(l0)*stride + t.get_j(l0));\n }\n }\n\n static __device__ __forceinline__ void load_ldmatrix(\n tile<32, 4, half2> & t, const half2 * __restrict__ xs0, const int stride) {\n#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride);\n#else\n GGML_UNUSED_VARS(t, xs0, stride);\n NO_DEVICE_CODE;\n#endif // __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n }\n\n template \n static __device__ __forceinline__ void load_ldmatrix_trans(\n tile<16, 8, T> & t, const T * __restrict__ xs0, const int stride) {\n#ifdef TURING_MMA_AVAILABLE\n int * xi = (int * ) t.x;\n const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2);\n asm volatile(\"ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];\"\n : \"=r\"(xi[0]), \"=r\"(xi[2]), \"=r\"(xi[1]), \"=r\"(xi[3])\n : \"l\"(xs));\n#else\n GGML_UNUSED_VARS(t, xs0, stride);\n NO_DEVICE_CODE;\n#endif // TURING_MMA_AVAILABLE\n }\n\n static __device__ __forceinline__ void mma(\n tile<16, 8, int> & D, const tile<16, 4, int> & A, const tile<8, 4, int> & B) {\n#ifdef TURING_MMA_AVAILABLE\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n asm(\"mma.sync.aligned.m16n8k16.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};\"\n : \"+r\"(D.x[0]), \"+r\"(D.x[1]), \"+r\"(D.x[2]), \"+r\"(D.x[3])\n : \"r\"(A.x[0]), \"r\"(A.x[1]), \"r\"(B.x[0]));\n#else\n // On Turing m16n8k16 mma is not available, use 2x m8n8k16 mma instead:\n asm(\"mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};\"\n : \"+r\"(D.x[0]), \"+r\"(D.x[1])\n : \"r\"(A.x[0]), \"r\"(B.x[0]));\n asm(\"mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};\"\n : \"+r\"(D.x[2]), \"+r\"(D.x[3])\n : \"r\"(A.x[1]), \"r\"(B.x[0]));\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // TURING_MMA_AVAILABLE\n }\n\n static __device__ __forceinline__ void mma(\n tile<16, 8, int> & D, const tile<16, 8, int> & A, const tile<8, 8, int> & B) {\n#ifdef TURING_MMA_AVAILABLE\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n asm(\"mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\"\n : \"+r\"(D.x[0]), \"+r\"(D.x[1]), \"+r\"(D.x[2]), \"+r\"(D.x[3])\n : \"r\"(A.x[0]), \"r\"(A.x[1]), \"r\"(A.x[2]), \"r\"(A.x[3]), \"r\"(B.x[0]), \"r\"(B.x[1]));\n#else\n // On Turing m16n8k32 mma is not available, use 4x m8n8k16 mma instead:\n asm(\"mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};\"\n : \"+r\"(D.x[0]), \"+r\"(D.x[1])\n : \"r\"(A.x[0]), \"r\"(B.x[0]));\n asm(\"mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};\"\n : \"+r\"(D.x[2]), \"+r\"(D.x[3])\n : \"r\"(A.x[1]), \"r\"(B.x[0]));\n asm(\"mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};\"\n : \"+r\"(D.x[0]), \"+r\"(D.x[1])\n : \"r\"(A.x[2]), \"r\"(B.x[1]));\n asm(\"mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};\"\n : \"+r\"(D.x[2]), \"+r\"(D.x[3])\n : \"r\"(A.x[3]), \"r\"(B.x[1]));\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // TURING_MMA_AVAILABLE\n }\n\n static __device__ __forceinline__ void mma(\n tile<16, 4, half2> & D, const tile<16, 8, half2> & A, const tile<8, 8, half2> & B) {\n#ifdef TURING_MMA_AVAILABLE\n const int * Axi = (const int *) A.x;\n const int * Bxi = (const int *) B.x;\n int * Dxi = (int *) D.x;\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n asm(\"mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[0]), \"r\"(Bxi[1]));\n#else\n // On Turing m16n8k16 mma is not available, use 2x m8n8k8 mma instead:\n asm(\"mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Bxi[0]));\n asm(\"mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1])\n : \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[1]));\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // TURING_MMA_AVAILABLE\n }\n\n static __device__ __forceinline__ void mma(\n tile<16, 8, half2> & D, const tile<16, 8, half2> & A, const tile<16, 8, half2> & B) {\n#ifdef TURING_MMA_AVAILABLE\n const int * Axi = (const int *) A.x;\n const int * Bxi = (const int *) B.x;\n int * Dxi = (int *) D.x;\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n asm(\"mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[0]), \"r\"(Bxi[2]));\n asm(\"mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};\"\n : \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[1]), \"r\"(Bxi[3]));\n#else\n // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead:\n asm(\"mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Bxi[0]));\n asm(\"mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1])\n : \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[2]));\n asm(\"mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};\"\n : \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Bxi[1]));\n asm(\"mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};\"\n : \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[3]));\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#elif defined(AMD_WMMA_AVAILABLE)\n#if defined(RDNA4)\n using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;\n halfx8_t& acc_frag = reinterpret_cast(D.x[0]);\n const halfx8_t& a_frag = reinterpret_cast(A.x[0]);\n const halfx8_t& b_frag = reinterpret_cast(B.x[0]);\n acc_frag = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag);\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // defined(RDNA4)\n#elif defined(AMD_MFMA_AVAILABLE)\n // MFMA: FP16 input, FP32 accumulate, convert back to half2.\n using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;\n using floatx4_t = __attribute__((ext_vector_type(4))) float;\n\n // Convert existing half2 accumulator to float for MFMA:\n floatx4_t acc_f32;\n {\n const halfx4_t acc_h = reinterpret_cast(D.x[0]);\n#pragma unroll\n for (int i = 0; i < 4; ++i) {\n acc_f32[i] = (float)acc_h[i];\n }\n }\n\n const halfx4_t& a_frag = reinterpret_cast(A.x[0]);\n const halfx4_t& b_frag = reinterpret_cast(B.x[0]);\n acc_f32 = __builtin_amdgcn_mfma_f32_16x16x16f16(a_frag, b_frag, acc_f32, 0, 0, 0);\n\n // Convert back to half2:\n {\n halfx4_t result_h;\n#pragma unroll\n for (int i = 0; i < 4; ++i) {\n result_h[i] = (_Float16)acc_f32[i];\n }\n reinterpret_cast(D.x[0]) = result_h;\n }\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // TURING_MMA_AVAILABLE\n }\n\n template \n static __device__ __forceinline__ void mma(\n tile<16, 8, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<8, 8, float, dl_ab> & B) {\n#ifdef AMPERE_MMA_AVAILABLE\n const int * Axi = (const int *) A.x;\n const int * Bxi = (const int *) B.x;\n int * Dxi = (int *) D.x;\n asm(\"mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[0]), \"r\"(Bxi[1]));\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // AMPERE_MMA_AVAILABLE\n }\n\n template \n static __device__ __forceinline__ void mma(\n tile<16, 16, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<16, 8, float, dl_ab> & B) {\n#ifdef AMD_MFMA_AVAILABLE\n using floatx4_t = __attribute__((ext_vector_type(4))) float;\n floatx4_t& acc_frag = reinterpret_cast(D.x[0]);\n#if defined(CDNA3)\n using floatx2_t = __attribute__((ext_vector_type(2))) float;\n const floatx2_t& a_frag = reinterpret_cast(A.x[0]);\n const floatx2_t& b_frag = reinterpret_cast(B.x[0]);\n acc_frag = __builtin_amdgcn_mfma_f32_16x16x8_xf32(a_frag, b_frag, acc_frag, 0, 0, 0);\n#elif defined(CDNA2) || defined(CDNA1)\n#pragma unroll\n for (int i = 0; i < 2; ++i) {\n acc_frag = __builtin_amdgcn_mfma_f32_16x16x4f32(A.x[i], B.x[i], acc_frag, 0, 0, 0);\n }\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // defined(CDNA3)\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // AMD_MFMA_AVAILABLE\n }\n\n static __device__ __forceinline__ void mma_block_scaled(tile<16, 8, float> & D,\n const tile<16, 8, int> & A,\n const tile<8, 8, int> & B,\n uint32_t a_scale,\n uint32_t b_scale) {\n#ifdef BLACKWELL_MMA_AVAILABLE\n const int * Axi = (const int *) A.x;\n const int * Bxi = (const int *) B.x;\n float * Dxi = (float *) D.x;\n\n asm volatile(\n \"mma.sync.aligned.kind::mxf4.block_scale.scale_vec::2X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue8m0 \"\n \"{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, \"\n \"%10, {0, 0}, %11, {0, 0};\"\n : \"+f\"(Dxi[0]), \"+f\"(Dxi[1]), \"+f\"(Dxi[2]), \"+f\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[0]), \"r\"(Bxi[1]), \"r\"(a_scale), \"r\"(b_scale));\n#else\n GGML_UNUSED_VARS(D, A, B, a_scale, b_scale);\n#endif // BLACKWELL_MMA_AVAILABLE\n }\n\n static __device__ __forceinline__ void mma(\n tile<16, 8, float> & D, const tile<16, 8, half2> & A, const tile<8, 8, half2> & B) {\n#ifdef TURING_MMA_AVAILABLE\n const int * Axi = (const int *) A.x;\n const int * Bxi = (const int *) B.x;\n int * Dxi = (int *) D.x;\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n asm(\"mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[0]), \"r\"(Bxi[1]));\n#else\n // On Turing m16n8k16 mma is not available, use 2x m8n8k8 mma instead:\n asm(\"mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Bxi[0]));\n asm(\"mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[1]));\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // TURING_MMA_AVAILABLE\n }\n\n static __device__ __forceinline__ void mma(\n tile<16, 8, float> & D, const tile<16, 8, nv_bfloat162> & A, const tile<8, 8, nv_bfloat162> & B) {\n#ifdef AMPERE_MMA_AVAILABLE\n const int * Axi = (const int *) A.x;\n const int * Bxi = (const int *) B.x;\n int * Dxi = (int *) D.x;\n asm(\"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[0]), \"r\"(Bxi[1]));\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // AMPERE_MMA_AVAILABLE\n }\n\n template \n static __device__ __forceinline__ void mma(\n tile<16, 16, float, dl_d> & D, const tile<16, 8, half2, dl_ab> & A, const tile<16, 8, half2, dl_ab> & B) {\n#ifdef TURING_MMA_AVAILABLE\n const int * Axi = (const int *) A.x;\n const int * Bxi = (const int *) B.x;\n int * Dxi = (int *) D.x;\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n asm(\"mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[0]), \"r\"(Bxi[2]));\n asm(\"mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[4]), \"+r\"(Dxi[5]), \"+r\"(Dxi[6]), \"+r\"(Dxi[7])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[1]), \"r\"(Bxi[3]));\n#else\n // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead:\n asm(\"mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Bxi[0]));\n asm(\"mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[2]));\n asm(\"mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[4]), \"+r\"(Dxi[5]), \"+r\"(Dxi[6]), \"+r\"(Dxi[7])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Bxi[1]));\n asm(\"mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[4]), \"+r\"(Dxi[5]), \"+r\"(Dxi[6]), \"+r\"(Dxi[7])\n : \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[3]));\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#elif defined(AMD_WMMA_AVAILABLE)\n#if defined(RDNA4)\n using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;\n using floatx8_t = __attribute__((ext_vector_type(8))) float;\n floatx8_t& acc_frag = reinterpret_cast(D.x[0]);\n const halfx8_t& a_frag = reinterpret_cast(A.x[0]);\n const halfx8_t& b_frag = reinterpret_cast(B.x[0]);\n acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag);\n#elif defined(RDNA3)\n using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;\n using floatx8_t = __attribute__((ext_vector_type(8))) float;\n floatx8_t& acc_frag = reinterpret_cast(D.x[0]);\n const halfx16_t& a_frag = reinterpret_cast(A.x[0]);\n const halfx16_t& b_frag = reinterpret_cast(B.x[0]);\n acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a_frag, b_frag, acc_frag);\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // RDNA4\n#elif defined(AMD_MFMA_AVAILABLE)\n using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;\n using floatx4_t = __attribute__((ext_vector_type(4))) float;\n floatx4_t& acc_frag = reinterpret_cast(D.x[0]);\n const halfx4_t& a_frag = reinterpret_cast(A.x[0]);\n const halfx4_t& b_frag = reinterpret_cast(B.x[0]);\n acc_frag = __builtin_amdgcn_mfma_f32_16x16x16f16(a_frag, b_frag, acc_frag, 0, 0, 0);\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // TURING_MMA_AVAILABLE\n }\n\n template \n static __device__ __forceinline__ void mma(\n tile<16, 16, float, dl_d> & D, const tile<16, 8, nv_bfloat162, dl_ab> & A, const tile<16, 8, nv_bfloat162, dl_ab> & B) {\n#if defined(AMD_WMMA_AVAILABLE)\n#if defined(RDNA4)\n using bf16x8_t = __attribute__((ext_vector_type(8))) __bf16;\n using floatx8_t = __attribute__((ext_vector_type(8))) float;\n floatx8_t& acc_frag = reinterpret_cast(D.x[0]);\n const bf16x8_t& a_frag = reinterpret_cast(A.x[0]);\n const bf16x8_t& b_frag = reinterpret_cast(B.x[0]);\n acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12(a_frag, b_frag, acc_frag);\n#elif defined(RDNA3)\n using bf16x16_t = __attribute__((ext_vector_type(16))) __bf16;\n using floatx8_t = __attribute__((ext_vector_type(8))) float;\n floatx8_t& acc_frag = reinterpret_cast(D.x[0]);\n const bf16x16_t& a_frag = reinterpret_cast(A.x[0]);\n const bf16x16_t& b_frag = reinterpret_cast(B.x[0]);\n acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32(a_frag, b_frag, acc_frag);\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // defined(RDNA4)\n#elif defined(AMD_MFMA_AVAILABLE)\n using floatx4_t = __attribute__((ext_vector_type(4))) float;\n floatx4_t& acc_frag = reinterpret_cast(D.x[0]);\n#if defined(CDNA3) || defined(CDNA2)\n using bf16x4_t = __attribute__((ext_vector_type(4))) __bf16;\n const bf16x4_t& a_frag = reinterpret_cast(A.x[0]);\n const bf16x4_t& b_frag = reinterpret_cast(B.x[0]);\n acc_frag = __builtin_amdgcn_mfma_f32_16x16x16bf16_1k(a_frag, b_frag, acc_frag, 0, 0, 0);\n#elif defined(CDNA1)\n#pragma unroll\n for (int i = 0; i < 2; ++i) {\n using bf16x2_t = __attribute__((ext_vector_type(2))) __bf16;\n const bf16x2_t& a_frag = reinterpret_cast(A.x[i]);\n const bf16x2_t& b_frag = reinterpret_cast(B.x[i]);\n acc_frag = __builtin_amdgcn_mfma_f32_16x16x8bf16(a_frag, b_frag, acc_frag, 0, 0, 0);\n }\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // defined(CDNA3) || defined(CDNA2)\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // defined(AMD_WMMA_AVAILABLE)\n }\n\n template \n static __device__ __forceinline__ void mma(\n tile<16, 16, int, dl_d> & D, const tile<16, 8, int, dl_ab> & A, const tile<16, 8, int, dl_ab> & B) {\n#if defined(AMD_MFMA_AVAILABLE)\n using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;\n int32x4_t * acc = (int32x4_t *) D.x;\n#if defined(CDNA3)\n acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(((int64_t *) A.x)[0],\n ((int64_t *) B.x)[0],\n acc[0],\n 0, 0, 0);\n#elif defined(CDNA2) || defined(CDNA)\n acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0],\n B.x[0],\n acc[0],\n 0, 0, 0);\n acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[1],\n B.x[1],\n acc[0],\n 0, 0, 0);\n#endif // defined(CDNA3)\n\n#elif defined(AMD_WMMA_AVAILABLE)\n\n using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;\n int32x8_t * acc = (int32x8_t *) D.x;\n\n#if defined(RDNA4)\n using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;\n int32x2_t * a_vec = (int32x2_t *) A.x;\n int32x2_t * b_vec = (int32x2_t *) B.x;\n\n acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(\n true,\n a_vec[0],\n true,\n b_vec[0],\n acc[0],\n true\n );\n\n acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(\n true,\n a_vec[1],\n true,\n b_vec[1],\n acc[0],\n true\n );\n\n#elif defined(RDNA3)\n using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;\n int32x4_t * a_vec = (int32x4_t *) A.x;\n int32x4_t * b_vec = (int32x4_t *) B.x;\n\n acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(\n true,\n a_vec[0],\n true,\n b_vec[0],\n acc[0],\n true\n );\n\n acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(\n true,\n a_vec[1],\n true,\n b_vec[1],\n acc[0],\n true\n );\n#endif // RDNA4\n\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // AMD_MFMA_AVAILABLE\n }\n\n static __device__ __forceinline__ void mma(\n tile<32, 32, int> & D, const tile<32, 4, int> & A, const tile<32, 4, int> & B) {\n#if defined(AMD_MFMA_AVAILABLE)\n using int32x16_t = __attribute__((__vector_size__(16 * sizeof(int)))) int;\n int32x16_t * acc = (int32x16_t *) D.x;\n#if defined(CDNA3)\n acc[0] = __builtin_amdgcn_mfma_i32_32x32x16_i8(((int64_t *) A.x)[0],\n ((int64_t *) B.x)[0],\n acc[0],\n 0, 0, 0);\n#elif defined(CDNA2) || defined(CDNA)\n acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[0],\n B.x[0],\n acc[0],\n 0, 0, 0);\n acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[1],\n B.x[1],\n acc[0],\n 0, 0, 0);\n#endif // defined(CDNA3)\n\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // AMD_MFMA_AVAILABLE\n }\n\n template \n static __device__ __forceinline__ void mma(\n tile<32, J, T1> & D, const tile<32, K, T2> & A, const tile & B) {\n tile <16, J, T1> * D16 = reinterpret_cast< tile<16, J, T1> *>(&D);\n const tile<16, K, T2> * A16 = reinterpret_cast *>(&A);\n mma(D16[0], A16[0], B);\n mma(D16[1], A16[1], B);\n }\n\n static __device__ __forceinline__ void mma(\n tile<32, 8, float> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) {\n#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n const int * Axi = (const int *) A.x;\n const int * Bxi = (const int *) B.x;\n int * Dxi = (int *) D.x;\n asm(\"mma.sync.aligned.m8n8k4.row.col.f32.f16.f16.f32 \"\n \"{%0, %1, %2, %3, %4, %5, %6, %7}, {%8, %9}, {%10, %11}, {%0, %1, %2, %3, %4, %5, %6, %7};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3]), \"+r\"(Dxi[4]), \"+r\"(Dxi[5]), \"+r\"(Dxi[6]), \"+r\"(Dxi[7])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Bxi[0]), \"r\"(Bxi[1]));\n asm(\"mma.sync.aligned.m8n8k4.row.col.f32.f16.f16.f32 \"\n \"{%0, %1, %2, %3, %4, %5, %6, %7}, {%8, %9}, {%10, %11}, {%0, %1, %2, %3, %4, %5, %6, %7};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3]), \"+r\"(Dxi[4]), \"+r\"(Dxi[5]), \"+r\"(Dxi[6]), \"+r\"(Dxi[7])\n : \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[2]), \"r\"(Bxi[3]));\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n }\n\n static __device__ __forceinline__ void mma(\n tile<32, 4, half2> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & B) {\n#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n const int * Axi = (const int *) A.x;\n const int * Bxi = (const int *) B.x;\n int * Dxi = (int *) D.x;\n asm(\"mma.sync.aligned.m8n8k4.row.row.f16.f16.f16.f16 \"\n \"{%0, %1, %2, %3}, {%4, %5}, {%6, %7}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[0]), \"r\"(Axi[1]), \"r\"(Bxi[0]), \"r\"(Bxi[1]));\n asm(\"mma.sync.aligned.m8n8k4.row.row.f16.f16.f16.f16 \"\n \"{%0, %1, %2, %3}, {%4, %5}, {%6, %7}, {%0, %1, %2, %3};\"\n : \"+r\"(Dxi[0]), \"+r\"(Dxi[1]), \"+r\"(Dxi[2]), \"+r\"(Dxi[3])\n : \"r\"(Axi[2]), \"r\"(Axi[3]), \"r\"(Bxi[2]), \"r\"(Bxi[3]));\n#else\n GGML_UNUSED_VARS(D, A, B);\n NO_DEVICE_CODE;\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n }\n\n template \n static __device__ __forceinline__ void mma(\n tile<16, 16, int, dl_d> & D, const tile<16, 4, int, dl_ab> & A, const tile<16, 4, int, dl_ab> & B) {\n#if defined(AMD_WMMA_AVAILABLE)\n using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;\n int32x8_t * acc = (int32x8_t *) D.x;\n#if defined(RDNA4)\n using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;\n int32x2_t * a_vec = (int32x2_t *) A.x;\n int32x2_t * b_vec = (int32x2_t *) B.x;\n\n acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(\n true,\n a_vec[0],\n true,\n b_vec[0],\n acc[0],\n false\n );\n#elif defined(RDNA3)\n using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;\n int32x4_t * a_vec = (int32x4_t *) A.x;\n int32x4_t * b_vec = (int32x4_t *) B.x;\n\n acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(\n true,\n a_vec[0],\n true,\n b_vec[0],\n acc[0],\n false\n );\n#endif // RDNA4\n#else\n GGML_UNUSED(D);\n GGML_UNUSED(A);\n GGML_UNUSED(B);\n NO_DEVICE_CODE;\n#endif // AMD_WMMA_AVAILABLE\n }\n}\n" }, { "path": "src/ggml-cuda/mmf.cu", "content": "#include \"ggml.h\"\n#include \"mmf.cuh\"\n#include \"mmid.cuh\"\n\nstatic __forceinline__ int mmf_get_rows_per_block(const int cc) {\n if (GGML_CUDA_CC_IS_CDNA(cc)) {\n return MMF_ROWS_PER_BLOCK_CDNA;\n } else {\n return MMF_ROWS_PER_BLOCK;\n }\n}\n\nvoid ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {\n GGML_ASSERT( src1->type == GGML_TYPE_F32);\n GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n const size_t ts_src0 = ggml_type_size(src0->type);\n const size_t ts_src1 = ggml_type_size(src1->type);\n const size_t ts_dst = ggml_type_size(dst->type);\n\n GGML_ASSERT(ne13 == ne3);\n\n GGML_ASSERT( nb00 == ts_src0);\n GGML_ASSERT( nb10 == ts_src1);\n GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));\n GGML_ASSERT( nb0 == ts_dst);\n\n const float * src1_d = (const float *) src1->data;\n const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr;\n float * dst_d = (float *) dst->data;\n\n const int64_t s01 = src0->nb[1] / ts_src0;\n const int64_t s11 = src1->nb[1] / ts_src1;\n const int64_t s1 = dst->nb[1] / ts_dst;\n const int64_t s02 = src0->nb[2] / ts_src0;\n const int64_t s12 = src1->nb[2] / ts_src1;\n const int64_t s2 = dst->nb[2] / ts_dst;\n const int64_t s03 = src0->nb[3] / ts_src0;\n const int64_t s13 = src1->nb[3] / ts_src1;\n const int64_t s3 = dst->nb[3] / ts_dst;\n\n const int64_t ids_s0 = ids ? ids->nb[0] / ggml_type_size(ids->type) : 0;\n const int64_t ids_s1 = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;\n\n mmf_ids_data ids_info{};\n mmf_ids_data * ids_info_ptr = nullptr;\n ggml_cuda_pool_alloc ids_src_compact_dev;\n ggml_cuda_pool_alloc ids_dst_compact_dev;\n ggml_cuda_pool_alloc expert_bounds_dev;\n\n // For MUL_MAT_ID the memory layout is different than for MUL_MAT:\n const int64_t ncols_dst = ids ? ne2 : ne1;\n const int64_t nchannels_dst = ids ? ne1 : ne2;\n\n const int64_t stride_col_dst = ids ? s2 : s1;\n const int64_t stride_col_y = ids ? s12 : s11;\n const int64_t stride_channel_dst = ids ? s1 : s2;\n\n int64_t stride_channel_y = ids ? s11 : s12;\n int64_t nchannels_y = ids ? ne11 : ne12;\n\n //mul_mat_id: handle broadcast\n if (ids && nchannels_y == 1) {\n stride_channel_y = 0;\n nchannels_y = ids->ne[0];\n }\n\n if (ids && ncols_dst > 16) {\n const int64_t n_expert_used = ids->ne[0];\n const int64_t n_experts = ne02;\n const int64_t n_tokens = ne12;\n const int64_t ne_get_rows = n_tokens * n_expert_used;\n\n ids_src_compact_dev.alloc(ctx.pool(), ne_get_rows);\n ids_dst_compact_dev.alloc(ctx.pool(), ne_get_rows);\n expert_bounds_dev.alloc(ctx.pool(), n_experts + 1);\n\n const int si1 = static_cast(ids_s1);\n const int sis1 = static_cast(src1->nb[2] / src1->nb[1]);\n\n GGML_ASSERT(sis1 > 0);\n\n ggml_cuda_launch_mm_ids_helper(ids_d, ids_src_compact_dev.get(), ids_dst_compact_dev.get(), expert_bounds_dev.get(),\n static_cast(n_experts), static_cast(n_tokens), static_cast(n_expert_used), static_cast(ne11), si1, sis1, ctx.stream());\n CUDA_CHECK(cudaGetLastError());\n\n ids_info.ids_src_compact = ids_src_compact_dev.get();\n ids_info.ids_dst_compact = ids_dst_compact_dev.get();\n ids_info.expert_bounds_dev = expert_bounds_dev.get();\n ids_info.n_experts = static_cast(n_experts);\n ids_info.sis1 = sis1;\n ids_info_ptr = &ids_info;\n }\n\n const int device = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[device].cc;\n const int rows_per_block = mmf_get_rows_per_block(cc);\n\n switch (src0->type) {\n case GGML_TYPE_F32: {\n const float * src0_d = (const float *) src0->data;\n constexpr int vals_per_T = 1;\n mul_mat_f_switch_rows_per_block(\n rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,\n ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,\n ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);\n } break;\n case GGML_TYPE_F16: {\n const half2 * src0_d = (const half2 *) src0->data;\n constexpr int vals_per_T = 2;\n mul_mat_f_switch_rows_per_block(\n rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,\n ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,\n ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);\n } break;\n case GGML_TYPE_BF16: {\n const nv_bfloat162 * src0_d = (const nv_bfloat162 *) src0->data;\n constexpr int vals_per_T = 2;\n mul_mat_f_switch_rows_per_block(\n rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,\n ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,\n ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);\n } break;\n default:\n GGML_ABORT(\"unsupported type: %s\", ggml_type_name(src0->type));\n }\n}\n\nbool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne,\n const size_t * src0_nb, const int src1_ncols, bool mul_mat_id) {\n if (ggml_is_quantized(type)) {\n return false;\n }\n\n const size_t ts = ggml_type_size(type);\n if (src0_ne[0] % (warp_size * (4/ts)) != 0) {\n return false;\n }\n\n if (src0_nb[0] != ts) {\n return false;\n }\n\n // Pointers not aligned to the size of half2/nv_bfloat162/float2 would result in a crash:\n for (size_t i = 1; i < GGML_MAX_DIMS; ++i) {\n if (src0_nb[i] % (2*ts) != 0) {\n return false;\n }\n }\n if (src0_ne[1] % mmf_get_rows_per_block(cc) != 0) {\n return false;\n }\n\n if (GGML_CUDA_CC_IS_CDNA3(cc) && type == GGML_TYPE_BF16) {\n return false;\n }\n\n if (mul_mat_id) {\n if (src0_ne[1] <= 1024 && src1_ncols > 512) {\n return false;\n } else if(src0_ne[1] > 1024 && src1_ncols > 128) {\n return false;\n }\n } else {\n if (GGML_CUDA_CC_IS_RDNA3_0(cc) && src1_ncols > 8) {\n return false;\n } else if (GGML_CUDA_CC_IS_CDNA2(cc) && (type == GGML_TYPE_F16 || type == GGML_TYPE_BF16)) {\n //TODO: truse CDNA2 as CDNA1, tune the perf when CDNA2 is available.\n return false;\n } else if (GGML_CUDA_CC_IS_CDNA1(cc) && (type == GGML_TYPE_F16 || type == GGML_TYPE_BF16)) {\n return false;\n } else if (src1_ncols > 16) {\n return false;\n }\n }\n\n switch (type) {\n case GGML_TYPE_F32:\n return ampere_mma_available(cc) || amd_mfma_available(cc);\n case GGML_TYPE_F16:\n return volta_mma_available(cc) || turing_mma_available(cc) || amd_wmma_available(cc) || amd_mfma_available(cc);\n case GGML_TYPE_BF16:\n return ampere_mma_available(cc) || amd_wmma_available(cc) || amd_mfma_available(cc);\n default:\n return false;\n }\n}\n" }, { "path": "src/ggml-cuda/mmf.cuh", "content": "#pragma once\n\n#include \"mma.cuh\"\n#include \"common.cuh\"\n#include \"convert.cuh\"\n\nusing namespace ggml_cuda_mma;\n\n#define MMF_ROWS_PER_BLOCK 32\n#define MMF_ROWS_PER_BLOCK_CDNA 64\n\nstatic __forceinline__ int64_t mmf_get_max_block_size(int cc) {\n if (GGML_CUDA_CC_IS_CDNA(cc)) {\n return 512;\n } else {\n return 256;\n }\n}\n\nstatic __forceinline__ int mmf_get_padding(int cc) {\n if (GGML_CUDA_CC_IS_CDNA(cc)) {\n return 2;\n } else {\n return 4;\n }\n}\n\nstatic constexpr __device__ int mmf_get_padding() {\n#if defined(AMD_MFMA_AVAILABLE)\n return 2;\n#else\n return 4;\n#endif // defined(AMD_MFMA_AVAILABLE)\n}\n\nstruct mmf_ids_data {\n const int32_t * ids_src_compact = nullptr;\n const int32_t * ids_dst_compact = nullptr;\n const int32_t * expert_bounds_dev = nullptr;\n int n_experts = 0;\n int sis1 = 0;\n};\n\nvoid ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);\n\nbool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const int64_t * scr0_ne, const size_t * src0_nb, const int src1_ncols, bool mul_mat_id);\n\ntemplate \n__launch_bounds__(ggml_cuda_get_physical_warp_size()*nwarps, 1)\nstatic __global__ void mul_mat_f(\n const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, float * __restrict__ dst,\n const int ncols, const int ncols_dst_total, const int nchannels_dst, const int stride_row, const int stride_col_y, const int stride_col_dst,\n const int stride_col_id, const int stride_row_id,\n const int channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,\n const int sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst) {\n// TODO: handle this in a consistent and simpler way after AMD MFMA support has been added\n#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n#if defined(AMD_WMMA_AVAILABLE)\n if constexpr (!(std::is_same_v || std::is_same_v) || rows_per_block != MMF_ROWS_PER_BLOCK) {NO_DEVICE_CODE;} else {\n typedef tile<16, 8, T, get_input_data_layout()> tile_A;\n typedef tile<16, 8, T, get_input_data_layout()> tile_B;\n typedef tile<16, 16, float, DATA_LAYOUT_J_MAJOR> tile_C;\n#elif defined(AMD_MFMA_AVAILABLE)\n if constexpr (rows_per_block != MMF_ROWS_PER_BLOCK_CDNA) {NO_DEVICE_CODE;} else {\n typedef tile<16, 8, T, DATA_LAYOUT_I_MAJOR> tile_A;\n typedef tile<16, 8, T, DATA_LAYOUT_I_MAJOR> tile_B;\n typedef tile<16, 16, float, DATA_LAYOUT_J_MAJOR> tile_C;\n#else\n#ifdef VOLTA_MMA_AVAILABLE\n if constexpr (!std::is_same_v || rows_per_block != MMF_ROWS_PER_BLOCK) {NO_DEVICE_CODE;} else {\n typedef tile<32, 4, T, DATA_LAYOUT_I_MAJOR> tile_A;\n typedef tile< 8, 4, T, DATA_LAYOUT_I_MAJOR_MIRRORED> tile_B;\n typedef tile<32, 8, float, DATA_LAYOUT_I_MAJOR> tile_C;\n#else\n if constexpr (rows_per_block != MMF_ROWS_PER_BLOCK) {NO_DEVICE_CODE;} else {\n typedef tile<16, 8, T> tile_A;\n typedef tile<8, 8, T> tile_B;\n typedef tile<16, 8, float> tile_C;\n#endif // VOLTA_MMA_AVAILABLE\n#endif // defined(AMD_WMMA_AVAILABLE)\n if constexpr (!tile_A::supported() || !tile_B::supported() || !tile_C::supported()) {\n NO_DEVICE_CODE;\n return;\n }\n\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n constexpr int tile_k_padded = warp_size + mmf_get_padding();\n constexpr int ntA = rows_per_block / tile_A::I;\n constexpr int ntB = (cols_per_block + tile_B::I - 1) / tile_B::I;\n\n const int row0 = blockIdx.x * rows_per_block;\n\n int expert_idx = 0;\n int col_base = 0;\n\n const int channel_dst = has_ids ? 0 : blockIdx.y;\n\n if constexpr (has_ids) {\n // experts + tiles of ncols_dst are packed in the y dimension\n int col_tiles = (ncols_dst_total + cols_per_block - 1) / cols_per_block;\n const int nchannels_x = gridDim.y / col_tiles;\n const int tile_idx = blockIdx.y / nchannels_x;\n expert_idx = blockIdx.y - tile_idx * nchannels_x;\n col_base = tile_idx * cols_per_block;\n }\n\n const int channel_x = has_ids ? expert_idx : (channel_dst / channel_ratio);\n const int channel_y = channel_dst;\n const int sample_dst = blockIdx.z;\n const int sample_x = sample_dst / sample_ratio;\n const int sample_y = sample_dst;\n\n x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row0*stride_row ;\n y += int64_t(sample_y) *stride_sample_y + (has_ids ? 0 : channel_y *stride_channel_y);\n dst += int64_t(sample_dst)*stride_sample_dst + (has_ids ? 0 : channel_dst*stride_channel_dst);\n\n if constexpr (has_ids) {\n constexpr int y_stride_scale = std::is_same_v ? 1 : 2;\n const int64_t col_offset = col_base;\n y += col_offset * stride_col_y * y_stride_scale;\n dst += col_offset * stride_col_dst;\n ids += col_offset * stride_row_id;\n }\n\n const float2 * y2 = (const float2 *) y;\n\n extern __shared__ char data_mmv[];\n\n char * shmem_base = data_mmv;\n int * slot_map = (int *) shmem_base;\n char * compute_base = has_ids ? (shmem_base + GGML_PAD(cols_per_block, 16) * sizeof(int)) : shmem_base;\n\n tile_C C[ntA][ntB];\n\n T * tile_xy = (T *) compute_base + threadIdx.y*(tile_A::I * tile_k_padded);\n\n if constexpr (has_ids) {\n int found = 0;\n\n for (int j0 = 0; j0 < cols_per_block; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (threadIdx.x == 0) {\n slot_map[j] = -1;\n }\n\n if (col_base + j >= ncols_dst_total) {\n continue;\n }\n\n const int32_t * __restrict__ id_row = ids + j*stride_row_id;\n\n for (int k = threadIdx.x; k < nchannels_dst; k += warp_size) {\n int match = id_row[k*stride_col_id] == expert_idx;\n\n if (match) {\n slot_map[j] = k;\n found = 1;\n break;\n }\n }\n }\n\n if (!__syncthreads_or(found)) {\n return;\n }\n }\n\n\n for (int col = threadIdx.y*warp_size + threadIdx.x; col < ncols; col += nwarps*warp_size) {\n tile_A A[ntA][warp_size / tile_A::J];\n#pragma unroll\n for (int itA = 0; itA < ntA; ++itA) {\n#pragma unroll\n for (int i = 0; i < tile_A::I; ++i) {\n tile_xy[i*tile_k_padded + threadIdx.x] = x[(itA*tile_A::I + i)*stride_row + col];\n }\n#pragma unroll\n for (int k0 = 0; k0 < warp_size; k0 += tile_A::J) {\n load_ldmatrix(A[itA][k0/tile_A::J], tile_xy + k0, tile_k_padded);\n }\n }\n\n#pragma unroll\n for (int itB = 0; itB < ntB; ++itB) {\n if constexpr (std::is_same_v) {\n#pragma unroll\n for (int j0 = 0; j0 < tile_B::I; ++j0) {\n const int j = j0 + itB*tile_B::I;\n\n if constexpr (!has_ids) {\n tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;\n } else {\n const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;\n tile_xy[j0*tile_k_padded + threadIdx.x] = valid ? y[slot_map[j]*stride_channel_y + j*stride_col_y + col] : 0.0f;\n }\n }\n } else if constexpr (std::is_same_v || std::is_same_v) {\n#pragma unroll\n for (int j0 = 0; j0 < tile_B::I; ++j0) {\n const int j = j0 + itB*tile_B::I;\n\n if constexpr (!has_ids) {\n const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);\n tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast(tmp);\n } else {\n const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;\n float2 tmp = valid ? *(const float2*) &y[slot_map[j]*stride_channel_y + 2*(j*stride_col_y + col)] : make_float2(0.0f, 0.0f);\n tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast(tmp);\n }\n }\n } else {\n static_assert(std::is_same_v, \"unsupported type\");\n }\n#pragma unroll\n for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) {\n tile_B B;\n load_ldmatrix(B, tile_xy + k0, tile_k_padded);\n#pragma unroll\n for (int itA = 0; itA < ntA; ++itA) {\n mma(C[itA][itB], A[itA][k0/tile_B::J], B);\n }\n }\n }\n }\n\n float * buf_iw = (float *) compute_base;\n constexpr int kiw = nwarps*rows_per_block + mmf_get_padding();\n\n if (nwarps > 1) {\n __syncthreads();\n }\n#pragma unroll\n for (int itB = 0; itB < ntB; ++itB) {\n#pragma unroll\n for (int itA = 0; itA < ntA; ++itA) {\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = threadIdx.y*rows_per_block + itA*tile_C::I + tile_C::get_i(l);\n const int j = itB*tile_C::J + tile_C::get_j(l);\n buf_iw[j*kiw + i] = C[itA][itB].x[l];\n }\n }\n }\n\n if (nwarps > 1) {\n __syncthreads();\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < cols_per_block; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (j0 + nwarps > cols_per_block && j >= cols_per_block) {\n return;\n }\n\n float sum[rows_per_block/warp_size] = {0.0f};\n static_assert((rows_per_block % warp_size) == 0, \"rows_per_block must be a multiple of warp_size.\");\n#pragma unroll\n for (int i0 = 0; i0 < nwarps*rows_per_block; i0 += rows_per_block) {\n#pragma unroll\n for (int i1 = 0; i1 < sizeof(sum)/sizeof(sum[0]); ++i1) {\n const int i = i0 + i1*warp_size + threadIdx.x;\n\n sum[i1] += buf_iw[j*kiw + i];\n }\n }\n\n if constexpr (!has_ids) {\n#pragma unroll\n for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {\n dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];\n }\n } else {\n const int slot = (j < cols_per_block) ? slot_map[j] : -1;\n if (slot >= 0 && (col_base + j) < ncols_dst_total) {\n#pragma unroll\n for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {\n dst[slot*stride_channel_dst + j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];\n }\n }\n }\n }\n }\n#else\n GGML_UNUSED_VARS(x, y, ids, dst,\n ncols, ncols_dst_total, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id,\n channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);\n NO_DEVICE_CODE;\n#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n}\n\n//This kernel is for larger batch sizes of mul_mat_id\ntemplate \n__launch_bounds__(ggml_cuda_get_physical_warp_size()*nwarps, 1)\nstatic __global__ void mul_mat_f_ids(\n const T * __restrict__ x, const float * __restrict__ y,\n const int32_t * __restrict__ ids_src_compact, const int32_t * __restrict__ ids_dst_compact,\n const int32_t * __restrict__ expert_bounds, float * __restrict__ dst,\n const int ncols, const int ncols_dst_total, const int nchannels_dst, const int stride_row, const int stride_col_y, const int stride_col_dst,\n const int channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,\n const int sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,\n const uint3 sis1_fd, const uint3 nch_fd) {\n// TODO: handle this in a consistent and simpler way after AMD MFMA support has been added\n#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n#if defined(AMD_WMMA_AVAILABLE)\n if constexpr (!(std::is_same_v || std::is_same_v) || rows_per_block != MMF_ROWS_PER_BLOCK) {NO_DEVICE_CODE;} else {\n typedef tile<16, 8, T, get_input_data_layout()> tile_A;\n typedef tile<16, 8, T, get_input_data_layout()> tile_B;\n typedef tile<16, 16, float, DATA_LAYOUT_J_MAJOR> tile_C;\n#elif defined(AMD_MFMA_AVAILABLE)\n if constexpr (rows_per_block != MMF_ROWS_PER_BLOCK_CDNA) {NO_DEVICE_CODE;} else {\n typedef tile<16, 8, T, DATA_LAYOUT_I_MAJOR> tile_A;\n typedef tile<16, 8, T, DATA_LAYOUT_I_MAJOR> tile_B;\n typedef tile<16, 16, float, DATA_LAYOUT_J_MAJOR> tile_C;\n#else\n#ifdef VOLTA_MMA_AVAILABLE\n if constexpr (!std::is_same_v || rows_per_block != MMF_ROWS_PER_BLOCK) {NO_DEVICE_CODE;} else {\n typedef tile<32, 4, T, DATA_LAYOUT_I_MAJOR> tile_A;\n typedef tile< 8, 4, T, DATA_LAYOUT_I_MAJOR_MIRRORED> tile_B;\n typedef tile<32, 8, float, DATA_LAYOUT_I_MAJOR> tile_C;\n#else\n if constexpr (rows_per_block != MMF_ROWS_PER_BLOCK) {NO_DEVICE_CODE;} else {\n typedef tile<16, 8, T> tile_A;\n typedef tile<8, 8, T> tile_B;\n typedef tile<16, 8, float> tile_C;\n#endif // VOLTA_MMA_AVAILABLE\n#endif // defined(AMD_WMMA_AVAILABLE)\n if constexpr (!tile_A::supported() || !tile_B::supported() || !tile_C::supported()) {\n NO_DEVICE_CODE;\n return;\n }\n\n\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n constexpr int tile_k_padded = warp_size + mmf_get_padding();\n constexpr int ntA = rows_per_block / tile_A::I;\n constexpr int ntB = (cols_per_block + tile_B::I - 1) / tile_B::I;\n\n const int row0 = blockIdx.x * rows_per_block;\n\n const int expert_idx = blockIdx.y;\n const int expert_start = expert_bounds[expert_idx];\n const int expert_end = expert_bounds[expert_idx + 1];\n const int ncols_expert = expert_end - expert_start;\n\n const int tiles_for_expert = (ncols_expert + cols_per_block - 1) / cols_per_block;\n const int tile_idx = blockIdx.z;\n if (tile_idx >= tiles_for_expert) {\n return;\n }\n\n const int col_base = tile_idx * cols_per_block;\n\n GGML_UNUSED(channel_ratio);\n\n const int channel_x = expert_idx;\n const int sample_dst = 0;\n const int sample_x = sample_dst / sample_ratio;\n const int sample_y = sample_dst;\n\n x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row0*stride_row;\n y += int64_t(sample_y) *stride_sample_y;\n dst += int64_t(sample_dst)*stride_sample_dst;\n\n const int32_t * ids_src_expert = ids_src_compact + expert_start;\n const int32_t * ids_dst_expert = ids_dst_compact + expert_start;\n\n extern __shared__ char data_mmv[];\n char * compute_base = data_mmv;\n\n //const float2 * y2 = (const float2 *) y;\n\n tile_C C[ntA][ntB];\n\n T * tile_xy = (T *) compute_base + threadIdx.y*(tile_A::I * tile_k_padded);\n\n for (int col = threadIdx.y*warp_size + threadIdx.x; col < ncols; col += nwarps*warp_size) {\n tile_A A[ntA][warp_size / tile_A::J];\n#pragma unroll\n for (int itA = 0; itA < ntA; ++itA) {\n#pragma unroll\n for (int i = 0; i < tile_A::I; ++i) {\n tile_xy[i*tile_k_padded + threadIdx.x] = x[(itA*tile_A::I + i)*stride_row + col];\n }\n#pragma unroll\n for (int k0 = 0; k0 < warp_size; k0 += tile_A::J) {\n load_ldmatrix(A[itA][k0/tile_A::J], tile_xy + k0, tile_k_padded);\n }\n }\n\n if constexpr (std::is_same_v) {\n float vals_buf[2][tile_B::I];\n auto gather_tile = [&](int tile_idx_local, float *vals) {\n#pragma unroll\n for (int j0 = 0; j0 < tile_B::I; ++j0) {\n const int j = j0 + tile_idx_local*tile_B::I;\n const int global_j = col_base + j;\n float val = 0.0f;\n if (j < cols_per_block && global_j < ncols_expert) {\n const int src_entry = ids_src_expert[global_j];\n const uint2 qrm = fast_div_modulo((uint32_t) src_entry, sis1_fd);\n const int token = (int) qrm.x;\n const int channel = (int) qrm.y;\n if (token < ncols_dst_total) {\n val = y[channel*stride_channel_y + token*stride_col_y + col];\n }\n }\n vals[j0] = val;\n }\n };\n\n gather_tile(0, vals_buf[0]);\n\n int curr_buf = 0;\n int next_buf = 1;\n#pragma unroll\n for (int itB = 0; itB < ntB; ++itB) {\n#pragma unroll\n for (int j0 = 0; j0 < tile_B::I; ++j0) {\n tile_xy[j0*tile_k_padded + threadIdx.x] = vals_buf[curr_buf][j0];\n }\n\n if (itB + 1 < ntB) {\n gather_tile(itB + 1, vals_buf[next_buf]);\n }\n\n#pragma unroll\n for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) {\n tile_B B;\n load_ldmatrix(B, tile_xy + k0, tile_k_padded);\n#pragma unroll\n for (int itA = 0; itA < ntA; ++itA) {\n mma(C[itA][itB], A[itA][k0/tile_B::J], B);\n }\n }\n\n if (itB + 1 < ntB) {\n curr_buf ^= 1;\n next_buf ^= 1;\n }\n }\n } else if constexpr (std::is_same_v || std::is_same_v) {\n float2 vals_buf[2][tile_B::I];\n auto gather_tile = [&](int tile_idx_local, float2 *vals) {\n#pragma unroll\n for (int j0 = 0; j0 < tile_B::I; ++j0) {\n const int j = j0 + tile_idx_local*tile_B::I;\n const int global_j = col_base + j;\n float2 tmp = make_float2(0.0f, 0.0f);\n if (j < cols_per_block && global_j < ncols_expert) {\n const int src_entry = ids_src_expert[global_j];\n const uint2 qrm = fast_div_modulo((uint32_t) src_entry, sis1_fd);\n const int token = (int) qrm.x;\n const int channel = (int) qrm.y;\n if (token < ncols_dst_total) {\n tmp = *(const float2*) &y[channel*stride_channel_y + 2*(token*stride_col_y + col)];\n }\n }\n vals[j0] = tmp;\n }\n };\n\n if (ntB > 0) {\n gather_tile(0, vals_buf[0]);\n }\n\n int curr_buf = 0;\n int next_buf = 1;\n#pragma unroll\n for (int itB = 0; itB < ntB; ++itB) {\n#pragma unroll\n for (int j0 = 0; j0 < tile_B::I; ++j0) {\n const float2 tmp = vals_buf[curr_buf][j0];\n tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast(tmp);\n }\n\n if (itB + 1 < ntB) {\n gather_tile(itB + 1, vals_buf[next_buf]);\n }\n\n#pragma unroll\n for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) {\n tile_B B;\n load_ldmatrix(B, tile_xy + k0, tile_k_padded);\n#pragma unroll\n for (int itA = 0; itA < ntA; ++itA) {\n mma(C[itA][itB], A[itA][k0/tile_B::J], B);\n }\n }\n\n if (itB + 1 < ntB) {\n curr_buf ^= 1;\n next_buf ^= 1;\n }\n }\n } else {\n static_assert(std::is_same_v, \"unsupported type\");\n }\n }\n\n float * buf_iw = (float *) compute_base;\n constexpr int kiw = nwarps*rows_per_block + mmf_get_padding();\n\n if (nwarps > 1) {\n __syncthreads();\n }\n#pragma unroll\n for (int itB = 0; itB < ntB; ++itB) {\n#pragma unroll\n for (int itA = 0; itA < ntA; ++itA) {\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = threadIdx.y*rows_per_block + itA*tile_C::I + tile_C::get_i(l);\n const int j = itB*tile_C::J + tile_C::get_j(l);\n buf_iw[j*kiw + i] = C[itA][itB].x[l];\n }\n }\n }\n\n if (nwarps > 1) {\n __syncthreads();\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < cols_per_block; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (j0 + nwarps > cols_per_block && j >= cols_per_block) {\n return;\n }\n\n float sum[rows_per_block/warp_size] = {0.0f};\n static_assert((rows_per_block % warp_size) == 0, \"rows_per_block must be a multiple of warp_size.\");\n#pragma unroll\n for (int i0 = 0; i0 < nwarps*rows_per_block; i0 += rows_per_block) {\n#pragma unroll\n for (int i1 = 0; i1 < sizeof(sum)/sizeof(sum[0]); ++i1) {\n const int i = i0 + i1*warp_size + threadIdx.x;\n\n sum[i1] += buf_iw[j * kiw + i];\n }\n }\n\n const int global_j = col_base + j;\n if (j < cols_per_block && global_j < ncols_expert && nchannels_dst > 0) {\n const int dst_entry = ids_dst_expert[global_j];\n const uint2 qrm = fast_div_modulo((uint32_t) dst_entry, nch_fd);\n const int token = (int) qrm.x;\n if (token < ncols_dst_total) {\n const int slot = (int) qrm.y;\n#pragma unroll\n for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {\n dst[slot * stride_channel_dst + token * stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];\n }\n }\n }\n }\n }\n#else\n GGML_UNUSED_VARS(x, y, ids_src_compact, ids_dst_compact, expert_bounds, dst,\n ncols, ncols_dst_total, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, sis1_fd, nch_fd);\n NO_DEVICE_CODE;\n#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n}\n\ntemplate\nstatic inline void mul_mat_f_switch_ids(\n const T * x, const float * y, const int32_t * ids, float * dst,\n const int64_t ncols_x, const int64_t ncols_dst, const int64_t nchannels_dst,\n const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,\n const int64_t stride_col_id, const int64_t stride_row_id,\n const int64_t channel_ratio, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst,\n const int64_t sample_ratio, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,\n const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared_total, cudaStream_t stream,\n const mmf_ids_data * ids_data) {\n const bool has_ids_data = ids_data && ids_data->ids_src_compact;\n\n // Use the compact-ids kernel only for larger tiles; for small ncols_dst (< 16)\n // we prefer the normal mul_mat_f path with has_ids=true.\n if (has_ids_data && ncols_dst > 16) {\n const int max_tiles = (int) ((ncols_dst + cols_per_block - 1) / cols_per_block);\n if (max_tiles == 0) {\n return;\n }\n dim3 block_nums_ids(block_nums.x, ids_data->n_experts, max_tiles);\n\n const uint3 sis1_fd = ids_data->sis1 > 0 ? init_fastdiv_values((uint32_t) ids_data->sis1) : make_uint3(0, 0, 1);\n const uint3 nch_fd = init_fastdiv_values((uint32_t) nchannels_dst);\n\n mul_mat_f_ids<<>>\n (x, y, ids_data->ids_src_compact, ids_data->ids_dst_compact, ids_data->expert_bounds_dev, dst,\n ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,\n sis1_fd, nch_fd);\n } else if (ids) {\n const int64_t col_tiles = (ncols_dst + cols_per_block - 1) / cols_per_block;\n dim3 block_nums_ids = block_nums;\n block_nums_ids.y *= col_tiles;\n\n mul_mat_f<<>>\n (x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);\n } else {\n mul_mat_f<<>>\n (x, y, ids, dst, ncols_x, cols_per_block, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);\n }\n}\n\ntemplate \nvoid mul_mat_f_cuda(\n const T * x, const float * y, const int32_t * ids, float * dst,\n const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,\n const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,\n const int64_t stride_col_id, const int64_t stride_row_id,\n const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,\n const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\n const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,\n cudaStream_t stream, const mmf_ids_data * ids_data) {\n typedef tile<16, 8, T> tile_A_16;\n typedef tile<32, 8, T> tile_A_32;\n typedef tile<16, 8, T> tile_B_16;\n typedef tile< 8, 8, T> tile_B_8;\n\n GGML_ASSERT(ncols_x % 2 == 0);\n GGML_ASSERT(stride_row % 2 == 0);\n GGML_ASSERT(stride_col_y % 2 == 0);\n GGML_ASSERT(ids || nchannels_dst % nchannels_x == 0);\n GGML_ASSERT( nsamples_dst % nsamples_x == 0);\n const int64_t channel_ratio = nchannels_dst / nchannels_x;\n const int64_t sample_ratio = nsamples_dst / nsamples_x;\n\n const int device = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[device].cc;\n const int warp_size = ggml_cuda_info().devices[device].warp_size;\n\n int64_t nwarps_best = 1;\n int64_t niter_best = (ncols_x + warp_size*2 - 1) / (warp_size*2);\n int64_t max_block_size = mmf_get_max_block_size(cc);\n for (int64_t nwarps = 2; nwarps <= max_block_size/warp_size; nwarps++) {\n const int64_t niter = (ncols_x + nwarps*warp_size*2 - 1) / (nwarps*warp_size*2);\n if (niter < niter_best) {\n niter_best = niter;\n nwarps_best = nwarps;\n }\n }\n\n const int nbytes_shared_iter = nwarps_best * (volta_mma_available(cc) ? tile_A_32::I : tile_A_16::I) * (warp_size + mmf_get_padding(cc)) * 4;\n const int nbytes_cols_per_block_pad = (amd_wmma_available(cc) || amd_mfma_available(cc)) ? tile_B_16::I : tile_B_8::I;\n const int nbytes_shared_combine = GGML_PAD(cols_per_block, nbytes_cols_per_block_pad) * (nwarps_best*rows_per_block + mmf_get_padding(cc)) * 4;\n const int nbytes_shared = std::max(nbytes_shared_iter, nbytes_shared_combine);\n const int nbytes_slotmap = ids ? GGML_PAD(cols_per_block, 16) * sizeof(int) : 0;\n const int nbytes_shared_total = nbytes_shared + nbytes_slotmap;\n const int64_t grid_y = ids ? nchannels_x : nchannels_dst;\n\n const dim3 block_nums(nrows_x/rows_per_block, grid_y, nsamples_dst);\n const dim3 block_dims(warp_size, nwarps_best, 1);\n\n switch (nwarps_best) {\n case 1: {\n mul_mat_f_switch_ids(\n x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,\n ids_data);\n } break;\n case 2: {\n mul_mat_f_switch_ids(\n x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,\n ids_data);\n } break;\n case 3: {\n mul_mat_f_switch_ids(\n x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,\n ids_data);\n } break;\n case 4: {\n mul_mat_f_switch_ids(\n x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,\n ids_data);\n } break;\n case 5: {\n mul_mat_f_switch_ids(\n x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,\n ids_data);\n } break;\n case 6: {\n mul_mat_f_switch_ids(\n x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,\n ids_data);\n } break;\n case 7: {\n mul_mat_f_switch_ids(\n x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,\n ids_data);\n } break;\n case 8: {\n mul_mat_f_switch_ids(\n x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,\n ids_data);\n } break;\n default: {\n GGML_ABORT(\"fatal error\");\n } break;\n }\n\n GGML_UNUSED_VARS(nchannels_y);\n}\n\ntemplate \nstatic void mul_mat_f_switch_cols_per_block(\n const T * x, const float * y, const int32_t * ids, float * dst,\n const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,\n const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,\n const int64_t stride_col_id, const int stride_row_id,\n const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,\n const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\n const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,\n cudaStream_t stream, const mmf_ids_data * ids_data) {\n\n const int ncols_case = (ids && ncols_dst > 16) ? 16 : ncols_dst;\n\n GGML_ASSERT(ids || ncols_dst <= 16);\n\n switch (ncols_case) {\n case 1: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 2: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 3: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 4: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 5: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 6: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 7: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 8: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 9: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 10: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 11: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 12: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 13: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 14: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 15: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case 16: {\n mul_mat_f_cuda(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n default: {\n GGML_ABORT(\"fatal error\");\n } break;\n }\n}\n\ntemplate \nstatic void mul_mat_f_switch_rows_per_block(\n const int rows_per_block, const T * x, const float * y, const int32_t * ids, float * dst,\n const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,\n const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,\n const int64_t stride_col_id, const int stride_row_id,\n const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,\n const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\n const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,\n cudaStream_t stream, const mmf_ids_data * ids_data) {\n switch (rows_per_block) {\n case MMF_ROWS_PER_BLOCK: {\n mul_mat_f_switch_cols_per_block(\n x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n case MMF_ROWS_PER_BLOCK_CDNA: {\n mul_mat_f_switch_cols_per_block(\n x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);\n } break;\n default:\n GGML_ABORT(\"unsupported rows_per_block: %i\", rows_per_block);\n }\n}\n\n#define DECL_MMF_CASE_HELPER(T, nrows_dst, ncols_dst) \\\n template void mul_mat_f_cuda( \\\n const T * x, const float * y, const int32_t * ids, float * dst, \\\n const int64_t ncols_x, const int64_t nrows_x, int64_t ncols_dst_total, const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst, \\\n const int64_t stride_col_id, const int64_t stride_row_id, \\\n const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst, \\\n const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\\\n const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst, \\\n cudaStream_t stream, const mmf_ids_data * ids_data);\n\n#if !defined(GGML_USE_MUSA)\n#define DECL_MMF_CASE_EXTERN(ncols_dst) \\\n extern DECL_MMF_CASE_HELPER(float, MMF_ROWS_PER_BLOCK, ncols_dst) \\\n extern DECL_MMF_CASE_HELPER(half2, MMF_ROWS_PER_BLOCK, ncols_dst) \\\n extern DECL_MMF_CASE_HELPER(nv_bfloat162, MMF_ROWS_PER_BLOCK, ncols_dst) \\\n extern DECL_MMF_CASE_HELPER(float, MMF_ROWS_PER_BLOCK_CDNA, ncols_dst) \\\n extern DECL_MMF_CASE_HELPER(half2, MMF_ROWS_PER_BLOCK_CDNA, ncols_dst) \\\n extern DECL_MMF_CASE_HELPER(nv_bfloat162, MMF_ROWS_PER_BLOCK_CDNA, ncols_dst)\n\n#define DECL_MMF_CASE(ncols_dst) \\\n DECL_MMF_CASE_HELPER(float, MMF_ROWS_PER_BLOCK, ncols_dst) \\\n DECL_MMF_CASE_HELPER(half2, MMF_ROWS_PER_BLOCK, ncols_dst) \\\n DECL_MMF_CASE_HELPER(nv_bfloat162, MMF_ROWS_PER_BLOCK, ncols_dst) \\\n DECL_MMF_CASE_HELPER(float, MMF_ROWS_PER_BLOCK_CDNA, ncols_dst) \\\n DECL_MMF_CASE_HELPER(half2, MMF_ROWS_PER_BLOCK_CDNA, ncols_dst) \\\n DECL_MMF_CASE_HELPER(nv_bfloat162, MMF_ROWS_PER_BLOCK_CDNA, ncols_dst)\n\nDECL_MMF_CASE_EXTERN(1);\nDECL_MMF_CASE_EXTERN(2);\nDECL_MMF_CASE_EXTERN(3);\nDECL_MMF_CASE_EXTERN(4);\nDECL_MMF_CASE_EXTERN(5);\nDECL_MMF_CASE_EXTERN(6);\nDECL_MMF_CASE_EXTERN(7);\nDECL_MMF_CASE_EXTERN(8);\nDECL_MMF_CASE_EXTERN(9);\nDECL_MMF_CASE_EXTERN(10);\nDECL_MMF_CASE_EXTERN(11);\nDECL_MMF_CASE_EXTERN(12);\nDECL_MMF_CASE_EXTERN(13);\nDECL_MMF_CASE_EXTERN(14);\nDECL_MMF_CASE_EXTERN(15);\nDECL_MMF_CASE_EXTERN(16);\n#else\n#define DECL_MMF_CASE(ncols_dst)\n#endif\n" }, { "path": "src/ggml-cuda/mmid.cu", "content": "#include \"common.cuh\"\n#include \"mmid.cuh\"\n\n// To reduce shared memory use, store \"it\" and \"iex_used\" with 22/10 bits each.\nstruct mm_ids_helper_store {\n uint32_t data;\n\n __device__ mm_ids_helper_store(const uint32_t it, const uint32_t iex_used) {\n data = (it & 0x003FFFFF) | (iex_used << 22);\n }\n\n __device__ uint32_t it() const {\n return data & 0x003FFFFF;\n }\n\n __device__ uint32_t iex_used() const {\n return data >> 22;\n }\n};\nstatic_assert(sizeof(mm_ids_helper_store) == 4, \"unexpected size for mm_ids_helper_store\");\n\n// Helper function for mul_mat_id, converts ids to a more convenient format.\n// ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert.\n// ids_dst describes the same mapping but for the dst tensor.\n// The upper and lower bounds for the ith expert in the compact src1 tensor are stored in expert_bounds[i:i+1].\ntemplate \n__launch_bounds__(ggml_cuda_get_physical_warp_size(), 1)\nstatic __global__ void mm_ids_helper(\n const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,\n const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1) {\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template;\n const int expert = blockIdx.x;\n\n extern __shared__ char data_mm_ids_helper[];\n mm_ids_helper_store * store = (mm_ids_helper_store *) data_mm_ids_helper;\n\n int nex_prev = 0; // Number of columns for experts with a lower index.\n int it_compact = 0; // Running index for the compact slice of this expert.\n\n if constexpr (n_expert_used_template == 0) {\n // Generic implementation:\n for (int it = 0; it < n_tokens; ++it) {\n int iex_used = -1; // The index at which the expert is used, if any.\n for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) {\n const int expert_used = ids[it*si1 + iex];\n nex_prev += expert_used < expert;\n if (expert_used == expert) {\n iex_used = iex;\n }\n }\n\n if (iex_used != -1) {\n store[it_compact] = mm_ids_helper_store(it, iex_used);\n }\n\n if (warp_reduce_any(iex_used != -1)) {\n it_compact++;\n }\n }\n } else {\n // Implementation optimized for specific numbers of experts used:\n static_assert(n_expert_used == 6 || warp_size % n_expert_used == 0, \"bad n_expert_used\");\n const int neu_padded = n_expert_used == 6 ? 8 : n_expert_used; // Padded to next higher power of 2.\n for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) {\n const int it = it0 + threadIdx.x / neu_padded;\n\n const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.\n const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?\n ids[it*si1 + iex] : INT_MAX;\n const int iex_used = expert_used == expert ? iex : -1;\n nex_prev += expert_used < expert;\n\n // Whether the threads at this token position have used the expert:\n const int it_compact_add_self = warp_reduce_any(iex_used != -1);\n\n // Do a scan over threads at lower token positions in warp to get the correct index for writing data:\n int it_compact_add_lower = 0;\n#pragma unroll\n for (int offset = neu_padded; offset < warp_size; offset += neu_padded) {\n const int tmp = __shfl_up_sync(0xFFFFFFFF, it_compact_add_self, offset, warp_size);\n if (threadIdx.x >= static_cast(offset)) {\n it_compact_add_lower += tmp;\n }\n }\n\n if (iex_used != -1) {\n store[it_compact + it_compact_add_lower] = mm_ids_helper_store(it, iex_used);\n }\n\n // The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads:\n it_compact += __shfl_sync(0xFFFFFFFF, it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);\n }\n }\n nex_prev = warp_reduce_sum(nex_prev);\n\n for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {\n const mm_ids_helper_store store_it = store[itc];\n const int it = store_it.it();\n const int iex_used = store_it.iex_used();\n ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y;\n ids_dst [nex_prev + itc] = it*n_expert_used + iex_used;\n }\n\n if (threadIdx.x != 0) {\n return;\n }\n\n expert_bounds[expert] = nex_prev;\n\n if (expert < static_cast(gridDim.x) - 1) {\n return;\n }\n\n expert_bounds[gridDim.x] = nex_prev + it_compact;\n}\n\ntemplate \nstatic void launch_mm_ids_helper(\n const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,\n const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, cudaStream_t stream) {\n GGML_ASSERT(n_tokens < (1 << 22) && \"too few bits in mm_ids_helper_store\");\n GGML_ASSERT(n_expert_used_var < (1 << 10) && \"too few bits in mm_ids_helper_store\");\n\n const int id = ggml_cuda_get_device();\n const int warp_size = ggml_cuda_info().devices[id].warp_size;\n const size_t smpbo = ggml_cuda_info().devices[id].smpbo;\n CUDA_SET_SHARED_MEMORY_LIMIT(mm_ids_helper, smpbo);\n\n const dim3 num_blocks(n_experts, 1, 1);\n const dim3 block_size(warp_size, 1, 1);\n const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store);\n GGML_ASSERT(nbytes_shared <= smpbo);\n mm_ids_helper<<>>\n (ids, ids_src1, ids_dst, expert_bounds, n_tokens, n_expert_used_var, nchannels_y, si1, sis1);\n}\n\nvoid ggml_cuda_launch_mm_ids_helper(\n const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,\n const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, cudaStream_t stream) {\n switch (n_expert_used) {\n case 2:\n launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream);\n break;\n case 4:\n launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream);\n break;\n case 6:\n launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream);\n break;\n case 8:\n launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream);\n break;\n case 16:\n launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream);\n break;\n case 32:\n launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream);\n break;\n default:\n launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream);\n break;\n }\n}\n" }, { "path": "src/ggml-cuda/mmid.cuh", "content": "#pragma once\n\nvoid ggml_cuda_launch_mm_ids_helper(\n const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds,\n int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, cudaStream_t stream);\n" }, { "path": "src/ggml-cuda/mmq.cu", "content": "#include \"common.cuh\"\n#include \"mmq.cuh\"\n#include \"quantize.cuh\"\n#include \"mmid.cuh\"\n\nstatic void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {\n switch (args.type_x) {\n case GGML_TYPE_Q4_0:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_Q4_1:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_Q5_0:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_Q5_1:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_Q8_0:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_MXFP4:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_Q2_K:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_Q3_K:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_Q4_K:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_Q5_K:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_Q6_K:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_IQ2_XXS:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_IQ2_XS:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_IQ2_S:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_IQ3_XXS:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_IQ3_S:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_IQ1_S:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_IQ4_XS:\n mul_mat_q_case(ctx, args, stream);\n break;\n case GGML_TYPE_IQ4_NL:\n mul_mat_q_case(ctx, args, stream);\n break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\nvoid ggml_cuda_mul_mat_q(\n ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {\n GGML_ASSERT( src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n cudaStream_t stream = ctx.stream();\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n\n const size_t ts_src0 = ggml_type_size(src0->type);\n const size_t ts_src1 = ggml_type_size(src1->type);\n const size_t ts_dst = ggml_type_size(dst->type);\n\n GGML_ASSERT( nb00 == ts_src0);\n GGML_ASSERT( nb10 == ts_src1);\n GGML_ASSERT( nb0 == ts_dst);\n GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));\n\n const char * src0_d = (const char *) src0->data;\n const float * src1_d = (const float *) src1->data;\n float * dst_d = (float *) dst->data;\n\n // If src0 is a temporary compute buffer, clear any potential padding.\n if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {\n const size_t size_data = ggml_nbytes(src0);\n const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0);\n if (size_alloc > size_data) {\n GGML_ASSERT(ggml_is_contiguously_allocated(src0));\n GGML_ASSERT(!src0->view_src);\n CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream));\n }\n }\n\n const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);\n\n const int64_t s01 = src0->nb[1] / ts_src0;\n const int64_t s1 = dst->nb[1] / ts_dst;\n const int64_t s02 = src0->nb[2] / ts_src0;\n const int64_t s2 = dst->nb[2] / ts_dst;\n const int64_t s03 = src0->nb[3] / ts_src0;\n const int64_t s3 = dst->nb[3] / ts_dst;\n\n const bool use_stream_k = (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA)\n || GGML_CUDA_CC_IS_CDNA(cc);\n\n // TODO: tighter pool buffer size vs q8 path\n const bool use_native_mxfp4 = blackwell_mma_available(cc) && src0->type == GGML_TYPE_MXFP4;\n\n if (!ids) {\n const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1 +\n get_mmq_x_max_host(cc)*sizeof(block_q8_1_mmq);\n ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), nbytes_src1_q8_1);\n\n {\n const int64_t s11 = src1->nb[1] / ts_src1;\n const int64_t s12 = src1->nb[2] / ts_src1;\n const int64_t s13 = src1->nb[3] / ts_src1;\n if (use_native_mxfp4) {\n static_assert(sizeof(block_fp4_mmq) == 4 * sizeof(block_q8_1));\n quantize_mmq_mxfp4_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,\n ne11, ne12, ne13, stream);\n\n } else {\n quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,\n ne11, ne12, ne13, stream);\n }\n CUDA_CHECK(cudaGetLastError());\n }\n\n // Stride depends on quantization format\n const int64_t s12 = use_native_mxfp4 ?\n ne11 * ne10_padded * sizeof(block_fp4_mmq) /\n (8 * QK_MXFP4 * sizeof(int)) // block_fp4_mmq holds 256 values (8 blocks of 32)\n :\n ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));\n const int64_t s13 = ne12*s12;\n\n const mmq_args args = {\n src0_d, src0->type, (const int *) src1_q8_1.ptr, nullptr, nullptr, dst_d,\n ne00, ne01, ne1, s01, ne11, s1,\n ne02, ne12, s02, s12, s2,\n ne03, ne13, s03, s13, s3,\n use_stream_k, ne1};\n ggml_cuda_mul_mat_q_switch_type(ctx, args, stream);\n return;\n }\n\n GGML_ASSERT(ne13 == 1);\n GGML_ASSERT(nb12 % nb11 == 0);\n GGML_ASSERT(nb2 % nb1 == 0);\n\n const int64_t n_expert_used = ids->ne[0];\n const int64_t ne_get_rows = ne12 * n_expert_used;\n GGML_ASSERT(ne1 == n_expert_used);\n\n ggml_cuda_pool_alloc ids_src1(ctx.pool(), ne_get_rows);\n ggml_cuda_pool_alloc ids_dst(ctx.pool(), ne_get_rows);\n ggml_cuda_pool_alloc expert_bounds(ctx.pool(), ne02 + 1);\n\n {\n GGML_ASSERT(ids->nb[0] == ggml_element_size(ids));\n const int si1 = ids->nb[1] / ggml_element_size(ids);\n const int sis1 = nb12 / nb11;\n\n ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(),\n ne02, ne12, n_expert_used, ne11, si1, sis1, stream);\n CUDA_CHECK(cudaGetLastError());\n }\n\n const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * sizeof(block_q8_1)/QK8_1 +\n get_mmq_x_max_host(cc)*sizeof(block_q8_1_mmq);\n ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), nbytes_src1_q8_1);\n\n const int64_t ne11_flat = ne12*n_expert_used;\n const int64_t ne12_flat = 1;\n const int64_t ne13_flat = 1;\n\n {\n const int64_t s11 = src1->nb[1] / ts_src1;\n const int64_t s12 = src1->nb[2] / ts_src1;\n const int64_t s13 = src1->nb[3] / ts_src1;\n\n if (use_native_mxfp4) {\n quantize_mmq_mxfp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,\n ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);\n } else {\n quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,\n ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);\n }\n CUDA_CHECK(cudaGetLastError());\n }\n\n const int64_t s12 = use_native_mxfp4 ? ne11 * ne10_padded * sizeof(block_fp4_mmq) / (8 * QK_MXFP4 * sizeof(int)) :\n ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));\n const int64_t s13 = ne12*s12;\n\n // Note that ne02 is used instead of ne12 because the number of y channels determines the z dimension of the CUDA grid.\n const mmq_args args = {\n src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), dst_d,\n ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,\n ne02, ne02, s02, s12, s2,\n ne03, ne13, s03, s13, s3,\n use_stream_k, ne12};\n\n ggml_cuda_mul_mat_q_switch_type(ctx, args, stream);\n}\n\nvoid ggml_cuda_op_mul_mat_q(\n ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,\n const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,\n const int64_t src1_padded_row_size, cudaStream_t stream) {\n\n const int64_t ne00 = src0->ne[0];\n\n const int64_t ne10 = src1->ne[0];\n const int64_t ne11 = src1->ne[1];\n GGML_ASSERT(ne10 % QK8_1 == 0);\n\n const int64_t ne0 = dst->ne[0];\n\n const int64_t row_diff = row_high - row_low;\n const int64_t stride01 = ne00 / ggml_blck_size(src0->type);\n\n const int id = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[id].cc;\n\n // the main device has a larger memory buffer to hold the results from all GPUs\n // nrows_dst == nrows of the matrix that the kernel writes into\n const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;\n\n // The stream-k decomposition is only faster for recent NVIDIA GPUs.\n // Also its fixup needs to allocate a temporary buffer in the memory pool.\n // There are multiple parallel CUDA streams for src1_ncols != ne11 which would introduce a race condition for this buffer.\n const bool use_stream_k = ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA)\n || GGML_CUDA_CC_IS_CDNA(cc))\n && src1_ncols == ne11;\n const mmq_args args = {\n src0_dd_i, src0->type, (const int *) src1_ddq_i, nullptr, nullptr, dst_dd_i,\n ne00, row_diff, src1_ncols, stride01, ne11, nrows_dst,\n 1, 1, 0, 0, 0,\n 1, 1, 0, 0, 0,\n use_stream_k, src1_ncols};\n\n ggml_cuda_mul_mat_q_switch_type(ctx, args, stream);\n\n GGML_UNUSED_VARS(src1, dst, src1_ddf_i, src1_padded_row_size);\n}\n\nbool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) {\n#ifdef GGML_CUDA_FORCE_CUBLAS\n return false;\n#endif // GGML_CUDA_FORCE_CUBLAS\n\n bool mmq_supported;\n\n switch (type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_MXFP4:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ2_S:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ3_S:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ4_NL:\n mmq_supported = true;\n break;\n default:\n mmq_supported = false;\n break;\n }\n\n if (!mmq_supported) {\n return false;\n }\n\n if (turing_mma_available(cc)) {\n return true;\n }\n\n if (ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_DP4A) {\n return false;\n }\n\n#ifdef GGML_CUDA_FORCE_MMQ\n return true;\n#endif //GGML_CUDA_FORCE_MMQ\n\n if (GGML_CUDA_CC_IS_NVIDIA(cc)) {\n return !fp16_mma_hardware_available(cc) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;\n }\n\n if (amd_mfma_available(cc)) {\n // As of ROCM 7.0 rocblas/tensile performs very poorly on CDNA3 and hipblaslt (via ROCBLAS_USE_HIPBLASLT)\n // performs better but is currently suffering from a crash on this architecture.\n // TODO: Revisit when hipblaslt is fixed on CDNA3\n if (GGML_CUDA_CC_IS_CDNA3(cc)) {\n return true;\n }\n if (n_experts > 64 || ne11 <= 128) {\n return true;\n }\n if (type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) {\n return true;\n }\n if (ne11 <= 256 && (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K)) {\n return true;\n }\n return false;\n }\n\n if (amd_wmma_available(cc)) {\n if (GGML_CUDA_CC_IS_RDNA3(cc)) {\n // High expert counts are almost always better on MMQ due to\n // the synchronization overhead in the cuBLAS/hipBLAS path:\n // https://github.com/ggml-org/llama.cpp/pull/18202\n if (n_experts >= 64) {\n return true;\n }\n\n // For some quantization types MMQ can have lower peak TOPS than hipBLAS\n // so it's only faster for sufficiently small batch sizes:\n switch (type) {\n case GGML_TYPE_Q2_K:\n return ne11 <= 128;\n case GGML_TYPE_Q6_K:\n return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ2_S:\n return GGML_CUDA_CC_IS_RDNA3_5(cc) || ne11 <= 128;\n default:\n return true;\n }\n }\n\n // For RDNA4 MMQ is consistently faster than dequantization + hipBLAS:\n // https://github.com/ggml-org/llama.cpp/pull/18537#issuecomment-3706422301\n return true;\n }\n\n return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;\n\n}\n" }, { "path": "src/ggml-cuda/mmq.cuh", "content": "#pragma once\n\n#include \"common.cuh\"\n#include \"vecdotq.cuh\"\n#include \"mma.cuh\"\n\n#include \n#include \n\nusing namespace ggml_cuda_mma;\n\n#define MMQ_DP4A_MAX_BATCH_SIZE 64 // Max. batch size to use for dp4a MMQ kernels when FP16 tensor cores are available.\n#define MMQ_ITER_K 256\n#define MMQ_ITER_K_MXFP4_FP4 512\n#define MMQ_NWARPS 8\n\ntypedef void (*load_tiles_mmq_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);\ntypedef void (*vec_dot_mmq_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);\ntypedef void (*mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,\n float * __restrict__ dst, const int stride, const int i_max, const int j_max);\n\nenum mmq_q8_1_ds_layout {\n MMQ_Q8_1_DS_LAYOUT_D4,\n MMQ_Q8_1_DS_LAYOUT_DS4,\n MMQ_Q8_1_DS_LAYOUT_D2S6,\n};\n\nstruct block_q8_1_mmq {\n // The y float data is converted to a data layout that can simply be copied to shared memory as a contiguous block.\n // The y float data is first grouped as blocks of 128 values.\n // These blocks are then treated as individual data values and transposed.\n //\n // To avoid shared memory bank conflicts each block is padded with 16 bytes.\n // This padding is also used to store block scales/partial sums.\n // The scales multiplied with the quantized data are equal to the unquantized values.\n // The partial sums are obtained by summing up a subgroup of the contained values (prior to quantization)\n // and are only needed for performance reasons.\n //\n // The exact data stored depends on the x data type.\n union {\n float d4[4]; // 1 32 bit scale per 32 values, stored as d0,d1,d2,d3\n half2 ds4[4]; // 1 16 bit scale + 1 16 bit partial sum per 32 values, stored as d0,s0,d1,s1,d2,s2,d3,s3\n half d2s6[8]; // 1 16 bit scale per 64 values + 1 16 bit partial sum per 16 values for the first 96 values,\n // stored as d0,d1,s1,s2,s3,s4,s5\n };\n int8_t qs[4*QK8_1]; // 128 values quantized to 8 bit each\n};\n\nstruct block_fp4_mmq {\n uint32_t d4[4]; // 8 E8M0 scales (1 per 32 values), 2 packed per uint32: d4[0]={s0,s1}, d4[1]={s2,s3}, etc.\n int8_t qs[4 * 32]; // 256 FP4 values packed as 4-bit pairs (2 per byte), 8 blocks of 32 values\n};\n\nstatic_assert(sizeof(block_q8_1_mmq) == 4*QK8_1 + 4*sizeof(half2), \"Unexpected block_q8_1_mmq size\");\nstatic_assert(sizeof(block_q8_1_mmq) == 4*sizeof(block_q8_1), \"Unexpected block_q8_1_mmq size\");\nstatic_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), \"Unexpected block_fp4_mmq size\");\n\nstatic mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {\n switch (type_x) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n return MMQ_Q8_1_DS_LAYOUT_DS4;\n case GGML_TYPE_Q5_0:\n return MMQ_Q8_1_DS_LAYOUT_D4;\n case GGML_TYPE_Q5_1:\n return MMQ_Q8_1_DS_LAYOUT_DS4;\n case GGML_TYPE_Q8_0:\n return MMQ_Q8_1_DS_LAYOUT_D4;\n case GGML_TYPE_MXFP4:\n return MMQ_Q8_1_DS_LAYOUT_D4;\n case GGML_TYPE_Q2_K:\n return MMQ_Q8_1_DS_LAYOUT_D2S6;\n case GGML_TYPE_Q3_K:\n return MMQ_Q8_1_DS_LAYOUT_D4;\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n return MMQ_Q8_1_DS_LAYOUT_DS4;\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ2_S:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ3_S:\n return MMQ_Q8_1_DS_LAYOUT_D4;\n case GGML_TYPE_IQ1_S:\n return MMQ_Q8_1_DS_LAYOUT_DS4;\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ4_NL:\n return MMQ_Q8_1_DS_LAYOUT_D4;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\nstruct tile_x_sizes {\n int qs;\n int dm;\n int sc;\n};\n\nstatic int get_mmq_x_max_host(const int cc) {\n return (amd_mfma_available(cc) || turing_mma_available(cc) || amd_wmma_available(cc)) ? 128 :\n GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA ?\n#ifdef GGML_CUDA_FORCE_MMQ\n 128 : 64;\n#else\n MMQ_DP4A_MAX_BATCH_SIZE : 64;\n#endif // GGML_CUDA_FORCE_MMQ\n}\n\nstatic constexpr __device__ int get_mmq_x_max_device() {\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n return 128;\n#else // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)\n\n#if defined(GGML_USE_HIP)\n return 64;\n#else // defined(GGML_USE_HIP)\n\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n#ifdef GGML_CUDA_FORCE_MMQ\n return 128;\n#else // GGML_CUDA_FORCE_MMQ\n return MMQ_DP4A_MAX_BATCH_SIZE;\n#endif // GGML_CUDA_FORCE_MMQ\n#else // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n return 64;\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n\n#endif // defined(GGML_USE_HIP)\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n}\n\nstatic int get_mmq_y_host(const int cc) {\n return GGML_CUDA_CC_IS_AMD(cc) ? (GGML_CUDA_CC_IS_RDNA1(cc) ? 64 : 128) :\n ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) ? 128 : 64);\n}\n\nstatic constexpr __device__ int get_iter_k([[maybe_unused]] const ggml_type type) {\n#if defined(BLACKWELL_MMA_AVAILABLE)\n return type == GGML_TYPE_MXFP4 ? MMQ_ITER_K_MXFP4_FP4 : MMQ_ITER_K;\n#else\n return MMQ_ITER_K;\n#endif // defined(BLACKWELL_MMA_AVAILABLE)\n}\n\nstatic constexpr __device__ int get_mmq_y_device() {\n#if defined(GGML_USE_HIP)\n#if defined(RDNA1)\n return 64;\n#else\n return 128;\n#endif // defined RDNA1\n#else\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n return 128;\n#else\n return 64;\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n#endif // defined(GGML_USE_HIP)\n}\n\n// Decouple shared memory tile sizes from WARP_SIZE to allow for different warp sizes.\n// The K dimension of the tiles has either,\n// 1*MMQ_TILE_NE_K==32 (always for TILE_Y_K) or 2*MMQ_TILE_NE_K==64 (typically for TILE_X_K),\n// 32 bit elements for the quantized data (does not include scales).\n// In other words, the size of the quantized data in the K dimension is a multiple of MMQ_TILE_NE_K.\n// The final tile size in K direction is padded to avoid shared memory bank conflicts,\n// in terms of 32 bit elements that means K % 2 == 1 for dp4a or K % 8 == 4 for mma.\n#define MMQ_TILE_NE_K 32\n\n#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{mmq_y*MMQ_TILE_NE_K + mmq_y, mmq_y*MMQ_TILE_NE_K/QI4_0 + mmq_y/QI4_0, 0}\n#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{mmq_y*MMQ_TILE_NE_K + mmq_y, mmq_y*MMQ_TILE_NE_K/QI4_1 + mmq_y/QI4_1, 0}\n#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K*2/QI8_0 + mmq_y/(QI8_0/2), 0}\n#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K*4/QI8_0 + mmq_y/(QI8_0/4), 0}\n#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K*2/QI8_1 + mmq_y/(QI8_1/2), 0}\n#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K + mmq_y, 0}\n#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8}\n#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K + mmq_y, mmq_y*MMQ_TILE_NE_K/QI4_K, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8}\n#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K/QI5_K + mmq_y/QI5_K, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8}\n#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K/QI6_K + mmq_y/QI6_K, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8}\n\nstatic constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int mmq_y) {\n switch (type) {\n case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0;\n case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1;\n case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0;\n case GGML_TYPE_Q5_1: return MMQ_DP4A_TXS_Q8_1;\n case GGML_TYPE_Q8_0: return MMQ_DP4A_TXS_Q8_0;\n case GGML_TYPE_MXFP4: return MMQ_DP4A_TXS_Q8_1;\n case GGML_TYPE_Q2_K: return MMQ_DP4A_TXS_Q2_K;\n case GGML_TYPE_Q3_K: return MMQ_DP4A_TXS_Q3_K;\n case GGML_TYPE_Q4_K: return MMQ_DP4A_TXS_Q4_K;\n case GGML_TYPE_Q5_K: return MMQ_DP4A_TXS_Q5_K;\n case GGML_TYPE_Q6_K: return MMQ_DP4A_TXS_Q6_K;\n case GGML_TYPE_IQ2_XXS: return MMQ_DP4A_TXS_Q8_0;\n case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_IQ2_S: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;\n case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;\n case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;\n case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;\n case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;\n default: return tile_x_sizes{0, 0, 0};\n }\n}\n\n#define MMQ_MMA_TILE_X_K_Q8_0 (2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4)\n#define MMQ_MMA_TILE_X_K_FP4 (2*MMQ_TILE_NE_K + 8 + 4)\n#define MMQ_MMA_TILE_X_K_Q8_1 (2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4)\n#define MMQ_MMA_TILE_X_K_Q2_K (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K + 4)\n#define MMQ_MMA_TILE_X_K_Q3_K (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4)\n#define MMQ_MMA_TILE_X_K_Q6_K (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7)\n\nstatic_assert(MMQ_MMA_TILE_X_K_Q8_0 % 8 == 4, \"Wrong padding.\");\nstatic_assert(MMQ_MMA_TILE_X_K_Q8_1 % 8 == 4, \"Wrong padding.\");\nstatic_assert(MMQ_MMA_TILE_X_K_Q2_K % 8 == 4, \"Wrong padding.\");\nstatic_assert(MMQ_MMA_TILE_X_K_Q3_K % 8 == 4, \"Wrong padding.\");\nstatic_assert(MMQ_MMA_TILE_X_K_Q6_K % 8 == 4, \"Wrong padding.\");\nstatic_assert(MMQ_MMA_TILE_X_K_FP4 % 8 == 4, \"Wrong padding.\");\nstatic_assert(MMQ_MMA_TILE_X_K_FP4 == MMQ_MMA_TILE_X_K_Q8_1, \"Wrong tile size for MXFP4\");\n\nstatic constexpr __host__ __device__ int mmq_get_mma_tile_x_k(ggml_type type) {\n switch (type) {\n case GGML_TYPE_Q4_0: return MMQ_MMA_TILE_X_K_Q8_0;\n case GGML_TYPE_Q4_1: return MMQ_MMA_TILE_X_K_Q8_1;\n case GGML_TYPE_Q5_0: return MMQ_MMA_TILE_X_K_Q8_0;\n case GGML_TYPE_Q5_1: return MMQ_MMA_TILE_X_K_Q8_1;\n case GGML_TYPE_Q8_0: return MMQ_MMA_TILE_X_K_Q8_0;\n // tile sizes are the same for Q8_1 and FP4 for blackwell\n case GGML_TYPE_MXFP4: return MMQ_MMA_TILE_X_K_Q8_1;\n case GGML_TYPE_Q2_K: return MMQ_MMA_TILE_X_K_Q2_K;\n case GGML_TYPE_Q3_K: return MMQ_MMA_TILE_X_K_Q3_K;\n case GGML_TYPE_Q4_K: return MMQ_MMA_TILE_X_K_Q8_1;\n case GGML_TYPE_Q5_K: return MMQ_MMA_TILE_X_K_Q8_1;\n case GGML_TYPE_Q6_K: return MMQ_MMA_TILE_X_K_Q6_K;\n case GGML_TYPE_IQ2_XXS: return MMQ_MMA_TILE_X_K_Q8_0;\n case GGML_TYPE_IQ2_XS: return MMQ_MMA_TILE_X_K_Q3_K;\n case GGML_TYPE_IQ2_S: return MMQ_MMA_TILE_X_K_Q3_K;\n case GGML_TYPE_IQ3_XXS: return MMQ_MMA_TILE_X_K_Q8_0;\n case GGML_TYPE_IQ3_S: return MMQ_MMA_TILE_X_K_Q8_0;\n case GGML_TYPE_IQ1_S: return MMQ_MMA_TILE_X_K_Q8_0;\n case GGML_TYPE_IQ4_XS: return MMQ_MMA_TILE_X_K_Q8_0;\n case GGML_TYPE_IQ4_NL: return MMQ_MMA_TILE_X_K_Q8_0;\n default: return 0;\n }\n}\n\n// block_q8_1_mmq has (128 8-bit ints == 32 32-bit ints + 4 32-bit scales)\n#define MMQ_TILE_Y_K (MMQ_TILE_NE_K + MMQ_TILE_NE_K / QI8_1)\n#define MMQ_TILE_Y_FP4_K MMQ_TILE_Y_K\n\nstatic int mmq_get_granularity_host(const int mmq_x, const int cc) {\n if (amd_mfma_available(cc) || amd_wmma_available(cc)) {\n return mmq_x >= 128 ? 32 : 16;\n } else if (turing_mma_available(cc) && mmq_x >= 48) {\n return 16;\n } else {\n return 8;\n }\n}\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\nstatic constexpr __device__ int mmq_get_granularity_device(const int mmq_x) {\n return mmq_x >= 128 ? 32 : 16;\n}\n#elif defined(TURING_MMA_AVAILABLE)\nstatic constexpr __device__ int mmq_get_granularity_device(const int mmq_x) {\n return mmq_x >= 48 ? 16 : 8;\n}\n#else\nstatic constexpr __device__ int mmq_get_granularity_device(const int /*mmq_x*/) {\n return 8;\n}\n#endif // AMD_MFMA_AVAILABLE\n\n#if defined(GGML_USE_HIP)\nstatic int mmq_get_nwarps_host(const int cc, const int warp_size) {\n return amd_mfma_available(cc) ? 8 : 256/warp_size;\n}\n#else\nstatic int mmq_get_nwarps_host(const int /*cc*/, const int warp_size) {\n return 256/warp_size;\n}\n#endif // (GGML_USE_HIP)\n\nstatic constexpr __device__ int mmq_get_nwarps_device() {\n#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n return 8;\n#else\n return 256/ggml_cuda_get_physical_warp_size();\n#endif // AMD_MFMA_AVAILABLE\n}\n\n// ------------------------------------------------------------\n\ntemplate static __device__ __forceinline__ void load_tiles_q4_0(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_0);\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n const int kbx = txi / QI4_0;\n const int kqsx = txi % QI4_0;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbx;\n const int qs0 = get_int_b2(bxi->qs, kqsx);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI4_0) + kqsx + 0] = __vsubss4((qs0 >> 0) & 0x0F0F0F0F, 0x08080808);\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI4_0) + kqsx + QI4_0] = __vsubss4((qs0 >> 4) & 0x0F0F0F0F, 0x08080808);\n#else\n x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)\n }\n\n constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_0;\n constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;\n const int kbxd = threadIdx.x % blocks_per_tile_x_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {\n int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbxd;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d;\n#else\n x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + kbxd] = bxi->d;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q4_0_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, mmq_y);\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + txs.qs;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_ds = (const half2 *) y;\n\n// #pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_0*VDR_Q4_0_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);\n\n int u[2*VDR_Q4_0_Q8_1_MMQ];\n\n#pragma unroll\n for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {\n u[2*l+0] = y_qs[j*MMQ_TILE_Y_K + kyqs + l];\n u[2*l+1] = y_qs[j*MMQ_TILE_Y_K + kyqs + (l + QI4_0)];\n }\n\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q4_0_q8_1_impl\n (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_0], u,\n x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + k0/(QR4_0*QI4_0)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n }\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_q4_1(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, mmq_y);\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_1);\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n const int kbx = txi / QI4_1;\n const int kqsx = txi % QI4_1;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbx;\n const int qs0 = get_int_b4(bxi->qs, kqsx);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI4_1) + kqsx + 0] = (qs0 >> 0) & 0x0F0F0F0F;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI4_1) + kqsx + QI4_1] = (qs0 >> 4) & 0x0F0F0F0F;\n#else\n x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_1;\n constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;\n const int kbxd = threadIdx.x % blocks_per_tile_x_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {\n int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbxd;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = bxi->dm;\n#else\n x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + kbxd] = bxi->dm;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q4_1_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, mmq_y);\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + txs.qs;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_ds = (const half2 *) y;\n\n// #pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_1*VDR_Q4_1_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);\n\n int u[2*VDR_Q4_1_Q8_1_MMQ];\n\n#pragma unroll\n for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {\n u[2*l+0] = y_qs[j*MMQ_TILE_Y_K + kyqs + l];\n u[2*l+1] = y_qs[j*MMQ_TILE_Y_K + kyqs + (l + QI4_1)];\n }\n\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q4_1_q8_1_impl\n (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_1], u,\n x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + k0/(QR4_1*QI4_1)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n }\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_q5_0(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_0, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_0);\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n const int kbx = txi / QI5_0;\n const int kqsx = txi % QI5_0;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbx;\n\n const int ql = get_int_b2(bxi->qs, kqsx);\n const int qh = get_int_b2(bxi->qh, 0) >> (4 * kqsx);\n\n int qs0 = (ql >> 0) & 0x0F0F0F0F;\n qs0 |= (qh << 4) & 0x00000010; // 0 -> 4\n qs0 |= (qh << 11) & 0x00001000; // 1 -> 12\n qs0 |= (qh << 18) & 0x00100000; // 2 -> 20\n qs0 |= (qh << 25) & 0x10000000; // 3 -> 28\n qs0 = __vsubss4(qs0, 0x10101010); // subtract 16\n\n int qs1 = (ql >> 4) & 0x0F0F0F0F;\n qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4\n qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12\n qs1 |= (qh << 2) & 0x00100000; // 18 -> 20\n qs1 |= (qh << 9) & 0x10000000; // 19 -> 28\n qs1 = __vsubss4(qs1, 0x10101010); // subtract 16\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI5_0) + kqsx + 0] = qs0;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + 0] = qs0;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_0;\n constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;\n const int kbxd = threadIdx.x % blocks_per_tile_x_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {\n int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbxd;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d;\n#else\n x_df[i*(MMQ_TILE_NE_K/QI5_0) + i/QI5_0 + kbxd] = bxi->d;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_q5_1(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, mmq_y);\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_1);\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n const int kbx = txi / QI5_1;\n const int kqsx = txi % QI5_1;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbx;\n\n const int ql = get_int_b4(bxi->qs, kqsx);\n const int qh = get_int_b4(bxi->qh, 0) >> (4 * kqsx);\n\n int qs0 = (ql >> 0) & 0x0F0F0F0F;\n qs0 |= (qh << 4) & 0x00000010; // 0 -> 4\n qs0 |= (qh << 11) & 0x00001000; // 1 -> 12\n qs0 |= (qh << 18) & 0x00100000; // 2 -> 20\n qs0 |= (qh << 25) & 0x10000000; // 3 -> 28\n\n int qs1 = (ql >> 4) & 0x0F0F0F0F;\n qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4\n qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12\n qs1 |= (qh << 2) & 0x00100000; // 18 -> 20\n qs1 |= (qh << 9) & 0x10000000; // 19 -> 28\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI5_1) + kqsx + 0] = qs0;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + 0] = qs0;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_1;\n constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;\n const int kbxd = threadIdx.x % blocks_per_tile_x_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {\n int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbxd;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = bxi->dm;\n#else\n x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + kbxd] = bxi->dm;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_q8_0(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n // MMQ_ITER_K / (4 * QR8_0) == 64 required. but NV has only 32 threads per warp\n constexpr int threads_per_row = 32;\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n const int kbx = txi / QI8_0;\n const int kqsx = txi % QI8_0;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0;\n constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;\n const int kbxd = threadIdx.x % blocks_per_tile_x_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {\n int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d;\n#else\n x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + kbxd] = bxi->d;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_mxfp4(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_MXFP4, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR_MXFP4);\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n const int kbx = txi / QI_MXFP4;\n const int kqsx = txi % QI_MXFP4;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbx;\n\n const int aux_q4 = get_int_b1(bxi->qs, kqsx);\n const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4);\n const int k0 = kbx * (2 * QI_MXFP4) + kqsx;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + k0 + 0] = v.x;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + k0 + QI_MXFP4] = v.y;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI_MXFP4] = v.y;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI_MXFP4;\n constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;\n const int kbxd = threadIdx.x % blocks_per_tile_x_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {\n int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbxd;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;\n#else\n x_df[i*(MMQ_TILE_NE_K/QI_MXFP4) + i/QI_MXFP4 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void load_tiles_mxfp4_fp4(const char * __restrict__ x,\n int * __restrict__ x_tile,\n const int kbx0,\n const int i_max,\n const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n int * x_qs = (int *) x_tile;\n uint32_t * x_sc = (uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);\n\n const int txi = threadIdx.x;\n\n constexpr int iter_k = get_iter_k(GGML_TYPE_MXFP4);\n\n constexpr int threads_per_row = iter_k / QK_MXFP4; // each thread processes 1 block\n constexpr int rows_per_warp = warp_size / threads_per_row;\n const int kbx = txi % threads_per_row;\n const int row_in_warp = txi / threads_per_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += rows_per_warp * nwarps) {\n int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;\n\n if constexpr (need_check) {\n i = min(i, i_max);\n }\n\n const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i * stride + kbx;\n\n // quantize_mxfp4_mmq permutes nibbles to match the quantized format\n const int k0 = kbx * 4;\n memcpy(x_qs + i * MMQ_MMA_TILE_X_K_FP4 + k0, bxi->qs, 16);\n\n // Load E8M0 scales: pack 2 consecutive scales into one uint32\n if (kbx % 2 == 0) {\n uint32_t e = bxi->e;\n e |= ((bxi + 1)->e << 8);\n x_sc[i * MMQ_MMA_TILE_X_K_FP4 + kbx / 2] = e;\n }\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q8_0_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y);\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + txs.qs;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n// #pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q8_0_q8_1_impl\n (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k0 % MMQ_TILE_NE_K],\n x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + k0/QI8_0], y_df[j*MMQ_TILE_Y_K + (k0/QI8_1) % (MMQ_TILE_NE_K/QI8_1)]);\n }\n }\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q8_0_q8_1_mma(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n constexpr data_layout input_layout = get_input_data_layout();\n typedef tile<16, 8, int, input_layout> tile_A;\n typedef tile<16, 8, int, input_layout> tile_B;\n typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n const half2 * y_ds = (const half2 *) y;\n\n const int i0 = (threadIdx.y / ntx) * rows_per_warp;\n\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {\n const int k0 = k00 + k01;\n\n tile_A A[ntx];\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_0 + k0, MMQ_MMA_TILE_X_K_Q8_0);\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n tile_B B;\n load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);\n\n float dB;\n const int j = j0 + tile_C::get_j(0);\n if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {\n dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];\n } else {\n dB = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C;\n mma(C, A[n], B);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = i0 + n*tile_A::I + tile_C::get_i(l);\n const float dA = x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + k0/QI8_0];\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA*dB;\n }\n }\n }\n }\n#else\n typedef tile<16, 8, int> tile_A;\n typedef tile< 8, 8, int> tile_B;\n typedef tile<16, 8, int> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = 2 * granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n const half2 * y_ds = (const half2 *) y;\n\n tile_A A[ntx][MMQ_TILE_NE_K/QI8_0];\n float dA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_0];\n\n const int i0 = (threadIdx.y/ntx)*rows_per_warp;\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {\n const int k0 = k00 + k01;\n\n load_ldmatrix(A[n][k01/QI8_0], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_0 + k0, MMQ_MMA_TILE_X_K_Q8_0);\n }\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);\n\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {\n const int k0 = k00 + k01;\n\n dA[n][l][k01/QI8_0] = x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + k0/QI8_0];\n }\n }\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {\n tile_B B;\n float dB[tile_C::ne/2];\n\n load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int j = j0 + tile_C::get_j(l);\n\n if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {\n dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];\n } else {\n dB[l] = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C;\n mma(C, A[n][k01/QI8_0], B);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA[n][l/2][k01/QI8_0]*dB[l%2];\n }\n }\n }\n }\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_mxfp4_mxfp4_mma(const int * __restrict__ x,\n const int * __restrict__ y,\n float * __restrict__ sum,\n const int k00) {\n typedef tile<16, 8, int> tile_A;\n typedef tile<8, 8, int> tile_B;\n typedef tile<16, 8, float> tile_C; // Output is float for native scaled MMA\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = 2 * granularity;\n constexpr int ntx = rows_per_warp / tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J * MMQ_TILE_Y_FP4_K);\n\n // Match layout from load_tiles_mxfp4_fp4\n const int * x_qs = (const int *) x;\n const uint32_t * x_sc = (const uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);\n const int * y_qs = (const int *) y + 4;\n const uint32_t * y_sc = (const uint32_t *) y;\n\n // tile_A has a length of 64 logical values vs. 32 values in block_mxfp4\n tile_A A[ntx][MMQ_TILE_NE_K / (2 * QI_MXFP4)];\n uint32_t scaleA[ntx][MMQ_TILE_NE_K / (2 * QI_MXFP4)];\n\n // Block scale\n // Each thread has to point to a 4 byte scale value\n // https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-block-scaling\n\n const int i0 = (threadIdx.y / ntx) * rows_per_warp;\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 2 * QI_MXFP4) {\n const int k0 = k00 + k01;\n\n load_ldmatrix(A[n][k01 / (2 * QI_MXFP4)], x_qs + (i0 + n * tile_A::I) * MMQ_MMA_TILE_X_K_FP4 + k0,\n MMQ_MMA_TILE_X_K_FP4);\n\n // based on block-scaling document, 2 threads in each quad need to supply to the scale value\n const int tidx = threadIdx.x / 4 + (threadIdx.x % 2) * 8;\n scaleA[n][k01 / (2 * QI_MXFP4)] =\n *(x_sc + (i0 + n * tile_A::I + tidx) * MMQ_MMA_TILE_X_K_FP4 + k0 / (2 * QI_MXFP4));\n }\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx * tile_C::J) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 2 * QI_MXFP4) {\n tile_B B;\n uint32_t scaleB; // 2xN scales\n\n load_generic(B, y_qs + j0 * MMQ_TILE_Y_FP4_K + k01, MMQ_TILE_Y_FP4_K);\n\n scaleB = y_sc[(j0 + threadIdx.x / 4) * MMQ_TILE_Y_FP4_K + k01 / (2 * QI_MXFP4)];\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C;\n\n mma_block_scaled(C, A[n][k01 / (2 * QI_MXFP4)], B, scaleA[n][k01 / (2 * QI_MXFP4)], scaleB);\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n sum[(j0 / tile_C::J + n) * tile_C::ne + l] += C.x[l];\n }\n }\n }\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q8_1_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, mmq_y);\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + txs.qs;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_ds = (const half2 *) y;\n\n// #pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q8_1_q8_1_impl\n (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],\n x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + k0/QI8_1], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n }\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q8_1_q8_1_mma(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n constexpr data_layout input_layout = get_input_data_layout();\n typedef tile<16, 8, int, input_layout> tile_A;\n typedef tile<16, 8, int, input_layout> tile_B;\n typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_dm = (const half2 *) y;\n\n const int i0 = (threadIdx.y / ntx) * rows_per_warp;\n\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {\n const int k0 = k00 + k01;\n\n tile_A A[ntx];\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_1 + k0, MMQ_MMA_TILE_X_K_Q8_1);\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n tile_B B;\n load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);\n\n const int j = j0 + tile_C::get_j(0);\n const float2 dsB = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C;\n mma(C, A[n], B);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = i0 + n*tile_A::I + tile_C::get_i(l);\n float2 dmA = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + k0/QI8_1]);\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.x*dsB.x*C.x[l];\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.y*dsB.y;\n }\n }\n }\n }\n#else\n typedef tile<16, 8, int> tile_A;\n typedef tile< 8, 8, int> tile_B;\n typedef tile<16, 8, int> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = 2 * granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_dm = (const half2 *) y;\n\n tile_A A[ntx][MMQ_TILE_NE_K/QI8_1];\n float2 dmA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_1];\n\n const int i0 = (threadIdx.y/ntx)*rows_per_warp;\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {\n const int k0 = k00 + k01;\n\n load_ldmatrix(A[n][k01/QI8_1], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_1 + k0, MMQ_MMA_TILE_X_K_Q8_1);\n }\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);\n\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {\n const int k0 = k00 + k01;\n\n dmA[n][l][k01/QI8_1] = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + k0/QI8_1]);\n }\n }\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {\n tile_B B;\n float2 dsB[tile_C::ne/2];\n\n load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int j = j0 + tile_C::get_j(l);\n\n dsB[l] = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C;\n mma(C, A[n][k01/QI8_1], B);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].x*dsB[l%2].x*C.x[l];\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].y*dsB[l%2].y;\n }\n }\n }\n }\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n}\n\n// Used for Q3_K, IQ2_S, and IQ2_XS\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q8_0_16_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = MMQ_DP4A_TXS_Q8_0_16;\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + txs.qs;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n// #pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q8_0_16_q8_1_impl(\n &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0],\n &y_qs[j*MMQ_TILE_Y_K + k01],\n &x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + k0/(QI8_0/2)],\n y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n }\n }\n}\n\n// Used for Q3_K, IQ2_S, and IQ2_XS:\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q8_0_16_q8_1_mma(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n#if defined(AMD_MFMA_AVAILABLE)\n constexpr data_layout input_layout = get_input_data_layout();\n typedef tile<16, 8, int, input_layout> tile_A;\n typedef tile<16, 8, int, input_layout> tile_B;\n typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;\n typedef tile<64, 2, int, input_layout> tile_load;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n const int i0 = (threadIdx.y / ntx) * rows_per_warp;\n\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {\n const int k0 = k00 + k01;\n\n tile_A A[ntx];\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n load_generic(((tile_load *) A)[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q3_K + k0, MMQ_MMA_TILE_X_K_Q3_K);\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n tile_B B[1];\n load_generic(((tile_load *) B)[0], y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);\n\n const int j = j0 + tile_C::get_j(0);\n const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1] / 2;\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C;\n mma(C, A[n], B[0]);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(l);\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*MMQ_MMA_TILE_X_K_Q3_K + k0/4] * dB;\n }\n }\n }\n }\n#elif defined(AMD_WMMA_AVAILABLE) //wmma instructions can handle 16x4 tiles, does not require loading 64x2 tiles\n constexpr data_layout input_layout = get_input_data_layout();\n typedef tile<16, 4, int, input_layout> tile_A;\n typedef tile<16, 4, int, input_layout> tile_B;\n typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n const int i0 = (threadIdx.y / ntx) * rows_per_warp;\n\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {\n const int k0 = k00 + k01;\n\n tile_A A[ntx];\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q3_K + k0, MMQ_MMA_TILE_X_K_Q3_K);\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n tile_B B;\n load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);\n\n const int j = j0 + tile_C::get_j(0);\n const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C;\n mma(C, A[n], B);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(l);\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*MMQ_MMA_TILE_X_K_Q3_K + k0/4] * dB;\n }\n }\n }\n }\n#elif defined(TURING_MMA_AVAILABLE)\n\n typedef tile<16, 4, int> tile_A;\n typedef tile<16, 8, int> tile_A_8;\n typedef tile< 8, 4, int> tile_B;\n typedef tile<16, 8, int> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = 2 * granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);\n\n tile_A A[ntx][8];\n float dA[ntx][tile_C::ne/2][8];\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {\n const int k0 = k00 + k01;\n\n load_ldmatrix(((tile_A_8 *) A[n])[k01/8], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q3_K + k0, MMQ_MMA_TILE_X_K_Q3_K);\n }\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);\n\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {\n const int k0 = k00 + k01;\n\n dA[n][l][k01/4] = x_df[i*MMQ_MMA_TILE_X_K_Q3_K + k0/4];\n }\n }\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {\n tile_B B[2];\n float dB[tile_C::ne/2];\n\n // Here load_generic is faster than load_ldmatrix.\n load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);\n load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int j = j0 + tile_C::get_j(l);\n\n dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C[2];\n mma(C[0], A[n][k01/4 + 0], B[0]);\n mma(C[1], A[n][k01/4 + 1], B[1]);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += dB[l%2]*(C[0].x[l]*dA[n][l/2][k01/4 + 0] + C[1].x[l]*dA[n][l/2][k01/4 + 1]);\n }\n }\n }\n }\n#else\n GGML_UNUSED_VARS(x, y, sum, k00);\n NO_DEVICE_CODE;\n#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_q2_K(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, mmq_y);\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR2_K);\n constexpr int nrows = ggml_cuda_get_physical_warp_size() / threads_per_row;\n const int kqsx = threadIdx.x % threads_per_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q2_K * bxi = (const block_q2_K *) x + kbx0 + i*stride;\n\n const int x_ql_0 = get_int_b2(bxi->qs, kqsx);\n\n#pragma unroll\n for (int l = 0; l < QR2_K; ++l) {\n const int k = (kqsx/8)*32 + l*8 + kqsx % 8;\n\n const int x_qs_k = (x_ql_0 >> (2*l)) & 0x03030303;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q2_K + k] = x_qs_k;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n const int sc_m = bxi->scales[kqsx];\n#ifdef FAST_FP16_AVAILABLE\n const half2 x_dm_ik = __hmul2(bxi->dm, make_half2(sc_m & 0x0F, sc_m >> 4));\n#else\n const float2 bxi_dmf = __half22float2(bxi->dm);\n const half2 x_dm_ik = make_half2(bxi_dmf.x*(sc_m & 0x0F), bxi_dmf.y*(sc_m >> 4));\n#endif // FAST_FP16_AVAILABLE\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + kqsx] = x_dm_ik;\n#else\n x_dm[i*(MMQ_TILE_NE_K + 1) + kqsx] = x_dm_ik;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q2_K_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, mmq_y);\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + txs.qs;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_ds = (const half2 *) y;\n\n float2 y_df[mmq_x/nwarps];\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n y_df[j0/nwarps] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);\n }\n\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K/2; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n constexpr int ns = 2;\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq(\n &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],\n &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,\n &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);\n }\n }\n }\n\n // Some compilers fail to unroll the loop over k01 if there is a conditional statement for ns in the inner loop.\n // As a workaround 2 separate loops are used instead.\n#pragma unroll\n for (int k01 = MMQ_TILE_NE_K/2; k01 < MMQ_TILE_NE_K; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n constexpr int ns = 1;\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq(\n &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],\n &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,\n &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);\n }\n }\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q2_K_q8_1_mma(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n#if defined(AMD_MFMA_AVAILABLE)\n constexpr data_layout input_layout = get_input_data_layout();\n typedef tile<16, 8, int, input_layout> tile_A;\n typedef tile<16, 8, int, input_layout> tile_B;\n typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;\n typedef tile<64, 2, int, input_layout> tile_load;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_ds = (const half2 *) y;\n\n const int i0 = (threadIdx.y / ntx) * rows_per_warp;\n\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {\n const int k0 = k00 + k01;\n\n tile_A A[ntx];\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n load_generic(((tile_load *) A)[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q2_K + k0, MMQ_MMA_TILE_X_K_Q2_K);\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n tile_B B[1];\n load_generic(((tile_load *) B)[0], y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);\n\n const int j = j0 + tile_C::get_j(0);\n const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x/2 : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y/2;\n const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0\n : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y\n : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x);\n\n tile_C Cm;\n if (k01 >= MMQ_TILE_NE_K * 3/4) {\n tile_A A1;\n A1.x[0] = 0x01010101;\n A1.x[1] = 0x01010101;\n mma(Cm, A1, B[0]);\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C Cd;\n mma(Cd, A[n], B[0]);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(l);\n const float2 dm = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + k0/4]);\n float tmp = Cd.x[l]*dm.x;\n if (k01 >= MMQ_TILE_NE_K * 3/4) {\n tmp -= Cm.x[l]*dm.y;\n }\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB;\n sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB;\n }\n }\n }\n }\n#elif defined(AMD_WMMA_AVAILABLE) //wmma instructions can handle 16x4 tiles, does not require loading 64x2 tiles\n constexpr data_layout input_layout = get_input_data_layout();\n typedef tile<16, 4, int, input_layout> tile_A;\n typedef tile<16, 4, int, input_layout> tile_B;\n typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_ds = (const half2 *) y;\n\n const int i0 = (threadIdx.y / ntx) * rows_per_warp;\n\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {\n const int k0 = k00 + k01;\n\n tile_A A[ntx];\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q2_K + k0, MMQ_MMA_TILE_X_K_Q2_K);\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n tile_B B;\n load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);\n\n const int j = j0 + tile_C::get_j(0);\n const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y;\n const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0\n : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y\n : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x);\n\n tile_C Cm;\n if (k01 >= MMQ_TILE_NE_K * 3/4) {\n tile_A A1;\n#pragma unroll\n for (int l = 0; l < tile_A::ne; ++l) {\n A1.x[l] = 0x01010101;\n }\n mma(Cm, A1, B);\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C Cd;\n mma(Cd, A[n], B);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(l);\n const float2 dm = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + k0/4]);\n float tmp = Cd.x[l]*dm.x;\n if (k01 >= MMQ_TILE_NE_K * 3/4) {\n tmp -= Cm.x[l]*dm.y;\n }\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB;\n sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB;\n }\n }\n }\n }\n#elif defined(TURING_MMA_AVAILABLE)\n\n typedef tile<16, 4, int> tile_A;\n typedef tile<16, 8, int> tile_A_8;\n typedef tile< 8, 4, int> tile_B;\n typedef tile<16, 8, int> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = 2 * granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_ds = (const half2 *) y;\n\n const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);\n\n tile_A A[ntx][8];\n float dA[ntx][tile_C::ne/2][8];\n float mA[ntx][tile_C::ne/2][8];\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {\n const int k0 = k00 + k01;\n\n load_ldmatrix(((tile_A_8 *) A[n])[k01/QI8_1], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q2_K + k0, MMQ_MMA_TILE_X_K_Q2_K);\n }\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);\n\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1/2) {\n const int k0 = k00 + k01;\n\n const float2 dm = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + k0/(QI8_1/2)]);\n\n dA[n][l][k01/(QI8_1/2)] = dm.x;\n mA[n][l][k01/(QI8_1/2)] = dm.y;\n }\n }\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n float2 dB[tile_C::ne/2];\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int j = j0 + tile_C::get_j(l);\n\n dB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);\n }\n\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {\n tile_B B[2];\n\n // Here load_generic is faster than load_ldmatrix.\n load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);\n load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);\n\n tile_C Cm[2];\n if (k01 >= MMQ_TILE_NE_K * 3/4) {\n tile_A A1;\n A1.x[0] = 0x01010101;\n A1.x[1] = 0x01010101;\n mma(Cm[0], A1, B[0]);\n mma(Cm[1], A1, B[1]);\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C Cd[2];\n\n mma(Cd[0], A[n][k01/4 + 0], B[0]);\n mma(Cd[1], A[n][k01/4 + 1], B[1]);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n float tmp = Cd[0].x[l]*dA[n][l/2][k01/4 + 0] + Cd[1].x[l]*dA[n][l/2][k01/4 + 1];\n if (k01 >= MMQ_TILE_NE_K * 3/4) {\n tmp -= Cm[0].x[l]*mA[n][l/2][k01/4 + 0] + Cm[1].x[l]*mA[n][l/2][k01/4 + 1];\n }\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*(k01 < MMQ_TILE_NE_K/2 ? dB[l%2].x : dB[l%2].y);\n }\n }\n }\n\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K * 3/4; k01 += QI8_1) {\n float2 sB[tile_C::ne/2];\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int j = j0 + tile_C::get_j(l);\n\n sB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 0]*sB[l%2].x;\n sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 1]*sB[l%2].y;\n }\n }\n }\n }\n#else\n GGML_UNUSED_VARS(x, y, sum, k00);\n NO_DEVICE_CODE;\n#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_q3_K(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n int * x_sc = (int *) (x_df + txs.dm);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR3_K);\n constexpr int nrows = warp_size / threads_per_row;\n const int kqsx = threadIdx.x % threads_per_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;\n\n const int x_ql_0 = get_int_b2(bxi->qs, kqsx);\n const int x_qh_0 = get_int_b2(bxi->hmask, kqsx % (QI3_K/2)) >> (4 * (kqsx / (QI3_K/2)));\n\n#pragma unroll\n for (int l = 0; l < QR3_K; ++l) {\n const int k = (kqsx/8)*32 + l*8 + kqsx % 8;\n\n const int x_ql_k = (x_ql_0 >> (2*l)) & 0x03030303;\n const int x_qh_k = ((x_qh_0 >> l) << 2) & 0x04040404;\n\n const int x_qs_k = __vsubss4(x_ql_k | x_qh_k, 0x04040404);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + k] = x_qs_k;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n }\n\n constexpr int rows_per_warp = warp_size / 4;\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {\n int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/4;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;\n\n const int ksc = threadIdx.x % 4;\n\n const int ksc_low = ksc % (QI3_K/8);\n const int shift_low = 4 * (ksc / (QI3_K/8));\n const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;\n\n const int ksc_high = QI3_K/8;\n const int shift_high = 2 * ksc;\n const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;\n\n const int sc = __vsubss4(sc_low | sc_high, 0x20202020);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n const int8_t * sc8 = (const int8_t *) ≻\n const float d = bxi->d;\n\n#pragma unroll\n for (int l = 0; l < int(sizeof(int)); ++l) {\n x_df[i*MMQ_MMA_TILE_X_K_Q3_K + sizeof(int)*ksc + l] = d*sc8[l];\n }\n#else\n x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = sc;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n#if !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE))\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) {\n int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;\n\n x_df[i] = bxi->d;\n }\n#endif // !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)) || defined(AMD_WMMA_AVAILABLE)\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q3_K_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, mmq_y);\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + txs.qs;\n const int * x_sc = (const int *) x_df + txs.dm;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n// #pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n const int8_t * scales = ((const int8_t *) (x_sc + i*(MMQ_TILE_NE_K/8) + i/8)) + k0/4;\n\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q3_K_q8_1_impl_mmq(\n &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], scales,\n x_df[i], y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n }\n }\n}\n\nstatic __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, const int ksc) {\n // scale arrangement after the following two lines:\n // - ksc == 0: sc0, sc1, sc2, sc3\n // - ksc == 1: sc4, sc5, sc6, sc7\n // - ksc == 2: m0, m1, m2, m3\n // - ksc == 3: m4, m5, m6, m7\n return ((scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F) | // lower 4 bits\n ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_q4_K(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, mmq_y);\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + txs.qs);\n int * x_sc = (int *) (x_dm + txs.dm);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_K);\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;\n const int qs0 = get_int_b4(bxi->qs, txi);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 16*(txi/8) + txi % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 16*(txi/8) + txi % 8 + 8] = (qs0 >> 4) & 0x0F0F0F0F;\n#else\n x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n constexpr int rows_per_warp = warp_size / 2;\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {\n#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n // Need if on AMD instead of % because warp_size == 64\n // This causes double work and throughput loss (MI300X)\n // H100 loses about 100 t/s with 'if' condition over '%'\n int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;\n if (i < mmq_y) {\n#else\n int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % mmq_y;\n {\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;\n\n const int * scales = (const int *) bxi->scales;\n const int ksc = threadIdx.x % 2;\n\n const int sc32 = unpack_scales_q45_K(scales, ksc + 0);\n const int m32 = unpack_scales_q45_K(scales, ksc + 2);\n\n const uint8_t * sc8 = (const uint8_t *) &sc32;\n const uint8_t * m8 = (const uint8_t *) &m32;\n\n const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);\n\n #pragma unroll\n for (int l = 0; l < sizeof(int); ++l) {\n x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);\n }\n }\n }\n#else\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) {\n int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;\n\n x_dm[i] = bxi->dm;\n }\n constexpr int rows_per_warp = warp_size / 4;\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {\n int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % mmq_y;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8);\n\n const int * scales = (const int *) bxi->scales;\n\n const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);\n const int scales8 = unpack_scales_q45_K(scales, ksc);\n\n x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;\n }\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q4_K_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, mmq_y);\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + txs.qs;\n const int * x_sc = (const int *) x_dm + txs.dm;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_ds = (const half2 *) y;\n\n// #pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_K*VDR_Q4_K_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n const uint8_t * sc = (const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/32] + 2*(k01/16);\n\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q4_K_q8_1_impl_mmq(\n &x_qs[i*(MMQ_TILE_NE_K + 1) + k0/2], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,\n x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n }\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_q5_K(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, mmq_y);\n int * x_qs = (int *) x_tile;\n half2 * x_dm = (half2 *) (x_qs + txs.qs);\n int * x_sc = (int *) (x_dm + txs.dm);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_K);\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;\n const int ky = QR5_K*txi;\n\n const int ql = get_int_b4(bxi->qs, txi);\n const int ql0 = (ql >> 0) & 0x0F0F0F0F;\n const int ql1 = (ql >> 4) & 0x0F0F0F0F;\n\n const int qh = get_int_b4(bxi->qh, txi % (QI5_K/4));\n const int qh0 = ((qh >> (2 * (txi / (QI5_K/4)) + 0)) << 4) & 0x10101010;\n const int qh1 = ((qh >> (2 * (txi / (QI5_K/4)) + 1)) << 4) & 0x10101010;\n\n const int kq0 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + 0;\n const int kq1 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + QI5_K/4;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kq0] = ql0 | qh0;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kq1] = ql1 | qh1;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = ql0 | qh0;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = ql1 | qh1;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n constexpr int rows_per_warp = warp_size / 2;\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {\n#if defined(AMD_MFMA_AVAILABLE)\n // Need if on AMD instead of % because warp_size == 64\n // This causes double work and throughput loss (MI300X)\n // H100 loses about 100 t/s with 'if' condition over '%'\n int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;\n if (i < mmq_y) {\n#else\n int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % mmq_y;\n {\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;\n\n const int * scales = (const int *) bxi->scales;\n const int ksc = threadIdx.x % 2;\n\n const int sc32 = unpack_scales_q45_K(scales, ksc + 0);\n const int m32 = unpack_scales_q45_K(scales, ksc + 2);\n\n const uint8_t * sc8 = (const uint8_t *) &sc32;\n const uint8_t * m8 = (const uint8_t *) &m32;\n\n const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);\n\n#pragma unroll\n for (int l = 0; l < int(sizeof(int)); ++l) {\n x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);\n }\n }\n }\n#else\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) {\n int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;\n\n x_dm[i] = bxi->dm;\n }\n\n constexpr int rows_per_warp = warp_size / 4;\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {\n int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % mmq_y;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;\n\n const int * scales = (const int *) bxi->scales;\n\n const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);\n const int scales8 = unpack_scales_q45_K(scales, ksc);\n\n x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;\n }\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q5_K_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, mmq_y);\n const int * x_qs = (const int *) x;\n const half2 * x_dm = (const half2 *) x_qs + txs.qs;\n const int * x_sc = (const int *) x_dm + txs.dm;\n const int * y_qs = (const int *) y + 4;\n const half2 * y_ds = (const half2 *) y;\n\n// #pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR5_K*VDR_Q5_K_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n const uint8_t * sc = ((const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k00/32]) + 2*(k01/16);\n\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q5_K_q8_1_impl_mmq(\n &x_qs[i*(QR5_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,\n x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n }\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_q6_K(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n int * x_sc = (int *) (x_df + MMQ_TILE_NE_K/QI6_K);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n int * x_sc = (int *) (x_df + txs.dm);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR6_K);\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;\n\n const int ql = get_int_b2(bxi->ql, txi);\n const int ql0 = (ql >> 0) & 0x0F0F0F0F;\n const int ql1 = (ql >> 4) & 0x0F0F0F0F;\n\n const int qh = get_int_b2(bxi->qh, (QI6_K/4) * (txi / (QI6_K/2)) + txi % (QI6_K/4));\n const int qh0 = ((qh >> ((txi & 0x08) >> 2)) << 4) & 0x30303030;\n const int qh1 = (qh >> ((txi & 0x08) >> 2)) & 0x30303030;\n\n const int kq0 = 2*txi - txi % (QI6_K/2) + 0;\n const int kq1 = 2*txi - txi % (QI6_K/2) + QI6_K/2;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q6_K + kq0] = __vsubss4(ql0 | qh0, 0x20202020);\n x_qs[i*MMQ_MMA_TILE_X_K_Q6_K + kq1] = __vsubss4(ql1 | qh1, 0x20202020);\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) {\n int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q6_K] = bxi->d;\n#else\n x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K] = bxi->d;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n constexpr int rows_per_warp = warp_size / 4;\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {\n int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % mmq_y;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_sc[i*MMQ_MMA_TILE_X_K_Q6_K + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8));\n#else\n x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + threadIdx.x%(MMQ_TILE_NE_K/8)] = get_int_b2(bxi->scales, threadIdx.x%(QI6_K/8));\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q6_K_q8_1_dp4a(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, mmq_y);\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + txs.qs;\n const int * x_sc = (const int *) x_df + txs.dm;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n// #pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR6_K*VDR_Q6_K_Q8_1_MMQ) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n const int8_t * sc = ((const int8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/16]);\n\n sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q6_K_q8_1_impl_mmq(\n &x_qs[i*(QR6_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc,\n x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K], &y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);\n }\n }\n }\n}\n\ntemplate \nstatic __device__ __forceinline__ void vec_dot_q6_K_q8_1_mma(\n const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {\n#if defined(AMD_MFMA_AVAILABLE)\n constexpr data_layout input_layout = get_input_data_layout();\n typedef tile<16, 8, int, input_layout> tile_A;\n typedef tile<16, 8, int, input_layout> tile_B;\n typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;\n typedef tile<64, 2, int, input_layout> tile_load;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;\n const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n const int i0 = (threadIdx.y / ntx) * rows_per_warp;\n\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {\n const int k0 = k00 + k01;\n\n tile_A A[ntx];\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n load_generic(((tile_load *) A)[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + k0, MMQ_MMA_TILE_X_K_Q6_K);\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n tile_B B[1];\n load_generic(((tile_load *) B)[0], y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);\n\n const int j = j0 + tile_C::get_j(0);\n const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1] / 2;\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C;\n mma(C, A[n], B[0]);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(l);\n const int8_t * sc = (const int8_t *) (x_sc + i*MMQ_MMA_TILE_X_K_Q6_K + k00/16);\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*MMQ_MMA_TILE_X_K_Q6_K] * dB;\n }\n }\n }\n }\n#elif defined(AMD_WMMA_AVAILABLE) //wmma instructions can handle 16x4 tiles, does not require loading 64x2 tiles\n constexpr data_layout input_layout = get_input_data_layout();\n typedef tile<16, 4, int, input_layout> tile_A;\n typedef tile<16, 4, int, input_layout> tile_B;\n typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;\n const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n const int i0 = (threadIdx.y / ntx) * rows_per_warp;\n\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {\n const int k0 = k00 + k01;\n\n tile_A A[ntx];\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + k0, MMQ_MMA_TILE_X_K_Q6_K);\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n tile_B B;\n load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);\n\n const int j = j0 + tile_C::get_j(0);\n const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C;\n mma(C, A[n], B);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(l);\n const int8_t * sc = (const int8_t *) (x_sc + i*MMQ_MMA_TILE_X_K_Q6_K + k00/16);\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*MMQ_MMA_TILE_X_K_Q6_K] * dB;\n }\n }\n }\n }\n#elif defined(TURING_MMA_AVAILABLE)\n\n typedef tile<16, 4, int> tile_A;\n typedef tile< 8, 4, int> tile_B;\n typedef tile<16, 8, int> tile_C;\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int rows_per_warp = 2 * granularity;\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);\n\n const int * x_qs = (const int *) x;\n const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;\n const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;\n const int * y_qs = (const int *) y + 4;\n const float * y_df = (const float *) y;\n\n const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);\n\n tile_A A[ntx][8];\n int scA[ntx][tile_C::ne/2][8];\n float dA[ntx][tile_C::ne/2];\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {\n const int k0 = k00 + k01;\n\n load_ldmatrix(A[n][k01/4 + 0], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + (k0 + 0), MMQ_MMA_TILE_X_K_Q6_K);\n load_ldmatrix(A[n][k01/4 + 1], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + (k0 + tile_A::J), MMQ_MMA_TILE_X_K_Q6_K);\n }\n\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 16) {\n const int k0 = k00 + k01;\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);\n\n const int sc_packed = x_sc[i*MMQ_MMA_TILE_X_K_Q6_K + k0/16];\n const int8_t * sc = (const int8_t *) &sc_packed;\n\n#pragma unroll\n for (int ksc = 0; ksc < sizeof(int); ++ksc) {\n scA[n][l][k01/4 + ksc] = sc[ksc];\n }\n }\n }\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);\n\n dA[n][l] = x_df[i*MMQ_MMA_TILE_X_K_Q6_K];\n }\n }\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n float tmp[ntx][tile_C::ne] = {{0.0f}};\n\n#pragma unroll\n for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {\n tile_B B[2];\n float dB[tile_C::ne/2];\n\n // Here load_generic is faster than load_ldmatrix.\n load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + 0 + k01, MMQ_TILE_Y_K);\n load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + tile_B::J + k01, MMQ_TILE_Y_K);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne/2; ++l) {\n const int j = j0 + tile_C::get_j(l);\n\n dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n tile_C C[2];\n mma(C[0], A[n][k01/4 + 0], B[0]);\n mma(C[1], A[n][k01/4 + 1], B[1]);\n\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n tmp[n][l] += (C[0].x[l]*scA[n][l/2][k01/4 + 0] + C[1].x[l]*scA[n][l/2][k01/4 + 1])*dB[l%2];\n }\n }\n }\n\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp[n][l]*dA[n][l/2];\n }\n }\n }\n#else\n GGML_UNUSED_VARS(x, y, sum, k00);\n NO_DEVICE_CODE;\n#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_iq4_nl(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_NL, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_NL);\n constexpr int nrows = warp_size / threads_per_row;\n const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n const int kbx = txi / QI4_NL;\n const int kqsx = txi % QI4_NL;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbx;\n\n const int aux_q4 = get_int_b2(bxi->qs, kqsx);\n const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);\n const int k0 = kbx * (2 * QI4_NL) + kqsx;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 0] = v.x;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + QI4_NL] = v.y;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI4_NL] = v.y;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_NL;\n constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;\n const int kbxd = threadIdx.x % blocks_per_tile_x_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {\n int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbxd;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = __half2float(bxi->d);\n#else\n x_df[i*(MMQ_TILE_NE_K/QI4_NL) + i/QI4_NL + kbxd] = __half2float(bxi->d);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_iq2_xxs(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XXS, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XXS)) / 2;\n constexpr int nrows = warp_size / threads_per_row;\n const int kqsx = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {\n int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq2_xxs * bxi = (const block_iq2_xxs *) x + kbx0 + i*stride;\n\n const int q2 = get_int_b2(bxi->qs, 2*kqsx+0);\n const uint8_t * aux8 = (const uint8_t *) &q2;\n const uint32_t aux32 = get_int_b2(bxi->qs, 2*kqsx+1);\n\n#pragma unroll\n for (int l = 0; l < QR2_XXS; ++l) {\n const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[l]];\n const uint32_t signs = unpack_ksigns(aux32 >> (7 * l));\n\n const int signs0 = __vcmpne4(signs & 0x08040201, 0);\n const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0);\n\n const int signs1 = __vcmpne4(signs & 0x80402010, 0);\n const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 0)] = grid0;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 1)] = grid1;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n const int ls = aux32 >> 27 | 1; // (scale * 2 + 1)\n const float d = bxi->d;\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4\n#else\n x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_iq2_xs(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = MMQ_DP4A_TXS_Q8_0_16;\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XS)) / 2;\n constexpr int nrows = warp_size / threads_per_row;\n const int kqsx = threadIdx.x % threads_per_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {\n int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq2_xs * bxi = (const block_iq2_xs *) x + kbx0 + i*stride;\n\n const int2 q2_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));\n const uint16_t * q2 = (const uint16_t *) &q2_packed;\n\n #pragma unroll\n for (int l = 0; l < QR2_XS; ++l) {\n const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l] & 0x1FF];\n const uint32_t signs = unpack_ksigns(q2[l] >> 9);\n\n const int signs0 = __vcmpne4(signs & 0x08040201, 0);\n const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);\n\n const int signs1 = __vcmpne4(signs & 0x80402010, 0);\n const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 0)] = grid_l;\n x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 1)] = grid_h;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n const int ls = bxi->scales[kqsx];\n const float d = bxi->d;\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;\n x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;\n#else\n x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;\n x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_iq2_s(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_S, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_S)) / 2;\n constexpr int nrows = warp_size / threads_per_row;\n const int kqsx = threadIdx.x % threads_per_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {\n int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq2_s * bxi = (const block_iq2_s *) x + kbx0 + i*stride;\n\n const int qs_packed = get_int_b2(bxi->qs, kqsx);\n const uint8_t * qs = (const uint8_t *) &qs_packed;\n\n const int qh = bxi->qh[kqsx];\n\n const int signs_packed_32 = get_int_b2(bxi->qs, QK_K/32 + kqsx);\n const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;\n\n#pragma unroll\n for (int l = 0; l < QR2_S; ++l) {\n const int * grid_pos = (const int *)(iq2s_grid + (qs[l] | ((qh << (8-2*l)) & 0x300)));\n\n const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);\n const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);\n\n const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);\n const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 0)] = grid_l;\n x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 1)] = grid_h;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n const int ls = bxi->scales[kqsx];\n const float d = bxi->d;\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;\n x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;\n#else\n x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;\n x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_iq3_xxs(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_XXS, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_XXS)) / 2;\n constexpr int nrows = warp_size / threads_per_row;\n const int kqsx = threadIdx.x % threads_per_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {\n int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq3_xxs * bxi = (const block_iq3_xxs *) x + kbx0 + i*stride;\n\n const int2 q3_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));\n const uint8_t * q3 = (const uint8_t *) &q3_packed;\n const uint32_t aux32 = get_int_b2(bxi->qs, QK_K/16 + kqsx);\n\n#pragma unroll\n for (int l = 0; l < QR3_XXS; ++l) {\n const int2 grid_pos = make_int2(iq3xxs_grid[q3[2*l+0]], iq3xxs_grid[q3[2*l+1]]);\n const uint32_t signs = unpack_ksigns(aux32 >> (7*l));\n\n const int signs0 = __vcmpne4(signs & 0x08040201, 0);\n const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);\n\n const int signs1 = __vcmpne4(signs & 0x80402010, 0);\n const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 0)] = grid_l;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 1)] = grid_h;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n const int ls = aux32 >> 28;\n const float d = bxi->d;\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = (ls*d + d/2)/2;\n#else\n x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = (ls*d + d/2)/2;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_iq3_s(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_S)) / 2;\n constexpr int nrows = warp_size / threads_per_row;\n const int kqsx = threadIdx.x % threads_per_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {\n int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq3_s * bxi = (const block_iq3_s *) x + kbx0 + i*stride;\n\n const int2 qs_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));\n const uint8_t * qs = (const uint8_t *) &qs_packed;\n\n const int qh = bxi->qh[kqsx];\n\n const int signs_packed_32 = get_int_b2(bxi->signs, kqsx);\n const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;\n\n#pragma unroll\n for (int l = 0; l < QR3_S; ++l) {\n const int2 grid_pos = make_int2(\n iq3s_grid[qs[2*l+0] | ((qh << (8 - 2*l)) & 0x100)],\n iq3s_grid[qs[2*l+1] | ((qh << (7 - 2*l)) & 0x100)]);\n\n const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);\n const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);\n\n const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);\n const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l+0)] = grid_l;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l+1)] = grid_h;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid_l;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid_h;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n const int ls = 1 + 2*((bxi->scales[kqsx/2] >> (((2*kqsx) << 1) & 0x04)) & 0x0F);\n const float d = bxi->d;\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = ls*d;\n#else\n x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = ls*d;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_iq1_s(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, mmq_y);\n int * x_qs = (int *) x_tile;\n half2 * x_ds = (half2 *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_S);\n constexpr int nrows = warp_size / threads_per_row;\n const int kqsx = threadIdx.x % threads_per_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {\n int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq1_s * bxi = (const block_iq1_s *) x + kbx0 + i*stride;\n\n const int qs_packed = get_int_b2(bxi->qs, kqsx);\n const uint8_t * qs = (const uint8_t *) &qs_packed;\n\n const int qh = bxi->qh[kqsx];\n\n #pragma unroll\n for (int l = 0; l < QR1_S/2; ++l) {\n const int grid = iq1s_grid_gpu[qs[l] | (((qh >> (3*l)) & 0x07) << 8)];\n\n const int grid0 = (grid >> 0) & 0x0F0F0F0F;\n const int grid1 = (grid >> 4) & 0x0F0F0F0F;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 8*kqsx + (2*l+0)] = grid0;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 8*kqsx + (2*l+1)] = grid1;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n const float d1q = __half2float(bxi->d) * (((qh >> 11) & 0x0E) + 1);\n const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_ds[i*MMQ_MMA_TILE_X_K_Q8_1 + kqsx] = make_half2(d1q, d1q*delta);\n#else\n x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate static __device__ __forceinline__ void load_tiles_iq4_xs(\n const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);\n#else\n constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_XS, mmq_y);\n int * x_qs = (int *) x_tile;\n float * x_df = (float *) (x_qs + txs.qs);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_XS);\n constexpr int nrows = warp_size / threads_per_row;\n const int kqsx = threadIdx.x % threads_per_row;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {\n int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;\n\n const int aux_q4 = get_int_b4(bxi->qs, kqsx);\n const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);\n const int k0 = 8 * (kqsx / 4) + kqsx % 4;\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 0] = v.x;\n x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 4] = v.y;\n#else\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;\n x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 4] = v.y;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n\n constexpr int rows_per_warp = warp_size / 8;\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {\n int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / (MMQ_TILE_NE_K/4);\n\n if (need_check) {\n i = min(i, i_max);\n }\n\n const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;\n\n const float d = __half2float(bxi->d);\n\n const int ls = ((bxi->scales_l[(threadIdx.x % 8)/2] >> (4*(threadIdx.x % 2))) & 0x0F)\n | (((bxi->scales_h >> (2*(threadIdx.x % 8))) & 0x03) << 4);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + threadIdx.x % 8] = d * (ls - 32);\n#else\n x_df[i*(MMQ_TILE_NE_K/4) + i/4 + threadIdx.x % 8] = d * (ls - 32);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n }\n}\n\ntemplate\nstatic __device__ __forceinline__ void mmq_write_back_dp4a(\n const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst,\n const int stride, const int i_max, const int j_max) {\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (j > j_max) {\n return;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n if (need_check && i > i_max) {\n continue;\n }\n\n dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (mmq_y/warp_size) + i0/warp_size];\n }\n }\n}\n\ntemplate\nstatic __device__ __forceinline__ void mmq_write_back_mma(\n const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst,\n const int stride, const int i_max, const int j_max) {\n\n constexpr int granularity = mmq_get_granularity_device(mmq_x);\n constexpr int nwarps = mmq_get_nwarps_device();\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n constexpr int tileC_IJ = mmq_get_granularity_device(0);\n typedef tile tile_C;\n constexpr int rows_per_warp = granularity;\n#else\n typedef tile<16, 8, int> tile_C;\n constexpr int rows_per_warp = 2 * granularity;\n#endif // defined(AMD_MFMA_AVAILABLE)\n constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.\n\n const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I);\n#if defined(TURING_MMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n static_assert(nwarps*tile_C::I == mmq_y, \"nwarps*tile_C::I != mmq_y\");\n#else\n GGML_UNUSED(nwarps);\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {\n#pragma unroll\n for (int n = 0; n < ntx; ++n) {\n#pragma unroll\n for (int l = 0; l < tile_C::ne; ++l) {\n const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l);\n\n if (j > j_max) {\n continue;\n }\n\n const int i = i0 + n*tile_C::I + tile_C::get_i(l);\n\n if (need_check && i > i_max) {\n continue;\n }\n\n dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];\n }\n }\n }\n}\n\n// -------------------------------------------------------------------------------------------------------------------------------------\n\ntemplate \nstruct mmq_type_traits;\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q4_0_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_0;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_0_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q4_1_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_1;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_1_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q5_0_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_0;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q5_1_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_1;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_1_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q8_0_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q8_0;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_MXFP4_Q8_1_MMQ;\n#ifdef BLACKWELL_MMA_AVAILABLE\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_mxfp4_fp4;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_mxfp4_mxfp4_mma;\n#else\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_mxfp4;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma;\n#endif // BLACKWELL_MMA_AVAILABLE\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q2_K_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q2_K;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q2_K_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q2_K_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q3_K_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q3_K;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q3_K_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q4_K_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_K;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_K_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q5_K_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_K;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q5_K_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_Q6_K_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_q6_K;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q6_K_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q6_K_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_IQ2_XXS_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_xxs;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_IQ2_XS_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_xs;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_IQ2_S_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_s;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_IQ3_XXS_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq3_xxs;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_IQ3_S_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq3_s;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_IQ1_S_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq1_s;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_1_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_IQ4_NL_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq4_nl;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a;\n};\n\ntemplate \nstruct mmq_type_traits {\n static constexpr int vdr = VDR_IQ4_XS_Q8_1_MMQ;\n static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq4_xs;\n static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma;\n static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a;\n};\n\ntemplate \nstatic __device__ __forceinline__ void mul_mat_q_process_tile(\n const char * __restrict__ x, const int offset_x, const int * __restrict__ y,\n const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup,\n const int stride_row_x, const int ncols_y, const int stride_col_dst,\n const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {\n\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int qk = ggml_cuda_type_traits::qk;\n constexpr int mmq_y = get_mmq_y_device();\n constexpr load_tiles_mmq_t load_tiles = mmq_type_traits::load_tiles;\n\n extern __shared__ int data_mul_mat_q[];\n int * tile_y = data_mul_mat_q + mmq_x;\n int * tile_x = tile_y + GGML_PAD(mmq_x*MMQ_TILE_Y_K, nwarps*warp_size);\n\n#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n constexpr vec_dot_mmq_t vec_dot = mmq_type_traits::vec_dot_mma;\n constexpr mmq_write_back_t write_back = mmq_write_back_mma;\n#else\n constexpr vec_dot_mmq_t vec_dot = mmq_type_traits::vec_dot_dp4a;\n constexpr mmq_write_back_t write_back = mmq_write_back_dp4a;\n#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)\n\n#if defined(BLACKWELL_MMA_AVAILABLE)\n // FP4 tile stores 8 blocks\n constexpr int ne_block = (type == GGML_TYPE_MXFP4) ? 8 * QK_MXFP4 : 4 * QK8_1;\n#else\n constexpr int ne_block = 4 * QK8_1;\n#endif // defined(BLACKWELL_MMA_AVAILABLE)\n\n constexpr int ITER_K = get_iter_k(type);\n constexpr int blocks_per_iter = ITER_K / qk;\n\n float sum[mmq_x*mmq_y / (nwarps*warp_size)] = {0.0f};\n\n constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int);\n\n for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) {\n load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x);\n {\n const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz;\n#pragma unroll\n for (int l0 = 0; l0 < mmq_x * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {\n int l = l0 + threadIdx.y*warp_size + threadIdx.x;\n\n tile_y[l] = by0[l];\n }\n }\n\n __syncthreads();\n\n vec_dot(tile_x, tile_y, sum, 0);\n\n __syncthreads();\n\n {\n const int * by0 = y + ncols_y * ((kb0 * qk / ne_block) * sz + sz);\n#pragma unroll\n for (int l0 = 0; l0 < mmq_x * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {\n int l = l0 + threadIdx.y*warp_size + threadIdx.x;\n\n tile_y[l] = by0[l];\n }\n }\n\n __syncthreads();\n\n vec_dot(tile_x, tile_y, sum, MMQ_TILE_NE_K);\n\n __syncthreads();\n }\n\n if (fixup) {\n write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(mmq_x*mmq_y), mmq_y, mmq_y, mmq_x);\n } else {\n write_back(sum, ids_dst, dst, stride_col_dst, tile_x_max_i, tile_y_max_j);\n }\n}\n\n\n// The mul_mat_q kernel implements \"stream-k\" work partitioning as described in https://arxiv.org/abs/2301.03598\n\ntemplate \n#if defined(GGML_USE_HIP)\n#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)\n __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 2)\n#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)\n#else\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 1)\n#else\n __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 2)\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n#endif // defined(GGML_USE_HIP)\nstatic __global__ void mul_mat_q(\n const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,\n const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup,\n const int ncols_x, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,\n const int channel_ratio, const int nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,\n const int sample_ratio, const int nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,\n const int ncols_max) {\n\n // Skip unused template specializations for faster compilation:\n if (mmq_x > get_mmq_x_max_device() || mmq_x % mmq_get_granularity_device(mmq_x) != 0) {\n NO_DEVICE_CODE;\n return;\n }\n\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr int qk = ggml_cuda_type_traits::qk;\n constexpr int mmq_y = get_mmq_y_device();\n\n const int ntx = (ncols_max + mmq_x - 1) / mmq_x; // Number of tiles x\n const int nty = (nrows_x + mmq_y - 1) / mmq_y; // Number of tiles y\n\n // Initialize the ids for writing back data with just the index.\n // For regular matrix multiplications this is never changed.\n // For MoE the correct indices are loaded from ids_dst.\n extern __shared__ int ids_dst_shared[]; // Stored at beginning of shared memory.\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) {\n const int j = j0 + threadIdx.y*warp_size + threadIdx.x;\n\n if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) {\n break;\n }\n\n ids_dst_shared[j] = j;\n }\n __syncthreads();\n\n // On non-CDNA AMD or old CUDA the performance with stream-k was worse, use conventional tiling instead:\n#if (defined(GGML_USE_HIP) && !defined(CDNA)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA\n {\n const int wt = blockIdx.z / nchannels_y;\n const int zt = blockIdx.z - wt*nchannels_y;\n const int jt = blockIdx.y;\n const int it = blockIdx.x;\n\n // Defaults for regular matrix multiplication:\n int col_low = 0;\n int col_high = ncols_dst;\n int col_diff = ncols_dst;\n int offset_y = wt*stride_sample_y + zt*stride_channel_y;\n int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst;\n\n if (ids_dst) {\n col_low = expert_bounds[zt + 0];\n col_high = expert_bounds[zt + 1];\n col_diff = col_high - col_low;\n\n offset_y = 0;\n offset_dst = 0;\n\n if (jt*mmq_x >= col_diff) {\n return;\n }\n\n // __syncthreads(); // There is no previous tile that could cause a race condition.\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) {\n const int j = j0 + threadIdx.y*warp_size + threadIdx.x;\n\n if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) {\n break;\n }\n\n ids_dst_shared[j] = ids_dst[col_low + jt*mmq_x + j];\n }\n __syncthreads();\n }\n\n offset_y += (col_low + jt*mmq_x)*(sizeof(block_q8_1_mmq)/sizeof(int));\n offset_dst += it*mmq_y;\n\n const int tile_x_max_i = nrows_x - it*mmq_y - 1;\n const int tile_y_max_j = col_diff - jt*mmq_x - 1;\n\n const int offset_x = (wt/sample_ratio)*stride_sample_x + (zt/channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;\n\n constexpr bool fixup = false;\n mul_mat_q_process_tile\n (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst,\n tile_x_max_i, tile_y_max_j, 0, ncols_x/qk);\n return;\n }\n#endif // (defined(GGML_USE_HIP) && !defined(CDNA3)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA\n\n constexpr int ITER_K = get_iter_k(type);\n\n const int64_t blocks_per_ne00 = ncols_x / qk;\n constexpr int blocks_per_iter = ITER_K / qk;\n\n // kbc == k block continuous, current index in continuous ijk space.\n int64_t kbc = (int64_t) blockIdx.x *nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x;\n int64_t kbc_stop = (int64_t)(blockIdx.x + 1)*nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x;\n\n kbc -= (kbc % blocks_per_ne00) % blocks_per_iter;\n kbc_stop -= (kbc_stop % blocks_per_ne00) % blocks_per_iter;\n\n // kb0 == k index when doing the matrix multiplication for an output tile.\n int kb0_start = kbc % blocks_per_ne00;\n int kb0_stop = min(blocks_per_ne00, kb0_start + kbc_stop - kbc);\n while (kbc < kbc_stop && kb0_stop == blocks_per_ne00) {\n int tmp = kbc;\n const int it = tmp / (nsamples_y*nchannels_y*ntx*blocks_per_ne00);\n tmp -= it * (nsamples_y*nchannels_y*ntx*blocks_per_ne00);\n const int wt = tmp / (nchannels_y*ntx*blocks_per_ne00);\n tmp -= wt * (nchannels_y*ntx*blocks_per_ne00);\n const int zt = tmp / (ntx*blocks_per_ne00);\n tmp -= zt * (ntx*blocks_per_ne00);\n const int jt = tmp / blocks_per_ne00;\n\n // Defaults for regular matrix multiplication:\n int col_low = 0;\n int col_high = ncols_dst;\n int col_diff = ncols_dst;\n int offset_y = wt*stride_sample_y + zt*stride_channel_y;\n int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst;\n\n if (ids_dst) {\n col_low = expert_bounds[zt + 0];\n col_high = expert_bounds[zt + 1];\n col_diff = col_high - col_low;\n\n offset_y = 0;\n offset_dst = 0;\n\n if (jt*mmq_x >= col_diff) {\n kbc += blocks_per_ne00;\n kbc -= kbc % blocks_per_ne00;\n\n kb0_start = 0;\n kb0_stop = min(blocks_per_ne00, kbc_stop - kbc);\n\n continue;\n }\n\n __syncthreads();\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) {\n const int j = j0 + threadIdx.y*warp_size + threadIdx.x;\n\n if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) {\n break;\n }\n\n ids_dst_shared[j] = ids_dst[col_low + jt*mmq_x + j];\n }\n __syncthreads();\n }\n\n offset_y += (col_low + jt * mmq_x) * (sizeof(block_q8_1_mmq) / sizeof(int));\n offset_dst += it*mmq_y;\n\n const int tile_x_max_i = nrows_x - it*mmq_y - 1;\n const int tile_y_max_j = col_diff - jt*mmq_x - 1;\n\n const int offset_x = (wt/sample_ratio)*stride_sample_x + (zt/channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;\n\n constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.\n mul_mat_q_process_tile\n (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst,\n tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);\n\n kbc += blocks_per_ne00;\n kbc -= kbc % blocks_per_ne00;\n\n kb0_start = 0;\n kb0_stop = min(blocks_per_ne00, kbc_stop - kbc);\n }\n\n if (kbc >= kbc_stop) {\n return;\n }\n\n int tmp = kbc;\n const int it = tmp / (nsamples_y*nchannels_y*ntx*blocks_per_ne00);\n tmp -= it * (nsamples_y*nchannels_y*ntx*blocks_per_ne00);\n const int wt = tmp / (nchannels_y*ntx*blocks_per_ne00);\n tmp -= wt * (nchannels_y*ntx*blocks_per_ne00);\n const int zt = tmp / (ntx*blocks_per_ne00);\n tmp -= zt * (ntx*blocks_per_ne00);\n const int jt = tmp / blocks_per_ne00;\n\n // Defaults for regular matrix multiplication:\n int col_low = 0;\n int col_high = ncols_dst;\n int col_diff = ncols_dst;\n int offset_y = wt*stride_sample_y + zt*stride_channel_y;\n int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst;\n\n if (ids_dst) {\n col_low = expert_bounds[zt + 0];\n col_high = expert_bounds[zt + 1];\n col_diff = col_high - col_low;\n\n offset_y = 0;\n offset_dst = 0;\n\n if (jt*mmq_x >= col_diff) {\n return;\n }\n\n // The memory layout for the fixup buffer is always contiguous, therefore reset ids:\n __syncthreads();\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) {\n const int j = j0 + threadIdx.y*warp_size + threadIdx.x;\n\n if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) {\n break;\n }\n\n ids_dst_shared[j] = j;\n }\n __syncthreads();\n }\n\n offset_y += (col_low + jt * mmq_x) * (sizeof(block_q8_1_mmq) / sizeof(int));\n offset_dst += it*mmq_y;\n\n const int tile_x_max_i = nrows_x - it*mmq_y - 1;\n const int tile_y_max_j = col_diff - jt*mmq_x - 1;\n\n const int offset_x = (wt/sample_ratio)*stride_sample_x + (zt/channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;\n\n constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.\n mul_mat_q_process_tile\n (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst,\n tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);\n}\n\ntemplate \nstatic __global__ void mul_mat_q_stream_k_fixup(const int32_t * ids_dst,\n const int32_t * expert_bounds,\n float * __restrict__ dst,\n const float * __restrict__ tmp_last_tile,\n const int ncols_x,\n const int nrows_x,\n const int ncols_dst,\n const size_t stride_col_dst,\n const int nchannels_y,\n const size_t stride_channel_dst,\n const int nsamples_y,\n const size_t stride_sample_dst,\n const int ncols_max) {\n constexpr int mmq_y = get_mmq_y_device();\n constexpr int qk = ggml_cuda_type_traits::qk;\n constexpr int ITER_K = get_iter_k(type);\n\n constexpr int blocks_per_iter = ITER_K / qk;\n const int64_t blocks_per_ne00 = ncols_x / qk;\n\n constexpr int nwarps = mmq_get_nwarps_device();\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n float sum[mmq_x*mmq_y / (nwarps*warp_size)] = {0.0f};\n\n const int ntx = (ncols_max + mmq_x - 1) / mmq_x;\n const int nty = (nrows_x + mmq_y - 1) / mmq_y;\n\n const int bidx0 = blockIdx.x;\n\n // kbc == k block continuous, current index in continuous ijk space.\n int64_t kbc0 = (int64_t) bidx0 *nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x;\n int64_t kbc0_stop = (int64_t)(bidx0 + 1)*nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x;\n\n kbc0 -= (kbc0 % blocks_per_ne00) % blocks_per_iter;\n kbc0_stop -= (kbc0_stop % blocks_per_ne00) % blocks_per_iter;\n\n const bool did_not_have_any_data = kbc0 == kbc0_stop;\n const bool wrote_beginning_of_tile = kbc0 % blocks_per_ne00 == 0;\n const bool did_not_write_last = kbc0/blocks_per_ne00 == kbc0_stop/blocks_per_ne00 && kbc0_stop % blocks_per_ne00 != 0;\n if (did_not_have_any_data || wrote_beginning_of_tile || did_not_write_last) {\n return;\n }\n\n bool any_fixup = false;\n\n // Iterate over previous blocks and sum up partial sums written to fixup buffer.\n // All CUDA blocks that get here must have a previous block that needs a fixup.\n int64_t bidx = bidx0 - 1;\n int64_t kbc_stop = kbc0;\n while(true) {\n int64_t kbc = bidx*nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x;\n kbc -= (kbc % blocks_per_ne00) % blocks_per_iter;\n\n if (kbc == kbc_stop) { // Did not have any data.\n bidx--;\n kbc_stop = kbc;\n continue;\n }\n\n any_fixup = true;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n sum[(j0/nwarps) * (mmq_y/warp_size) + i0/warp_size] += tmp_last_tile[bidx*(mmq_x*mmq_y) + j*mmq_y + i];\n }\n }\n\n // If this block started in a previous tile we are done and don't need to combine additional partial results.\n if (kbc % blocks_per_ne00 == 0 || kbc/blocks_per_ne00 < kbc0/blocks_per_ne00) {\n break;\n }\n bidx--;\n kbc_stop = kbc;\n }\n\n if (!any_fixup) {\n return;\n }\n\n int tmp = kbc0;\n const int it = tmp / (nsamples_y*nchannels_y*ntx*blocks_per_ne00);\n tmp -= it * (nsamples_y*nchannels_y*ntx*blocks_per_ne00);\n const int wt = tmp / (nchannels_y*ntx*blocks_per_ne00);\n tmp -= wt * (nchannels_y*ntx*blocks_per_ne00);\n const int zt = tmp / (ntx*blocks_per_ne00);\n tmp -= zt * (ntx*blocks_per_ne00);\n const int jt = tmp / blocks_per_ne00;\n\n if (!ids_dst) {\n const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst + it*mmq_y;\n dst += offset_dst;\n\n const int i_max = nrows_x - it*mmq_y - 1;\n const int j_max = ncols_dst - jt*mmq_x - 1;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (j > j_max) {\n return;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n if (need_check && i > i_max) {\n continue;\n }\n\n dst[j*stride_col_dst + i] += sum[(j0/nwarps) * (mmq_y/warp_size) + i0/warp_size];\n }\n }\n return;\n }\n\n __shared__ int ids_dst_shared[mmq_x];\n const int col_low = expert_bounds[zt + 0];\n const int col_high = expert_bounds[zt + 1];\n const int col_diff = col_high - col_low;\n\n for (int j = threadIdx.y*warp_size + threadIdx.x; j < mmq_x; j += nwarps*warp_size) {\n ids_dst_shared[j] = ids_dst[col_low + jt*mmq_x + j];\n }\n __syncthreads();\n\n const int offset_dst = it*mmq_y;\n dst += offset_dst;\n\n const int i_max = nrows_x - it*mmq_y - 1;\n const int j_max = col_diff - jt*mmq_x - 1;\n\n#pragma unroll\n for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {\n const int j = j0 + threadIdx.y;\n\n if (j > j_max) {\n return;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {\n const int i = i0 + threadIdx.x;\n\n if (need_check && i > i_max) {\n continue;\n }\n\n dst[ids_dst_shared[j]*stride_col_dst + i] += sum[(j0/nwarps) * (mmq_y/warp_size) + i0/warp_size];\n }\n }\n}\n\nstruct mmq_args {\n const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst;\n int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst;\n int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst;\n int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst;\n bool use_stream_k; int64_t ncols_max;\n};\n\ntemplate\nstatic size_t mmq_get_nbytes_shared(const int mmq_x, const int mmq_y, const int cc, const int warp_size, const int nwarps) {\n const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, mmq_y);\n const int mmq_tile_x_k = mmq_get_mma_tile_x_k(type);\n const size_t nbs_ids = mmq_x*sizeof(int);\n const size_t nbs_x = (turing_mma_available(cc) || amd_mfma_available(cc) || amd_wmma_available(cc)) ? mmq_y*mmq_tile_x_k*sizeof(int) : txs.qs*sizeof(int) + txs.dm*sizeof(half2) + txs.sc*sizeof(int);\n const size_t nbs_y = mmq_x * (sizeof(block_q8_1_mmq));\n return nbs_ids + nbs_x + GGML_PAD(nbs_y, nwarps*warp_size*sizeof(int));\n}\n\ntemplate \nstatic void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {\n const int id = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[id].cc;\n const int nsm = ggml_cuda_info().devices[id].nsm;\n const int warp_size = ggml_cuda_info().devices[id].warp_size;\n const int nwarps = mmq_get_nwarps_host(cc, warp_size);\n const int mmq_y = get_mmq_y_host(cc);\n\n const dim3 block_dims(warp_size, nwarps, 1);\n\n const int nbytes_shared = mmq_get_nbytes_shared(mmq_x, mmq_y, cc, warp_size, nwarps);\n\n CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared);\n CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared);\n\n const int nty = (args.nrows_x + mmq_y - 1) / mmq_y;\n const int ntx = (args.ncols_max + mmq_x - 1) / mmq_x;\n const int ntzw = args.nchannels_y * args.nsamples_y;\n const dim3 block_nums_xy_tiling(nty, ntx, ntzw);\n\n GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);\n GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0);\n const int channel_ratio = args.nchannels_y / args.nchannels_x;\n const int sample_ratio = args.nsamples_y / args.nsamples_x;\n\n if (!args.use_stream_k) {\n if (args.nrows_x % mmq_y == 0) {\n constexpr bool need_check = false;\n mul_mat_q<<>>\n (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr,\n args.ncols_x, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,\n channel_ratio, args.nchannels_y, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,\n sample_ratio, args.nsamples_y, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,\n args.ncols_max);\n } else {\n constexpr bool need_check = true;\n mul_mat_q<<>>\n (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr,\n args.ncols_x, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,\n channel_ratio, args.nchannels_y, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,\n sample_ratio, args.nsamples_y, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,\n args.ncols_max);\n }\n return;\n }\n\n const dim3 block_nums_stream_k(nsm, 1, 1);\n const bool fixup_needed = ntx*nty*ntzw % nsm != 0;\n\n ggml_cuda_pool & pool = ctx.pool(id);\n ggml_cuda_pool_alloc tmp_fixup(pool);\n if (fixup_needed) {\n tmp_fixup.alloc(block_nums_stream_k.x * mmq_x*mmq_y);\n }\n\n if (args.nrows_x % mmq_y == 0) {\n constexpr bool need_check = false;\n mul_mat_q<<>>\n (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr,\n args.ncols_x, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,\n channel_ratio, args.nchannels_y, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,\n sample_ratio, args.nsamples_y, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,\n args.ncols_max);\n\n if (!fixup_needed) {\n return;\n }\n\n mul_mat_q_stream_k_fixup<<>>\n (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.ncols_x, args.nrows_x, args.ncols_dst,\n args.nrows_dst, args.nchannels_y, args.stride_channel_dst, args.nsamples_y, args.stride_sample_dst,\n args.ncols_max);\n } else {\n constexpr bool need_check = true;\n mul_mat_q<<>>\n (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr,\n args.ncols_x, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,\n channel_ratio, args.nchannels_y, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,\n sample_ratio, args.nsamples_y, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,\n args.ncols_max);\n\n if (!fixup_needed) {\n return;\n }\n\n mul_mat_q_stream_k_fixup<<>>\n (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.ncols_x, args.nrows_x, args.ncols_dst,\n args.nrows_dst, args.nchannels_y, args.stride_channel_dst, args.nsamples_y, args.stride_sample_dst,\n args.ncols_max);\n }\n}\n\ntemplate \nvoid mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {\n const int id = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[id].cc;\n const size_t smpbo = ggml_cuda_info().devices[id].smpbo;\n const int warp_size = ggml_cuda_info().devices[id].warp_size;\n const int nwarps = mmq_get_nwarps_host(cc, warp_size);\n\n const int mmq_x_max = get_mmq_x_max_host(cc);\n const int mmq_y = get_mmq_y_host(cc);\n\n int mmq_x_best = 0;\n int ntiles_x_best = INT_MAX;\n\n for (int mmq_x = 8; mmq_x <= mmq_x_max && ntiles_x_best > 1; mmq_x += 8) {\n const int granularity = mmq_get_granularity_host(mmq_x, cc);\n\n if (mmq_x % granularity != 0 || mmq_get_nbytes_shared(mmq_x, mmq_y, cc, warp_size, nwarps) > smpbo) {\n continue;\n }\n\n const int ntiles_x = (args.ncols_max + mmq_x - 1) / mmq_x;\n\n if (ntiles_x < ntiles_x_best) {\n mmq_x_best = mmq_x;\n ntiles_x_best = ntiles_x;\n }\n }\n\n switch (mmq_x_best) {\n case 8:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 16:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 24:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 32:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 40:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 48:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 56:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 64:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 72:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 80:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 88:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 96:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 104:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 112:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 120:\n launch_mul_mat_q(ctx, args, stream);\n break;\n case 128:\n launch_mul_mat_q(ctx, args, stream);\n break;\n default:\n fprintf(stderr, \"mmq_x_best=%d\\n\", mmq_x_best);\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\n#define DECL_MMQ_CASE(type) \\\n template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \\\n\nextern DECL_MMQ_CASE(GGML_TYPE_Q4_0);\nextern DECL_MMQ_CASE(GGML_TYPE_Q4_1);\nextern DECL_MMQ_CASE(GGML_TYPE_Q5_0);\nextern DECL_MMQ_CASE(GGML_TYPE_Q5_1);\nextern DECL_MMQ_CASE(GGML_TYPE_Q8_0);\nextern DECL_MMQ_CASE(GGML_TYPE_MXFP4);\nextern DECL_MMQ_CASE(GGML_TYPE_Q2_K);\nextern DECL_MMQ_CASE(GGML_TYPE_Q3_K);\nextern DECL_MMQ_CASE(GGML_TYPE_Q4_K);\nextern DECL_MMQ_CASE(GGML_TYPE_Q5_K);\nextern DECL_MMQ_CASE(GGML_TYPE_Q6_K);\nextern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);\nextern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);\nextern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);\nextern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);\nextern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);\nextern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);\nextern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);\nextern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);\n\n// -------------------------------------------------------------------------------------------------------------------------\n\nvoid ggml_cuda_mul_mat_q(\n ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);\n\nvoid ggml_cuda_op_mul_mat_q(\n ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,\n const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,\n const int64_t src1_padded_row_size, cudaStream_t stream);\n\nbool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);\n" }, { "path": "src/ggml-cuda/mmvf.cu", "content": "#include \"ggml.h\"\n#include \"common.cuh\"\n#include \"unary.cuh\"\n#include \"mmvf.cuh\"\n#include \"convert.cuh\"\n\ntemplate \nstatic __global__ void mul_mat_vec_f(\n const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, const ggml_cuda_mm_fusion_args_device fusion, float * __restrict__ dst,\n const int ncols2, const uint3 nchannels_y, const int stride_row, const int stride_col_y2, const int stride_col_dst,\n const uint3 channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,\n const uint3 sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,\n const int ids_stride) {\n const int row = blockIdx.x;\n // for MUL_MAT_ID - blockIdx.y = n_expert_used, blockIdx.z = ncols_dst (tokens)\n const int channel_dst = blockIdx.y;\n const int tid = threadIdx.x;\n\n int token_idx;\n int channel_x;\n int channel_y;\n int sample_dst;\n\n if constexpr (is_multi_token_id) {\n // Multi-token MUL_MAT_ID path, adding these in the normal path causes a perf regression for n_tokens=1 case\n token_idx = blockIdx.z;\n channel_x = ids[channel_dst + token_idx * ids_stride];\n channel_y = fastmodulo(channel_dst, nchannels_y);\n sample_dst = 0;\n } else {\n token_idx = ids ? blockIdx.z : 0;\n channel_x = ids ? ids[blockIdx.y + token_idx * ids_stride] : fastdiv((uint32_t) channel_dst, channel_ratio);\n channel_y = ids ? fastmodulo(blockIdx.y, nchannels_y) : channel_dst;\n sample_dst = ids ? 0 : blockIdx.z;\n }\n\n const int sample_x = fastdiv((uint32_t) sample_dst, sample_ratio);\n const int sample_y = sample_dst;\n\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row*stride_row;\n y += int64_t(sample_y) *stride_sample_y + channel_y *stride_channel_y;\n dst += int64_t(sample_dst)*stride_sample_dst + channel_dst*stride_channel_dst;\n if constexpr (is_multi_token_id) {\n y += token_idx*stride_col_y2*2;\n dst += token_idx*stride_col_dst;\n }\n\n bool use_gate = false;\n bool use_bias = false;\n bool use_gate_bias = false;\n ggml_glu_op glu_op = ggml_glu_op::GGML_GLU_OP_SWIGLU;\n const T * gate_x = nullptr;\n const float * x_bias = nullptr;\n const float * gate_bias = nullptr;\n\n if constexpr (has_fusion) {\n use_gate = fusion.gate != nullptr;\n use_bias = fusion.x_bias != nullptr;\n use_gate_bias = fusion.gate_bias != nullptr;\n glu_op = fusion.glu_op;\n\n if (use_gate) {\n gate_x = static_cast(fusion.gate);\n }\n if (use_bias) {\n x_bias = static_cast(fusion.x_bias);\n }\n if (use_gate_bias) {\n gate_bias = static_cast(fusion.gate_bias);\n use_gate_bias = use_gate;\n } else {\n use_gate_bias = false;\n }\n }\n\n if (use_gate) {\n gate_x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row*stride_row;\n }\n\n const int channel_bias = ids ? channel_x : channel_dst;\n\n if constexpr (has_fusion) {\n if (use_bias) {\n x_bias += int64_t(sample_dst)*stride_sample_dst + channel_bias*stride_channel_dst;\n }\n if (use_gate_bias) {\n gate_bias += int64_t(sample_dst)*stride_sample_dst + channel_bias*stride_channel_dst;\n }\n }\n\n const float2 * y2 = (const float2 *) y;\n\n extern __shared__ char data_mmv[];\n float * buf_iw = (float *) data_mmv;\n float * buf_iw_gate = nullptr;\n if constexpr (has_fusion) {\n buf_iw_gate = (float *) (data_mmv + warp_size*sizeof(float));\n }\n\n if (block_size > warp_size) {\n if (tid < warp_size) {\n buf_iw[tid] = 0.0f;\n if constexpr (has_fusion) {\n if (use_gate) {\n buf_iw_gate[tid] = 0.0f;\n }\n }\n }\n __syncthreads();\n }\n\n float sumf[ncols_dst] = {0.0f};\n float sumf_gate[ncols_dst];\n if constexpr (has_fusion) {\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n sumf_gate[j] = 0.0f;\n }\n }\n\n if constexpr (std::is_same_v) {\n const float2 * x2 = (const float2 *) x;\n const float2 * gate_x2 = nullptr;\n if constexpr (has_fusion) {\n if (use_gate) {\n gate_x2 = (const float2 *) gate_x;\n }\n }\n\n for (int col2 = tid; col2 < ncols2; col2 += block_size) {\n const float2 tmpx = x2[col2];\n float2 tmpx_gate = make_float2(0.0f, 0.0f);\n if constexpr (has_fusion) {\n if (use_gate) {\n tmpx_gate = gate_x2[col2];\n }\n }\n\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n const float2 tmpy = y2[j*stride_col_y2 + col2];\n ggml_cuda_mad(sumf[j], tmpx.x, tmpy.x);\n ggml_cuda_mad(sumf[j], tmpx.y, tmpy.y);\n\n if constexpr (has_fusion) {\n if (use_gate) {\n ggml_cuda_mad(sumf_gate[j], tmpx_gate.x, tmpy.x);\n ggml_cuda_mad(sumf_gate[j], tmpx_gate.y, tmpy.y);\n }\n }\n }\n }\n } else if constexpr (std::is_same_v) {\n const half2 * x2 = (const half2 *) x;\n const half2 * gate_x2 = nullptr;\n if constexpr (has_fusion) {\n if (use_gate) {\n gate_x2 = (const half2 *) gate_x;\n }\n }\n\n if (std::is_same_v) {\n for (int col2 = tid; col2 < ncols2; col2 += block_size) {\n const float2 tmpx = __half22float2(x2[col2]);\n float2 tmpx_gate = make_float2(0.0f, 0.0f);\n if constexpr (has_fusion) {\n if (use_gate) {\n tmpx_gate = __half22float2(gate_x2[col2]);\n }\n }\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n const float2 tmpy = y2[j*stride_col_y2 + col2];\n ggml_cuda_mad(sumf[j], tmpx.x, tmpy.x);\n ggml_cuda_mad(sumf[j], tmpx.y, tmpy.y);\n\n if constexpr (has_fusion) {\n if (use_gate) {\n ggml_cuda_mad(sumf_gate[j], tmpx_gate.x, tmpy.x);\n ggml_cuda_mad(sumf_gate[j], tmpx_gate.y, tmpy.y);\n }\n }\n }\n }\n } else {\n#ifdef FP16_AVAILABLE\n half2 sumh2[ncols_dst] = {{0.0f, 0.0f}};\n half2 sumh2_gate[ncols_dst] = {{0.0f, 0.0f}};\n\n for (int col2 = tid; col2 < ncols2; col2 += block_size) {\n const half2 tmpx = x2[col2];\n half2 tmpx_gate = make_half2(0.0f, 0.0f);\n if constexpr (has_fusion) {\n if (use_gate) {\n tmpx_gate = gate_x2[col2];\n }\n }\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n const float2 tmpy = y2[j*stride_col_y2 + col2];\n sumh2[j] += tmpx * make_half2(tmpy.x, tmpy.y);\n\n if constexpr (has_fusion) {\n if (use_gate) {\n sumh2_gate[j] += tmpx_gate * make_half2(tmpy.x, tmpy.y);\n }\n }\n }\n }\n\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n sumf[j] = __low2float(sumh2[j]) + __high2float(sumh2[j]);\n }\n\n if constexpr (has_fusion) {\n if (use_gate) {\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n sumf_gate[j] = __low2float(sumh2_gate[j]) + __high2float(sumh2_gate[j]);\n }\n }\n }\n#else\n NO_DEVICE_CODE;\n#endif // FP16_AVAILABLE\n }\n } else if constexpr (std::is_same_v) {\n//TODO: add support for ggml_cuda_mad for hip_bfloat162\n#if defined(GGML_USE_HIP)\n const int * x2 = (const int *) x;\n const int * gate_x2 = nullptr;\n if constexpr (has_fusion) {\n if (use_gate) {\n gate_x2 = (const int *) gate_x;\n }\n }\n for (int col2 = tid; col2 < ncols2; col2 += block_size) {\n const int tmpx = x2[col2];\n int tmpx_gate = 0;\n if constexpr (has_fusion) {\n if (use_gate) {\n tmpx_gate = gate_x2[col2];\n }\n }\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n const float2 tmpy = y2[j*stride_col_y2 + col2];\n const float tmpx0 = ggml_cuda_cast(reinterpret_cast(&tmpx)[0]);\n const float tmpx1 = ggml_cuda_cast(reinterpret_cast(&tmpx)[1]);\n ggml_cuda_mad(sumf[j], tmpx0, tmpy.x);\n ggml_cuda_mad(sumf[j], tmpx1, tmpy.y);\n\n if constexpr (has_fusion) {\n if (use_gate) {\n const float tmpx0_gate = ggml_cuda_cast(reinterpret_cast(&tmpx_gate)[0]);\n const float tmpx1_gate = ggml_cuda_cast(reinterpret_cast(&tmpx_gate)[1]);\n ggml_cuda_mad(sumf_gate[j], tmpx0_gate, tmpy.x);\n ggml_cuda_mad(sumf_gate[j], tmpx1_gate, tmpy.y);\n }\n }\n }\n }\n#else\n const nv_bfloat162 * x2 = (const nv_bfloat162 *) x;\n const nv_bfloat162 * gate_x2 = nullptr;\n if constexpr (has_fusion) {\n if (use_gate) {\n gate_x2 = (const nv_bfloat162 *) gate_x;\n }\n }\n for (int col2 = tid; col2 < ncols2; col2 += block_size) {\n const nv_bfloat162 tmpx = x2[col2];\n nv_bfloat162 tmpx_gate;\n if constexpr (has_fusion) {\n if (use_gate) {\n tmpx_gate = gate_x2[col2];\n }\n }\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n const float2 tmpy = y2[j*stride_col_y2 + col2];\n ggml_cuda_mad(sumf[j], tmpx.x, tmpy.x);\n ggml_cuda_mad(sumf[j], tmpx.y, tmpy.y);\n\n if constexpr (has_fusion) {\n if (use_gate) {\n ggml_cuda_mad(sumf_gate[j], tmpx_gate.x, tmpy.x);\n ggml_cuda_mad(sumf_gate[j], tmpx_gate.y, tmpy.y);\n }\n }\n }\n }\n#endif\n } else {\n static_assert(std::is_same_v, \"unsupported type\");\n }\n\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n sumf[j] = warp_reduce_sum(sumf[j]);\n\n if constexpr (has_fusion) {\n if (use_gate) {\n sumf_gate[j] = warp_reduce_sum(sumf_gate[j]);\n }\n }\n\n if (block_size > warp_size) {\n buf_iw[tid/warp_size] = sumf[j];\n if constexpr (has_fusion) {\n if (use_gate) {\n buf_iw_gate[tid/warp_size] = sumf_gate[j];\n }\n }\n __syncthreads();\n if (tid < warp_size) {\n sumf[j] = buf_iw[tid];\n sumf[j] = warp_reduce_sum(sumf[j]);\n if constexpr (has_fusion) {\n if (use_gate) {\n sumf_gate[j] = buf_iw_gate[tid];\n sumf_gate[j] = warp_reduce_sum(sumf_gate[j]);\n }\n }\n }\n\n if (j < ncols_dst) {\n __syncthreads();\n }\n }\n }\n\n if (tid >= ncols_dst) {\n return;\n }\n\n float value = sumf[tid];\n\n if constexpr (has_fusion) {\n if (use_bias) {\n value += x_bias[tid*stride_col_dst + row];\n }\n\n if (use_gate) {\n float gate_value = sumf_gate[tid];\n if (use_gate_bias) {\n gate_value += gate_bias[tid*stride_col_dst + row];\n }\n switch (glu_op) {\n case GGML_GLU_OP_SWIGLU:\n value *= ggml_cuda_op_silu_single(gate_value);\n break;\n case GGML_GLU_OP_GEGLU:\n value *= ggml_cuda_op_gelu_single(gate_value);\n break;\n case GGML_GLU_OP_SWIGLU_OAI: {\n value = ggml_cuda_op_swiglu_oai_single(gate_value, value);\n break;\n }\n default:\n break;\n }\n }\n }\n\n dst[tid*stride_col_dst + row] = value;\n\n if constexpr (!has_fusion) {\n GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, gate_x, x_bias, gate_bias, sumf_gate);\n }\n}\n\ntemplate\nstatic void mul_mat_vec_f_switch_fusion(\n const T * x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,\n const int64_t ncols, const uint3 nchannels_y,\n const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,\n const uint3 channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,\n const uint3 sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,\n const dim3 & block_dims, const dim3 & block_nums, const int nbytes_shared, const int ids_stride, const cudaStream_t stream) {\n\n const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;\n if constexpr (ncols_dst == 1) {\n if (has_fusion) {\n mul_mat_vec_f<<>>\n (x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst,\n channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);\n return;\n }\n }\n\n GGML_ASSERT(!has_fusion && \"fusion only supported for ncols_dst=1\");\n\n mul_mat_vec_f<<>>\n (x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst,\n channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);\n\n}\n\ntemplate \nvoid launch_mul_mat_vec_f_cuda(\n const T * x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,\n const int64_t ncols, const int64_t nrows,\n const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,\n const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,\n const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\n const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,\n const int64_t nsamples_or_ntokens, const int64_t ids_stride, cudaStream_t stream) {\n GGML_ASSERT(ncols % 2 == 0);\n GGML_ASSERT(stride_row % 2 == 0);\n GGML_ASSERT(stride_col_y % 2 == 0);\n GGML_ASSERT(ids || nchannels_dst % nchannels_x == 0);\n GGML_ASSERT( nsamples_dst % nsamples_x == 0);\n const uint3 nchannels_y_fd = ids ? init_fastdiv_values(nchannels_y) : make_uint3(0, 0, 0);\n const uint3 channel_ratio_fd = ids ? make_uint3(0, 0, 0) : init_fastdiv_values(nchannels_dst / nchannels_x);\n const uint3 sample_ratio_fd = init_fastdiv_values(nsamples_dst / nsamples_x);\n\n const int device = ggml_cuda_get_device();\n const int warp_size = ggml_cuda_info().devices[device].warp_size;\n\n int64_t block_size_best = warp_size;\n int64_t niter_best = (ncols + 2*warp_size - 1) / (2*warp_size);\n int64_t max_block_size = 256;\n if(ggml_cuda_info().devices[device].cc > GGML_CUDA_CC_OFFSET_AMD && ggml_cuda_info().devices[device].cc < GGML_CUDA_CC_RDNA1) {\n max_block_size = 128;\n }\n for (int64_t block_size = 2*warp_size; block_size <= max_block_size; block_size += warp_size) {\n const int64_t niter = (ncols + 2*block_size - 1) / (2*block_size);\n if (niter < niter_best) {\n niter_best = niter;\n block_size_best = block_size;\n }\n }\n\n const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;\n\n const int nbytes_shared = warp_size*sizeof(float) + (has_fusion ? warp_size*sizeof(float) : 0);\n const dim3 block_nums(nrows, nchannels_dst, nsamples_or_ntokens);\n const dim3 block_dims(block_size_best, 1, 1);\n switch (block_size_best) {\n case 32: {\n mul_mat_vec_f_switch_fusion\n (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);\n } break;\n case 64: {\n mul_mat_vec_f_switch_fusion\n (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);\n } break;\n case 96: {\n mul_mat_vec_f_switch_fusion\n (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);\n } break;\n case 128: {\n mul_mat_vec_f_switch_fusion\n (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);\n } break;\n case 160: {\n mul_mat_vec_f_switch_fusion\n (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);\n } break;\n case 192: {\n mul_mat_vec_f_switch_fusion\n (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);\n } break;\n case 224: {\n mul_mat_vec_f_switch_fusion\n (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);\n } break;\n case 256: {\n mul_mat_vec_f_switch_fusion\n (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);\n } break;\n default: {\n GGML_ABORT(\"fatal error\");\n } break;\n }\n}\n\ntemplate \nstatic void mul_mat_vec_f_cuda_switch_ncols_dst(\n const T * x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,\n const int64_t ncols, const int64_t nrows, const int64_t ncols_dst,\n const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,\n const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,\n const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\n const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,\n const int64_t ids_stride, cudaStream_t stream) {\n\n const bool has_ids = ids != nullptr;\n\n if (has_ids && ncols_dst > 1) {\n // Multi-token MUL_MAT_ID path only - single-token goes through regular path below\n constexpr int c_ncols_dst = 1;\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n ncols_dst, ids_stride, stream);\n return;\n }\n\n if (has_ids) {\n // Single-token MUL_MAT_ID path\n constexpr int c_ncols_dst = 1;\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n ncols_dst, ids_stride, stream);\n return;\n }\n\n switch (ncols_dst) {\n case 1:\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n nsamples_dst, ids_stride, stream);\n break;\n case 2:\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n nsamples_dst, ids_stride, stream);\n break;\n case 3:\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n nsamples_dst, ids_stride, stream);\n break;\n case 4:\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n nsamples_dst, ids_stride, stream);\n break;\n case 5:\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n nsamples_dst, ids_stride, stream);\n break;\n case 6:\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n nsamples_dst, ids_stride, stream);\n break;\n case 7:\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n nsamples_dst, ids_stride, stream);\n break;\n case 8:\n launch_mul_mat_vec_f_cuda\n (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,\n nsamples_dst, ids_stride, stream);\n break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\ntemplate\nstatic void mul_mat_vec_f_cuda(\n const T * x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,\n const int64_t ncols, const int64_t nrows, const int64_t ncols_dst,\n const int64_t stride_row, const int64_t stride_col_y, const int stride_col_dst,\n const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,\n const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\n const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,\n const int64_t ids_stride, enum ggml_prec prec, cudaStream_t stream) {\n\n if constexpr(std::is_same_v) {\n if (prec == GGML_PREC_DEFAULT) {\n mul_mat_vec_f_cuda_switch_ncols_dst\n (x, y, ids, fusion, dst, ncols, nrows, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n return;\n }\n }\n mul_mat_vec_f_cuda_switch_ncols_dst\n (x, y, ids, fusion, dst, ncols, nrows, ncols_dst, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,\n stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n}\n\nvoid ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,\n const ggml_cuda_mm_fusion_args_host * fusion) {\n GGML_ASSERT( src1->type == GGML_TYPE_F32);\n GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n const size_t ts_src0 = ggml_type_size(src0->type);\n const size_t ts_src1 = ggml_type_size(src1->type);\n const size_t ts_dst = ggml_type_size(dst->type);\n\n GGML_ASSERT(!ids || ne12 <= MMVF_MAX_BATCH_SIZE);\n GGML_ASSERT(ne13 == ne3);\n\n GGML_ASSERT( nb00 == ts_src0);\n GGML_ASSERT( nb10 == ts_src1);\n GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));\n GGML_ASSERT( nb0 == ts_dst);\n\n const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32;\n\n const float * src1_d = (const float *) src1->data;\n const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr;\n float * dst_d = (float *) dst->data;\n\n ggml_cuda_mm_fusion_args_device fusion_local{};\n\n if (fusion) {\n GGML_ASSERT( !ids || dst->ne[2] == 1);\n GGML_ASSERT( ids || dst->ne[1] == 1);\n if (fusion->x_bias) {\n GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32);\n GGML_ASSERT(fusion->x_bias->ne[0] == dst->ne[0]);\n GGML_ASSERT(!ids || fusion->x_bias->ne[1] == src0->ne[2]);\n fusion_local.x_bias = fusion->x_bias->data;\n }\n if (fusion->gate) {\n GGML_ASSERT(fusion->gate->type == src0->type && ggml_are_same_stride(fusion->gate, src0));\n fusion_local.gate = fusion->gate->data;\n }\n if (fusion->gate_bias) {\n GGML_ASSERT(fusion->gate_bias->type == GGML_TYPE_F32);\n GGML_ASSERT(fusion->gate_bias->ne[0] == dst->ne[0]);\n GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);\n fusion_local.gate_bias = fusion->gate_bias->data;\n }\n fusion_local.glu_op = fusion->glu_op;\n }\n\n const int64_t s01 = src0->nb[1] / ts_src0;\n const int64_t s11 = src1->nb[1] / ts_src1;\n const int64_t s1 = dst->nb[1] / ts_dst;\n const int64_t s02 = src0->nb[2] / ts_src0;\n const int64_t s12 = src1->nb[2] / ts_src1;\n const int64_t s2 = dst->nb[2] / ts_dst;\n const int64_t s03 = src0->nb[3] / ts_src0;\n const int64_t s13 = src1->nb[3] / ts_src1;\n const int64_t s3 = dst->nb[3] / ts_dst;\n\n // For MUL_MAT_ID the memory layout is different than for MUL_MAT:\n const int64_t ncols_dst = ids ? ne2 : ne1;\n const int64_t nchannels_y = ids ? ne11 : ne12;\n const int64_t nchannels_dst = ids ? ne1 : ne2;\n const int64_t stride_col_dst = ids ? s2 : s1;\n const int64_t stride_col_y = ids ? s12 : s11;\n const int64_t stride_channel_dst = ids ? s1 : s2;\n const int64_t stride_channel_y = ids ? s11 : s12;\n\n const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;\n\n switch (src0->type) {\n case GGML_TYPE_F32: {\n const float * src0_d = (const float *) src0->data;\n mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst,\n ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,\n ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream());\n } break;\n case GGML_TYPE_F16: {\n const half * src0_d = (const half *) src0->data;\n mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst,\n ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,\n ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream());\n } break;\n case GGML_TYPE_BF16: {\n const nv_bfloat16 * src0_d = (const nv_bfloat16 *) src0->data;\n mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst,\n ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,\n ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream());\n } break;\n default:\n GGML_ABORT(\"unsupported type: %s\", ggml_type_name(src0->type));\n }\n}\n\nvoid ggml_cuda_op_mul_mat_vec_f(\n ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,\n const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,\n const int64_t src1_padded_row_size, cudaStream_t stream) {\n\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n const int64_t ne00 = src0->ne[0];\n const int64_t ne10 = src1->ne[0];\n const int64_t ne0 = dst->ne[0];\n const int64_t row_diff = row_high - row_low;\n\n const int id = ggml_cuda_get_device();\n const int cc = ggml_cuda_info().devices[id].cc;\n const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32;\n\n // ggml_cuda_op provides single, contiguous matrices\n const int64_t stride_row = ne00;\n const int64_t stride_col_y = ne10;\n const int64_t stride_col_dst = id == ctx.device ? ne0 : row_diff; // main device has larger memory buffer\n const int64_t nchannels_x = 1;\n const int64_t nchannels_y = 1;\n const int64_t nchannels_dst = 1;\n const int64_t stride_channel_x = 0;\n const int64_t stride_channel_y = 0;\n const int64_t stride_channel_dst = 0;\n const int64_t nsamples_x = 1;\n const int64_t nsamples_dst = 1;\n const int64_t stride_sample_x = 0;\n const int64_t stride_sample_y = 0;\n const int64_t stride_sample_dst = 0;\n\n ggml_cuda_mm_fusion_args_device empty{};\n switch (src0->type) {\n case GGML_TYPE_F32: {\n const float * src0_d = (const float *) src0_dd_i;\n mul_mat_vec_f_cuda(src0_d, src1_ddf_i, nullptr, empty, dst_dd_i, ne00, row_diff, src1_ncols, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, stream);\n } break;\n case GGML_TYPE_F16: {\n const half * src0_d = (const half *) src0_dd_i;\n mul_mat_vec_f_cuda(src0_d, src1_ddf_i, nullptr, empty, dst_dd_i, ne00, row_diff, src1_ncols, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, stream);\n } break;\n case GGML_TYPE_BF16: {\n const nv_bfloat16 * src0_d = (const nv_bfloat16 *) src0_dd_i;\n mul_mat_vec_f_cuda(src0_d, src1_ddf_i, nullptr, empty, dst_dd_i, ne00, row_diff, src1_ncols, stride_row, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, stream);\n } break;\n default:\n GGML_ABORT(\"unsupported type: %s\", ggml_type_name(src0->type));\n }\n\n GGML_UNUSED_VARS(ctx, src1, dst, src1_ddq_i, src1_ncols, src1_padded_row_size);\n}\n\nbool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) {\n if (src0_ne[0] % 2 != 0) {\n return false;\n }\n\n const size_t ts = ggml_type_size(type);\n if (src0_nb[0] != ts) {\n return false;\n }\n\n // Pointers not aligned to the size of half2/nv_bfloat162/float2 would result in a crash:\n for (size_t i = 1; i < GGML_MAX_DIMS; ++i) {\n if (src0_nb[i] % (2*ts) != 0) {\n return false;\n }\n }\n\n switch (type) {\n case GGML_TYPE_F32:\n if (GGML_CUDA_CC_IS_NVIDIA(cc)) {\n if (ampere_mma_available(cc)) {\n return ne11 <= 3;\n }\n if (cc >= GGML_CUDA_CC_TURING) {\n return ne11 <= 4;\n }\n return ne11 <= 3;\n } else if (GGML_CUDA_CC_IS_AMD(cc)) {\n if (fp32_mma_hardware_available(cc)) {\n return ne11 <= 3;\n }\n return ne11 <= 8;\n }\n return ne11 <= 8;\n case GGML_TYPE_F16:\n if (GGML_CUDA_CC_IS_NVIDIA(cc)) {\n const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1);\n if (ampere_mma_available(cc)) {\n return src0_small && ne11 == 1;\n }\n if (cc >= GGML_CUDA_CC_ADA_LOVELACE) {\n return src0_small && ne11 <= 4;\n }\n if (fp16_mma_hardware_available(cc)) {\n return src0_small && ne11 <= 3;\n }\n return ne11 <= 8;\n } else if (GGML_CUDA_CC_IS_AMD(cc)) {\n if (fp16_mma_hardware_available(cc)) {\n if (GGML_CUDA_CC_IS_RDNA3(cc)) {\n return ne11 <= 3;\n }\n if (GGML_CUDA_CC_IS_RDNA4(cc)) {\n return ne11 <= 5;\n }\n return ne11 <= 2;\n }\n return ne11 <= 8;\n }\n return ne11 <= 8;\n case GGML_TYPE_BF16:\n if (GGML_CUDA_CC_IS_NVIDIA(cc)) {\n const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1);\n if (ampere_mma_available(cc)) {\n return src0_small && ne11 == 1;\n }\n if (cc >= GGML_CUDA_CC_ADA_LOVELACE) {\n return src0_small && ne11 <= 4;\n }\n if (bf16_mma_hardware_available(cc)) {\n return src0_small && ne11 <= 3;\n }\n return ne11 <= 8;\n } else if (GGML_CUDA_CC_IS_AMD(cc)) {\n if (bf16_mma_hardware_available(cc)) {\n return ne11 <= 3;\n }\n return ne11 <= 8;\n }\n return ne11 <= 8;\n default:\n return false;\n }\n}\n" }, { "path": "src/ggml-cuda/mmvf.cuh", "content": "#include \"common.cuh\"\n\n#define MMVF_MAX_BATCH_SIZE 8 // Max. batch size for which to use MMVF kernels.\n\nvoid ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,\n const ggml_cuda_mm_fusion_args_host * fusion = nullptr);\n\nvoid ggml_cuda_op_mul_mat_vec_f(\n ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,\n const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,\n const int64_t src1_padded_row_size, cudaStream_t stream);\n\nbool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11);\n" }, { "path": "src/ggml-cuda/mmvq.cu", "content": "#include \"mmvq.cuh\"\n#include \"quantize.cuh\"\n#include \"unary.cuh\"\n#include \"vecdotq.cuh\"\n\n#include \n\ntypedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs);\n\nstatic constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) {\n switch (type) {\n case GGML_TYPE_Q4_0: return vec_dot_q4_0_q8_1;\n case GGML_TYPE_Q4_1: return vec_dot_q4_1_q8_1;\n case GGML_TYPE_Q5_0: return vec_dot_q5_0_q8_1;\n case GGML_TYPE_Q5_1: return vec_dot_q5_1_q8_1;\n case GGML_TYPE_Q8_0: return vec_dot_q8_0_q8_1;\n case GGML_TYPE_MXFP4: return vec_dot_mxfp4_q8_1;\n case GGML_TYPE_Q2_K: return vec_dot_q2_K_q8_1;\n case GGML_TYPE_Q3_K: return vec_dot_q3_K_q8_1;\n case GGML_TYPE_Q4_K: return vec_dot_q4_K_q8_1;\n case GGML_TYPE_Q5_K: return vec_dot_q5_K_q8_1;\n case GGML_TYPE_Q6_K: return vec_dot_q6_K_q8_1;\n case GGML_TYPE_IQ2_XXS: return vec_dot_iq2_xxs_q8_1;\n case GGML_TYPE_IQ2_XS: return vec_dot_iq2_xs_q8_1;\n case GGML_TYPE_IQ2_S: return vec_dot_iq2_s_q8_1;\n case GGML_TYPE_IQ3_XXS: return vec_dot_iq3_xxs_q8_1;\n case GGML_TYPE_IQ1_S: return vec_dot_iq1_s_q8_1;\n case GGML_TYPE_IQ1_M: return vec_dot_iq1_m_q8_1;\n case GGML_TYPE_IQ4_NL: return vec_dot_iq4_nl_q8_1;\n case GGML_TYPE_IQ4_XS: return vec_dot_iq4_xs_q8_1;\n case GGML_TYPE_IQ3_S: return vec_dot_iq3_s_q8_1;\n default: return nullptr;\n }\n}\n\nstatic constexpr __device__ int get_vdr_mmvq(ggml_type type) {\n switch (type) {\n case GGML_TYPE_Q4_0: return VDR_Q4_0_Q8_1_MMVQ;\n case GGML_TYPE_Q4_1: return VDR_Q4_1_Q8_1_MMVQ;\n case GGML_TYPE_Q5_0: return VDR_Q5_0_Q8_1_MMVQ;\n case GGML_TYPE_Q5_1: return VDR_Q5_1_Q8_1_MMVQ;\n case GGML_TYPE_Q8_0: return VDR_Q8_0_Q8_1_MMVQ;\n case GGML_TYPE_MXFP4: return VDR_MXFP4_Q8_1_MMVQ;\n case GGML_TYPE_Q2_K: return VDR_Q2_K_Q8_1_MMVQ;\n case GGML_TYPE_Q3_K: return VDR_Q3_K_Q8_1_MMVQ;\n case GGML_TYPE_Q4_K: return VDR_Q4_K_Q8_1_MMVQ;\n case GGML_TYPE_Q5_K: return VDR_Q5_K_Q8_1_MMVQ;\n case GGML_TYPE_Q6_K: return VDR_Q6_K_Q8_1_MMVQ;\n case GGML_TYPE_IQ2_XXS: return VDR_IQ2_XXS_Q8_1_MMVQ;\n case GGML_TYPE_IQ2_XS: return VDR_IQ2_XS_Q8_1_MMVQ;\n case GGML_TYPE_IQ2_S: return VDR_IQ2_S_Q8_1_MMVQ;\n case GGML_TYPE_IQ3_XXS: return VDR_IQ3_XXS_Q8_1_MMVQ;\n case GGML_TYPE_IQ3_S: return VDR_IQ3_S_Q8_1_MMVQ;\n case GGML_TYPE_IQ4_NL: return VDR_IQ4_NL_Q8_1_MMVQ;\n case GGML_TYPE_IQ4_XS: return VDR_IQ4_XS_Q8_1_MMVQ;\n default: return 1;\n }\n}\n\nenum mmvq_parameter_table_id {\n MMVQ_PARAMETERS_GENERIC = 0,\n MMVQ_PARAMETERS_GCN,\n MMVQ_PARAMETERS_RDNA2,\n MMVQ_PARAMETERS_RDNA3_0,\n MMVQ_PARAMETERS_RDNA4\n};\n\nstatic constexpr __device__ mmvq_parameter_table_id get_device_table_id() {\n#if defined(RDNA4)\n return MMVQ_PARAMETERS_RDNA4;\n#elif defined(RDNA3_0)\n return MMVQ_PARAMETERS_RDNA3_0;\n#elif defined(RDNA2) || defined(RDNA3_5)\n return MMVQ_PARAMETERS_RDNA2;\n#elif defined(GCN) || defined(CDNA)\n return MMVQ_PARAMETERS_GCN;\n#else\n return MMVQ_PARAMETERS_GENERIC;\n#endif\n}\n\nstatic __host__ mmvq_parameter_table_id get_device_table_id(int cc) {\n if (GGML_CUDA_CC_IS_RDNA4(cc)) {\n return MMVQ_PARAMETERS_RDNA4;\n }\n if (GGML_CUDA_CC_IS_RDNA3_0(cc)) {\n return MMVQ_PARAMETERS_RDNA3_0;\n }\n if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) {\n return MMVQ_PARAMETERS_RDNA2;\n }\n if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) {\n return MMVQ_PARAMETERS_GCN;\n }\n return MMVQ_PARAMETERS_GENERIC;\n}\n\nstatic constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_dst, mmvq_parameter_table_id table_id) {\n if (table_id == MMVQ_PARAMETERS_GENERIC) {\n switch (ncols_dst) {\n case 1:\n case 2:\n case 3:\n case 4:\n return 4;\n case 5:\n case 6:\n case 7:\n case 8:\n return 2;\n default:\n return 1;\n }\n } else if (table_id == MMVQ_PARAMETERS_GCN) {\n switch (ncols_dst) {\n case 1:\n case 2:\n case 3:\n case 4:\n return 2;\n case 5:\n case 6:\n case 7:\n case 8:\n default:\n return 1;\n }\n }\n if (table_id == MMVQ_PARAMETERS_RDNA4) {\n // nwarps=8 benefits types with simple vec_dot on RDNA4 (ncols_dst=1).\n // Types with complex vec_dot (Q3_K, IQ2_*, IQ3_*) regress due to register\n // pressure and lookup table contention at higher thread counts.\n if (ncols_dst == 1) {\n switch (type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_IQ4_XS:\n return 8;\n default:\n return 1;\n }\n }\n return 1;\n }\n if (table_id == MMVQ_PARAMETERS_RDNA3_0) {\n // RDNA3 (W7900): stricter whitelist than RDNA4.\n // Q2_K / Q5_K / IQ4_XS regress in full quant sweeps.\n if (ncols_dst == 1) {\n switch (type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_IQ4_NL:\n return 8;\n default:\n return 1;\n }\n }\n return 1;\n }\n return 1;\n}\n\nstatic constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id) {\n if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN) {\n switch (ncols_dst) {\n case 1:\n return 1;\n case 2:\n case 3:\n case 4:\n case 5:\n case 6:\n case 7:\n case 8:\n return 2;\n default:\n return 1;\n }\n }\n return 1;\n}\n\ntemplate \n__launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id())*ggml_cuda_get_physical_warp_size(), 1)\nstatic __global__ void mul_mat_vec_q(\n const void * __restrict__ vx, const void * __restrict__ vy, const int32_t * __restrict__ ids, const ggml_cuda_mm_fusion_args_device fusion, float * __restrict__ dst,\n const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,\n const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,\n const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,\n const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst,\n const uint32_t ids_stride) {\n\n constexpr int qk = ggml_cuda_type_traits::qk;\n constexpr int qi = ggml_cuda_type_traits::qi;\n constexpr int vdr = get_vdr_mmvq(type);\n constexpr mmvq_parameter_table_id table_id = get_device_table_id();\n constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id);\n constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_dst, table_id);\n constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n\n constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);\n\n const int tid = warp_size*threadIdx.y + threadIdx.x;\n const int row0 = rows_per_cuda_block*blockIdx.x;\n const int blocks_per_row_x = ncols_x / qk;\n constexpr int blocks_per_iter = vdr * nwarps*warp_size / qi;\n\n const uint32_t channel_dst = blockIdx.y;\n\n uint32_t token_idx = 0;\n uint32_t channel_x;\n uint32_t channel_y;\n uint32_t sample_dst;\n\n if constexpr (is_multi_token_id) {\n // Multi-token MUL_MAT_ID path, adding these in the normal path causes a perf regression for n_tokens=1 case\n token_idx = blockIdx.z;\n channel_x = ids[channel_dst + token_idx * ids_stride];\n channel_y = fastmodulo(channel_dst, nchannels_y);\n sample_dst = 0;\n } else {\n channel_x = ncols_dst == 1 && ids ? ids[channel_dst] : fastdiv(channel_dst, channel_ratio);\n channel_y = ncols_dst == 1 && ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst;\n sample_dst = blockIdx.z;\n }\n\n const uint32_t sample_x = fastdiv(sample_dst, sample_ratio);\n const uint32_t sample_y = sample_dst;\n\n bool use_gate = false;\n bool use_bias = false;\n bool use_gate_bias = false;\n const void * vgate = nullptr;\n const float * x_bias = nullptr;\n const float * gate_bias = nullptr;\n ggml_glu_op active_glu;\n\n if constexpr (has_fusion) {\n use_gate = fusion.gate != nullptr;\n use_bias = fusion.x_bias != nullptr;\n use_gate_bias = fusion.gate_bias != nullptr && use_gate;\n vgate = fusion.gate;\n x_bias = (const float *) fusion.x_bias;\n gate_bias = (const float *) fusion.gate_bias;\n active_glu = fusion.glu_op;\n }\n\n\n float x_biases[ncols_dst] = { 0.0f };\n float gate_biases[ncols_dst] = { 0.0f };\n if constexpr (has_fusion) {\n const uint32_t channel_bias = ids ? channel_x : channel_dst;\n if (use_bias) {\n x_bias = x_bias + sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row0;\n // 1. Hide latency by prefetching bias and gate here\n // 2. load only on threads that won't die after partial sum calculation\n if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 &&\n (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n x_biases[j] = x_bias[j * stride_col_dst + threadIdx.x];\n }\n }\n }\n if (use_gate_bias) {\n gate_bias = gate_bias + sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row0;\n if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 &&\n (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n gate_biases[j] = gate_bias[j * stride_col_dst + threadIdx.x];\n }\n }\n }\n }\n\n // partial sum for each thread\n float tmp[ncols_dst][rows_per_cuda_block] = {{0.0f}};\n float tmp_gate[ncols_dst][rows_per_cuda_block] = {{0.0f}};\n\n const block_q8_1 * y = ((const block_q8_1 *) vy) + sample_y*stride_sample_y + channel_y*stride_channel_y;\n if constexpr (is_multi_token_id) {\n y += token_idx*stride_col_y;\n }\n const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row0*stride_row_x;\n\n for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {\n const int kby = kbx * (qk/QK8_1); // y block index that aligns with kbx\n\n // x block quant index when casting the quants to int\n const int kqs = vdr * (tid % (qi/vdr));\n\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n#pragma unroll\n for (int i = 0; i < rows_per_cuda_block; ++i) {\n tmp[j][i] += vec_dot_q_cuda(\n vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);\n if constexpr (has_fusion) {\n if (use_gate) {\n tmp_gate[j][i] += vec_dot_q_cuda(\n vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);\n }\n }\n }\n }\n }\n\n __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];\n __shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];\n if constexpr (!has_fusion) {\n (void) tmp_shared_gate;\n } else if (!use_gate) {\n (void) tmp_shared_gate;\n }\n\n if (threadIdx.y > 0) {\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n#pragma unroll\n for (int i = 0; i < rows_per_cuda_block; ++i) {\n tmp_shared[threadIdx.y-1][j][i][threadIdx.x] = tmp[j][i];\n if constexpr (has_fusion) {\n if (use_gate) {\n tmp_shared_gate[threadIdx.y-1][j][i][threadIdx.x] = tmp_gate[j][i];\n }\n }\n }\n }\n }\n __syncthreads();\n if (threadIdx.y > 0) {\n return;\n }\n\n dst += sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row0;\n\n if constexpr (is_multi_token_id) {\n dst += token_idx*stride_col_dst;\n }\n\n // sum up partial sums and write back result\n#pragma unroll\n for (int j = 0; j < ncols_dst; ++j) {\n#pragma unroll\n for (int i = 0; i < rows_per_cuda_block; ++i) {\n#pragma unroll\n for (int l = 0; l < nwarps-1; ++l) {\n tmp[j][i] += tmp_shared[l][j][i][threadIdx.x];\n if constexpr (has_fusion) {\n if (use_gate) {\n tmp_gate[j][i] += tmp_shared_gate[l][j][i][threadIdx.x];\n }\n }\n }\n tmp[j][i] = warp_reduce_sum(tmp[j][i]);\n if constexpr (has_fusion) {\n if (use_gate) {\n tmp_gate[j][i] = warp_reduce_sum(tmp_gate[j][i]);\n }\n }\n }\n\n if (threadIdx.x < rows_per_cuda_block && (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {\n float result = tmp[j][threadIdx.x];\n if constexpr (has_fusion) {\n if (use_bias) {\n result += x_biases[j];\n }\n if (use_gate) {\n float gate_value = tmp_gate[j][threadIdx.x];\n if (use_gate_bias) {\n gate_value += gate_biases[j];\n }\n switch (active_glu) {\n case GGML_GLU_OP_SWIGLU:\n result *= ggml_cuda_op_silu_single(gate_value);\n break;\n case GGML_GLU_OP_GEGLU:\n result *= ggml_cuda_op_gelu_single(gate_value);\n break;\n case GGML_GLU_OP_SWIGLU_OAI: {\n result = ggml_cuda_op_swiglu_oai_single(gate_value, result);\n break;\n }\n default:\n result = result * gate_value;\n break;\n }\n }\n }\n dst[j*stride_col_dst + threadIdx.x] = result;\n }\n }\n\n if constexpr (!has_fusion) {\n GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, active_glu, gate_bias, x_bias, tmp_gate);\n }\n}\n\ntemplate\nstatic std::pair calc_launch_params(\n const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens,\n const int warp_size, const mmvq_parameter_table_id table_id) {\n const int64_t nblocks = (nrows_x + calc_rows_per_block(ncols_dst, table_id) - 1) / calc_rows_per_block(ncols_dst, table_id);\n const dim3 block_nums(nblocks, nchannels_dst, nsamples_or_ntokens);\n const dim3 block_dims(warp_size, calc_nwarps(type, ncols_dst, table_id), 1);\n return {block_nums, block_dims};\n}\n\ntemplate\nstatic void mul_mat_vec_q_switch_fusion(\n const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,\n const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,\n const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,\n const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,\n const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst,\n const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared,\n const uint32_t ids_stride, cudaStream_t stream) {\n\n const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;\n if constexpr (c_ncols_dst == 1) {\n if (has_fusion) {\n mul_mat_vec_q<<>>\n (vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);\n return;\n }\n }\n\n GGML_ASSERT(!has_fusion && \"fusion only supported for ncols_dst=1\");\n\n mul_mat_vec_q<<>>\n (vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);\n}\n\ntemplate \nstatic void mul_mat_vec_q_switch_ncols_dst(\n const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,\n const int ncols_x, const int nrows_x, const int ncols_dst,\n const int stride_row_x, const int stride_col_y, const int stride_col_dst,\n const int nchannels_x, const int nchannels_y, const int nchannels_dst,\n const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,\n const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,\n const int ids_stride, cudaStream_t stream) {\n\n GGML_ASSERT(ncols_x % ggml_blck_size(type) == 0);\n GGML_ASSERT(ncols_dst <= MMVQ_MAX_BATCH_SIZE);\n\n const uint3 nchannels_y_fd = ids ? init_fastdiv_values(nchannels_y) : make_uint3(0, 0, 0);\n const uint3 channel_ratio_fd = ids ? make_uint3(0, 0, 0) : init_fastdiv_values(nchannels_dst / nchannels_x);\n const uint3 sample_ratio_fd = init_fastdiv_values(nsamples_dst / nsamples_x);\n\n const int device = ggml_cuda_get_device();\n const int warp_size = ggml_cuda_info().devices[device].warp_size;\n const mmvq_parameter_table_id table_id = get_device_table_id(ggml_cuda_info().devices[device].cc);\n\n const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;\n const bool has_ids = ids != nullptr;\n\n if (has_ids && ncols_dst > 1) {\n // Multi-token MUL_MAT_ID path only - single-token goes through regular path below\n constexpr int c_ncols_dst = 1;\n std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, ncols_dst, warp_size, table_id);\n mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,\n dims.first, dims.second, 0, ids_stride, stream);\n return;\n }\n\n switch (ncols_dst) {\n case 1: {\n constexpr int c_ncols_dst = 1;\n std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);\n mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,\n dims.first, dims.second, 0, ids_stride, stream);\n } break;\n case 2: {\n constexpr int c_ncols_dst = 2;\n std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);\n mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,\n dims.first, dims.second, 0, ids_stride, stream);\n } break;\n case 3: {\n constexpr int c_ncols_dst = 3;\n std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);\n mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,\n dims.first, dims.second, 0, ids_stride, stream);\n } break;\n case 4: {\n constexpr int c_ncols_dst = 4;\n std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);\n mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,\n dims.first, dims.second, 0, ids_stride, stream);\n } break;\n case 5: {\n constexpr int c_ncols_dst = 5;\n std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);\n mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,\n dims.first, dims.second, 0, ids_stride, stream);\n } break;\n case 6: {\n constexpr int c_ncols_dst = 6;\n std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);\n mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,\n dims.first, dims.second, 0, ids_stride, stream);\n } break;\n case 7: {\n constexpr int c_ncols_dst = 7;\n std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);\n mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,\n dims.first, dims.second, 0, ids_stride, stream);\n } break;\n case 8: {\n constexpr int c_ncols_dst = 8;\n std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);\n mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,\n channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,\n sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,\n dims.first, dims.second, 0, ids_stride, stream);\n } break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n\n GGML_UNUSED(has_fusion);\n}\nstatic void mul_mat_vec_q_switch_type(\n const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,\n const int ncols_x, const int nrows_x, const int ncols_dst,\n const int stride_row_x, const int stride_col_y, const int stride_col_dst,\n const int nchannels_x, const int nchannels_y, const int nchannels_dst,\n const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,\n const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,\n const int ids_stride, cudaStream_t stream) {\n switch (type_x) {\n case GGML_TYPE_Q4_0:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_Q4_1:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_Q5_0:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_Q5_1:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_Q8_0:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_MXFP4:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_Q2_K:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_Q3_K:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_Q4_K:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_Q5_K:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_Q6_K:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_IQ2_XXS:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_IQ2_XS:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_IQ2_S:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_IQ3_XXS:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_IQ1_S:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_IQ1_M:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_IQ4_NL:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_IQ4_XS:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n case GGML_TYPE_IQ3_S:\n mul_mat_vec_q_switch_ncols_dst\n (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,\n nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,\n nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);\n break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\nvoid ggml_cuda_mul_mat_vec_q(\n ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,\n const ggml_cuda_mm_fusion_args_host * fusion) {\n GGML_ASSERT( src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.\n\n GGML_TENSOR_BINARY_OP_LOCALS;\n\n cudaStream_t stream = ctx.stream();\n\n const size_t ts_src0 = ggml_type_size(src0->type);\n const size_t ts_src1 = ggml_type_size(src1->type);\n const size_t ts_dst = ggml_type_size(dst->type);\n\n GGML_ASSERT( nb00 == ts_src0);\n GGML_ASSERT( nb10 == ts_src1);\n GGML_ASSERT( nb0 == ts_dst);\n GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));\n\n GGML_ASSERT(!ids || ne12 <= MMVQ_MAX_BATCH_SIZE);\n\n const float * src1_d = (const float *) src1->data;\n const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr;\n float * dst_d = (float *) dst->data;\n\n ggml_cuda_mm_fusion_args_device fusion_local{};\n\n if (fusion) {\n GGML_ASSERT( !ids || dst->ne[2] == 1);\n GGML_ASSERT( ids || dst->ne[1] == 1);\n\n if (fusion->x_bias) {\n GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32);\n GGML_ASSERT(fusion->x_bias->ne[0] == dst->ne[0]);\n GGML_ASSERT(!ids || fusion->x_bias->ne[1] == src0->ne[2]);\n fusion_local.x_bias = fusion->x_bias->data;\n }\n if (fusion->gate) {\n GGML_ASSERT(fusion->gate->type == src0->type && ggml_are_same_stride(fusion->gate, src0));\n fusion_local.gate = fusion->gate->data;\n }\n if (fusion->gate_bias) {\n GGML_ASSERT(fusion->gate_bias->type == GGML_TYPE_F32);\n GGML_ASSERT(fusion->gate_bias->ne[0] == dst->ne[0]);\n GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);\n fusion_local.gate_bias = fusion->gate_bias->data;\n }\n fusion_local.glu_op = fusion->glu_op;\n }\n\n // If src0 is a temporary compute buffer, clear any potential padding.\n if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {\n const size_t size_data = ggml_nbytes(src0);\n const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0);\n if (size_alloc > size_data) {\n GGML_ASSERT(ggml_is_contiguously_allocated(src0));\n GGML_ASSERT(!src0->view_src);\n CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream));\n }\n }\n\n const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);\n ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);\n {\n const int64_t s11 = src1->nb[1] / ts_src1;\n const int64_t s12 = src1->nb[2] / ts_src1;\n const int64_t s13 = src1->nb[3] / ts_src1;\n quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);\n }\n\n const int64_t s01 = src0->nb[1] / ts_src0;\n const int64_t s11 = ne10_padded / QK8_1;\n const int64_t s1 = dst->nb[1] / ts_dst;\n const int64_t s02 = src0->nb[2] / ts_src0;\n const int64_t s2 = dst->nb[2] / ts_dst;\n const int64_t s03 = src0->nb[3] / ts_src0;\n const int64_t s3 = dst->nb[3] / ts_dst;\n\n const int64_t s12 = ne11*s11;\n const int64_t s13 = ne12*s12;\n\n // For MUL_MAT_ID the memory layout is different than for MUL_MAT:\n const int64_t ncols_dst = ids ? ne2 : ne1;\n const int64_t nchannels_y = ids ? ne11 : ne12;\n const int64_t nchannels_dst = ids ? ne1 : ne2;\n const int64_t stride_col_dst = ids ? s2 : s1;\n const int64_t stride_col_y = ids ? s12 : s11;\n const int64_t stride_channel_dst = ids ? s1 : s2;\n const int64_t stride_channel_y = ids ? s11 : s12;\n\n const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;\n\n mul_mat_vec_q_switch_type(\n src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00,\n ne01, ncols_dst, s01, stride_col_y, stride_col_dst,\n ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,\n ne03, ne3, s03, s13, s3, ids_stride, stream);\n}\n\nvoid ggml_cuda_op_mul_mat_vec_q(\n ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,\n const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,\n const int64_t src1_padded_row_size, cudaStream_t stream) {\n\n const int64_t ne00 = src0->ne[0];\n const int64_t row_diff = row_high - row_low;\n\n const int64_t ne10 = src1->ne[0];\n GGML_ASSERT(ne10 % QK8_1 == 0);\n\n const int64_t ne0 = dst->ne[0];\n\n int id = ggml_cuda_get_device();\n\n // the main device has a larger memory buffer to hold the results from all GPUs\n // nrows_dst == nrows of the matrix that the kernel writes into\n const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;\n\n const int stride_row_x = ne00 / ggml_blck_size(src0->type);\n const int stride_col_y = src1_padded_row_size / QK8_1;\n\n ggml_cuda_mm_fusion_args_device fusion_local{};\n mul_mat_vec_q_switch_type(\n src0_dd_i, src0->type, src1_ddq_i, nullptr, fusion_local, dst_dd_i, ne00, row_diff, src1_ncols, stride_row_x, stride_col_y, nrows_dst,\n 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, stream);\n\n GGML_UNUSED_VARS(src1, dst, src1_ddf_i, src1_ncols, src1_padded_row_size);\n}\n" }, { "path": "src/ggml-cuda/mmvq.cuh", "content": "#include \"common.cuh\"\n\n#define MMVQ_MAX_BATCH_SIZE 8 // Max. batch size for which to use MMVQ kernels.\n#define MMVQ_MMID_MAX_BATCH_SIZE 4 // Max. batch size for which to use MMVQ kernels for MUL_MAT_ID\n\nvoid ggml_cuda_mul_mat_vec_q(ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr);\n\nvoid ggml_cuda_op_mul_mat_vec_q(\n ggml_backend_cuda_context & ctx,\n const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,\n const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,\n const int64_t src1_padded_row_size, cudaStream_t stream);\n" }, { "path": "src/ggml-cuda/norm.cu", "content": "#include \"norm.cuh\"\n#include \n\ntemplate \nstatic __global__ void norm_f32(\n const float * x, float * dst, const int ncols, const int64_t stride_row, const int64_t stride_channel,\n const int64_t stride_sample, const float eps) {\n const int nrows = gridDim.x;\n const int nchannels = gridDim.y;\n\n const int row = blockIdx.x;\n const int channel = blockIdx.y;\n const int sample = blockIdx.z;\n const int tid = threadIdx.x;\n\n x += sample*stride_sample + channel*stride_channel + row*stride_row;\n dst += ((sample*nchannels + channel)*nrows + row)*ncols;\n\n float2 mean_var = make_float2(0.0f, 0.0f);\n\n for (int col = tid; col < ncols; col += block_size) {\n const float xi = x[col];\n mean_var.x += xi;\n mean_var.y += xi * xi;\n }\n\n // sum up partial sums\n extern __shared__ float2 s_sum2[];\n mean_var = block_reduce(mean_var, s_sum2);\n\n const float mean = mean_var.x / ncols;\n const float var = mean_var.y / ncols - mean * mean;\n const float inv_std = rsqrtf(var + eps);\n\n for (int col = tid; col < ncols; col += block_size) {\n dst[col] = (x[col] - mean) * inv_std;\n }\n}\n\ntemplate \nstatic __global__ void group_norm_f32(const float * x, float * dst, const int group_size, const int ne_elements, const float eps) {\n // blockIdx.x: num_groups idx\n // threadIdx.x: block_size idx\n const int start = blockIdx.x*group_size + threadIdx.x;\n const int end = min(blockIdx.x*group_size + group_size, ne_elements);\n\n float tmp = 0.0f; // partial sum for thread in warp\n\n for (int j = start; j < end; j += block_size) {\n tmp += x[j];\n }\n\n extern __shared__ float s_sum[];\n tmp = block_reduce(tmp, s_sum);\n\n const float mean = tmp / group_size;\n tmp = 0.0f;\n\n for (int j = start; j < end; j += block_size) {\n const float xi = x[j] - mean;\n dst[j] = xi;\n tmp += xi * xi;\n }\n\n tmp = block_reduce(tmp, s_sum);\n\n const float variance = tmp / group_size;\n const float scale = rsqrtf(variance + eps);\n for (int j = start; j < end; j += block_size) {\n dst[j] *= scale;\n }\n}\n\ntemplate \nstatic __global__ void rms_norm_f32(const float * x,\n float * dst,\n const int ncols,\n const int64_t stride_row,\n const int64_t stride_channel,\n const int64_t stride_sample,\n const float eps,\n const float * mul = nullptr,\n const int64_t mul_stride_row = 0,\n const int64_t mul_stride_channel = 0,\n const int64_t mul_stride_sample = 0,\n const uint3 mul_ncols_packed = make_uint3(0, 0, 0),\n const uint3 mul_nrows_packed = make_uint3(0, 0, 0),\n const uint3 mul_nchannels_packed = make_uint3(0, 0, 0),\n const uint3 mul_nsamples_packed = make_uint3(0, 0, 0),\n const float * add = nullptr,\n const int64_t add_stride_row = 0,\n const int64_t add_stride_channel = 0,\n const int64_t add_stride_sample = 0,\n const uint3 add_ncols_packed = make_uint3(0, 0, 0),\n const uint3 add_nrows_packed = make_uint3(0, 0, 0),\n const uint3 add_nchannels_packed = make_uint3(0, 0, 0),\n const uint3 add_nsamples_packed = make_uint3(0, 0, 0)) {\n const int nrows = gridDim.x;\n const int nchannels = gridDim.y;\n\n const int row = blockIdx.x;\n const int channel = blockIdx.y;\n const int sample = blockIdx.z;\n const int tid = threadIdx.x;\n\n static_assert(!do_add || do_multiply, \"fusing add is not supported without multiplying\");\n\n x += sample*stride_sample + channel*stride_channel + row*stride_row;\n dst += ((sample*nchannels + channel)*nrows + row)*ncols;\n\n if constexpr (do_multiply) {\n const uint32_t mul_row = fastmodulo(row, mul_nrows_packed);\n const uint32_t mul_channel = fastmodulo(channel, mul_nchannels_packed);\n const uint32_t mul_sample = fastmodulo(sample, mul_nsamples_packed);\n mul += mul_sample * mul_stride_sample + mul_channel * mul_stride_channel + mul_row * mul_stride_row;\n }\n\n if constexpr (do_add) {\n const int add_row = fastmodulo(row, add_nrows_packed);\n const int add_channel = fastmodulo(channel, add_nchannels_packed);\n const int add_sample = fastmodulo(sample, add_nsamples_packed);\n add += add_sample * add_stride_sample + add_channel * add_stride_channel + add_row * add_stride_row;\n }\n\n float tmp = 0.0f; // partial sum for thread in warp\n\n for (int col = tid; col < ncols; col += block_size) {\n const float xi = x[col];\n tmp += xi * xi;\n }\n\n // sum up partial sums\n extern __shared__ float s_sum[];\n tmp = block_reduce(tmp, s_sum);\n\n const float mean = tmp / ncols;\n const float scale = rsqrtf(mean + eps);\n\n for (int col = tid; col < ncols; col += block_size) {\n if constexpr (do_multiply && do_add) {\n const int mul_col = fastmodulo(col, mul_ncols_packed);\n const int add_col = fastmodulo(col, add_ncols_packed);\n dst[col] = scale * x[col] * mul[mul_col] + add[add_col];\n } else if constexpr (do_multiply) {\n const int mul_col = fastmodulo(col, mul_ncols_packed);\n dst[col] = scale * x[col] * mul[mul_col];\n } else {\n dst[col] = scale * x[col];\n }\n }\n}\n\ntemplate \nstatic __global__ void rms_norm_back_f32(\n const float * grad, const float * xf, float * dst, const int ncols, const float eps) {\n const int row = blockIdx.x*blockDim.y + threadIdx.y;\n const int tid = threadIdx.x;\n\n grad += int64_t(row)*ncols;\n xf += int64_t(row)*ncols;\n dst += int64_t(row)*ncols;\n\n float sum_xx = 0.0f; // sum for squares of x, equivalent to forward pass\n float sum_xg = 0.0f; // sum for x * gradient, needed because RMS norm mixes inputs\n\n for (int col = tid; col < ncols; col += block_size) {\n const float xfi = xf[col];\n sum_xx += xfi * xfi;\n sum_xg += xfi * grad[col];\n }\n\n // sum up partial sums\n sum_xx = warp_reduce_sum(sum_xx);\n sum_xg = warp_reduce_sum(sum_xg);\n if constexpr (block_size > WARP_SIZE) {\n static_assert(block_size == 1024, \"unexpected block_size\");\n __shared__ float s_sum_xx[32];\n __shared__ float s_sum_xg[32];\n const int warp_id = threadIdx.x / WARP_SIZE;\n const int lane_id = threadIdx.x % WARP_SIZE;\n if (lane_id == 0) {\n s_sum_xx[warp_id] = sum_xx;\n s_sum_xg[warp_id] = sum_xg;\n }\n __syncthreads();\n\n sum_xx = s_sum_xx[lane_id];\n sum_xx = warp_reduce_sum(sum_xx);\n\n sum_xg = s_sum_xg[lane_id];\n sum_xg = warp_reduce_sum(sum_xg);\n }\n\n const float mean_eps = sum_xx / ncols + eps;\n const float sum_eps = sum_xx + ncols*eps;\n\n const float scale_grad = rsqrtf(mean_eps);\n const float scale_x = -scale_grad * sum_xg/sum_eps;\n\n for (int col = tid; col < ncols; col += block_size) {\n dst[col] = scale_grad*grad[col] + scale_x*xf[col];\n }\n}\n\n// template \n// static __global__ void l2_norm_f32(const float * x, float * dst, const int ncols, const float eps) {\n// const int row = blockIdx.x*blockDim.y + threadIdx.y;\n// const int tid = threadIdx.x;\n\n// float tmp = 0.0f; // partial sum for thread in warp\n\n// for (int col = tid; col < ncols; col += block_size) {\n// const float xi = x[row*ncols + col];\n// tmp += xi * xi;\n// }\n\n// // sum up partial sums\n// tmp = warp_reduce_sum(tmp);\n// if (block_size > WARP_SIZE) {\n// __shared__ float s_sum[32];\n// int warp_id = threadIdx.x / WARP_SIZE;\n// int lane_id = threadIdx.x % WARP_SIZE;\n// if (lane_id == 0) {\n// s_sum[warp_id] = tmp;\n// }\n// __syncthreads();\n// tmp = s_sum[lane_id];\n// tmp = warp_reduce_sum(tmp);\n// }\n\n// // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html\n// const float scale = rsqrtf(fmaxf(tmp, eps * eps));\n\n// for (int col = tid; col < ncols; col += block_size) {\n// dst[row*ncols + col] = scale * x[row*ncols + col];\n// }\n// }\n\ntemplate \nstatic __global__ void l2_norm_f32(\n const float * x, float * dst, const int ncols, const int64_t stride_row, const int64_t stride_channel,\n const int64_t stride_sample, const float eps) {\n const int nrows = gridDim.x;\n const int nchannels = gridDim.y;\n\n const int row = blockIdx.x;\n const int channel = blockIdx.y;\n const int sample = blockIdx.z;\n const int tid = threadIdx.x;\n\n x += sample*stride_sample + channel*stride_channel + row*stride_row;\n dst += ((sample*nchannels + channel)*nrows + row)*ncols;\n\n float tmp = 0.0f; // partial sum for thread in warp\n\n for (int col = tid; col < ncols; col += block_size) {\n const float xi = x[col];\n tmp += xi * xi;\n }\n\n // sum up partial sums\n extern __shared__ float s_sum[];\n tmp = block_reduce(tmp, s_sum);\n\n // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html\n const float scale = rsqrtf(fmaxf(tmp, eps * eps));\n\n for (int col = tid; col < ncols; col += block_size) {\n dst[col] = scale * x[col];\n }\n}\n\nstatic void norm_f32_cuda(\n const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,\n const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) {\n const dim3 blocks_num(nrows, nchannels, nsamples);\n if (ncols < 1024) {\n const dim3 block_dims(WARP_SIZE, 1, 1);\n norm_f32<<>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps);\n } else {\n const dim3 block_dims(1024, 1, 1);\n norm_f32<1024><< WARP_SIZE ? 32 * sizeof(float2): 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps);\n }\n}\n\nstatic void group_norm_f32_cuda(\n const float * x, float * dst, const int num_groups, const float eps, const int group_size, const int ne_elements, cudaStream_t stream) {\n if (group_size < 1024) {\n const dim3 block_dims(WARP_SIZE, 1, 1);\n group_norm_f32<<>>(x, dst, group_size, ne_elements, eps);\n } else {\n const dim3 block_dims(1024, 1, 1);\n group_norm_f32<1024><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, group_size, ne_elements, eps);\n }\n}\n\nstatic void rms_norm_f32_cuda(\n const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,\n const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) {\n const dim3 blocks_num(nrows, nchannels, nsamples);\n if (ncols < 1024) {\n const dim3 block_dims(256, 1, 1);\n rms_norm_f32<256, false><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps);\n } else {\n const dim3 block_dims(1024, 1, 1);\n rms_norm_f32<1024, false><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps);\n }\n}\n\nstatic void rms_norm_mul_f32_cuda(const float * x,\n const float * mul,\n const float * add,\n float * dst,\n const int ncols,\n const int nrows,\n const int nchannels,\n const int nsamples,\n const int64_t stride_row,\n const int64_t stride_channel,\n const int64_t stride_sample,\n const int64_t mul_stride_row,\n const int64_t mul_stride_channel,\n const int64_t mul_stride_sample,\n const uint32_t mul_ncols,\n const uint32_t mul_nrows,\n const uint32_t mul_nchannels,\n const uint32_t mul_nsamples,\n const int64_t add_stride_row,\n const int64_t add_stride_channel,\n const int64_t add_stride_sample,\n const uint32_t add_ncols,\n const uint32_t add_nrows,\n const uint32_t add_nchannels,\n const uint32_t add_nsamples,\n const float eps,\n cudaStream_t stream) {\n const dim3 blocks_num(nrows, nchannels, nsamples);\n if (mul == nullptr) {\n rms_norm_f32_cuda(x, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream);\n return;\n }\n if (add == nullptr) {\n const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols);\n const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows);\n const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);\n const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples);\n if (ncols < 1024) {\n const dim3 block_dims(256, 1, 1);\n rms_norm_f32<256, true><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(\n x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,\n mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed);\n } else {\n const dim3 block_dims(1024, 1, 1);\n rms_norm_f32<1024, true><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(\n x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,\n mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed);\n }\n } else {\n const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols);\n const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows);\n const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);\n const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples);\n\n const uint3 add_ncols_packed = init_fastdiv_values(add_ncols);\n const uint3 add_nrows_packed = init_fastdiv_values(add_nrows);\n const uint3 add_nchannels_packed = init_fastdiv_values(add_nchannels);\n const uint3 add_nsamples_packed = init_fastdiv_values(add_nsamples);\n if (ncols < 1024) {\n const dim3 block_dims(256, 1, 1);\n rms_norm_f32<256, true, true><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(\n x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,\n mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add,\n add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed,\n add_nchannels_packed, add_nsamples_packed);\n } else {\n const dim3 block_dims(1024, 1, 1);\n rms_norm_f32<1024, true, true><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(\n x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,\n mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add,\n add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed,\n add_nchannels_packed, add_nsamples_packed);\n }\n }\n}\n\nstatic void rms_norm_back_f32_cuda(const float * grad, const float * xf, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {\n if (ncols < 1024) {\n const dim3 block_dims(WARP_SIZE, 1, 1);\n rms_norm_back_f32<<>>(grad, xf, dst, ncols, eps);\n } else {\n const dim3 block_dims(1024, 1, 1);\n rms_norm_back_f32<1024><<>>(grad, xf, dst, ncols, eps);\n }\n}\n\nstatic void l2_norm_f32_cuda(\n const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,\n const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) {\n const dim3 blocks_num(nrows, nchannels, nsamples);\n if (ncols < 1024) {\n const dim3 block_dims(WARP_SIZE, 1, 1);\n l2_norm_f32<<>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps);\n } else {\n const dim3 block_dims(1024, 1, 1);\n l2_norm_f32<1024><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps);\n }\n}\n\nvoid ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *) src0->data;\n float * dst_d = (float *) dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_UNARY_OP_LOCALS;\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n GGML_ASSERT(eps >= 0.0f);\n\n const size_t ts0 = ggml_type_size(src0->type);\n GGML_ASSERT(nb00 == ts0);\n const int64_t s01 = nb01 / ts0;\n const int64_t s02 = nb02 / ts0;\n const int64_t s03 = nb03 / ts0;\n\n norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);\n}\n\nvoid ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *)src0->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n int num_groups = dst->op_params[0];\n\n float eps;\n memcpy(&eps, dst->op_params + 1, sizeof(float));\n GGML_ASSERT(eps >= 0.0f);\n\n int group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups);\n group_norm_f32_cuda(src0_d, dst_d, num_groups * src0->ne[3], eps, group_size, ggml_nelements(src0), stream);\n}\n\nvoid ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *) src0->data;\n float * dst_d = (float *) dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_UNARY_OP_LOCALS;\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n GGML_ASSERT(eps >= 0.0f);\n\n const size_t ts0 = ggml_type_size(src0->type);\n GGML_ASSERT(nb00 == ts0);\n const int64_t s01 = nb01 / ts0;\n const int64_t s02 = nb02 / ts0;\n const int64_t s03 = nb03 / ts0;\n\n rms_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);\n}\n\nvoid ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor) {\n const ggml_tensor * rms_norm_src = (ggml_tensor *) dst->src[0];\n float eps = 0.0f;\n\n memcpy(&eps, dst->op_params, sizeof(float));\n\n const float * src0_d = (const float *) rms_norm_src->data;\n const float * mul_d = nullptr;\n const ggml_tensor * mul_src = nullptr;\n\n if (mul_tensor->src[0] == dst) {\n mul_d = (float *) mul_tensor->src[1]->data;\n mul_src = mul_tensor->src[1];\n } else if(mul_tensor->src[1] == dst) {\n mul_d = (float *) mul_tensor->src[0]->data;\n mul_src = mul_tensor->src[0];\n } else {\n GGML_ASSERT(false);\n }\n\n float * dst_d = (float *) mul_tensor->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(rms_norm_src->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n GGML_ASSERT(mul_tensor->type == GGML_TYPE_F32);\n GGML_ASSERT(eps >= 0.0f);\n\n const int64_t ne00 = rms_norm_src->ne[0];\n const int64_t ne01 = rms_norm_src->ne[1];\n const int64_t ne02 = rms_norm_src->ne[2];\n const int64_t ne03 = rms_norm_src->ne[3];\n\n const size_t ts0 = ggml_type_size(rms_norm_src->type);\n GGML_ASSERT(rms_norm_src->nb[0] == ts0);\n const int64_t s01 = rms_norm_src->nb[1] / ts0;\n const int64_t s02 = rms_norm_src->nb[2] / ts0;\n const int64_t s03 = rms_norm_src->nb[3] / ts0;\n\n const size_t ts_mul = ggml_type_size(mul_src->type);\n GGML_ASSERT(mul_src->nb[0] == ts_mul);\n const int64_t mul_s01 = mul_src->nb[1] / ts_mul;\n const int64_t mul_s02 = mul_src->nb[2] / ts_mul;\n const int64_t mul_s03 = mul_src->nb[3] / ts_mul;\n\n const int mul_ncols = mul_src->ne[0];\n const int mul_nrows = mul_src->ne[1];\n const int mul_nchannels = mul_src->ne[2];\n const int mul_nsamples = mul_src->ne[3];\n\n rms_norm_mul_f32_cuda(src0_d, mul_d, nullptr, dst_d,\n ne00, ne01, ne02, ne03,\n /*s00*/ s01, s02, s03,\n /*mul_s00*/ mul_s01, mul_s02, mul_s03,\n mul_ncols, mul_nrows, mul_nchannels, mul_nsamples,\n /*add_s00*/ 0, 0, 0,\n 0, 0, 0, 0,\n eps, stream);\n}\n\nvoid ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,\n ggml_tensor * dst,\n ggml_tensor * mul_tensor,\n ggml_tensor * add_tensor) {\n const ggml_tensor * rms_norm_src = (ggml_tensor *) dst->src[0];\n float eps = 0.0f;\n\n memcpy(&eps, dst->op_params, sizeof(float));\n\n const float * src0_d = (const float *) rms_norm_src->data;\n const float * mul_d = nullptr;\n const ggml_tensor * mul_src = nullptr;\n\n if (mul_tensor->src[0] == dst) {\n mul_d = (float *) mul_tensor->src[1]->data;\n mul_src = mul_tensor->src[1];\n } else if (mul_tensor->src[1] == dst) {\n mul_d = (float *) mul_tensor->src[0]->data;\n mul_src = mul_tensor->src[0];\n } else {\n GGML_ASSERT(false);\n }\n\n const float * add_d = nullptr;\n const ggml_tensor * add_src = nullptr;\n\n if (add_tensor->src[0] == mul_tensor) {\n add_d = (float *) add_tensor->src[1]->data;\n add_src = add_tensor->src[1];\n } else if (add_tensor->src[1] == mul_tensor) {\n add_d = (float *) add_tensor->src[0]->data;\n add_src = add_tensor->src[0];\n } else {\n GGML_ASSERT(false);\n }\n\n float * dst_d = (float *) add_tensor->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(rms_norm_src->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n GGML_ASSERT(mul_tensor->type == GGML_TYPE_F32);\n GGML_ASSERT(add_tensor->type == GGML_TYPE_F32);\n GGML_ASSERT(eps >= 0.0f);\n\n const int64_t ne00 = rms_norm_src->ne[0];\n const int64_t ne01 = rms_norm_src->ne[1];\n const int64_t ne02 = rms_norm_src->ne[2];\n const int64_t ne03 = rms_norm_src->ne[3];\n\n const size_t ts0 = ggml_type_size(rms_norm_src->type);\n GGML_ASSERT(rms_norm_src->nb[0] == ts0);\n const int64_t s01 = rms_norm_src->nb[1] / ts0;\n const int64_t s02 = rms_norm_src->nb[2] / ts0;\n const int64_t s03 = rms_norm_src->nb[3] / ts0;\n\n const size_t ts_mul = ggml_type_size(mul_src->type);\n GGML_ASSERT(mul_src->nb[0] == ts_mul);\n const int64_t mul_s01 = mul_src->nb[1] / ts_mul;\n const int64_t mul_s02 = mul_src->nb[2] / ts_mul;\n const int64_t mul_s03 = mul_src->nb[3] / ts_mul;\n\n const int mul_ncols = mul_src->ne[0];\n const int mul_nrows = mul_src->ne[1];\n const int mul_nchannels = mul_src->ne[2];\n const int mul_nsamples = mul_src->ne[3];\n\n const size_t ts_add = ggml_type_size(add_src->type);\n GGML_ASSERT(add_src->nb[0] == ts_add);\n const int64_t add_s01 = add_src->nb[1] / ts_add;\n const int64_t add_s02 = add_src->nb[2] / ts_add;\n const int64_t add_s03 = add_src->nb[3] / ts_add;\n\n const int add_ncols = add_src->ne[0];\n const int add_nrows = add_src->ne[1];\n const int add_nchannels = add_src->ne[2];\n const int add_nsamples = add_src->ne[3];\n\n rms_norm_mul_f32_cuda(src0_d, mul_d,add_d,dst_d,\n ne00,ne01, ne02, ne03,\n /*s00*/ s01, s02, s03,\n /*mul_s00*/ mul_s01, mul_s02, mul_s03,\n mul_ncols, mul_nrows, mul_nchannels, mul_nsamples,\n /*add_s00*/ add_s01, add_s02, add_s03,\n add_ncols, add_nrows, add_nchannels, add_nsamples,\n eps, stream);\n}\n\nvoid ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * grad = dst->src[0]; // gradients\n const ggml_tensor * src0f = dst->src[1]; // src0 from forward pass\n\n const float * grad_d = (const float *) grad->data;\n const float * src0f_d = (const float *) src0f->data;\n float * dst_d = (float *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous(grad));\n\n GGML_ASSERT( grad->type == GGML_TYPE_F32);\n GGML_ASSERT(src0f->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n const int64_t ne00 = src0f->ne[0];\n const int64_t nrows = ggml_nrows(src0f);\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n GGML_ASSERT(eps >= 0.0f);\n\n rms_norm_back_f32_cuda(grad_d, src0f_d, dst_d, ne00, nrows, eps, stream);\n}\n\nvoid ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *) src0->data;\n float * dst_d = (float *) dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_TENSOR_UNARY_OP_LOCALS;\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n GGML_ASSERT(eps >= 0.0f);\n\n const size_t ts0 = ggml_type_size(src0->type);\n GGML_ASSERT(nb00 == ts0);\n const int64_t s01 = nb01 / ts0;\n const int64_t s02 = nb02 / ts0;\n const int64_t s03 = nb03 / ts0;\n\n l2_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);\n}\n" }, { "path": "src/ggml-cuda/norm.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor);\n\nvoid ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,\n ggml_tensor * dst,\n ggml_tensor * mul_tensor,\n ggml_tensor * add_tensor);\n\nvoid ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/opt-step-adamw.cu", "content": "#include \"ggml-impl.h\"\n#include \"opt-step-adamw.cuh\"\n\n#include \n\nstatic __global__ void opt_step_adamw_f32(\n float * __restrict__ x, const float * __restrict__ g, float * __restrict__ g_m, float * __restrict__ g_v,\n const float * __restrict__ pars, const int64_t k) {\n\n const int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n\n const float alpha = pars[0];\n const float beta1 = pars[1];\n const float beta2 = pars[2];\n const float eps = pars[3];\n const float wd = pars[4];\n const float beta1h = pars[5];\n const float beta2h = pars[6];\n\n const float gi = g[i];\n const float gmi = g_m[i]*beta1 + gi*(1.0f - beta1);\n const float gvi = g_v[i]*beta2 + gi*gi*(1.0f - beta2);\n\n g_m[i] = gmi;\n g_v[i] = gvi;\n\n const float mh = gmi*beta1h;\n const float vh = sqrtf(gvi*beta2h) + eps;\n\n x[i] = x[i]*(1.0f - alpha*wd) - alpha*mh/vh;\n}\n\nstatic void opt_step_adamw_f32_cuda(\n float * x, const float * g, float * g_m, float * g_v, const float * pars, const int64_t k, cudaStream_t stream) {\n\n const dim3 block_dims(CUDA_OPT_STEP_ADAMW_BLOCK_SIZE, 1, 1);\n const dim3 block_nums((k + CUDA_OPT_STEP_ADAMW_BLOCK_SIZE - 1) / CUDA_OPT_STEP_ADAMW_BLOCK_SIZE, 1, 1);\n opt_step_adamw_f32<<>>(x, g, g_m, g_v, pars, k);\n}\n\nvoid ggml_cuda_opt_step_adamw(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src0_grad = dst->src[1];\n const ggml_tensor * src0_grad_m = dst->src[2];\n const ggml_tensor * src0_grad_v = dst->src[3];\n const ggml_tensor * adamw_params = dst->src[4];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src0_grad->type == GGML_TYPE_F32);\n GGML_ASSERT(src0_grad_m->type == GGML_TYPE_F32);\n GGML_ASSERT(src0_grad_v->type == GGML_TYPE_F32);\n GGML_ASSERT(adamw_params->type == GGML_TYPE_F32);\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src0_grad));\n GGML_ASSERT(ggml_is_contiguous(src0_grad_m));\n GGML_ASSERT(ggml_is_contiguous(src0_grad_v));\n GGML_ASSERT(ggml_is_contiguous(adamw_params));\n GGML_ASSERT(ggml_are_same_shape(src0, src0_grad));\n GGML_ASSERT(ggml_are_same_shape(src0, src0_grad_m));\n GGML_ASSERT(ggml_are_same_shape(src0, src0_grad_v));\n GGML_ASSERT(ggml_nelements(adamw_params) == 7);\n\n float * src0_d = (float *) src0->data;\n const float * src0_grad_d = (const float *) src0_grad->data;\n float * src0_grad_m_d = (float *) src0_grad_m->data;\n float * src0_grad_v_d = (float *) src0_grad_v->data;\n const float * adamw_params_d = (const float *) adamw_params->data;\n\n cudaStream_t stream = ctx.stream();\n\n const int64_t ne = ggml_nelements(src0);\n\n opt_step_adamw_f32_cuda(src0_d, src0_grad_d, src0_grad_m_d, src0_grad_v_d, adamw_params_d, ne, stream);\n}\n" }, { "path": "src/ggml-cuda/opt-step-adamw.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_OPT_STEP_ADAMW_BLOCK_SIZE 256\n\nvoid ggml_cuda_opt_step_adamw(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/opt-step-sgd.cu", "content": "#include \"ggml-impl.h\"\n#include \"opt-step-sgd.cuh\"\n\n#include \n\nstatic __global__ void opt_step_sgd_f32(\n float * __restrict__ x, const float * __restrict__ g,\n const float * __restrict__ pars, const int64_t k) {\n\n const int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n x[i] = x[i] * (1.0f - pars[0] * pars[1]) - pars[0] * g[i];\n}\n\nstatic void opt_step_sgd_f32_cuda(\n float * x, const float * g, const float * __restrict__ pars, const int64_t k, cudaStream_t stream) {\n\n const dim3 block_dims(CUDA_OPT_STEP_SGD_BLOCK_SIZE, 1, 1);\n const dim3 block_nums((k + CUDA_OPT_STEP_SGD_BLOCK_SIZE - 1) / CUDA_OPT_STEP_SGD_BLOCK_SIZE, 1, 1);\n opt_step_sgd_f32<<>>(x, g, pars, k);\n}\n\nvoid ggml_cuda_opt_step_sgd(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src0_grad = dst->src[1];\n const ggml_tensor * params = dst->src[2];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src0_grad->type == GGML_TYPE_F32);\n GGML_ASSERT(params->type == GGML_TYPE_F32);\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src0_grad));\n GGML_ASSERT(ggml_is_contiguous(params));\n GGML_ASSERT(ggml_are_same_shape(src0, src0_grad));\n GGML_ASSERT(ggml_nelements(params) == 2);\n\n float * src0_d = (float *) src0->data;\n const float * src0_grad_d = (const float *) src0_grad->data;\n const float * params_d = (const float *) params->data;\n\n cudaStream_t stream = ctx.stream();\n\n const int64_t ne = ggml_nelements(src0);\n\n opt_step_sgd_f32_cuda(src0_d, src0_grad_d, params_d, ne, stream);\n}\n" }, { "path": "src/ggml-cuda/opt-step-sgd.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_OPT_STEP_SGD_BLOCK_SIZE 256\n\nvoid ggml_cuda_opt_step_sgd(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/out-prod.cu", "content": "#include \"out-prod.cuh\"\n\n#include \n\nvoid ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ne01 == ne11);\n GGML_ASSERT(ne0 == ne00);\n GGML_ASSERT(ne1 == ne10);\n\n GGML_ASSERT(ne2 % src0->ne[2] == 0);\n GGML_ASSERT(ne3 % src0->ne[3] == 0);\n\n GGML_ASSERT(ne2 == src1->ne[2]);\n GGML_ASSERT(ne3 == src1->ne[3]);\n\n const float * src0_d = (const float *) src0->data;\n const float * src1_d = (const float *) src1->data;\n float * dst_d = (float *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n cublasHandle_t handle = ctx.cublas_handle();\n\n const float alpha = 1.0f;\n const float beta = 0.0f;\n\n CUBLAS_CHECK(cublasSetStream(handle, stream));\n\n const int64_t lda = nb01 / sizeof(float);\n const int64_t ldc = nb1 / sizeof(float);\n\n const bool src1_T = ggml_is_transposed(src1);\n const cublasOperation_t src1_cublas_op = src1_T ? CUBLAS_OP_N : CUBLAS_OP_T;\n const int64_t ldb = (src1_T ? nb10 : nb11) / sizeof(float);\n GGML_ASSERT( (src1_T ? nb11 : nb10) == sizeof(float));\n\n // data strides in dimensions 2/3\n const size_t s02 = nb02 / sizeof(float);\n const size_t s03 = nb03 / sizeof(float);\n const size_t s12 = nb12 / sizeof(float);\n const size_t s13 = nb13 / sizeof(float);\n const size_t s2 = nb2 / sizeof(float);\n const size_t s3 = nb3 / sizeof(float);\n\n // dps == dst per src0, used for group query attention\n const int64_t dps2 = ne2 / ne02;\n const int64_t dps3 = ne3 / ne03;\n\n // TODO batched matrix multiplication\n for (int64_t i3 = 0; i3 < ne3; ++i3) {\n for (int64_t i2 = 0; i2 < ne2; ++i2) {\n CUBLAS_CHECK(\n cublasSgemm(handle, CUBLAS_OP_N, src1_cublas_op,\n ne0, ne1, ne01,\n &alpha, src0_d + (i3/dps3)*s03 + (i2/dps2)*s02, lda,\n src1_d + i3 *s13 + i2 *s12, ldb,\n &beta, dst_d + i3 *s3 + i2 *s2, ldc));\n }\n }\n}\n" }, { "path": "src/ggml-cuda/out-prod.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/pad.cu", "content": "#include \"pad.cuh\"\n\n#include \n\n__device__ __forceinline__ int64_t wrap_around(int64_t coord, int64_t size) {\n // + size ensures negatives are handled properly\n return (coord + size) % size;\n}\n\nstatic __global__ void pad_f32(const float * src, size_t s00, size_t s01, size_t s02, size_t s03, float * dst,\n const int lp0, const int rp0, const int lp1, const int rp1,\n const int lp2, const int rp2, const int lp3, const int rp3,\n const int ne0, const int ne1, const int ne2, const int ne3,\n const bool circular) {\n // blockIdx.z: i3*ne2+i2\n // blockIdx.y: i1\n // blockIDx.x: i0 / CUDA_PAD_BLOCK_SIZE\n // gridDim.y: ne1\n int i0 = threadIdx.x + blockIdx.x * blockDim.x;\n int i1 = blockIdx.y;\n int i2 = blockIdx.z % ne2;\n int i3 = blockIdx.z / ne2;\n\n if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {\n return;\n }\n\n const int64_t dst_idx = i3 * (ne0 * ne1 * ne2) + i2 * (ne0 * ne1) + i1 * ne0 + i0;\n\n if (!circular) {\n if ((i0 >= lp0 && i0 < ne0 - rp0) && (i1 >= lp1 && i1 < ne1 - rp1) && (i2 >= lp2 && i2 < ne2 - rp2) &&\n (i3 >= lp3 && i3 < ne3 - rp3)) {\n const int64_t i00 = i0 - lp0;\n const int64_t i01 = i1 - lp1;\n const int64_t i02 = i2 - lp2;\n const int64_t i03 = i3 - lp3;\n\n const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00;\n\n dst[dst_idx] = src[src_idx];\n } else {\n dst[dst_idx] = 0.0f;\n }\n }\n // circular means on a torus, so x and y wrap around\n else {\n const int64_t ne00 = ne0 - lp0 - rp0;\n const int64_t ne01 = ne1 - lp1 - rp1;\n const int64_t ne02 = ne2 - lp2 - rp2;\n const int64_t ne03 = ne3 - lp3 - rp3;\n\n const int64_t i00 = wrap_around(i0 - lp0, ne00);\n const int64_t i01 = wrap_around(i1 - lp1, ne01);\n const int64_t i02 = wrap_around(i2 - lp2, ne02);\n const int64_t i03 = wrap_around(i3 - lp3, ne03);\n\n const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00;\n\n dst[dst_idx] = src[src_idx];\n }\n}\n\n\nstatic void pad_f32_cuda(const float * src, size_t s00, size_t s01, size_t s02, size_t s03, float * dst,\n const int lp0, const int rp0, const int lp1, const int rp1,\n const int lp2, const int rp2, const int lp3, const int rp3,\n const int ne0, const int ne1, const int ne2, const int ne3,\n const bool circular, cudaStream_t stream) {\n int num_blocks = (ne0 + CUDA_PAD_BLOCK_SIZE - 1) / CUDA_PAD_BLOCK_SIZE;\n dim3 gridDim(num_blocks, ne1, ne2 * ne3);\n pad_f32<<>>(src, s00, s01, s02, s03, dst,\n lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3,\n ne0, ne1, ne2, ne3, circular);\n}\n\nvoid ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *) src0->data;\n float * dst_d = (float *) dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_TENSOR_UNARY_OP_LOCALS;\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n const int32_t lp0 = ((const int32_t *) (dst->op_params))[0];\n const int32_t rp0 = ((const int32_t *) (dst->op_params))[1];\n const int32_t lp1 = ((const int32_t *) (dst->op_params))[2];\n const int32_t rp1 = ((const int32_t *) (dst->op_params))[3];\n const int32_t lp2 = ((const int32_t *) (dst->op_params))[4];\n const int32_t rp2 = ((const int32_t *) (dst->op_params))[5];\n const int32_t lp3 = ((const int32_t *) (dst->op_params))[6];\n const int32_t rp3 = ((const int32_t *) (dst->op_params))[7];\n const int32_t circular = ((const int32_t *) (dst->op_params))[8];\n\n const size_t s00 = nb00 / ggml_type_size(src0->type);\n const size_t s01 = nb01 / ggml_type_size(src0->type);\n const size_t s02 = nb02 / ggml_type_size(src0->type);\n const size_t s03 = nb03 / ggml_type_size(src0->type);\n\n pad_f32_cuda(src0_d, s00, s01, s02, s03, dst_d,\n lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3,\n dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n (bool) circular, stream);\n}\n" }, { "path": "src/ggml-cuda/pad.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_PAD_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/pad_reflect_1d.cu", "content": "#include \"pad_reflect_1d.cuh\"\n\nstatic __global__ __launch_bounds__(CUDA_PAD_REFLECT_1D_BLOCK_SIZE, 1) void\n pad_reflect_1d_kernel_f32(\n const void * __restrict__ src0,\n void * __restrict__ dst,\n const int64_t ne0,\n const int64_t ne00,\n const uint3 ne01,\n const int64_t ne02,\n const int64_t ne03,\n const int64_t nb00,\n const int64_t nb01,\n const int64_t nb02,\n const int64_t nb03,\n const int64_t nb0,\n const int64_t nb1,\n const int64_t nb2,\n const int64_t nb3,\n const int p0,\n const int p1) {\n const int64_t i3 = blockIdx.z;\n const int64_t i2 = blockIdx.y;\n\n const uint2 div_mod_packed = fast_div_modulo(blockIdx.x, ne01);\n const int64_t tile1 = div_mod_packed.y; // i1\n const int64_t tile0 = div_mod_packed.x; // nth i0 tile\n const int64_t i1 = tile1;\n const int64_t i0 = threadIdx.x + tile0 * blockDim.x;\n\n // ne01.z is original value of unpacked ne01 (see init_fastdiv_values in common.cuh)\n if (i0 >= ne0 || i1 >= ne01.z || i2 >= ne02 || i3 >= ne03) {\n return;\n }\n\n const char * src0_ptr = (const char *) src0 + i3 * nb03 + i2 * nb02 + i1 * nb01;\n char * dst_ptr = (char *) dst + i3 * nb3 + i2 * nb2 + i1 * nb1;\n\n const int64_t rel_i0 = i0 - p0; // relative i0 in src0\n int64_t src_idx;\n\n if (rel_i0 < 0) {\n // Left padding - reflect\n src_idx = -rel_i0;\n } else if (rel_i0 < ne00) {\n // Middle - copy\n src_idx = rel_i0;\n } else {\n // Right padding - reflect\n src_idx = 2 * ne00 - 2 - rel_i0;\n }\n const float value = *(const float *) (src0_ptr + src_idx * nb00);\n *(float *) (dst_ptr + i0 * nb0) = value;\n\n GGML_UNUSED(p1);\n}\n\nvoid ggml_cuda_op_pad_reflect_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n const int32_t * opts = (const int32_t *) dst->op_params;\n const int p0 = opts[0];\n const int p1 = opts[1];\n\n const int64_t ne00 = src0->ne[0];\n const int64_t ne01 = src0->ne[1];\n const uint3 ne01_packed = init_fastdiv_values(ne01);\n const int64_t ne02 = src0->ne[2];\n const int64_t ne03 = src0->ne[3];\n\n const int64_t ne0 = dst->ne[0];\n\n // sanity: padded length matches\n GGML_ASSERT(ne0 == ne00 + p0 + p1);\n\n constexpr int64_t bx = CUDA_PAD_REFLECT_1D_BLOCK_SIZE; // threads per block (x)\n const int64_t tiles0 = (ne0 + bx - 1) / bx; // number of tiles along i0\n // grid.x covers i1 and all tiles of i0: [ne01 * tiles0]\n // grid.y covers i2: [ne02]\n // grid.z covers i3: [ne03]\n const dim3 grid_dims((unsigned) (ne01 * tiles0), (unsigned) ne02, (unsigned) ne03);\n const dim3 block_dims((unsigned) bx, 1, 1);\n\n pad_reflect_1d_kernel_f32<<>>(\n src0->data, dst->data, ne0, ne00, ne01_packed, ne02, ne03, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], p0, p1);\n}\n" }, { "path": "src/ggml-cuda/pad_reflect_1d.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_PAD_REFLECT_1D_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_pad_reflect_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/pool2d.cu", "content": "#include \"pool2d.cuh\"\n\ntemplate \nstatic __global__ void pool2d_nchw_kernel(\n const int ih, const int iw, const int oh, const int ow,\n const int kh, const int kw, const int sh, const int sw,\n const int ph, const int pw, const int parallel_elements,\n const Ti* src, To* dst, const enum ggml_op_pool op) {\n int idx = threadIdx.x + blockIdx.x * blockDim.x;\n if (idx >= parallel_elements) {\n return;\n }\n\n const int I_HW = ih * iw;\n const int O_HW = oh * ow;\n const int nc = idx / O_HW;\n const int cur_oh = idx % O_HW / ow;\n const int cur_ow = idx % O_HW % ow;\n const Ti* i_ptr = src + nc * I_HW;\n To* o_ptr = dst + nc * O_HW;\n const int start_h = cur_oh * sh - ph;\n const int bh = max(0, start_h);\n const int eh = min(ih, start_h + kh);\n const int start_w = cur_ow * sw - pw;\n const int bw = max(0, start_w);\n const int ew = min(iw, start_w + kw);\n const To scale = 1. / (kh * kw);\n To res = 0;\n\n switch (op) {\n case GGML_OP_POOL_AVG: res = 0; break;\n case GGML_OP_POOL_MAX: res = -FLT_MAX; break;\n default: assert(false);\n }\n\n for (int i = bh; i < eh; i += 1) {\n for (int j = bw; j < ew; j += 1) {\n#if __CUDA_ARCH__ >= 350\n Ti cur = __ldg(i_ptr + i * iw + j);\n#else\n Ti cur = i_ptr[i * iw + j];\n#endif\n switch (op) {\n case GGML_OP_POOL_AVG: res += cur * scale; break;\n case GGML_OP_POOL_MAX: res = max(res, (To)cur); break;\n default: assert(false);\n }\n }\n }\n o_ptr[cur_oh * ow + cur_ow] = res;\n}\n\nstatic void pool2d_nchw_kernel_f32_f32_cuda(\n const int ih, const int iw, const int oh, const int ow,\n const int kh, const int kw, const int sh, const int sw,\n const int ph, const int pw, const int parallel_elements,\n const float * src, float * dst, const enum ggml_op_pool op,\n cudaStream_t stream) {\n\n const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE;\n dim3 block_nums(num_blocks);\n pool2d_nchw_kernel<<>>(ih, iw, oh, ow, kh, kw, sh, sw, ph, pw, parallel_elements, src, dst, op);\n}\n\nvoid ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *)src0->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n const int32_t * opts = (const int32_t *)dst->op_params;\n enum ggml_op_pool op = static_cast(opts[0]);\n const int k0 = opts[1];\n const int k1 = opts[2];\n const int s0 = opts[3];\n const int s1 = opts[4];\n const int p0 = opts[5];\n const int p1 = opts[6];\n\n const int64_t IH = src0->ne[1];\n const int64_t IW = src0->ne[0];\n\n const int64_t N = dst->ne[3];\n const int64_t OC = dst->ne[2];\n const int64_t OH = dst->ne[1];\n const int64_t OW = dst->ne[0];\n\n const int parallel_elements = N * OC * OH * OW;\n\n pool2d_nchw_kernel_f32_f32_cuda(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_d, dst_d, op, stream);\n}\n" }, { "path": "src/ggml-cuda/pool2d.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_POOL2D_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/quantize.cu", "content": "#include \"quantize.cuh\"\n#include \n\n__launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1)\nstatic __global__ void quantize_q8_1(\n const float * __restrict__ x, void * __restrict__ vy,\n const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,\n const int64_t ne0, const uint32_t ne1, const uint3 ne2) {\n const int64_t i0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i0 >= ne0) {\n return;\n }\n\n const int64_t i3 = fastdiv(blockIdx.z, ne2);\n const int64_t i2 = blockIdx.z - i3*ne2.z;\n const int64_t i1 = blockIdx.y;\n\n const int64_t & i00 = i0;\n const int64_t & i01 = i1;\n const int64_t & i02 = i2;\n const int64_t & i03 = i3;\n\n const int64_t i_cont = ((i3*ne2.z + i2) * ne1 + i1) * ne0 + i0;\n\n block_q8_1 * y = (block_q8_1 *) vy;\n\n const int64_t ib = i_cont / QK8_1; // block index\n const int64_t iqs = i_cont % QK8_1; // quant index\n\n const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f;\n float amax = fabsf(xi);\n float sum = xi;\n\n amax = warp_reduce_max(amax);\n sum = warp_reduce_sum(sum);\n\n const float d = amax / 127.0f;\n const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);\n\n y[ib].qs[iqs] = q;\n\n if (iqs > 0) {\n return;\n }\n\n y[ib].ds = make_half2(d, sum);\n}\n\n__device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) {\n if (!(amax > 0.0f)) {\n return 0;\n }\n\n // FP4 E2M1: max exponent (unbiased) is 2.\n constexpr int FP4_E2M1_EMAX = 2;\n\n const float e = log2f(amax);\n\n // \"even\" -> round-to-nearest integer, ties-to-even\n const int e_int = __float2int_rn(e);\n\n const int shared_exp = e_int - FP4_E2M1_EMAX;\n\n int biased = shared_exp + 127;\n\n biased = max(biased, 0);\n biased = min(biased, 254);\n\n return static_cast(biased);\n}\n\n// quantize values in the format mxfp4 is stored which is interleaved nibbles\n// i.e. a block a0-a31 is represented as a0a16,a1a17 ...a15a31\nstatic __global__ void quantize_mmq_mxfp4(const float * __restrict__ x,\n const int32_t * __restrict__ ids,\n void * __restrict__ vy,\n const int64_t ne00,\n const int64_t s01,\n const int64_t s02,\n const int64_t s03,\n const int64_t ne0,\n const int ne1,\n const int ne2) {\n constexpr int vals_per_scale = 32;\n constexpr int vals_per_warp = 2 * vals_per_scale; // Each warp processes 2 blocks of 32 = 64 values\n\n const int warp_id = threadIdx.y;\n const int lane_id_32 = threadIdx.x;\n\n const int nwarps = blockDim.y;\n\n const int64_t warp_start_offset = (blockIdx.y * nwarps + warp_id) * vals_per_warp;\n\n if (warp_start_offset >= ne0) {\n return;\n }\n\n const int64_t i1 = blockIdx.x;\n const int64_t i2 = blockIdx.z % ne2;\n const int64_t i3 = blockIdx.z / ne2;\n\n const int64_t i01 = ids ? ids[i1] : i1;\n const int64_t i02 = i2;\n const int64_t i03 = i3;\n\n block_fp4_mmq * y = (block_fp4_mmq *) vy;\n\n const int64_t block_fp4_mmq_size = 8 * QK_MXFP4; // 256 values\n const int64_t ib0 = blockIdx.z * ((int64_t) ne1 * (ne0 / block_fp4_mmq_size));\n const int64_t ib = ib0 + (warp_start_offset / block_fp4_mmq_size) * ne1 + blockIdx.x;\n const int64_t quad_idx_in_block = (warp_start_offset % block_fp4_mmq_size) / vals_per_warp;\n\n const int group_id = lane_id_32 / 4;\n const int lane_in_group = lane_id_32 % 4;\n const int base = group_id * 2;\n char2 * yqs2 = (char2 *) y[ib].qs;\n\n const int64_t base_pos = i03 * s03 + i02 * s02 + i01 * s01;\n\n uint8_t scales[2];\n\n#pragma unroll\n for (int b = 0; b < 2; ++b) {\n const int64_t i0 = warp_start_offset + b * vals_per_scale + lane_id_32;\n const float xi = (i0 < ne00) ? x[base_pos + i0] : 0.0f;\n\n float amax = fabsf(xi);\n#pragma unroll\n for (int mask = 16; mask > 0; mask >>= 1) {\n amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, mask, WARP_SIZE));\n }\n\n const uint8_t e = compute_e8m0_scale(amax);\n scales[b] = e;\n const float inv_s = (amax == 0.0f) ? 0.0f : __frcp_rn(ggml_cuda_e8m0_to_fp32(e));\n\n#if CUDART_VERSION >= 12080\n const float scaled_val = xi * inv_s;\n\n const float val0 = __shfl_sync(0xFFFFFFFF, scaled_val, base, WARP_SIZE);\n const float val1 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 16, WARP_SIZE);\n const float val2 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 1, WARP_SIZE);\n const float val3 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 17, WARP_SIZE);\n\n if (lane_in_group == 0) {\n __nv_fp4x4_e2m1 fp4_packed(make_float4(val0, val1, val2, val3));\n\n yqs2[quad_idx_in_block * 16 + b * 8 + group_id] = *(char2 *) &fp4_packed;\n }\n#else\n // Fallback: manual FP4 conversion using LUT\n const uint8_t q_val = ggml_cuda_float_to_fp4_e2m1(xi, inv_s);\n\n const uint8_t q_lo_0 = __shfl_sync(0xFFFFFFFF, q_val, base, WARP_SIZE);\n const uint8_t q_lo_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 1, WARP_SIZE);\n const uint8_t q_hi_0 = __shfl_sync(0xFFFFFFFF, q_val, base + 16, WARP_SIZE);\n const uint8_t q_hi_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 17, WARP_SIZE);\n\n if (lane_in_group == 0) {\n char2 q;\n q.x = (q_hi_0 << 4) | q_lo_0;\n q.y = (q_hi_1 << 4) | q_lo_1;\n yqs2[quad_idx_in_block * 16 + b * 8 + group_id] = q;\n }\n#endif // CUDART_VERSION >= 12080\n }\n\n if (lane_id_32 == 0) {\n // Store 2 scales packed into 1 uint32\n y[ib].d4[quad_idx_in_block] = (scales[1] << 8) | scales[0];\n }\n}\n\ntemplate \nstatic __global__ void quantize_mmq_q8_1(\n const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy,\n const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,\n const int64_t ne0, const int ne1, const int ne2) {\n\n constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32;\n constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32;\n\n const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4;\n\n if (i0 >= ne0) {\n return;\n }\n\n const int64_t i1 = blockIdx.x;\n const int64_t i2 = blockIdx.z % ne2;\n const int64_t i3 = blockIdx.z / ne2;\n\n const int64_t i00 = i0;\n const int64_t i01 = ids ? ids[i1] : i1;\n const int64_t i02 = i2;\n const int64_t i03 = i3;\n\n const float4 * x4 = (const float4 *) x;\n\n block_q8_1_mmq * y = (block_q8_1_mmq *) vy;\n\n const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel\n const int64_t ib = ib0 + (i0 / (4*QK8_1))*ne1 + blockIdx.x; // block index in channel\n const int64_t iqs = i0 % (4*QK8_1); // quant index in block\n\n // Load 4 floats per thread and calculate max. abs. value between them:\n const float4 xi = i0 < ne00 ? x4[(i03*s03 + i02*s02 + i01*s01 + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f);\n float amax = fabsf(xi.x);\n amax = fmaxf(amax, fabsf(xi.y));\n amax = fmaxf(amax, fabsf(xi.z));\n amax = fmaxf(amax, fabsf(xi.w));\n\n // Exchange max. abs. value between vals_per_scale/4 threads.\n#pragma unroll\n for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) {\n amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));\n }\n\n float sum;\n if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {\n sum = xi.x + xi.y + xi.z + xi.w;\n\n // Calculate sums across vals_per_sum/4 threads.\n#pragma unroll\n for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) {\n sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE);\n }\n }\n\n const float d_inv = 127.0f / amax;\n char4 q;\n q.x = roundf(xi.x*d_inv);\n q.y = roundf(xi.y*d_inv);\n q.z = roundf(xi.z*d_inv);\n q.w = roundf(xi.w*d_inv);\n\n // Write back 4 int8 values as a single 32 bit value for better memory bandwidth:\n char4 * yqs4 = (char4 *) y[ib].qs;\n yqs4[iqs/4] = q;\n\n if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) {\n if (iqs % 16 != 0 || iqs >= 96) {\n return;\n }\n\n y[ib].d2s6[2 + iqs/16] = sum;\n\n if (iqs % 64 != 0) {\n return;\n }\n\n const float d = 1.0f / d_inv;\n\n y[ib].d2s6[iqs/64] = d;\n\n return;\n }\n\n if (iqs % 32 != 0) {\n return;\n }\n\n const float d = 1.0f / d_inv;\n\n if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) {\n y[ib].ds4[iqs/32] = make_half2(d, sum);\n } else {\n y[ib].d4[iqs/32] = d;\n }\n}\n\nvoid quantize_row_q8_1_cuda(\n const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,\n const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,\n const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {\n GGML_ASSERT(!ids);\n GGML_ASSERT(ne0 % QK8_1 == 0);\n\n const uint3 ne2_fastdiv = init_fastdiv_values(ne2);\n\n const int64_t block_num_x = (ne0 + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;\n const dim3 num_blocks(block_num_x, ne1, ne2*ne3);\n const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);\n quantize_q8_1<<>>(x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv);\n GGML_UNUSED(type_src0);\n}\n\nvoid quantize_mmq_q8_1_cuda(\n const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,\n const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,\n const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {\n GGML_ASSERT(ne00 % 4 == 0);\n GGML_ASSERT(ne0 % (4*QK8_1) == 0);\n\n // ne1 tends to assume the highest values, therefore use it as the \"x\" dimension of the CUDA grid:\n const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);\n const dim3 num_blocks(ne1, block_num_y, ne2*ne3);\n const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);\n switch (mmq_get_q8_1_ds_layout(type_src0)) {\n case MMQ_Q8_1_DS_LAYOUT_D4:\n quantize_mmq_q8_1\n <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2);\n break;\n case MMQ_Q8_1_DS_LAYOUT_DS4:\n quantize_mmq_q8_1\n <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2);\n break;\n case MMQ_Q8_1_DS_LAYOUT_D2S6:\n quantize_mmq_q8_1\n <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2);\n break;\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\nvoid quantize_mmq_mxfp4_cuda(const float * x,\n const int32_t * ids,\n void * vy,\n [[maybe_unused]] const ggml_type type_src0,\n const int64_t ne00,\n const int64_t s01,\n const int64_t s02,\n const int64_t s03,\n const int64_t ne0,\n const int64_t ne1,\n const int64_t ne2,\n const int64_t ne3,\n cudaStream_t stream) {\n GGML_ASSERT(ne0 % (2 * QK_MXFP4) == 0);\n\n constexpr int nwarps = 8;\n constexpr int vals_per_warp = 2 * QK_MXFP4;\n constexpr int vals_per_block = nwarps * vals_per_warp;\n\n const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block;\n const dim3 num_blocks(ne1, block_num_y, ne2 * ne3);\n const dim3 block_size(WARP_SIZE, nwarps, 1);\n\n quantize_mmq_mxfp4<<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2);\n}\n" }, { "path": "src/ggml-cuda/quantize.cuh", "content": "#pragma once\n\n#include \"common.cuh\"\n#include \"mmq.cuh\"\n\n#include \n\n#define CUDA_QUANTIZE_BLOCK_SIZE 256\n#define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128\n\nstatic_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, \"Risk of out-of-bounds access.\");\nstatic_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, \"Risk of out-of-bounds access.\");\n\ntypedef void (*quantize_cuda_t)(\n const float * x, const int32_t * ids, void * vy,\n ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,\n int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);\n\nvoid quantize_row_q8_1_cuda(\n const float * x, const int32_t * ids, void * vy,\n ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,\n int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);\n\nvoid quantize_mmq_q8_1_cuda(\n const float * x, const int32_t * ids, void * vy,\n ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,\n int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);\n\nvoid quantize_mmq_mxfp4_cuda(const float * x,\n const int32_t * ids,\n void * vy,\n ggml_type type_src0,\n int64_t ne00,\n int64_t s01,\n int64_t s02,\n int64_t s03,\n int64_t ne0,\n int64_t ne1,\n int64_t ne2,\n int64_t ne3,\n cudaStream_t stream);\n" }, { "path": "src/ggml-cuda/reduce_rows.cuh", "content": "#include \"common.cuh\"\n\n// Row reduction kernel template - compute sum (norm=false) or mean (norm=true)\ntemplate \nstatic __global__ void reduce_rows_f32(const float * __restrict__ x, float * __restrict__ dst, const int ncols) {\n const int row = blockIdx.x;\n const int col = threadIdx.x;\n\n float sum = 0.0f;\n const int num_unroll = 8;\n float temp[num_unroll];\n float sum_temp[num_unroll] = { 0.0f };\n for (int i = col; i < ncols;) {\n for (int j = 0; j < num_unroll; ++j) {\n if (i < ncols) {\n temp[j] = x[row * ncols + i];\n } else {\n temp[j] = 0;\n }\n i += blockDim.x;\n }\n for (int j = 0; j < num_unroll; ++j) {\n sum_temp[j] += temp[j];\n }\n }\n for (int j = 0; j < num_unroll; ++j) {\n sum += sum_temp[j];\n }\n\n // sum up partial sums\n __shared__ float shared_vals[32];\n sum = block_reduce(sum, shared_vals);\n\n if (col != 0) {\n return;\n }\n\n dst[row] = norm ? sum / ncols : sum;\n}\n" }, { "path": "src/ggml-cuda/roll.cu", "content": "#include \"ggml-cuda/common.cuh\"\n#include \"roll.cuh\"\n\nstatic __forceinline__ __device__ int64_t wrap_index(const int64_t idx, const int64_t ne) {\n if (idx < 0) {\n return idx + ne;\n }\n if (idx >= ne) {\n return idx - ne;\n }\n return idx;\n}\n\nstatic __global__ void roll_f32_cuda(const float * __restrict__ src,\n float * __restrict__ dst,\n const int64_t ne00,\n const int64_t ne01,\n const int64_t ne02,\n const int64_t ne03,\n const int s0,\n const int s1,\n const int s2,\n const int s3) {\n const int64_t idx = int64_t(blockDim.x) * blockIdx.x + threadIdx.x;\n const int64_t n_elements = ne00 * ne01 * ne02 * ne03;\n\n if (idx >= n_elements) {\n return;\n }\n\n const int64_t i0 = idx % ne00;\n const int64_t i1 = (idx / ne00) % ne01;\n const int64_t i2 = (idx / (ne00 * ne01)) % ne02;\n const int64_t i3 = (idx / (ne00 * ne01 * ne02)) % ne03;\n\n const int64_t d0 = wrap_index(i0 - s0, ne00);\n const int64_t d1 = wrap_index(i1 - s1, ne01);\n const int64_t d2 = wrap_index(i2 - s2, ne02);\n const int64_t d3 = wrap_index(i3 - s3, ne03);\n\n dst[i3 * (ne00 * ne01 * ne02) + i2 * (ne01 * ne00) + i1 * ne00 + i0] =\n src[d3 * (ne00 * ne01 * ne02) + d2 * (ne01 * ne00) + d1 * ne00 + d0];\n}\n\nvoid ggml_cuda_op_roll(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n int s0 = dst->op_params[0];\n int s1 = dst->op_params[1];\n int s2 = dst->op_params[2];\n int s3 = dst->op_params[3];\n\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *) dst->src[0]->data;\n float * dst_d = (float *) dst->data;\n\n GGML_TENSOR_UNARY_OP_LOCALS;\n\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(ggml_are_same_shape(dst->src[0], dst));\n\n cudaStream_t stream = ctx.stream();\n\n int64_t sz = (ne00 * ne01 * ne02 * ne03);\n int64_t num_blocks = (sz + CUDA_ROLL_BLOCK_SIZE - 1) / CUDA_ROLL_BLOCK_SIZE;\n\n roll_f32_cuda<<>>(\n src0_d, dst_d, ne00, ne01, ne02, ne03, s0, s1, s2, s3);\n}\n" }, { "path": "src/ggml-cuda/roll.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_ROLL_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_roll(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/rope.cu", "content": "#include \"convert.cuh\"\n#include \"ggml-cuda/common.cuh\"\n#include \"ggml.h\"\n#include \"rope.cuh\"\n\nstruct rope_corr_dims {\n float v[2];\n};\n\n\nstruct mrope_sections {\n int v[4];\n};\n\nstatic __device__ float rope_yarn_ramp(const float low, const float high, const int i0) {\n const float y = (i0 / 2 - low) / max(0.001f, high - low);\n return 1.0f - min(1.0f, max(0.0f, y));\n}\n\n// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn\n// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.\ntemplate\nstatic __device__ void rope_yarn(\n const float theta_extrap, const float freq_scale, const rope_corr_dims corr_dims, const int64_t i0, const float ext_factor,\n float mscale, float & cos_theta, float & sin_theta) {\n // Get n-d rotational scaling corrected for extrapolation\n float theta_interp = freq_scale * theta_extrap;\n float theta = theta_interp;\n if (ext_factor != 0.0f) {\n float ramp_mix = rope_yarn_ramp(corr_dims.v[0], corr_dims.v[1], i0) * ext_factor;\n theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;\n\n // Get n-d magnitude scaling corrected for interpolation\n mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);\n }\n cos_theta = cosf(theta) * mscale;\n sin_theta = sinf(theta) * mscale;\n if (!forward) {\n sin_theta *= -1.0f;\n }\n}\n\ntemplate \nstatic __global__ void rope_norm(const T * x,\n D * dst,\n const int ne00,\n const int ne01,\n const int ne02,\n const int s01,\n const int s02,\n const int s03,\n const int s1,\n const int s2,\n const int s3,\n const int n_dims,\n const int32_t * pos,\n const float freq_scale,\n const float ext_factor,\n const float attn_factor,\n const rope_corr_dims corr_dims,\n const float theta_scale,\n const float * freq_factors,\n const int64_t * row_indices,\n const int set_rows_stride) {\n const int i0 = 2*(blockDim.y*blockIdx.y + threadIdx.y);\n\n if (i0 >= ne00) {\n return;\n }\n\n const int row_dst = blockDim.x*blockIdx.x + threadIdx.x;\n\n const uint32_t i3 = row_dst / (ne01 * ne02);\n const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;\n const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;\n\n int idst = i0 + i1 * s1 + i2 * s2 + i3 * s3;\n const int ix = i0 + i1 * s01 + i2 * s02 + i3 * s03;\n // Fusion optimization: ROPE + VIEW + SET_ROWS.\n // The rope output is viewed as a 1D tensor and offset based on a row index in row_indices.\n if (set_rows_stride != 0) {\n idst = i1 * s1 + i0;\n idst += row_indices[i2] * set_rows_stride;\n }\n\n const auto & store_coaelsced = [&](float x0, float x1) {\n if constexpr (std::is_same_v) {\n float2 v = make_float2(x0, x1);\n ggml_cuda_memcpy_1<8>(dst + idst, &v);\n } else if constexpr (std::is_same_v) {\n half2 v = make_half2(x0, x1);\n ggml_cuda_memcpy_1<4>(dst + idst, &v);\n }\n };\n if (i0 >= n_dims) {\n store_coaelsced(x[ix + 0], x[ix + 1]);\n return;\n }\n\n const float theta_base = pos[i2]*powf(theta_scale, i0/2.0f);\n\n const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f;\n\n float cos_theta;\n float sin_theta;\n\n rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);\n\n const float x0 = x[ix + 0];\n const float x1 = x[ix + 1];\n\n store_coaelsced(x0 * cos_theta - x1 * sin_theta, x0 * sin_theta + x1 * cos_theta);\n}\n\ntemplate \nstatic __global__ void rope_neox(const T * x,\n D * dst,\n const int ne00,\n const int ne01,\n const int ne02,\n const int s01,\n const int s02,\n const int s03,\n const int s1,\n const int s2,\n const int s3,\n const int n_dims,\n const int32_t * pos,\n const float freq_scale,\n const float ext_factor,\n const float attn_factor,\n const rope_corr_dims corr_dims,\n const float theta_scale,\n const float * freq_factors,\n const int64_t * row_indices,\n const int set_rows_stride) {\n const int i0 = 2*(blockDim.y*blockIdx.y + threadIdx.y);\n\n if (i0 >= ne00) {\n return;\n }\n\n const int row_dst = blockDim.x*blockIdx.x + threadIdx.x;\n\n const uint32_t i3 = row_dst / (ne01 * ne02);\n const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;\n const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;\n\n int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;\n const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;\n\n // Fusion optimization: ROPE + VIEW + SET_ROWS.\n // The rope output is viewed as a 1D tensor and offset based on a row index in row_indices.\n if (set_rows_stride != 0) {\n idst = i1 * s1 + i0 / 2;\n idst += row_indices[i2] * set_rows_stride;\n }\n\n if (i0 >= n_dims) {\n dst[idst + i0 / 2 + 0] = ggml_cuda_cast(x[ix + i0 / 2 + 0]);\n dst[idst + i0 / 2 + 1] = ggml_cuda_cast(x[ix + i0 / 2 + 1]);\n\n return;\n }\n\n const float theta_base = pos[i2]*powf(theta_scale, i0/2.0f);\n\n const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f;\n\n float cos_theta;\n float sin_theta;\n\n rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);\n\n const float x0 = x[ix + 0];\n const float x1 = x[ix + n_dims/2];\n\n dst[idst + 0] = ggml_cuda_cast(x0 * cos_theta - x1 * sin_theta);\n dst[idst + n_dims / 2] = ggml_cuda_cast(x0 * sin_theta + x1 * cos_theta);\n}\n\ntemplate \nstatic __global__ void rope_multi(const T * x,\n T * dst,\n const int ne00,\n const int ne01,\n const int ne02,\n const int s01,\n const int s02,\n const int s03,\n const int s1,\n const int s2,\n const int s3,\n const int n_dims,\n const int32_t * pos,\n const float freq_scale,\n const float ext_factor,\n const float attn_factor,\n const rope_corr_dims corr_dims,\n const float theta_scale,\n const float * freq_factors,\n const mrope_sections sections,\n const bool is_imrope) {\n const int i0 = 2 * (blockDim.y * blockIdx.y + threadIdx.y);\n\n if (i0 >= ne00) {\n return;\n }\n\n const int row_dst = blockDim.x*blockIdx.x + threadIdx.x;\n\n const uint32_t i3 = row_dst / (ne01 * ne02);\n const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;\n const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;\n\n int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;\n const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;\n\n if (i0 >= n_dims) {\n dst[idst + i0/2 + 0] = x[ix + i0/2 + 0];\n dst[idst + i0/2 + 1] = x[ix + i0/2 + 1];\n\n return;\n }\n\n const int sect_dims = sections.v[0] + sections.v[1] + sections.v[2] + sections.v[3];\n const int sec_w = sections.v[1] + sections.v[0];\n const int sector = (i0 / 2) % sect_dims;\n\n float theta_base = 0.0;\n if (is_imrope) {\n if (sector % 3 == 1 && sector < 3 * sections.v[1]) { // h\n theta_base = pos[i2 + ne02 * 1] * powf(theta_scale, i0 / 2.0f);\n } else if (sector % 3 == 2 && sector < 3 * sections.v[2]) { // w\n theta_base = pos[i2 + ne02 * 2] * powf(theta_scale, i0 / 2.0f);\n } else if (sector % 3 == 0 && sector < 3 * sections.v[0]) { // t\n theta_base = pos[i2] * powf(theta_scale, i0 / 2.0f);\n } else {\n theta_base = pos[i2 + ne02 * 3] * powf(theta_scale, i0 / 2.0f);\n }\n } else {\n if (sector < sections.v[0]) {\n theta_base = pos[i2] * powf(theta_scale, i0 / 2.0f);\n } else if (sector >= sections.v[0] && sector < sec_w) {\n theta_base = pos[i2 + ne02 * 1] * powf(theta_scale, i0 / 2.0f);\n } else if (sector >= sec_w && sector < sec_w + sections.v[2]) {\n theta_base = pos[i2 + ne02 * 2] * powf(theta_scale, i0 / 2.0f);\n } else if (sector >= sec_w + sections.v[2]) {\n theta_base = pos[i2 + ne02 * 3] * powf(theta_scale, i0 / 2.0f);\n }\n }\n\n const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f;\n\n float cos_theta;\n float sin_theta;\n\n rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);\n\n const float x0 = x[ix + 0];\n const float x1 = x[ix + n_dims/2];\n\n dst[idst + 0] = x0*cos_theta - x1*sin_theta;\n dst[idst + n_dims/2] = x0*sin_theta + x1*cos_theta;\n}\n\ntemplate \nstatic __global__ void rope_vision(const T * x,\n T * dst,\n const int ne00,\n const int ne01,\n const int ne02,\n const int s01,\n const int s02,\n const int s03,\n const int s1,\n const int s2,\n const int s3,\n const int n_dims,\n const int32_t * pos,\n const float freq_scale,\n const float ext_factor,\n const float attn_factor,\n const rope_corr_dims corr_dims,\n const float theta_scale,\n const float * freq_factors,\n const mrope_sections sections) {\n const int i0 = 2*(blockDim.y*blockIdx.y + threadIdx.y);\n\n if (i0 >= ne00) {\n return;\n }\n\n const int row_dst = blockDim.x*blockIdx.x + threadIdx.x;\n\n const uint32_t i3 = row_dst / (ne01 * ne02);\n const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;\n const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;\n\n int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;\n const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;\n\n const int sect_dims = sections.v[0] + sections.v[1];\n const int sec_w = sections.v[1] + sections.v[0];\n const int sector = (i0 / 2) % sect_dims;\n\n float theta_base = 0.0;\n if (sector < sections.v[0]) {\n const int p = sector;\n theta_base = pos[i2] * powf(theta_scale, p);\n } else if (sector >= sections.v[0] && sector < sec_w) {\n const int p = sector - sections.v[0];\n theta_base = pos[i2 + ne02] * powf(theta_scale, p);\n }\n\n const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f;\n\n float cos_theta;\n float sin_theta;\n\n rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);\n\n const float x0 = x[ix + 0];\n const float x1 = x[ix + n_dims];\n\n dst[idst + 0] = x0*cos_theta - x1*sin_theta;\n dst[idst + n_dims] = x0*sin_theta + x1*cos_theta;\n}\n\ntemplate \nstatic void rope_norm_cuda(const T * x,\n D * dst,\n const int ne00,\n const int ne01,\n const int ne02,\n const int s01,\n const int s02,\n const int s03,\n const int s1,\n const int s2,\n const int s3,\n const int n_dims,\n const int nr,\n const int32_t * pos,\n const float freq_scale,\n const float freq_base,\n const float ext_factor,\n const float attn_factor,\n const rope_corr_dims corr_dims,\n const float * freq_factors,\n const int64_t * row_indices,\n const int set_rows_stride,\n cudaStream_t stream) {\n GGML_ASSERT(ne00 % 2 == 0);\n const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);\n const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE);\n const dim3 block_nums(nr, n_blocks_x, 1);\n\n const float theta_scale = powf(freq_base, -2.0f / n_dims);\n\n if (freq_factors == nullptr) {\n rope_norm<<>>(\n x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor,\n attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride);\n } else {\n rope_norm<<>>(\n x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor,\n attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride);\n }\n}\n\ntemplate \nstatic void rope_neox_cuda(const T * x,\n D * dst,\n const int ne00,\n const int ne01,\n const int ne02,\n const int s01,\n const int s02,\n const int s03,\n const int s1,\n const int s2,\n const int s3,\n const int n_dims,\n const int nr,\n const int32_t * pos,\n const float freq_scale,\n const float freq_base,\n const float ext_factor,\n const float attn_factor,\n const rope_corr_dims corr_dims,\n const float * freq_factors,\n const int64_t * row_indices,\n const int set_rows_stride,\n cudaStream_t stream) {\n GGML_ASSERT(ne00 % 2 == 0);\n const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);\n const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE);\n const dim3 block_nums(nr, n_blocks_x, 1);\n\n const float theta_scale = powf(freq_base, -2.0f / n_dims);\n\n if (freq_factors == nullptr) {\n rope_neox<<>>(\n x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor,\n attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride);\n } else {\n rope_neox<<>>(\n x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor,\n attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride);\n }\n}\n\ntemplate \nstatic void rope_multi_cuda(const T * x,\n T * dst,\n const int ne00,\n const int ne01,\n const int ne02,\n const int s01,\n const int s02,\n const int s03,\n const int s1,\n const int s2,\n const int s3,\n const int n_dims,\n const int nr,\n const int32_t * pos,\n const float freq_scale,\n const float freq_base,\n const float ext_factor,\n const float attn_factor,\n const rope_corr_dims corr_dims,\n const float * freq_factors,\n const mrope_sections sections,\n const bool is_imrope,\n cudaStream_t stream) {\n GGML_ASSERT(ne00 % 2 == 0);\n const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);\n const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE);\n const dim3 block_nums(nr, n_blocks_x, 1);\n\n const float theta_scale = powf(freq_base, -2.0f / n_dims);\n\n if (freq_factors == nullptr) {\n rope_multi<<>>(\n x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor,\n attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope);\n } else {\n rope_multi<<>>(\n x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor,\n attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope);\n }\n}\n\ntemplate \nstatic void rope_vision_cuda(const T * x,\n T * dst,\n const int ne00,\n const int ne01,\n const int ne02,\n const int s01,\n const int s02,\n const int s03,\n const int s1,\n const int s2,\n const int s3,\n const int n_dims,\n const int nr,\n const int32_t * pos,\n const float freq_scale,\n const float freq_base,\n const float ext_factor,\n const float attn_factor,\n const rope_corr_dims corr_dims,\n const float * freq_factors,\n const mrope_sections sections,\n cudaStream_t stream) {\n GGML_ASSERT(ne00 % 2 == 0);\n const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);\n const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE);\n const dim3 block_nums(nr, n_blocks_x, 1);\n // break down (head_dim, heads, seq) into (CUDA_ROPE_BLOCK_SIZE, x, heads * seq)\n // where x ~= ceil(head_dim / CUDA_ROPE_BLOCK_SIZE);\n\n const float theta_scale = powf(freq_base, -2.0f/n_dims);\n\n if (freq_factors == nullptr) {\n rope_vision<<>>(\n x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor,\n attn_factor, corr_dims, theta_scale, freq_factors, sections);\n } else {\n rope_vision<<>>(\n x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor,\n attn_factor, corr_dims, theta_scale, freq_factors, sections);\n }\n}\n\ntemplate \nvoid ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,\n ggml_tensor * dst,\n const ggml_tensor * set_rows = nullptr) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const ggml_tensor * src2 = dst->src[2];\n\n const float * src0_d = (const float *)src0->data;\n const float * src1_d = (const float *)src1->data;\n\n void * dst_d = dst->data;\n const int64_t * row_indices = nullptr;\n ggml_type dst_type = dst->type;\n int set_rows_stride = 0;\n\n if (set_rows != nullptr) {\n GGML_ASSERT(forward);\n dst_d = set_rows->data;\n row_indices = (const int64_t *) set_rows->src[1]->data;\n dst_type = set_rows->type;\n set_rows_stride = set_rows->nb[1] / ggml_type_size(set_rows->type);\n }\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n // When not fused, src0 and dst types must match\n // When fused (ROPE+VIEW+SET_ROWS), src0 may be F32 and dst may be F16\n GGML_ASSERT(src0->type == dst->type || (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16));\n\n const int64_t ne00 = src0->ne[0]; // head dims\n const int64_t ne01 = src0->ne[1]; // num heads\n const int64_t ne02 = src0->ne[2]; // num heads\n const int64_t nr = ggml_nrows(src0);\n\n const size_t s01 = src0->nb[1] / ggml_type_size(src0->type);\n const size_t s02 = src0->nb[2] / ggml_type_size(src0->type);\n const size_t s03 = src0->nb[3] / ggml_type_size(src0->type);\n\n const size_t s1 = dst->nb[1] / ggml_type_size(dst->type);\n const size_t s2 = dst->nb[2] / ggml_type_size(dst->type);\n const size_t s3 = dst->nb[3] / ggml_type_size(dst->type);\n\n //const int n_past = ((int32_t *) dst->op_params)[0];\n const int n_dims = ((int32_t *) dst->op_params)[1];\n const int mode = ((int32_t *) dst->op_params)[2];\n //const int n_ctx = ((int32_t *) dst->op_params)[3];\n const int n_ctx_orig = ((int32_t *) dst->op_params)[4];\n mrope_sections sections;\n\n // RoPE alteration for extended context\n float freq_base;\n float freq_scale;\n float ext_factor;\n float attn_factor;\n float beta_fast;\n float beta_slow;\n\n memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));\n memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));\n memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));\n memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));\n memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));\n memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));\n memcpy(§ions.v, (int32_t *) dst->op_params + 11, sizeof(int)*4);\n\n const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;\n const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;\n const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE;\n const bool is_vision = mode == GGML_ROPE_TYPE_VISION;\n\n if (is_mrope) {\n GGML_ASSERT(sections.v[0] > 0 || sections.v[1] > 0 || sections.v[2] > 0);\n }\n\n if (is_vision) {\n GGML_ASSERT(n_dims == ne00/2);\n }\n\n const int32_t * pos = (const int32_t *) src1_d;\n\n const float * freq_factors = nullptr;\n if (src2 != nullptr) {\n freq_factors = (const float *) src2->data;\n }\n\n rope_corr_dims corr_dims;\n ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims.v);\n\n // compute\n if (is_neox) {\n if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {\n rope_neox_cuda((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02,\n s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,\n ext_factor, attn_factor, corr_dims, freq_factors, row_indices,\n set_rows_stride, stream);\n } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {\n rope_neox_cuda((const float *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,\n s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,\n ext_factor, attn_factor, corr_dims, freq_factors, row_indices,\n set_rows_stride, stream);\n } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {\n rope_neox_cuda((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,\n s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,\n ext_factor, attn_factor, corr_dims, freq_factors, row_indices,\n set_rows_stride, stream);\n } else {\n GGML_ABORT(\"fatal error\");\n }\n } else if (is_mrope && !is_vision) {\n if (src0->type == GGML_TYPE_F32) {\n rope_multi_cuda((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,\n s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,\n corr_dims, freq_factors, sections, is_imrope, stream);\n } else if (src0->type == GGML_TYPE_F16) {\n rope_multi_cuda((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,\n s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,\n corr_dims, freq_factors, sections, is_imrope, stream);\n } else {\n GGML_ABORT(\"fatal error\");\n }\n } else if (is_vision) {\n if (src0->type == GGML_TYPE_F32) {\n rope_vision_cuda((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,\n s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,\n corr_dims, freq_factors, sections, stream);\n } else if (src0->type == GGML_TYPE_F16) {\n rope_vision_cuda((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,\n s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,\n corr_dims, freq_factors, sections, stream);\n } else {\n GGML_ABORT(\"fatal error\");\n }\n } else {\n if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {\n rope_norm_cuda((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02,\n s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,\n ext_factor, attn_factor, corr_dims, freq_factors, row_indices,\n set_rows_stride, stream);\n } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {\n rope_norm_cuda((const float *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,\n s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,\n ext_factor, attn_factor, corr_dims, freq_factors, row_indices,\n set_rows_stride, stream);\n } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {\n rope_norm_cuda((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,\n s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,\n ext_factor, attn_factor, corr_dims, freq_factors, row_indices,\n set_rows_stride, stream);\n } else {\n GGML_ABORT(\"fatal error\");\n }\n }\n}\n\nvoid ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_rope_impl(ctx, dst);\n}\n\nvoid ggml_cuda_op_rope_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_rope_impl(ctx, dst);\n}\n\nvoid ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rope, ggml_tensor * set_rows) {\n ggml_cuda_op_rope_impl(ctx, rope, set_rows);\n}\n" }, { "path": "src/ggml-cuda/rope.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_ROPE_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_rope_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * set_rows);\n" }, { "path": "src/ggml-cuda/scale.cu", "content": "#include \"scale.cuh\"\n\n#define MAX_GRIDDIM_X 0x7FFFFFFF\n\nstatic __global__ void scale_f32(const float * x, float * dst, const float scale, const float bias, const int64_t nelements) {\n int64_t tid = (int64_t)blockIdx.x * (int64_t)blockDim.x + (int64_t)threadIdx.x;\n int64_t stride = (int64_t)blockDim.x * (int64_t)gridDim.x;\n\n for (int64_t i = tid; i < nelements; i += stride) {\n dst[i] = scale * x[i] + bias;\n }\n}\n\nstatic void scale_f32_cuda(const float * x, float * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) {\n const int64_t num_blocks = (nelements + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;\n scale_f32<<>>(x, dst, scale, bias, nelements);\n}\n\nvoid ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *)src0->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n float scale;\n float bias;\n memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&bias, (float *) dst->op_params + 1, sizeof(float));\n\n scale_f32_cuda(src0_d, dst_d, scale, bias, ggml_nelements(src0), stream);\n}\n" }, { "path": "src/ggml-cuda/scale.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_SCALE_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/set-rows.cu", "content": "#include \"set-rows.cuh\"\n#include \"cpy-utils.cuh\"\n\ntypedef void (*set_rows_kernel_t)(const char * src, char * dst);\n\n// Generic quantized set_rows kernel template\ntemplate \nstatic __global__ void k_set_rows_quant(const float * __restrict__ src0,\n const idx_t * __restrict__ src1,\n block_type * __restrict__ dst,\n const int64_t ne_total,\n const int64_t ne10,\n const int64_t ne11,\n const int64_t ne12,\n const int64_t ne13,\n const int64_t s01,\n const int64_t s02,\n const int64_t s03,\n const int64_t s10,\n const int64_t s11,\n const int64_t s12,\n const int64_t s1,\n const int64_t s2,\n const int64_t s3,\n const uint3 ne00,\n const uint3 ne01,\n const uint3 ne02,\n const uint3 ne11_fd,\n const uint3 ne12_fd) {\n const int64_t i = int64_t(blockDim.x) * blockIdx.x + threadIdx.x;\n\n if (i >= ne_total) {\n return;\n }\n\n const int64_t i_base = i * qk;\n uint32_t tmp = (uint32_t) i_base;\n uint2 div_mod;\n\n div_mod = fast_div_modulo(tmp, ne00);\n const int64_t i00 = div_mod.y;\n tmp = div_mod.x;\n\n div_mod = fast_div_modulo(tmp, ne01);\n const int64_t i01 = div_mod.y;\n tmp = div_mod.x;\n\n div_mod = fast_div_modulo(tmp, ne02);\n const int64_t i02 = div_mod.y;\n const int64_t i03 = div_mod.x;\n\n const int64_t i12 = fastmodulo((uint32_t) i03, ne12_fd);\n const int64_t i11 = fastmodulo((uint32_t) i02, ne11_fd);\n const int64_t i10 = i01;\n\n const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);\n\n const float * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;\n block_type * dst_row_ptr = dst + (dst_row*s1 + i02*s2 + i03*s3) / sizeof(block_type);\n\n const float * src_block = src0_row + i00;\n block_type * dst_block = dst_row_ptr + i00 / qk;\n\n quantize_func(src_block, dst_block);\n\n GGML_UNUSED(ne10);\n GGML_UNUSED(ne11);\n GGML_UNUSED(ne12);\n GGML_UNUSED(ne13);\n}\n\n// Template dispatch function for quantized set_rows\ntemplate\nstatic void set_rows_cuda_quant(\n const float * src0_d, const idx_t * src1_d, block_type * dst_d,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,\n const size_t nb01, const size_t nb02, const size_t nb03,\n const size_t nb10, const size_t nb11, const size_t nb12,\n const size_t nb1, const size_t nb2, const size_t nb3,\n cudaStream_t stream) {\n\n GGML_ASSERT(ne00 % qk == 0);\n const int64_t ne_total = (ne00 * ne01 * ne02 * ne03) / qk;\n const int num_blocks = (ne_total + CUDA_SET_ROWS_BLOCK_SIZE - 1) / CUDA_SET_ROWS_BLOCK_SIZE;\n const dim3 block_size(CUDA_SET_ROWS_BLOCK_SIZE);\n const dim3 grid_size(num_blocks);\n\n const int64_t s01 = nb01/sizeof(float);\n const int64_t s02 = nb02/sizeof(float);\n const int64_t s03 = nb03/sizeof(float);\n const int64_t s10 = nb10/sizeof(idx_t);\n const int64_t s11 = nb11/sizeof(idx_t);\n const int64_t s12 = nb12/sizeof(idx_t);\n const int64_t s1 = nb1;\n const int64_t s2 = nb2;\n const int64_t s3 = nb3;\n\n if (ne_total > 0 && ne00 > 0 && ne01 > 0 && ne02 > 0 && ne11 > 0 && ne12 > 0) {\n const uint3 ne00_fd = init_fastdiv_values((uint32_t) ne00);\n const uint3 ne01_fd = init_fastdiv_values((uint32_t) ne01);\n const uint3 ne02_fd = init_fastdiv_values((uint32_t) ne02);\n const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11);\n const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12);\n\n k_set_rows_quant<<>>(\n src0_d, src1_d, dst_d, ne_total, ne10, ne11, ne12, ne13, s01, s02, s03, s10, s11, s12, s1, s2, s3, ne00_fd,\n ne01_fd, ne02_fd, ne11_fd, ne12_fd);\n }\n}\n\ntemplate \nstatic __global__ void k_set_rows(const src_t * __restrict__ src0,\n const idx_t * __restrict__ src1,\n dst_t * __restrict__ dst,\n const int64_t ne_total,\n const int64_t ne10,\n const int64_t ne11,\n const int64_t ne12,\n const int64_t ne13,\n const int64_t s01,\n const int64_t s02,\n const int64_t s03,\n const int64_t s10,\n const int64_t s11,\n const int64_t s12,\n const int64_t s1,\n const int64_t s2,\n const int64_t s3,\n const uint3 ne00,\n const uint3 ne01,\n const uint3 ne02,\n const uint3 ne11_fd,\n const uint3 ne12_fd) {\n const int64_t i = int64_t(blockDim.x) * blockIdx.x + threadIdx.x;\n\n if (i >= ne_total) {\n return;\n }\n\n uint32_t tmp = (uint32_t) i;\n uint2 div_mod;\n\n div_mod = fast_div_modulo(tmp, ne00);\n const int64_t i00 = div_mod.y;\n tmp = div_mod.x;\n\n div_mod = fast_div_modulo(tmp, ne01);\n const int64_t i01 = div_mod.y;\n tmp = div_mod.x;\n\n div_mod = fast_div_modulo(tmp, ne02);\n const int64_t i02 = div_mod.y;\n const int64_t i03 = div_mod.x;\n\n const int64_t i12 = fastmodulo((uint32_t) i03, ne12_fd);\n const int64_t i11 = fastmodulo((uint32_t) i02, ne11_fd);\n const int64_t i10 = i01;\n\n const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);\n\n const src_t * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;\n dst_t * dst_row_ptr = dst + dst_row*s1 + i02*s2 + i03*s3;\n\n dst_row_ptr[i00] = ggml_cuda_cast(src0_row[i00]);\n\n GGML_UNUSED(ne10);\n GGML_UNUSED(ne11);\n GGML_UNUSED(ne12);\n GGML_UNUSED(ne13);\n}\n\ntemplate\nstatic void set_rows_cuda(\n const src_t * src0_d, const idx_t * src1_d, dst_t * dst_d,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,\n const size_t nb01, const size_t nb02, const size_t nb03,\n const size_t nb10, const size_t nb11, const size_t nb12,\n const size_t nb1, const size_t nb2, const size_t nb3,\n cudaStream_t stream) {\n\n const int64_t ne_total = ne00 * ne01 * ne02 * ne03;\n const int num_blocks = (ne_total + CUDA_SET_ROWS_BLOCK_SIZE - 1) / CUDA_SET_ROWS_BLOCK_SIZE;\n const dim3 block_size(CUDA_SET_ROWS_BLOCK_SIZE);\n const dim3 grid_size(num_blocks);\n\n\n const int64_t s01 = nb01/sizeof(src_t);\n const int64_t s02 = nb02/sizeof(src_t);\n const int64_t s03 = nb03/sizeof(src_t);\n const int64_t s10 = nb10/sizeof(idx_t);\n const int64_t s11 = nb11/sizeof(idx_t);\n const int64_t s12 = nb12/sizeof(idx_t);\n const int64_t s1 = nb1/sizeof(dst_t);\n const int64_t s2 = nb2/sizeof(dst_t);\n const int64_t s3 = nb3/sizeof(dst_t);\n\n if (ne_total > 0 && ne00 > 0 && ne01 > 0 && ne02 > 0 && ne11 > 0 && ne12 > 0) {\n const uint3 ne00_fd = init_fastdiv_values((uint32_t) ne00);\n const uint3 ne01_fd = init_fastdiv_values((uint32_t) ne01);\n const uint3 ne02_fd = init_fastdiv_values((uint32_t) ne02);\n const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11);\n const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12);\n\n k_set_rows<<>>(src0_d, src1_d, dst_d, ne_total, ne10, ne11, ne12, ne13, s01,\n s02, s03, s10, s11, s12, s1, s2, s3, ne00_fd, ne01_fd, ne02_fd,\n ne11_fd, ne12_fd);\n }\n}\n\ntemplate\nstatic void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n const src_t * src0_d = (const src_t *)src0->data;\n const idx_t * src1_d = (const idx_t *)src1->data;\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n cudaStream_t stream = ctx.stream();\n\n\n if (dst->type == GGML_TYPE_F32) {\n set_rows_cuda(\n src0_d, src1_d, (float*)dst->data,\n ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13,\n nb01, nb02, nb03,\n nb10, nb11, nb12,\n nb1, nb2, nb3,\n stream\n );\n } else if (dst->type == GGML_TYPE_F16) {\n set_rows_cuda(\n src0_d, src1_d, (half*)dst->data,\n ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13,\n nb01, nb02, nb03,\n nb10, nb11, nb12,\n nb1, nb2, nb3,\n stream\n );\n } else if (dst->type == GGML_TYPE_BF16) {\n set_rows_cuda(\n src0_d, src1_d, (nv_bfloat16*)dst->data,\n ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13,\n nb01, nb02, nb03,\n nb10, nb11, nb12,\n nb1, nb2, nb3,\n stream\n );\n } else if (dst->type == GGML_TYPE_Q4_0) {\n set_rows_cuda_quant(\n src0_d, src1_d, (block_q4_0*)dst->data,\n ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13,\n nb01, nb02, nb03,\n nb10, nb11, nb12,\n nb1, nb2, nb3,\n stream\n );\n } else if (dst->type == GGML_TYPE_Q4_1) {\n set_rows_cuda_quant(\n src0_d, src1_d, (block_q4_1*)dst->data,\n ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13,\n nb01, nb02, nb03,\n nb10, nb11, nb12,\n nb1, nb2, nb3,\n stream\n );\n } else if (dst->type == GGML_TYPE_Q5_0) {\n set_rows_cuda_quant(\n src0_d, src1_d, (block_q5_0*)dst->data,\n ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13,\n nb01, nb02, nb03,\n nb10, nb11, nb12,\n nb1, nb2, nb3,\n stream\n );\n } else if (dst->type == GGML_TYPE_Q5_1) {\n set_rows_cuda_quant(\n src0_d, src1_d, (block_q5_1*)dst->data,\n ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13,\n nb01, nb02, nb03,\n nb10, nb11, nb12,\n nb1, nb2, nb3,\n stream\n );\n } else if (dst->type == GGML_TYPE_Q8_0) {\n set_rows_cuda_quant(\n src0_d, src1_d, (block_q8_0*)dst->data,\n ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13,\n nb01, nb02, nb03,\n nb10, nb11, nb12,\n nb1, nb2, nb3,\n stream\n );\n } else if (dst->type == GGML_TYPE_IQ4_NL) {\n set_rows_cuda_quant(\n src0_d, src1_d, (block_iq4_nl*)dst->data,\n ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13,\n nb01, nb02, nb03,\n nb10, nb11, nb12,\n nb1, nb2, nb3,\n stream\n );\n } else {\n GGML_ABORT(\"unsupported type %s\", ggml_type_name(dst->type));\n }\n}\n\n\nvoid ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32);\n\n if (src1->type == GGML_TYPE_I64) {\n set_rows_cuda(ctx, src0, src1, dst);\n } else {\n set_rows_cuda(ctx, src0, src1, dst);\n }\n}\n" }, { "path": "src/ggml-cuda/set-rows.cuh", "content": "#pragma once\n\n#include \"common.cuh\"\n\n#define CUDA_SET_ROWS_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/set.cu", "content": "#include \"set.cuh\"\n#include \"cpy.cuh\"\n\nvoid ggml_cuda_op_set(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32));\n GGML_ASSERT(src1->type == src0->type);\n GGML_ASSERT(dst ->type == src0->type);\n\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src1));\n\n const size_t nb1 = ((int32_t *) dst->op_params)[0];\n const size_t nb2 = ((int32_t *) dst->op_params)[1];\n const size_t nb3 = ((int32_t *) dst->op_params)[2];\n const size_t offset = ((int32_t *) dst->op_params)[3];\n const bool inplace= (bool) ((int32_t *) dst->op_params)[4];\n\n if (!inplace) {\n ggml_cuda_cpy(ctx, src0, dst);\n }\n\n ggml_tensor dst_view = *dst;\n dst_view.data = (void *)((char *)dst->data + offset);\n dst_view.ne[0] = src1->ne[0];\n dst_view.ne[1] = src1->ne[1];\n dst_view.ne[2] = src1->ne[2];\n dst_view.ne[3] = src1->ne[3];\n\n dst_view.nb[0] = ggml_element_size(dst);\n dst_view.nb[1] = nb1;\n dst_view.nb[2] = nb2;\n dst_view.nb[3] = nb3;\n\n ggml_cuda_cpy(ctx, src1, &dst_view);\n}\n" }, { "path": "src/ggml-cuda/set.cuh", "content": "#pragma once\n\n#include \"common.cuh\"\n\n#define CUDA_SET_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_set(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/softcap.cu", "content": "#include \"softcap.cuh\"\n\nstatic __global__ void softcap_f32(const float * x, float * dst, const float scale, const float softcap, const int k) {\n const int i = blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n\n dst[i] = tanhf(scale * x[i]) * softcap;\n}\n\nstatic void softcap_f32_cuda(const float * x, float * dst, const float scale, const float softcap, const int k, cudaStream_t stream) {\n const int num_blocks = (k + CUDA_SOFTCAP_BLOCK_SIZE - 1) / CUDA_SOFTCAP_BLOCK_SIZE;\n softcap_f32<<>>(x, dst, scale, softcap, k);\n}\n\n// fused GGML_OP_SCALE + GGML_UNARY_OP_TANH + GGML_OP_SCALE\nvoid ggml_cuda_op_softcap(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * src) {\n const ggml_tensor * src0 = src->src[0];\n const float * src0_d = (const float *)src0->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n float scale;\n float softcap;\n memcpy(&scale, (float *) src->op_params + 0, sizeof(float));\n memcpy(&softcap, (float *) dst->op_params + 0, sizeof(float));\n\n softcap_f32_cuda(src0_d, dst_d, scale, softcap, ggml_nelements(src0), stream);\n}\n" }, { "path": "src/ggml-cuda/softcap.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_SOFTCAP_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_softcap(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * src);\n" }, { "path": "src/ggml-cuda/softmax.cu", "content": "#include \"common.cuh\"\n#include \"ggml.h\"\n#include \"softmax.cuh\"\n\n#ifdef GGML_USE_HIP\n#include \n#else\n#include \n#include \n#endif // GGML_USE_HIP\n\n#include \n#include \n\ntemplate \nstatic __device__ __forceinline__ float t2f32(T val) {\n return (float) val;\n}\n\ntemplate <>\n__device__ float __forceinline__ t2f32(half val) {\n return __half2float(val);\n}\n\nstruct soft_max_params {\n\n int64_t nheads;\n uint32_t n_head_log2;\n int64_t ncols;\n int64_t nrows_x;\n int64_t nrows_y;\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n int64_t ne03;\n int64_t nb11;\n int64_t nb12;\n int64_t nb13;\n\n int64_t ne12;\n int64_t ne13;\n float scale;\n float max_bias;\n float m0;\n float m1;\n};\n\n// When ncols_template == 0 the bounds for the loops in this function are not known and can't be unrolled.\n// As we want to keep pragma unroll for all other cases we suppress the clang transformation warning here.\n#ifdef __clang__\n#pragma clang diagnostic push\n#pragma clang diagnostic ignored \"-Wpass-failed\"\n#endif // __clang__\ntemplate \nstatic __global__ void soft_max_f32(\n const float * x, const T * mask, const float * sinks, float * dst, const soft_max_params p) {\n const int ncols = ncols_template == 0 ? p.ncols : ncols_template;\n\n const int tid = threadIdx.x;\n\n const int64_t i03 = blockIdx.z;\n const int64_t i02 = blockIdx.y;\n const int64_t i01 = blockIdx.x;\n\n //TODO: noncontigous inputs/outputs\n const int rowx = blockIdx.x + blockIdx.y * gridDim.x + blockIdx.z * gridDim.x * gridDim.y;\n\n const int64_t i11 = i01;\n const int64_t i12 = i02 % p.ne12;\n const int64_t i13 = i03 % p.ne13;\n\n x += int64_t(rowx)*ncols;\n mask += (i11*p.nb11 + i12*p.nb12 + i13*p.nb13) / sizeof(T) * (mask != nullptr);\n dst += int64_t(rowx)*ncols;\n\n const int block_size = block_size_template == 0 ? blockDim.x : block_size_template;\n\n const float slope = get_alibi_slope(p.max_bias, i02, p.n_head_log2, p.m0, p.m1);\n\n extern __shared__ float data_soft_max_f32[];\n float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication\n // shared memory buffer to cache values between iterations:\n float * vals = use_shared ? buf_iw + WARP_SIZE : dst;\n\n float max_val = sinks ? sinks[i02] : -INFINITY;\n\n#pragma unroll\n for (int col0 = 0; col0 < ncols; col0 += block_size) {\n const int col = col0 + tid;\n\n if (ncols_template == 0 && col >= ncols) {\n break;\n }\n\n const float val = x[col]*p.scale + (mask ? slope*t2f32(mask[col]) : 0.0f);\n\n vals[col] = val;\n max_val = max(max_val, val);\n }\n\n // find the max value in the block\n max_val = block_reduce(max_val, buf_iw);\n\n float tmp = 0.0f; // partial sum\n\n#pragma unroll\n for (int col0 = 0; col0 < ncols; col0 += block_size) {\n const int col = col0 + tid;\n\n if (ncols_template == 0 && col >= ncols) {\n break;\n }\n\n const float val = expf(vals[col] - max_val);\n tmp += val;\n vals[col] = val;\n }\n\n // find the sum of exps in the block\n tmp = block_reduce(tmp, buf_iw);\n\n if (sinks) {\n tmp += expf(sinks[i02] - max_val);\n }\n\n const float inv_sum = 1.0f / tmp;\n\n#pragma unroll\n for (int col0 = 0; col0 < ncols; col0 += block_size) {\n const int col = col0 + tid;\n\n if (ncols_template == 0 && col >= ncols) {\n return;\n }\n\n dst[col] = vals[col] * inv_sum;\n }\n}\n\n// TODO: Template to allow keeping ncols in registers if they fit\nstatic __device__ void soft_max_f32_parallelize_cols_single_row(const float * __restrict__ x,\n float * __restrict__ dst,\n float * __restrict__ tmp_maxs,\n float * __restrict__ tmp_sums,\n const soft_max_params p) {\n namespace cg = cooperative_groups;\n\n const cg::grid_group g = cg::this_grid();\n\n const int tid = threadIdx.x;\n const int col_start = blockIdx.x * blockDim.x + tid;\n const int n_elem_per_thread = 4;\n\n float local_vals[n_elem_per_thread] = { -INFINITY, -INFINITY, -INFINITY, -INFINITY };\n float local_max = -INFINITY;\n const int step_size = gridDim.x * blockDim.x;\n __shared__ float shared_vals[32];\n\n // Compute thread-local max\n for (int col = col_start; col < p.ncols;) {\n#pragma unroll\n for (int i = 0; i < n_elem_per_thread; i++) {\n const int idx = col + i * step_size;\n local_vals[i] = idx < p.ncols ? x[idx] : -INFINITY;\n }\n#pragma unroll\n for (int i = 0; i < n_elem_per_thread; i++) {\n local_max = fmaxf(local_max, local_vals[i]);\n }\n col += step_size * n_elem_per_thread;\n }\n\n // Compute CTA-level max\n local_max = block_reduce(local_max, shared_vals);\n\n // Store CTA-level max to GMEM\n if (tid == 0) {\n tmp_maxs[blockIdx.x] = local_max;\n }\n g.sync();\n\n // Compute compute global max from CTA-level maxs\n assert(gridDim.x < blockDim.x); // currently we only support this case\n if (tid < gridDim.x) {\n local_max = tmp_maxs[tid];\n } else {\n local_max = -INFINITY;\n }\n local_max = block_reduce(local_max, shared_vals);\n\n // Compute softmax dividends, accumulate divisor\n float tmp_expf = 0.0f;\n for (int col = col_start; col < p.ncols;) {\n#pragma unroll\n for (int i = 0; i < n_elem_per_thread; i++) {\n const int idx = col + i * step_size;\n local_vals[i] = idx < p.ncols ? x[idx] : -INFINITY;\n }\n#pragma unroll\n for (int i = 0; i < n_elem_per_thread; i++) {\n const int idx = col + i * step_size;\n if (idx < p.ncols) {\n const float tmp = expf(local_vals[i] - local_max);\n tmp_expf += tmp;\n dst[idx] = tmp;\n }\n }\n col += step_size * n_elem_per_thread;\n }\n\n // Reduce divisor within CTA\n tmp_expf = block_reduce(tmp_expf, shared_vals);\n\n // Store CTA-level sum to GMEM\n if (tid == 0) {\n tmp_sums[blockIdx.x] = tmp_expf;\n }\n g.sync();\n\n // Compute global sum from CTA-level sums\n if (tid < gridDim.x) {\n tmp_expf = tmp_sums[tid];\n } else {\n tmp_expf = 0.0f;\n }\n tmp_expf = block_reduce(tmp_expf, shared_vals);\n\n // Divide dividend by global sum + store data\n for (int col = col_start; col < p.ncols;) {\n#pragma unroll\n for (int i = 0; i < n_elem_per_thread; i++) {\n const int idx = col + i * step_size;\n local_vals[i] = idx < p.ncols ? dst[idx] : -INFINITY;\n }\n#pragma unroll\n for (int i = 0; i < n_elem_per_thread; i++) {\n const int idx = col + i * step_size;\n if (idx < p.ncols) {\n dst[idx] = local_vals[i] / tmp_expf;\n }\n }\n col += step_size * n_elem_per_thread;\n }\n}\n\n#ifdef __clang__\n#pragma clang diagnostic pop\n#endif // __clang__\n\nstatic __global__ void soft_max_back_f32(\n const float * grad, const float * dstf, float * dst, const int ncols, const float scale) {\n const int tid = threadIdx.x;\n const int rowx = blockIdx.x;\n\n grad += int64_t(rowx)*ncols;\n dstf += int64_t(rowx)*ncols;\n dst += int64_t(rowx)*ncols;\n\n float dgf_dot = 0.0f; // dot product of dst from forward pass and gradients\n\n for (int col = tid; col < ncols; col += WARP_SIZE) {\n dgf_dot += dstf[col]*grad[col];\n }\n\n dgf_dot = warp_reduce_sum(dgf_dot);\n\n for (int col = tid; col < ncols; col += WARP_SIZE) {\n dst[col] = scale * (grad[col] - dgf_dot) * dstf[col];\n }\n}\n\ntemplate\nstatic void launch_soft_max_kernels(const float * x, const T * mask, const float * sinks, float * dst,\n const soft_max_params & p, cudaStream_t stream, dim3 block_dims, dim3 block_nums, size_t nbytes_shared)\n{\n const int id = ggml_cuda_get_device();\n const size_t smpbo = ggml_cuda_info().devices[id].smpbo;\n\n auto launch_kernel = [=](auto I) -> bool {\n constexpr int ncols = decltype(I)::value;\n constexpr int block = (ncols > 1024 ? 1024 : ncols);\n\n if (p.ncols == ncols) {\n CUDA_SET_SHARED_MEMORY_LIMIT((soft_max_f32), smpbo);\n soft_max_f32<<>>\n (x, mask, sinks, dst, p);\n return true;\n }\n return false;\n };\n\n // unary fold over launch_kernel\n if ((launch_kernel(std::integral_constant{}) || ...)) {\n return;\n }\n\n //default case\n CUDA_SET_SHARED_MEMORY_LIMIT((soft_max_f32), smpbo);\n soft_max_f32<<>>(x, mask, sinks, dst, p);\n}\n\n__launch_bounds__(8*WARP_SIZE, 1) static __global__ void soft_max_f32_parallelize_cols(const float * __restrict__ x,\n float * __restrict__ dst,\n float * __restrict__ tmp_maxs,\n float * __restrict__ tmp_sums,\n const soft_max_params p)\n// We loop over all instead of parallelizing across gridDim.y as cooperative groups\n// currently only support synchronizing the complete grid if not launched as a cluster group\n// (which requires CC > 9.0)\n// https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#grid-synchronization\n// https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#class-cluster-group\n{\n for (int rowx = 0; rowx < p.ne01 * p.ne02 * p.ne03; rowx++) {\n soft_max_f32_parallelize_cols_single_row(x + int64_t(rowx) * p.ncols, dst + int64_t(rowx) * p.ncols, tmp_maxs,\n tmp_sums, p);\n }\n}\n\ntemplate \nstatic void soft_max_f32_cuda(const float * x,\n const T * mask,\n const float * sinks,\n float * dst,\n const soft_max_params & params,\n cudaStream_t stream,\n [[maybe_unused]] ggml_backend_cuda_context & ctx) {\n int nth = WARP_SIZE;\n const int64_t ncols_x = params.ncols;\n\n while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;\n const dim3 block_dims(nth, 1, 1);\n const dim3 block_nums(params.ne01, params.ne02, params.ne03);\n const size_t nbytes_shared = (GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE)*sizeof(float);\n static_assert(CUDA_SOFT_MAX_BLOCK_SIZE == 1024, \"These values need to be adjusted.\");\n\n\n const int id = ggml_cuda_get_device();\n const size_t smpbo = ggml_cuda_info().devices[id].smpbo;\n\n\n if (nbytes_shared <= smpbo) {\n launch_soft_max_kernels<32, 64, 128, 256, 512, 1024, 2048, 4096>(x, mask, sinks, dst, params, stream, block_dims, block_nums, nbytes_shared);\n } else {\n // Parallelize across SMs for top-p/dist-sampling\n // The heuristic for parallelizing rows across SMs vs parallelizing single row & looping over all rows was done on the basis of a B6000 GPU and\n // Can be adapted further for lower-SM-count GPUs, though keeping data in registers should be implemented first as that is the optimal solution.\n if (ggml_cuda_info().devices[id].supports_cooperative_launch &&\n ncols_x / (params.ne01 * params.ne02 * params.ne03) > 8192 && mask == nullptr && sinks == nullptr &&\n params.scale == 1.0f && params.max_bias == 0.0f) {\n ggml_cuda_pool_alloc tmp_maxs_alloc(ctx.pool(), ggml_cuda_info().devices[id].nsm * sizeof(float));\n ggml_cuda_pool_alloc tmp_sums_alloc(ctx.pool(), ggml_cuda_info().devices[id].nsm * sizeof(float));\n\n void * kernel_args[] = { (void *) &x, (void *) &dst, (void *) &tmp_maxs_alloc.ptr,\n (void *) &tmp_sums_alloc.ptr, (void *) const_cast(¶ms) };\n CUDA_CHECK(cudaLaunchCooperativeKernel((void *) soft_max_f32_parallelize_cols,\n dim3(ggml_cuda_info().devices[id].nsm, 1, 1),\n dim3(WARP_SIZE * 8, 1, 1), kernel_args, 0, stream));\n } else {\n const size_t nbytes_shared_low = WARP_SIZE * sizeof(float);\n soft_max_f32\n <<>>(x, mask, sinks, dst, params);\n }\n }\n}\n\nstatic void soft_max_back_f32_cuda(\n const float * grad, const float * dstf, float * dst,\n const int ncols, const int nrows, const float scale, cudaStream_t stream) {\n const dim3 block_dims(WARP_SIZE, 1, 1);\n const dim3 block_nums(nrows, 1, 1);\n\n soft_max_back_f32<<>>(grad, dstf, dst, ncols, scale);\n}\n\nvoid ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const ggml_tensor * src2 = dst->src[2];\n\n const float * src0_d = (const float *) src0->data;\n const void * src1_d = src1 ? (const void *) src1->data : nullptr;\n const void * src2_d = src2 ? (const void *) src2->data : nullptr;\n float * dst_d = (float *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional\n\n const int64_t nrows_x = ggml_nrows(src0);\n const int64_t nrows_y = src0->ne[1];\n\n const int64_t ne00 = src0->ne[0];\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n\n memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float));\n memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));\n\n const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);\n\n const int64_t nb11 = src1 ? src1->nb[1] : 1;\n const int64_t nb12 = src1 ? src1->nb[2] : 1;\n const int64_t nb13 = src1 ? src1->nb[3] : 1;\n\n const int64_t ne12 = src1 ? src1->ne[2] : 1;\n const int64_t ne13 = src1 ? src1->ne[3] : 1;\n\n const uint32_t n_head = src0->ne[2];\n const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));\n\n const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n\n soft_max_params params = {};\n params.nheads = src0->ne[2];\n params.n_head_log2 = n_head_log2;\n params.ncols = ne00;\n params.nrows_x = nrows_x;\n params.nrows_y = nrows_y;\n params.ne00 = src0->ne[0];\n params.ne01 = src0->ne[1];\n params.ne02 = src0->ne[2];\n params.ne03 = src0->ne[3];\n params.nb11 = nb11;\n params.nb12 = nb12;\n params.nb13 = nb13;\n params.ne12 = ne12;\n params.ne13 = ne13;\n params.scale = scale;\n params.max_bias = max_bias;\n params.m0 = m0;\n params.m1 = m1;\n\n if (use_f16) {\n soft_max_f32_cuda(src0_d, (const half *) src1_d, (const float *) src2_d, dst_d, params, stream, ctx);\n } else {\n soft_max_f32_cuda(src0_d, (const float *) src1_d, (const float *) src2_d, dst_d, params, stream, ctx);\n }\n}\n\nvoid ggml_cuda_op_soft_max_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0]; // grad\n const ggml_tensor * src1 = dst->src[1]; // forward pass output\n\n const float * src0_d = (const float *) src0->data;\n const float * src1_d = (const float *) src1->data;\n float * dst_d = (float *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n const int64_t ncols = src0->ne[0];\n const int64_t nrows = ggml_nrows(src0);\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n\n memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float));\n memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));\n\n GGML_ASSERT(max_bias == 0.0f);\n\n soft_max_back_f32_cuda(src0_d, src1_d, dst_d, ncols, nrows, scale, stream);\n}\n" }, { "path": "src/ggml-cuda/softmax.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_SOFT_MAX_BLOCK_SIZE 1024\n\nvoid ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_soft_max_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/solve_tri.cu", "content": "#include \"common.cuh\"\n#include \"ggml.h\"\n#include \"solve_tri.cuh\"\n\n#define MAX_N_FAST 64\n#define MAX_K_FAST 32\n\nstatic __global__ void get_batch_pointers(const float * A,\n float * X,\n const float ** A_ptrs,\n float ** X_ptrs,\n int64_t ne02,\n int64_t total_batches,\n size_t s02,\n size_t s03,\n size_t s2,\n size_t s3) {\n const int idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (idx >= total_batches) {\n return;\n }\n\n const int64_t i3 = idx / ne02;\n const int64_t i2 = idx % ne02;\n\n A_ptrs[idx] = A + i3 * s03 + i2 * s02;\n X_ptrs[idx] = X + i3 * s3 + i2 * s2;\n}\n\nstatic void solve_tri_f32_cublas(ggml_backend_cuda_context & ctx,\n const float * A,\n const float * B,\n float * X,\n int n,\n int k,\n int64_t ne02,\n int64_t ne03,\n size_t s02,\n size_t s03,\n size_t s12,\n size_t s13,\n size_t s2,\n size_t s3,\n cudaStream_t stream) {\n const float alpha = 1.0f;\n const int64_t total_batches = ne02 * ne03;\n if (total_batches == 0) {\n return;\n }\n\n // Bulk copy B -> X (contiguous tensors)\n if (X != B) {\n const int64_t total_elements_BX = n * k * total_batches;\n CUDA_CHECK(cudaMemcpyAsync(X, B, total_elements_BX * sizeof(float), cudaMemcpyDeviceToDevice, stream));\n }\n\n const int id = ggml_cuda_get_device();\n\n ggml_cuda_pool_alloc A_ptrs_alloc(ctx.pool(id), total_batches);\n ggml_cuda_pool_alloc X_ptrs_alloc(ctx.pool(id), total_batches);\n\n const float ** A_ptrs_dev = A_ptrs_alloc.get();\n float ** X_ptrs_dev = X_ptrs_alloc.get();\n\n get_batch_pointers<<<(total_batches + 255) / 256, 256, 0, stream>>>(A, X, A_ptrs_dev, X_ptrs_dev, ne02,\n total_batches, s02, s03, s2, s3);\n\n CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));\n\n // Yes, this is necessary, without this we get RMSE errors\n CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_DEFAULT_MATH));\n CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(id), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N,\n CUBLAS_DIAG_NON_UNIT, k, n, &alpha, A_ptrs_dev, n, X_ptrs_dev, k, total_batches));\n\n // revert to standard mode from common.cuh\n CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_TF32_TENSOR_OP_MATH));\n\n GGML_UNUSED_VARS(s12, s13);\n}\n\n// ======================\n// Fast Kernel (n <= 64, k <= 32) - Warp-based parallel reduction\n// ======================\n// When ncols_template == 0 the bounds for the loops in this function are not\n// known and can't be unrolled. As we want to keep pragma unroll for all other\n// cases we suppress the clang transformation warning here.\n#ifdef __clang__\n# pragma clang diagnostic push\n# pragma clang diagnostic ignored \"-Wpass-failed\"\n#endif // __clang__\ntemplate \nstatic __global__ void solve_tri_f32_fast(const float * __restrict__ A,\n const float * __restrict__ B,\n float * __restrict__ X,\n const uint3 ne02,\n const size_t nb02,\n const size_t nb03,\n const size_t nb12,\n const size_t nb13,\n const size_t nb2,\n const size_t nb3,\n const int n_arg,\n const int k_arg) {\n const int n = n_template == 0 ? n_arg : n_template;\n const int k = k_template == 0 ? k_arg : k_template;\n\n const int batch_idx = blockIdx.x;\n const int lane = threadIdx.x;\n const int col_idx = threadIdx.y;\n\n if (col_idx >= k) {\n return;\n }\n\n const uint2 i02_i03 = fast_div_modulo(batch_idx, ne02);\n const int64_t i02 = i02_i03.y;\n const int64_t i03 = i02_i03.x;\n\n const float * const A_batch = (const float *) (A + i02 * nb02 + i03 * nb03);\n const float * const B_batch = (const float *) (B + i02 * nb12 + i03 * nb13);\n float * X_batch = (float *) (X + i02 * nb2 + i03 * nb3);\n\n __shared__ float sA[MAX_N_FAST * MAX_N_FAST];\n\n const int offset = threadIdx.x + threadIdx.y * blockDim.x;\n\n#pragma unroll\n for (int i = 0; i < n * n; i += k * WARP_SIZE) {\n const int i0 = i + offset;\n if (i0 < n * n) {\n sA[i0] = A_batch[i0];\n }\n }\n\n __syncthreads();\n\n float x_low = (lane < n) ? B_batch[lane * k + col_idx] : 0.0f;\n float x_high = (WARP_SIZE + lane < n) ? B_batch[(WARP_SIZE + lane) * k + col_idx] : 0.0f;\n\n const int half = WARP_SIZE;\n const int nrows_low = (n < half) ? n : half;\n\n#pragma unroll\n for (int row = 0; row < nrows_low; ++row) {\n float sum = 0.0f;\n if (lane < row) {\n sum += sA[row * n + lane] * x_low;\n }\n sum = warp_reduce_sum(sum);\n\n if (lane == row) {\n x_low = (x_low - sum) / sA[row * n + row];\n }\n }\n\n#pragma unroll\n for (int row = half; row < n; ++row) {\n float sum = sA[row * n + lane] * x_low;\n const int j = half + lane;\n if (j < row) {\n sum += sA[row * n + j] * x_high;\n }\n sum = warp_reduce_sum(sum);\n\n if (lane == row - half) {\n x_high = (x_high - sum) / sA[row * n + row];\n }\n }\n\n#pragma unroll\n for (int rr = 0; rr < 2; ++rr) {\n const int row = rr * WARP_SIZE + lane;\n if (row < n) {\n const float val = (row < half) ? x_low : x_high;\n X_batch[row * k + col_idx] = val;\n }\n }\n}\n#ifdef __clang__\n# pragma clang diagnostic pop\n#endif // __clang__\n\nstatic void solve_tri_f32_cuda(const float * A,\n const float * B,\n float * X,\n int n,\n int k,\n int64_t ne02,\n int64_t ne03,\n size_t nb02,\n size_t nb03,\n size_t nb12,\n size_t nb13,\n size_t nb2,\n size_t nb3,\n cudaStream_t stream) {\n const uint3 ne02_fd = init_fastdiv_values((uint32_t) ne02);\n dim3 threads(WARP_SIZE, k);\n dim3 grid(ne02 * ne03);\n if (n == 64) {\n switch (k) {\n case 32:\n solve_tri_f32_fast<64, 32>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n case 16:\n solve_tri_f32_fast<64, 16>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n case 14:\n solve_tri_f32_fast<64, 14>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n case 12:\n solve_tri_f32_fast<64, 12>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n case 10:\n solve_tri_f32_fast<64, 10>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n case 8:\n solve_tri_f32_fast<64, 8>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n case 6:\n solve_tri_f32_fast<64, 6>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n case 4:\n solve_tri_f32_fast<64, 4>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n case 2:\n solve_tri_f32_fast<64, 2>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n case 1:\n solve_tri_f32_fast<64, 1>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, 0, 0);\n break;\n default:\n solve_tri_f32_fast<0, 0>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, n, k);\n }\n } else { // run general case\n solve_tri_f32_fast<0, 0>\n <<>>(A, B, X, ne02_fd, nb02, nb03, nb12, nb13, nb2, nb3, n, k);\n }\n}\n\nvoid ggml_cuda_op_solve_tri(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0]; // A (n×n, lower triangular)\n const ggml_tensor * src1 = dst->src[1]; // B (n×k)\n\n ggml_is_contiguous(src0);\n ggml_is_contiguous(src1);\n\n const int64_t n = src0->ne[0];\n const int64_t k = src1->ne[0];\n const int64_t ne02 = src0->ne[2];\n const int64_t ne03 = src0->ne[3];\n\n if (n <= MAX_N_FAST && k <= MAX_K_FAST) {\n solve_tri_f32_cuda((const float *) src0->data, (const float *) src1->data, (float *) dst->data, n, k,\n src0->ne[2], src0->ne[3], src0->nb[2] / sizeof(float), src0->nb[3] / sizeof(float),\n src1->nb[2] / sizeof(float), src1->nb[3] / sizeof(float), dst->nb[2] / sizeof(float),\n dst->nb[3] / sizeof(float), ctx.stream());\n } else {\n solve_tri_f32_cublas(ctx, (const float *) src0->data, (const float *) src1->data, (float *) dst->data, n, k,\n ne02, ne03, src0->nb[2] / sizeof(float), src0->nb[3] / sizeof(float),\n src1->nb[2] / sizeof(float), src1->nb[3] / sizeof(float), dst->nb[2] / sizeof(float),\n dst->nb[3] / sizeof(float), ctx.stream());\n }\n}\n" }, { "path": "src/ggml-cuda/solve_tri.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_solve_tri(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/ssm-conv.cu", "content": "#include \"ssm-conv.cuh\"\n#include \"unary.cuh\"\n\ntemplate \nstatic __global__ void ssm_conv_f32(const float * __restrict__ src0, const float * __restrict__ src1,\n const int src0_nb0, const int src0_nb1, const int src0_nb2, const int src1_nb1,\n float * __restrict__ dst, const int dst_nb0, const int dst_nb1, const int dst_nb2,\n const int64_t n_t) {\n GGML_UNUSED(src0_nb0);\n const int tid = threadIdx.x;\n const int bidx = blockIdx.x;\n const int bidy = blockIdx.y;\n\n const float * x_block = (const float *) ((const char *) src0 + bidx * src0_nb2 + bidy * split_d_inner * src0_nb1);\n const float * w_block = (const float *) ((const char *) src1 + bidy * split_d_inner * src1_nb1);\n float * y_block = (float *) ((char *) dst + bidx * dst_nb2 + bidy * split_d_inner * dst_nb0);\n\n const int stride_x = src0_nb1 / sizeof(float);\n const int stride_w = src1_nb1 / sizeof(float);\n const int stride_y = dst_nb1 / sizeof(float);\n\n float x[d_conv] = { 0.0f };\n float w[d_conv] = { 0.0f };\n\n#pragma unroll\n for (size_t j = 0; j < d_conv; j++) {\n w[j] = w_block[tid * stride_w + j];\n }\n\n for (int64_t i = 0; i < n_t; i++) {\n float sumf = 0.0f;\n\n if (i == 0) {\n for (size_t j = 0; j < d_conv; j++) {\n x[j] = x_block[tid * stride_x + j];\n }\n } else {\n x[(i - 1) % d_conv] = x_block[tid * stride_x + i + d_conv - 1];\n }\n\n#pragma unroll\n for (size_t j = 0; j < d_conv; j++) {\n sumf += x[(i + j) % d_conv] * w[j];\n }\n y_block[i * stride_y + tid] = apply_silu ? ggml_cuda_op_silu_single(sumf) : sumf;\n }\n}\n\ntemplate \nstatic __global__ void ssm_conv_long_token_f32(const float * __restrict__ src0, const float * __restrict__ src1,\n const int src0_nb0, const int src0_nb1, const int src0_nb2,\n const int src1_nb1, float * __restrict__ dst, const int dst_nb0,\n const int dst_nb1, const int dst_nb2, const int64_t n_t) {\n const int tid = threadIdx.x;\n const int bidx = blockIdx.x;\n const int bidy = blockIdx.y;\n const int bidz = blockIdx.z;\n\n const float * x_block = (const float *) ((const char *) src0 + bidx * src0_nb2 + bidy * split_d_inner * src0_nb1 +\n bidz * split_n_t * src0_nb0);\n const float * w_block = (const float *) ((const char *) src1 + bidy * split_d_inner * src1_nb1);\n float * y_block =\n (float *) ((char *) dst + bidx * dst_nb2 + bidz * split_n_t * dst_nb1 + bidy * split_d_inner * dst_nb0);\n\n const int stride_x = src0_nb1 / sizeof(float);\n const int stride_w = src1_nb1 / sizeof(float);\n const int stride_y = dst_nb1 / sizeof(float);\n\n const int64_t local_n_t = min(split_n_t, n_t - bidz * split_n_t);\n const int n_cols = d_conv - 1 + split_n_t;\n\n extern __shared__ float smem[];\n\n constexpr int load_cols = d_conv - 1 + split_n_t;\n constexpr int total_elems = split_d_inner * load_cols;\n int row = tid / load_cols;\n int col = tid % load_cols;\n#pragma unroll\n for (int idx = 0; idx < total_elems; idx += split_d_inner) {\n if (row < (int)split_d_inner) {\n smem[row * n_cols + col] = x_block[row * stride_x + col];\n }\n\n col += split_d_inner;\n row += col / load_cols;\n col = col % load_cols;\n if (idx >= total_elems - tid - split_d_inner) {\n break;\n }\n }\n __syncthreads();\n\n // Load weights into registers (done once, small)\n float w[d_conv] = { 0.0f };\n#pragma unroll\n for (size_t j = 0; j < d_conv; j++) {\n w[j] = w_block[tid * stride_w + j];\n }\n\n // Compute from shared memory\n for (int64_t i = 0; i < local_n_t; i++) {\n float sumf = 0.0f;\n#pragma unroll\n for (size_t j = 0; j < d_conv; j++) {\n sumf += smem[tid * n_cols + i + j] * w[j];\n }\n y_block[i * stride_y + tid] = apply_silu ? ggml_cuda_op_silu_single(sumf) : sumf;\n }\n}\n\ntemplate \nstatic void ssm_conv_f32_cuda(const float * src0, const float * src1, const int src0_nb0, const int src0_nb1,\n const int src0_nb2, const int src1_nb1, float * dst, const int dst_nb0, const int dst_nb1,\n const int dst_nb2, const int64_t nc, const int64_t nr, const int64_t n_t,\n const int64_t n_s, cudaStream_t stream) {\n const int threads = 128;\n GGML_ASSERT(nr % threads == 0);\n\n auto launch_kernel = [&](auto NC) {\n constexpr int kNC = decltype(NC)::value;\n if (n_t <= 32) {\n const dim3 blocks(n_s, (nr + threads - 1) / threads, 1);\n ssm_conv_f32<<>>(src0, src1, src0_nb0, src0_nb1, src0_nb2, src1_nb1,\n dst, dst_nb0, dst_nb1, dst_nb2, n_t);\n } else {\n const int64_t split_n_t = 32;\n dim3 blocks(n_s, (nr + threads - 1) / threads, (n_t + split_n_t - 1) / split_n_t);\n const size_t smem_size = threads * (kNC - 1 + split_n_t) * sizeof(float);\n ssm_conv_long_token_f32<<>>(\n src0, src1, src0_nb0, src0_nb1, src0_nb2, src1_nb1, dst, dst_nb0, dst_nb1, dst_nb2, n_t);\n }\n };\n\n switch (nc) {\n case 3: launch_kernel(std::integral_constant{}); break;\n case 4: launch_kernel(std::integral_constant{}); break;\n case 9: launch_kernel(std::integral_constant{}); break;\n default: GGML_ABORT(\"Only support kernel sizes 3, 4, 9 right now.\");\n }\n}\n\nvoid ggml_cuda_op_ssm_conv(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * silu_dst) {\n const struct ggml_tensor * src0 = dst->src[0]; // conv_x\n const struct ggml_tensor * src1 = dst->src[1]; // conv1d.weight\n const bool fuse_silu = silu_dst != nullptr;\n\n // When fusing, write to silu_dst (the node downstream references).\n const struct ggml_tensor * out = fuse_silu ? silu_dst : dst;\n\n const int64_t nc = src1->ne[0]; // d_conv\n const int64_t nr = src0->ne[1]; // d_inner\n const int64_t n_t = out->ne[1]; // tokens per sequence\n const int64_t n_s = out->ne[2]; // number of sequences in the batch\n\n GGML_ASSERT(out->ne[0] == nr);\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n GGML_ASSERT(src1->nb[0] == sizeof(float));\n GGML_ASSERT(src0->nb[1] == src0->ne[0] * sizeof(float));\n\n const float * src0_d = (const float *) src0->data;\n const float * src1_d = (const float *) src1->data;\n float * dst_d = (float *) out->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(out->type == GGML_TYPE_F32);\n if (fuse_silu) {\n ssm_conv_f32_cuda(src0_d, src1_d, src0->nb[0], src0->nb[1], src0->nb[2], src1->nb[1], dst_d, out->nb[0], out->nb[1],\n out->nb[2], nc, nr, n_t, n_s, stream);\n } else {\n ssm_conv_f32_cuda(src0_d, src1_d, src0->nb[0], src0->nb[1], src0->nb[2], src1->nb[1], dst_d, out->nb[0], out->nb[1],\n out->nb[2], nc, nr, n_t, n_s, stream);\n }\n}\n" }, { "path": "src/ggml-cuda/ssm-conv.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_ssm_conv(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * silu_dst = nullptr);\n" }, { "path": "src/ggml-cuda/ssm-scan.cu", "content": "#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070\n#define USE_CUB\n#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070\n\n#ifdef USE_CUB\n#include \nusing namespace cub;\n#endif // USE_CUB\n\n#include \"ssm-scan.cuh\"\n\n// We would like to keep pragma unroll for cases where L_template is not 0,\n// so we suppress the clang transformation warning.\n#ifdef __clang__\n#pragma clang diagnostic push\n#pragma clang diagnostic ignored \"-Wpass-failed\"\n#endif // __clang__\ntemplate \n__global__ void __launch_bounds__(splitD, 1)\n ssm_scan_f32(const float *__restrict__ src0, const float *__restrict__ src1, const float *__restrict__ src2,\n const float *__restrict__ src3, const float *__restrict__ src4, const float *__restrict__ src5,\n const int32_t * __restrict__ src6, float * __restrict__ dst,\n const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3,\n const int src2_nb1, const int src2_nb2, const int src3_nb1,\n const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3,\n const int64_t s_off, const int64_t d_inner, const int64_t L_param)\n{\n const size_t L = L_template == 0 ? L_param : L_template;\n const float *s0_block = (const float *)((const char *)src0 + src6[blockIdx.x] * src0_nb3 + blockIdx.y * splitD * src0_nb2);\n const float *x_block = (const float *)((const char *)src1 + (blockIdx.x * src1_nb3) + blockIdx.y * splitD * sizeof(float));\n const float *dt_block = (const float *)((const char *)src2 + (blockIdx.x * src2_nb2) + blockIdx.y * splitD * sizeof(float));\n const float *A_block = (const float *)((const char *)src3 + blockIdx.y * splitD * src3_nb1);\n const float *B_block = (const float *)((const char *)src4 + (blockIdx.x * src4_nb3));\n const float *C_block = (const float *)((const char *)src5 + (blockIdx.x * src5_nb3));\n float *y_block = (float *)((char *)dst + (blockIdx.x * d_inner * L * sizeof(float)) + blockIdx.y * splitD * sizeof(float));\n float *s_block = (float *)((char *)dst + s_off + blockIdx.x * src0_nb3 + blockIdx.y * splitD * src0_nb2);\n\n const int stride_x = src1_nb2 / sizeof(float);\n const int stride_dt = src2_nb1 / sizeof(float);\n const int stride_B = src4_nb2 / sizeof(float);\n const int stride_C = src5_nb2 / sizeof(float);\n const int stride_y = d_inner;\n\n float regA[N];\n float regs0[N];\n\n __shared__ float smemB[N];\n __shared__ float smemC[N];\n\n#ifdef USE_CUB\n using BlockLoad = cub::BlockLoad;\n using BlockStore = cub::BlockStore;\n\n union CubTempStorage {\n typename BlockLoad::TempStorage load_temp;\n typename BlockStore::TempStorage store_temp;\n };\n __shared__ CubTempStorage cub_temp_storage;\n\n BlockLoad(cub_temp_storage.load_temp).Load(A_block, regA);\n BlockLoad(cub_temp_storage.load_temp).Load(s0_block, regs0);\n#else\n const int stride_s0 = src0_nb2 / sizeof(float);\n const int stride_A = src3_nb1 / sizeof(float);\n#pragma unroll\n for (size_t n = 0; n < N; ++n)\n {\n regA[n] = A_block[threadIdx.x * stride_A + n];\n regs0[n] = s0_block[threadIdx.x * stride_s0 + n];\n }\n#endif\n\n#pragma unroll\n for (size_t i = 0; i < L; i++)\n {\n if (threadIdx.x < N)\n {\n smemB[threadIdx.x] = B_block[i * stride_B + threadIdx.x];\n smemC[threadIdx.x] = C_block[i * stride_C + threadIdx.x];\n }\n __syncthreads();\n\n float dt_soft_plus = dt_block[i * stride_dt + threadIdx.x];\n if (dt_soft_plus <= 20.0f)\n {\n dt_soft_plus = log1pf(expf(dt_soft_plus));\n }\n float x_dt = x_block[i * stride_x + threadIdx.x] * dt_soft_plus;\n\n float sumf = 0.0f;\n#pragma unroll\n for (size_t n = 0; n < N; n++)\n {\n float state = regs0[n] * expf(dt_soft_plus * regA[n]) + smemB[n] * x_dt;\n sumf += state * smemC[n];\n regs0[n] = state;\n }\n y_block[i * stride_y + threadIdx.x] = sumf;\n }\n\n#ifdef USE_CUB\n BlockStore(cub_temp_storage.store_temp).Store(s_block, regs0);\n#else\n const int stride_s = stride_s0;\n#pragma unroll\n for (size_t n = 0; n < N; ++n)\n {\n s_block[threadIdx.x * stride_s + n] = regs0[n];\n }\n#endif\n}\n#ifdef __clang__\n#pragma clang diagnostic pop\n#endif // __clang__\n\n// assumes as many threads as d_state\ntemplate \n__global__ void __launch_bounds__(d_state, 1)\n ssm_scan_f32_group(\n const float * __restrict__ src0, const float * __restrict__ src1, const float * __restrict__ src2,\n const float * __restrict__ src3, const float * __restrict__ src4, const float * __restrict__ src5,\n const int32_t * __restrict__ src6, float * __restrict__ dst,\n const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3,\n const int src2_nb1, const int src2_nb2, const int src3_nb1,\n const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3,\n const int64_t s_off, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok) {\n\n const int warp = threadIdx.x / WARP_SIZE;\n const int lane = threadIdx.x % WARP_SIZE;\n const int warp_idx = blockIdx.x * c_factor + warp;\n\n const int head_idx = warp_idx / d_head;\n const int head_off = (warp_idx % d_head) * sizeof(float);\n const int seq_idx = blockIdx.y;\n\n const int group_off = (head_idx / (n_head / n_group)) * d_state * sizeof(float);\n\n // TODO: refactor strides to be in elements/floats instead of bytes to be cleaner and consistent with the rest of the codebase\n const float * s0_warp = (const float *) ((const char *) src0 + src6[seq_idx] * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);\n const float * x_warp = (const float *) ((const char *) src1 + (seq_idx * src1_nb3) + (warp_idx * sizeof(float)));\n const float * dt_warp = (const float *) ((const char *) src2 + (seq_idx * src2_nb2) + head_idx * sizeof(float));\n const float * A_warp = (const float *) ((const char *) src3 + head_idx * src3_nb1);\n const float * B_warp = (const float *) ((const char *) src4 + (seq_idx * src4_nb3) + (group_off));\n const float * C_warp = (const float *) ((const char *) src5 + (seq_idx * src5_nb3) + (group_off));\n float * y_warp = dst + (seq_idx * n_tok * n_head * d_head) + warp_idx;\n float * s_warp = (float *) ((char *) dst + s_off + seq_idx * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);\n\n // strides across n_seq_tokens\n const int stride_x = src1_nb2 / sizeof(float);\n const int stride_dt = src2_nb1 / sizeof(float);\n const int stride_B = src4_nb2 / sizeof(float);\n const int stride_C = src5_nb2 / sizeof(float);\n const int stride_y = n_head * d_head;\n\n float state[c_factor];\n float state_sum = 0.0f;\n\n#pragma unroll\n for (int j = 0; j < c_factor; j++) {\n state[j] = s0_warp[WARP_SIZE * j + lane];\n }\n\n for (int64_t i = 0; i < n_tok; i++) {\n // NOTE: dt_soft_plus, dA and x_dt have the same value for a warp here.\n // Recalculation is intentional; sharing via shuffles/smem proved slower due to sync overhead.\n const float dt_soft_plus = (dt_warp[i * stride_dt] <= 20.0f ? log1pf(expf(dt_warp[i * stride_dt])) : dt_warp[i * stride_dt]);\n\n state_sum = 0.0f;\n const float dA = expf(dt_soft_plus * A_warp[0]);\n const float x_dt = x_warp[i * stride_x] * dt_soft_plus;\n#pragma unroll\n for (int j = 0; j < c_factor; j++) {\n const float B_val = B_warp[i * stride_B + WARP_SIZE * j + lane];\n const float C_val = C_warp[i * stride_C + WARP_SIZE * j + lane];\n state[j] = (state[j] * dA) + (B_val * x_dt);\n state_sum += state[j] * C_val;\n }\n\n // parallel accumulation for output\n state_sum = warp_reduce_sum(state_sum);\n\n if (lane == 0) {\n y_warp[i * stride_y] = state_sum;\n }\n }\n\n // write back the state\n#pragma unroll\n for (int j = 0; j < c_factor; j++) {\n s_warp[WARP_SIZE * j + lane] = state[j];\n }\n}\n\nstatic void ssm_scan_f32_cuda(const float * src0, const float * src1, const float * src2, const float * src3,\n const float * src4, const float * src5, const int32_t * src6, float * dst,\n const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3, const int src2_nb1,\n const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2,\n const int src5_nb3, const int64_t s_off, const int64_t d_state, const int64_t head_dim,\n const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq,\n cudaStream_t stream) {\n // NOTE: if you change conditions here, be sure to update the corresponding supports_op condition!\n if (src3_nb1 == sizeof(float)) {\n // Mamba-2\n if (d_state == 128) {\n constexpr int threads = 128;\n constexpr int num_warps = threads/WARP_SIZE;\n\n const dim3 blocks((n_head * head_dim + (num_warps - 1)) / num_warps, n_seq, 1);\n ssm_scan_f32_group<128/WARP_SIZE, 128><<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,\n src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok);\n } else if (d_state == 256) { // Falcon-H1\n constexpr int threads = 256;\n constexpr int num_warps = threads/WARP_SIZE;\n\n const dim3 blocks((n_head * head_dim + (num_warps - 1)) / num_warps, n_seq, 1);\n ssm_scan_f32_group<256/WARP_SIZE, 256><<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,\n src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok);\n } else {\n GGML_ABORT(\"doesn't support d_state!=(128 or 256).\");\n }\n } else {\n // Mamba-1\n constexpr int threads = 128;\n GGML_ASSERT(n_head % threads == 0);\n GGML_ASSERT(head_dim == 1);\n GGML_ASSERT(n_group == 1);\n const dim3 blocks(n_seq, (n_head + threads - 1) / threads, 1);\n const int smem_size = (threads * (d_state + 1) * 2) * sizeof(float);\n if (d_state == 16) {\n switch (n_tok)\n {\n case 1:\n ssm_scan_f32<<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2,\n src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok);\n break;\n case 2:\n ssm_scan_f32<<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2,\n src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok);\n break;\n case 3:\n ssm_scan_f32<<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2,\n src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok);\n break;\n case 4:\n ssm_scan_f32<<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2,\n src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok);\n break;\n case 5:\n ssm_scan_f32<<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2,\n src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok);\n break;\n case 6:\n ssm_scan_f32<<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2,\n src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok);\n break;\n case 7:\n ssm_scan_f32<<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2,\n src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok);\n break;\n case 8:\n ssm_scan_f32<<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2,\n src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok);\n break;\n default:\n ssm_scan_f32<<>>(\n src0, src1, src2, src3, src4, src5, src6, dst,\n src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2,\n src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok);\n break;\n }\n } else {\n GGML_ABORT(\"doesn't support d_state!=16.\");\n }\n }\n}\n\nvoid ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const struct ggml_tensor * src0 = dst->src[0]; // s\n const struct ggml_tensor * src1 = dst->src[1]; // x\n const struct ggml_tensor * src2 = dst->src[2]; // dt\n const struct ggml_tensor * src3 = dst->src[3]; // A\n const struct ggml_tensor * src4 = dst->src[4]; // B\n const struct ggml_tensor * src5 = dst->src[5]; // C\n const struct ggml_tensor * src6 = dst->src[6]; // ids\n\n const int64_t nc = src0->ne[0]; // d_state\n const int64_t nr = src0->ne[1]; // head_dim or 1\n const int64_t nh = src1->ne[1]; // n_head\n const int64_t ng = src4->ne[1]; // n_group\n const int64_t n_t = src1->ne[2]; // number of tokens per sequence\n const int64_t n_s = src1->ne[3]; // number of sequences in the batch\n\n const int64_t s_off = ggml_nelements(src1) * sizeof(float);\n\n GGML_ASSERT(ggml_nelements(src1) + nc*nr*nh*n_s == ggml_nelements(dst));\n GGML_ASSERT(src0->nb[0] == sizeof(float));\n GGML_ASSERT(src1->nb[0] == sizeof(float));\n GGML_ASSERT(src2->nb[0] == sizeof(float));\n GGML_ASSERT(src3->nb[0] == sizeof(float));\n GGML_ASSERT(src4->nb[0] == sizeof(float));\n GGML_ASSERT(src5->nb[0] == sizeof(float));\n GGML_ASSERT(src6->nb[0] == sizeof(int32_t));\n\n const float * src0_d = (const float *) src0->data;\n const float * src1_d = (const float *) src1->data;\n const float * src2_d = (const float *) src2->data;\n const float * src3_d = (const float *) src3->data;\n const float * src4_d = (const float *) src4->data;\n const float * src5_d = (const float *) src5->data;\n const int32_t * src6_d = (const int32_t *) src6->data;\n float * dst_d = (float *) dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src6->type == GGML_TYPE_I32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n ssm_scan_f32_cuda(src0_d, src1_d, src2_d, src3_d, src4_d, src5_d, src6_d, dst_d,\n src0->nb[2], src0->nb[3], src1->nb[2], src1->nb[3], src2->nb[1], src2->nb[2],\n src3->nb[1], src4->nb[2], src4->nb[3], src5->nb[2], src5->nb[3],\n s_off, nc, nr, nh, ng, n_t, n_s, stream);\n}\n" }, { "path": "src/ggml-cuda/ssm-scan.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/sum.cu", "content": "#include \"sum.cuh\"\n#include \"sumrows.cuh\"\n\n#ifdef GGML_CUDA_USE_CUB\n#include \nusing namespace cub;\n#endif // GGML_CUDA_USE_CUB\n\n#include \n\nvoid sum_f32_cuda(ggml_cuda_pool & pool, const float * x, float * dst, const int64_t ne, cudaStream_t stream) {\n#ifdef GGML_CUDA_USE_CUB\n size_t tmp_size = 0;\n DeviceReduce::Sum(nullptr, tmp_size, x, dst, ne, stream);\n ggml_cuda_pool_alloc tmp_alloc(pool, tmp_size);\n DeviceReduce::Sum(tmp_alloc.ptr, tmp_size, x, dst, ne, stream);\n#else\n // Use (inefficient) sum_rows implementation as a fallback.\n // For AMD there is rocPRIM which could be used as a drop-in replacement via hipcub but this would require C++11 -> C++14.\n sum_rows_f32_cuda(x, dst, ne, 1, stream);\n GGML_UNUSED(pool);\n#endif // GGML_CUDA_USE_CUB\n}\n\nvoid ggml_cuda_op_sum(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n GGML_ASSERT(ggml_is_contiguously_allocated(src0));\n\n const float * src0_d = (const float *) src0->data;\n float * dst_d = (float *) dst->data;\n\n const int64_t ne = ggml_nelements(src0);\n\n ggml_cuda_pool & pool = ctx.pool();\n cudaStream_t stream = ctx.stream();\n\n sum_f32_cuda(pool, src0_d, dst_d, ne, stream);\n}\n" }, { "path": "src/ggml-cuda/sum.cuh", "content": "#include \"common.cuh\"\n\nvoid sum_f32_cuda(ggml_cuda_pool & pool, const float * x, float * dst, const int64_t ne, cudaStream_t stream);\n\nvoid ggml_cuda_op_sum(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/sumrows.cu", "content": "#include \"reduce_rows.cuh\"\n#include \"sumrows.cuh\"\n\nvoid sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) {\n const int id = ggml_cuda_get_device();\n const int nsm = ggml_cuda_info().devices[id].nsm;\n const dim3 block_nums(nrows, 1, 1);\n if ((nrows / nsm) < 2) {\n const dim3 block_dims(512, 1, 1);\n reduce_rows_f32<<>>(x, dst, ncols);\n } else {\n const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1);\n reduce_rows_f32<<>>(x, dst, ncols);\n }\n}\n\nvoid ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *)src0->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n const int64_t ncols = src0->ne[0];\n const int64_t nrows = ggml_nrows(src0);\n\n const dim3 block_nums(nrows, 1, 1);\n\n const int id = ggml_cuda_get_device();\n const int nsm = ggml_cuda_info().devices[id].nsm;\n if ((nrows / nsm) < 2) {\n // Increase num threads to 512 for small nrows to better hide the latency\n const dim3 block_dims(512, 1, 1);\n reduce_rows_f32<<>>(src0_d, dst_d, ncols);\n } else {\n // Enough active SMs to hide latency, use smaller blocks to allow better scheduling\n const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1);\n reduce_rows_f32<<>>(src0_d, dst_d, ncols);\n }\n}\n" }, { "path": "src/ggml-cuda/sumrows.cuh", "content": "#include \"common.cuh\"\n\nvoid sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream);\nvoid ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(576, 512, 1, 16);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(576, 512, 1, 32);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 1, 8);\nDECL_FATTN_MMA_F16_CASE(80, 80, 1, 8);\nDECL_FATTN_MMA_F16_CASE(96, 96, 1, 8);\nDECL_FATTN_MMA_F16_CASE(112, 112, 1, 8);\nDECL_FATTN_MMA_F16_CASE(128, 128, 1, 8);\nDECL_FATTN_MMA_F16_CASE(256, 256, 1, 8);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 16, 1);\nDECL_FATTN_MMA_F16_CASE(80, 80, 16, 1);\nDECL_FATTN_MMA_F16_CASE(96, 96, 16, 1);\nDECL_FATTN_MMA_F16_CASE(112, 112, 16, 1);\nDECL_FATTN_MMA_F16_CASE(128, 128, 16, 1);\nDECL_FATTN_MMA_F16_CASE(256, 256, 16, 1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 16, 2);\nDECL_FATTN_MMA_F16_CASE(80, 80, 16, 2);\nDECL_FATTN_MMA_F16_CASE(96, 96, 16, 2);\nDECL_FATTN_MMA_F16_CASE(112, 112, 16, 2);\nDECL_FATTN_MMA_F16_CASE(128, 128, 16, 2);\nDECL_FATTN_MMA_F16_CASE(256, 256, 16, 2);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 16, 4);\nDECL_FATTN_MMA_F16_CASE(80, 80, 16, 4);\nDECL_FATTN_MMA_F16_CASE(96, 96, 16, 4);\nDECL_FATTN_MMA_F16_CASE(112, 112, 16, 4);\nDECL_FATTN_MMA_F16_CASE(128, 128, 16, 4);\nDECL_FATTN_MMA_F16_CASE(256, 256, 16, 4);\nDECL_FATTN_MMA_F16_CASE(576, 512, 16, 4);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(576, 512, 2, 16);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(576, 512, 2, 32);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 2, 4);\nDECL_FATTN_MMA_F16_CASE(80, 80, 2, 4);\nDECL_FATTN_MMA_F16_CASE(96, 96, 2, 4);\nDECL_FATTN_MMA_F16_CASE(112, 112, 2, 4);\nDECL_FATTN_MMA_F16_CASE(128, 128, 2, 4);\nDECL_FATTN_MMA_F16_CASE(256, 256, 2, 4);\nDECL_FATTN_MMA_F16_CASE(576, 512, 2, 4);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 2, 8);\nDECL_FATTN_MMA_F16_CASE(80, 80, 2, 8);\nDECL_FATTN_MMA_F16_CASE(96, 96, 2, 8);\nDECL_FATTN_MMA_F16_CASE(112, 112, 2, 8);\nDECL_FATTN_MMA_F16_CASE(128, 128, 2, 8);\nDECL_FATTN_MMA_F16_CASE(256, 256, 2, 8);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 32, 1);\nDECL_FATTN_MMA_F16_CASE(80, 80, 32, 1);\nDECL_FATTN_MMA_F16_CASE(96, 96, 32, 1);\nDECL_FATTN_MMA_F16_CASE(112, 112, 32, 1);\nDECL_FATTN_MMA_F16_CASE(128, 128, 32, 1);\nDECL_FATTN_MMA_F16_CASE(256, 256, 32, 1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 32, 2);\nDECL_FATTN_MMA_F16_CASE(80, 80, 32, 2);\nDECL_FATTN_MMA_F16_CASE(96, 96, 32, 2);\nDECL_FATTN_MMA_F16_CASE(112, 112, 32, 2);\nDECL_FATTN_MMA_F16_CASE(128, 128, 32, 2);\nDECL_FATTN_MMA_F16_CASE(256, 256, 32, 2);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(576, 512, 4, 16);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 4, 2);\nDECL_FATTN_MMA_F16_CASE(80, 80, 4, 2);\nDECL_FATTN_MMA_F16_CASE(96, 96, 4, 2);\nDECL_FATTN_MMA_F16_CASE(112, 112, 4, 2);\nDECL_FATTN_MMA_F16_CASE(128, 128, 4, 2);\nDECL_FATTN_MMA_F16_CASE(256, 256, 4, 2);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 4, 4);\nDECL_FATTN_MMA_F16_CASE(80, 80, 4, 4);\nDECL_FATTN_MMA_F16_CASE(96, 96, 4, 4);\nDECL_FATTN_MMA_F16_CASE(112, 112, 4, 4);\nDECL_FATTN_MMA_F16_CASE(128, 128, 4, 4);\nDECL_FATTN_MMA_F16_CASE(256, 256, 4, 4);\nDECL_FATTN_MMA_F16_CASE(576, 512, 4, 4);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 4, 8);\nDECL_FATTN_MMA_F16_CASE(80, 80, 4, 8);\nDECL_FATTN_MMA_F16_CASE(96, 96, 4, 8);\nDECL_FATTN_MMA_F16_CASE(112, 112, 4, 8);\nDECL_FATTN_MMA_F16_CASE(128, 128, 4, 8);\nDECL_FATTN_MMA_F16_CASE(256, 256, 4, 8);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 64, 1);\nDECL_FATTN_MMA_F16_CASE(80, 80, 64, 1);\nDECL_FATTN_MMA_F16_CASE(96, 96, 64, 1);\nDECL_FATTN_MMA_F16_CASE(112, 112, 64, 1);\nDECL_FATTN_MMA_F16_CASE(128, 128, 64, 1);\nDECL_FATTN_MMA_F16_CASE(256, 256, 64, 1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 8, 1);\nDECL_FATTN_MMA_F16_CASE(80, 80, 8, 1);\nDECL_FATTN_MMA_F16_CASE(96, 96, 8, 1);\nDECL_FATTN_MMA_F16_CASE(112, 112, 8, 1);\nDECL_FATTN_MMA_F16_CASE(128, 128, 8, 1);\nDECL_FATTN_MMA_F16_CASE(256, 256, 8, 1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 8, 2);\nDECL_FATTN_MMA_F16_CASE(80, 80, 8, 2);\nDECL_FATTN_MMA_F16_CASE(96, 96, 8, 2);\nDECL_FATTN_MMA_F16_CASE(112, 112, 8, 2);\nDECL_FATTN_MMA_F16_CASE(128, 128, 8, 2);\nDECL_FATTN_MMA_F16_CASE(256, 256, 8, 2);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 8, 4);\nDECL_FATTN_MMA_F16_CASE(80, 80, 8, 4);\nDECL_FATTN_MMA_F16_CASE(96, 96, 8, 4);\nDECL_FATTN_MMA_F16_CASE(112, 112, 8, 4);\nDECL_FATTN_MMA_F16_CASE(128, 128, 8, 4);\nDECL_FATTN_MMA_F16_CASE(256, 256, 8, 4);\nDECL_FATTN_MMA_F16_CASE(576, 512, 8, 4);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\nDECL_FATTN_MMA_F16_CASE(64, 64, 8, 8);\nDECL_FATTN_MMA_F16_CASE(80, 80, 8, 8);\nDECL_FATTN_MMA_F16_CASE(96, 96, 8, 8);\nDECL_FATTN_MMA_F16_CASE(112, 112, 8, 8);\nDECL_FATTN_MMA_F16_CASE(128, 128, 8, 8);\nDECL_FATTN_MMA_F16_CASE(256, 256, 8, 8);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE(112, 112);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE(128, 128);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE(256, 256);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE(40, 40);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE(576, 512);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE(64, 64);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE(72, 72);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE(80, 80);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE(96, 96);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q4_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q4_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q5_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q5_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q8_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_F16);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_F16);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_F16);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_Q4_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_Q8_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_1, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_1, GGML_TYPE_F16);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_1, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_1, GGML_TYPE_Q4_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_1, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_1, GGML_TYPE_Q4_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F16);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_F16);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q4_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_Q4_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1);\n" }, { "path": "src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0);\nDECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0);\n" }, { "path": "src/ggml-cuda/template-instances/generate_cu_files.py", "content": "#!/usr/bin/env python3\n\nfrom glob import glob\nimport os\n\nHEAD_SIZES_KQ = [40, 64, 72, 80, 96, 112, 128, 256, 576]\n\nTYPES_KV = [\"GGML_TYPE_F16\", \"GGML_TYPE_Q4_0\", \"GGML_TYPE_Q4_1\", \"GGML_TYPE_Q5_0\", \"GGML_TYPE_Q5_1\", \"GGML_TYPE_Q8_0\"]\n\nSOURCE_FATTN_TILE = \"\"\"// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-tile.cuh\"\n\nDECL_FATTN_TILE_CASE({head_size_kq}, {head_size_v});\n\"\"\"\n\nSOURCE_FATTN_VEC = \"\"\"// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-vec.cuh\"\n\nDECL_FATTN_VEC_CASE( 64, {type_k}, {type_v});\nDECL_FATTN_VEC_CASE(128, {type_k}, {type_v});\nDECL_FATTN_VEC_CASE(256, {type_k}, {type_v});\n\"\"\"\n\nSOURCE_FATTN_MMA_START = \"\"\"// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../fattn-mma-f16.cuh\"\n\n\"\"\"\n\nSOURCE_FATTN_MMA_CASE = \"DECL_FATTN_MMA_F16_CASE({head_size_kq}, {head_size_v}, {ncols1}, {ncols2});\\n\"\n\nTYPES_MMQ = [\n \"GGML_TYPE_Q4_0\", \"GGML_TYPE_Q4_1\", \"GGML_TYPE_Q5_0\", \"GGML_TYPE_Q5_1\", \"GGML_TYPE_Q8_0\",\n \"GGML_TYPE_Q2_K\", \"GGML_TYPE_Q3_K\", \"GGML_TYPE_Q4_K\", \"GGML_TYPE_Q5_K\", \"GGML_TYPE_Q6_K\",\n \"GGML_TYPE_IQ2_XXS\", \"GGML_TYPE_IQ2_XS\", \"GGML_TYPE_IQ2_S\", \"GGML_TYPE_IQ3_XXS\", \"GGML_TYPE_IQ3_S\",\n \"GGML_TYPE_IQ1_S\", \"GGML_TYPE_IQ4_NL\", \"GGML_TYPE_IQ4_XS\", \"GGML_TYPE_MXFP4\"\n]\n\nSOURCE_MMQ = \"\"\"// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE({type});\n\"\"\"\n\nSOURCE_MMF = \"\"\"// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE({type});\n\"\"\"\n\n\ndef get_short_name(long_quant_name):\n return long_quant_name.replace(\"GGML_TYPE_\", \"\").lower()\n\n\nfor filename in glob(\"*.cu\"):\n os.remove(filename)\n\nfor head_size_kq in HEAD_SIZES_KQ:\n head_size_v = head_size_kq if head_size_kq != 576 else 512\n with open(f\"fattn-tile-instance-dkq{head_size_kq}-dv{head_size_v}.cu\", \"w\") as f:\n f.write(SOURCE_FATTN_TILE.format(head_size_kq=head_size_kq, head_size_v=head_size_v))\n\nfor type_k in TYPES_KV:\n for type_v in TYPES_KV:\n with open(f\"fattn-vec-instance-{get_short_name(type_k)}-{get_short_name(type_v)}.cu\", \"w\") as f:\n f.write(SOURCE_FATTN_VEC.format(type_k=type_k, type_v=type_v))\n\nfor ncols in [8, 16, 32, 64]:\n for ncols2 in [1, 2, 4, 8, 16, 32]:\n if ncols2 > ncols:\n continue\n ncols1 = ncols // ncols2\n with open(f\"fattn-mma-f16-instance-ncols1_{ncols1}-ncols2_{ncols2}.cu\", \"w\") as f:\n f.write(SOURCE_FATTN_MMA_START)\n\n for head_size_kq in HEAD_SIZES_KQ:\n if head_size_kq == 40:\n continue\n if head_size_kq == 72:\n continue\n if head_size_kq != 576 and ncols2 in (16, 32):\n continue\n if head_size_kq == 576 and ncols2 not in (4, 16, 32):\n continue\n head_size_v = head_size_kq if head_size_kq != 576 else 512\n f.write(SOURCE_FATTN_MMA_CASE.format(ncols1=ncols1, ncols2=ncols2, head_size_kq=head_size_kq, head_size_v=head_size_v))\n\nfor type in TYPES_MMQ:\n with open(f\"mmq-instance-{get_short_name(type)}.cu\", \"w\") as f:\n f.write(SOURCE_MMQ.format(type=type))\n\nfor type in range(1, 17):\n with open(f\"mmf-instance-ncols_{type}.cu\", \"w\") as f:\n f.write(SOURCE_MMF.format(type=type))\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(1);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(10);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(11);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(12);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(13);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(14);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(15);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(16);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(2);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(3);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(4);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(5);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(6);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(7);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(8);\n" }, { "path": "src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmf.cuh\"\n\nDECL_MMF_CASE(9);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_IQ1_S);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_IQ2_S);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_IQ2_XS);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_IQ3_S);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_IQ4_NL);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_IQ4_XS);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_MXFP4);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q2_k.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q2_K);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q3_k.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q3_K);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q4_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q4_0);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q4_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q4_1);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q4_k.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q4_K);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q5_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q5_0);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q5_1.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q5_1);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q5_k.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q5_K);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q6_k.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q6_K);\n" }, { "path": "src/ggml-cuda/template-instances/mmq-instance-q8_0.cu", "content": "// This file has been autogenerated by generate_cu_files.py, do not edit manually.\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_Q8_0);\n" }, { "path": "src/ggml-cuda/top-k.cu", "content": "#include \"argsort.cuh\"\n#include \"top-k.cuh\"\n\n#ifdef GGML_CUDA_USE_CUB\n# include \n# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2)\n# define CUB_TOP_K_AVAILABLE\nusing namespace cub;\n# endif // CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2\n#endif // GGML_CUDA_USE_CUB\n\n#ifdef CUB_TOP_K_AVAILABLE\n\nstatic void top_k_cub(ggml_cuda_pool & pool,\n const float * src,\n int * dst,\n const int ncols,\n const int k,\n cudaStream_t stream) {\n auto requirements = cuda::execution::require(cuda::execution::determinism::not_guaranteed,\n cuda::execution::output_ordering::unsorted);\n auto stream_env = cuda::stream_ref{ stream };\n auto env = cuda::std::execution::env{ stream_env, requirements };\n\n auto indexes_in = cuda::make_counting_iterator(0);\n\n size_t temp_storage_bytes = 0;\n DeviceTopK::MaxPairs(nullptr, temp_storage_bytes, src, cuda::discard_iterator(), indexes_in, dst, ncols, k,\n env);\n\n ggml_cuda_pool_alloc temp_storage_alloc(pool, temp_storage_bytes);\n void * d_temp_storage = temp_storage_alloc.get();\n\n DeviceTopK::MaxPairs(d_temp_storage, temp_storage_bytes, src, cuda::discard_iterator(), indexes_in, dst,\n ncols, k, env);\n}\n\n#elif defined(GGML_CUDA_USE_CUB) // CUB_TOP_K_AVAILABLE\n\nstatic int next_power_of_2(int x) {\n int n = 1;\n while (n < x) {\n n *= 2;\n }\n return n;\n}\n\n#endif // CUB_TOP_K_AVAILABLE\n\nvoid ggml_cuda_op_top_k(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *) src0->data;\n int * dst_d = (int *) dst->data;\n cudaStream_t stream = ctx.stream();\n\n // are these asserts truly necessary?\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_I32);\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n const int64_t ncols = src0->ne[0];\n const int64_t nrows = ggml_nrows(src0);\n const int64_t k = dst->ne[0];\n ggml_cuda_pool & pool = ctx.pool();\n#ifdef CUB_TOP_K_AVAILABLE\n // TODO: Switch to `DeviceSegmentedTopK` for multi-row TopK once implemented\n // https://github.com/NVIDIA/cccl/issues/6391\n // TODO: investigate if there exists a point where parallelized argsort is faster than sequential top-k\n for (int i = 0; i < nrows; i++) {\n top_k_cub(pool, src0_d + i * ncols, dst_d + i * k, ncols, k, stream);\n }\n#elif defined(GGML_CUDA_USE_CUB) // CUB_TOP_K_AVAILABLE\n // Fall back to argsort + copy\n const int ncols_pad = next_power_of_2(ncols);\n const size_t shared_mem = ncols_pad * sizeof(int);\n const size_t max_shared_mem = ggml_cuda_info().devices[ggml_cuda_get_device()].smpb;\n\n ggml_cuda_pool_alloc temp_dst_alloc(pool, ncols * nrows);\n int * tmp_dst = temp_dst_alloc.get();\n\n if (shared_mem > max_shared_mem || ncols > 1024) {\n argsort_f32_i32_cuda_cub(pool, src0_d, tmp_dst, ncols, nrows, GGML_SORT_ORDER_DESC, stream);\n } else {\n argsort_f32_i32_cuda_bitonic(src0_d, tmp_dst, ncols, nrows, GGML_SORT_ORDER_DESC, stream);\n }\n CUDA_CHECK(cudaMemcpy2DAsync(dst_d, k * sizeof(int), tmp_dst, ncols * sizeof(int), k * sizeof(int), nrows,\n cudaMemcpyDeviceToDevice, stream));\n#else // GGML_CUDA_USE_CUB\n ggml_cuda_pool_alloc temp_dst_alloc(pool, ncols * nrows);\n int * tmp_dst = temp_dst_alloc.get();\n argsort_f32_i32_cuda_bitonic(src0_d, tmp_dst, ncols, nrows, GGML_SORT_ORDER_DESC, stream);\n CUDA_CHECK(cudaMemcpy2DAsync(dst_d, k * sizeof(int), tmp_dst, ncols * sizeof(int), k * sizeof(int), nrows,\n cudaMemcpyDeviceToDevice, stream));\n#endif\n}\n" }, { "path": "src/ggml-cuda/top-k.cuh", "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_top_k(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/topk-moe.cu", "content": "#include \"ggml-cuda/common.cuh\"\n#include \"ggml.h\"\n#include \"topk-moe.cuh\"\n\n#include \n#include \n\n// Kernel config struct - passed by value to CUDA kernel\nstruct topk_moe_config {\n bool use_sigmoid;\n bool with_norm;\n bool delayed_softmax;\n};\n\n// Warp-local softmax used for both the pre-top-k logits and the post-top-k delayed path.\ntemplate \n__device__ void softmax_warp_inplace(float (&vals)[experts_per_thread], const int limit, const int lane) {\n float max_val = -INFINITY;\n\n#pragma unroll\n for (int i = 0; i < experts_per_thread; i++) {\n const int idx = lane + i * WARP_SIZE;\n const bool active = !use_limit || (idx < limit);\n if (active) {\n max_val = max(max_val, vals[i]);\n }\n }\n\n max_val = warp_reduce_max(max_val);\n\n float sum = 0.f;\n\n#pragma unroll\n for (int i = 0; i < experts_per_thread; i++) {\n const int idx = lane + i * WARP_SIZE;\n const bool active = !use_limit || (idx < limit);\n if (active) {\n const float val = expf(vals[i] - max_val);\n vals[i] = val;\n sum += val;\n } else {\n vals[i] = 0.f;\n }\n }\n\n sum = warp_reduce_sum(sum);\n\n const float inv_sum = 1.0f / sum;\n\n#pragma unroll\n for (int i = 0; i < experts_per_thread; i++) {\n const int idx = lane + i * WARP_SIZE;\n const bool active = !use_limit || (idx < limit);\n if (active) {\n vals[i] *= inv_sum;\n }\n }\n}\n\ntemplate \n__device__ void sigmoid_warp_inplace(float (&vals)[experts_per_thread], const int limit, const int lane) {\n#pragma unroll\n for (int i = 0; i < experts_per_thread; i++) {\n const int idx = lane + i * WARP_SIZE;\n const bool active = !use_limit || (idx < limit);\n vals[i] = active ? 1.f / (1.f + expf(-vals[i])) : -INFINITY;\n }\n}\n\n/*\n This kernel does the following:\n 1. optionally softmax over the logits per token [n_experts, n_tokens]\n 2. argmax reduce over the top-k (n_experts_used) logits\n 3. write weights + ids to global memory\n 4. optionally normalize the weights or apply softmax over the selected logits\n\n It is intended as fusion of softmax->top-k->get_rows pipeline for MoE models\n*/\ntemplate \n__launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * logits,\n float * weights,\n int32_t * ids,\n float * bias,\n const int n_rows,\n const int n_expert_used,\n const float clamp_val,\n const float scale_val,\n const topk_moe_config config) {\n const int row = blockIdx.x * blockDim.y + threadIdx.y;\n if (row >= n_rows) {\n return;\n }\n\n logits += n_experts * row;\n weights += n_expert_used * row;\n ids += n_experts * row;\n\n constexpr int experts_per_thread = (n_experts > WARP_SIZE) ? n_experts / WARP_SIZE : 1;\n\n float wt[experts_per_thread];\n\n // Initialize all slots to -INFINITY\n#pragma unroll\n for (int i = 0; i < experts_per_thread; i++) {\n wt[i] = -INFINITY;\n }\n\n#pragma unroll\n for (int i = 0; i < n_experts; i += WARP_SIZE) {\n const int expert = i + threadIdx.x;\n wt[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? logits[expert] : -INFINITY;\n }\n\n if (!config.delayed_softmax) {\n if (config.use_sigmoid) {\n sigmoid_warp_inplace(wt, n_experts, threadIdx.x);\n } else {\n softmax_warp_inplace(wt, n_experts, threadIdx.x);\n }\n }\n\n // Sanitize NaN to -FLT_MAX so the iterative argmax produces unique expert IDs.\n // NaN comparisons always return false, which would cause the same expert to be\n // selected repeatedly. -FLT_MAX compares normally and is still excluded by the\n // -INFINITY sentinel used after each selection round.\n // More relevant for the cuBLAS path. See https://github.com/ggml-org/llama.cpp/issues/19659\n#pragma unroll\n for (int i = 0; i < experts_per_thread; i++) {\n if (__isnanf(wt[i])) {\n wt[i] = -FLT_MAX;\n }\n }\n\n // selection_wt is only needed when bias is present (selection uses wt + bias)\n // when no bias, we use wt directly for both selection and weight values\n float selection_wt[has_bias ? experts_per_thread : 1];\n\n if constexpr (has_bias) {\n#pragma unroll\n for (int i = 0; i < experts_per_thread; i++) {\n selection_wt[i] = -INFINITY;\n }\n#pragma unroll\n for (int i = 0; i < n_experts; i += WARP_SIZE) {\n const int expert = i + threadIdx.x;\n selection_wt[i / WARP_SIZE] =\n (n_experts % WARP_SIZE == 0 || expert < n_experts) ? wt[i / WARP_SIZE] + bias[expert] : -INFINITY;\n }\n }\n\n //at this point, each thread holds either a portion of the softmax distribution\n //or the raw logits. We do the argmax reduce over n_expert_used, each time marking\n //the expert weight as -inf to exclude from the next iteration\n\n float wt_sum = 0.f;\n\n float output_weights[experts_per_thread];\n\n#pragma unroll\n for (int i = 0; i < experts_per_thread; i++) {\n output_weights[i] = 0.f;\n }\n\n for (int k = 0; k < n_expert_used; k++) {\n float max_val = wt[0];\n int max_expert = threadIdx.x;\n\n if constexpr (has_bias) {\n float max_val_s = selection_wt[0];\n\n#pragma unroll\n for (int i = 1; i < experts_per_thread; i++) {\n const int expert = threadIdx.x + i * WARP_SIZE;\n if ((n_experts % WARP_SIZE == 0 || expert < n_experts) && selection_wt[i] > max_val_s) {\n max_val = wt[i];\n max_val_s = selection_wt[i];\n max_expert = expert;\n }\n }\n\n#pragma unroll\n for (int mask = WARP_SIZE / 2; mask > 0; mask /= 2) {\n const float val = __shfl_xor_sync(0xFFFFFFFF, max_val, mask, WARP_SIZE);\n const float val_s = __shfl_xor_sync(0xFFFFFFFF, max_val_s, mask, WARP_SIZE);\n const int expert = __shfl_xor_sync(0xFFFFFFFF, max_expert, mask, WARP_SIZE);\n if (val_s > max_val_s || (val_s == max_val_s && expert < max_expert)) {\n max_val = val;\n max_val_s = val_s;\n max_expert = expert;\n }\n }\n\n if ((max_expert & (WARP_SIZE - 1)) == threadIdx.x) {\n selection_wt[max_expert / WARP_SIZE] = -INFINITY;\n }\n } else {\n#pragma unroll\n for (int i = 1; i < experts_per_thread; i++) {\n const int expert = threadIdx.x + i * WARP_SIZE;\n if ((n_experts % WARP_SIZE == 0 || expert < n_experts) && wt[i] > max_val) {\n max_val = wt[i];\n max_expert = expert;\n }\n }\n\n#pragma unroll\n for (int mask = WARP_SIZE / 2; mask > 0; mask /= 2) {\n const float val = __shfl_xor_sync(0xFFFFFFFF, max_val, mask, WARP_SIZE);\n const int expert = __shfl_xor_sync(0xFFFFFFFF, max_expert, mask, WARP_SIZE);\n if (val > max_val || (val == max_val && expert < max_expert)) {\n max_val = val;\n max_expert = expert;\n }\n }\n\n if ((max_expert & (WARP_SIZE - 1)) == threadIdx.x) {\n wt[max_expert / WARP_SIZE] = -INFINITY;\n }\n }\n\n if ((k & (WARP_SIZE - 1)) == threadIdx.x) {\n output_weights[k / WARP_SIZE] = max_val;\n }\n\n if ((max_expert & (WARP_SIZE - 1)) == threadIdx.x) {\n ids[k] = max_expert;\n if (config.with_norm) {\n wt_sum += max_val;\n }\n }\n }\n\n if (config.with_norm) {\n wt_sum = warp_reduce_sum(wt_sum);\n wt_sum = max(wt_sum, clamp_val);\n const float inv_sum = 1.0f / wt_sum;\n\n for (int i = 0; i < experts_per_thread; i++) {\n output_weights[i] *= inv_sum;\n }\n }\n\n if (config.delayed_softmax) {\n softmax_warp_inplace(output_weights, n_expert_used, threadIdx.x);\n }\n\n#pragma unroll\n for (int i = 0; i < experts_per_thread; i++) {\n const int idx = i * WARP_SIZE + threadIdx.x;\n if (idx < n_expert_used) {\n weights[idx] = output_weights[i] * scale_val;\n }\n }\n}\n\ntemplate\nstatic void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx,\n const float * logits,\n float * weights,\n int32_t * ids,\n float * bias,\n const int n_rows,\n const int n_expert,\n const int n_expert_used,\n const float clamp_val,\n const float scale_val,\n const topk_moe_config config) {\n GGML_ASSERT(!(config.with_norm && config.delayed_softmax) &&\n \"delayed softmax is not supported with weight normalization\");\n const int rows_per_block = 4;\n dim3 grid_dims((n_rows + rows_per_block - 1) / rows_per_block, 1, 1);\n dim3 block_dims(WARP_SIZE, rows_per_block, 1);\n cudaStream_t stream = ctx.stream();\n\n switch (n_expert) {\n case 1:\n topk_moe_cuda<1, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 2:\n topk_moe_cuda<2, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 4:\n topk_moe_cuda<4, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 8:\n topk_moe_cuda<8, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 16:\n topk_moe_cuda<16, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 32:\n topk_moe_cuda<32, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 64:\n topk_moe_cuda<64, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 128:\n topk_moe_cuda<128, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 256:\n topk_moe_cuda<256, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 512:\n topk_moe_cuda<512, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n case 576:\n topk_moe_cuda<576, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used,\n clamp_val, scale_val, config);\n break;\n default:\n GGML_ASSERT(false && \"fatal error\");\n break;\n }\n}\n\nvoid ggml_cuda_op_topk_moe(ggml_backend_cuda_context & ctx,\n const ggml_tensor * logits,\n ggml_tensor * weights,\n ggml_tensor * ids,\n const ggml_tensor * clamp,\n const ggml_tensor * scale,\n const ggml_tensor * bias,\n const ggml_cuda_topk_moe_args & args) {\n GGML_ASSERT(logits->type == GGML_TYPE_F32);\n GGML_ASSERT(weights->type == GGML_TYPE_F32);\n GGML_ASSERT(ids->type == GGML_TYPE_I32);\n\n const int n_experts = logits->ne[0];\n const int n_rows = logits->ne[1];\n\n const float * logits_d = (const float *) logits->data;\n float * weights_d = (float *) weights->data;\n int32_t * ids_d = (int32_t *) ids->data;\n float * bias_d = bias ? (float *) bias->data : nullptr;\n\n float scale_val = scale ? ggml_get_op_params_f32(scale, 0) : 1.0f;\n\n GGML_ASSERT(ids->nb[1] / ggml_type_size(ids->type) == (size_t) n_experts);\n\n const int n_expert_used = weights->ne[1];\n\n const bool with_norm = clamp != nullptr;\n\n float clamp_val = -INFINITY;\n if (clamp) {\n clamp_val = ggml_get_op_params_f32(clamp, 0);\n }\n\n topk_moe_config config;\n config.use_sigmoid = args.sigmoid;\n config.with_norm = with_norm;\n config.delayed_softmax = args.delayed_softmax;\n\n if (bias) {\n launch_topk_moe_cuda(ctx, logits_d, weights_d, ids_d, bias_d, n_rows, n_experts, n_expert_used, clamp_val,\n scale_val, config);\n } else {\n launch_topk_moe_cuda(ctx, logits_d, weights_d, ids_d, bias_d, n_rows, n_experts, n_expert_used, clamp_val,\n scale_val, config);\n }\n}\n\nbool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op,\n const ggml_tensor * weights,\n const ggml_tensor * logits,\n const ggml_tensor * ids) {\n const int n_expert = ids->nb[1] / ids->nb[0];\n if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 576) {\n return false;\n }\n\n if (!ggml_is_contiguous(weights) || !ggml_is_contiguous(logits)) {\n return false;\n }\n\n if (gating_op->op == GGML_OP_SOFT_MAX) {\n const ggml_tensor * softmax = gating_op;\n float scale = 1.0f;\n float max_bias = 0.0f;\n\n memcpy(&scale, (const float *) softmax->op_params + 0, sizeof(float));\n memcpy(&max_bias, (const float *) softmax->op_params + 1, sizeof(float));\n\n if (!ggml_is_contiguous(softmax->src[0])) {\n return false;\n }\n\n if (scale != 1.0f || max_bias != 0.0f) {\n return false;\n }\n\n // don't fuse when masks or sinks are present\n if (softmax->src[1] || softmax->src[2]) {\n return false;\n }\n } else if (gating_op->op == GGML_OP_UNARY) {\n ggml_unary_op op = ggml_get_unary_op(gating_op);\n\n if (op != GGML_UNARY_OP_SIGMOID) {\n return false;\n }\n }\n\n return true;\n}\n" }, { "path": "src/ggml-cuda/topk-moe.cuh", "content": "#include \"common.cuh\"\n#include \"ggml.h\"\n\n#include \n\nstruct ggml_cuda_topk_moe_args {\n bool sigmoid{};\n bool softmax{};\n bool delayed_softmax{};\n bool prob_bias{};\n bool norm{};\n bool scale{};\n};\n\nvoid ggml_cuda_op_topk_moe(ggml_backend_cuda_context & ctx,\n const ggml_tensor * logits,\n ggml_tensor * weights,\n ggml_tensor * ids,\n const ggml_tensor * clamp,\n const ggml_tensor * scale,\n const ggml_tensor * bias,\n const ggml_cuda_topk_moe_args & args);\n\nbool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op,\n const ggml_tensor * weights,\n const ggml_tensor * logits,\n const ggml_tensor * ids);\n" }, { "path": "src/ggml-cuda/tri.cu", "content": "#include \"common.cuh\"\n#include \"convert.cuh\"\n#include \"tri.cuh\"\n#include \"ggml.h\"\n\ntemplate\nstatic __global__ void tri_kernel(\n const T * src, T * dst,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03,\n const int64_t nb0, const int64_t nb1, const int64_t nb2, const int64_t nb3) {\n const int64_t i3 = blockIdx.z;\n const int64_t i2 = blockIdx.y;\n const int64_t i1 = blockIdx.x;\n const int64_t split_point = i1 + add_to_split;\n\n GGML_UNUSED_VARS(nb00, nb0);\n\n if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {\n return;\n }\n\n const T * src_row = src + i1*nb01 + i2*nb02 + i3*nb03;\n T * dst_row = dst + i1*nb1 + i2*nb2 + i3*nb3;\n\n if constexpr (prefix_keep) {\n for (int64_t i0 = threadIdx.x; i0 < split_point; i0 += blockDim.x) {\n dst_row[i0] = src_row[i0];\n }\n for (int64_t i0 = threadIdx.x + split_point; i0 < ne00; i0 += blockDim.x) {\n dst_row[i0] = ggml_cuda_cast(0.0f);\n }\n } else {\n for (int64_t i0 = threadIdx.x; i0 < split_point; i0 += blockDim.x) {\n dst_row[i0] = ggml_cuda_cast(0.0f);\n }\n for (int64_t i0 = threadIdx.x + split_point; i0 < ne00; i0 += blockDim.x) {\n dst_row[i0] = src_row[i0];\n }\n }\n}\n\ntemplate\nstatic void tri_cuda(\n const T * src, T * dst,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03,\n const int64_t nb0, const int64_t nb1, const int64_t nb2, const int64_t nb3,\n const ggml_tri_type ttype,\n cudaStream_t stream) {\n\n dim3 block_dims(CUDA_TRI_BLOCK_SIZE, 1, 1);\n dim3 grid_dims(ne01, ne02, ne03);\n const size_t type_size = sizeof(T);\n\n const int add_to_split = (ttype == GGML_TRI_TYPE_LOWER_DIAG || ttype == GGML_TRI_TYPE_UPPER) ? 1 : 0;\n const bool prefix_keep = (ttype == GGML_TRI_TYPE_LOWER || ttype == GGML_TRI_TYPE_LOWER_DIAG);\n\n if (prefix_keep) {\n if (add_to_split == 0) {\n tri_kernel<<>>(\n src, dst,\n ne00, ne01, ne02, ne03,\n nb00 / type_size, nb01 / type_size, nb02 / type_size, nb03 / type_size,\n nb0 / type_size, nb1 / type_size, nb2 / type_size, nb3 / type_size\n );\n } else { // only 0 and 1 supported\n tri_kernel<<>>(\n src, dst,\n ne00, ne01, ne02, ne03,\n nb00 / type_size, nb01 / type_size, nb02 / type_size, nb03 / type_size,\n nb0 / type_size, nb1 / type_size, nb2 / type_size, nb3 / type_size\n );\n }\n } else {\n if (add_to_split == 0) {\n tri_kernel<<>>(\n src, dst,\n ne00, ne01, ne02, ne03,\n nb00 / type_size, nb01 / type_size, nb02 / type_size, nb03 / type_size,\n nb0 / type_size, nb1 / type_size, nb2 / type_size, nb3 / type_size\n );\n } else {\n tri_kernel<<>>(\n src, dst,\n ne00, ne01, ne02, ne03,\n nb00 / type_size, nb01 / type_size, nb02 / type_size, nb03 / type_size,\n nb0 / type_size, nb1 / type_size, nb2 / type_size, nb3 / type_size\n );\n }\n }\n}\n\nvoid ggml_cuda_op_tri(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n cudaStream_t stream = ctx.stream();\n\n const ggml_tri_type ttype = static_cast(ggml_get_op_params_i32(dst, 0));\n\n GGML_ASSERT(src0->type == dst->type);\n\n switch(src0->type) {\n case GGML_TYPE_F32:\n {\n tri_cuda(\n (const float *)src0->data, (float *)dst->data,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3],\n ttype, stream\n );\n } break;\n case GGML_TYPE_F16:\n {\n tri_cuda(\n (const half *)src0->data, (half *)dst->data,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3],\n ttype, stream\n );\n } break;\n case GGML_TYPE_BF16:\n {\n tri_cuda(\n (const nv_bfloat16 *)src0->data, (nv_bfloat16 *)dst->data,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3],\n ttype, stream\n );\n } break;\n default:\n GGML_ABORT(\"fatal error\");\n }\n}\n" }, { "path": "src/ggml-cuda/tri.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_TRI_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_tri(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/tsembd.cu", "content": "#include \"tsembd.cuh\"\n\nstatic __global__ void timestep_embedding_f32(const float * timesteps, float * dst, const int nb1, const int dim, const int max_period) {\n // blockIDx.y: idx of timesteps->ne[0]\n // blockIDx.x: idx of ((dim + 1) / 2) / BLOCK_SIZE\n int i = blockIdx.y;\n int j = threadIdx.x + blockIdx.x * blockDim.x;\n float * embed_data = (float *)((char *)dst + i*nb1);\n\n int half = dim / 2;\n if (dim % 2 != 0 && j == half) {\n embed_data[2 * half] = 0.f;\n }\n\n if (j >= half) {\n return;\n }\n\n float timestep = timesteps[i];\n float freq = (float)expf(-logf(max_period) * j / half);\n float arg = timestep * freq;\n embed_data[j] = cosf(arg);\n embed_data[j + half] = sinf(arg);\n}\n\nstatic void timestep_embedding_f32_cuda(const float * x, float * dst, const int ne00, const int nb1,\n const int dim, const int max_period, cudaStream_t stream) {\n int half_ceil = (dim + 1) / 2;\n int num_blocks = (half_ceil + CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE - 1) / CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE;\n dim3 gridDim(num_blocks, ne00, 1);\n timestep_embedding_f32<<>>(x, dst, nb1, dim, max_period);\n}\n\nvoid ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *)src0->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n const int dim = dst->op_params[0];\n const int max_period = dst->op_params[1];\n\n timestep_embedding_f32_cuda(src0_d, dst_d, src0->ne[0], dst->nb[1], dim, max_period, stream);\n}\n" }, { "path": "src/ggml-cuda/tsembd.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/unary.cu", "content": "#include \"unary.cuh\"\n#include \"convert.cuh\"\n\nstatic __device__ __forceinline__ float op_abs(float x) {\n return fabsf(x);\n}\n\nstatic __device__ __forceinline__ float op_sgn(float x) {\n return (x > 0.f ? 1.f : ((x < 0.f ? -1.f : 0.f)));\n}\n\nstatic __device__ __forceinline__ float op_neg(float x) {\n return -x;\n}\n\nstatic __device__ __forceinline__ float op_step(float x) {\n return x > 0.0f;\n}\n\nstatic __device__ __forceinline__ float op_gelu(float x) {\n return ggml_cuda_op_gelu_single(x);\n}\n\nstatic __device__ __forceinline__ float op_gelu_erf(float x) {\n const float SQRT_2_INV = 0.70710678118654752440084436210484f;\n\n return 0.5f*x*(1.0f + erff(x*SQRT_2_INV));\n}\n\nstatic __device__ __forceinline__ float op_gelu_quick(float x) {\n const float GELU_QUICK_COEF = -1.702f;\n\n return x * (1.0f / (1.0f + expf(GELU_QUICK_COEF * x)));\n}\n\nstatic __device__ __forceinline__ float op_silu(float x) {\n return ggml_cuda_op_silu_single(x);\n}\n\nstatic __device__ __forceinline__ float op_tanh(float x) {\n return tanhf(x);\n}\n\nstatic __device__ __forceinline__ float op_relu(float x) {\n return fmaxf(x, 0);\n}\n\nstatic __device__ __forceinline__ float op_sigmoid(float x) {\n return 1.0f / (1.0f + expf(-x));\n}\n\nstatic __device__ __forceinline__ float op_hardsigmoid(float x) {\n return fminf(1.0f, fmaxf(0.0f, (x + 3.0f) / 6.0f));\n}\n\nstatic __device__ __forceinline__ float op_hardswish(float x) {\n return x * fminf(1.0f, fmaxf(0.0f, (x + 3.0f) / 6.0f));\n}\n\nstatic __device__ __forceinline__ float op_exp(float x) {\n return expf(x);\n}\n\nstatic __device__ __forceinline__ float op_sqr(float x) {\n return x * x;\n}\n\nstatic __device__ __forceinline__ float op_sqrt(float x) {\n return sqrtf(x);\n}\n\nstatic __device__ __forceinline__ float op_sin(float x) {\n return sinf(x);\n}\n\nstatic __device__ __forceinline__ float op_cos(float x) {\n return cosf(x);\n}\n\nstatic __device__ __forceinline__ float op_log(float x) {\n return logf(x);\n}\n\nstatic __device__ __forceinline__ float op_expm1(float x) {\n return expm1f(x);\n}\n\nstatic __device__ __forceinline__ float op_softplus(float x) {\n return (x > 20.0f) ? x : logf(1.0f + expf(x));\n}\n\nstatic __device__ __forceinline__ float op_elu(float x) {\n return (x > 0.f) ? x : expm1f(x);\n}\n\nstatic __device__ __forceinline__ float op_floor(float x) {\n return floorf(x);\n}\n\nstatic __device__ __forceinline__ float op_ceil(float x) {\n return ceilf(x);\n}\n\nstatic __device__ __forceinline__ float op_round(float x) {\n return round(x);\n}\n\nstatic __device__ __forceinline__ float op_trunc(float x) {\n return trunc(x);\n}\n\ntemplate \nstatic __global__ void unary_op_kernel(const T * x, T * dst, const int k) {\n const int i = blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n\n dst[i] = (T)op((float)x[i]);\n}\n\ntemplate \nstatic void unary_cuda(const T * x, T * dst, const int k, cudaStream_t stream) {\n const int num_blocks = (k + CUDA_NEG_BLOCK_SIZE - 1) / CUDA_NEG_BLOCK_SIZE;\n unary_op_kernel<<>>(x, dst, k);\n}\n\ntemplate \nvoid ggml_cuda_op_unary(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const void * src0_d = src0->data;\n void * dst_d = dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n GGML_ASSERT(src0->type == dst->type);\n\n if (src0->type == GGML_TYPE_F16) {\n unary_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream);\n } else {\n unary_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream);\n }\n}\n\nvoid ggml_cuda_op_abs(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_sgn(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_neg(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_step(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_gelu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_gelu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_gelu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_silu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_tanh(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_sigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_hardswish(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_exp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_sqrt(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_sin(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_cos(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_log(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_elu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_floor(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_ceil(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_round(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_trunc(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_expm1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n\nvoid ggml_cuda_op_softplus(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary(ctx, dst);\n}\n/* gated ops */\n\ntemplate \nstatic __global__ void unary_gated_op_kernel(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1) {\n const int64_t i = int64_t(blockDim.x)*blockIdx.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n\n // perform base op and multiply with gate (either offset in same tensor or a separate one)\n const int64_t j0 = (i / n) * o0 + (i % n);\n const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);\n\n dst[i] = (T)(op((float)x[j0]) * (float)g[j1]);\n}\n\ntemplate \nstatic void unary_gated_cuda(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, cudaStream_t stream) {\n const int64_t num_blocks = (k + CUDA_GLU_BLOCK_SIZE - 1) / CUDA_GLU_BLOCK_SIZE;\n unary_gated_op_kernel<<>>(x, g, dst, k, n, o0, o1);\n}\n\ntemplate \nvoid ggml_cuda_op_unary_gated(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n void * src0_d = src0->data;\n void * src1_d = src1 ? src1->data : src0->data;\n const int64_t src0_o = src0->nb[1];\n const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n void * dst_d = dst->data;\n const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(src0->nb[0] == ggml_element_size(src0));\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n GGML_ASSERT(src0->type == dst->type);\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src1->nb[0] == ggml_element_size(src1));\n GGML_ASSERT(src1->ne[0] == nc);\n GGML_ASSERT(src0->type == src1->type);\n }\n\n const int32_t swapped = ((const int32_t *) dst->op_params)[1];\n\n if (src0->type == GGML_TYPE_F16) {\n half * src0_p = (half *) src0_d;\n half * src1_p = (half *) src1_d;\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n unary_gated_cuda(src0_p, src1_p, (half *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(half), src1_o / sizeof(half), stream);\n } else {\n float * src0_p = (float *) src0_d;\n float * src1_p = (float *) src1_d;\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n unary_gated_cuda(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), stream);\n }\n}\n\nvoid ggml_cuda_op_reglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary_gated(ctx, dst);\n}\n\nvoid ggml_cuda_op_geglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary_gated(ctx, dst);\n}\n\nvoid ggml_cuda_op_swiglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary_gated(ctx, dst);\n}\n\nvoid ggml_cuda_op_geglu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary_gated(ctx, dst);\n}\n\nvoid ggml_cuda_op_geglu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n ggml_cuda_op_unary_gated(ctx, dst);\n}\n\n// swiglu_oai\n\ntemplate \nstatic __global__ void swiglu_oai_kernel(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, float alpha, float limit) {\n const int64_t i = int64_t(blockDim.x)*blockIdx.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n\n // perform base op and multiply with gate (either offset in same tensor or a separate one)\n const int64_t j0 = (i / n) * o0 + (i % n);\n const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);\n\n float xi = x[j0];\n float gi = g[j1];\n\n dst[i] = ggml_cuda_op_swiglu_oai_single(xi, gi, alpha, limit);\n}\n\ntemplate \nstatic void swiglu_oai_cuda(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, const float alpha, const float limit, cudaStream_t stream) {\n const int64_t num_blocks = (k + CUDA_GLU_BLOCK_SIZE - 1) / CUDA_GLU_BLOCK_SIZE;\n swiglu_oai_kernel<<>>(x, g, dst, k, n, o0, o1, alpha, limit);\n}\n\nvoid ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n void * src0_d = src0->data;\n void * src1_d = src1 ? src1->data : src0->data;\n const int64_t src0_o = src0->nb[1];\n const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n void * dst_d = dst->data;\n const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(src0->nb[0] == ggml_element_size(src0));\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n GGML_ASSERT(src0->type == dst->type);\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src1->nb[0] == ggml_element_size(src1));\n GGML_ASSERT(src1->ne[0] == nc);\n GGML_ASSERT(src0->type == src1->type);\n }\n\n //const int32_t swapped = ((const int32_t *) dst->op_params)[1];\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n const float alpha = ggml_get_op_params_f32(dst, 2);\n const float limit = ggml_get_op_params_f32(dst, 3);\n\n float * src0_p = (float *) src0_d;\n float * src1_p = (float *) src1_d;\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n swiglu_oai_cuda(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream);\n}\n\n/* CUDA kernel + launcher for xIELU */\n\ntemplate \nstatic __global__ void xielu_kernel(const T * x, T * dst, const int k, float alpha_n, float alpha_p, float beta, float eps) {\n const int i = blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n\n const float xi = ggml_cuda_cast(x[i]);\n\n const float gate_pos = (xi > 0.0f);\n const float y_pos = alpha_p * xi * xi + beta * xi;\n const float min_v_eps = fminf(xi, eps);\n const float y_neg = (expm1f(min_v_eps) - xi) * alpha_n + beta * xi;\n const float out = gate_pos * y_pos + (1.0f - gate_pos) * y_neg;\n\n dst[i] = ggml_cuda_cast(out);\n}\n\ntemplate \nstatic void xielu_cuda(const T * x, T * dst, const int k, float alpha_n, float alpha_p, float beta, float eps, cudaStream_t stream) {\n const int num_blocks = (k + CUDA_XIELU_BLOCK_SIZE) / CUDA_XIELU_BLOCK_SIZE;\n xielu_kernel<<>>(x, dst, k, alpha_n, alpha_p, beta, eps);\n}\n\nvoid ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const void * src0_d = src0->data;\n void * dst_d = dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n GGML_ASSERT(src0->type == dst->type);\n\n const float alpha_n = ggml_get_op_params_f32(dst, 1);\n const float alpha_p = ggml_get_op_params_f32(dst, 2);\n const float beta = ggml_get_op_params_f32(dst, 3);\n const float eps = ggml_get_op_params_f32(dst, 4);\n\n if (src0->type == GGML_TYPE_F16) {\n xielu_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream);\n } else {\n xielu_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream);\n }\n}\n\n\n\n/* silu_back */\n\nstatic __device__ __forceinline__ float op_silu_back(float grad, float x) {\n const float s = 1.0f / (1.0f + expf(-x));\n return grad * s * (1.0f + x * (1.0f - s));\n}\n\ntemplate \nstatic __global__ void silu_back_kernel(const T * grad, const T * xf, T * dst, const int k) {\n const int i = blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n\n dst[i] = (T)op_silu_back((float)grad[i], (float)xf[i]);\n}\n\ntemplate \nstatic void silu_back_cuda(const T * grad, const T * x, T * dst, const int k, cudaStream_t stream) {\n const int num_blocks = (k + CUDA_SILU_BACK_BLOCK_SIZE - 1) / CUDA_SILU_BLOCK_SIZE;\n silu_back_kernel<<>>(grad, x, dst, k);\n}\n\nvoid ggml_cuda_op_silu_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0]; // input from forward pass\n const ggml_tensor * src1 = dst->src[1]; // grads of forward pass output\n\n const float * src0_d = (const float *) src0->data;\n const float * src1_d = (const float *) src1->data;\n float * dst_d = (float *) dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n GGML_ASSERT(src0->type == dst->type);\n\n if (src0->type == GGML_TYPE_F16) {\n silu_back_cuda((const half *)src0_d, (const half *)src1_d, (half *)dst_d, ggml_nelements(src0), stream);\n } else {\n silu_back_cuda((const float*)src0_d, (const float*)src1_d, (float *)dst_d, ggml_nelements(src0), stream);\n }\n}\n\n/* leaky relu */\n\nstatic __device__ __forceinline__ float op_leaky_relu(float x, const float negative_slope) {\n return fmaxf(x, 0) + fminf(x, 0.0f) * negative_slope;\n}\n\ntemplate \nstatic __global__ void leaky_relu_kernel(const T * x, T * dst, const int k, const float negative_slope) {\n const int i = blockDim.x*blockIdx.x + threadIdx.x;\n\n if (i >= k) {\n return;\n }\n\n dst[i] = (T)op_leaky_relu((float)x[i], negative_slope);\n}\n\ntemplate \nstatic void leaky_relu_cuda(const T * x, T * dst, const int k, const float negative_slope, cudaStream_t stream) {\n const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE;\n leaky_relu_kernel<<>>(x, dst, k, negative_slope);\n}\n\nvoid ggml_cuda_op_leaky_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const void * src0_d = src0->data;\n void * dst_d = dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n GGML_ASSERT(src0->type == dst->type);\n\n float negative_slope;\n memcpy(&negative_slope, dst->op_params, sizeof(float));\n\n if (src0->type == GGML_TYPE_F16) {\n leaky_relu_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), negative_slope, stream);\n } else {\n leaky_relu_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), negative_slope, stream);\n }\n}\n\n/* fused unary + mul */\n\ntemplate \nstatic void ggml_cuda_op_unary_mul_impl(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node) {\n // unary_node: UNARY op applied to unary_node->src[0]\n // mul_node: MUL(a, b) where one of a/b is unary_node\n // Output goes to mul_node->data\n\n const ggml_tensor * unary_src = unary_node->src[0]; // input to the unary op\n const ggml_tensor * other_src = (mul_node->src[0] == unary_node) ? mul_node->src[1] : mul_node->src[0];\n\n GGML_ASSERT(ggml_is_contiguous_1(unary_src));\n GGML_ASSERT(unary_src->nb[0] == ggml_element_size(unary_src));\n GGML_ASSERT(ggml_is_contiguous_1(other_src));\n GGML_ASSERT(other_src->nb[0] == ggml_element_size(other_src));\n GGML_ASSERT(ggml_are_same_shape(unary_src, other_src));\n\n GGML_ASSERT(unary_src->type == GGML_TYPE_F32 || unary_src->type == GGML_TYPE_F16);\n GGML_ASSERT(unary_src->type == other_src->type);\n GGML_ASSERT(unary_src->type == mul_node->type);\n\n cudaStream_t stream = ctx.stream();\n\n const int64_t k = ggml_nelements(mul_node);\n const int64_t nc = unary_src->ne[0];\n const int64_t unary_stride = unary_src->nb[1];\n const int64_t other_stride = other_src->nb[1];\n\n if (unary_src->type == GGML_TYPE_F16) {\n unary_gated_cuda((const half *) unary_src->data, (const half *) other_src->data,\n (half *) mul_node->data, k, nc,\n unary_stride / sizeof(half), other_stride / sizeof(half), stream);\n } else {\n unary_gated_cuda((const float *) unary_src->data, (const float *) other_src->data,\n (float *) mul_node->data, k, nc,\n unary_stride / sizeof(float), other_stride / sizeof(float), stream);\n }\n}\n\nvoid ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node) {\n switch (ggml_get_unary_op(unary_node)) {\n case GGML_UNARY_OP_SILU:\n ggml_cuda_op_unary_mul_impl(ctx, unary_node, mul_node);\n break;\n case GGML_UNARY_OP_SIGMOID:\n ggml_cuda_op_unary_mul_impl(ctx, unary_node, mul_node);\n break;\n case GGML_UNARY_OP_SOFTPLUS:\n ggml_cuda_op_unary_mul_impl(ctx, unary_node, mul_node);\n break;\n default:\n GGML_ABORT(\"Unsupported unary op for fused unary+mul\");\n }\n}\n" }, { "path": "src/ggml-cuda/unary.cuh", "content": "#pragma once\n#include \"common.cuh\"\n\n#define CUDA_NEG_BLOCK_SIZE 256\n#define CUDA_STEP_BLOCK_SIZE 256\n#define CUDA_GELU_BLOCK_SIZE 256\n#define CUDA_SILU_BLOCK_SIZE 256\n#define CUDA_SILU_BACK_BLOCK_SIZE 256\n#define CUDA_TANH_BLOCK_SIZE 256\n#define CUDA_RELU_BLOCK_SIZE 256\n#define CUDA_SIGMOID_BLOCK_SIZE 256\n#define CUDA_HARDSIGMOID_BLOCK_SIZE 256\n#define CUDA_EXP_BLOCK_SIZE 256\n#define CUDA_HARDSWISH_BLOCK_SIZE 256\n#define CUDA_SQR_BLOCK_SIZE 256\n#define CUDA_SQRT_BLOCK_SIZE 256\n#define CUDA_SIN_BLOCK_SIZE 256\n#define CUDA_COS_BLOCK_SIZE 256\n#define CUDA_GLU_BLOCK_SIZE 256\n#define CUDA_XIELU_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_abs(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_sgn(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_neg(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_step(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_gelu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_silu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_silu_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_gelu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_gelu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_tanh(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_sigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_exp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_hardswish(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_leaky_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_sqrt(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_sin(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_cos(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_log(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_expm1(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_softplus(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_elu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_floor(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_ceil(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_round(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_trunc(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_reglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_geglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_swiglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_geglu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_geglu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node);\n\n__device__ __forceinline__ float ggml_cuda_op_silu_single(float x) {\n return x / (1.0f + expf(-x));\n}\n\n__device__ __forceinline__ float ggml_cuda_op_gelu_single(float x) {\n const float GELU_COEF_A = 0.044715f;\n const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;\n\n return 0.5f * x * (1.0f + tanhf(SQRT_2_OVER_PI * x * (1.0f + GELU_COEF_A * x * x)));\n}\n\n__device__ __forceinline__ float ggml_cuda_op_swiglu_oai_single(float x, float g, float alpha = 1.702f, float limit = 7.0f) {\n x = fminf(x, limit);\n g = fmaxf(fminf(g, limit), -limit);\n\n float out_glu = x / (1.0f + expf(-x * alpha));\n out_glu = out_glu * (1.0f + g);\n return out_glu;\n}\n" }, { "path": "src/ggml-cuda/upscale.cu", "content": "#include \"upscale.cuh\"\n\nstatic __global__ void upscale_f32(const float * x, float * dst,\n const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int ne13,\n const float sf0, const float sf1, const float sf2, const float sf3) {\n int index = threadIdx.x + blockIdx.x * blockDim.x;\n if (index >= ne10 * ne11 * ne12 * ne13) {\n return;\n }\n\n int i10 = index % ne10;\n int i11 = (index / ne10) % ne11;\n int i12 = (index / (ne10 * ne11)) % ne12;\n int i13 = (index / (ne10 * ne11 * ne12)) % ne13;\n\n int i00 = i10 / sf0;\n int i01 = i11 / sf1;\n int i02 = i12 / sf2;\n int i03 = i13 / sf3;\n\n dst[index] = *( (const float *)((const char *)x + i03 * nb03 + i02 * nb02 + i01 * nb01 + i00 * nb00) );\n}\n\nstatic __global__ void upscale_f32_bilinear(const float * x, float * dst,\n const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne00_src, const int ne01_src,\n const int ne10_dst, const int ne11_dst, const int ne12_dst, const int ne13_dst,\n const float sf0, const float sf1, const float sf2, const float sf3,\n const float pixel_offset) {\n const int64_t index = threadIdx.x + blockIdx.x * blockDim.x;\n const int64_t dst_total_elements = ne10_dst * ne11_dst * ne12_dst * ne13_dst;\n\n if (index >= dst_total_elements) {\n return;\n }\n\n const int i10_dst = index % ne10_dst;\n const int i11_dst = (index / ne10_dst) % ne11_dst;\n const int i12_dst = (index / (ne10_dst * ne11_dst)) % ne12_dst;\n const int i13_dst = index / (ne10_dst * ne11_dst * ne12_dst);\n\n const int i02_src = (int)(i12_dst / sf2);\n const int i03_src = (int)(i13_dst / sf3);\n\n const float y_src_f = ((float)i11_dst + pixel_offset) / sf1 - pixel_offset;\n int y0_src = (int)floorf(y_src_f);\n int y1_src = y0_src + 1;\n\n y0_src = max(0, min(y0_src, ne01_src - 1));\n y1_src = max(0, min(y1_src, ne01_src - 1));\n\n float dy = y_src_f - (float)y0_src;\n dy = max(0.0f, min(dy, 1.0f));\n\n float x_src_f = ((float)i10_dst + pixel_offset) / sf0 - pixel_offset;\n int x0_src = (int)floorf(x_src_f);\n int x1_src = x0_src + 1;\n\n x0_src = max(0, min(x0_src, ne00_src - 1));\n x1_src = max(0, min(x1_src, ne00_src - 1));\n\n float dx = x_src_f - (float)x0_src;\n dx = max(0.0f, min(dx, 1.0f));\n\n const float * p_a = (const float *)((const char *)x + (int64_t)x0_src * nb00 + (int64_t)y0_src * nb01 + (int64_t)i02_src * nb02 + (int64_t)i03_src * nb03);\n const float * p_b = (const float *)((const char *)x + (int64_t)x1_src * nb00 + (int64_t)y0_src * nb01 + (int64_t)i02_src * nb02 + (int64_t)i03_src * nb03);\n const float * p_c = (const float *)((const char *)x + (int64_t)x0_src * nb00 + (int64_t)y1_src * nb01 + (int64_t)i02_src * nb02 + (int64_t)i03_src * nb03);\n const float * p_d = (const float *)((const char *)x + (int64_t)x1_src * nb00 + (int64_t)y1_src * nb01 + (int64_t)i02_src * nb02 + (int64_t)i03_src * nb03);\n\n const float val_a = *p_a;\n const float val_b = *p_b;\n const float val_c = *p_c;\n const float val_d = *p_d;\n\n float result = val_a * (1.0f - dx) * (1.0f - dy) +\n val_b * dx * (1.0f - dy) +\n val_c * (1.0f - dx) * dy +\n val_d * dx * dy;\n\n dst[index] = result;\n}\n\n// Similar to F.interpolate(..., mode=\"bilinear\", align_corners=False, antialias=True)\n// https://github.com/pytorch/pytorch/blob/8871ff29b743948d1225389d5b7068f37b22750b/aten/src/ATen/native/cpu/UpSampleKernel.cpp\nstatic __global__ void upscale_f32_bilinear_antialias(const float * src0, float * dst,\n const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne00_src, const int ne01_src,\n const int ne10_dst, const int ne11_dst, const int ne12_dst, const int ne13_dst,\n const float sf0, const float sf1, const float sf2, const float sf3,\n const float pixel_offset) {\n const int64_t index = threadIdx.x + blockIdx.x * blockDim.x;\n const int64_t dst_total_elements = ne10_dst * ne11_dst * ne12_dst * ne13_dst;\n\n if (index >= dst_total_elements) {\n return;\n }\n\n const int i10_dst = index % ne10_dst;\n const int i11_dst = (index / ne10_dst) % ne11_dst;\n const int i12_dst = (index / (ne10_dst * ne11_dst)) % ne12_dst;\n const int i13_dst = index / (ne10_dst * ne11_dst * ne12_dst);\n\n const int i02_src = (int)(i12_dst / sf2);\n const int i03_src = (int)(i13_dst / sf3);\n\n const float y = ((float)i11_dst + pixel_offset) / sf1;\n const float x = ((float)i10_dst + pixel_offset) / sf0;\n\n // support and invscale, minimum 1 pixel for bilinear\n const float support1 = max(1.0f / sf1, 1.0f);\n const float invscale1 = 1.0f / support1;\n const float support0 = max(1.0f / sf0, 1.0f);\n const float invscale0 = 1.0f / support0;\n\n // the range of source pixels that contribute\n const int64_t x_min = max(int64_t(0), int64_t(x - support0 + pixel_offset));\n const int64_t x_max = min(int64_t(ne00_src), int64_t(x + support0 + pixel_offset));\n const int64_t y_min = max(int64_t(0), int64_t(y - support1 + pixel_offset));\n const int64_t y_max = min(int64_t(ne01_src), int64_t(y + support1 + pixel_offset));\n\n // bilinear filter with antialiasing\n float val = 0.0f;\n float total_weight = 0.0f;\n\n auto triangle_filter = [](float x) -> float {\n return max(1.0f - fabsf(x), 0.0f);\n };\n\n for (int64_t sy = y_min; sy < y_max; sy++) {\n const float weight_y = triangle_filter((sy - y + pixel_offset) * invscale1);\n\n for (int64_t sx = x_min; sx < x_max; sx++) {\n const float weight_x = triangle_filter((sx - x + pixel_offset) * invscale0);\n const float weight = weight_x * weight_y;\n\n if (weight <= 0.0f) {\n continue;\n }\n\n const float pixel = *(const float *)((const char *)src0 + sx*nb00 + sy*nb01 + i02_src*nb02 + i03_src*nb03);\n val += pixel * weight;\n total_weight += weight;\n }\n }\n\n if (total_weight > 0.0f) {\n val /= total_weight;\n }\n\n dst[index] = val;\n}\n\nnamespace bicubic_interpolation {\n// https://en.wikipedia.org/wiki/Bicubic_interpolation#Bicubic_convolution_algorithm\n__device__ const float a = -0.75f; // use alpha = -0.75 (same as PyTorch)\n\nstatic __device__ float weight1(float x) { return ((a + 2) * x - (a + 3)) * x * x + 1; };\nstatic __device__ float weight2(float x) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; };\n\nstatic __device__ float bicubic(float p0, float p1, float p2, float p3, float x) {\n const float w0 = weight2(x + 1);\n const float w1 = weight1(x + 0);\n const float w2 = weight1(1 - x);\n const float w3 = weight2(2 - x);\n return p0 * w0 + p1 * w1 + p2 * w2 + p3 * w3;\n};\n} // namespace bicubic_interpolation\n\nstatic __global__ void upscale_f32_bicubic(const float * x, float * dst,\n const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne00_src, const int ne01_src,\n const int ne10_dst, const int ne11_dst, const int ne12_dst, const int ne13_dst,\n const float sf0, const float sf1, const float sf2, const float sf3,\n const float pixel_offset) {\n using bicubic_interpolation::bicubic;\n\n const int64_t index = threadIdx.x + blockIdx.x * blockDim.x;\n const int64_t dst_total_elements = ne10_dst * ne11_dst * ne12_dst * ne13_dst;\n\n if (index >= dst_total_elements) {\n return;\n }\n\n const int i10_dst = index % ne10_dst;\n const int i11_dst = (index / ne10_dst) % ne11_dst;\n const int i12_dst = (index / (ne10_dst * ne11_dst)) % ne12_dst;\n const int i13_dst = index / (ne10_dst * ne11_dst * ne12_dst);\n\n const int i02_src = (int)(i12_dst / sf2);\n const int i03_src = (int)(i13_dst / sf3);\n\n const float y_src_f = ((float)i11_dst + pixel_offset) / sf1 - pixel_offset;\n const int y0_src = (int)floorf(y_src_f);\n const float dy = y_src_f - (float)y0_src;\n\n const float x_src_f = ((float)i10_dst + pixel_offset) / sf0 - pixel_offset;\n const int x0_src = (int)floorf(x_src_f);\n const float dx = x_src_f - (float)x0_src;\n\n const char * x_base = (const char *)x + (int64_t)i02_src * nb02 + (int64_t)i03_src * nb03;\n\n auto load = [=](int x_off, int y_off) -> float {\n int i00_src = max(0, min(x0_src + x_off, ne00_src - 1));\n int i01_src = max(0, min(y0_src + y_off, ne01_src - 1));\n return *(const float *)(x_base + (int64_t)i00_src * nb00 + (int64_t)i01_src * nb01);\n };\n\n const float result = bicubic(\n bicubic(load(-1,-1), load(0,-1), load(1,-1), load(2,-1), dx),\n bicubic(load(-1, 0), load(0, 0), load(1, 0), load(2, 0), dx),\n bicubic(load(-1, 1), load(0, 1), load(1, 1), load(2, 1), dx),\n bicubic(load(-1, 2), load(0, 2), load(1, 2), load(2, 2), dx), dy);\n\n dst[index] = result;\n}\n\nstatic void upscale_f32_cuda(const float * x, float * dst,\n const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int ne13,\n const float sf0, const float sf1, const float sf2, const float sf3,\n cudaStream_t stream) {\n const int64_t dst_size = ne10 * ne11 * ne12 * ne13;\n const int64_t num_blocks = (dst_size + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE;\n\n upscale_f32<<>>(x, dst, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3);\n}\n\nstatic void upscale_f32_bilinear_cuda(const float * x, float * dst,\n const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne00_src, const int ne01_src,\n const int ne10_dst, const int ne11_dst, const int ne12_dst, const int ne13_dst,\n const float sf0, const float sf1, const float sf2, const float sf3,\n const float pixel_offset, bool antialias, cudaStream_t stream) {\n const int64_t dst_size = ne10_dst * ne11_dst * ne12_dst * ne13_dst;\n const int64_t num_blocks = (dst_size + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE;\n\n if (antialias) {\n upscale_f32_bilinear_antialias<<>>(x, dst, nb00, nb01, nb02, nb03, ne00_src, ne01_src, ne10_dst, ne11_dst, ne12_dst, ne13_dst, sf0, sf1, sf2, sf3, pixel_offset);\n } else {\n upscale_f32_bilinear<<>>(x, dst, nb00, nb01, nb02, nb03, ne00_src, ne01_src, ne10_dst, ne11_dst, ne12_dst, ne13_dst, sf0, sf1, sf2, sf3, pixel_offset);\n }\n}\n\nstatic void upscale_f32_bicubic_cuda(const float * x, float * dst,\n const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne00_src, const int ne01_src,\n const int ne10_dst, const int ne11_dst, const int ne12_dst, const int ne13_dst,\n const float sf0, const float sf1, const float sf2, const float sf3,\n const float pixel_offset, cudaStream_t stream) {\n const int64_t dst_size = ne10_dst * ne11_dst * ne12_dst * ne13_dst;\n const int64_t num_blocks = (dst_size + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE;\n\n upscale_f32_bicubic<<>>(x, dst, nb00, nb01, nb02, nb03, ne00_src, ne01_src, ne10_dst, ne11_dst, ne12_dst, ne13_dst, sf0, sf1, sf2, sf3, pixel_offset);\n}\n\nvoid ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const float * src0_d = (const float *)src0->data;\n float * dst_d = (float *)dst->data;\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n const int mode_flags = dst->op_params[0];\n const ggml_scale_mode mode = (ggml_scale_mode)(mode_flags & 0xFF);\n\n float sf0 = (float)dst->ne[0]/src0->ne[0];\n float sf1 = (float)dst->ne[1]/src0->ne[1];\n float sf2 = (float)dst->ne[2]/src0->ne[2];\n const float sf3 = (float)dst->ne[3]/src0->ne[3];\n\n float pixel_offset = 0.5f;\n if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) {\n sf0 = dst->ne[0] > 1 && src0->ne[0] > 1 ? (float)(dst->ne[0] - 1) / (src0->ne[0] - 1) : sf0;\n sf1 = dst->ne[1] > 1 && src0->ne[1] > 1 ? (float)(dst->ne[1] - 1) / (src0->ne[1] - 1) : sf1;\n pixel_offset = 0.0f;\n }\n\n if (mode == GGML_SCALE_MODE_NEAREST) {\n upscale_f32_cuda(src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], sf0, sf1, sf2, sf3, stream);\n } else if (mode == GGML_SCALE_MODE_BILINEAR) {\n const bool antialias = (mode_flags & GGML_SCALE_FLAG_ANTIALIAS);\n upscale_f32_bilinear_cuda(src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n src0->ne[0], src0->ne[1], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n sf0, sf1, sf2, sf3, pixel_offset, antialias, stream);\n } else if (mode == GGML_SCALE_MODE_BICUBIC) {\n upscale_f32_bicubic_cuda(src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n src0->ne[0], src0->ne[1], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n sf0, sf1, sf2, sf3, pixel_offset, stream);\n }\n}\n" }, { "path": "src/ggml-cuda/upscale.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_UPSCALE_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-cuda/vecdotq.cuh", "content": "#pragma once\n\n#include \"common.cuh\"\n\n#include \n\nstatic __device__ __forceinline__ int get_int_b1(const void * x, const int & i32) {\n const uint8_t * x8 = (const uint8_t *) x;\n\n int x32 = x8[4*i32 + 0] << 0;\n x32 |= x8[4*i32 + 1] << 8;\n x32 |= x8[4*i32 + 2] << 16;\n x32 |= x8[4*i32 + 3] << 24;\n\n return x32;\n}\n\nstatic __device__ __forceinline__ int get_int_b2(const void * x, const int & i32) {\n const uint16_t * x16 = (const uint16_t *) x; // assume at least 2 byte alignment\n\n int x32 = x16[2*i32 + 0] << 0;\n x32 |= x16[2*i32 + 1] << 16;\n\n return x32;\n}\n\nstatic __device__ __forceinline__ int get_int_b4(const void * x, const int & i32) {\n return ((const int *) x)[i32]; // assume at least 4 byte alignment\n}\n\n// q4 contains 8 indices with 4 bit each.\n// This function selects those bytes from table that are at those indices and returns them as int2.\n// The first int contains the bytes with even indices in q4, the second int contains the bytes with odd indices in q4.\nstatic __device__ __forceinline__ int2 get_int_from_table_16(const int & q4, const int8_t * table) {\n#if defined(GGML_USE_HIP)\n // Load the 16-byte table into four 32-bit unsigned integers.\n const uint32_t *values = (const uint32_t *)table;\n\n const uint32_t q_even = q4;\n const uint32_t q_odd = (q4 >> 4);\n\n // Perform lookups in the lower half of the table (indices 0-7).\n uint32_t v_even_low = __builtin_amdgcn_perm(values[1], values[0], q_even & 0x07070707);\n uint32_t v_odd_low = __builtin_amdgcn_perm(values[1], values[0], q_odd & 0x07070707);\n\n // Perform lookups in the upper half of the table (indices 8-15).\n uint32_t v_even_high = __builtin_amdgcn_perm(values[3], values[2], q_even & 0x07070707);\n uint32_t v_odd_high = __builtin_amdgcn_perm(values[3], values[2], q_odd & 0x07070707);\n\n // Select between the low and high results based on the MSB of each index nibble.\n uint32_t mask_even = 0x03020100 | ((q_even & 0x08080808) >> 1);\n uint32_t res_x = __builtin_amdgcn_perm(v_even_high, v_even_low, mask_even);\n uint32_t mask_odd = 0x03020100 | ((q_odd & 0x08080808) >> 1);\n uint32_t res_y = __builtin_amdgcn_perm(v_odd_high, v_odd_low, mask_odd);\n\n return make_int2(res_x, res_y);\n#elif !defined(GGML_USE_MUSA)\n // CUDA does not have an instruction for selecting bytes with 4 bit indices.\n // However, __byte_perm is an instruction that selects bytes with 3 bit indices that can be used instead.\n const uint32_t * table32 = (const uint32_t *) table;\n\n // __byte_perm selects bytes based on the lower 16 bits in its third argument.\n // Therefore, do 2 iterations over the 32 bits in q4 with 0 and 16 shift.\n // To handle the fourth bit, first call _byte_perm both for the low and the high 64 bit of table, using the low 3 bits.\n // Then, call __byte_perm again to select from the low and high bytes based on the fourth bit.\n uint32_t tmp[2];\n const uint32_t low_high_selection_indices = (0x32103210 | ((q4 & 0x88888888) >> 1));\n#pragma unroll\n for (uint32_t i = 0; i < 2; ++i) {\n const uint32_t shift = 16 * i;\n\n const uint32_t low = __byte_perm(table32[0], table32[1], q4 >> shift);\n const uint32_t high = __byte_perm(table32[2], table32[3], q4 >> shift);\n tmp[i] = __byte_perm(low, high, low_high_selection_indices >> shift);\n }\n\n // tmp contains the bytes from tyble in the same order as the 4 bit indices in q4.\n // However, for the result we need ints with all even/odd 4 bit indices in q4.\n // Therefore, 2 more calls to __byte_perm to put the bytes in the correct order.\n return make_int2(__byte_perm(tmp[0], tmp[1], 0x6420), __byte_perm(tmp[0], tmp[1], 0x7531));\n#else\n // Generic implementation.\n const int q0_32 = (q4 >> 0) & 0x0F0F0F0F;\n const int8_t * q0_8 = (const int8_t *) &q0_32;\n const char4 val0_8 = make_char4(\n table[q0_8[0]], table[q0_8[1]], table[q0_8[2]], table[q0_8[3]]);\n\n const int q1_32 = (q4 >> 4) & 0x0F0F0F0F;\n const int8_t * q1_8 = (const int8_t *) &q1_32;\n const char4 val1_8 = make_char4(\n table[q1_8[0]], table[q1_8[1]], table[q1_8[2]], table[q1_8[3]]);\n\n return make_int2(*((const int *) &val0_8), *((const int *) &val1_8));\n#endif\n}\n\nstatic __device__ __forceinline__ uint32_t unpack_ksigns(const uint8_t v) {\n // v is a 7 bit int, with the 8th sign being encodable as popcnt\n // with xor we can \"correct\" the bit instead of having to mask\n const uint32_t p = __popc(v) & 1;\n const uint32_t s = v ^ p << 7;\n // broadcast over uint to allow for 0x08040201 / 0x80402010 as selectors\n return s * 0x01010101;\n}\n\n// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called\n// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q\n\n#define VDR_Q4_0_Q8_1_MMVQ 2\n#define VDR_Q4_0_Q8_1_MMQ 4\n\ntemplate static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(\n const int * v, const int * u, const float & d4, const half2 & ds8) {\n\n int sumi = 0;\n\n#pragma unroll\n for (int i = 0; i < vdr; ++i) {\n const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;\n const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;\n\n // SIMD dot product of quantized values\n sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);\n sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);\n }\n\n const float2 ds8f = __half22float2(ds8);\n\n // second part effectively subtracts 8 from each quant value\n return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);\n}\n\n#define VDR_Q4_1_Q8_1_MMVQ 2\n#define VDR_Q4_1_Q8_1_MMQ 4\n\ntemplate static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(\n const int * v, const int * u, const half2 & dm4, const half2 & ds8) {\n\n int sumi = 0;\n\n#pragma unroll\n for (int i = 0; i < vdr; ++i) {\n const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;\n const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;\n\n // SIMD dot product of quantized values\n sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);\n sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);\n }\n\n#ifdef FAST_FP16_AVAILABLE\n const float2 tmp = __half22float2(__hmul2(dm4, ds8));\n const float d4d8 = tmp.x;\n const float m4s8 = tmp.y;\n#else\n const float2 dm4f = __half22float2(dm4);\n const float2 ds8f = __half22float2(ds8);\n const float d4d8 = dm4f.x * ds8f.x;\n const float m4s8 = dm4f.y * ds8f.y;\n#endif // FAST_FP16_AVAILABLE\n\n // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it\n return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));\n}\n\n#define VDR_Q5_0_Q8_1_MMVQ 2\n#define VDR_Q5_0_Q8_1_MMQ 4\n\ntemplate static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(\n const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {\n\n int sumi = 0;\n\n#pragma unroll\n for (int i = 0; i < vdr; ++i) {\n int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits\n vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4\n vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12\n vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20\n vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28\n sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values\n\n int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits\n vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4\n vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12\n vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20\n vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28\n sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values\n }\n\n const float2 ds8f = __half22float2(ds8);\n\n // second part effectively subtracts 16 from each quant value\n return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);\n}\n\n#define VDR_Q5_1_Q8_1_MMVQ 2\n#define VDR_Q5_1_Q8_1_MMQ 4\n\ntemplate static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(\n const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {\n\n int sumi = 0;\n\n#pragma unroll\n for (int i = 0; i < vdr; ++i) {\n int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits\n vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4\n vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12\n vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20\n vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28\n sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values\n\n int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits\n vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4\n vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12\n vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20\n vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28\n sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values\n }\n\n#ifdef FAST_FP16_AVAILABLE\n const float2 tmp = __half22float2(__hmul2(dm5, ds8));\n const float d5d8 = tmp.x;\n const float m5s8 = tmp.y;\n#else\n const float2 dm5f = __half22float2(dm5);\n const float2 ds8f = __half22float2(ds8);\n const float d5d8 = dm5f.x * ds8f.x;\n const float m5s8 = dm5f.y * ds8f.y;\n#endif // FAST_FP16_AVAILABLE\n\n // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it\n return sumi*d5d8 + m5s8 / (QI5_1 / vdr);\n}\n\n#define VDR_Q8_0_Q8_1_MMVQ 2\n#define VDR_Q8_0_Q8_1_MMQ 8\n\ntemplate static __device__ __forceinline__ T vec_dot_q8_0_q8_1_impl(\n const int * v, const int * u, const T & d8_0, const T & d8_1) {\n\n int sumi = 0;\n\n#pragma unroll\n for (int i = 0; i < vdr; ++i) {\n // SIMD dot product of quantized values\n sumi = ggml_cuda_dp4a(v[i], u[i], sumi);\n }\n\n return d8_0*d8_1 * ((T) sumi);\n}\n\ntemplate static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(\n const int * v, const int * u, const half2 & dm8, const half2 & ds8) {\n\n int sumi = 0;\n\n#pragma unroll\n for (int i = 0; i < vdr; ++i) {\n // SIMD dot product of quantized values\n sumi = ggml_cuda_dp4a(v[i], u[i], sumi);\n }\n\n#ifdef FAST_FP16_AVAILABLE\n const float2 tmp = __half22float2(__hmul2(dm8, ds8));\n const float d8d8 = tmp.x;\n const float m8s8 = tmp.y;\n#else\n const float2 dm8f = __half22float2(dm8);\n const float2 ds8f = __half22float2(ds8);\n const float d8d8 = dm8f.x * ds8f.x;\n const float m8s8 = dm8f.y * ds8f.y;\n#endif // FAST_FP16_AVAILABLE\n\n // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it\n return sumi*d8d8 + m8s8 / (QI8_1 / vdr);\n}\n\ntemplate static __device__ __forceinline__ float vec_dot_q8_0_16_q8_1_impl(\n const int * v, const int * u, const float * d8_0, const float & d8_1) {\n\n float sumf = 0.0f;\n\n#pragma unroll\n for (int i0 = 0; i0 < vdr; i0 += QI8_0/2) {\n int sumi = 0;\n\n#pragma unroll\n for (int i = i0; i < i0 + QI8_0/2; ++i) {\n // SIMD dot product of quantized values\n sumi = ggml_cuda_dp4a(v[i], u[i], sumi);\n }\n\n sumf += d8_0[i0/(QI8_0/2)]*sumi;\n }\n\n return d8_1*sumf;\n}\n\n#define VDR_MXFP4_Q8_1_MMVQ 2\n#define VDR_MXFP4_Q8_1_MMQ 4\n\nstatic __device__ __forceinline__ float vec_dot_mxfp4_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_mxfp4 * bq4 = (const block_mxfp4 *) vbq + kbx;\n\n const int * q8 = (const int *) bq8_1->qs + iqs;\n\n int sumi = 0;\n#pragma unroll\n for (int l = 0; l < VDR_MXFP4_Q8_1_MMVQ; ++l) {\n const int aux_q4 = get_int_b1(bq4->qs, iqs + l);\n const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4);\n\n sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi);\n sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi);\n }\n\n const float d = ggml_cuda_e8m0_to_fp32(bq4->e) * 0.5f * __low2float(bq8_1->ds);\n return d * sumi;\n}\n\n#define VDR_Q2_K_Q8_1_MMVQ 1\n#define VDR_Q2_K_Q8_1_MMQ 4\n\n// contiguous v/x values\nstatic __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(\n const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,\n const half2 & dm2, const float * __restrict__ d8) {\n\n float sumf_d = 0.0f;\n float sumf_m = 0.0f;\n\n#pragma unroll\n for (int i = 0; i < QR2_K; ++i) {\n const int sc = scales[2*i];\n\n const int vi = (v >> (2*i)) & 0x03030303;\n\n sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product\n\n // fill int with 4x m\n int m = sc >> 4;\n m |= m << 8;\n m |= m << 16;\n sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values\n }\n\n const float2 dm2f = __half22float2(dm2);\n\n return dm2f.x*sumf_d - dm2f.y*sumf_m;\n}\n\n// contiguous v/x + u/y values\ntemplate \nstatic __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(\n const int * __restrict__ v, const int * __restrict__ u, const half2 * dm2, const float & d8, const half2 * s8) {\n\n float sumf = 0.0f;\n float sumf_d8 = 0.0f;\n\n#pragma unroll\n for (int i0 = 0; i0 < QR2_K*VDR_Q2_K_Q8_1_MMQ; i0 += QI8_1) {\n const float2 dm2f0 = __half22float2(dm2[i0/(QI8_1/2) + 0]);\n int sumi_d0 = 0;\n\n const float2 dm2f1 = __half22float2(dm2[i0/(QI8_1/2) + 1]);\n int sumi_d1 = 0;\n\n#pragma unroll\n for (int i = i0; i < i0 + QI8_1/2; ++i) {\n sumi_d0 = ggml_cuda_dp4a(v[i], u[i], sumi_d0);\n }\n sumf_d8 += dm2f0.x * sumi_d0;\n\n#pragma unroll\n for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {\n sumi_d1 = ggml_cuda_dp4a(v[i], u[i], sumi_d1);\n }\n sumf_d8 += dm2f1.x * sumi_d1;\n\n if (i0/QI8_1 < ns8) {\n const float2 s8f = __half22float2(s8[i0/QI8_1]);\n sumf -= dm2f0.y*s8f.x;\n sumf -= dm2f1.y*s8f.y;\n } else {\n int sumi_m0 = 0;\n#pragma unroll\n for (int i = i0; i < i0 + QI8_1/2; ++i) {\n sumi_m0 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m0);\n }\n sumf_d8 -= dm2f0.y * sumi_m0;\n\n int sumi_m1 = 0;\n#pragma unroll\n for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {\n sumi_m1 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m1);\n }\n sumf_d8 -= dm2f1.y * sumi_m1;\n }\n }\n\n return sumf + d8*sumf_d8;\n}\n\n#define VDR_Q3_K_Q8_1_MMVQ 1\n#define VDR_Q3_K_Q8_1_MMQ 2\n\n// contiguous v/x values\nstatic __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(\n const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,\n const int & scale_offset, const float & d3, const float * __restrict__ d8) {\n\n float sumf = 0.0f;\n\n#pragma unroll\n for (int i = 0; i < QR3_K; ++i) {\n const int isc = scale_offset + 2*i;\n\n const int isc_low = isc % (QK_K/32);\n const int sc_shift_low = 4 * (isc / (QK_K/32));\n const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;\n\n const int isc_high = isc % (QK_K/64);\n const int sc_shift_high = 2 * (isc / (QK_K/64));\n const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;\n\n const int sc = (sc_low | sc_high) - 32;\n\n const int vil = (vl >> (2*i)) & 0x03030303;\n\n const int vih = ((vh >> i) << 2) & 0x04040404;\n\n const int vi = __vsubss4(vil, vih);\n\n sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product\n }\n\n return d3 * sumf;\n}\n\n// contiguous v/x + u/y values\nstatic __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(\n const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,\n const float & d3, const float & d8) {\n\n int sumi = 0;\n\n#pragma unroll\n for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {\n int sumi_sc = 0;\n\n#pragma unroll\n for (int i = i0; i < i0 + QI8_1/2; ++i) {\n sumi_sc = ggml_cuda_dp4a(v[i], u[i], sumi_sc); // SIMD dot product\n }\n\n sumi += sumi_sc * scales[i0 / (QI8_1/2)];\n }\n\n return d3*d8 * sumi;\n}\n\n#define VDR_Q4_K_Q8_1_MMVQ 2\n#define VDR_Q4_K_Q8_1_MMQ 8\n\n// contiguous v/x values\nstatic __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(\n const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,\n const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {\n\n float sumf_d = 0.0f;\n float sumf_m = 0.0f;\n\n#pragma unroll\n for (int i = 0; i < QR4_K; ++i) {\n const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;\n const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;\n\n const int dot1 = ggml_cuda_dp4a(v1i, u[2*i+1], ggml_cuda_dp4a(v0i, u[2*i+0], 0)); // SIMD dot product\n const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+1], ggml_cuda_dp4a(0x01010101, u[2*i+0], 0)); // sum of u\n\n sumf_d += d8[i] * (dot1 * sc[i]);\n sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values\n }\n\n const float2 dm4f = __half22float2(dm4);\n\n return dm4f.x*sumf_d - dm4f.y*sumf_m;\n}\n\n// contiguous v/x + u/y values\nstatic __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(\n const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,\n const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {\n\n float sumf_d = 0.0f;\n float sumf_m = 0.0f;\n\n#pragma unroll\n for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {\n int sumi_d = 0;\n\n#pragma unroll\n for (int j = 0; j < QI8_1; ++j) {\n sumi_d = ggml_cuda_dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product\n }\n\n const float2 ds8f = __half22float2(ds8[i]);\n\n sumf_d += ds8f.x * (sc[i] * sumi_d);\n sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val\n }\n\n const float2 dm4f = __half22float2(dm4);\n\n return dm4f.x*sumf_d - dm4f.y*sumf_m;\n}\n\n#define VDR_Q5_K_Q8_1_MMVQ 2\n#define VDR_Q5_K_Q8_1_MMQ 8\n\n// contiguous v/x values\nstatic __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(\n const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,\n const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {\n\n float sumf_d = 0.0f;\n float sumf_m = 0.0f;\n\n#pragma unroll\n for (int i = 0; i < QR5_K; ++i) {\n const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;\n const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;\n\n const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;\n const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;\n\n const int v0i = vl0i | vh0i;\n const int v1i = vl1i | vh1i;\n\n const int dot1 = ggml_cuda_dp4a(v0i, u[2*i+0], ggml_cuda_dp4a(v1i, u[2*i+1], 0)); // SIMD dot product\n const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+0], ggml_cuda_dp4a(0x01010101, u[2*i+1], 0)); // sum of u\n\n sumf_d += d8[i] * (dot1 * sc[i]);\n sumf_m += d8[i] * (dot2 * m[i]);\n\n }\n\n const float2 dm5f = __half22float2(dm5);\n\n return dm5f.x*sumf_d - dm5f.y*sumf_m;\n}\n\n// contiguous v/x + u/y values\nstatic __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(\n const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,\n const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {\n\n float sumf_d = 0.0f;\n float sumf_m = 0.0f;\n\n#pragma unroll\n for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {\n int sumi_d = 0;\n\n#pragma unroll\n for (int j = 0; j < QI8_1; ++j) {\n sumi_d = ggml_cuda_dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product\n }\n\n const float2 ds8f = __half22float2(ds8[i]);\n\n sumf_d += ds8f.x * (sc[i] * sumi_d);\n sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val\n }\n\n const float2 dm4f = __half22float2(dm4);\n\n return dm4f.x*sumf_d - dm4f.y*sumf_m;\n}\n\n#define VDR_Q6_K_Q8_1_MMVQ 1\n#define VDR_Q6_K_Q8_1_MMQ 8\n\n// contiguous v/x values\nstatic __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(\n const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,\n const float & d, const float * __restrict__ d8) {\n\n float sumf = 0.0f;\n\n#pragma unroll\n for (int i = 0; i < QR6_K; ++i) {\n const int sc = scales[4*i];\n\n const int vil = (vl >> (4*i)) & 0x0F0F0F0F;\n\n const int vih = ((vh >> (4*i)) << 4) & 0x30303030;\n\n const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32\n\n sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product\n }\n\n return d*sumf;\n}\n\n// contiguous v/x + u/y values\nstatic __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(\n const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,\n const float & d6, const float * __restrict__ d8) {\n\n float sumf_d = 0.0f;\n\n const int sc_packed = get_int_b4(sc, 0);\n const int8_t * sc_reg = (const int8_t *) &sc_packed;\n\n#pragma unroll\n for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {\n int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale\n\n#pragma unroll\n for (int i = i0; i < i0 + 2; ++i) {\n sumi_d.x = ggml_cuda_dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product\n sumi_d.x = ggml_cuda_dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product\n\n sumi_d.y = ggml_cuda_dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product\n sumi_d.y = ggml_cuda_dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product\n }\n\n sumf_d += d8[i0/4] * (sc_reg[i0/2+0]*sumi_d.x + sc_reg[i0/2+1]*sumi_d.y);\n }\n\n return d6 * sumf_d;\n}\n\nstatic __device__ __forceinline__ float vec_dot_q4_0_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq + kbx;\n\n int v[VDR_Q4_0_Q8_1_MMVQ];\n int u[2*VDR_Q4_0_Q8_1_MMVQ];\n\n#pragma unroll\n for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {\n v[i] = get_int_b2(bq4_0->qs, iqs + i);\n u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);\n u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_0);\n }\n\n return vec_dot_q4_0_q8_1_impl(v, u, bq4_0->d, bq8_1->ds);\n}\n\n\nstatic __device__ __forceinline__ float vec_dot_q4_1_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq + kbx;\n\n int v[VDR_Q4_1_Q8_1_MMVQ];\n int u[2*VDR_Q4_1_Q8_1_MMVQ];\n\n#pragma unroll\n for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {\n v[i] = get_int_b4(bq4_1->qs, iqs + i);\n u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);\n u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_1);\n }\n\n return vec_dot_q4_1_q8_1_impl(v, u, bq4_1->dm, bq8_1->ds);\n}\n\nstatic __device__ __forceinline__ float vec_dot_q5_0_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq + kbx;\n\n int vl[VDR_Q5_0_Q8_1_MMVQ];\n int vh[VDR_Q5_0_Q8_1_MMVQ];\n int u[2*VDR_Q5_0_Q8_1_MMVQ];\n\n#pragma unroll\n for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {\n vl[i] = get_int_b2(bq5_0->qs, iqs + i);\n vh[i] = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i));\n u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);\n u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_0);\n }\n\n return vec_dot_q5_0_q8_1_impl(vl, vh, u, bq5_0->d, bq8_1->ds);\n}\n\nstatic __device__ __forceinline__ float vec_dot_q5_1_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq + kbx;\n\n int vl[VDR_Q5_1_Q8_1_MMVQ];\n int vh[VDR_Q5_1_Q8_1_MMVQ];\n int u[2*VDR_Q5_1_Q8_1_MMVQ];\n\n#pragma unroll\n for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {\n vl[i] = get_int_b4(bq5_1->qs, iqs + i);\n vh[i] = get_int_b4(bq5_1->qh, 0) >> (4 * (iqs + i));\n u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);\n u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_1);\n }\n\n return vec_dot_q5_1_q8_1_impl(vl, vh, u, bq5_1->dm, bq8_1->ds);\n}\n\nstatic __device__ __forceinline__ float vec_dot_q8_0_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq + kbx;\n\n int v[VDR_Q8_0_Q8_1_MMVQ];\n int u[VDR_Q8_0_Q8_1_MMVQ];\n\n#pragma unroll\n for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {\n v[i] = get_int_b2(bq8_0->qs, iqs + i);\n u[i] = get_int_b4(bq8_1->qs, iqs + i);\n }\n\n return vec_dot_q8_0_q8_1_impl(v, u, bq8_0->d, __low2half(bq8_1->ds));\n}\n\nstatic __device__ __forceinline__ float vec_dot_q2_K_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx;\n\n const int bq8_offset = QR2_K * (iqs / QI8_1);\n const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);\n\n const uint8_t * scales = bq2_K->scales + scale_offset;\n\n const int v = get_int_b4(bq2_K->qs, iqs);\n int u[QR2_K];\n float d8[QR2_K];\n\n#pragma unroll\n for (int i = 0; i < QR2_K; ++ i) {\n u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);\n d8[i] = __low2float(bq8_1[bq8_offset + i].ds);\n }\n\n return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);\n}\n\nstatic __device__ __forceinline__ float vec_dot_q3_K_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx;\n\n const int bq8_offset = QR3_K * (iqs / (QI3_K/2));\n const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);\n\n const float d = bq3_K->d;\n\n const int vl = get_int_b2(bq3_K->qs, iqs);\n\n // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted\n const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;\n\n int u[QR3_K];\n float d8[QR3_K];\n\n#pragma unroll\n for (int i = 0; i < QR3_K; ++i) {\n u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);\n d8[i] = __low2float(bq8_1[bq8_offset + i].ds);\n }\n\n return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);\n}\n\nstatic __device__ __forceinline__ float vec_dot_q4_K_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q4_K * bq4_K = (const block_q4_K *) vbq + kbx;\n\n int v[2];\n int u[2*QR4_K];\n float d8[QR4_K];\n\n // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6\n const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));\n\n // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12\n // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44\n // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76\n // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108\n\n const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));\n v[0] = q4[0];\n v[1] = q4[4];\n\n const uint16_t * scales = (const uint16_t *)bq4_K->scales;\n uint16_t aux[2];\n const int j = bq8_offset/2;\n if (j < 2) {\n aux[0] = scales[j+0] & 0x3f3f;\n aux[1] = scales[j+2] & 0x3f3f;\n } else {\n aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);\n aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);\n }\n const uint8_t * sc = (const uint8_t *)aux;\n const uint8_t * m = sc + 2;\n\n for (int i = 0; i < QR4_K; ++i) {\n const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;\n d8[i] = __low2float(bq8i->ds);\n\n const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);\n u[2*i+0] = q8[0];\n u[2*i+1] = q8[4];\n }\n\n return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);\n}\n\nstatic __device__ __forceinline__ float vec_dot_q5_K_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q5_K * bq5_K = (const block_q5_K *) vbq + kbx;\n\n int vl[2];\n int vh[2];\n int u[2*QR5_K];\n float d8[QR5_K];\n\n const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));\n const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));\n const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));\n\n vl[0] = ql[0];\n vl[1] = ql[4];\n\n vh[0] = qh[0] >> bq8_offset;\n vh[1] = qh[4] >> bq8_offset;\n\n const uint16_t * scales = (const uint16_t *)bq5_K->scales;\n uint16_t aux[2];\n const int j = bq8_offset/2;\n if (j < 2) {\n aux[0] = scales[j+0] & 0x3f3f;\n aux[1] = scales[j+2] & 0x3f3f;\n } else {\n aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);\n aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);\n }\n const uint8_t * sc = (const uint8_t *)aux;\n const uint8_t * m = sc + 2;\n\n#pragma unroll\n for (int i = 0; i < QR5_K; ++i) {\n const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;\n d8[i] = __low2float(bq8i->ds);\n\n const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);\n u[2*i+0] = q8[0];\n u[2*i+1] = q8[4];\n }\n\n return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);\n}\n\nstatic __device__ __forceinline__ float vec_dot_q6_K_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_q6_K * bq6_K = (const block_q6_K *) vbq + kbx;\n\n const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);\n const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);\n const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));\n\n const int vl = get_int_b2(bq6_K->ql, iqs);\n const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;\n\n const int8_t * scales = bq6_K->scales + scale_offset;\n\n int u[QR6_K];\n float d8[QR6_K];\n\n#pragma unroll\n for (int i = 0; i < QR6_K; ++i) {\n u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);\n d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);\n }\n\n return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);\n}\n\n#define VDR_IQ2_XXS_Q8_1_MMVQ 2\n#define VDR_IQ2_XXS_Q8_1_MMQ 2\n\nstatic __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq + kbx;\n\n const int q2 = get_int_b2(bq2->qs, iqs);\n const uint8_t * aux8 = (const uint8_t *) &q2;\n const uint32_t aux32 = get_int_b2(bq2->qs, iqs + 1);\n\n int sumi = 0;\n#pragma unroll\n for (int k0 = 0; k0 < 8; k0 += 2) {\n const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[k0/2]];\n const uint32_t signs = unpack_ksigns(aux32 >> (7 * k0 / 2));\n\n const int signs0 = __vcmpne4(signs & 0x08040201, 0);\n const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0);\n const int u0 = get_int_b4(bq8_1[iqs/2].qs, k0 + 0);\n sumi = ggml_cuda_dp4a(grid0, u0, sumi);\n\n const int signs1 = __vcmpne4(signs & 0x80402010, 0);\n const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1);\n const int u1 = get_int_b4(bq8_1[iqs/2].qs, k0 + 1);\n sumi = ggml_cuda_dp4a(grid1, u1, sumi);\n }\n\n const int ls = aux32 >> 27 | 1; // (scale * 2 + 1)\n sumi = sumi * ls / 8; // (sumi * scale + sumi / 2) / 4\n const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);\n return d * sumi;\n}\n\n#define VDR_IQ2_XS_Q8_1_MMVQ 2\n#define VDR_IQ2_XS_Q8_1_MMQ 2\n\nstatic __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq + kbx;\n\n const int2 q2_packed = make_int2(get_int_b2(bq2->qs, iqs + 0), get_int_b2(bq2->qs, iqs + 1));\n const uint16_t * q2 = (const uint16_t *) &q2_packed;\n const int ls0 = bq2->scales[iqs/2] & 0x0F;\n const int ls1 = bq2->scales[iqs/2] >> 4;\n\n int sumi0 = 0;\n int sumi1 = 0;\n#pragma unroll\n for (int l0 = 0; l0 < 8; l0 += 2) {\n const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l0/2] & 0x1FF];\n const uint32_t signs = unpack_ksigns(q2[l0/2] >> 9);\n\n const int signs0 = __vcmpne4(signs & 0x08040201, 0);\n const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);\n const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);\n\n const int signs1 = __vcmpne4(signs & 0x80402010, 0);\n const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);\n const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);\n\n if (l0 < 4) {\n sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);\n sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);\n } else {\n sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);\n sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);\n }\n }\n const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;\n const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);\n return d * sumi;\n}\n\n#define VDR_IQ2_S_Q8_1_MMVQ 2\n#define VDR_IQ2_S_Q8_1_MMQ 2\n\nstatic __device__ __forceinline__ float vec_dot_iq2_s_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_iq2_s * bq2 = (const block_iq2_s *) vbq + kbx;\n\n const int qs_packed = get_int_b2(bq2->qs, iqs/2);\n const uint8_t * qs = (const uint8_t *) &qs_packed;\n\n const int qh = bq2->qh[iqs/2];\n\n const int signs_packed_32 = get_int_b2(bq2->qs, QK_K/32 + iqs/2);\n const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;\n\n const int ls0 = bq2->scales[iqs/2] & 0x0F;\n const int ls1 = bq2->scales[iqs/2] >> 4;\n\n int sumi0 = 0;\n int sumi1 = 0;\n#pragma unroll\n for (int l0 = 0; l0 < 8; l0 += 2) {\n const int * grid_pos = (const int *)(iq2s_grid + (qs[l0/2] | ((qh << (8-l0)) & 0x300)));\n\n const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);\n const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);\n\n const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);\n const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);\n\n const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);\n const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);\n\n if (l0 < 4) {\n sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);\n sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);\n } else {\n sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);\n sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);\n }\n }\n const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;\n\n const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);\n return d * sumi;\n}\n\n#define VDR_IQ3_XXS_Q8_1_MMVQ 2\n#define VDR_IQ3_XXS_Q8_1_MMQ 2\n\nstatic __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_iq3_xxs * bq3 = (const block_iq3_xxs *) vbq + kbx;\n\n const int2 q3_packed = make_int2(get_int_b2(bq3->qs, iqs), get_int_b2(bq3->qs, iqs+1));\n const uint8_t * q3 = (const uint8_t *) &q3_packed;\n const uint32_t aux32 = get_int_b2(bq3->qs, QK_K/16 + iqs/2);\n\n int sumi = 0;\n#pragma unroll\n for (int l0 = 0; l0 < 8; l0 += 2) {\n const int2 grid_pos = make_int2(iq3xxs_grid[q3[l0 + 0]], iq3xxs_grid[q3[l0 + 1]]);\n const uint32_t signs = unpack_ksigns(aux32 >> (7*l0/2));\n\n const int signs0 = __vcmpne4(signs & 0x08040201, 0);\n const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);\n\n const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);\n\n const int signs1 = __vcmpne4(signs & 0x80402010, 0);\n const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);\n\n const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);\n\n sumi = ggml_cuda_dp4a(grid_l, u0, sumi);\n sumi = ggml_cuda_dp4a(grid_h, u1, sumi);\n }\n\n const int ls = aux32 >> 28;\n sumi = (ls*sumi + sumi/2)/2;\n const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);\n return d * sumi;\n}\n\n#define VDR_IQ3_S_Q8_1_MMVQ 2\n#define VDR_IQ3_S_Q8_1_MMQ 2\n\n// TODO: don't use lookup table for signs\nstatic __device__ __forceinline__ float vec_dot_iq3_s_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_iq3_s * bq3 = (const block_iq3_s *) vbq + kbx;\n\n const int2 qs_packed = make_int2(get_int_b2(bq3->qs, iqs + 0), get_int_b2(bq3->qs, iqs + 1));\n const uint8_t * qs = (const uint8_t *) &qs_packed;\n\n const int qh = bq3->qh[iqs/2];\n\n const int signs_packed_32 = get_int_b2(bq3->signs, iqs/2);\n const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;\n\n int sumi = 0;\n#pragma unroll\n for (int l0 = 0; l0 < 8; l0 += 2) {\n const int2 grid_pos = make_int2(\n iq3s_grid[qs[l0 + 0] | ((qh << (8 - l0)) & 0x100)],\n iq3s_grid[qs[l0 + 1] | ((qh << (7 - l0)) & 0x100)]);\n\n const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);\n const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);\n\n const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);\n const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);\n\n const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);\n const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);\n\n sumi = ggml_cuda_dp4a(grid_l, u0, sumi);\n sumi = ggml_cuda_dp4a(grid_h, u1, sumi);\n }\n\n sumi *= 1 + 2*((bq3->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F);\n\n const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);\n return d * sumi;\n}\n\n#define VDR_IQ1_S_Q8_1_MMVQ 1\n#define VDR_IQ1_S_Q8_1_MMQ 1\n\nstatic __device__ __forceinline__ float vec_dot_iq1_s_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n const block_iq1_s * bq1 = (const block_iq1_s *) vbq + kbx;\n\n const int qs_packed = get_int_b2(bq1->qs, iqs);\n const uint8_t * qs = (const uint8_t *) &qs_packed;\n\n const int qh = bq1->qh[iqs];\n\n int sumi = 0;\n#pragma unroll\n for (int l0 = 0; l0 < 8; l0 += 2) {\n const int grid = iq1s_grid_gpu[qs[l0/2] | (((qh >> 3*(l0/2)) & 0x07) << 8)];\n\n const int grid0 = (grid >> 0) & 0x0F0F0F0F;\n const int grid1 = (grid >> 4) & 0x0F0F0F0F;\n\n const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);\n const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);\n\n sumi = ggml_cuda_dp4a(grid0, u0, sumi);\n sumi = ggml_cuda_dp4a(grid1, u1, sumi);\n }\n\n const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1);\n const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);\n const float2 ds = __half22float2(bq8_1[iqs].ds);\n return d1q * (ds.x*sumi + ds.y*delta);\n}\n\n#define VDR_IQ1_M_Q8_1_MMVQ 1\n#define VDR_IQ1_M_Q8_1_MMQ 1\n\nstatic __device__ __forceinline__ float vec_dot_iq1_m_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_iq1_m * bq1 = (const block_iq1_m *) vbq + kbx;\n\n const int qs_packed = get_int_b4(bq1->qs, iqs);\n const uint8_t * qs = (const uint8_t *) &qs_packed;\n\n int sumi[2] = {0};\n float sumf[2] = {0.0f};\n#pragma unroll\n for (int l0 = 0; l0 < 8; l0 += 2) {\n const int qhl = bq1->qh[2*iqs + l0/4] >> (4 * ((l0/2) % 2));\n\n const int grid = iq1s_grid_gpu[qs[l0/2] | ((qhl & 0x07) << 8)];\n\n const int grid0 = (grid >> 0) & 0x0F0F0F0F;\n const int grid1 = (grid >> 4) & 0x0F0F0F0F;\n\n const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);\n const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);\n\n sumi[l0/4] = ggml_cuda_dp4a(grid0, u0, sumi[l0/4]);\n sumi[l0/4] = ggml_cuda_dp4a(grid1, u1, sumi[l0/4]);\n\n const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f*IQ1M_DELTA/0x08);\n int sumy = 0;\n sumy = ggml_cuda_dp4a(u0, 0x01010101, sumy);\n sumy = ggml_cuda_dp4a(u1, 0x01010101, sumy);\n sumf[l0/4] += delta*sumy;\n }\n\n const uint16_t * sc = (const uint16_t *) bq1->scales;\n\n iq1m_scale_t scale;\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);\n const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds);\n\n const int tmp = sc[iqs/2] >> (6*(iqs%2));\n const int sc0 = 2*((tmp >> 0) & 0x07) + 1;\n const int sc1 = 2*((tmp >> 3) & 0x07) + 1;\n return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);\n}\n\n#define VDR_IQ4_NL_Q8_1_MMVQ 2\n#define VDR_IQ4_NL_Q8_1_MMQ 4\n\nstatic __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_iq4_nl * bq4 = (const block_iq4_nl *) vbq + kbx;\n\n const int * q8 = (const int *) bq8_1->qs + iqs;\n\n int sumi = 0;\n#pragma unroll\n for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {\n const int aux_q4 = get_int_b2(bq4->qs, iqs + l);\n const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);\n\n sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi);\n sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi);\n }\n\n const float d = __half2float(bq4->d) * __low2float(bq8_1->ds);\n return d * sumi;\n}\n\n#define VDR_IQ4_XS_Q8_1_MMVQ 4\n#define VDR_IQ4_XS_Q8_1_MMQ 4\n\nstatic __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(\n const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {\n\n const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq + kbx;\n\n int sumi = 0;\n#pragma unroll\n for (int j = 0; j < 4; ++j) {\n const int aux_q4 = get_int_b4(bq4->qs, iqs + j);\n const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);\n\n const int u0 = get_int_b4(bq8_1[iqs/4].qs, j + 0);\n const int u1 = get_int_b4(bq8_1[iqs/4].qs, j + 4);\n\n sumi = ggml_cuda_dp4a(v.x, u0, sumi);\n sumi = ggml_cuda_dp4a(v.y, u1, sumi);\n }\n\n const int ls = ((bq4->scales_l[iqs/8] >> (iqs & 0x04)) & 0x0F) | (((bq4->scales_h >> (iqs/2)) & 0x03) << 4);\n sumi *= ls - 32;\n\n const float d = __half2float(bq4->d) * __low2float(bq8_1[iqs/4].ds);\n return d * sumi;\n}\n" }, { "path": "src/ggml-cuda/vendors/cuda.h", "content": "#pragma once\n\n#include \n#include \n#include \n#include \n#include \n\n#if CUDART_VERSION >= 12050\n#include \n#endif // CUDART_VERSION >= 12050\n\n#if CUDART_VERSION >= 12080\n#include \n#endif // CUDART_VERSION >= 12080\n\n#if CUDART_VERSION < 11020\n#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED\n#define CUBLAS_TF32_TENSOR_OP_MATH CUBLAS_TENSOR_OP_MATH\n#define CUBLAS_COMPUTE_16F CUDA_R_16F\n#define CUBLAS_COMPUTE_32F CUDA_R_32F\n#define cublasComputeType_t cudaDataType_t\n#endif // CUDART_VERSION < 11020\n" }, { "path": "src/ggml-cuda/vendors/hip.h", "content": "#pragma once\n\n#define HIP_DISABLE_WARP_SYNC_BUILTINS 1\n#include \n#include \n#include \n#include \n\n#if defined(GGML_HIP_ROCWMMA_FATTN)\n#include \n#endif // defined(GGML_HIP_ROCWMMA_FATTN)\n\n#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT\n#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT\n#define CUBLAS_OP_N HIPBLAS_OP_N\n#define CUBLAS_OP_T HIPBLAS_OP_T\n#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS\n#define CUBLAS_TF32_TENSOR_OP_MATH 0\n#define CUDA_R_16F HIPBLAS_R_16F\n#define CUDA_R_16BF HIPBLAS_R_16B\n#define CUDA_R_32F HIPBLAS_R_32F\n#define CUBLAS_SIDE_RIGHT HIPBLAS_SIDE_RIGHT\n#define CUBLAS_FILL_MODE_UPPER HIPBLAS_FILL_MODE_UPPER\n#define CUBLAS_DIAG_NON_UNIT HIPBLAS_DIAG_NON_UNIT\n#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED hipDeviceAttributeVirtualMemoryManagementSupported\n#define CU_MEM_ALLOC_GRANULARITY_RECOMMENDED hipMemAllocationGranularityRecommended\n#define CU_MEM_ALLOCATION_TYPE_PINNED hipMemAllocationTypePinned\n#define CU_MEM_LOCATION_TYPE_DEVICE hipMemLocationTypeDevice\n#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE hipMemAccessFlagsProtReadWrite\n#define CU_CHECK(fn) {hipError_t err = fn; if(err != hipSuccess) { GGML_ABORT(\"HipVMM Failure: %s\\n\", hipGetErrorString(err)); }}\n#define __shfl_sync(mask, var, laneMask, width) __shfl(var, laneMask, width)\n#define __shfl_up_sync(mask, var, laneMask, width) __shfl_up(var, laneMask, width)\n#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)\n#define __all_sync(mask, var) __all(var)\n#define __any_sync(mask, var) __any(var)\n#define cublasStrsmBatched hipblasStrsmBatched\n#define cublasCreate hipblasCreate\n#define cublasDestroy hipblasDestroy\n#define cublasGemmEx hipblasGemmEx\n#define cublasGemmBatchedEx hipblasGemmBatchedEx\n#define cublasGemmStridedBatchedEx hipblasGemmStridedBatchedEx\n#define cublasHandle_t hipblasHandle_t\n#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS\n#define cublasSetStream hipblasSetStream\n#define cublasSgemm hipblasSgemm\n#define cublasStatus_t hipblasStatus_t\n#define cublasOperation_t hipblasOperation_t\n#define cudaDevAttrCooperativeLaunch hipDeviceAttributeCooperativeLaunch\n#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer\n#define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess\n#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess\n#define cudaDeviceGetAttribute hipDeviceGetAttribute\n#define cudaDeviceProp hipDeviceProp_t\n#define cudaDeviceSynchronize hipDeviceSynchronize\n#define cudaError_t hipError_t\n#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled\n#define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled\n#define cudaEventCreateWithFlags hipEventCreateWithFlags\n#define cudaEventDisableTiming hipEventDisableTiming\n#define cudaEventRecord hipEventRecord\n#define cudaEventSynchronize hipEventSynchronize\n#define cudaEvent_t hipEvent_t\n#define cudaEventDestroy hipEventDestroy\n#define cudaFree hipFree\n#define cudaFreeHost hipHostFree\n#define cudaGetDevice hipGetDevice\n#define cudaGetDeviceCount hipGetDeviceCount\n#define cudaGetDeviceProperties hipGetDeviceProperties\n#define cudaGetErrorString hipGetErrorString\n#define cudaGetLastError hipGetLastError\n#define cudaHostRegister hipHostRegister\n#define cudaHostRegisterPortable hipHostRegisterPortable\n#define cudaHostRegisterReadOnly hipHostRegisterReadOnly\n#define cudaHostUnregister hipHostUnregister\n#define cudaLaunchCooperativeKernel hipLaunchCooperativeKernel\n#define cudaLaunchHostFunc hipLaunchHostFunc\n#define cudaMalloc hipMalloc\n#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size, hipHostMallocDefault)\n#define cudaMallocManaged hipMallocManaged\n#define cudaMemAdvise hipMemAdvise\n#define cudaMemcpy hipMemcpy\n#define cudaMemcpyAsync hipMemcpyAsync\n#define cudaMemcpyPeerAsync hipMemcpyPeerAsync\n#define cudaMemcpy2DAsync hipMemcpy2DAsync\n#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice\n#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost\n#define cudaMemcpyHostToDevice hipMemcpyHostToDevice\n#define cudaMemcpyKind hipMemcpyKind\n#define cudaMemset hipMemset\n#define cudaMemsetAsync hipMemsetAsync\n#define cudaMemGetInfo hipMemGetInfo\n#define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize\n#define cudaSetDevice hipSetDevice\n#define cuDeviceGet hipDeviceGet\n#define CUdevice hipDevice_t\n#define CUdeviceptr hipDeviceptr_t\n#define cuMemUnmap hipMemUnmap\n#define CUmemAccessDesc hipMemAccessDesc\n#define cuMemAddressFree hipMemAddressFree\n#define cuMemRelease hipMemRelease\n#define CUmemGenericAllocationHandle hipMemGenericAllocationHandle_t\n#define cuMemCreate hipMemCreate\n#define cuMemAddressReserve hipMemAddressReserve\n#define cuMemMap hipMemMap\n#define cuMemSetAccess hipMemSetAccess\n#define cuMemGetAllocationGranularity hipMemGetAllocationGranularity\n#define CUmemAllocationProp hipMemAllocationProp\n#define cuDeviceGetAttribute hipDeviceGetAttribute\n#define cudaStreamCreateWithFlags hipStreamCreateWithFlags\n#define cudaStreamDestroy hipStreamDestroy\n#define cudaStreamFireAndForget hipStreamFireAndForget\n#define cudaStreamNonBlocking hipStreamNonBlocking\n#define cudaStreamPerThread hipStreamPerThread\n#define cudaStreamSynchronize hipStreamSynchronize\n#define cudaStreamWaitEvent hipStreamWaitEvent\n#define cudaGraphExec_t hipGraphExec_t\n#define cudaGraphNode_t hipGraphNode_t\n#define cudaKernelNodeParams hipKernelNodeParams\n#define cudaKernelNodeParams hipKernelNodeParams\n#define cudaGraphExecDestroy hipGraphExecDestroy\n#define cudaGraphLaunch hipGraphLaunch\n#define cudaErrorGraphExecUpdateFailure hipErrorGraphExecUpdateFailure\n#define cudaGraphExecUpdateResult hipGraphExecUpdateResult\n#define cudaGraphNodeType hipGraphNodeType\n#define cudaGraphNodeTypeKernel hipGraphNodeTypeKernel\n#define cudaGraphInstantiate hipGraphInstantiate\n#define cudaStreamEndCapture hipStreamEndCapture\n#define cudaGraphDestroy hipGraphDestroy\n#define cudaGraphKernelNodeSetParams hipGraphKernelNodeSetParams\n#define cudaErrorInvalidDeviceFunction hipErrorInvalidDeviceFunction\n#define cudaGraphKernelNodeGetParams hipGraphKernelNodeGetParams\n#define cudaGraphNodeGetType hipGraphNodeGetType\n#define cudaGraphGetNodes hipGraphGetNodes\n#define cudaGraphExecUpdate hipGraphExecUpdate\n#define cudaStreamCaptureModeRelaxed hipStreamCaptureModeRelaxed\n#define cudaStreamBeginCapture hipStreamBeginCapture\n#define cudaGraph_t hipGraph_t\n#define cudaStream_t hipStream_t\n#define cudaSuccess hipSuccess\n#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor\n#define cudaFuncSetAttribute hipFuncSetAttribute\n#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize\n#define __trap() do { abort(); __builtin_unreachable(); } while(0)\n#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS\n#define CUBLAS_STATUS_NOT_INITIALIZED HIPBLAS_STATUS_NOT_INITIALIZED\n#define CUBLAS_STATUS_ALLOC_FAILED HIPBLAS_STATUS_ALLOC_FAILED\n#define CUBLAS_STATUS_INVALID_VALUE HIPBLAS_STATUS_INVALID_VALUE\n#define CUBLAS_STATUS_ARCH_MISMATCH HIPBLAS_STATUS_ARCH_MISMATCH\n#define CUBLAS_STATUS_MAPPING_ERROR HIPBLAS_STATUS_MAPPING_ERROR\n#define CUBLAS_STATUS_EXECUTION_FAILED HIPBLAS_STATUS_EXECUTION_FAILED\n#define CUBLAS_STATUS_INTERNAL_ERROR HIPBLAS_STATUS_INTERNAL_ERROR\n#define CUBLAS_STATUS_NOT_SUPPORTED HIPBLAS_STATUS_NOT_SUPPORTED\n\n#if HIP_VERSION >= 60500000\n#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F\n#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F\n#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F\n#define cublasComputeType_t hipblasComputeType_t\n#define cudaDataType_t hipDataType\n#else\n#define CUBLAS_COMPUTE_16F HIPBLAS_R_16F\n#define CUBLAS_COMPUTE_32F HIPBLAS_R_32F\n#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_R_32F\n#define cublasComputeType_t hipblasDatatype_t\n#define cudaDataType_t hipblasDatatype_t\n#endif // HIP_VERSION >= 6050000\n\n#if !defined(__HIP_PLATFORM_AMD__)\n#error \"The HIP backend supports only AMD targets\"\n#endif // !defined(__HIP_PLATFORM_AMD__)\n\n#define __CUDA_ARCH__ 1300\n\n#if defined(__gfx900__) || defined(__gfx906__)\n#define GCN5\n#endif // defined(__gfx900__) || defined(__gfx906__)\n\n#if defined(__gfx803__)\n#define GCN4\n#endif // defined(__gfx803__)\n\n#if defined(GCN5) || defined(GCN4)\n#define GCN\n#endif // defined(GCN5) || defined(GCN4)\n\n#if defined(__gfx942__)\n#define CDNA3\n#endif // defined(__gfx942__)\n\n#if defined(__gfx90a__)\n#define CDNA2\n#endif // defined(__gfx90a__)\n\n#if defined(__gfx908__)\n#define CDNA1\n#endif // defined(__gfx908__)\n\n#if defined(CDNA3) || defined(CDNA2) || defined(CDNA1)\n#define CDNA // For the entire family\n#endif // defined(CDNA3) || defined(CDNA2) || defined(CDNA1)\n\n#if defined(__GFX12__)\n#define RDNA4\n#endif // defined(__GFX12__)\n\n#if defined(__GFX11__)\n#define RDNA3\n#endif // defined(__GFX11__)\n\n#if defined(__gfx1150__) || defined(__gfx1151__)\n#define RDNA3_5\n#endif // defined(__gfx1150__) || defined(__gfx1151__)\n\n#if defined(RDNA3) && !defined(RDNA3_5)\n#define RDNA3_0\n#endif // defined(RDNA3) && !defined(RDNA3_5)\n\n#if defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) || \\\n defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) || defined(__gfx1037__)\n#define RDNA2\n#endif\n\n#if defined(__gfx1010__) || defined(__gfx1012__)\n#define RDNA1\n#endif // defined(__gfx1010__) || defined(__gfx1012__)\n\n#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)\n#define RDNA // For the entire family\n#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)\n\n#ifndef __has_builtin\n #define __has_builtin(x) 0\n#endif\n\ntypedef __hip_bfloat16 nv_bfloat16;\ntypedef __hip_bfloat162 nv_bfloat162;\n\ntypedef int8_t int8x4_t __attribute__((ext_vector_type(4)));\ntypedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));\nstatic __device__ __forceinline__ int __vsubss4(const int a, const int b) {\n const int8x4_t va = reinterpret_cast(a);\n const int8x4_t vb = reinterpret_cast(b);\n#if __has_builtin(__builtin_elementwise_sub_sat)\n const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);\n return reinterpret_cast(c);\n#else\n int8x4_t c;\n int16_t tmp;\n#pragma unroll\n for (int i = 0; i < 4; i++) {\n tmp = va[i] - vb[i];\n if(tmp > std::numeric_limits::max()) tmp = std::numeric_limits::max();\n if(tmp < std::numeric_limits::min()) tmp = std::numeric_limits::min();\n c[i] = tmp;\n }\n return reinterpret_cast(c);\n#endif // __has_builtin(__builtin_elementwise_sub_sat)\n}\n\nstatic __device__ __forceinline__ int __vsub4(const int a, const int b) {\n return __vsubss4(a, b);\n}\n\nstatic __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) {\n const uint8x4_t& va = reinterpret_cast(a);\n const uint8x4_t& vb = reinterpret_cast(b);\n unsigned int c;\n uint8x4_t& vc = reinterpret_cast(c);\n#pragma unroll\n for (int i = 0; i < 4; ++i) {\n vc[i] = va[i] == vb[i] ? 0xff : 0x00;\n }\n return c;\n}\n\nstatic __device__ __forceinline__ unsigned int __vcmpne4(unsigned int a, unsigned int b) {\n const uint8x4_t& va = reinterpret_cast(a);\n const uint8x4_t& vb = reinterpret_cast(b);\n unsigned int c;\n uint8x4_t& vc = reinterpret_cast(c);\n#pragma unroll\n for (int i = 0; i < 4; ++i) {\n vc[i] = va[i] == vb[i] ? 0x00 : 0xff;\n }\n return c;\n}\n" }, { "path": "src/ggml-cuda/vendors/musa.h", "content": "#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#define CUBLAS_COMPUTE_16F CUDA_R_16F\n#define CUBLAS_COMPUTE_32F CUDA_R_32F\n#define CUBLAS_COMPUTE_32F_FAST_16F MUBLAS_COMPUTE_32F_FAST_16F\n#define CUBLAS_GEMM_DEFAULT MUBLAS_GEMM_DEFAULT\n#define CUBLAS_GEMM_DEFAULT_TENSOR_OP MUBLAS_GEMM_DEFAULT\n#define CUBLAS_OP_N MUBLAS_OP_N\n#define CUBLAS_OP_T MUBLAS_OP_T\n#define CUBLAS_DEFAULT_MATH MUBLAS_DEFAULT_MATH\n#define CUBLAS_SIDE_RIGHT MUBLAS_SIDE_RIGHT\n#define CUBLAS_FILL_MODE_UPPER MUBLAS_FILL_MODE_UPPER\n#define CUBLAS_DIAG_NON_UNIT MUBLAS_DIAG_NON_UNIT\n#define CUBLAS_STATUS_SUCCESS MUBLAS_STATUS_SUCCESS\n#define CUBLAS_TF32_TENSOR_OP_MATH MUBLAS_TENSOR_OP_MATH\n#define CUDA_R_16F MUSA_R_16F\n#define CUDA_R_16BF MUSA_R_16BF\n#define CUDA_R_32F MUSA_R_32F\n#define cublasStrsmBatched mublasStrsmBatched\n#define cublasComputeType_t cudaDataType_t\n#define cublasCreate mublasCreate\n#define cublasDestroy mublasDestroy\n#define cublasGemmEx mublasGemmEx\n#define cublasGemmBatchedEx mublasGemmBatchedEx\n#define cublasGemmStridedBatchedEx mublasGemmStridedBatchedEx\n#define cublasHandle_t mublasHandle_t\n#define cublasSetMathMode mublasSetMathMode\n#define cublasSetStream mublasSetStream\n#define cublasSgemm mublasSgemm\n#define cublasStatus_t mublasStatus_t\n#define cublasOperation_t mublasOperation_t\n#define cublasGetStatusString mublasGetStatusString\n#define cudaDataType_t musaDataType_t\n#define cudaDeviceCanAccessPeer musaDeviceCanAccessPeer\n#define cudaDeviceDisablePeerAccess musaDeviceDisablePeerAccess\n#define cudaDeviceEnablePeerAccess musaDeviceEnablePeerAccess\n#define cudaDeviceProp musaDeviceProp\n#define cudaDeviceSynchronize musaDeviceSynchronize\n#define cudaError_t musaError_t\n#define cudaErrorPeerAccessAlreadyEnabled musaErrorPeerAccessAlreadyEnabled\n#define cudaErrorPeerAccessNotEnabled musaErrorPeerAccessNotEnabled\n#define cudaEventCreateWithFlags musaEventCreateWithFlags\n#define cudaEventDisableTiming musaEventDisableTiming\n#define cudaEventRecord musaEventRecord\n#define cudaEventSynchronize musaEventSynchronize\n#define cudaEvent_t musaEvent_t\n#define cudaEventDestroy musaEventDestroy\n#define cudaFree musaFree\n#define cudaFreeHost musaFreeHost\n#define cudaGetDevice musaGetDevice\n#define cudaGetDeviceCount musaGetDeviceCount\n#define cudaGetDeviceProperties musaGetDeviceProperties\n#define cudaGetErrorString musaGetErrorString\n#define cudaGetLastError musaGetLastError\n#define cudaHostRegister musaHostRegister\n#define cudaHostRegisterPortable musaHostRegisterPortable\n#define cudaHostRegisterReadOnly musaHostRegisterReadOnly\n#define cudaHostUnregister musaHostUnregister\n#define cudaLaunchCooperativeKernel musaLaunchCooperativeKernel\n#define cudaLaunchHostFunc musaLaunchHostFunc\n#define cudaMalloc musaMalloc\n#define cudaMallocHost musaMallocHost\n#define cudaMallocManaged musaMallocManaged\n#define cudaMemcpy musaMemcpy\n#define cudaMemcpyAsync musaMemcpyAsync\n#define cudaMemcpyPeerAsync musaMemcpyPeerAsync\n#define cudaMemcpy2DAsync musaMemcpy2DAsync\n#define cudaMemcpyDeviceToDevice musaMemcpyDeviceToDevice\n#define cudaMemcpyDeviceToHost musaMemcpyDeviceToHost\n#define cudaMemcpyHostToDevice musaMemcpyHostToDevice\n#define cudaMemcpyKind musaMemcpyKind\n#define cudaMemset musaMemset\n#define cudaMemsetAsync musaMemsetAsync\n#define cudaMemGetInfo musaMemGetInfo\n#define cudaOccupancyMaxPotentialBlockSize musaOccupancyMaxPotentialBlockSize\n#define cudaSetDevice musaSetDevice\n#define cudaStreamCreateWithFlags musaStreamCreateWithFlags\n#define cudaStreamDestroy musaStreamDestroy\n#define cudaStreamFireAndForget musaStreamFireAndForget\n#define cudaStreamNonBlocking musaStreamNonBlocking\n#define cudaStreamPerThread musaStreamPerThread\n#define cudaStreamSynchronize musaStreamSynchronize\n#define cudaStreamWaitEvent musaStreamWaitEvent\n#define cudaStream_t musaStream_t\n#define cudaSuccess musaSuccess\n\n// Additional mappings for MUSA virtual memory pool\n#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED MU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED\n#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE MU_MEM_ACCESS_FLAGS_PROT_READWRITE\n#define CU_MEM_ALLOC_GRANULARITY_RECOMMENDED MU_MEM_ALLOC_GRANULARITY_RECOMMENDED\n#define CU_MEM_ALLOCATION_TYPE_PINNED MU_MEM_ALLOCATION_TYPE_PINNED\n#define CU_MEM_LOCATION_TYPE_DEVICE MU_MEM_LOCATION_TYPE_DEVICE\n#define CUdevice MUdevice\n#define CUdeviceptr MUdeviceptr\n#define CUmemAccessDesc MUmemAccessDesc\n#define CUmemAllocationProp MUmemAllocationProp\n#define CUmemGenericAllocationHandle MUmemGenericAllocationHandle\n#define cuDeviceGet muDeviceGet\n#define cuDeviceGetAttribute muDeviceGetAttribute\n#define cuMemAddressFree muMemAddressFree\n#define cuMemAddressReserve muMemAddressReserve\n#define cuMemCreate muMemCreate\n#define cuMemGetAllocationGranularity muMemGetAllocationGranularity\n#define cuMemMap muMemMap\n#define cuMemRelease muMemRelease\n#define cuMemSetAccess muMemSetAccess\n#define cuMemUnmap muMemUnmap\n#define cudaFuncAttributeMaxDynamicSharedMemorySize musaFuncAttributeMaxDynamicSharedMemorySize\n#define cudaFuncSetAttribute musaFuncSetAttribute\n#define cudaMemcpy3DPeerParms musaMemcpy3DPeerParms\n#define make_cudaExtent make_musaExtent\n#define make_cudaPitchedPtr make_musaPitchedPtr\n\n// Additional mappings for MUSA graphs\n#define CUDA_SUCCESS MUSA_SUCCESS\n#define CUresult MUresult\n#define cuGetErrorString muGetErrorString\n#define cudaErrorGraphExecUpdateFailure musaErrorGraphExecUpdateFailure\n#define cudaErrorInvalidDeviceFunction musaErrorInvalidDeviceFunction\n#define cudaGraphDestroy musaGraphDestroy\n#define cudaGraphExecDestroy musaGraphExecDestroy\n#define cudaGraphExec_t musaGraphExec_t\n#define cudaGraphExecUpdate musaGraphExecUpdate\n#define cudaGraphExecUpdateResult musaGraphExecUpdateResult\n#define cudaGraphGetNodes musaGraphGetNodes\n#define cudaGraphInstantiate musaGraphInstantiate\n#define cudaGraphKernelNodeGetParams musaGraphKernelNodeGetParams\n#define cudaGraphKernelNodeSetParams musaGraphKernelNodeSetParams\n#define cudaGraphLaunch musaGraphLaunch\n#define cudaGraphNodeGetType musaGraphNodeGetType\n#define cudaGraphNode_t musaGraphNode_t\n#define cudaGraphNodeType musaGraphNodeType\n#define cudaGraphNodeTypeKernel musaGraphNodeTypeKernel\n#define cudaGraph_t musaGraph_t\n#define cudaKernelNodeParams musaKernelNodeParams\n#define cudaStreamCaptureModeRelaxed musaStreamCaptureModeRelaxed\n#define cudaStreamBeginCapture musaStreamBeginCapture\n#define cudaStreamEndCapture musaStreamEndCapture\n#define cudaOccupancyMaxActiveBlocksPerMultiprocessor musaOccupancyMaxActiveBlocksPerMultiprocessor\n\ntypedef __mt_bfloat16 nv_bfloat16;\ntypedef __mt_bfloat162 nv_bfloat162;\n" }, { "path": "src/ggml-cuda/wkv.cu", "content": "#include \"common.cuh\"\n#include \"wkv.cuh\"\n\ntemplate \nstatic __global__ void rwkv_wkv_f32(const int B, const int T, const int C, const int H, const float * k, const float * v, const float * r, const float * tf, const float * td, const float * s, float * dst) {\n const int tid = threadIdx.x;\n const int bid = blockIdx.x;\n\n const int head_size = block_size;\n const int batch_i = bid / H;\n const int head_i = bid % H;\n const int state_size = C * head_size;\n const int n_seq_tokens = T / B;\n\n float state[head_size];\n __shared__ float _k[head_size], _r[head_size], _tf[head_size], _td[head_size];\n\n #pragma unroll\n for (int i = 0; i < head_size; i++) {\n state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid];\n }\n\n __syncthreads();\n _tf[tid] = tf[head_i * head_size + tid];\n __syncthreads();\n\n for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) {\n __syncthreads();\n _k[tid] = k[t];\n _r[tid] = r[t];\n _td[tid] = td[t];\n __syncthreads();\n\n const float _v = v[t];\n float y = 0;\n for (int j = 0; j < head_size; j += 4) {\n const float4& k = (float4&)(_k[j]);\n const float4& r = (float4&)(_r[j]);\n const float4& tf = (float4&)(_tf[j]);\n const float4& td = (float4&)(_td[j]);\n float4& s = (float4&)(state[j]);\n float4 kv;\n\n kv.x = k.x * _v;\n kv.y = k.y * _v;\n kv.z = k.z * _v;\n kv.w = k.w * _v;\n\n y += r.x * (tf.x * kv.x + s.x);\n y += r.y * (tf.y * kv.y + s.y);\n y += r.z * (tf.z * kv.z + s.z);\n y += r.w * (tf.w * kv.w + s.w);\n\n s.x = s.x * td.x + kv.x;\n s.y = s.y * td.y + kv.y;\n s.z = s.z * td.z + kv.z;\n s.w = s.w * td.w + kv.w;\n }\n dst[t] = y;\n }\n\n #pragma unroll\n for (int i = 0; i < head_size; i++) {\n dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i];\n }\n}\n\ntemplate \nstatic __global__ void rwkv_wkv7_f32(const int B, const int T, const int C, const int H, const float * r, const float * w, const float * k, const float * v, const float * a, const float * b, const float * s, float * dst) {\n const int tid = threadIdx.x;\n const int bid = blockIdx.x;\n\n const int head_size = block_size;\n const int batch_i = bid / H;\n const int head_i = bid % H;\n const int state_size = C * head_size;\n const int n_seq_tokens = T / B;\n\n float state[head_size];\n __shared__ float _r[head_size], _w[head_size], _k[head_size], _a[head_size], _b[head_size];\n\n#ifndef GGML_USE_MUSA\n #pragma unroll\n#endif\n for (int i = 0; i < head_size; i++) {\n state[i] = s[batch_i * state_size + head_i * head_size * head_size + tid * head_size + i];\n }\n\n for (int t = batch_i * n_seq_tokens * C + head_i * head_size + tid; t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) {\n __syncthreads();\n _r[tid] = r[t];\n _w[tid] = w[t];\n _k[tid] = k[t];\n _a[tid] = a[t];\n _b[tid] = b[t];\n __syncthreads();\n\n float sa = 0;\n #pragma unroll\n for (int j = 0; j < head_size; j += 4)\n {\n const float4& a = (float4&)(_a[j]);\n const float4& s = (float4&)(state[j]);\n sa += a.x * s.x;\n sa += a.y * s.y;\n sa += a.z * s.z;\n sa += a.w * s.w;\n }\n\n const float _v = v[t];\n float y = 0;\n for (int j = 0; j < head_size; j += 4) {\n const float4& r = (float4&)(_r[j]);\n const float4& w = (float4&)(_w[j]);\n const float4& k = (float4&)(_k[j]);\n const float4& b = (float4&)(_b[j]);\n float4& s = (float4&)(state[j]);\n float4 kv;\n\n kv.x = k.x * _v;\n kv.y = k.y * _v;\n kv.z = k.z * _v;\n kv.w = k.w * _v;\n\n s.x = s.x * w.x + kv.x + sa * b.x;\n s.y = s.y * w.y + kv.y + sa * b.y;\n s.z = s.z * w.z + kv.z + sa * b.z;\n s.w = s.w * w.w + kv.w + sa * b.w;\n\n y += s.x * r.x;\n y += s.y * r.y;\n y += s.z * r.z;\n y += s.w * r.w;\n }\n dst[t] = y;\n }\n\n #pragma unroll\n for (int i = 0; i < head_size; i++) {\n dst[T * C + batch_i * state_size + head_i * head_size * head_size + tid * head_size + i] = state[i];\n }\n}\n\nvoid ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const float * k_d = (const float *)dst->src[0]->data;\n const float * v_d = (const float *)dst->src[1]->data;\n const float * r_d = (const float *)dst->src[2]->data;\n const float * tf_d = (const float *)dst->src[3]->data;\n const float * td_d = (const float *)dst->src[4]->data;\n const float * s_d = (const float *)dst->src[5]->data;\n\n const int64_t B = dst->src[5]->ne[1];\n const int64_t T = dst->src[0]->ne[2];\n const int64_t C = dst->ne[0];\n const int64_t H = dst->src[0]->ne[1];\n\n float * dst_d = (float *)dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(dst->src[5]->type == GGML_TYPE_F32);\n GGML_ASSERT(C % H == 0);\n GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE || C / H == CUDA_WKV_BLOCK_SIZE * 2);\n\n if (C / H == CUDA_WKV_BLOCK_SIZE) {\n rwkv_wkv_f32<<>>(B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d);\n } else {\n rwkv_wkv_f32<<>>(B, T, C, H, k_d, v_d, r_d, tf_d, td_d, s_d, dst_d);\n }\n}\n\nvoid ggml_cuda_op_rwkv_wkv7(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n const float * r_d = (const float *)dst->src[0]->data;\n const float * w_d = (const float *)dst->src[1]->data;\n const float * k_d = (const float *)dst->src[2]->data;\n const float * v_d = (const float *)dst->src[3]->data;\n const float * a_d = (const float *)dst->src[4]->data;\n const float * b_d = (const float *)dst->src[5]->data;\n const float * s_d = (const float *)dst->src[6]->data;\n\n const int64_t B = dst->src[6]->ne[1];\n const int64_t T = dst->src[0]->ne[2];\n const int64_t C = dst->ne[0];\n const int64_t H = dst->src[0]->ne[1];\n\n float * dst_d = (float *)dst->data;\n\n cudaStream_t stream = ctx.stream();\n\n GGML_ASSERT(dst->src[6]->type == GGML_TYPE_F32);\n GGML_ASSERT(C % H == 0);\n GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE || C / H == CUDA_WKV_BLOCK_SIZE * 2);\n\n if (C / H == CUDA_WKV_BLOCK_SIZE) {\n rwkv_wkv7_f32<<>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d);\n } else {\n rwkv_wkv7_f32<<>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d);\n }\n}\n" }, { "path": "src/ggml-cuda/wkv.cuh", "content": "#include \"common.cuh\"\n\n#define CUDA_WKV_BLOCK_SIZE 64\n\nvoid ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_rwkv_wkv7(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-hexagon/CMakeLists.txt", "content": "file(TO_CMAKE_PATH \"${HEXAGON_SDK_ROOT}\" HEXAGON_SDK_ROOT)\nfile(TO_CMAKE_PATH \"${HEXAGON_TOOLS_ROOT}\" HEXAGON_TOOLS_ROOT)\n\nif (NOT IS_DIRECTORY \"${HEXAGON_SDK_ROOT}\")\n message(FATAL_ERROR \"Make sure HEXAGON_SDK_ROOT point to the correct Hexagon SDK installation.\")\nendif()\n\nif (NOT IS_DIRECTORY \"${HEXAGON_TOOLS_ROOT}\")\n message(\"Try to read HEXAGON_TOOLS_ROOT from hexagon_sdk.json\")\n file(READ \"${HEXAGON_SDK_ROOT}/hexagon_sdk.json\" HEXAGON_SDK_CONFIG_PATH)\n string(JSON HEXAGON_TOOLS_PATH GET ${HEXAGON_SDK_CONFIG_PATH} \"root\" \"tools\" \"info\" 0 \"path\")\n message(\"Found HEXAGON_TOOLS_PATH: ${HEXAGON_TOOLS_PATH}\")\n set(HEXAGON_TOOLS_ROOT \"${HEXAGON_SDK_ROOT}/${HEXAGON_TOOLS_PATH}\")\n file(TO_CMAKE_PATH \"${HEXAGON_TOOLS_ROOT}\" HEXAGON_TOOLS_ROOT)\n if (NOT IS_DIRECTORY \"${HEXAGON_TOOLS_ROOT}\")\n message(FATAL_ERROR \"Make sure HEXAGON_TOOLS_ROOT point to the correct Hexagon SDK installation.\")\n endif()\nendif()\n\nmessage(STATUS \"hexagon: using ${HEXAGON_SDK_ROOT} and ${HEXAGON_TOOLS_ROOT} for building libggml-htp skels\")\n\ninclude(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake)\ninclude(ExternalProject)\n\noption(GGML_HEXAGON_HTP_DEBUG \"ggml-hexagon: enable HTP debug output\" OFF)\nset(GGML_HEXAGON_HTP_CERT \"$ENV{HEXAGON_HTP_CERT}\" CACHE PATH \"ggml-hexagon: enable HTP library signing using certificate\")\nset(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING \"ggml-hexagon: quantize group size (32, 64, or 128)\")\n\nadd_library(htp_iface OBJECT\n ${CMAKE_CURRENT_BINARY_DIR}/htp_iface_stub.c)\n\nset_target_properties(htp_iface PROPERTIES POSITION_INDEPENDENT_CODE ON)\ntarget_include_directories(htp_iface PUBLIC\n ${HEXAGON_SDK_ROOT}/incs\n ${HEXAGON_SDK_ROOT}/incs/stddef\n ${HEXAGON_SDK_ROOT}/utils/examples\n ${CMAKE_CURRENT_SOURCE_DIR}/htp\n ${CMAKE_CURRENT_BINARY_DIR})\n\nbuild_idl(htp/htp_iface.idl htp_iface)\n\nif (CMAKE_SYSTEM_NAME MATCHES Android)\n target_link_options(htp_iface PUBLIC -llog -ldl)\nelseif (CMAKE_SYSTEM_NAME MATCHES Windows)\n target_precompile_headers(htp_iface PUBLIC )\nelse()\n target_link_options(htp_iface PUBLIC -ldl)\nendif()\n\nset(TARGET_NAME ggml-hexagon)\nggml_add_backend_library(${TARGET_NAME}\n ggml-hexagon.cpp\n htp-drv.cpp\n htp-drv.h\n libdl.h\n ../../include/ggml-hexagon.h)\n\ntarget_link_libraries(${TARGET_NAME} PRIVATE htp_iface)\ntarget_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/htp ${CMAKE_CURRENT_BINARY_DIR})\n\n# Build HTP skels\nset(HTP_SKELS)\nfunction(build_htp_skel V)\n ExternalProject_Add(htp-${V}\n SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/htp BUILD_ALWAYS ON\n BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so\n CMAKE_ARGS\n -DCMAKE_BUILD_TYPE=Release\n -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/htp/cmake-toolchain.cmake\n -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR}\n -DHEXAGON_SDK_ROOT=${HEXAGON_SDK_ROOT}\n -DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_ROOT}\n -DHEXAGON_HTP_DEBUG=${GGML_HEXAGON_HTP_DEBUG}\n -DGGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE=${GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE}\n -DDSP_VERSION=${V}\n -DPREBUILT_LIB_DIR=\"toolv19_${V}\")\n list(APPEND HTP_SKELS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so)\n set(HTP_SKELS ${HTP_SKELS} PARENT_SCOPE)\nendfunction()\n\nbuild_htp_skel(v68)\nbuild_htp_skel(v69)\nbuild_htp_skel(v73)\nbuild_htp_skel(v75)\nbuild_htp_skel(v79)\nbuild_htp_skel(v81)\n\n# Install Hexagon skels required at runtime\ninstall(FILES ${HTP_SKELS} TYPE LIB)\n\nif (CMAKE_SYSTEM_NAME MATCHES Windows AND GGML_HEXAGON_HTP_CERT)\n file(TO_CMAKE_PATH \"$ENV{WINDOWS_SDK_BIN}/arm64\" WINSDK_BIN0_ARM64)\n file(TO_CMAKE_PATH \"$ENV{WINDOWS_SDK_BIN}/x86\" WINSDK_BIN0_X86)\n file(TO_CMAKE_PATH \"$ENV{WindowsSdkVerBinPath}/arm64\" WINSDK_BIN1_ARM64)\n file(TO_CMAKE_PATH \"$ENV{WindowsSdkVerBinPath}/x86\" WINSDK_BIN1_X86)\n\n set(WINSDK_PATHS ${WINSDK_BIN0_ARM64} ${WINSDK_BIN0_X86} ${WINSDK_BIN1_ARM64} ${WINSDK_BIN1_X86})\n\n find_program(INF2CAT NAMES inf2cat.exe PATHS ${WINSDK_PATHS} REQUIRED)\n find_program(SIGNTOOL NAMES signtool.exe PATHS ${WINSDK_PATHS} REQUIRED)\n\n message(STATUS \"hexagon: using ${GGML_HEXAGON_HTP_CERT} to sign libggml-htp skels\")\n\n set(LIBGGML_HTP_CAT ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp.cat)\n add_custom_target(libggml-htp-cat\n BYPRODUCTS ${LIBGGML_HTP_CAT}\n DEPENDS libggml-htp.inf ${HTP_SKELS}\n COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/libggml-htp.inf ${CMAKE_CURRENT_BINARY_DIR}\n COMMAND ${INF2CAT} /driver:${CMAKE_CURRENT_BINARY_DIR} /os:10_25H2_ARM64\n COMMAND ${SIGNTOOL} sign /fd sha256 /f ${GGML_HEXAGON_HTP_CERT} ${LIBGGML_HTP_CAT}\n COMMENT \"generating and signing libggml-htp.cat file\"\n VERBATIM\n )\n\n add_dependencies(${TARGET_NAME} libggml-htp-cat)\n install(FILES ${LIBGGML_HTP_CAT} TYPE LIB)\nendif()\n" }, { "path": "src/ggml-hexagon/ggml-hexagon.cpp", "content": "#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#ifdef _WIN32\n# include \n#else\n# include \n# include \n#endif\n\n#pragma clang diagnostic ignored \"-Wnested-anon-types\"\n#pragma clang diagnostic ignored \"-Wgnu-anonymous-struct\"\n\n#include \n#include \n#include \n\n#define GGML_COMMON_IMPL_CPP\n#include \"ggml-backend-impl.h\"\n#include \"ggml-common.h\"\n#include \"ggml-hexagon.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-quants.h\"\n#include \"op-desc.h\"\n#include \"htp-msg.h\"\n#include \"htp_iface.h\"\n#include \"htp-drv.h\"\n\nstatic size_t opt_ndev = 1;\nstatic size_t opt_nhvx = 0; // use all\nstatic int opt_arch = 0; // autodetect\nstatic int opt_etm = 0;\nstatic int opt_verbose = 0;\nstatic int opt_profile = 0;\nstatic int opt_hostbuf = 1; // hostbuf ON by default\nstatic int opt_experimental = 0;\n\n// Enable all stages by default\nstatic int opt_opmask = HTP_OPMASK_QUEUE | HTP_OPMASK_QUANTIZE | HTP_OPMASK_COMPUTE;\nstatic int opt_opsync = 0; // synchronous ops\n\n#define HEX_VERBOSE(...) \\\n if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__)\n\nstatic inline uint64_t hex_is_aligned(void * addr, uint32_t align) {\n return ((size_t) addr & (align - 1)) == 0;\n}\n\nstatic inline size_t hex_round_up(size_t n, size_t m) {\n return m * ((n + m - 1) / m);\n}\n\nstatic const char * status_to_str(uint32_t status) {\n switch (status) {\n case HTP_STATUS_OK:\n return \"OK\";\n case HTP_STATUS_NO_SUPPORT:\n return \"NO-SUPPORT\";\n case HTP_STATUS_INVAL_PARAMS:\n return \"INVAL-PARAMS\";\n case HTP_STATUS_VTCM_TOO_SMALL:\n return \"VTCM-TOO-SMALL\";\n case HTP_STATUS_INTERNAL_ERR:\n return \"INTERNAL-ERROR\";\n default:\n return \"UNKNOWN\";\n }\n}\n\n// ** debug helpers\n\nstatic void ggml_hexagon_dump_op_exec(const std::string &sess_name, const ggml_tensor * op, const uint32_t req_flags) {\n if (!opt_verbose) return;\n\n op_desc desc(op);\n GGML_LOG_DEBUG(\"ggml-hex: %s execute-op %s: %s : %s : %s : %s : %s : flags 0x%x\\n\", sess_name.c_str(),\n ggml_op_name(op->op), desc.names, desc.dims, desc.types, desc.strides, desc.buffs, req_flags);\n}\n\nstatic void ggml_hexagon_dump_op_supp(const std::string &sess_name, const struct ggml_tensor * op, bool supp) {\n if (!opt_verbose) return;\n\n op_desc desc(op);\n GGML_LOG_DEBUG(\"ggml-hex: %s supports-op %s : %s : %s : %s : %s : %s : %s\\n\", sess_name.c_str(),\n ggml_op_name(op->op), desc.names, desc.dims, desc.types, desc.strides, desc.buffs, supp ? \"yes\" : \"no\");\n}\n\nstatic void ggml_hexagon_dump_op_prof(const std::string &sess_name, const ggml_tensor * op,\n uint32_t op_usec, uint32_t op_cycles, uint32_t op_pkts, uint64_t call_usec) {\n if (!opt_profile) return;\n\n op_desc desc(op);\n GGML_LOG_DEBUG(\"ggml-hex: %s profile-op %s: %s : %s : %s : %s : %s : op-usec %u op-cycles %u op-pkts %u (%f) call-usec %llu\\n\", sess_name.c_str(),\n ggml_op_name(op->op), desc.names, desc.dims, desc.types, desc.strides, desc.buffs,\n op_usec, op_cycles, op_pkts, (float) op_cycles / op_pkts, (unsigned long long) call_usec);\n}\n\n// ** backend sessions\n\nstruct ggml_hexagon_session {\n ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false);\n ~ggml_hexagon_session() noexcept(true);\n\n void allocate(int dev_id) noexcept(false);\n void release() noexcept(true);\n\n void enqueue(struct htp_general_req &req, struct dspqueue_buffer *bufs, uint32_t n_bufs, bool sync = false);\n void flush();\n\n ggml_backend_buffer_type buffer_type = {};\n ggml_backend_buffer_type repack_buffer_type = {};\n\n std::string name;\n remote_handle64 handle;\n dspqueue_t queue;\n uint32_t session_id;\n uint32_t domain_id;\n uint64_t queue_id;\n int dev_id;\n bool valid_session;\n bool valid_handle;\n bool valid_queue;\n bool valid_iface;\n std::atomic op_pending;\n uint32_t prof_usecs;\n uint32_t prof_cycles;\n uint32_t prof_pkts;\n};\n\nvoid ggml_hexagon_session::enqueue(struct htp_general_req &req, struct dspqueue_buffer *bufs, uint32_t n_bufs, bool sync) {\n // Bump pending flag (cleared in the session::flush once we get the response)\n this->op_pending++; // atomic inc\n\n int err = dspqueue_write(this->queue,\n 0, // flags - the framework will autoset this\n n_bufs, // number of buffers\n bufs, // buffer references\n sizeof(req), // Message length\n (const uint8_t *) &req, // Message\n DSPQUEUE_TIMEOUT // Timeout\n );\n\n if (err != 0) {\n GGML_ABORT(\"ggml-hex: %s dspqueue_write failed: 0x%08x\\n\", this->name.c_str(), (unsigned) err);\n }\n\n if (sync) {\n flush();\n }\n}\n\n// Flush HTP response queue i.e wait for all outstanding requests to complete\nvoid ggml_hexagon_session::flush() {\n dspqueue_t q = this->queue;\n\n // Repeatedly read packets from the queue until it's empty. We don't\n // necessarily get a separate callback for each packet, and new packets\n // may arrive while we're processing the previous one.\n\n while (this->op_pending) {\n struct htp_general_rsp rsp;\n uint32_t rsp_size;\n uint32_t flags;\n\n struct dspqueue_buffer bufs[HTP_MAX_PACKET_BUFFERS];\n uint32_t n_bufs;\n\n // Read response packet from queue\n int err = dspqueue_read(q, &flags,\n HTP_MAX_PACKET_BUFFERS, // Maximum number of buffer references\n &n_bufs, // Number of buffer references\n bufs, // Buffer references\n sizeof(rsp), // Max message length\n &rsp_size, // Message length\n (uint8_t *) &rsp, // Message\n DSPQUEUE_TIMEOUT); // Timeout\n\n if (err == AEE_EEXPIRED) {\n // TODO: might need to bail out if the HTP is stuck on something\n continue;\n }\n\n if (err != 0) {\n GGML_ABORT(\"ggml-hex: dspqueue_read failed: 0x%08x\\n\", (unsigned) err);\n }\n\n // Basic sanity checks\n if (rsp_size != sizeof(rsp)) {\n GGML_ABORT(\"ggml-hex: dspcall : bad response (size)\\n\");\n }\n\n if (rsp.status != HTP_STATUS_OK) {\n GGML_LOG_ERROR(\"ggml-hex: dspcall : dsp-rsp: %s\\n\", status_to_str(rsp.status));\n // TODO: handle errors\n }\n\n // TODO: update profiling implementation, currently only works for opt_opsync mode\n this->prof_usecs = rsp.prof_usecs;\n this->prof_cycles = rsp.prof_cycles;\n this->prof_pkts = rsp.prof_pkts;\n\n this->op_pending--; // atomic dec\n }\n}\n\n// ** backend buffers\n\nstruct ggml_backend_hexagon_buffer_type_context {\n ggml_backend_hexagon_buffer_type_context(const std::string & name, ggml_hexagon_session * sess) {\n this->sess = sess;\n this->name = name;\n }\n\n ggml_hexagon_session * sess;\n std::string name;\n};\n\nstruct ggml_backend_hexagon_buffer_context {\n bool mmap_to(ggml_hexagon_session * s) {\n HEX_VERBOSE(\"ggml-hex: %s mmaping buffer: base %p domain-id %d session-id %d size %zu fd %d repack %d\\n\",\n s->name.c_str(), (void *) this->base, s->domain_id, s->session_id, this->size, this->fd,\n (int) this->repack);\n\n int err = fastrpc_mmap(s->domain_id, this->fd, (void *) this->base, 0, this->size, FASTRPC_MAP_FD);\n if (err != 0) {\n GGML_LOG_ERROR(\"ggml-hex: buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\\n\",\n s->domain_id, this->size, this->fd, (unsigned) err);\n return false;\n }\n\n return true;\n }\n\n bool mmap() {\n if (this->mapped) {\n return true;\n }\n if (!mmap_to(this->sess)) {\n return false;\n }\n this->mapped = true;\n return true;\n }\n\n void munmap() {\n if (!this->mapped) {\n return;\n }\n\n fastrpc_munmap(this->sess->domain_id, this->fd, this->base, this->size);\n this->mapped = false;\n }\n\n ggml_backend_hexagon_buffer_context(ggml_hexagon_session * sess, size_t size, bool repack) {\n size += 4 * 1024; // extra page for padding\n\n this->base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS | RPCMEM_HEAP_NOREG, size);\n if (!this->base) {\n GGML_LOG_ERROR(\"ggml-hex: %s failed to allocate buffer : size %zu\\n\", sess->name.c_str(), size);\n throw std::runtime_error(\"ggml-hex: rpcmem_alloc failed (see log for details)\");\n }\n\n this->fd = rpcmem_to_fd(this->base);\n if (this->fd < 0) {\n GGML_LOG_ERROR(\"ggml-hex: %s failed to get FD for buffer %p\\n\", sess->name.c_str(), (void *) this->base);\n rpcmem_free(this->base);\n this->base = NULL;\n throw std::runtime_error(\"ggml-hex: rpcmem_to_fd failed (see log for details)\");\n }\n\n HEX_VERBOSE(\"ggml-hex: %s allocated buffer: base %p size %zu fd %d repack %d\\n\", sess->name.c_str(),\n (void *) this->base, size, this->fd, (int) repack);\n\n this->sess = sess;\n this->size = size;\n this->mapped = false;\n this->repack = repack;\n }\n\n ~ggml_backend_hexagon_buffer_context() {\n munmap();\n if (this->base) {\n rpcmem_free(this->base);\n this->base = NULL;\n }\n }\n\n ggml_hexagon_session * sess; // primary session\n uint8_t * base;\n size_t size;\n int fd;\n bool mapped; // mmap is done\n bool repack; // repacked buffer\n};\n\nstatic ggml_hexagon_session * ggml_backend_hexagon_buffer_get_sess(ggml_backend_buffer_t buffer) {\n return static_cast(buffer->buft->context)->sess;\n}\n\nstatic void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n auto ctx = static_cast(buffer->context);\n delete ctx;\n}\n\nstatic void * ggml_backend_hexagon_buffer_get_base(ggml_backend_buffer_t buffer) {\n auto ctx = static_cast(buffer->context);\n return ctx->base;\n}\n\nstatic enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {\n auto ctx = static_cast(buffer->context);\n auto sess = ctx->sess;\n\n HEX_VERBOSE(\"ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu usage %d repack %d\\n\", sess->name.c_str(),\n tensor->name, (void *) ctx->base, tensor->data, ggml_nbytes(tensor), (int) buffer->usage,\n (int) ctx->repack);\n\n if (tensor->view_src != NULL && tensor->view_offs == 0) {\n ; // nothing to do for the view\n } else {\n if (!ctx->mapped) {\n ctx->mmap();\n }\n }\n return GGML_STATUS_SUCCESS;\n}\n\n// ======== Q4x4x2 ====================\nstruct x2_q4 {\n int v[2];\n};\n\nstatic x2_q4 unpack_q4(uint8_t v) {\n x2_q4 x = { (int) (v & 0x0f) - 8, (int) (v >> 4) - 8 };\n return x;\n}\n\nstatic void dump_block_q4_0(const block_q4_0 * b, int i) {\n HEX_VERBOSE(\"ggml-hex: repack q4_0 %d: %d %d %d %d ... %d %d %d %d : %.6f\\n\", i, unpack_q4(b->qs[0]).v[0],\n unpack_q4(b->qs[1]).v[0], unpack_q4(b->qs[2]).v[0], unpack_q4(b->qs[3]).v[0], unpack_q4(b->qs[12]).v[1],\n unpack_q4(b->qs[13]).v[1], unpack_q4(b->qs[14]).v[1], unpack_q4(b->qs[15]).v[1],\n GGML_FP16_TO_FP32(b->d));\n}\n\nstatic void dump_packed_block_q4x4x2(const uint8_t * v, unsigned int i, size_t k) {\n static const int qk = QK_Q4_0x4x2;\n const int dblk_size = 8 * 2; // 8x __fp16\n const int qblk_size = qk / 2; // int4\n const int qrow_size = k / 2; // int4 (not padded)\n\n const uint8_t * v_q = v + 0; // quants first\n const uint8_t * v_d = v + qrow_size; // then scales\n\n const uint8_t * q = v_q + i * qblk_size;\n const ggml_half * d = (const ggml_half *) (v_d + i * dblk_size);\n\n HEX_VERBOSE(\"ggml-hex: repack q4x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\\n\", i,\n unpack_q4(q[0]).v[0], unpack_q4(q[1]).v[0], unpack_q4(q[2]).v[0], unpack_q4(q[3]).v[0],\n unpack_q4(q[60]).v[0], unpack_q4(q[61]).v[0], unpack_q4(q[62]).v[0], unpack_q4(q[63]).v[0],\n unpack_q4(q[124]).v[0], unpack_q4(q[125]).v[0], unpack_q4(q[126]).v[0], unpack_q4(q[127]).v[0],\n GGML_FP16_TO_FP32(d[0]), GGML_FP16_TO_FP32(d[1]), GGML_FP16_TO_FP32(d[2]), GGML_FP16_TO_FP32(d[3]));\n\n HEX_VERBOSE(\"ggml-hex: repack q4x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\\n\",\n i + 1, unpack_q4(q[0]).v[1], unpack_q4(q[1]).v[1], unpack_q4(q[2]).v[1], unpack_q4(q[3]).v[1],\n unpack_q4(q[60]).v[1], unpack_q4(q[61]).v[1], unpack_q4(q[62]).v[1], unpack_q4(q[63]).v[1],\n unpack_q4(q[124]).v[1], unpack_q4(q[125]).v[1], unpack_q4(q[126]).v[1], unpack_q4(q[127]).v[1],\n GGML_FP16_TO_FP32(d[4]), GGML_FP16_TO_FP32(d[5]), GGML_FP16_TO_FP32(d[6]), GGML_FP16_TO_FP32(d[7]));\n}\n\nstatic void unpack_q4_0_quants(uint8_t * qs, const block_q4_0 * x, unsigned int bi) {\n static const int qk = QK4_0;\n\n for (unsigned int i = 0; i < qk / 2; ++i) {\n const int x0 = (x->qs[i] & 0x0F);\n const int x1 = (x->qs[i] >> 4);\n qs[bi * qk + i + 0] = x0;\n qs[bi * qk + i + qk / 2] = x1;\n }\n}\n\nstatic void pack_q4_0_quants(block_q4_0 * x, const uint8_t * qs, unsigned int bi) {\n static const int qk = QK4_0;\n\n for (unsigned int i = 0; i < qk / 2; ++i) {\n const uint8_t x0 = qs[bi * qk + i + 0];\n const uint8_t x1 = qs[bi * qk + i + qk / 2];\n x->qs[i] = x0 | (x1 << 4);\n }\n}\n\nstatic void repack_row_q4x4x2(uint8_t * y, const block_q4_0 * x, int64_t k) {\n static const int qk = QK_Q4_0x4x2;\n const int nb = (k + qk - 1) / qk; // number of blocks (padded)\n const int nloe = k % qk; // leftovers\n\n const int dblk_size = 8 * 2; // 8x __fp16\n const int qblk_size = qk / 2; // int4\n const int qrow_size = k / 2; // int4 (not padded to blocks)\n\n uint8_t * y_q = y + 0; // quants first\n uint8_t * y_d = y + qrow_size; // then scales\n\n if (opt_verbose > 2) {\n for (int i = 0; i < nb; i++) {\n dump_block_q4_0(&x[i * 8 + 0], 0);\n dump_block_q4_0(&x[i * 8 + 1], 1);\n dump_block_q4_0(&x[i * 8 + 2], 2);\n dump_block_q4_0(&x[i * 8 + 3], 3);\n dump_block_q4_0(&x[i * 8 + 4], 4);\n dump_block_q4_0(&x[i * 8 + 5], 5);\n dump_block_q4_0(&x[i * 8 + 6], 6);\n dump_block_q4_0(&x[i * 8 + 7], 7);\n }\n }\n\n // Repack the quants\n for (int i = 0; i < nb; i++) {\n uint8_t qs[QK_Q4_0x4x2]; // unpacked quants\n unpack_q4_0_quants(qs, &x[i * 8 + 0], 0);\n unpack_q4_0_quants(qs, &x[i * 8 + 1], 1);\n unpack_q4_0_quants(qs, &x[i * 8 + 2], 2);\n unpack_q4_0_quants(qs, &x[i * 8 + 3], 3);\n unpack_q4_0_quants(qs, &x[i * 8 + 4], 4);\n unpack_q4_0_quants(qs, &x[i * 8 + 5], 5);\n unpack_q4_0_quants(qs, &x[i * 8 + 6], 6);\n unpack_q4_0_quants(qs, &x[i * 8 + 7], 7);\n\n bool partial = (nloe && i == nb-1);\n\n uint8_t * q = y_q + (i * qblk_size);\n for (int j = 0; j < qk / 2; j++) {\n q[j] = partial ? (qs[j*2+1] << 4) | qs[j*2+0] : (qs[j+128] << 4) | qs[j+000];\n }\n }\n\n // Repack the scales\n // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2)\n // the last block is truncated and overridden by the scales.\n for (int i = 0; i < nb; i++) {\n // Repack the scales\n ggml_half * d = (ggml_half *) (y_d + i * dblk_size);\n d[0] = x[i * 8 + 0].d;\n d[1] = x[i * 8 + 1].d;\n d[2] = x[i * 8 + 2].d;\n d[3] = x[i * 8 + 3].d;\n d[4] = x[i * 8 + 4].d;\n d[5] = x[i * 8 + 5].d;\n d[6] = x[i * 8 + 6].d;\n d[7] = x[i * 8 + 7].d;\n }\n\n if (opt_verbose > 1) {\n for (int i = 0; i < nb; i++) {\n dump_packed_block_q4x4x2(y, i, k);\n }\n }\n}\n\nstatic void unpack_row_q4x4x2(block_q4_0 * x, const uint8_t * y, int64_t k) {\n static const int qk = QK_Q4_0x4x2;\n const int nb = (k + qk - 1) / qk; // number of blocks (padded)\n const int nloe = k % qk; // leftovers\n\n const int dblk_size = 8 * 2; // 8x __fp16\n const int qblk_size = qk / 2; // int4\n const int qrow_size = k / 2; // int4 (not padded to blocks)\n\n const uint8_t * y_q = y + 0; // quants first\n const uint8_t * y_d = y + qrow_size; // then scales\n\n if (opt_verbose > 1) {\n for (int i = 0; i < nb; i++) {\n dump_packed_block_q4x4x2(y, i, k);\n }\n }\n\n // Unpack the quants\n for (int i = 0; i < nb; i++) {\n uint8_t qs[QK_Q4_0x4x2]; // unpacked quants\n\n bool partial = (nloe && i == nb-1);\n\n const uint8_t * q = y_q + (i * qblk_size);\n for (int j = 0; j < qk / 2; j++) {\n if (partial) {\n qs[j*2+0] = q[j] & 0xf;\n qs[j*2+1] = q[j] >> 4;\n } else {\n qs[j+000] = q[j] & 0xf;\n qs[j+128] = q[j] >> 4;\n }\n }\n\n pack_q4_0_quants(&x[i * 8 + 0], qs, 0);\n pack_q4_0_quants(&x[i * 8 + 1], qs, 1);\n pack_q4_0_quants(&x[i * 8 + 2], qs, 2);\n pack_q4_0_quants(&x[i * 8 + 3], qs, 3);\n pack_q4_0_quants(&x[i * 8 + 4], qs, 4);\n pack_q4_0_quants(&x[i * 8 + 5], qs, 5);\n pack_q4_0_quants(&x[i * 8 + 6], qs, 6);\n pack_q4_0_quants(&x[i * 8 + 7], qs, 7);\n }\n\n // Repack the scales\n // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2)\n // the last block is truncated and overridden by the scales.\n for (int i = 0; i < nb; i++) {\n // Unpack the scales\n const ggml_half * d = (const ggml_half *) (y_d + i * dblk_size);\n x[i * 8 + 0].d = d[0];\n x[i * 8 + 1].d = d[1];\n x[i * 8 + 2].d = d[2];\n x[i * 8 + 3].d = d[3];\n x[i * 8 + 4].d = d[4];\n x[i * 8 + 5].d = d[5];\n x[i * 8 + 6].d = d[6];\n x[i * 8 + 7].d = d[7];\n }\n\n if (opt_verbose > 2) {\n for (int i = 0; i < nb; i++) {\n dump_block_q4_0(&x[i * 8 + 0], 0);\n dump_block_q4_0(&x[i * 8 + 1], 1);\n dump_block_q4_0(&x[i * 8 + 2], 2);\n dump_block_q4_0(&x[i * 8 + 3], 3);\n dump_block_q4_0(&x[i * 8 + 4], 4);\n dump_block_q4_0(&x[i * 8 + 5], 5);\n dump_block_q4_0(&x[i * 8 + 6], 6);\n dump_block_q4_0(&x[i * 8 + 7], 7);\n }\n }\n}\n\nstatic void init_row_q4x4x2(block_q4_0 * x, int64_t k) {\n static const int qk = QK_Q4_0x4x2;\n const int nb = (k + qk - 1) / qk; // number of blocks (padded)\n\n // Init the quants such that they unpack into zeros\n uint8_t qs[QK_Q4_0x4x2]; // unpacked quants\n memset(qs, 8, sizeof(qs));\n\n for (int i = 0; i < nb; i++) {\n pack_q4_0_quants(&x[i * 8 + 0], qs, 0);\n pack_q4_0_quants(&x[i * 8 + 1], qs, 1);\n pack_q4_0_quants(&x[i * 8 + 2], qs, 2);\n pack_q4_0_quants(&x[i * 8 + 3], qs, 3);\n pack_q4_0_quants(&x[i * 8 + 4], qs, 4);\n pack_q4_0_quants(&x[i * 8 + 5], qs, 5);\n pack_q4_0_quants(&x[i * 8 + 6], qs, 6);\n pack_q4_0_quants(&x[i * 8 + 7], qs, 7);\n }\n\n // Init the scales\n // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2)\n // the last block is truncated and overridden by the scales.\n for (int i = 0; i < nb; i++) {\n // Unpack the scales\n x[i * 8 + 0].d = 0;\n x[i * 8 + 1].d = 0;\n x[i * 8 + 2].d = 0;\n x[i * 8 + 3].d = 0;\n x[i * 8 + 4].d = 0;\n x[i * 8 + 5].d = 0;\n x[i * 8 + 6].d = 0;\n x[i * 8 + 7].d = 0;\n }\n}\n\n// repack q4_0 data into q4x4x2 tensor\nstatic void repack_q4_0_q4x4x2(ggml_tensor * t, const void * data, size_t size) {\n int64_t nrows = ggml_nrows(t);\n\n size_t row_size = ggml_row_size(t->type, t->ne[0]);\n size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_Q4_0x4x2)); // extra elements for the pad\n size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any)\n\n // Ensure we don't try to read more data than is available in the source buffer 'data'\n // or write more than the tensor can hold.\n const size_t total_tensor_size = (size_t)nrows * row_size;\n const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size;\n\n // Calculate how many full rows and how many remaining bytes we need to process.\n const int64_t n_full_rows = n_bytes_to_copy / row_size;\n const size_t n_rem_bytes = n_bytes_to_copy % row_size;\n\n void * buf_pd = ggml_aligned_malloc(row_size_pd);\n GGML_ASSERT(buf_pd != NULL);\n\n void * buf_rp = ggml_aligned_malloc(row_size_rp);\n GGML_ASSERT(buf_rp != NULL);\n\n HEX_VERBOSE(\"ggml-hex: repack-q4_0-q4x4x2 %s : data %p size %zu dims %ldx%ld row-size %zu\\n\", t->name, data, size,\n t->ne[0], nrows, row_size);\n\n init_row_q4x4x2((block_q4_0 *) buf_pd, t->ne[0]); // init padded buffer to make sure the tail is all zeros\n\n // 1. Process all the full rows\n for (int64_t i = 0; i < n_full_rows; i++) {\n const uint8_t * src = (const uint8_t *) data + (i * row_size);\n uint8_t * dst = (uint8_t *) t->data + (i * row_size);\n\n memcpy(buf_pd, src, row_size);\n repack_row_q4x4x2((uint8_t *) buf_rp, (const block_q4_0 *) buf_pd, t->ne[0]);\n memcpy(dst, buf_rp, row_size);\n }\n\n // 2. Process the final, potentially partial, row\n if (n_rem_bytes > 0) {\n const int64_t i = n_full_rows;\n const uint8_t * src = (const uint8_t *) data + (i * row_size);\n uint8_t * dst = (uint8_t *) t->data + (i * row_size);\n\n // re-init the row because we are potentially copying a partial row\n init_row_q4x4x2((block_q4_0 *) buf_pd, t->ne[0]);\n\n // Copy only the remaining bytes from the source.\n memcpy(buf_pd, src, n_rem_bytes);\n\n // Repack the entire buffer\n repack_row_q4x4x2((uint8_t *) buf_rp, (const block_q4_0 *) buf_pd, t->ne[0]);\n\n // Write only the corresponding remaining bytes to the destination tensor.\n memcpy(dst, buf_rp, n_rem_bytes);\n }\n\n ggml_aligned_free(buf_pd, row_size_pd);\n ggml_aligned_free(buf_rp, row_size_rp);\n}\n\n// repack q4x4x2 tensor into q4_0 data\nstatic void repack_q4x4x2_q4_0(void * data, const ggml_tensor * t, size_t size) {\n int64_t nrows = ggml_nrows(t);\n\n size_t row_size = ggml_row_size(t->type, t->ne[0]);\n size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_Q4_0x4x2)); // extra elements for the pad\n size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any)\n\n // Ensure we don't try to copy more data than the tensor actually contains.\n const size_t total_tensor_size = (size_t)nrows * row_size;\n const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size;\n\n // Calculate how many full rows and how many remaining bytes we need to process.\n const int64_t n_full_rows = n_bytes_to_copy / row_size;\n const size_t n_rem_bytes = n_bytes_to_copy % row_size;\n\n void * buf_pd = ggml_aligned_malloc(row_size_pd);\n GGML_ASSERT(buf_pd != NULL);\n\n void * buf_rp = ggml_aligned_malloc(row_size_rp);\n GGML_ASSERT(buf_rp != NULL);\n\n HEX_VERBOSE(\"ggml-hex: repack-q4x4x2-q4_0 %s : data %p size %zu dims %ldx%ld row-size %zu\\n\", t->name, data, size,\n t->ne[0], nrows, row_size);\n\n memset(buf_pd, 0, row_size_pd); // clear-out padded buffer to make sure the tail is all zeros\n\n // 1. Process all the full rows\n for (int64_t i = 0; i < n_full_rows; i++) {\n const uint8_t * src = (const uint8_t *) t->data + (i * row_size);\n uint8_t * dst = (uint8_t *) data + (i * row_size);\n\n memcpy(buf_pd, src, row_size);\n unpack_row_q4x4x2((block_q4_0 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]);\n memcpy(dst, buf_rp, row_size);\n }\n\n // 2. Process the final, potentially partial, row\n if (n_rem_bytes > 0) {\n const int64_t i = n_full_rows;\n const uint8_t * src = (const uint8_t *) t->data + (i * row_size);\n uint8_t * dst = (uint8_t *) data + (i * row_size);\n\n // We still need to read and unpack the entire source row because quantization is block-based.\n memcpy(buf_pd, src, row_size);\n unpack_row_q4x4x2((block_q4_0 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]);\n\n // But we only copy the remaining number of bytes to the destination.\n memcpy(dst, buf_rp, n_rem_bytes);\n }\n\n ggml_aligned_free(buf_pd, row_size_pd);\n ggml_aligned_free(buf_rp, row_size_rp);\n}\n\n// ======== Q8x4x2 ====================\nstatic void dump_block_q8_0(const block_q8_0 * b, int i) {\n HEX_VERBOSE(\"ggml-hex: repack q8_0 %d: %d %d %d %d ... %d %d %d %d : %.6f\\n\", i, b->qs[0], b->qs[1], b->qs[2],\n b->qs[3], b->qs[28], b->qs[29], b->qs[30], b->qs[31], GGML_FP16_TO_FP32(b->d));\n}\n\nstatic void dump_packed_block_q8x4x2(const uint8_t * v, unsigned int i, size_t k) {\n static const int qk = QK_Q8_0x4x2;\n const int dblk_size = 8 * 2; // 8x __fp16\n const int qblk_size = qk; // int8\n const int qrow_size = k; // int8 (not padded)\n\n const uint8_t * v_q = v + 0; // quants first\n const uint8_t * v_d = v + qrow_size; // then scales\n\n const uint8_t * q = v_q + i * qblk_size;\n const ggml_half * d = (const ggml_half *) (v_d + i * dblk_size);\n\n HEX_VERBOSE(\"ggml-hex: repack q8x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\\n\", i,\n q[0], q[1], q[2], q[3], q[60], q[61], q[62], q[63], q[124], q[125], q[126], q[127],\n GGML_FP16_TO_FP32(d[0]), GGML_FP16_TO_FP32(d[1]), GGML_FP16_TO_FP32(d[2]), GGML_FP16_TO_FP32(d[3]));\n\n HEX_VERBOSE(\"ggml-hex: repack q8x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\\n\",\n i + 1, q[128], q[129], q[130], q[131], q[192], q[193], q[194], q[195], q[252], q[253], q[254], q[255],\n GGML_FP16_TO_FP32(d[4]), GGML_FP16_TO_FP32(d[5]), GGML_FP16_TO_FP32(d[6]), GGML_FP16_TO_FP32(d[7]));\n}\n\nstatic void unpack_q8_0_quants(uint8_t * qs, const block_q8_0 * x, unsigned int bi) {\n static const int qk = QK8_0;\n\n for (unsigned int i = 0; i < qk; ++i) {\n qs[bi * qk + i] = x->qs[i];\n }\n}\n\nstatic void pack_q8_0_quants(block_q8_0 * x, const uint8_t * qs, unsigned int bi) {\n static const int qk = QK8_0;\n\n for (unsigned int i = 0; i < qk; ++i) {\n x->qs[i] = qs[bi * qk + i];\n }\n}\n\nstatic void repack_row_q8x4x2(uint8_t * y, const block_q8_0 * x, int64_t k) {\n static const int qk = QK_Q8_0x4x2;\n const int nb = (k + qk - 1) / qk; // number of blocks (padded)\n\n const int dblk_size = 8 * 2; // 8x __fp16\n const int qblk_size = qk; // int8\n const int qrow_size = k; // int8 (not padded to blocks)\n\n uint8_t * y_q = y + 0; // quants first\n uint8_t * y_d = y + qrow_size; // then scales\n\n if (opt_verbose > 2) {\n for (int i = 0; i < nb; i++) {\n dump_block_q8_0(&x[i * 8 + 0], 0);\n dump_block_q8_0(&x[i * 8 + 1], 1);\n dump_block_q8_0(&x[i * 8 + 2], 2);\n dump_block_q8_0(&x[i * 8 + 3], 3);\n dump_block_q8_0(&x[i * 8 + 4], 4);\n dump_block_q8_0(&x[i * 8 + 5], 5);\n dump_block_q8_0(&x[i * 8 + 6], 6);\n dump_block_q8_0(&x[i * 8 + 7], 7);\n }\n }\n\n // Repack the quants\n for (int i = 0; i < nb; i++) {\n uint8_t qs[QK_Q8_0x4x2]; // unpacked quants\n\n unpack_q8_0_quants(qs, &x[i * 8 + 0], 0);\n unpack_q8_0_quants(qs, &x[i * 8 + 1], 1);\n unpack_q8_0_quants(qs, &x[i * 8 + 2], 2);\n unpack_q8_0_quants(qs, &x[i * 8 + 3], 3);\n unpack_q8_0_quants(qs, &x[i * 8 + 4], 4);\n unpack_q8_0_quants(qs, &x[i * 8 + 5], 5);\n unpack_q8_0_quants(qs, &x[i * 8 + 6], 6);\n unpack_q8_0_quants(qs, &x[i * 8 + 7], 7);\n\n uint8_t * q = y_q + (i * qblk_size);\n for (int j = 0; j < qk; j++) {\n q[j] = qs[j];\n }\n }\n\n // Repack the scales\n // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2)\n // the last block is truncated and overridden by the scales.\n for (int i = 0; i < nb; i++) {\n // Repack the scales\n ggml_half * d = (ggml_half *) (y_d + i * dblk_size);\n d[0] = x[i * 8 + 0].d;\n d[1] = x[i * 8 + 1].d;\n d[2] = x[i * 8 + 2].d;\n d[3] = x[i * 8 + 3].d;\n d[4] = x[i * 8 + 4].d;\n d[5] = x[i * 8 + 5].d;\n d[6] = x[i * 8 + 6].d;\n d[7] = x[i * 8 + 7].d;\n }\n\n if (opt_verbose > 1) {\n for (int i = 0; i < nb; i++) {\n dump_packed_block_q8x4x2(y, i, k);\n }\n }\n}\n\nstatic void unpack_row_q8x4x2(block_q8_0 * x, const uint8_t * y, int64_t k) {\n static const int qk = QK_Q8_0x4x2;\n const int nb = (k + qk - 1) / qk; // number of blocks (padded)\n\n const int dblk_size = 8 * 2; // 8x __fp16\n const int qblk_size = qk; // int8\n const int qrow_size = k; // int8 (not padded to blocks)\n\n const uint8_t * y_q = y + 0; // quants first\n const uint8_t * y_d = y + qrow_size; // then scales\n\n if (opt_verbose > 1) {\n for (int i = 0; i < nb; i++) {\n dump_packed_block_q8x4x2(y, i, k);\n }\n }\n\n // Unpack the quants\n for (int i = 0; i < nb; i++) {\n uint8_t qs[QK_Q4_0x4x2]; // unpacked quants\n\n const uint8_t * q = y_q + (i * qblk_size);\n for (int j = 0; j < qk; j++) {\n qs[j] = q[j];\n }\n\n pack_q8_0_quants(&x[i * 8 + 0], qs, 0);\n pack_q8_0_quants(&x[i * 8 + 1], qs, 1);\n pack_q8_0_quants(&x[i * 8 + 2], qs, 2);\n pack_q8_0_quants(&x[i * 8 + 3], qs, 3);\n pack_q8_0_quants(&x[i * 8 + 4], qs, 4);\n pack_q8_0_quants(&x[i * 8 + 5], qs, 5);\n pack_q8_0_quants(&x[i * 8 + 6], qs, 6);\n pack_q8_0_quants(&x[i * 8 + 7], qs, 7);\n }\n\n // Repack the scales\n // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2)\n // the last block is truncated and overridden by the scales.\n for (int i = 0; i < nb; i++) {\n // Unpack the scales\n const ggml_half * d = (const ggml_half *) (y_d + i * dblk_size);\n x[i * 8 + 0].d = d[0];\n x[i * 8 + 1].d = d[1];\n x[i * 8 + 2].d = d[2];\n x[i * 8 + 3].d = d[3];\n x[i * 8 + 4].d = d[4];\n x[i * 8 + 5].d = d[5];\n x[i * 8 + 6].d = d[6];\n x[i * 8 + 7].d = d[7];\n }\n\n if (opt_verbose > 2) {\n for (int i = 0; i < nb; i++) {\n dump_block_q8_0(&x[i * 8 + 0], 0);\n dump_block_q8_0(&x[i * 8 + 1], 1);\n dump_block_q8_0(&x[i * 8 + 2], 2);\n dump_block_q8_0(&x[i * 8 + 3], 3);\n dump_block_q8_0(&x[i * 8 + 4], 4);\n dump_block_q8_0(&x[i * 8 + 5], 5);\n dump_block_q8_0(&x[i * 8 + 6], 6);\n dump_block_q8_0(&x[i * 8 + 7], 7);\n }\n }\n}\n\nstatic void init_row_q8x4x2(block_q8_0 * x, int64_t k) {\n static const int qk = QK_Q8_0x4x2;\n const int nb = (k + qk - 1) / qk; // number of blocks (padded)\n\n // Init the quants such that they unpack into zeros\n uint8_t qs[QK_Q8_0x4x2]; // unpacked quants\n memset(qs, 0, sizeof(qs));\n\n for (int i = 0; i < nb; i++) {\n pack_q8_0_quants(&x[i * 8 + 0], qs, 0);\n pack_q8_0_quants(&x[i * 8 + 1], qs, 1);\n pack_q8_0_quants(&x[i * 8 + 2], qs, 2);\n pack_q8_0_quants(&x[i * 8 + 3], qs, 3);\n pack_q8_0_quants(&x[i * 8 + 4], qs, 4);\n pack_q8_0_quants(&x[i * 8 + 5], qs, 5);\n pack_q8_0_quants(&x[i * 8 + 6], qs, 6);\n pack_q8_0_quants(&x[i * 8 + 7], qs, 7);\n }\n\n // Init the scales\n // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q8_0x4x2)\n // the last block is truncated and overridden by the scales.\n for (int i = 0; i < nb; i++) {\n // Unpack the scales\n x[i * 8 + 0].d = 0;\n x[i * 8 + 1].d = 0;\n x[i * 8 + 2].d = 0;\n x[i * 8 + 3].d = 0;\n x[i * 8 + 4].d = 0;\n x[i * 8 + 5].d = 0;\n x[i * 8 + 6].d = 0;\n x[i * 8 + 7].d = 0;\n }\n}\n\n// repack q8_0 data into q8x4x2 tensor\nstatic void repack_q8_0_q8x4x2(ggml_tensor * t, const void * data, size_t size) {\n int64_t nrows = ggml_nrows(t);\n\n size_t row_size = ggml_row_size(t->type, t->ne[0]);\n size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_Q8_0x4x2)); // extra elements for the pad\n size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any)\n\n // Ensure we don't try to read more data than is available in the source buffer 'data'\n // or write more than the tensor can hold.\n const size_t total_tensor_size = (size_t)nrows * row_size;\n const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size;\n\n // Calculate how many full rows and how many remaining bytes we need to process.\n const int64_t n_full_rows = n_bytes_to_copy / row_size;\n const size_t n_rem_bytes = n_bytes_to_copy % row_size;\n\n void * buf_pd = ggml_aligned_malloc(row_size_pd);\n GGML_ASSERT(buf_pd != NULL);\n\n void * buf_rp = ggml_aligned_malloc(row_size_rp);\n GGML_ASSERT(buf_rp != NULL);\n\n HEX_VERBOSE(\"ggml-hex: repack-q8_0-q8x4x2 %s : data %p size %zu dims %ldx%ld row-size %zu\\n\", t->name, data, size,\n t->ne[0], nrows, row_size);\n\n init_row_q8x4x2((block_q8_0 *) buf_pd, t->ne[0]); // init padded buffer to make sure the tail is all zeros\n\n // 1. Process all the full rows\n for (int64_t i = 0; i < n_full_rows; i++) {\n const uint8_t * src = (const uint8_t *) data + (i * row_size);\n uint8_t * dst = (uint8_t *) t->data + (i * row_size);\n\n memcpy(buf_pd, src, row_size);\n repack_row_q8x4x2((uint8_t *) buf_rp, (const block_q8_0 *) buf_pd, t->ne[0]);\n memcpy(dst, buf_rp, row_size);\n }\n\n // 2. Process the final, potentially partial, row\n if (n_rem_bytes > 0) {\n const int64_t i = n_full_rows;\n const uint8_t * src = (const uint8_t *) data + (i * row_size);\n uint8_t * dst = (uint8_t *) t->data + (i * row_size);\n\n // re-init the row because we are potentially copying a partial row\n init_row_q8x4x2((block_q8_0 *) buf_pd, t->ne[0]);\n\n // Copy only the remaining bytes from the source.\n memcpy(buf_pd, src, n_rem_bytes);\n\n // Repack the entire buffer\n repack_row_q8x4x2((uint8_t *) buf_rp, (const block_q8_0 *) buf_pd, t->ne[0]);\n\n // Write only the corresponding remaining bytes to the destination tensor.\n memcpy(dst, buf_rp, n_rem_bytes);\n }\n\n ggml_aligned_free(buf_pd, row_size_pd);\n ggml_aligned_free(buf_rp, row_size_rp);\n}\n\n// repack q8x4x2 tensor into q8_0 data\nstatic void repack_q8x4x2_q8_0(void * data, const ggml_tensor * t, size_t size) {\n int64_t nrows = ggml_nrows(t);\n\n size_t row_size = ggml_row_size(t->type, t->ne[0]);\n size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_Q8_0x4x2)); // extra elements for the pad\n size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any)\n\n // Ensure we don't try to copy more data than the tensor actually contains.\n const size_t total_tensor_size = (size_t)nrows * row_size;\n const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size;\n\n // Calculate how many full rows and how many remaining bytes we need to process.\n const int64_t n_full_rows = n_bytes_to_copy / row_size;\n const size_t n_rem_bytes = n_bytes_to_copy % row_size;\n\n void * buf_pd = ggml_aligned_malloc(row_size_pd);\n GGML_ASSERT(buf_pd != NULL);\n\n void * buf_rp = ggml_aligned_malloc(row_size_rp);\n GGML_ASSERT(buf_rp != NULL);\n\n HEX_VERBOSE(\"ggml-hex: repack-q8x4x2-q8_0 %s : data %p size %zu dims %ldx%ld row-size %zu\\n\", t->name, data, size,\n t->ne[0], nrows, row_size);\n\n memset(buf_pd, 0, row_size_pd); // clear-out padded buffer to make sure the tail is all zeros\n\n // 1. Process all the full rows\n for (int64_t i = 0; i < n_full_rows; i++) {\n const uint8_t * src = (const uint8_t *) t->data + (i * row_size);\n uint8_t * dst = (uint8_t *) data + (i * row_size);\n\n memcpy(buf_pd, src, row_size);\n unpack_row_q8x4x2((block_q8_0 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]);\n memcpy(dst, buf_rp, row_size);\n }\n\n // 2. Process the final, potentially partial, row\n if (n_rem_bytes > 0) {\n const int64_t i = n_full_rows;\n const uint8_t * src = (const uint8_t *) t->data + (i * row_size);\n uint8_t * dst = (uint8_t *) data + (i * row_size);\n\n // We still need to read and unpack the entire source row because quantization is block-based.\n memcpy(buf_pd, src, row_size);\n unpack_row_q8x4x2((block_q8_0 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]);\n\n // But we only copy the remaining number of bytes to the destination.\n memcpy(dst, buf_rp, n_rem_bytes);\n }\n\n ggml_aligned_free(buf_pd, row_size_pd);\n ggml_aligned_free(buf_rp, row_size_rp);\n}\n\n// ======== MXFP4x4x2 ====================\nstruct x2_mxfp4 {\n int v[2];\n};\n\nstatic x2_mxfp4 unpack_mxfp4(uint8_t v) {\n x2_mxfp4 x;\n x.v[0] = kvalues_mxfp4[(v & 0x0f)];\n x.v[1] = kvalues_mxfp4[(v >> 4)];\n return x;\n}\n\nstatic void dump_block_mxfp4(const block_mxfp4 * b, int i) {\n HEX_VERBOSE(\"ggml-hex: repack mxfp4 %d: %d %d %d %d ... %d %d %d %d : %.6f\\n\", i, unpack_mxfp4(b->qs[0]).v[0],\n unpack_mxfp4(b->qs[1]).v[0], unpack_mxfp4(b->qs[2]).v[0], unpack_mxfp4(b->qs[3]).v[0],\n unpack_mxfp4(b->qs[12]).v[1], unpack_mxfp4(b->qs[13]).v[1], unpack_mxfp4(b->qs[14]).v[1],\n unpack_mxfp4(b->qs[15]).v[1], GGML_E8M0_TO_FP32_HALF(b->e));\n}\n\nstatic void dump_packed_block_mxfp4x4x2(const uint8_t * v, unsigned int i, size_t k) {\n static const int qk = QK_MXFP4x4x2;\n const int eblk_size = 8 * 1; // 8x E8M0\n const int qblk_size = qk / 2; // int4\n const int qrow_size = k / 2; // int4 (not padded)\n\n const uint8_t * v_q = v + 0; // quants first\n const uint8_t * v_e = v + qrow_size; // then scales\n\n const uint8_t * q = v_q + i * qblk_size;\n const uint8_t * e = (const uint8_t *) (v_e + i * eblk_size);\n\n HEX_VERBOSE(\"ggml-hex: repack mxfp4x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\\n\", i,\n unpack_mxfp4(q[0]).v[0], unpack_mxfp4(q[1]).v[0], unpack_mxfp4(q[2]).v[0], unpack_mxfp4(q[3]).v[0],\n unpack_mxfp4(q[60]).v[0], unpack_mxfp4(q[61]).v[0], unpack_mxfp4(q[62]).v[0], unpack_mxfp4(q[63]).v[0],\n unpack_mxfp4(q[124]).v[0], unpack_mxfp4(q[125]).v[0], unpack_mxfp4(q[126]).v[0],\n unpack_mxfp4(q[127]).v[0], GGML_E8M0_TO_FP32_HALF(e[0]), GGML_E8M0_TO_FP32_HALF(e[1]),\n GGML_E8M0_TO_FP32_HALF(e[2]), GGML_E8M0_TO_FP32_HALF(e[3]));\n\n HEX_VERBOSE(\"ggml-hex: repack mxfp4x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\\n\",\n i + 1, unpack_mxfp4(q[0]).v[1], unpack_mxfp4(q[1]).v[1], unpack_mxfp4(q[2]).v[1],\n unpack_mxfp4(q[3]).v[1], unpack_mxfp4(q[60]).v[1], unpack_mxfp4(q[61]).v[1], unpack_mxfp4(q[62]).v[1],\n unpack_mxfp4(q[63]).v[1], unpack_mxfp4(q[124]).v[1], unpack_mxfp4(q[125]).v[1],\n unpack_mxfp4(q[126]).v[1], unpack_mxfp4(q[127]).v[1], GGML_E8M0_TO_FP32_HALF(e[4]),\n GGML_E8M0_TO_FP32_HALF(e[5]), GGML_E8M0_TO_FP32_HALF(e[6]), GGML_E8M0_TO_FP32_HALF(e[7]));\n}\n\nstatic void unpack_mxfp4_quants(uint8_t * qs, const block_mxfp4 * x, unsigned int bi) {\n static const int qk = QK_MXFP4;\n\n for (unsigned int i = 0; i < qk / 2; ++i) {\n const uint8_t x0 = (x->qs[i] & 0x0F);\n const uint8_t x1 = (x->qs[i] >> 4);\n qs[bi * qk + i + 0] = x0;\n qs[bi * qk + i + qk / 2] = x1;\n }\n}\n\nstatic void pack_mxfp4_quants(block_mxfp4 * x, const uint8_t * qs, unsigned int bi) {\n static const int qk = QK4_0;\n\n for (unsigned int i = 0; i < qk / 2; ++i) {\n const uint8_t x0 = qs[bi * qk + i + 0];\n const uint8_t x1 = qs[bi * qk + i + qk / 2];\n x->qs[i] = x0 | (x1 << 4);\n }\n}\n\nstatic void repack_row_mxfp4x4x2(uint8_t * y, const block_mxfp4 * x, int64_t k) {\n static const int qk = QK_MXFP4x4x2;\n const int nb = (k + qk - 1) / qk; // number of blocks (padded)\n const int nloe = k % qk; // leftovers\n\n const int eblk_size = 8 * 1; // 8x E8M0\n const int qblk_size = qk / 2; // int4\n const int qrow_size = k / 2; // int4 (not padded to blocks)\n\n uint8_t * y_q = y + 0; // quants first\n uint8_t * y_e = y + qrow_size; // then scales\n\n if (opt_verbose > 2) {\n for (int i = 0; i < nb; i++) {\n dump_block_mxfp4(&x[i * 8 + 0], 0);\n dump_block_mxfp4(&x[i * 8 + 1], 1);\n dump_block_mxfp4(&x[i * 8 + 2], 2);\n dump_block_mxfp4(&x[i * 8 + 3], 3);\n dump_block_mxfp4(&x[i * 8 + 4], 4);\n dump_block_mxfp4(&x[i * 8 + 5], 5);\n dump_block_mxfp4(&x[i * 8 + 6], 6);\n dump_block_mxfp4(&x[i * 8 + 7], 7);\n }\n }\n\n // Repack the quants\n for (int i = 0; i < nb; i++) {\n uint8_t qs[QK_MXFP4x4x2]; // unpacked quants\n\n unpack_mxfp4_quants(qs, &x[i * 8 + 0], 0);\n unpack_mxfp4_quants(qs, &x[i * 8 + 1], 1);\n unpack_mxfp4_quants(qs, &x[i * 8 + 2], 2);\n unpack_mxfp4_quants(qs, &x[i * 8 + 3], 3);\n unpack_mxfp4_quants(qs, &x[i * 8 + 4], 4);\n unpack_mxfp4_quants(qs, &x[i * 8 + 5], 5);\n unpack_mxfp4_quants(qs, &x[i * 8 + 6], 6);\n unpack_mxfp4_quants(qs, &x[i * 8 + 7], 7);\n\n bool partial = (nloe && i == nb-1);\n\n uint8_t * q = y_q + (i * qblk_size);\n for (int j = 0; j < qk / 2; j++) {\n q[j] = partial ? (qs[j*2+1] << 4) | qs[j*2+0] : (qs[j+128] << 4) | qs[j+000];\n }\n }\n\n // Repack the scales\n // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_MXFP4x4x2)\n // the last block is truncated and overridden by the scales.\n for (int i = 0; i < nb; i++) {\n // Repack the scales\n uint8_t * e = (uint8_t *) (y_e + i * eblk_size);\n e[0] = x[i * 8 + 0].e;\n e[1] = x[i * 8 + 1].e;\n e[2] = x[i * 8 + 2].e;\n e[3] = x[i * 8 + 3].e;\n e[4] = x[i * 8 + 4].e;\n e[5] = x[i * 8 + 5].e;\n e[6] = x[i * 8 + 6].e;\n e[7] = x[i * 8 + 7].e;\n }\n\n if (opt_verbose > 1) {\n for (int i = 0; i < nb; i++) {\n dump_packed_block_mxfp4x4x2(y, i, k);\n }\n }\n}\n\nstatic void unpack_row_mxfp4x4x2(block_mxfp4 * x, const uint8_t * y, int64_t k) {\n static const int qk = QK_MXFP4x4x2;\n const int nb = (k + qk - 1) / qk; // number of blocks (padded)\n const int nloe = k % qk; // leftovers\n\n const int eblk_size = 8 * 1; // 8x E8M0\n const int qblk_size = qk / 2; // int4\n const int qrow_size = k / 2; // int4 (not padded to blocks)\n\n const uint8_t * y_q = y + 0; // quants first\n const uint8_t * y_e = y + qrow_size; // then scales\n\n if (opt_verbose > 1) {\n for (int i = 0; i < nb; i++) {\n dump_packed_block_mxfp4x4x2(y, i, k);\n }\n }\n\n // Unpack the quants\n for (int i = 0; i < nb; i++) {\n uint8_t qs[QK_MXFP4x4x2]; // unpacked quants\n\n bool partial = (nloe && i == nb-1);\n\n const uint8_t * q = y_q + (i * qblk_size);\n for (int j = 0; j < qk / 2; j++) {\n if (partial) {\n qs[j*2+0] = q[j] & 0xf;\n qs[j*2+1] = q[j] >> 4;\n } else {\n qs[j+000] = q[j] & 0xf;\n qs[j+128] = q[j] >> 4;\n }\n }\n\n pack_mxfp4_quants(&x[i * 8 + 0], qs, 0);\n pack_mxfp4_quants(&x[i * 8 + 1], qs, 1);\n pack_mxfp4_quants(&x[i * 8 + 2], qs, 2);\n pack_mxfp4_quants(&x[i * 8 + 3], qs, 3);\n pack_mxfp4_quants(&x[i * 8 + 4], qs, 4);\n pack_mxfp4_quants(&x[i * 8 + 5], qs, 5);\n pack_mxfp4_quants(&x[i * 8 + 6], qs, 6);\n pack_mxfp4_quants(&x[i * 8 + 7], qs, 7);\n }\n\n // Repack the scales\n // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_MXFP4_0x4x2)\n // the last block is truncated and overridden by the scales.\n for (int i = 0; i < nb; i++) {\n // Unpack the scales\n const uint8_t * e = (const uint8_t *) (y_e + i * eblk_size);\n x[i * 8 + 0].e = e[0];\n x[i * 8 + 1].e = e[1];\n x[i * 8 + 2].e = e[2];\n x[i * 8 + 3].e = e[3];\n x[i * 8 + 4].e = e[4];\n x[i * 8 + 5].e = e[5];\n x[i * 8 + 6].e = e[6];\n x[i * 8 + 7].e = e[7];\n }\n\n if (opt_verbose > 2) {\n for (int i = 0; i < nb; i++) {\n dump_block_mxfp4(&x[i * 8 + 0], 0);\n dump_block_mxfp4(&x[i * 8 + 1], 1);\n dump_block_mxfp4(&x[i * 8 + 2], 2);\n dump_block_mxfp4(&x[i * 8 + 3], 3);\n dump_block_mxfp4(&x[i * 8 + 4], 4);\n dump_block_mxfp4(&x[i * 8 + 5], 5);\n dump_block_mxfp4(&x[i * 8 + 6], 6);\n dump_block_mxfp4(&x[i * 8 + 7], 7);\n }\n }\n}\n\nstatic void init_row_mxfp4x4x2(block_mxfp4 * x, int64_t k) {\n static const int qk = QK_MXFP4x4x2;\n const int nb = (k + qk - 1) / qk; // number of blocks (padded)\n\n // Init the quants such that they unpack into zeros\n uint8_t qs[QK_MXFP4x4x2]; // unpacked quants\n memset(qs, 0, sizeof(qs));\n\n for (int i = 0; i < nb; i++) {\n pack_mxfp4_quants(&x[i * 8 + 0], qs, 0);\n pack_mxfp4_quants(&x[i * 8 + 1], qs, 1);\n pack_mxfp4_quants(&x[i * 8 + 2], qs, 2);\n pack_mxfp4_quants(&x[i * 8 + 3], qs, 3);\n pack_mxfp4_quants(&x[i * 8 + 4], qs, 4);\n pack_mxfp4_quants(&x[i * 8 + 5], qs, 5);\n pack_mxfp4_quants(&x[i * 8 + 6], qs, 6);\n pack_mxfp4_quants(&x[i * 8 + 7], qs, 7);\n }\n\n // Init the scales\n // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_MXFP4x4x2)\n // the last block is truncated and overridden by the scales.\n for (int i = 0; i < nb; i++) {\n // Unpack the scales\n x[i * 8 + 0].e = 0;\n x[i * 8 + 1].e = 0;\n x[i * 8 + 2].e = 0;\n x[i * 8 + 3].e = 0;\n x[i * 8 + 4].e = 0;\n x[i * 8 + 5].e = 0;\n x[i * 8 + 6].e = 0;\n x[i * 8 + 7].e = 0;\n }\n}\n\n// repack mxfp4 data into mxfp4x4x2 tensor\nstatic void repack_mxfp4_mxfp4x4x2(ggml_tensor * t, const void * data, size_t size) {\n int64_t nrows = ggml_nrows(t);\n\n size_t row_size = ggml_row_size(t->type, t->ne[0]);\n size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_MXFP4x4x2)); // extra elements for the pad\n size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any)\n\n // Ensure we don't try to read more data than is available in the source buffer 'data'\n // or write more than the tensor can hold.\n const size_t total_tensor_size = (size_t)nrows * row_size;\n const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size;\n\n // Calculate how many full rows and how many remaining bytes we need to process.\n const int64_t n_full_rows = n_bytes_to_copy / row_size;\n const size_t n_rem_bytes = n_bytes_to_copy % row_size;\n\n void * buf_pd = ggml_aligned_malloc(row_size_pd);\n GGML_ASSERT(buf_pd != NULL);\n\n void * buf_rp = ggml_aligned_malloc(row_size_rp);\n GGML_ASSERT(buf_rp != NULL);\n\n HEX_VERBOSE(\"ggml-hex: repack-mxfp4-mxfp4x4x2 %s : data %p size %zu dims %ldx%ld row-size %zu\\n\", t->name, data,\n size, t->ne[0], nrows, row_size);\n\n init_row_mxfp4x4x2((block_mxfp4 *) buf_pd, t->ne[0]); // init padded buffer to make sure the tail is all zeros\n\n // 1. Process all the full rows\n for (int64_t i = 0; i < n_full_rows; i++) {\n const uint8_t * src = (const uint8_t *) data + (i * row_size);\n uint8_t * dst = (uint8_t *) t->data + (i * row_size);\n\n memcpy(buf_pd, src, row_size);\n repack_row_mxfp4x4x2((uint8_t *) buf_rp, (const block_mxfp4 *) buf_pd, t->ne[0]);\n memcpy(dst, buf_rp, row_size);\n }\n\n // 2. Process the final, potentially partial, row\n if (n_rem_bytes > 0) {\n const int64_t i = n_full_rows;\n const uint8_t * src = (const uint8_t *) data + (i * row_size);\n uint8_t * dst = (uint8_t *) t->data + (i * row_size);\n\n // re-init the row because we are potentially copying a partial row\n init_row_mxfp4x4x2((block_mxfp4 *) buf_pd, t->ne[0]);\n\n // Copy only the remaining bytes from the source.\n memcpy(buf_pd, src, n_rem_bytes);\n\n // Repack the entire buffer (partial data + zero padding).\n repack_row_mxfp4x4x2((uint8_t *) buf_rp, (const block_mxfp4 *) buf_pd, t->ne[0]);\n\n // Write only the corresponding remaining bytes to the destination tensor.\n memcpy(dst, buf_rp, n_rem_bytes);\n }\n\n ggml_aligned_free(buf_pd, row_size_pd);\n ggml_aligned_free(buf_rp, row_size_rp);\n}\n\n// repack mxfp4x4x2 tensor into mxfp4 data\nstatic void repack_mxfp4x4x2_mxfp4(void * data, const ggml_tensor * t, size_t size) {\n int64_t nrows = ggml_nrows(t);\n\n size_t row_size = ggml_row_size(t->type, t->ne[0]);\n size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_MXFP4x4x2)); // extra elements for the pad\n size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any)\n\n // Ensure we don't try to copy more data than the tensor actually contains.\n const size_t total_tensor_size = (size_t)nrows * row_size;\n const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size;\n\n // Calculate how many full rows and how many remaining bytes we need to process.\n const int64_t n_full_rows = n_bytes_to_copy / row_size;\n const size_t n_rem_bytes = n_bytes_to_copy % row_size;\n\n void * buf_pd = ggml_aligned_malloc(row_size_pd);\n GGML_ASSERT(buf_pd != NULL);\n\n void * buf_rp = ggml_aligned_malloc(row_size_rp);\n GGML_ASSERT(buf_rp != NULL);\n\n HEX_VERBOSE(\"ggml-hex: repack-mxfp4x4x2-mxfp4 %s : data %p size %zu dims %ldx%ld row-size %zu\\n\", t->name, data,\n size, t->ne[0], nrows, row_size);\n\n memset(buf_pd, 0, row_size_pd); // clear-out padded buffer to make sure the tail is all zeros\n\n // 1. Process all the full rows\n for (int64_t i = 0; i < n_full_rows; i++) {\n const uint8_t * src = (const uint8_t *) t->data + (i * row_size);\n uint8_t * dst = (uint8_t *) data + (i * row_size);\n\n memcpy(buf_pd, src, row_size);\n unpack_row_mxfp4x4x2((block_mxfp4 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]);\n memcpy(dst, buf_rp, row_size);\n }\n\n // 2. Process the final, potentially partial, row\n if (n_rem_bytes > 0) {\n const int64_t i = n_full_rows;\n const uint8_t * src = (const uint8_t *) t->data + (i * row_size);\n uint8_t * dst = (uint8_t *) data + (i * row_size);\n\n // We still need to read and unpack the entire source row because the format is block-based.\n memcpy(buf_pd, src, row_size);\n unpack_row_mxfp4x4x2((block_mxfp4 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]);\n\n // But we only copy the remaining number of bytes to the destination to respect the size limit.\n memcpy(dst, buf_rp, n_rem_bytes);\n }\n\n ggml_aligned_free(buf_pd, row_size_pd);\n ggml_aligned_free(buf_rp, row_size_rp);\n}\n\nstatic void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer,\n ggml_tensor * tensor,\n const void * data,\n size_t offset,\n size_t size) {\n auto ctx = (ggml_backend_hexagon_buffer_context *) buffer->context;\n auto sess = ctx->sess;\n\n HEX_VERBOSE(\"ggml-hex: %s set-tensor %s : data %p offset %zu size %zu\\n\", sess->name.c_str(), tensor->name, data,\n offset, size);\n\n switch (tensor->type) {\n case GGML_TYPE_Q4_0:\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor));\n repack_q4_0_q4x4x2(tensor, data, size);\n break;\n\n case GGML_TYPE_Q8_0:\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor));\n repack_q8_0_q8x4x2(tensor, data, size);\n break;\n\n case GGML_TYPE_MXFP4:\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor));\n repack_mxfp4_mxfp4x4x2(tensor, data, size);\n break;\n\n default:\n memcpy((char *) tensor->data + offset, data, size);\n break;\n }\n}\n\nstatic void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,\n const ggml_tensor * tensor,\n void * data,\n size_t offset,\n size_t size) {\n auto ctx = (ggml_backend_hexagon_buffer_context *) buffer->context;\n auto sess = ctx->sess;\n\n HEX_VERBOSE(\"ggml-hex: %s get-tensor %s : data %p offset %zu size %zu\\n\", sess->name.c_str(), tensor->name, data,\n offset, size);\n\n switch (tensor->type) {\n case GGML_TYPE_Q4_0:\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor));\n repack_q4x4x2_q4_0(data, tensor, size);\n break;\n\n case GGML_TYPE_Q8_0:\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor));\n repack_q8x4x2_q8_0(data, tensor, size);\n break;\n\n case GGML_TYPE_MXFP4:\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(offset + size <= ggml_nbytes(tensor));\n repack_mxfp4x4x2_mxfp4(data, tensor, size);\n break;\n\n default:\n memcpy(data, (const char *) tensor->data + offset, size);\n break;\n }\n}\n\nstatic bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t buffer,\n const struct ggml_tensor * src,\n struct ggml_tensor * dst) {\n GGML_UNUSED(buffer);\n GGML_UNUSED(src);\n GGML_UNUSED(dst);\n // we might optimize this later, for now take the slow path (ie get/set_tensor)\n return false;\n}\n\nstatic void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n auto ctx = (ggml_backend_hexagon_buffer_context *) buffer->context;\n auto sess = ctx->sess;\n HEX_VERBOSE(\"ggml-hex: %s clear-buff base %p size %zu\\n\", sess->name.c_str(), (void *) ctx->base, ctx->size);\n memset(ctx->base, value, ctx->size);\n}\n\nstatic ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = {\n /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer,\n /* .get_base = */ ggml_backend_hexagon_buffer_get_base,\n /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor,\n /* .memset_tensor = */ NULL,\n /* .set_tensor = */ ggml_backend_hexagon_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_hexagon_buffer_get_tensor,\n /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor,\n /* .clear = */ ggml_backend_hexagon_buffer_clear,\n /* .reset = */ NULL,\n};\n\n// ** backend buffer type\n\nstatic const char * ggml_backend_hexagon_buffer_type_name(ggml_backend_buffer_type_t buffer_type) {\n return static_cast(buffer_type->context)->name.c_str();\n}\n\nstatic ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer(\n ggml_backend_buffer_type_t buffer_type, size_t size) {\n auto sess = static_cast(buffer_type->context)->sess;\n try {\n ggml_backend_hexagon_buffer_context * ctx = new ggml_backend_hexagon_buffer_context(sess, size, false /*repack*/);\n return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, ctx, size);\n } catch (const std::exception & exc) {\n GGML_LOG_ERROR(\"ggml-hex: %s failed to allocate buffer context: %s\\n\", sess->name.c_str(), exc.what());\n return nullptr;\n }\n}\n\nstatic ggml_backend_buffer_t ggml_backend_hexagon_repack_buffer_type_alloc_buffer(\n ggml_backend_buffer_type_t buffer_type, size_t size) {\n auto sess = static_cast(buffer_type->context)->sess;\n try {\n ggml_backend_hexagon_buffer_context * ctx = new ggml_backend_hexagon_buffer_context(sess, size, true /*repack*/);\n return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, ctx, size);\n } catch (const std::exception & exc) {\n GGML_LOG_ERROR(\"ggml-hex: %s failed to allocate buffer context: %s\\n\", sess->name.c_str(), exc.what());\n return nullptr;\n }\n}\n\nstatic size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) {\n return 128; // HVX alignment\n GGML_UNUSED(buffer_type);\n}\n\nstatic size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) {\n return ggml_nbytes(t);\n}\n\nstatic size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) {\n return 1 * 1024 * 1024 * 1024; // 1GB per buffer\n GGML_UNUSED(buffer_type);\n}\n\nstatic bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) {\n return opt_hostbuf;\n GGML_UNUSED(buft);\n}\n\nstatic bool ggml_backend_hexagon_repack_buffer_type_is_host(ggml_backend_buffer_type_t buft) {\n return false;\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_type_i ggml_backend_hexagon_buffer_type_interface = {\n /* .get_name = */ ggml_backend_hexagon_buffer_type_name,\n /* .alloc_buffer = */ ggml_backend_hexagon_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment,\n /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size,\n /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size,\n /* .is_host = */ ggml_backend_hexagon_buffer_type_is_host,\n};\n\nstatic ggml_backend_buffer_type_i ggml_backend_hexagon_repack_buffer_type_interface = {\n /* .get_name = */ ggml_backend_hexagon_buffer_type_name,\n /* .alloc_buffer = */ ggml_backend_hexagon_repack_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment,\n /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size,\n /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size,\n /* .is_host = */ ggml_backend_hexagon_repack_buffer_type_is_host,\n};\n\nvoid ggml_hexagon_session::allocate(int dev_id) noexcept(false) {\n this->valid_session = false;\n this->valid_handle = false;\n this->valid_queue = false;\n this->valid_iface = false;\n\n this->domain_id = 3; // Default for CDSP, updated after the session is created\n this->session_id = 0; // Default for CDSP, updated after the session is created\n this->dev_id = dev_id;\n this->name = std::string(\"HTP\") + std::to_string(dev_id);\n\n this->op_pending = 0;\n this->prof_usecs = 0;\n this->prof_cycles = 0;\n this->prof_pkts = 0;\n\n GGML_LOG_INFO(\"ggml-hex: allocating new session: %s\\n\", this->name.c_str());\n\n domain * my_domain = get_domain(this->domain_id);\n if (my_domain == NULL) {\n GGML_LOG_ERROR(\"ggml-hex: unable to get domain struct for CDSP\\n\");\n throw std::runtime_error(\"ggml-hex: failed to get CDSP domain (see log for details)\");\n }\n\n // Create new session\n if (dev_id != 0) {\n struct remote_rpc_reserve_new_session n;\n n.domain_name_len = strlen(CDSP_DOMAIN_NAME);\n n.domain_name = const_cast(CDSP_DOMAIN_NAME);\n n.session_name = const_cast(this->name.c_str());\n n.session_name_len = this->name.size();\n\n int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n));\n if (err != AEE_SUCCESS) {\n GGML_LOG_ERROR(\"ggml-hex: failed to reserve new session %d : error 0x%x\\n\", dev_id, err);\n throw std::runtime_error(\"ggml-hex: remote_session_control(new-sess) failed (see log for details)\");\n }\n\n // Save the IDs\n this->session_id = n.session_id;\n this->domain_id = n.effective_domain_id;\n this->valid_session = true;\n }\n\n // Get session URI\n\n char session_uri[256];\n {\n char htp_uri[256];\n snprintf(htp_uri, sizeof(htp_uri), \"file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0\", opt_arch);\n\n struct remote_rpc_get_uri u = {};\n u.session_id = this->session_id;\n u.domain_name = const_cast(CDSP_DOMAIN_NAME);\n u.domain_name_len = strlen(CDSP_DOMAIN_NAME);\n u.module_uri = const_cast(htp_uri);\n u.module_uri_len = strlen(htp_uri);\n u.uri = session_uri;\n u.uri_len = sizeof(session_uri);\n\n int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u));\n if (err != AEE_SUCCESS) {\n // fallback to single session uris\n int htp_URI_domain_len = strlen(htp_uri) + MAX_DOMAIN_NAMELEN;\n\n snprintf(session_uri, htp_URI_domain_len, \"%s%s\", htp_uri, my_domain->uri);\n\n GGML_LOG_WARN(\"ggml-hex: failed to get URI for session %d : error 0x%x. Falling back to single session URI: %s\\n\", dev_id, err, session_uri);\n }\n }\n\n // Enable Unsigned PD\n {\n struct remote_rpc_control_unsigned_module u;\n u.domain = this->domain_id;\n u.enable = 1;\n int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u));\n if (err != AEE_SUCCESS) {\n GGML_LOG_ERROR(\"ggml-hex: failed to enable unsigned PD for session %d : error 0x%x\\n\", dev_id, err);\n throw std::runtime_error(\"ggml-hex: remote_session_control(unsign) failed (see log for details)\");\n }\n }\n\n // Open session\n int err = htp_iface_open(session_uri, &this->handle);\n if (err != AEE_SUCCESS) {\n GGML_LOG_ERROR(\"ggml-hex: failed to open session %d : error 0x%x\\n\", dev_id, err);\n throw std::runtime_error(\"ggml-hex: failed to open session (see log for details)\");\n }\n\n this->valid_handle = true;\n\n GGML_LOG_INFO(\"ggml-hex: new session: %s : session-id %d domain-id %d uri %s handle 0x%lx\\n\", this->name.c_str(),\n this->session_id, this->domain_id, session_uri, (unsigned long) this->handle);\n\n // Enable FastRPC QoS mode\n {\n struct remote_rpc_control_latency l;\n l.enable = 1;\n\n int err = remote_handle64_control(this->handle, DSPRPC_CONTROL_LATENCY, (void *) &l, sizeof(l));\n if (err != 0) {\n GGML_LOG_WARN(\"ggml-hex: failed to enable fastrpc QOS mode: 0x%08x\\n\", (unsigned) err);\n }\n }\n\n // Now let's setup the DSP queue\n err = dspqueue_create(this->domain_id,\n 0, // Flags\n 128 * 1024, // Request queue size (in bytes)\n 64 * 1024, // Response queue size (in bytes)\n nullptr, // Read packet callback (we handle reads explicitly)\n nullptr, // Error callback (we handle errors during reads)\n (void *) this, // Callback context\n &queue);\n if (err != 0) {\n GGML_LOG_ERROR(\"ggml-hex: %s dspqueue_create failed: 0x%08x\\n\", this->name.c_str(), (unsigned) err);\n throw std::runtime_error(\"ggml-hex: failed to create dspqueue (see log for details)\");\n }\n\n this->valid_queue = true;\n\n // Export queue for use on the DSP\n err = dspqueue_export(queue, &this->queue_id);\n if (err != 0) {\n GGML_LOG_ERROR(\"ggml-hex: dspqueue_export failed: 0x%08x\\n\", (unsigned) err);\n throw std::runtime_error(\"ggml-hex: dspqueue export failed (see log for details)\");\n }\n\n if (opt_etm) {\n err = htp_iface_enable_etm(this->handle);\n if (err != 0) {\n GGML_LOG_ERROR(\"ggml-hex: failed to enable ETM tracing: 0x%08x\\n\", (unsigned) err);\n }\n }\n\n // Start the DSP-side service. We need to pass the queue ID to the\n // DSP in a FastRPC call; the DSP side will import the queue and start\n // listening for packets in a callback.\n err = htp_iface_start(this->handle, dev_id, this->queue_id, opt_nhvx);\n if (err != 0) {\n GGML_LOG_ERROR(\"ggml-hex: failed to start session: 0x%08x\\n\", (unsigned) err);\n throw std::runtime_error(\"ggml-hex: iface start failed (see log for details)\");\n }\n this->valid_iface = true;\n}\n\nvoid ggml_hexagon_session::release() noexcept(true) {\n GGML_LOG_INFO(\"ggml-hex: releasing session: %s\\n\", this->name.c_str());\n\n int err;\n\n // Stop the DSP-side service and close the queue\n if (this->valid_iface) {\n err = htp_iface_stop(this->handle);\n if (err != 0) {\n GGML_ABORT(\"ggml-hex: htp_iface_stop failed: 0x%08x\\n\", (unsigned) err);\n }\n }\n\n if (opt_etm) {\n err = htp_iface_disable_etm(this->handle);\n if (err != 0) {\n GGML_LOG_ERROR(\"ggml-hex: warn : failed to disable ETM tracing: 0x%08x\\n\", (unsigned) err);\n }\n }\n\n if (this->valid_queue) {\n err = dspqueue_close(queue);\n if (err != 0) {\n GGML_ABORT(\"ggml-hex: dspqueue_close failed: 0x%08x\\n\", (unsigned) err);\n }\n }\n\n if (this->valid_handle) {\n htp_iface_close(this->handle);\n }\n}\n\nggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false) {\n buffer_type.device = dev;\n repack_buffer_type.device = dev;\n\n try {\n allocate(dev_id);\n\n buffer_type.iface = ggml_backend_hexagon_buffer_type_interface;\n buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name, this);\n\n repack_buffer_type.iface = ggml_backend_hexagon_repack_buffer_type_interface;\n repack_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name + \"-REPACK\", this);\n } catch (const std::exception & exc) {\n release();\n throw;\n }\n}\n\nggml_hexagon_session::~ggml_hexagon_session() noexcept(true) {\n release();\n\n delete static_cast(buffer_type.context);\n delete static_cast(repack_buffer_type.context);\n}\n\n// ** backend interface\n\nstatic bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) {\n return b->buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment;\n}\n\nstatic inline bool ggml_backend_buffer_is_hexagon_repack(const struct ggml_backend_buffer * b) {\n if (!opt_hostbuf) {\n return ggml_backend_buffer_is_hexagon(b);\n }\n return b->buft->iface.alloc_buffer == ggml_backend_hexagon_repack_buffer_type_alloc_buffer;\n}\n\nstatic bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n const struct ggml_tensor * src2 = op->src[2];\n const struct ggml_tensor * src3 = op->src[3];\n const struct ggml_tensor * src4 = op->src[4];\n const struct ggml_tensor * dst = op;\n\n // Check for F16 support only as requested\n if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || src1->type != GGML_TYPE_F16 || src2->type != GGML_TYPE_F16) {\n return false;\n }\n\n if (src3 && src3->type != GGML_TYPE_F16) { // mask\n return false;\n }\n\n if (src4 && src4->type != GGML_TYPE_F32) { // sinks\n return false;\n }\n\n // For now we support F32 or F16 output as htp backend often converts output on the fly if needed,\n // but the op implementation writes to F16 or F32.\n // Let's assume dst can be F32 or F16.\n if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) {\n return false;\n }\n\n return opt_experimental;\n}\n\n\nstatic bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {\n const struct ggml_tensor * src0 = dst->src[0];\n const struct ggml_tensor * src1 = dst->src[1];\n\n if (dst->type != GGML_TYPE_F32) {\n return false;\n }\n\n if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) {\n return false;\n }\n\n switch (src0->type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_MXFP4:\n if (src0->ne[0] % 32) {\n return false;\n }\n\n if (ggml_nrows(src0) > 16 * 1024) {\n return false; // typically the lm-head which would be too large for VTCM\n }\n\n if (ggml_nrows(src1) > 1024 || src1->ne[2] != 1 || src1->ne[3] != 1) {\n return false; // no huge batches or broadcasting (for now)\n }\n\n // src0 (weights) must be repacked\n if (src0->buffer && !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) {\n return false;\n }\n break;\n\n case GGML_TYPE_F16:\n if (src0->nb[1] < src0->nb[0]) {\n GGML_LOG_DEBUG(\"ggml_hexagon_supported_mul_mat: permuted F16 src0 not supported\\n\");\n return false;\n }\n if (ggml_nrows(src1) > 1024) {\n return false; // no huge batches (for now)\n }\n break;\n\n default:\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n const struct ggml_tensor * src2 = op->src[2];\n const struct ggml_tensor * dst = op;\n\n if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) {\n return false;\n }\n\n switch (src0->type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_MXFP4:\n if ((src0->ne[0] % 32)) {\n return false;\n }\n\n // src0 (weights) must be repacked\n if (src0->buffer && !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) {\n return false;\n }\n break;\n\n default:\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n const struct ggml_tensor * dst = op;\n\n if (src0->type == GGML_TYPE_F32) {\n if (src1->type != GGML_TYPE_F32) {\n return false;\n }\n if (dst->type != GGML_TYPE_F32) {\n return false;\n }\n }\n else if (src0->type == GGML_TYPE_F16) {\n if (src1->type != GGML_TYPE_F16) {\n return false;\n }\n if (dst->type != GGML_TYPE_F16) {\n return false;\n }\n }\n else {\n return false;\n }\n\n if (!ggml_are_same_shape(src0, dst)) {\n return false;\n }\n if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) {\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n const struct ggml_tensor * dst = op;\n\n if (src0->type != GGML_TYPE_F32) {\n return false;\n }\n if (src1->type != GGML_TYPE_F32) {\n return false;\n }\n if (dst->type != GGML_TYPE_F32) {\n return false;\n }\n if (!ggml_are_same_shape(src0, dst)) {\n return false;\n }\n\n // REVISIT: add support for non-contigiuos tensors\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * dst = op;\n\n if (src0->type != GGML_TYPE_F32) {\n return false;\n }\n if (dst->type != GGML_TYPE_F32) {\n return false;\n }\n if (!ggml_are_same_shape(src0, dst)) {\n return false;\n }\n\n // TODO: add support for non-contigiuos tensors\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * dst = op;\n\n if (src0->type != GGML_TYPE_F32) {\n return false;\n }\n if (dst->type != GGML_TYPE_F32) {\n return false;\n }\n\n // TODO: add support for non-contigiuos tensors\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess,\n const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n const struct ggml_tensor * dst = op;\n\n if (src0->type != GGML_TYPE_F32) {\n return false;\n }\n if (dst->type != GGML_TYPE_F32) {\n return false;\n }\n\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {\n return false;\n }\n\n if (src1) {\n if (src1->type != GGML_TYPE_F32) {\n return false;\n }\n if (!ggml_are_same_shape(src0, src1)) {\n return false;\n }\n if (!ggml_is_contiguous(src1)) {\n return false;\n }\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n const struct ggml_tensor * src2 = op->src[2];\n const struct ggml_tensor * dst = op;\n\n if (src2) {\n return false; // FIXME: add support for sinks\n }\n\n if (src0->type != GGML_TYPE_F32) {\n return false;\n }\n if (dst->type != GGML_TYPE_F32) {\n return false;\n }\n\n if (src1) {\n if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) {\n return false;\n }\n if (src0->ne[0] != src1->ne[0]) {\n return false;\n }\n if (src1->ne[1] < src0->ne[1]) {\n return false;\n }\n if (src0->ne[2] % src1->ne[2] != 0) {\n return false;\n }\n if (src0->ne[3] % src1->ne[3] != 0) {\n return false;\n }\n }\n\n if (src1) {\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {\n return false;\n }\n } else {\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {\n return false;\n }\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0]; // values\n const struct ggml_tensor * src1 = op->src[1]; // indices\n const struct ggml_tensor * dst = op;\n\n if (src0->type != GGML_TYPE_F32) {\n return false;\n }\n\n if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {\n return false;\n }\n\n if (dst->type != GGML_TYPE_F16) {\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0]; // values\n const struct ggml_tensor * src1 = op->src[1]; // indices\n const struct ggml_tensor * dst = op;\n\n if (src0->type != GGML_TYPE_F32) {\n return false;\n }\n\n if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {\n return false;\n }\n\n if (dst->type != GGML_TYPE_F32) {\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0]; // values\n const struct ggml_tensor * dst = op; // indices\n\n if (src0->type != GGML_TYPE_F32) {\n return false;\n }\n\n if (dst->type != GGML_TYPE_I32) {\n return false;\n }\n\n if (src0->ne[0] > (16*1024)) {\n // reject tensors with huge rows for now\n return false;\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const int32_t * op_params = &op->op_params[0];\n\n int mode = op_params[2];\n\n if ((mode & GGML_ROPE_TYPE_MROPE) || (mode & GGML_ROPE_TYPE_VISION)) {\n return false;\n }\n if (mode & 1) {\n return false;\n }\n\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n const struct ggml_tensor * src2 = op->src[2];\n const struct ggml_tensor * dst = op;\n\n if (src0->type != GGML_TYPE_F32) {\n return false; // FIXME: add support for GGML_TYPE_F16 for src0\n }\n if (dst->type != GGML_TYPE_F32) {\n return false;\n }\n if (src1->type != GGML_TYPE_I32) {\n return false;\n }\n if (src2) {\n if (src2->type != GGML_TYPE_F32) {\n return false;\n }\n int n_dims = op_params[1];\n if (src2->ne[0] < (n_dims / 2)) {\n return false;\n }\n }\n\n if (src2) {\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(src2) ||\n !ggml_is_contiguous(dst)) {\n return false;\n }\n } else {\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {\n return false;\n }\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * src1 = op->src[1];\n const struct ggml_tensor * dst = op;\n\n // Only support FP32 for now\n if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {\n return false;\n }\n\n // Check IO tensor shapes and dims\n if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) {\n return false; // src0 should be effectively 3D\n }\n\n const int d_conv = src1->ne[0];\n const int d_inner = src0->ne[1];\n const int n_t = dst->ne[1];\n const int n_s = dst->ne[2];\n\n if (src0->ne[0] != d_conv - 1 + n_t || src0->ne[1] != d_inner || src0->ne[2] != n_s) {\n return false;\n }\n if (src1->ne[0] != d_conv || src1->ne[1] != d_inner) {\n return false;\n }\n if (dst->ne[0] != d_inner || dst->ne[1] != n_t || dst->ne[2] != n_s) {\n return false;\n }\n\n // TODO: add support for non-contiguous tensors\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {\n return false;\n }\n\n return true;\n}\n\nenum dspqbuf_type {\n DSPQBUF_TYPE_DSP_WRITE_CPU_READ = 0,\n DSPQBUF_TYPE_CPU_WRITE_DSP_READ,\n DSPQBUF_TYPE_CONSTANT,\n};\n\nstatic void dspqbuf_dump(dspqueue_buffer * d, const struct ggml_tensor * t, dspqbuf_type type) {\n if (opt_verbose < 2) return;\n\n auto buf = static_cast(t->buffer->context);\n auto sess = buf->sess;\n\n GGML_LOG_DEBUG(\"ggml-hex: %s dspqbuf : %s base-addr %p base-size %zu data %p offset %u size %u\\n\", sess->name.c_str(),\n t->name, (void *) buf->base, buf->size, (void *) d->ptr, (unsigned int) d->offset,\n (unsigned int) d->size);\n}\n\n// Init hexagon tensor from GGML tensor and Hexagon buffer\nstatic void htp_req_tensor_init(htp_tensor * h, const ggml_tensor * t) {\n h->data = 0; // updated by the receiver\n h->type = t->type;\n h->ne[0] = t->ne[0];\n h->ne[1] = t->ne[1];\n h->ne[2] = t->ne[2];\n h->ne[3] = t->ne[3];\n h->nb[0] = t->nb[0];\n h->nb[1] = t->nb[1];\n h->nb[2] = t->nb[2];\n h->nb[3] = t->nb[3];\n}\n\nstatic size_t htp_req_buff_init(htp_tensor *h, dspqueue_buffer * d, const ggml_tensor * t, dspqbuf_type type) {\n if (!t) {\n return 0;\n }\n\n auto buf = static_cast(t->buffer->context);\n\n memset(d, 0, sizeof(*d));\n d->fd = buf->fd;\n d->ptr = t->data;\n d->offset = (uint8_t *) t->data - buf->base;\n d->size = ggml_nbytes(t);\n\n if (!d->size) {\n // Some requests contain srcs where ggml_nbytes() returns 0 but the rest of the op is non-empty\n d->size = 64;\n }\n\n switch (type) {\n case DSPQBUF_TYPE_DSP_WRITE_CPU_READ:\n // Flush CPU\n d->flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER;\n break;\n case DSPQBUF_TYPE_CPU_WRITE_DSP_READ:\n // Flush CPU, Invalidate DSP\n d->flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;\n break;\n default:\n // Constant buffer, no cache maintenance\n d->flags = 0;\n break;\n }\n\n htp_req_tensor_init(h, t);\n\n dspqbuf_dump(d, t, type);\n\n return 1;\n}\n\ntypedef size_t (*htp_req_init_func_t)(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * op);\n\ntemplate \nstatic inline void ggml_hexagon_dispatch_op(ggml_hexagon_session *sess, const struct ggml_tensor * op, uint32_t flags) {\n uint64_t t = ggml_time_us();\n\n // Construct HTP request\n htp_general_req req;\n memset(&req, 0, sizeof(req));\n\n req.flags = flags;\n if (!(opt_opmask & HTP_OPMASK_QUANTIZE)) {\n req.flags |= HTP_OPFLAGS_SKIP_QUANTIZE;\n }\n if (!(opt_opmask & HTP_OPMASK_COMPUTE)) {\n req.flags |= HTP_OPFLAGS_SKIP_COMPUTE;\n }\n\n ggml_hexagon_dump_op_exec(sess->name, op, req.flags);\n\n if ((opt_opmask & HTP_OPMASK_QUEUE)) {\n dspqueue_buffer bufs[HTP_MAX_PACKET_BUFFERS];\n size_t n_bufs = _init_req_func(&req, bufs, op);\n sess->enqueue(req, bufs, n_bufs, opt_opsync);\n }\n\n t = ggml_time_us() - t;\n\n ggml_hexagon_dump_op_prof(sess->name, op, sess->prof_usecs, sess->prof_cycles, sess->prof_pkts, t);\n}\n\ntemplate \nstatic inline size_t init_binary_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n switch (t->op) {\n case GGML_OP_MUL_MAT:\n req->op = HTP_OP_MUL_MAT;\n break;\n case GGML_OP_MUL:\n req->op = HTP_OP_MUL;\n break;\n case GGML_OP_ADD:\n req->op = HTP_OP_ADD;\n break;\n case GGML_OP_SUB:\n req->op = HTP_OP_SUB;\n break;\n case GGML_OP_DIV:\n req->op = HTP_OP_DIV;\n break;\n default:\n GGML_ABORT(\"ggml-hex: binary : unsupported op: %d\\n\", t->op);\n break;\n }\n\n // src0: Weights (mulmat) or First Operand (binary op).\n // If constant (e.g. weights), no cache management is needed.\n // src1: Input Activations (mulmat) or Second Operand (binary op).\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], _is_src0_constant ? DSPQBUF_TYPE_CONSTANT : DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src1, &bufs[n_bufs], t->src[1], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic inline size_t init_cpy_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n req->op = HTP_OP_CPY;\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic inline size_t init_get_rows_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n req->op = HTP_OP_GET_ROWS;\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src1, &bufs[n_bufs], t->src[1], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic inline size_t init_argsort_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n req->op = HTP_OP_ARGSORT;\n memcpy(&req->op_params, &t->op_params, sizeof(t->op_params));\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\ntemplate \nstatic inline size_t init_binary_id_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n switch (t->op) {\n case GGML_OP_MUL_MAT_ID:\n req->op = HTP_OP_MUL_MAT_ID;\n break;\n case GGML_OP_ADD_ID:\n req->op = HTP_OP_ADD_ID;\n break;\n default:\n GGML_ABORT(\"ggml-hex: unsupported op: %d\\n\", t->op);\n }\n\n // src0: Weights (mulmat) or Input Activations (other op).\n // If constant, no cache management is needed.\n // src1: Input Activations (mulmat) or Second Operand (binary op).\n // src2: Expert IDs (mulmat) or Activated Experts (other op).\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], _is_src0_constant ? DSPQBUF_TYPE_CONSTANT : DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src1, &bufs[n_bufs], t->src[1], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src2, &bufs[n_bufs], t->src[2], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic inline size_t init_set_rows_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n req->op = HTP_OP_SET_ROWS;\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src1, &bufs[n_bufs], t->src[1], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic inline size_t init_unary_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n memcpy(&req->op_params, &t->op_params, sizeof(t->op_params));\n\n bool supported = false;\n\n switch (t->op) {\n case GGML_OP_RMS_NORM:\n req->op = HTP_OP_RMS_NORM;\n supported = true;\n break;\n\n case GGML_OP_SCALE:\n req->op = HTP_OP_SCALE;\n supported = true;\n break;\n\n case GGML_OP_SQR:\n req->op = HTP_OP_SQR;\n supported = true;\n break;\n\n case GGML_OP_SQRT:\n req->op = HTP_OP_SQRT;\n supported = true;\n break;\n\n case GGML_OP_UNARY:\n if (ggml_get_unary_op(t) == GGML_UNARY_OP_SILU) {\n req->op = HTP_OP_UNARY_SILU;\n supported = true;\n } else if (ggml_get_unary_op(t) == GGML_UNARY_OP_GELU) {\n req->op = HTP_OP_UNARY_GELU;\n supported = true;\n }\n break;\n\n case GGML_OP_GLU:\n if (ggml_get_glu_op(t) == GGML_GLU_OP_SWIGLU) {\n req->op = HTP_OP_GLU_SWIGLU;\n supported = true;\n } else if (ggml_get_glu_op(t) == GGML_GLU_OP_SWIGLU_OAI) {\n req->op = HTP_OP_GLU_SWIGLU_OAI;\n supported = true;\n } else if (ggml_get_glu_op(t) == GGML_GLU_OP_GEGLU) {\n req->op = HTP_OP_GLU_GEGLU;\n supported = true;\n }\n break;\n\n case GGML_OP_SOFT_MAX:\n req->op = HTP_OP_SOFTMAX;\n supported = true;\n break;\n\n default:\n break;\n }\n\n if (!supported) {\n GGML_ABORT(\"ggml-hex: unary : unsupported op: %d\\n\", t->op);\n }\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src1, &bufs[n_bufs], t->src[1], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic inline size_t init_sum_rows_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n memcpy(&req->op_params, &t->op_params, sizeof(t->op_params));\n req->op = HTP_OP_SUM_ROWS;\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic inline size_t init_rope_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n memcpy(&req->op_params, &t->op_params, sizeof(t->op_params));\n req->op = HTP_OP_ROPE;\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src1, &bufs[n_bufs], t->src[1], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src2, &bufs[n_bufs], t->src[2], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic inline size_t init_flash_attn_ext_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n memcpy(&req->op_params, &t->op_params, sizeof(t->op_params));\n req->op = HTP_OP_FLASH_ATTN_EXT;\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src1, &bufs[n_bufs], t->src[1], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src2, &bufs[n_bufs], t->src[2], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src3, &bufs[n_bufs], t->src[3], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src4, &bufs[n_bufs], t->src[4], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic inline size_t init_ssm_conv_req(htp_general_req * req, dspqueue_buffer * bufs, const ggml_tensor * t) {\n req->op = HTP_OP_SSM_CONV;\n\n size_t n_bufs = 0;\n n_bufs += htp_req_buff_init(&req->src0, &bufs[n_bufs], t->src[0], DSPQBUF_TYPE_CPU_WRITE_DSP_READ);\n n_bufs += htp_req_buff_init(&req->src1, &bufs[n_bufs], t->src[1], DSPQBUF_TYPE_CONSTANT);\n n_bufs += htp_req_buff_init(&req->dst, &bufs[n_bufs], t, DSPQBUF_TYPE_DSP_WRITE_CPU_READ);\n\n return n_bufs;\n}\n\nstatic const char * ggml_backend_hexagon_name(ggml_backend_t backend) {\n auto sess = static_cast(backend->context);\n return sess->name.c_str();\n}\n\nstatic void ggml_backend_hexagon_free(ggml_backend_t backend) {\n // we just need to delete the backend here\n // the sessions are allocated & freed as part of the registry\n delete backend;\n}\n\nstatic inline bool op_reuse_src1(const ggml_tensor * op1, const ggml_tensor * op0) {\n return (op0 && op0->src[1] == op1->src[1] && ggml_is_quantized(op0->src[0]->type));\n}\n\nstatic inline bool is_compute_op(ggml_tensor *node)\n{\n return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE);\n}\n\n// scan the graph and figure out last compute op index\nstatic inline int last_compute_op(ggml_cgraph * graph) {\n int last = 0;\n for (int i = 0; i < graph->n_nodes; ++i) {\n if (is_compute_op(graph->nodes[i])) {\n last = i;\n }\n }\n\n return last;\n}\n\nstatic ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) {\n auto sess = static_cast(backend->context);\n\n HEX_VERBOSE(\"ggml-hex: %s graph-compute n_nodes %d\\n\", sess->name.c_str(), graph->n_nodes);\n\n const int last = last_compute_op(graph);\n\n const struct ggml_tensor * prev_op = nullptr; // prev executed op\n\n for (int i = 0; i < graph->n_nodes; ++i) {\n ggml_tensor * node = graph->nodes[i];\n\n if (!is_compute_op(node)) {\n continue;\n }\n\n uint32_t flags = 0;\n\n // skip quantizer if src1 is reused\n if (op_reuse_src1(node, prev_op)) {\n flags |= HTP_OPFLAGS_SKIP_QUANTIZE;\n }\n\n prev_op = node;\n\n // ask for early notification for the last Op\n if (i == last) {\n flags |= HTP_OPFLAGS_EARLY_WAKEUP;\n }\n\n switch (node->op) {\n case GGML_OP_MUL_MAT:\n if (ggml_is_quantized(node->src[0]->type)) {\n ggml_hexagon_dispatch_op>(sess, node, flags);\n } else {\n ggml_hexagon_dispatch_op>(sess, node, flags);\n }\n break;\n case GGML_OP_MUL_MAT_ID:\n if (ggml_is_quantized(node->src[0]->type)) {\n ggml_hexagon_dispatch_op>(sess, node, flags);\n } else {\n ggml_hexagon_dispatch_op>(sess, node, flags);\n }\n break;\n case GGML_OP_MUL:\n case GGML_OP_ADD:\n case GGML_OP_SUB:\n case GGML_OP_DIV:\n ggml_hexagon_dispatch_op>(sess, node, flags);\n break;\n case GGML_OP_ADD_ID:\n ggml_hexagon_dispatch_op>(sess, node, flags);\n break;\n case GGML_OP_RMS_NORM:\n case GGML_OP_SCALE:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n case GGML_OP_SQR:\n case GGML_OP_SQRT:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n case GGML_OP_SUM_ROWS:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n case GGML_OP_UNARY:\n if ((ggml_get_unary_op(node) == GGML_UNARY_OP_SILU) ||\n (ggml_get_unary_op(node) == GGML_UNARY_OP_GELU)) {\n ggml_hexagon_dispatch_op(sess, node, flags);\n }\n break;\n case GGML_OP_GLU:\n if ((ggml_get_glu_op(node) == GGML_GLU_OP_SWIGLU) ||\n (ggml_get_glu_op(node) == GGML_GLU_OP_SWIGLU_OAI) ||\n (ggml_get_glu_op(node) == GGML_GLU_OP_GEGLU)) {\n ggml_hexagon_dispatch_op(sess, node, flags);\n }\n break;\n case GGML_OP_SOFT_MAX:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n\n case GGML_OP_ROPE:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n\n case GGML_OP_FLASH_ATTN_EXT:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n\n case GGML_OP_SET_ROWS:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n\n case GGML_OP_GET_ROWS:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n\n case GGML_OP_CPY:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n\n case GGML_OP_ARGSORT:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n\n case GGML_OP_SSM_CONV:\n ggml_hexagon_dispatch_op(sess, node, flags);\n break;\n\n default:\n GGML_ABORT(\"\\nggml-hex: graph-compute %s is not supported\\n\", ggml_op_desc(node));\n }\n }\n\n // Wait until all pending ops complete\n sess->flush();\n\n return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_hexagon_synchronize(ggml_backend_t backend) {\n auto sess = static_cast(backend->context);\n\n HEX_VERBOSE(\"ggml-hex: %s synchronize\\n\", sess->name.c_str());\n\n // Wait until all pending ops complete\n sess->flush();\n}\n\nstruct node_info {\n ggml_tensor * node;\n\n std::vector fused;\n\n ggml_op op() const {\n return node->op;\n }\n\n const ggml_tensor * dst() const {\n return fused.empty() ? node : fused.back();\n }\n\n const ggml_tensor * src0() const {\n return node->src[0];\n }\n\n const ggml_tensor * src1() const {\n return node->src[1];\n }\n\n bool is_empty() const {\n return ggml_op_is_empty(node->op);\n }\n\n void add_fused(ggml_tensor * t) {\n fused.push_back(t);\n }\n\n bool stackable() const {\n switch (this->op()) {\n case GGML_OP_MUL_MAT:\n case GGML_OP_MUL_MAT_ID:\n return ggml_is_quantized(this->src0()->type);\n default:\n return false;\n }\n }\n\n bool same_input(const node_info& n) const {\n return n.src1() == this->src1();\n }\n};\n\nstatic std::vector ggml_hexagon_graph_optimize_reorder(const std::vector & nodes) {\n const int n = nodes.size();\n\n std::vector res;\n res.reserve(n);\n\n std::vector used(n, false);\n\n // The main goal here is to stack the MUL_MAT ops with the same src1 input.\n // This allows use to reuse dynamically quantized src1 in VTCM.\n\n // TODO: the current version might do incorrect reordering in cases where quantized src0\n // input is an output of another Op.\n\n for (int i0 = 0; i0 < n; i0++) {\n if (used[i0]) {\n continue;\n }\n\n res.push_back(i0);\n\n const auto & node0 = nodes[i0];\n\n if (!node0.stackable()) {\n continue;\n }\n\n // that many nodes forward to search for stackable nodes that can reuse VTCM\n constexpr int N_FORWARD = 16;\n\n for (int i1 = i0 + 1; i1 < i0 + N_FORWARD && i1 < n; i1++) {\n if (used[i1]) {\n continue;\n }\n\n const auto & node1 = nodes[i1];\n\n if (node1.stackable() && node1.same_input(node0)) {\n res.push_back(i1);\n used[i1] = true;\n }\n }\n }\n\n return res;\n}\n\nstatic void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgraph * gf) {\n const int n = gf->n_nodes;\n\n constexpr int MAX_FUSE = 16;\n\n enum ggml_op ops[MAX_FUSE];\n\n std::vector nodes;\n nodes.reserve(gf->n_nodes);\n\n // fuse nodes:\n // we don't want to make reorders that break fusing, so we first pack all fusable tensors\n // and perform the reorder over the fused nodes. after the reorder is done, we unfuse\n for (int i = 0; i < n; i++) {\n node_info node = {\n /*.node =*/gf->nodes[i],\n /*.fused =*/{},\n };\n\n // fuse only ops that start with these operations\n // can be expanded when needed\n if (node.op() == GGML_OP_ADD ||\n node.op() == GGML_OP_NORM ||\n node.op() == GGML_OP_RMS_NORM) {\n ops[0] = node.op();\n\n int f = i + 1;\n while (f < n && f < i + MAX_FUSE) {\n // conservatively allow fusing only these ops\n // can be expanded when needed\n if (gf->nodes[f]->op != GGML_OP_ADD &&\n gf->nodes[f]->op != GGML_OP_MUL &&\n gf->nodes[f]->op != GGML_OP_NORM &&\n gf->nodes[f]->op != GGML_OP_RMS_NORM) {\n break;\n }\n ops[f - i] = gf->nodes[f]->op;\n f++;\n }\n\n f -= i;\n for (; f > 1; f--) {\n if (ggml_can_fuse(gf, i, ops, f)) {\n break;\n }\n }\n\n // add the fused tensors into the node info so we can unfuse them later\n for (int k = 1; k < f; k++) {\n ++i;\n\n // the .dst() becomes the last fused tensor\n node.add_fused(gf->nodes[i]);\n }\n }\n\n nodes.push_back(std::move(node));\n }\n\n const auto order = ggml_hexagon_graph_optimize_reorder(nodes);\n\n // unfuse\n {\n int j = 0;\n for (const auto i : order) {\n const auto & node = nodes[i];\n\n gf->nodes[j++] = node.node;\n\n for (auto * fused : node.fused) {\n gf->nodes[j++] = fused;\n }\n }\n }\n}\n\nstatic struct ggml_backend_i hexagon_backend_i = {\n /* .get_name = */ ggml_backend_hexagon_name,\n /* .free = */ ggml_backend_hexagon_free,\n /* .set_tensor_async = */ NULL,\n /* .get_tensor_async = */ NULL,\n /* .cpy_tensor_async = */ NULL,\n /* .synchronize = */ ggml_backend_hexagon_synchronize,\n /* .graph_plan_create = */ NULL,\n /* .graph_plan_free = */ NULL,\n /* .graph_plan_update = */ NULL,\n /* .graph_plan_compute = */ NULL,\n /* .graph_compute = */ ggml_backend_hexagon_graph_compute,\n /* .event_record = */ NULL,\n /* .event_wait = */ NULL,\n /* .graph_optimize = */ ggml_backend_hexagon_graph_optimize,\n};\n\nstatic ggml_guid_t ggml_backend_hexagon_guid() {\n static ggml_guid guid = { 0x7b, 0x57, 0xdc, 0xaf, 0xde, 0x12, 0x1d, 0x49,\n 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 };\n return &guid;\n}\n\nbool ggml_backend_is_hexagon(ggml_backend_t backend) {\n return backend && backend->iface.get_name == ggml_backend_hexagon_name;\n}\n\n// device interface\n\nstatic ggml_backend_t ggml_backend_hexagon_device_init(ggml_backend_dev_t dev, const char * params) {\n auto sess = static_cast(dev->context);\n\n return new ggml_backend{\n /* .guid = */ ggml_backend_hexagon_guid(),\n /* .interface = */ hexagon_backend_i,\n /* .device = */ dev,\n /* .context = */ sess,\n };\n\n GGML_UNUSED(params);\n}\n\nstatic const char * ggml_backend_hexagon_device_get_name(ggml_backend_dev_t dev) {\n auto sess = static_cast(dev->context);\n return sess->name.c_str();\n\n GGML_UNUSED(dev);\n}\n\nstatic const char * ggml_backend_hexagon_device_get_description(ggml_backend_dev_t dev) {\n return \"Hexagon\";\n GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_hexagon_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n // ~2GB per session for now\n *free = 2ULL * 1024 * 1024 * 1024;\n *total = *free;\n\n GGML_UNUSED(dev);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_hexagon_device_get_type(ggml_backend_dev_t dev) {\n return GGML_BACKEND_DEVICE_TYPE_GPU;\n\n GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_hexagon_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {\n props->name = ggml_backend_hexagon_device_get_name(dev);\n props->description = ggml_backend_hexagon_device_get_description(dev);\n props->type = ggml_backend_hexagon_device_get_type(dev);\n ggml_backend_hexagon_device_get_memory(dev, &props->memory_free, &props->memory_total);\n props->caps = {\n /* .async = */ true,\n /* .host_buffer = */ (bool) opt_hostbuf,\n /* .buffer_from_host_ptr = */ false,\n /* .events = */ false,\n };\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_buffer_type(ggml_backend_dev_t dev) {\n auto sess = static_cast(dev->context);\n return &sess->buffer_type;\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_repack_buffer_type(ggml_backend_dev_t dev) {\n auto sess = static_cast(dev->context);\n return &sess->repack_buffer_type;\n}\n\nstatic bool ggml_hexagon_supported_buffer(ggml_hexagon_session *sess, const struct ggml_tensor * t) {\n if (t && t->buffer) {\n if (ggml_backend_buffer_is_hexagon(t->buffer) == false) return false; // not our buffer\n if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != sess) return false; // wrong session\n }\n return true;\n}\n\nstatic bool ggml_hexagon_supported_buffers(ggml_hexagon_session *sess, const struct ggml_tensor * t) {\n // all srcs & dsts must be mapped to the same session\n if (!ggml_hexagon_supported_buffer(sess, t)) {\n return false;\n }\n\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (!ggml_hexagon_supported_buffer(sess, t->src[i])) {\n return false;\n }\n }\n\n return true;\n}\n\nstatic bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {\n const struct ggml_tensor * src0 = op->src[0];\n const struct ggml_tensor * dst = op;\n\n // for now we can do f32 -> f16 and f16 -> f32 (without reshaping)\n if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) return false;\n if ( dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) return false;\n\n const bool sametype = (src0->type == dst->type);\n const bool transposed = ggml_is_transposed(src0) || ggml_is_transposed(dst);\n const bool sameshape = !transposed && ggml_are_same_shape(src0, dst);\n\n // can handle any shape and any same-type (pretty slow if reshaping is required)\n if (sametype) return true;\n\n // cannot handle re-shaping and type conversion at the same time\n if (!sameshape) return false;\n\n return true;\n}\n\nstatic bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {\n auto sess = static_cast(dev->context);\n\n // all srcs & dsts must be mapped to the same session\n if (!ggml_hexagon_supported_buffers(sess, op)) {\n ggml_hexagon_dump_op_supp(sess->name, op, false);\n return false;\n }\n\n bool supp = false;\n switch (op->op) {\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n supp = true;\n break;\n\n case GGML_OP_MUL_MAT:\n supp = ggml_hexagon_supported_mul_mat(sess, op);\n break;\n\n case GGML_OP_MUL_MAT_ID:\n supp = ggml_hexagon_supported_mul_mat_id(sess, op);\n break;\n\n case GGML_OP_MUL:\n case GGML_OP_ADD:\n case GGML_OP_SUB:\n case GGML_OP_DIV:\n supp = ggml_hexagon_supported_binary(sess, op);\n break;\n\n case GGML_OP_ADD_ID:\n supp = ggml_hexagon_supported_add_id(sess, op);\n break;\n\n case GGML_OP_RMS_NORM:\n case GGML_OP_SCALE:\n supp = ggml_hexagon_supported_unary(sess, op);\n break;\n\n case GGML_OP_SQR:\n case GGML_OP_SQRT:\n supp = ggml_hexagon_supported_unary(sess, op);\n break;\n\n case GGML_OP_SUM_ROWS:\n supp = ggml_hexagon_supported_sum_rows(sess, op);\n break;\n\n case GGML_OP_SOFT_MAX:\n supp = ggml_hexagon_supported_softmax(sess, op);\n break;\n\n case GGML_OP_UNARY:\n {\n const auto unary_op = ggml_get_unary_op(op);\n if (unary_op == GGML_UNARY_OP_SILU || unary_op == GGML_UNARY_OP_GELU) {\n supp = ggml_hexagon_supported_activations(sess, op);\n }\n break;\n }\n case GGML_OP_GLU:\n {\n const auto glu_op = ggml_get_glu_op(op);\n if ((glu_op == GGML_GLU_OP_SWIGLU) || (glu_op == GGML_GLU_OP_SWIGLU_OAI) || (glu_op == GGML_GLU_OP_GEGLU)) {\n supp = ggml_hexagon_supported_activations(sess, op);\n }\n break;\n }\n case GGML_OP_ROPE:\n supp = ggml_hexagon_supported_rope(sess, op);\n break;\n\n case GGML_OP_FLASH_ATTN_EXT:\n supp = ggml_hexagon_supported_flash_attn_ext(sess, op);\n break;\n\n case GGML_OP_SET_ROWS:\n supp = ggml_hexagon_supported_set_rows(sess, op);\n break;\n\n case GGML_OP_GET_ROWS:\n supp = ggml_hexagon_supported_get_rows(sess, op);\n break;\n\n case GGML_OP_CPY:\n supp = ggml_hexagon_supported_cpy(sess, op);\n break;\n\n case GGML_OP_ARGSORT:\n supp = ggml_hexagon_supported_argsort(sess, op);\n break;\n\n case GGML_OP_SSM_CONV:\n supp = ggml_hexagon_supported_ssm_conv(sess, op);\n break;\n\n default:\n break;\n }\n\n ggml_hexagon_dump_op_supp(sess->name, op, supp);\n return supp;\n}\n\nstatic bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n if (buft->iface.get_alignment != ggml_backend_hexagon_buffer_type_get_alignment) {\n return false;\n }\n\n auto s0 = static_cast(dev->context);\n auto s1 = static_cast(buft->context)->sess;\n\n // Need session/domain-id for buffers to be compatible\n bool supp = (s0->session_id == s1->session_id);\n\n HEX_VERBOSE(\"ggml-hex: %s device-supports-buft %s (%d)\\n\", s0->name.c_str(), s1->name.c_str(), (int) supp);\n\n return supp;\n}\n\nstatic ggml_backend_buffer_type_t * ggml_backend_hexagon_device_get_extra_buffers_type(ggml_backend_dev_t dev) {\n auto s0 = static_cast(dev->context);\n HEX_VERBOSE(\"ggml-hex: device-get-extra-buft : %s \\n\", s0->name.c_str());\n\n static ggml_backend_buffer_type_t bufts[2];\n bufts[0] = ggml_backend_hexagon_device_get_repack_buffer_type(dev);\n bufts[1] = NULL;\n return bufts;\n}\n\nstatic const struct ggml_backend_device_i ggml_backend_hexagon_device_i = {\n /* .get_name = */ ggml_backend_hexagon_device_get_name,\n /* .get_description = */ ggml_backend_hexagon_device_get_description,\n /* .get_memory = */ ggml_backend_hexagon_device_get_memory,\n /* .get_type = */ ggml_backend_hexagon_device_get_type,\n /* .get_props = */ ggml_backend_hexagon_device_get_props,\n /* .init_backend = */ ggml_backend_hexagon_device_init,\n /* .get_buffer_type = */ ggml_backend_hexagon_device_get_buffer_type,\n /* .get_host_buffer_type = */ NULL, // ggml_backend_hexagon_device_get_host_buffer_type,\n /* .buffer_from_host_ptr = */ NULL, // ggml_backend_hexagon_device_buffer_from_ptr,\n /* .supports_op = */ ggml_backend_hexagon_device_supports_op,\n /* .supports_buft = */ ggml_backend_hexagon_device_supports_buft,\n /* .offload_op = */ NULL, // ggml_backend_hexagon_device_offload_op,\n /* .event_new = */ NULL,\n /* .event_free = */ NULL,\n /* .event_synchronize = */ NULL,\n};\n\n//** backend registry\n\n#define GGML_HEXAGON_MAX_SESSIONS 16\n\nstruct ggml_hexagon_registry {\n ggml_hexagon_registry(ggml_backend_reg_t reg);\n ~ggml_hexagon_registry();\n\n ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS];\n};\n\nggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {\n GGML_LOG_INFO(\"ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev %zu\\n\", opt_ndev);\n\n if (!opt_arch) {\n int err = get_hex_arch_ver(CDSP_DOMAIN_ID, &opt_arch);\n if (err != 0) {\n GGML_LOG_ERROR(\"ggml-hex: failed to query HTP version (err %d) defaulting to v73\\n\", err);\n opt_arch = 73;\n }\n }\n\n#if defined(__ANDROID__)\n if (opt_arch < 75) {\n opt_ndev = 1;\n GGML_LOG_WARN(\"ggml-hex: forcing ndev to 1 for SoCs archs lower than v75.\\n\");\n }\n#endif\n\n GGML_LOG_INFO(\"ggml-hex: Hexagon Arch version v%d\\n\", opt_arch);\n\n // Create devices / sessions\n for (size_t i = 0; i < opt_ndev; i++) {\n devices[i].iface = ggml_backend_hexagon_device_i;\n devices[i].reg = reg;\n try {\n devices[i].context = new ggml_hexagon_session(i, &devices[i]);\n } catch (const std::exception & exc) {\n GGML_LOG_ERROR(\"ggml-hex: failed to create device/session %zu\\n\", i);\n devices[i].context = nullptr;\n }\n }\n}\n\nggml_hexagon_registry::~ggml_hexagon_registry() {\n GGML_LOG_INFO(\"ggml-hex: releasing registry\\n\");\n\n // Release devices / sessions\n for (size_t i = 0; i < opt_ndev; i++) {\n auto sess = static_cast(devices[i].context);\n delete sess;\n }\n}\n\nstatic const char * ggml_backend_hexagon_reg_get_name(ggml_backend_reg_t reg) {\n return \"HTP\";\n GGML_UNUSED(reg);\n}\n\nstatic size_t ggml_backend_hexagon_reg_get_device_count(ggml_backend_reg_t reg) {\n return opt_ndev;\n GGML_UNUSED(reg);\n}\n\nstatic ggml_backend_dev_t ggml_backend_hexagon_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n auto hreg = static_cast(reg->context);\n\n if (index >= opt_ndev || !hreg->devices[index].context) {\n return nullptr;\n }\n\n return &hreg->devices[index];\n}\n\nstatic void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n if (strcmp(name, \"ggml_backend_dev_get_extra_bufts\") == 0 && opt_hostbuf) {\n ggml_backend_dev_get_extra_bufts_t fct = ggml_backend_hexagon_device_get_extra_buffers_type;\n return (void *) fct;\n }\n\n return NULL;\n}\n\nstatic void ggml_hexagon_init(ggml_backend_reg * reg) {\n // Basic sanity checks to make sure definitions match\n static_assert((unsigned int) HTP_TYPE_Q4_0 == (unsigned int) GGML_TYPE_Q4_0,\n \"please update hexagon_type to match ggml_type\");\n static_assert((unsigned int) HTP_TYPE_Q8_0 == (unsigned int) GGML_TYPE_Q8_0,\n \"please update hexagon_type to match ggml_type\");\n static_assert((unsigned int) HTP_TYPE_MXFP4 == (unsigned int) GGML_TYPE_MXFP4,\n \"please update hexagon_type to match ggml_type\");\n\n const char * str_experimental = getenv(\"GGML_HEXAGON_EXPERIMENTAL\");\n const char * str_verbose = getenv(\"GGML_HEXAGON_VERBOSE\");\n const char * str_hostbuf = getenv(\"GGML_HEXAGON_HOSTBUF\");\n const char * str_opmask = getenv(\"GGML_HEXAGON_OPMASK\");\n const char * str_opsync = getenv(\"GGML_HEXAGON_OPSYNC\");\n const char * str_profile = getenv(\"GGML_HEXAGON_PROFILE\");\n const char * str_etm = getenv(\"GGML_HEXAGON_ETM\");\n const char * str_nhvx = getenv(\"GGML_HEXAGON_NHVX\");\n const char * str_ndev = getenv(\"GGML_HEXAGON_NDEV\");\n const char * str_arch = getenv(\"GGML_HEXAGON_ARCH\");\n\n opt_experimental = str_experimental ? atoi(str_experimental) : 0;\n opt_verbose = str_verbose ? atoi(str_verbose) : 0;\n opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) : opt_hostbuf;\n opt_opmask = str_opmask ? strtoul(str_opmask, NULL, 0) : opt_opmask;\n opt_opsync = str_opsync ? atoi(str_opsync) : 0;\n opt_profile = str_profile ? atoi(str_profile) : 0;\n opt_etm = str_etm ? atoi(str_etm) : 0;\n opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx;\n opt_ndev = str_ndev ? strtoul(str_ndev, NULL, 0) : opt_ndev;\n\n if (opt_ndev > GGML_HEXAGON_MAX_SESSIONS) {\n opt_ndev = GGML_HEXAGON_MAX_SESSIONS;\n }\n\n if (str_arch) {\n if (str_arch[0] == 'v') {\n str_arch++;\n }\n opt_arch = strtoul(str_arch, NULL, 0);\n }\n\n opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) : 1;\n\n reg->context = new ggml_hexagon_registry(reg);\n\n HEX_VERBOSE(\"ggml-hex: size-of-general-req %zu size-of-general-rsp %zu\\n\", sizeof(struct htp_general_req),\n sizeof(struct htp_general_rsp));\n}\n\nstatic const struct ggml_backend_reg_i ggml_backend_hexagon_reg_i = {\n /* .get_name = */ ggml_backend_hexagon_reg_get_name,\n /* .get_device_count = */ ggml_backend_hexagon_reg_get_device_count,\n /* .get_device = */ ggml_backend_hexagon_reg_get_device,\n /* .get_proc_address = */ ggml_backend_hexagon_get_proc_address,\n};\n\nggml_backend_reg_t ggml_backend_hexagon_reg(void) {\n static bool initialized = false;\n\n static ggml_backend_reg reg = { /* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_hexagon_reg_i,\n /* .context = */ NULL };\n\n {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n if (!initialized) {\n auto nErr = htpdrv_init();\n if (nErr != AEE_SUCCESS) {\n return NULL;\n }\n\n ggml_hexagon_init(®);\n }\n\n initialized = true;\n }\n\n return ®\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_hexagon_reg)\n" }, { "path": "src/ggml-hexagon/htp/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.22.2)\nproject(ggml-htp C CXX ASM)\n\ninclude(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake)\n\ninclude_directories(\n ${HEXAGON_SDK_ROOT}/incs\n ${HEXAGON_SDK_ROOT}/incs/stddef\n ${CMAKE_CURRENT_SOURCE_DIR}/../../../include\n ${CMAKE_CURRENT_SOURCE_DIR}/../..\n ${CMAKE_CURRENT_SOURCE_DIR}/..\n ${CMAKE_CURRENT_SOURCE_DIR}\n ${CMAKE_CURRENT_BINARY_DIR})\n\nset(HTP_LIB ggml-htp-${DSP_VERSION})\n\nadd_library(${HTP_LIB} SHARED\n main.c\n htp_iface_skel.c\n worker-pool.c\n hex-dma.c\n matmul-ops.c\n binary-ops.c\n unary-ops.c\n sum-rows-ops.c\n softmax-ops.c\n act-ops.c\n rope-ops.c\n flash-attn-ops.c\n set-rows-ops.c\n get-rows-ops.c\n cpy-ops.c\n argsort-ops.c\n ssm-conv.c\n)\n\ntarget_compile_definitions(${HTP_LIB} PRIVATE\n $,HTP_DEBUG=1,NDEBUG=1>\n $,FARF_HIGH=1,>\n FP32_QUANTIZE_GROUP_SIZE=${GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE})\n\nbuild_idl(htp_iface.idl ${HTP_LIB})\n\nset_target_properties(${HTP_LIB} PROPERTIES EXPORT_COMPILE_COMMANDS ON)\n\ninstall(TARGETS ${HTP_LIB})\n" }, { "path": "src/ggml-hexagon/htp/act-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hvx-utils.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\n#define htp_act_preamble3 \\\n const uint32_t ne00 = src0->ne[0]; \\\n const uint32_t ne01 = src0->ne[1]; \\\n const uint32_t ne02 = src0->ne[2]; \\\n const uint32_t ne03 = src0->ne[3]; \\\n \\\n const uint32_t ne10 = src1->ne[0]; \\\n const uint32_t ne11 = src1->ne[1]; \\\n const uint32_t ne12 = src1->ne[2]; \\\n const uint32_t ne13 = src1->ne[3]; \\\n \\\n const uint32_t ne0 = dst->ne[0]; \\\n const uint32_t ne1 = dst->ne[1]; \\\n const uint32_t ne2 = dst->ne[2]; \\\n const uint32_t ne3 = dst->ne[3]; \\\n \\\n const uint32_t nb00 = src0->nb[0]; \\\n const uint32_t nb01 = src0->nb[1]; \\\n const uint32_t nb02 = src0->nb[2]; \\\n const uint32_t nb03 = src0->nb[3]; \\\n \\\n const uint32_t nb10 = src1->nb[0]; \\\n const uint32_t nb11 = src1->nb[1]; \\\n const uint32_t nb12 = src1->nb[2]; \\\n const uint32_t nb13 = src1->nb[3]; \\\n \\\n const uint32_t nb0 = dst->nb[0]; \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3];\n\n#define htp_act_preamble2 \\\n const uint32_t ne00 = src0->ne[0]; \\\n const uint32_t ne01 = src0->ne[1]; \\\n const uint32_t ne02 = src0->ne[2]; \\\n const uint32_t ne03 = src0->ne[3]; \\\n \\\n const uint32_t ne0 = dst->ne[0]; \\\n const uint32_t ne1 = dst->ne[1]; \\\n const uint32_t ne2 = dst->ne[2]; \\\n const uint32_t ne3 = dst->ne[3]; \\\n \\\n const uint32_t nb00 = src0->nb[0]; \\\n const uint32_t nb01 = src0->nb[1]; \\\n const uint32_t nb02 = src0->nb[2]; \\\n const uint32_t nb03 = src0->nb[3]; \\\n \\\n const uint32_t nb0 = dst->nb[0]; \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3];\n\nstruct htp_act_context {\n struct htp_ops_context * octx;\n\n // Precomputed values\n const uint8_t * data_src0;\n const uint8_t * data_src1;\n uint8_t * data_dst;\n\n size_t src0_row_size;\n size_t src1_row_size;\n size_t dst_row_size;\n\n size_t src0_row_size_aligned;\n size_t src1_row_size_aligned;\n size_t dst_row_size_aligned;\n\n size_t src0_spad_half_size;\n size_t src1_spad_half_size;\n size_t dst_spad_half_size;\n\n uint32_t block;\n uint32_t src0_nrows;\n uint32_t src0_nrows_per_thread;\n int nc;\n};\n\nstatic void glu_swiglu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {\n struct htp_act_context * actx = (struct htp_act_context *) data;\n const struct htp_tensor * src0 = &actx->octx->src0;\n const struct htp_tensor * src1 = &actx->octx->src1;\n const struct htp_tensor * dst = &actx->octx->dst;\n htp_act_preamble3;\n\n size_t src0_row_size = actx->src0_row_size;\n size_t src1_row_size = actx->src1_row_size;\n size_t dst_row_size = actx->dst_row_size;\n\n const uint32_t src0_nrows = actx->src0_nrows;\n const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const uint8_t * restrict data_src0 = actx->data_src0;\n const uint8_t * restrict data_src1 = actx->data_src1;\n uint8_t * restrict data_dst = actx->data_dst;\n\n const int nc = actx->nc;\n\n const size_t src0_row_size_aligned = actx->src0_row_size_aligned;\n const size_t src1_row_size_aligned = actx->src1_row_size_aligned;\n const size_t dst_row_size_aligned = actx->dst_row_size_aligned;\n\n uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);\n uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread);\n uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);\n\n size_t src0_spad_half_size = actx->src0_spad_half_size;\n size_t src1_spad_half_size = actx->src1_spad_half_size;\n size_t dst_spad_half_size = actx->dst_spad_half_size;\n\n const int BLOCK = actx->block;\n if (BLOCK == 0) {\n FARF(ERROR,\n \"swiglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\\n\",\n actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);\n return;\n }\n\n dma_queue * dma_queue = actx->octx->ctx->dma[ith];\n\n // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379\n for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)\n dma_queue_push_vtcm_to_ddr(dma_queue,\n dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),\n dst_row_size, dst_row_size_aligned, 0);\n\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),\n src0_row_size_aligned, src0_row_size, block_size);\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)),\n src1_row_size_aligned, src1_row_size, block_size);\n }\n\n for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n float * dst_spad = (float *) dma_queue_pop(dma_queue).src;\n float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;\n float * src1_spad = (float *) dma_queue_pop(dma_queue).dst;\n\n for (uint32_t ib = 0; ib < block_size; ib++) {\n const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));\n const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float));\n float * dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float));\n\n //swiglu(x) = x1 * sigmoid(x0)\n hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, nc);\n hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr,\n (const uint8_t *) src1_spad_ptr, nc);\n }\n\n dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size,\n dst_row_size_aligned, block_size);\n\n // prefetch N+2 loop iteration if any\n const uint32_t pref_block = (ir + BLOCK * 2);\n if (pref_block < src0_end_row) {\n const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),\n src0_row_size_aligned, src0_row_size, pref_block_size);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)),\n src1_row_size_aligned, src1_row_size, pref_block_size);\n }\n }\n\n dma_queue_flush(dma_queue);\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"swiglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\\n\", ith, nth,\n ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3,\n (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\nstatic void glu_swiglu_oai_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {\n struct htp_act_context * actx = (struct htp_act_context *) data;\n const struct htp_tensor * src0 = &actx->octx->src0;\n const struct htp_tensor * src1 = &actx->octx->src1;\n const struct htp_tensor * dst = &actx->octx->dst;\n htp_act_preamble3;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n size_t src0_row_size = actx->src0_row_size;\n size_t src1_row_size = actx->src1_row_size;\n size_t dst_row_size = actx->dst_row_size;\n\n const uint32_t src0_nrows = actx->src0_nrows;\n const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n const uint8_t * restrict data_src0 = actx->data_src0;\n const uint8_t * restrict data_src1 = actx->data_src1;\n uint8_t * restrict data_dst = actx->data_dst;\n\n const int nc = actx->nc;\n\n const size_t src0_row_size_aligned = actx->src0_row_size_aligned;\n const size_t src1_row_size_aligned = actx->src1_row_size_aligned;\n const size_t dst_row_size_aligned = actx->dst_row_size_aligned;\n\n uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);\n uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread);\n uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);\n\n size_t src0_spad_half_size = actx->src0_spad_half_size;\n size_t src1_spad_half_size = actx->src1_spad_half_size;\n size_t dst_spad_half_size = actx->dst_spad_half_size;\n\n const int BLOCK = actx->block;\n if (BLOCK == 0) {\n FARF(ERROR,\n \"swiglu-oai-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least \"\n \"%zu\\n\",\n actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);\n return;\n }\n const float alpha = ((const float *) (actx->octx->op_params))[2];\n const float limit = ((const float *) (actx->octx->op_params))[3];\n\n dma_queue * dma_queue = actx->octx->ctx->dma[ith];\n\n // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379\n for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)\n dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),\n dst_row_size, dst_row_size_aligned, 0);\n\n dma_queue_push_ddr_to_vtcm(\n dma_queue,\n dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),\n src0_row_size_aligned, src0_row_size, block_size);\n dma_queue_push_ddr_to_vtcm(\n dma_queue,\n dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)),\n src1_row_size_aligned, src1_row_size, block_size);\n }\n\n for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n float * dst_spad = (float *) dma_queue_pop(dma_queue).src;\n float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;\n float * src1_spad = (float *) dma_queue_pop(dma_queue).dst;\n\n for (uint32_t ib = 0; ib < block_size; ib++) {\n const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));\n const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float));\n float * dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float));\n\n // x (src0_spad_data) = std::min(src0_p[k], limit);\n hvx_min_scalar_f32((uint8_t *) src0_spad_ptr, (const uint8_t *) src0_spad_ptr, limit, nc);\n // y1 (src1_spad_data) = std::clamp(src1_p[k], -limit, limit);\n hvx_clamp_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, -limit, limit, nc);\n // y (src1_spad_data) = y1 + 1.f\n hvx_add_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, 1.0, nc);\n // x1 (dst_spad_data) = alpha * (x)\n hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, alpha, nc);\n // x2 (dst_spad_data) = sigmoid(x1) = 1/(1+exp(-x1))\n hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc);\n // out = x * sigmoid(alpha * x) * (y + 1.f)\n hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr,\n (const uint8_t *) src1_spad_ptr, nc);\n }\n\n dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size,\n dst_row_size_aligned, block_size);\n\n // prefetch N+2 loop iteration if any\n const uint32_t pref_block = (ir + BLOCK * 2);\n if (pref_block < src0_end_row) {\n const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),\n src0_row_size_aligned, src0_row_size, pref_block_size);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)),\n src1_row_size_aligned, src1_row_size, pref_block_size);\n }\n }\n\n dma_queue_flush(dma_queue);\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"swiglu-oai-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\\n\", ith, nth, src0->ne[0],\n src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], src1->ne[1], src1->ne[2],\n src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n\nstatic void unary_gelu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {\n struct htp_act_context * actx = (struct htp_act_context *) data;\n const struct htp_tensor * src0 = &actx->octx->src0;\n const struct htp_tensor * dst = &actx->octx->dst;\n htp_act_preamble2;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const size_t src0_row_size = actx->src0_row_size;\n const size_t dst_row_size = actx->dst_row_size;\n const size_t src0_row_size_aligned = actx->src0_row_size_aligned;\n const size_t dst_row_size_aligned = actx->dst_row_size_aligned;\n\n const uint32_t src0_nrows = actx->src0_nrows;\n const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n const uint8_t * data_src0 = actx->data_src0;\n uint8_t * data_dst = actx->data_dst;\n\n // nc/ne0 matches.\n const int ne0_val = actx->nc; // == dst->ne[0]\n\n uint8_t * src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);\n uint8_t * dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);\n\n size_t src0_spad_half_size = actx->src0_spad_half_size;\n size_t dst_spad_half_size = actx->dst_spad_half_size;\n\n // In gelu = x*sigmoid(x*1.702)\n const int BLOCK = actx->block;\n\n if (BLOCK == 0) {\n FARF(ERROR, \"gelu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\\n\",\n actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);\n return;\n }\n\n dma_queue * dma_queue = actx->octx->ctx->dma[ith];\n\n // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379\n for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)\n dma_queue_push_vtcm_to_ddr(dma_queue,\n dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),\n dst_row_size, dst_row_size_aligned, 0);\n\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),\n src0_row_size_aligned, src0_row_size, block_size);\n }\n\n for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n float* dst_spad = (float *) dma_queue_pop(dma_queue).src;\n float* src0_spad = (float *) dma_queue_pop(dma_queue).dst;\n\n for (uint32_t ib = 0; ib < block_size; ib++) {\n const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));\n float* dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float));\n\n // gelu = x * sigmoid(1.702 * x) // current implementation\n hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (float) 1.702, ne0_val);\n hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val);\n hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val);\n }\n\n dma_queue_push_vtcm_to_ddr(dma_queue,\n dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad),\n dst_row_size, dst_row_size_aligned, block_size);\n\n // prefetch N+2 loop iteration if any\n const uint32_t pref_block = (ir + BLOCK * 2);\n if (pref_block < src0_end_row) {\n const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),\n src0_row_size_aligned, src0_row_size, pref_block_size);\n }\n }\n\n dma_queue_flush(dma_queue);\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"gelu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\\n\", ith, nth, ne00, ne01, ne02,\n ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n\nstatic void unary_silu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {\n struct htp_act_context * actx = (struct htp_act_context *) data;\n const struct htp_tensor * src0 = &actx->octx->src0;\n const struct htp_tensor * dst = &actx->octx->dst;\n htp_act_preamble2;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const size_t src0_row_size = actx->src0_row_size;\n const size_t dst_row_size = actx->dst_row_size;\n const size_t src0_row_size_aligned = actx->src0_row_size_aligned;\n const size_t dst_row_size_aligned = actx->dst_row_size_aligned;\n\n const uint32_t src0_nrows = actx->src0_nrows;\n const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n const uint8_t * data_src0 = actx->data_src0;\n uint8_t * data_dst = actx->data_dst;\n\n const int ne0_val = actx->nc; // == dst->ne[0]\n\n uint8_t * src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);\n uint8_t * dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);\n\n size_t src0_spad_half_size = actx->src0_spad_half_size;\n size_t dst_spad_half_size = actx->dst_spad_half_size;\n\n const int BLOCK = actx->block;\n\n if (BLOCK == 0) {\n FARF(ERROR, \"silu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\\n\",\n actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);\n return;\n }\n\n dma_queue * dma_queue = actx->octx->ctx->dma[ith];\n\n // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379\n for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)\n dma_queue_push_vtcm_to_ddr(dma_queue,\n dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),\n dst_row_size, dst_row_size_aligned, 0);\n\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),\n src0_row_size_aligned, src0_row_size, block_size);\n }\n\n for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n float* dst_spad = (float *) dma_queue_pop(dma_queue).src;\n float* src0_spad = (float *) dma_queue_pop(dma_queue).dst;\n\n for (uint32_t ib = 0; ib < block_size; ib++) {\n const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));\n float* dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float));\n\n // silu = x * sigmoid(x)\n hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, ne0_val);\n hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val);\n }\n\n dma_queue_push_vtcm_to_ddr(dma_queue,\n dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad),\n dst_row_size, dst_row_size_aligned, block_size);\n\n // prefetch N+2 loop iteration if any\n const uint32_t pref_block = (ir + BLOCK * 2);\n if (pref_block < src0_end_row) {\n const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),\n src0_row_size_aligned, src0_row_size, pref_block_size);\n }\n }\n\n dma_queue_flush(dma_queue);\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"silu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\\n\", ith, nth, ne00, ne01, ne02,\n ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\nstatic const float GELU_COEF_A = 0.044715f;\nstatic const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;\n\nstatic void glu_geglu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {\n struct htp_act_context * actx = (struct htp_act_context *) data;\n const struct htp_tensor * src0 = &actx->octx->src0;\n const struct htp_tensor * src1 = &actx->octx->src1;\n const struct htp_tensor * dst = &actx->octx->dst;\n htp_act_preamble3;\n\n size_t src0_row_size = actx->src0_row_size;\n size_t src1_row_size = actx->src1_row_size;\n size_t dst_row_size = actx->dst_row_size;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const uint32_t src0_nrows = actx->src0_nrows;\n const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n const uint8_t * restrict data_src0 = actx->data_src0;\n const uint8_t * restrict data_src1 = actx->data_src1;\n uint8_t * restrict data_dst = actx->data_dst;\n\n const int nc = actx->nc;\n\n const size_t src0_row_size_aligned = actx->src0_row_size_aligned;\n const size_t src1_row_size_aligned = actx->src1_row_size_aligned;\n const size_t dst_row_size_aligned = actx->dst_row_size_aligned;\n\n uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);\n uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread);\n uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);\n\n size_t src0_spad_half_size = actx->src0_spad_half_size;\n size_t src1_spad_half_size = actx->src1_spad_half_size;\n size_t dst_spad_half_size = actx->dst_spad_half_size;\n\n const int BLOCK = actx->block;\n if (BLOCK == 0) {\n FARF(ERROR,\n \"geglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\\n\",\n actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);\n return;\n }\n\n dma_queue * dma_queue = actx->octx->ctx->dma[ith];\n\n // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379\n for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)\n dma_queue_push_vtcm_to_ddr(dma_queue,\n dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),\n dst_row_size, dst_row_size_aligned, 0);\n\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),\n src0_row_size_aligned, src0_row_size, block_size);\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)),\n src1_row_size_aligned, src1_row_size, block_size);\n }\n\n for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n float * dst_spad = (float *) dma_queue_pop(dma_queue).src;\n float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;\n float * src1_spad = (float *) dma_queue_pop(dma_queue).dst;\n\n for (uint32_t ib = 0; ib < block_size; ib++) {\n const uint8_t * src0_spad_ptr = (const uint8_t *)(src0_spad + ib * (src0_row_size_aligned / sizeof(float)));\n const uint8_t * src1_spad_ptr = (const uint8_t *)(src1_spad + ib * (src1_row_size_aligned / sizeof(float)));\n uint8_t * dst_spad_ptr = (uint8_t *)(dst_spad + ib * (dst_row_size_aligned / sizeof(float)));\n\n // geglu tanh implementation\n // geglu(x, g) = gelu(x) * g\n // gelu(x) = 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)))\n hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, src0_spad_ptr, nc); // res = x*x\n hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, GELU_COEF_A, nc); // res = res * GELU_COEF_A\n hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 1.0f, nc); // res = res + 1.0f\n hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc); // res = res * x\n hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, SQRT_2_OVER_PI, nc); // res = result * SQRT_2_OVER_PI\n hvx_tanh_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc); // res = tanh(res)\n hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, 1.0f, nc); // res = res + 1.0f\n hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc); // res = res * x\n hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 0.5f, nc); // res = res + 0.5f\n hvx_mul_f32_aaa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, src1_spad_ptr, nc); // res = res * g\n }\n\n dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size,\n dst_row_size_aligned, block_size);\n\n // prefetch N+2 loop iteration if any\n const uint32_t pref_block = (ir + BLOCK * 2);\n if (pref_block < src0_end_row) {\n const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),\n src0_row_size_aligned, src0_row_size, pref_block_size);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)),\n src1_row_size_aligned, src1_row_size, pref_block_size);\n }\n }\n\n dma_queue_flush(dma_queue);\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"geglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\\n\", ith, nth,\n ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3,\n (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\nstatic int execute_op_activations_f32(struct htp_ops_context * octx) {\n const struct htp_tensor * src0 = &octx->src0;\n const struct htp_tensor * src1 = &octx->src1;\n struct htp_tensor * dst = &octx->dst;\n\n if (((src0->ne[0] * SIZEOF_FP32) != src0->nb[1]) || ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1])) {\n FARF(ERROR, \"Non-contiguous tensors are not supported at this time \\n\");\n return HTP_STATUS_NO_SUPPORT;\n }\n\n worker_callback_t act_op_func;\n const char * op_type = NULL;\n\n switch (octx->op) {\n case HTP_OP_UNARY_SILU:\n act_op_func = (worker_callback_t)unary_silu_f32_per_thread;\n op_type = \"silu-f32\";\n break;\n\n case HTP_OP_GLU_SWIGLU:\n act_op_func = (worker_callback_t)glu_swiglu_f32_per_thread;\n op_type = \"swiglu-f32\";\n break;\n\n case HTP_OP_GLU_SWIGLU_OAI:\n act_op_func = (worker_callback_t)glu_swiglu_oai_f32_per_thread;\n op_type = \"swiglu-oai-f32\";\n break;\n case HTP_OP_UNARY_GELU:\n act_op_func = (worker_callback_t)unary_gelu_f32_per_thread;\n op_type = \"gelu-f32\";\n break;\n\n case HTP_OP_GLU_GEGLU:\n act_op_func = (worker_callback_t)glu_geglu_f32_per_thread;\n op_type = \"geglu-f32\";\n break;\n default:\n FARF(ERROR, \"Unsupported activations Op %u\\n\", octx->op);\n return HTP_STATUS_NO_SUPPORT;\n }\n\n const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];\n const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);\n\n size_t src0_row_size = src0->nb[1];\n size_t src1_row_size = src1->nb[1]; // zero bytes if src1 is not used\n size_t dst_row_size = dst->nb[1];\n\n const bool src1_valid = src1->ne[0];\n if (!src1_valid) {\n src1_row_size = src0_row_size;\n }\n\n const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);\n const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);\n const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);\n // VTCM scratchpads for all tensors\n // N rows per thread, padded to HVX vector size\n\n size_t spad_size_per_row = (src0_row_size_aligned + src1_row_size_aligned) + dst_row_size_aligned;\n size_t vtcm_row_per_thread = (octx->ctx->vtcm_size)/ (n_threads* spad_size_per_row);\n\n // Make sure the reserved vtcm size is sufficient\n if(vtcm_row_per_thread ==0){\n FARF(ERROR, \"act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\\n\", op_type, octx->ctx->vtcm_size,\n spad_size_per_row * n_threads);\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n octx->src0_spad.size_per_thread = src0_row_size_aligned * vtcm_row_per_thread;\n octx->src1_spad.size_per_thread = src1_row_size_aligned * vtcm_row_per_thread;\n octx->dst_spad.size_per_thread = dst_row_size_aligned * vtcm_row_per_thread;\n\n octx->dst_spad.size = n_threads* octx->dst_spad.size_per_thread;\n octx->src0_spad.size = n_threads* octx->src0_spad.size_per_thread;\n octx->src1_spad.size = n_threads* octx->src1_spad.size_per_thread;\n\n octx->src0_spad.data = octx->ctx->vtcm_base;\n octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;\n octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size;\n\n if (src1->ne[0]) {\n FARF(HIGH, \"%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\\n\",\n op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2],\n src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], octx->src0_spad.size, octx->src1_spad.size,\n octx->dst_spad.size);\n } else {\n FARF(HIGH, \"%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\\n\", op_type,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size);\n }\n\n if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {\n return HTP_STATUS_OK;\n }\n\n // Prepare context\n struct htp_act_context actx;\n actx.octx = octx;\n\n actx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;\n\n actx.src0_row_size = src0_row_size;\n actx.src1_row_size = src1_row_size;\n actx.dst_row_size = dst_row_size;\n\n actx.src0_row_size_aligned = src0_row_size_aligned;\n actx.src1_row_size_aligned = src1_row_size_aligned;\n actx.dst_row_size_aligned = dst_row_size_aligned;\n\n actx.src0_spad_half_size = octx->src0_spad.size_per_thread / 2;\n actx.src1_spad_half_size = octx->src1_spad.size_per_thread / 2;\n actx.dst_spad_half_size = octx->dst_spad.size_per_thread / 2;\n\n actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;\n actx.src0_nrows = src0_nrows;\n\n actx.nc = dst->ne[0];\n\n // Pointers and GLU logic\n const uint8_t * data_src0 = (const uint8_t *) src0->data;\n const uint8_t * data_src1 = (const uint8_t *) src1->data;\n\n if (!src1_valid && (octx->op == HTP_OP_GLU_SWIGLU || octx->op == HTP_OP_GLU_SWIGLU_OAI || octx->op == HTP_OP_GLU_GEGLU)) {\n const int32_t swapped = octx->op_params[1];\n data_src1 = data_src0;\n actx.src1_row_size = actx.src0_row_size;\n\n size_t nc_in_bytes = actx.nc * SIZEOF_FP32;\n if (swapped) {\n data_src0 += nc_in_bytes;\n } else {\n data_src1 += nc_in_bytes;\n }\n }\n\n actx.data_src0 = data_src0;\n actx.data_src1 = data_src1;\n actx.data_dst = (uint8_t *) dst->data;\n\n worker_pool_run_func(octx->ctx->worker_pool, act_op_func, &actx, n_threads);\n return HTP_STATUS_OK;\n}\n\nint op_activations(struct htp_ops_context * octx) {\n int err = HTP_STATUS_OK;\n\n switch (octx->src0.type) {\n case HTP_TYPE_F32:\n err = execute_op_activations_f32(octx);\n break;\n\n default:\n err = HTP_STATUS_NO_SUPPORT;\n break;\n }\n\n return err;\n}\n" }, { "path": "src/ggml-hexagon/htp/argsort-ops.c", "content": "#include \n#include \n#include \n#include \n#include \n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"ggml.h\"\n\n#include \"hvx-utils.h\"\n#include \"hex-dma.h\"\n\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\n#ifndef MIN\n#define MIN(a, b) ((a) < (b) ? (a) : (b))\n#endif\n\nstruct htp_argsort_context {\n struct htp_ops_context * octx;\n uint32_t nrows_per_thread;\n};\n\nstatic inline bool all_greater_f32(HVX_Vector x, HVX_Vector y)\n{\n const HVX_Vector one = Q6_V_vsplat_R(1);\n const HVX_Vector zero = Q6_V_vzero();\n\n HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(x, y);\n HVX_Vector matches = Q6_V_vmux_QVV(pred, one, zero);\n HVX_Vector sum = hvx_vec_reduce_sum_i32(matches);\n return hvx_vec_get_i32(sum) == 32;\n}\n\n// Sorts values and mirrors swaps to indices.\nstatic void quicksort_values_indices_asc(float * values, int32_t * indices, int left, int right) {\n if (left >= right) return;\n\n int pivot_idx = (left + right) / 2;\n float pivot = values[pivot_idx];\n int i = left;\n int j = right;\n\n HVX_Vector pivot_vec = hvx_vec_splat_f32(pivot);\n while (i <= j) {\n // Vectorized scan for i\n while (i <= j) {\n // Check if we have at least one full vector\n if (i + 32 <= j) {\n HVX_Vector vals_vec = *(HVX_UVector *)(values + i);\n if (all_greater_f32(pivot_vec, vals_vec)) {\n // If all elements are < pivot, we can skip this whole block\n i += 32;\n continue;\n }\n }\n\n // Scalar fallback / cleanup\n if (values[i] < pivot) {\n i++;\n } else {\n break;\n }\n }\n\n // Vectorized scan for j\n while (i <= j) {\n if (j - 32 >= i) {\n // Load 32 elements ending at j.\n // Since we want `values[j] > pivot`, let's load from j-31 to j.\n HVX_Vector vals_vec = *(HVX_UVector *)(values + j - 31);\n if (all_greater_f32(vals_vec, pivot_vec)) {\n j -= 32;\n continue;\n }\n }\n\n if (values[j] > pivot) {\n j--;\n } else {\n break;\n }\n }\n\n if (i <= j) {\n float tmp_val = values[i];\n values[i] = values[j];\n values[j] = tmp_val;\n\n int32_t tmp_idx = indices[i];\n indices[i] = indices[j];\n indices[j] = tmp_idx;\n i++;\n j--;\n }\n }\n\n if (left < j) quicksort_values_indices_asc(values, indices, left, j);\n if (i < right) quicksort_values_indices_asc(values, indices, i, right);\n}\n\nstatic void quicksort_values_indices_desc(float * values, int32_t * indices, int left, int right) {\n if (left >= right) return;\n\n int pivot_idx = (left + right) / 2;\n float pivot = values[pivot_idx];\n int i = left;\n int j = right;\n\n HVX_Vector pivot_vec = hvx_vec_splat_f32(pivot);\n\n while (i <= j) {\n // Vectorized scan for i (values[i] > pivot)\n while (i <= j) {\n if (i + 32 <= j) {\n HVX_Vector vals_vec = *(HVX_UVector *)(values + i);\n if (all_greater_f32(vals_vec, pivot_vec)) {\n i += 32;\n continue;\n }\n }\n\n if (values[i] > pivot) {\n i++;\n } else {\n break;\n }\n }\n\n // Vectorized scan for j (values[j] < pivot)\n while (i <= j) {\n if (j - 32 >= i) {\n HVX_Vector vals_vec = *(HVX_UVector *)(values + j - 31);\n if (all_greater_f32(pivot_vec, vals_vec)) {\n j -= 32;\n continue;\n }\n }\n\n if (values[j] < pivot) {\n j--;\n } else {\n break;\n }\n }\n\n if (i <= j) {\n float tmp_val = values[i];\n values[i] = values[j];\n values[j] = tmp_val;\n\n int32_t tmp_idx = indices[i];\n indices[i] = indices[j];\n indices[j] = tmp_idx;\n i++;\n j--;\n }\n }\n\n if (left < j) quicksort_values_indices_desc(values, indices, left, j);\n if (i < right) quicksort_values_indices_desc(values, indices, i, right);\n}\n\nstatic void htp_argsort_f32(unsigned int n, unsigned int i, void * data) {\n struct htp_argsort_context * actx = (struct htp_argsort_context *)data;\n struct htp_ops_context * octx = actx->octx;\n\n // Unpack context\n const struct htp_tensor * src0 = &octx->src0;\n const struct htp_tensor * dst = &octx->dst;\n\n // Scratchpad memory\n uint8_t * spad = octx->src0_spad.data + octx->src0_spad.size_per_thread * i;\n\n // Dimensions\n uint32_t ne00 = src0->ne[0];\n uint32_t ne01 = src0->ne[1];\n uint32_t ne02 = src0->ne[2];\n uint32_t ne03 = src0->ne[3];\n\n uint32_t nb01 = src0->nb[1];\n //uint32_t nb02 = src0->nb[2];\n //uint32_t nb03 = src0->nb[3];\n\n uint32_t nb1 = dst->nb[1];\n //uint32_t nb2 = dst->nb[2];\n //uint32_t nb3 = dst->nb[3];\n\n // Sort order\n enum ggml_sort_order order = (enum ggml_sort_order) octx->op_params[0];\n\n // Rows to process\n uint32_t total_rows = ne01 * ne02 * ne03;\n uint32_t rows_per_thread = actx->nrows_per_thread;\n uint32_t start_row = rows_per_thread * i;\n uint32_t end_row = MIN(start_row + rows_per_thread, total_rows);\n\n // Scratchpad layout:\n // We need space for one row of float data (values) and one row of int32 indices.\n // values: ne00 * sizeof(float)\n // indices: ne00 * sizeof(int32_t)\n // Padded to 128 bytes.\n\n size_t values_size = hex_round_up(ne00 * sizeof(float), 128);\n float * values_buf = (float *) spad;\n int32_t * indices_buf = (int32_t *) (spad + values_size);\n\n for (uint32_t r = start_row; r < end_row; r++) {\n uint32_t src_offset = r * nb01;\n uint32_t dst_offset = r * nb1;\n\n uint8_t * src_ptr = (uint8_t *) src0->data + src_offset;\n uint8_t * dst_ptr = (uint8_t *) dst->data + dst_offset;\n\n hex_l2fetch(src_ptr, ne00 * sizeof(float), ne00 * sizeof(float), 1);\n hvx_copy_f32_au((uint8_t*)values_buf, src_ptr, ne00);\n\n // Initialize indices\n for (uint32_t j = 0; j < ne00; j++) {\n indices_buf[j] = j;\n }\n\n // Sort values and mirror swaps to indices\n if (order == GGML_SORT_ORDER_ASC) {\n quicksort_values_indices_asc(values_buf, indices_buf, 0, ne00 - 1);\n } else {\n quicksort_values_indices_desc(values_buf, indices_buf, 0, ne00 - 1);\n }\n\n // Copy indices back to DDR\n hvx_copy_f32_ua(dst_ptr, (const uint8_t *) indices_buf, ne00);\n }\n}\n\nint op_argsort(struct htp_ops_context * octx) {\n // Check supported types\n if (octx->src0.type != HTP_TYPE_F32) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n const uint32_t total_rows = octx->src0.ne[1] * octx->src0.ne[2] * octx->src0.ne[3];\n const uint32_t n_threads = MIN(total_rows, octx->n_threads);\n\n // Allocate scratchpad\n // We need 1 row of float + 1 row of int32 per thread.\n uint32_t ne00 = octx->src0.ne[0];\n size_t values_size = hex_round_up(ne00 * sizeof(float), 128);\n size_t indices_size = hex_round_up(ne00 * sizeof(int32_t), 128);\n size_t spad_per_thread = values_size + indices_size;\n\n // Make sure we round up to 256 for alignment requirements\n spad_per_thread = hex_round_up(spad_per_thread, 256);\n\n size_t total_spad_size = spad_per_thread * n_threads;\n\n if (octx->ctx->vtcm_size < total_spad_size) {\n FARF(ERROR, \"argsort: VTCM size too small. Needed %zu, have %zu\", total_spad_size, octx->ctx->vtcm_size);\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n octx->src0_spad.data = octx->ctx->vtcm_base;\n octx->src0_spad.size = total_spad_size;\n octx->src0_spad.size_per_thread = spad_per_thread;\n\n FARF(HIGH, \"argsort: %ux%ux%ux%u -> %ux%ux%ux%u (0x%x, 0x%x)\",\n octx->src0.ne[0], octx->src0.ne[1], octx->src0.ne[2], octx->src0.ne[3],\n octx->dst.ne[0], octx->dst.ne[1], octx->dst.ne[2], octx->dst.ne[3],\n octx->src0.data, octx->dst.data);\n\n struct htp_argsort_context actx;\n actx.octx = octx;\n actx.nrows_per_thread = (total_rows + n_threads - 1) / n_threads;\n\n // Run jobs\n worker_pool_run_func(octx->ctx->worker_pool, htp_argsort_f32, &actx, n_threads);\n\n return HTP_STATUS_OK;\n}\n" }, { "path": "src/ggml-hexagon/htp/binary-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hvx-utils.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\n#ifndef MIN\n#define MIN(a, b) ((a) < (b) ? (a) : (b))\n#endif\n\n// Context for binary operations\nstruct htp_binary_context {\n struct htp_ops_context * octx;\n struct fastdiv_values dim1_div;\n struct fastdiv_values dim2_div;\n struct fastdiv_values dim12_div;\n\n struct fastdiv_values src1_dim1_div; // ne11\n struct fastdiv_values src1_dim2_div; // ne12\n struct fastdiv_values src1_dim3_div; // ne13\n\n uint32_t nrows_per_thread;\n bool split_at_ne01;\n bool split_at_ne02;\n\n // Precomputed values\n uint32_t block_max;\n size_t src0_row_size_aligned;\n size_t src1_row_size_aligned;\n size_t dst_row_size_aligned;\n uint32_t src1_fetch_rows; // 1 or block_max\n uint32_t src1_dma_stride; // 0 or stride\n};\n\n#define htp_binary_preamble \\\n const struct htp_tensor * src0 = &octx->src0; \\\n const struct htp_tensor * src1 = &octx->src1; \\\n struct htp_tensor * dst = &octx->dst; \\\n \\\n const uint32_t ne00 = src0->ne[0]; \\\n const uint32_t ne01 = src0->ne[1]; \\\n const uint32_t ne02 = src0->ne[2]; \\\n const uint32_t ne03 = src0->ne[3]; \\\n \\\n const uint32_t ne10 = src1->ne[0]; \\\n const uint32_t ne11 = src1->ne[1]; \\\n const uint32_t ne12 = src1->ne[2]; \\\n const uint32_t ne13 = src1->ne[3]; \\\n \\\n const uint32_t nb01 = src0->nb[1]; \\\n const uint32_t nb02 = src0->nb[2]; \\\n const uint32_t nb03 = src0->nb[3]; \\\n \\\n const uint32_t nb11 = src1->nb[1]; \\\n const uint32_t nb12 = src1->nb[2]; \\\n const uint32_t nb13 = src1->nb[3]; \\\n \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3];\n\nstatic inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_t ir, uint32_t end_row,\n uint32_t ne01, uint32_t ne02) {\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir, &bctx->dim12_div);\n rem = ir - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n uint32_t rows_left = end_row - ir;\n uint32_t block_limit = rows_left;\n\n if (bctx->split_at_ne01) {\n block_limit = MIN(block_limit, ne01 - i01);\n }\n if (bctx->split_at_ne02) {\n uint32_t rows_in_plane = (ne02 * ne01) - rem;\n block_limit = MIN(block_limit, rows_in_plane);\n }\n\n return MIN(bctx->block_max, block_limit);\n}\n\n// Macro for scalar op switch\n#define COMPUTE_SCALAR_OP(DST, SRC, VAL, TYPE, N) \\\n if(TYPE == HTP_TYPE_F32) { \\\n switch (octx->op) { \\\n case HTP_OP_ADD: hvx_add_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \\\n case HTP_OP_SUB: hvx_sub_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \\\n case HTP_OP_MUL: hvx_mul_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \\\n case HTP_OP_DIV: hvx_mul_scalar_f32_aa(DST, SRC, 1.0f / (*(float *)VAL), N); break; \\\n default: break; \\\n } \\\n } \\\n else { \\\n switch (octx->op) { \\\n case HTP_OP_ADD: hvx_add_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \\\n case HTP_OP_SUB: hvx_sub_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \\\n case HTP_OP_MUL: hvx_mul_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \\\n case HTP_OP_DIV: hvx_div_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \\\n default: break; \\\n } \\\n }\n\n// Macro for vector op switch (All Aligned)\n#define COMPUTE_VECTOR_OP_AAA(DST, SRC0, SRC1, TYPE, N) \\\n if(TYPE == HTP_TYPE_F32) { \\\n switch (octx->op) { \\\n case HTP_OP_ADD: hvx_add_f32_aaa(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_SUB: hvx_sub_f32_aaa(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_MUL: hvx_mul_f32_aaa(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_DIV: hvx_div_f32_aaa(DST, SRC0, SRC1, N); break; \\\n default: break; \\\n } \\\n } \\\n else { \\\n switch (octx->op) { \\\n case HTP_OP_ADD: hvx_add_f16_aaa(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_SUB: hvx_sub_f16_aaa(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_MUL: hvx_mul_f16_aaa(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_DIV: hvx_div_f16_aaa(DST, SRC0, SRC1, N); break; \\\n default: break; \\\n } \\\n }\n\n// Macro for vector op switch (Dst Aligned, Src0 Aligned, Src1 Unaligned)\n#define COMPUTE_VECTOR_OP_AAU(DST, SRC0, SRC1, TYPE, N) \\\n if(TYPE == HTP_TYPE_F32) { \\\n switch (octx->op) { \\\n case HTP_OP_ADD: hvx_add_f32_aau(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_SUB: hvx_sub_f32_aau(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_MUL: hvx_mul_f32_aau(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_DIV: hvx_div_f32_aau(DST, SRC0, SRC1, N); break; \\\n default: break; \\\n } \\\n } \\\n else { \\\n switch (octx->op) { \\\n case HTP_OP_ADD: hvx_add_f16_aau(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_SUB: hvx_sub_f16_aau(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_MUL: hvx_mul_f16_aau(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_DIV: hvx_div_f16_aau(DST, SRC0, SRC1, N); break; \\\n default: break; \\\n } \\\n }\n\n// Macro for vector op switch (All Unaligned - generic loop used in element repeat)\n#define COMPUTE_VECTOR_OP_UUU(DST, SRC0, SRC1, TYPE, N) \\\n if(TYPE == HTP_TYPE_F32) { \\\n switch (octx->op) { \\\n case HTP_OP_ADD: hvx_add_f32_uuu(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_SUB: hvx_sub_f32_uuu(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_MUL: hvx_mul_f32_uuu(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_DIV: hvx_div_f32_uuu(DST, SRC0, SRC1, N); break; \\\n default: break; \\\n } \\\n } \\\n else { \\\n switch (octx->op) { \\\n case HTP_OP_ADD: hvx_add_f16_uuu(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_SUB: hvx_sub_f16_uuu(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_MUL: hvx_mul_f16_uuu(DST, SRC0, SRC1, N); break; \\\n case HTP_OP_DIV: hvx_div_f16_uuu(DST, SRC0, SRC1, N); break; \\\n default: break; \\\n } \\\n }\n\n// 1. Scalar src1 (ne10 == 1)\nstatic void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) {\n struct htp_binary_context * bctx = (struct htp_binary_context *) data;\n struct htp_ops_context * octx = bctx->octx;\n htp_binary_preamble;\n\n const uint32_t src0_type = octx->src0.type;\n const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16);\n const uint32_t total_rows = ne01 * ne02 * ne03;\n const uint32_t start_row = bctx->nrows_per_thread * ith;\n const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);\n if (start_row >= end_row) return;\n\n uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);\n uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);\n size_t src0_spad_half = octx->src0_spad.size_per_thread / 2;\n size_t dst_spad_half = octx->dst_spad.size_per_thread / 2;\n\n dma_queue * q = octx->ctx->dma[ith];\n uint32_t ir_prefetch = start_row;\n int spad_idx = 0;\n\n // Preamble\n for (int k = 0; k < 2 && ir_prefetch < end_row; k++) {\n uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n rem = ir_prefetch - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n\n uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;\n uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half;\n\n dma_queue_push_vtcm_to_ddr(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, 0);\n dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size);\n ir_prefetch += current_block_size;\n spad_idx ^= 1;\n }\n\n // Main loop\n for (uint32_t ir = start_row; ir < end_row; ) {\n uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);\n\n uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;\n uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst;\n\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir, &bctx->dim12_div);\n rem = ir - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n // src1 indices (broadcast/repeat)\n uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);\n uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div);\n uint32_t i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div);\n\n uint8_t * src1_ptr = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11;\n uint32_t s1_stride = (ne11 == 1) ? 0 : nb11;\n\n for (uint32_t r = 0; r < current_block_size; r++) {\n uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned;\n uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned;\n COMPUTE_SCALAR_OP(r_dst, r_src0, src1_ptr, src0_type, ne00);\n src1_ptr += s1_stride;\n }\n\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);\n\n if (ir_prefetch < end_row) {\n uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t p03, p02, p01, prem;\n p03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n prem = ir_prefetch - p03 * (ne02 * ne01);\n p02 = fastdiv(prem, &bctx->dim1_div);\n p01 = prem - p02 * ne01;\n uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;\n\n dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size);\n ir_prefetch += next_block_size;\n }\n ir += current_block_size;\n }\n dma_queue_flush(q);\n}\n\n// 2. Vector Same Shape (ne1x == ne0x) or Simple Broadcast\nstatic void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, void * data) {\n struct htp_binary_context * bctx = (struct htp_binary_context *) data;\n struct htp_ops_context * octx = bctx->octx;\n htp_binary_preamble;\n\n const uint32_t src0_type = octx->src0.type;\n const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16);\n const uint32_t total_rows = ne01 * ne02 * ne03;\n const uint32_t start_row = bctx->nrows_per_thread * ith;\n const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);\n if (start_row >= end_row) return;\n\n uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);\n uint8_t * src1_spad_base = octx->src1_spad.data + (ith * octx->src1_spad.size_per_thread);\n uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);\n\n size_t src0_spad_half = octx->src0_spad.size_per_thread / 2;\n size_t src1_spad_half = octx->src1_spad.size_per_thread / 2;\n size_t dst_spad_half = octx->dst_spad.size_per_thread / 2;\n\n dma_queue * q = octx->ctx->dma[ith];\n uint32_t ir_prefetch = start_row;\n int spad_idx = 0;\n\n for (int k = 0; k < 2 && ir_prefetch < end_row; k++) {\n uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n rem = ir_prefetch - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n uint32_t i13 = (ne13 == 1) ? 0 : i03;\n uint32_t i12 = (ne12 == 1) ? 0 : i02;\n uint32_t i11 = (ne11 == 1) ? 0 : i01;\n\n uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;\n uint8_t * src1_base = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11;\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n\n uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;\n uint8_t * s1_spad = src1_spad_base + spad_idx * src1_spad_half;\n uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half;\n\n dma_queue_push_vtcm_to_ddr(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, 0);\n dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size);\n dma_queue_push(q, dma_make_ptr(s1_spad, src1_base), bctx->src1_row_size_aligned, bctx->src1_dma_stride, row_size_bytes, current_block_size);\n ir_prefetch += current_block_size;\n spad_idx ^= 1;\n }\n\n for (uint32_t ir = start_row; ir < end_row; ) {\n uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);\n uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;\n uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst;\n uint8_t * s1_spad = (uint8_t *) dma_queue_pop(q).dst;\n\n for (uint32_t r = 0; r < current_block_size; r++) {\n uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned;\n uint8_t * r_src1 = s1_spad + r * bctx->src1_row_size_aligned;\n uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned;\n COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00);\n }\n\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir, &bctx->dim12_div);\n rem = ir - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);\n\n if (ir_prefetch < end_row) {\n uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t p03, p02, p01, prem;\n p03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n prem = ir_prefetch - p03 * (ne02 * ne01);\n p02 = fastdiv(prem, &bctx->dim1_div);\n p01 = prem - p02 * ne01;\n\n uint32_t p13 = (ne13 == 1) ? 0 : p03;\n uint32_t p12 = (ne12 == 1) ? 0 : p02;\n uint32_t p11 = (ne11 == 1) ? 0 : p01;\n\n uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;\n uint8_t * s1_next = (uint8_t *)src1->data + p13 * nb13 + p12 * nb12 + p11 * nb11;\n\n dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size);\n dma_queue_push(q, dma_make_ptr(s1_spad, s1_next), bctx->src1_row_size_aligned, bctx->src1_dma_stride, row_size_bytes, next_block_size);\n\n ir_prefetch += next_block_size;\n }\n ir += current_block_size;\n }\n dma_queue_flush(q);\n}\n\n// 3. Row Broadcast (ne11 == 1, ne12 == 1, single row src1)\nstatic void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, void * data) {\n struct htp_binary_context * bctx = (struct htp_binary_context *) data;\n struct htp_ops_context * octx = bctx->octx;\n htp_binary_preamble;\n\n const uint32_t src0_type = octx->src0.type;\n const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16);\n const uint32_t total_rows = ne01 * ne02 * ne03;\n const uint32_t start_row = bctx->nrows_per_thread * ith;\n const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);\n if (start_row >= end_row) return;\n\n uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);\n uint8_t * src1_spad = octx->src1_spad.data + (ith * octx->src1_spad.size_per_thread);\n uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);\n\n size_t src0_spad_half = octx->src0_spad.size_per_thread / 2;\n size_t dst_spad_half = octx->dst_spad.size_per_thread / 2;\n\n dma_queue * q = octx->ctx->dma[ith];\n uint32_t ir_prefetch = start_row;\n int spad_idx = 0;\n\n void * s1_ptr = (void *) src1_spad;\n\n for (int k = 0; k < 2 && ir_prefetch < end_row; k++) {\n uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n rem = ir_prefetch - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n\n uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;\n uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half;\n\n dma_queue_push_vtcm_to_ddr(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, 0);\n dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size);\n ir_prefetch += current_block_size;\n spad_idx ^= 1;\n }\n\n for (uint32_t ir = start_row; ir < end_row; ) {\n uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);\n uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;\n uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst;\n\n for (uint32_t r = 0; r < current_block_size; r++) {\n uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned;\n uint8_t * r_src1 = (uint8_t *)s1_ptr; // Constant\n uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned;\n COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00);\n }\n\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir, &bctx->dim12_div);\n rem = ir - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);\n\n if (ir_prefetch < end_row) {\n uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t p03, p02, p01, prem;\n p03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n prem = ir_prefetch - p03 * (ne02 * ne01);\n p02 = fastdiv(prem, &bctx->dim1_div);\n p01 = prem - p02 * ne01;\n uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;\n dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size);\n ir_prefetch += next_block_size;\n }\n ir += current_block_size;\n }\n dma_queue_flush(q);\n}\n\n// 4. Vector Complex (ne10 == ne00, complex broadcast)\nstatic void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * data) {\n struct htp_binary_context * bctx = (struct htp_binary_context *) data;\n struct htp_ops_context * octx = bctx->octx;\n htp_binary_preamble;\n\n const uint32_t src0_type = octx->src0.type;\n const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16);\n const uint32_t total_rows = ne01 * ne02 * ne03;\n const uint32_t start_row = bctx->nrows_per_thread * ith;\n const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);\n if (start_row >= end_row) return;\n\n uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);\n uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);\n size_t src0_spad_half = octx->src0_spad.size_per_thread / 2;\n size_t dst_spad_half = octx->dst_spad.size_per_thread / 2;\n\n dma_queue * q = octx->ctx->dma[ith];\n uint32_t ir_prefetch = start_row;\n int spad_idx = 0;\n\n for (int k = 0; k < 2 && ir_prefetch < end_row; k++) {\n uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n rem = ir_prefetch - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n\n uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;\n uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half;\n\n dma_queue_push_vtcm_to_ddr(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, 0);\n dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size);\n ir_prefetch += current_block_size;\n spad_idx ^= 1;\n }\n\n for (uint32_t ir = start_row; ir < end_row; ) {\n uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);\n uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;\n uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst;\n\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir, &bctx->dim12_div);\n rem = ir - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n for (uint32_t r = 0; r < current_block_size; r++) {\n uint32_t r_i01 = i01 + r;\n uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);\n uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div);\n uint32_t i11 = fastmodulo(r_i01, ne11, &bctx->src1_dim1_div);\n\n uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned;\n uint8_t * r_src1 = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11;\n uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned;\n\n // Read src1 from DDR (unaligned)\n COMPUTE_VECTOR_OP_AAU(r_dst, r_src0, r_src1, src0_type, ne00);\n }\n\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);\n\n if (ir_prefetch < end_row) {\n uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t p03, p02, p01, prem;\n p03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n prem = ir_prefetch - p03 * (ne02 * ne01);\n p02 = fastdiv(prem, &bctx->dim1_div);\n p01 = prem - p02 * ne01;\n uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;\n dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size);\n ir_prefetch += next_block_size;\n }\n ir += current_block_size;\n }\n dma_queue_flush(q);\n}\n\n// 5. Element Repeat (ne10 != ne00)\nstatic void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * data) {\n struct htp_binary_context * bctx = (struct htp_binary_context *) data;\n struct htp_ops_context * octx = bctx->octx;\n htp_binary_preamble;\n\n const uint32_t src0_type = octx->src0.type;\n const uint32_t elem_size_bytes = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);\n const uint32_t row_size_bytes = ne00 * elem_size_bytes;;\n const uint32_t total_rows = ne01 * ne02 * ne03;\n const uint32_t start_row = bctx->nrows_per_thread * ith;\n const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);\n if (start_row >= end_row) return;\n\n uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);\n uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);\n size_t src0_spad_half = octx->src0_spad.size_per_thread / 2;\n size_t dst_spad_half = octx->dst_spad.size_per_thread / 2;\n\n dma_queue * q = octx->ctx->dma[ith];\n uint32_t ir_prefetch = start_row;\n int spad_idx = 0;\n\n for (int k = 0; k < 2 && ir_prefetch < end_row; k++) {\n uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n rem = ir_prefetch - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n\n uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;\n uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half;\n\n dma_queue_push_vtcm_to_ddr(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, 0);\n dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size);\n ir_prefetch += current_block_size;\n spad_idx ^= 1;\n }\n\n for (uint32_t ir = start_row; ir < end_row; ) {\n uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);\n uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;\n uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst;\n\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir, &bctx->dim12_div);\n rem = ir - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n for (uint32_t r = 0; r < current_block_size; r++) {\n uint32_t r_i01 = i01 + r;\n uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);\n uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div);\n uint32_t i11 = fastmodulo(r_i01, ne11, &bctx->src1_dim1_div);\n\n uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned;\n uint8_t * r_src1_row = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11;\n uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned;\n\n // Repeat src1 row\n for (uint32_t c = 0; c < ne00; c += ne10) {\n uint32_t len = MIN(ne10, ne00 - c);\n // Use UUU for speed and simplicity\n COMPUTE_VECTOR_OP_UUU(r_dst + c * elem_size_bytes, r_src0 + c * elem_size_bytes, r_src1_row, src0_type, len);\n }\n }\n\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);\n\n if (ir_prefetch < end_row) {\n uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t p03, p02, p01, prem;\n p03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n prem = ir_prefetch - p03 * (ne02 * ne01);\n p02 = fastdiv(prem, &bctx->dim1_div);\n p01 = prem - p02 * ne01;\n uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;\n dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size);\n ir_prefetch += next_block_size;\n }\n ir += current_block_size;\n }\n dma_queue_flush(q);\n}\n\n// 6. ADD_ID (src1 gathered via src2 indices)\nstatic void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) {\n struct htp_binary_context * bctx = (struct htp_binary_context *) data;\n struct htp_ops_context * octx = bctx->octx;\n\n const struct htp_tensor * src0 = &octx->src0;\n const struct htp_tensor * src1 = &octx->src1;\n const struct htp_tensor * src2 = &octx->src2;\n struct htp_tensor * dst = &octx->dst;\n\n const uint32_t ne00 = src0->ne[0];\n const uint32_t ne01 = src0->ne[1];\n const uint32_t ne02 = src0->ne[2];\n const uint32_t ne03 = src0->ne[3];\n const uint32_t ne11 = src1->ne[1]; // for bounds check\n\n const uint32_t nb01 = src0->nb[1];\n const uint32_t nb02 = src0->nb[2];\n const uint32_t nb03 = src0->nb[3];\n const uint32_t nb11 = src1->nb[1]; // src1 row stride\n const uint32_t nb1 = dst->nb[1];\n const uint32_t nb2 = dst->nb[2];\n const uint32_t nb3 = dst->nb[3];\n\n const uint32_t total_rows = ne01 * ne02 * ne03;\n const uint32_t start_row = bctx->nrows_per_thread * ith;\n const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);\n if (start_row >= end_row) return;\n\n uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);\n uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);\n size_t src0_spad_half = octx->src0_spad.size_per_thread / 2;\n size_t dst_spad_half = octx->dst_spad.size_per_thread / 2;\n\n dma_queue * q = octx->ctx->dma[ith];\n uint32_t ir_prefetch = start_row;\n int spad_idx = 0;\n\n for (int k = 0; k < 2 && ir_prefetch < end_row; k++) {\n uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n rem = ir_prefetch - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n\n uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;\n uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half;\n\n dma_queue_push_vtcm_to_ddr(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, 0);\n dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, ne00 * sizeof(float), current_block_size);\n ir_prefetch += current_block_size;\n spad_idx ^= 1;\n }\n\n for (uint32_t ir = start_row; ir < end_row; ) {\n uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);\n uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;\n uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst;\n\n uint32_t i03, i02, i01, rem;\n i03 = fastdiv(ir, &bctx->dim12_div);\n rem = ir - i03 * (ne02 * ne01);\n i02 = fastdiv(rem, &bctx->dim1_div);\n i01 = rem - i02 * ne01;\n\n for (uint32_t r = 0; r < current_block_size; r++) {\n uint32_t r_i01 = i01 + r; // linear within block since we split at ne01\n\n const int32_t idx = *(int32_t *)((char *)src2->data + r_i01 * src2->nb[0] + i02 * src2->nb[1]);\n\n uint8_t * r_src1 = (uint8_t *)src1->data + idx * nb11;\n uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned;\n uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned;\n\n hvx_add_f32_aau(r_dst, r_src0, r_src1, ne00);\n }\n\n uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;\n dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, ne00 * sizeof(float), current_block_size);\n\n if (ir_prefetch < end_row) {\n uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);\n uint32_t p03, p02, p01, prem;\n p03 = fastdiv(ir_prefetch, &bctx->dim12_div);\n prem = ir_prefetch - p03 * (ne02 * ne01);\n p02 = fastdiv(prem, &bctx->dim1_div);\n p01 = prem - p02 * ne01;\n uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;\n dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, ne00 * sizeof(float), next_block_size);\n ir_prefetch += next_block_size;\n }\n ir += current_block_size;\n }\n dma_queue_flush(q);\n}\n\nstatic int execute_op_binary(struct htp_ops_context * octx) {\n const struct htp_tensor * src0 = &octx->src0;\n const struct htp_tensor * src1 = &octx->src1;\n struct htp_tensor * dst = &octx->dst;\n\n const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];\n const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);\n\n // Use packed row sizes for VTCM allocation\n const uint32_t src0_type = octx->src0.type;\n const size_t elem_size = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);\n const size_t src0_row_size = src0->ne[0] * elem_size;\n const size_t src1_row_size = src1->ne[0] * elem_size;\n const size_t dst_row_size = dst->ne[0] * elem_size;\n\n // Align to VLEN\n const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);\n const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);\n size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);\n\n bool is_add_id = (octx->op == HTP_OP_ADD_ID);\n bool is_scalar = !is_add_id && (src1->ne[0] == 1);\n\n // Determine which kernel we will use to alloc memory and dispatch\n bool use_vector_same = !is_add_id && !is_scalar && ((src0->nb[1] % VLEN) == 0) && (src1->ne[0] == src0->ne[0]) &&\n (src1->ne[1] == src0->ne[1] || src1->ne[1] == 1) &&\n (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) &&\n (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1);\n\n bool is_row_bcast = use_vector_same && (src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1);\n bool use_complex = !is_add_id && !is_scalar && !use_vector_same && (src1->ne[0] == src0->ne[0]);\n bool use_repeat = !is_add_id && !is_scalar && !use_vector_same && (src1->ne[0] != src0->ne[0]);\n\n size_t spad_row_total;\n if (is_scalar) {\n spad_row_total = 2 * (src0_row_size_aligned + dst_row_size_aligned);\n } else if (is_row_bcast) {\n spad_row_total = 2 * (src0_row_size_aligned + dst_row_size_aligned);\n } else if (use_vector_same) {\n spad_row_total = 2 * (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned);\n } else if (is_add_id) {\n spad_row_total = 2 * (src0_row_size_aligned + dst_row_size_aligned); // src1 read directly\n } else {\n spad_row_total = 2 * (src0_row_size_aligned + dst_row_size_aligned);\n }\n\n size_t rows_per_buffer = octx->ctx->vtcm_size / (n_threads * spad_row_total);\n // Adjust for static src1 in row_bcast case\n if (is_row_bcast) {\n size_t needed_static = src1_row_size_aligned;\n if (octx->ctx->vtcm_size < needed_static) return HTP_STATUS_VTCM_TOO_SMALL;\n size_t avail = octx->ctx->vtcm_size - needed_static;\n rows_per_buffer = avail / (n_threads * spad_row_total);\n }\n\n if (rows_per_buffer < 1) {\n FARF(ERROR, \"binary: VTCM too small\\n\");\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n octx->src0_spad.size_per_thread = rows_per_buffer * 2 * src0_row_size_aligned;\n octx->dst_spad.size_per_thread = rows_per_buffer * 2 * dst_row_size_aligned;\n\n if (is_scalar || use_complex || use_repeat || is_add_id) {\n octx->src1_spad.size_per_thread = 0;\n } else if (is_row_bcast) {\n octx->src1_spad.size_per_thread = 0;\n } else {\n octx->src1_spad.size_per_thread = rows_per_buffer * 2 * src1_row_size_aligned;\n }\n\n octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;\n if (is_row_bcast) {\n octx->src1_spad.size = src1_row_size_aligned;\n } else {\n octx->src1_spad.size = n_threads * octx->src1_spad.size_per_thread;\n }\n octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;\n\n if (octx->ctx->vtcm_size < (octx->src0_spad.size + octx->src1_spad.size + octx->dst_spad.size)) {\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n octx->src0_spad.data = octx->ctx->vtcm_base;\n octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;\n octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size;\n\n if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {\n return HTP_STATUS_OK;\n }\n\n dma_queue * q = octx->ctx->dma[0];\n if (is_row_bcast) {\n dma_queue_push(q, dma_make_ptr(octx->src1_spad.data, (const void *) src1->data), src1_row_size_aligned, 0, src1->ne[0] * elem_size, 1);\n }\n\n struct htp_binary_context bctx;\n bctx.octx = octx;\n bctx.nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;\n bctx.block_max = rows_per_buffer;\n bctx.src0_row_size_aligned = src0_row_size_aligned;\n bctx.src1_row_size_aligned = src1_row_size_aligned;\n bctx.dst_row_size_aligned = dst_row_size_aligned;\n\n bctx.dim1_div = init_fastdiv_values(src0->ne[1]);\n bctx.dim2_div = init_fastdiv_values(src0->ne[2]);\n bctx.dim12_div = init_fastdiv_values(src0->ne[1] * src0->ne[2]);\n\n bctx.src1_dim1_div = init_fastdiv_values(src1->ne[1]);\n bctx.src1_dim2_div = init_fastdiv_values(src1->ne[2]);\n bctx.src1_dim3_div = init_fastdiv_values(src1->ne[3]);\n\n bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);\n bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);\n\n bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);\n bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);\n\n bctx.split_at_ne01 = (src0->ne[2] > 1) &&\n ((src1->ne[1] > 1) || (src1->ne[2] > 1) || !src0_contig_dim1 || !dst_contig_dim1);\n\n bctx.split_at_ne02 = (src0->ne[3] > 1) &&\n ((src1->ne[2] > 1) || (src1->ne[3] > 1) || !src0_contig_dim2 || !dst_contig_dim2);\n\n // Precompute specific kernel parameters\n if (use_vector_same) {\n bctx.src1_dma_stride = (src1->ne[1] == 1) ? 0 : src1->nb[1];\n bctx.src1_fetch_rows = (src1->ne[1] == 1) ? 1 : rows_per_buffer;\n }\n\n worker_callback_t worker_func;\n if (is_add_id) worker_func = binary_job_add_id;\n else if (is_scalar) worker_func = binary_job_scalar;\n else if (is_row_bcast) worker_func = binary_job_vector_row_broadcast;\n else if (use_vector_same) worker_func = binary_job_vector_same_shape;\n else if (use_complex) worker_func = binary_job_vector_complex;\n else worker_func = binary_job_element_repeat;\n\n if (is_row_bcast) {\n dma_queue_pop(q);\n }\n\n worker_pool_run_func(octx->ctx->worker_pool, worker_func, &bctx, n_threads);\n\n return HTP_STATUS_OK;\n}\n\nint op_binary(struct htp_ops_context * octx) {\n\n // Does not support permutations of src1\n const struct htp_tensor * src1 = &octx->src1;\n if (src1->nb[1] < src1->nb[0]) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n const uint32_t src0_type = octx->src0.type;\n if ((src0_type == HTP_TYPE_F32) || (src0_type == HTP_TYPE_F16)) {\n return execute_op_binary(octx);\n }\n\n return HTP_STATUS_NO_SUPPORT;\n}\n\n" }, { "path": "src/ggml-hexagon/htp/cmake-toolchain.cmake", "content": "if (HEXAGON_TOOLCHAIN_INCLUDED)\n return()\nendif()\nset(HEXAGON_TOOLCHAIN_INCLUDED true)\n\n#Cross Compiling for Hexagon\nset(HEXAGON TRUE)\nset(CMAKE_SYSTEM_NAME QURT)\nset(CMAKE_SYSTEM_PROCESSOR Hexagon)\nset(CMAKE_SYSTEM_VERSION \"1\") #${HEXAGON_PLATFORM_LEVEL})\nset(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)\nset(CUSTOM_RUNELF_PATH \"\")\n\n#To fix backward compatibility with EAI addon.\nif (NOT HEXAGON_SDK_ROOT)\n set(HEXAGON_SDK_ROOT $ENV{HEXAGON_SDK_ROOT})\nendif()\n\nif (NOT HEXAGON_TOOLS_ROOT)\n if (DEFINED ENV{HEXAGON_TOOLS_ROOT})\n set(HEXAGON_TOOLS_ROOT $ENV{HEXAGON_TOOLS_ROOT})\n endif()\n if(NOT HEXAGON_TOOLS_ROOT)\n set(HEXAGON_TOOLS_ROOT $ENV{DEFAULT_HEXAGON_TOOLS_ROOT})\n endif()\nendif()\n\nfile(TO_CMAKE_PATH \"${HEXAGON_TOOLS_ROOT}\" HEXAGON_TOOLS_ROOT)\nfile(TO_CMAKE_PATH \"${HEXAGON_SDK_ROOT}\" HEXAGON_SDK_ROOT)\n\n#Get the Binary extension of the Hexagon Toolchain\nif(CMAKE_HOST_SYSTEM_NAME STREQUAL Windows)\n set(HEXAGON_TOOLCHAIN_SUFFIX .exe)\nendif()\nmessage(DEBUG \"CMAKE_HOST_SYSTEM_NAME:${CMAKE_HOST_SYSTEM_NAME}\")\n\ninclude(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_arch.cmake)\n\nset(HEXAGON_TOOLCHAIN ${HEXAGON_TOOLS_ROOT})\nset(HEXAGON_LIB_DIR \"${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib\")\nset(HEXAGON_ISS_DIR ${HEXAGON_TOOLCHAIN}/Tools/lib/iss)\n\nset(CMAKE_TRY_COMPILE_PLATFORM_VARIABLES\n HEXAGON_SDK_ROOT\n HEXAGON_TOOLS_ROOT\n)\n\n#QURT Related includes and linker flags\nset(V_ARCH ${HEXAGON_ARCH})\nset(_QURT_INSTALL_DIR \"${HEXAGON_SDK_ROOT}/rtos/qurt/ADSP${V_ARCH}MP${V_ARCH_EXTN}\")\nset(_QURT_INSTALL_DIR \"${HEXAGON_SDK_ROOT}/rtos/qurt/compute${V_ARCH}${V_ARCH_EXTN}\")\n\nif( ${TREE} MATCHES PAKMAN )\n set(_QURT_INSTALL_DIR \"${QURT_IMAGE_DIR}/compute${V_ARCH}${V_ARCH_EXTN}\")\nendif()\nmessage(DEBUG \"_QURT_INSTALL_DIR:${_QURT_INSTALL_DIR}\")\nset(RTOS_DIR ${_QURT_INSTALL_DIR})\nset(QCC_DIR \"${HEXAGON_QCC_DIR}/${V_ARCH}/G0\")\nset(TARGET_DIR \"${HEXAGON_LIB_DIR}/${V_ARCH}/G0\")\n\ninclude_directories(\n ${_QURT_INSTALL_DIR}/include\n ${_QURT_INSTALL_DIR}/include/qurt\n ${_QURT_INSTALL_DIR}/include/posix\n )\n\nset(QURT_START_LINK_LIBS)\nset(QURT_START_LINK_LIBS\n \"${TARGET_DIR}/init.o\"\n \"${RTOS_DIR}/lib/crt1.o\"\n \"${RTOS_DIR}/lib/debugmon.o\"\n \"${RTOS_DIR}/lib/libqurt.a\"\n \"${TARGET_DIR}/libc.a\"\n \"${TARGET_DIR}/libqcc.a\"\n \"${TARGET_DIR}/libhexagon.a\"\n \"${RTOS_DIR}/lib/libqurtcfs.a\"\n \"${RTOS_DIR}/lib/libtimer_island.a\"\n \"${RTOS_DIR}/lib/libtimer_main.a\"\n \"${RTOS_DIR}/lib/libposix.a\"\n )\nSTRING(REPLACE \";\" \" \" QURT_START_LINK_LIBS \"${QURT_START_LINK_LIBS}\")\n\nset(QURT_END_LINK_LIBS\n ${TARGET_DIR}/fini.o\n )\n\n#Non QURT related includes and linker flags\n\nset(TARGET_DIR_NOOS \"${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib/${HEXAGON_ARCH}\")\n\nif (NOT NO_WRAP_MEM_API)\n set(WRAP_MALLOC -Wl,--wrap=malloc)\n set(WRAP_CALLOC -Wl,--wrap=calloc)\n set(WRAP_FREE -Wl,--wrap=free)\n set(WRAP_REALLOC -Wl,--wrap=realloc)\n set(WRAP_MEMALIGN -Wl,--wrap=memalign)\nendif()\n\nset(PIC_SHARED_LD_FLAGS\n -mcpu=${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH}\n -G0\n -fpic\n -Wl,-Bsymbolic\n -Wl,-L${TARGET_DIR_NOOS}/G0/pic\n -Wl,-L${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib/\n -Wl,--no-threads ${WRAP_MALLOC} ${WRAP_CALLOC} ${WRAP_FREE} ${WRAP_REALLOC} ${WRAP_MEMALIGN}\n -shared\n \"-o \"\n \"\"\n -Wl,--start-group\n \"\"\n \"\"\n -Wl,--end-group\n -lc\n )\nSTRING(REPLACE \";\" \" \" PIC_SHARED_LD_FLAGS \"${PIC_SHARED_LD_FLAGS}\")\n\nset(HEXAGON_PIC_SHARED_LINK_OPTIONS \"${PIC_SHARED_LD_FLAGS}\")\n\n#System include paths\ninclude_directories(SYSTEM ${HEXAGON_SDK_ROOT}/incs)\ninclude_directories(SYSTEM ${HEXAGON_SDK_ROOT}/incs/stddef)\ninclude_directories(SYSTEM ${HEXAGON_SDK_ROOT}/ipc/fastrpc/incs)\n\n#LLVM toolchain setup\n#Compiler paths, options and architecture\nset(CMAKE_C_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang${HEXAGON_TOOLCHAIN_SUFFIX})\nset(CMAKE_CXX_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang++${HEXAGON_TOOLCHAIN_SUFFIX})\nset(CMAKE_AR ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-ar${HEXAGON_TOOLCHAIN_SUFFIX})\nset(CMAKE_ASM_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang++${HEXAGON_TOOLCHAIN_SUFFIX})\nset(HEXAGON_LINKER ${CMAKE_C_COMPILER})\nset(CMAKE_PREFIX_PATH ${HEXAGON_TOOLCHAIN}/Tools/target/hexagon)\n\nset(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG \"-Wl,-soname,\")\nset(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG \"-Wl,-soname,\")\n\n#Compiler Options\nset(COMMON_FLAGS \"-mcpu=hexagon${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH} -fvectorize -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}\")\n\nset(CMAKE_CXX_FLAGS_DEBUG \"${COMMON_FLAGS} -O0 -D_DEBUG -g\")\nset(CMAKE_CXX_FLAGS_RELWITHDEBINFO \"${COMMON_FLAGS} -O3 -g\")\nset(CMAKE_CXX_FLAGS_RELEASE \"${COMMON_FLAGS} -O3\")\n\nset(CMAKE_C_FLAGS_DEBUG \"${COMMON_FLAGS} -O0 -D_DEBUG -g\")\nset(CMAKE_C_FLAGS_RELWITHDEBINFO \"${COMMON_FLAGS} -O3 -g\")\nset(CMAKE_C_FLAGS_RELEASE \"${COMMON_FLAGS} -O3\")\n\nset(CMAKE_ASM_FLAGS_DEBUG \"${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_DEBUG}\")\nset(CMAKE_ASM_FLAGS_RELEASE \"${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE}\")\nset(CMAKE_ASM_FLAGS_RELWITHDEBINFO \"${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}\" )\n\n#Linker Options\nset(CMAKE_C_CREATE_SHARED_LIBRARY \"${HEXAGON_LINKER} ${HEXAGON_PIC_SHARED_LINK_OPTIONS}\")\nset(CMAKE_CXX_CREATE_SHARED_LIBRARY \"${HEXAGON_LINKER} ${HEXAGON_PIC_SHARED_LINK_OPTIONS}\")\n" }, { "path": "src/ggml-hexagon/htp/cpy-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n#include \"hvx-utils.h\"\n\nstruct htp_copy_context {\n struct htp_ops_context * octx;\n\n uint32_t src0_type_size;\n uint32_t src0_block_size;\n\n uint32_t dst_type_size;\n uint32_t dst_block_size;\n\n uint32_t src0_blocks_per_row;\n uint32_t dst_blocks_per_row;\n\n uint32_t src0_nrows_per_thread;\n\n void (*copy)(struct htp_copy_context * ct, struct htp_ops_context * octx, int nth, int ith);\n};\n\n#define cpy_preamble \\\n struct htp_tensor *src0 = &octx->src0; \\\n struct htp_tensor *dst = &octx->dst; \\\n \\\n const uint32_t ne00 = src0->ne[0]; \\\n const uint32_t ne01 = src0->ne[1]; \\\n const uint32_t ne02 = src0->ne[2]; \\\n const uint32_t ne03 = src0->ne[3]; \\\n \\\n const uint32_t nb00 = src0->nb[0]; \\\n const uint32_t nb01 = src0->nb[1]; \\\n const uint32_t nb02 = src0->nb[2]; \\\n const uint32_t nb03 = src0->nb[3]; \\\n \\\n const uint32_t ne0 = dst->ne[0]; \\\n const uint32_t ne1 = dst->ne[1]; \\\n const uint32_t ne2 = dst->ne[2]; \\\n const uint32_t ne3 = dst->ne[3]; \\\n \\\n const uint32_t nb0 = dst->nb[0]; \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3]; \\\n \\\n const uint32_t nr = ne01;\n\nstatic void cpy_thread_sametype_sameshape(struct htp_copy_context * ct, struct htp_ops_context * octx, const int nth, const int ith) {\n cpy_preamble;\n\n // parallelize by src0 rows\n const uint32_t dr = ct->src0_nrows_per_thread;\n const uint32_t ir0 = dr * ith;\n const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr;\n\n // copy by rows\n for (uint32_t i03 = 0; i03 < ne03; i03++) {\n for (uint32_t i02 = 0; i02 < ne02; i02++) {\n #pragma unroll(2)\n for (uint32_t i01 = ir0; i01 < ir1; i01++) {\n uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;\n uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;\n hex_l2fetch(src0_ptr, ne00 * ct->src0_type_size, nb01, 2);\n hvx_copy_uu(dst_ptr, src0_ptr, ne00, ct->src0_type_size);\n }\n }\n }\n}\n\nstatic void cpy_thread_sametype_reshape(struct htp_copy_context * ct, struct htp_ops_context * octx, int nth, int ith) {\n cpy_preamble;\n\n // parallelize by src0 rows\n const uint32_t dr = ct->src0_nrows_per_thread;\n const uint32_t ir0 = dr * ith;\n const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr;\n\n // dst counters\n int64_t k10 = 0;\n int64_t i11 = 0;\n int64_t i12 = 0;\n int64_t i13 = 0;\n\n // number of blocks in a row\n const int64_t nk00 = ct->src0_blocks_per_row;\n const int64_t nk0 = ct->dst_blocks_per_row;\n\n for (int64_t i03 = 0; i03 < ne03; i03++) {\n for (int64_t i02 = 0; i02 < ne02; i02++) {\n k10 += nk00 * ir0;\n while (k10 >= nk0) {\n k10 -= nk0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n for (int64_t i01 = ir0; i01 < ir1; i01++) {\n for (int64_t k00 = 0; k00 < nk00; k00++) {\n const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n char * dst_ptr = ((char *) dst->data + k10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);\n memcpy(dst_ptr, src0_ptr, ct->dst_type_size);\n\n if (++k10 == nk0) {\n k10 = 0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n }\n }\n k10 += nk00 * (ne01 - ir1);\n while (k10 >= nk0) {\n k10 -= nk0;\n if (++i11 == ne1) {\n i11 = 0;\n if (++i12 == ne2) {\n i12 = 0;\n if (++i13 == ne3) {\n i13 = 0;\n }\n }\n }\n }\n }\n }\n}\n\nstatic void cpy_thread_f16_f32_sameshape(struct htp_copy_context * ct, struct htp_ops_context * octx, const int nth, const int ith) {\n cpy_preamble;\n\n // parallelize by src0 rows\n const uint32_t dr = ct->src0_nrows_per_thread;\n const uint32_t ir0 = dr * ith;\n const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr;\n\n // copy by rows\n for (uint32_t i03 = 0; i03 < ne03; i03++) {\n for (uint32_t i02 = 0; i02 < ne02; i02++) {\n #pragma unroll(2)\n for (uint32_t i01 = ir0; i01 < ir1; i01++) {\n uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;\n uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;\n hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2);\n hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00);\n }\n }\n }\n}\n\nstatic void cpy_thread_f32_f16_sameshape(struct htp_copy_context * ct, struct htp_ops_context * octx, const int nth, const int ith) {\n cpy_preamble;\n\n // parallelize by src0 rows\n const uint32_t dr = ct->src0_nrows_per_thread;\n const uint32_t ir0 = dr * ith;\n const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr;\n\n // copy by rows\n for (uint32_t i03 = 0; i03 < ne03; i03++) {\n for (uint32_t i02 = 0; i02 < ne02; i02++) {\n #pragma unroll(2)\n for (uint32_t i01 = ir0; i01 < ir1; i01++) {\n uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;\n uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;\n hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2);\n hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00);\n }\n }\n }\n}\n\nstatic void cpy_work_func(unsigned int n, unsigned int i, void *data) {\n struct htp_copy_context *ct = (struct htp_copy_context *) data;\n ct->copy(ct, ct->octx, n, i);\n}\n\nint op_cpy(struct htp_ops_context * octx) {\n cpy_preamble;\n\n const uint32_t n_threads = MIN(nr, octx->n_threads);\n\n struct htp_copy_context ct;\n ct.octx = octx;\n\n switch (src0->type) {\n case HTP_TYPE_F32: ct.src0_type_size = 4; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break;\n case HTP_TYPE_F16: ct.src0_type_size = 2; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break;\n default:\n return HTP_STATUS_NO_SUPPORT;\n }\n\n switch (dst->type) {\n case HTP_TYPE_F32: ct.dst_type_size = 4; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break;\n case HTP_TYPE_F16: ct.dst_type_size = 2; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break;\n default:\n return HTP_STATUS_NO_SUPPORT;\n }\n\n if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {\n return HTP_STATUS_OK;\n }\n\n const bool sametype = (src0->type == dst->type);\n const bool transposed = (nb00 > nb01) || (nb0 > nb1);\n const bool sameshape = !transposed && (ne00 == ne0 && ne01 == ne1 && ne02 == ne2 && ne03 == ne3);\n\n ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads;\n\n if (sametype && sameshape) {\n ct.copy = cpy_thread_sametype_sameshape;\n } else if (sameshape) {\n /**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32)\n ct.copy = cpy_thread_f16_f32_sameshape;\n else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16)\n ct.copy = cpy_thread_f32_f16_sameshape;\n else\n return HTP_STATUS_NO_SUPPORT;\n } else if (sametype) {\n ct.copy = cpy_thread_sametype_reshape;\n } else {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n worker_pool_run_func(octx->ctx->worker_pool, cpy_work_func, &ct, n_threads);\n\n return HTP_STATUS_OK;\n}\n" }, { "path": "src/ggml-hexagon/htp/flash-attn-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n#include \n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hvx-utils.h\"\n#include \"hvx-dump.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\n// Must be multiple of 32\n#define FLASH_ATTN_BLOCK_SIZE (32 * 2)\n\n// This is a bit of a hack because the compiler is strugling to properly inline\n// the default hvx_vec_f32_to_f16 with output into the local array.\nstatic void __attribute__((noinline)) hvx_vec_f32_to_f16_a(void *ptr, HVX_Vector v0, HVX_Vector v1)\n{\n *(HVX_Vector *) ptr = hvx_vec_f32_to_f16(v0, v1);\n}\n\n// Dot product of two F16 vectors, accumulating to float\nstatic inline void hvx_dot_f16_f16_aa(float * restrict r, const void * restrict x, const void * restrict y, unsigned int n, float s) {\n const HVX_Vector * restrict vx = (const HVX_Vector * restrict) x; // fp16\n const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16\n\n uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors\n uint32_t nloe = n % VLEN_FP16; // leftover elements\n\n HVX_VectorPair rsum_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));\n\n uint32_t i = 0;\n\n #pragma unroll(4)\n for (i = 0; i < nvec; i++) {\n rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, vx[i], vy[i]);\n }\n\n if (nloe) {\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);\n HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]);\n HVX_Vector x_hf = Q6_V_vand_QV(bmask, vx[i]);\n\n rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf);\n }\n\n HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p)));\n rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(hvx_vec_splat_f32(s), hvx_vec_reduce_sum_f32(rsum)));\n hvx_vec_store_u(r, 4, rsum);\n}\n\nstatic inline HVX_Vector hvx_dot_f16_f16_aa_rx4(const void * restrict y,\n const uint8_t * restrict x,\n const size_t stride_x,\n const size_t nvec,\n const size_t nloe) {\n const HVX_Vector * restrict vx0 = (const HVX_Vector * restrict) x; // fp16\n const HVX_Vector * restrict vx1 = (const HVX_Vector * restrict) (x + stride_x); // fp16\n const HVX_Vector * restrict vx2 = (const HVX_Vector * restrict) (x + stride_x * 2); // fp16\n const HVX_Vector * restrict vx3 = (const HVX_Vector * restrict) (x + stride_x * 3); // fp16\n const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16\n\n HVX_VectorPair rsum0_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));\n HVX_VectorPair rsum1_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));\n HVX_VectorPair rsum2_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));\n HVX_VectorPair rsum3_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));\n\n uint32_t i = 0;\n\n for (i = 0; i < nvec; i++) {\n HVX_Vector y_hf = vy[i];\n HVX_Vector x0_hf = vx0[i];\n HVX_Vector x1_hf = vx1[i];\n HVX_Vector x2_hf = vx2[i];\n HVX_Vector x3_hf = vx3[i];\n\n rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf);\n rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf);\n rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf);\n rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf);\n }\n\n if (nloe) {\n // Load x (fp16) and zero-out unused elements\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);\n HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]);\n HVX_Vector x0_hf = Q6_V_vand_QV(bmask, vx0[i]);\n HVX_Vector x1_hf = Q6_V_vand_QV(bmask, vx1[i]);\n HVX_Vector x2_hf = Q6_V_vand_QV(bmask, vx2[i]);\n HVX_Vector x3_hf = Q6_V_vand_QV(bmask, vx3[i]);\n\n rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf);\n rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf);\n rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf);\n rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf);\n }\n\n HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p)));\n HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p)));\n HVX_Vector rsum2 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum2_p), Q6_V_hi_W(rsum2_p)));\n HVX_Vector rsum3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum3_p), Q6_V_hi_W(rsum3_p)));\n\n HVX_Vector_x4 rsum0123 = { .v = { rsum0, rsum1, rsum2, rsum3 } };\n return hvx_vec_reduce_sum_f32x4(rsum0123);\n}\n\nstatic inline HVX_Vector hvx_dot_f16_f16_aa_rx32(const void * restrict y,\n const uint8_t * restrict x,\n const size_t stride_x,\n const size_t n,\n float s) {\n\n const size_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors\n const size_t nloe = n % VLEN_FP16; // leftover elements\n\n HVX_Vector sums; // initialize at j = 0\n const size_t stride_x_4 = stride_x * 4;\n for (uint32_t j = 0; j < VLEN_FP32; j += 4) {\n HVX_Vector sums_x4 = hvx_dot_f16_f16_aa_rx4(y, x, stride_x, nvec, nloe);\n HVX_VectorPred pred = Q6_Q_vsetq_R(j * SIZEOF_FP32);\n sums = Q6_V_vmux_QVV(pred, sums, sums_x4);\n x += stride_x_4;\n }\n\n sums = Q6_Vqf32_vmpy_VsfVsf(hvx_vec_splat_f32(s), sums);\n return Q6_Vsf_equals_Vqf32(sums);\n}\n\n// MAD: y (F32) += x (F16) * s (F16)\nstatic inline void hvx_mad_f32_f16_aa(float * restrict y, const void * restrict x, const __fp16 * restrict s, int n) {\n const HVX_Vector * restrict vx0 = (const HVX_Vector *) x;\n\n HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y;\n HVX_Vector * restrict vy = (HVX_Vector *) y;\n\n uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors\n uint32_t nloe = n % VLEN_FP16; // leftover elements\n\n HVX_Vector S0 = hvx_vec_splat_f16(*s);\n\n uint32_t i = 0;\n\n #pragma unroll(2)\n for (i = 0; i < nvec; ++i) {\n vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0);\n }\n\n if (nloe) {\n HVX_VectorPair xy_p = vy_p[i];\n xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0);\n\n HVX_Vector xy = Q6_V_lo_W(xy_p);\n i = 2 * i; // index for vy\n\n if (nloe >= VLEN_FP32) {\n vy[i] = xy;\n nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p);\n }\n\n if (nloe) {\n hvx_vec_store_a(&vy[i], nloe * 4, xy);\n }\n }\n}\n\n// MAD: y (F32) += x0 (F16) * s0 (F16) + x1 (F16) * s1 (F16)\nstatic inline void hvx_mad_f32_f16_aa_rx2(float * restrict y, const void * restrict x0, const void * restrict x1,\n const __fp16 * restrict s0, const __fp16 * restrict s1, int n) {\n const HVX_Vector * restrict vx0 = (const HVX_Vector *) x0;\n const HVX_Vector * restrict vx1 = (const HVX_Vector *) x1;\n\n HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y;\n HVX_Vector * restrict vy = (HVX_Vector *) y;\n\n uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors\n uint32_t nloe = n % VLEN_FP16; // leftover elements\n\n HVX_Vector S0 = hvx_vec_splat_f16(*s0);\n HVX_Vector S1 = hvx_vec_splat_f16(*s1);\n\n uint32_t i = 0;\n\n #pragma unroll(2)\n for (i = 0; i < nvec; ++i) {\n vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0);\n vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx1[i]), S1);\n }\n\n if (nloe) {\n HVX_VectorPair xy_p = vy_p[i];\n xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0);\n xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx1[i]), S1);\n\n HVX_Vector xy = Q6_V_lo_W(xy_p);\n i = 2 * i; // index for vy\n\n if (nloe >= VLEN_FP32) {\n vy[i] = xy;\n nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p);\n }\n\n if (nloe) {\n hvx_vec_store_a(&vy[i], nloe * 4, xy);\n }\n }\n}\n\nstruct htp_fa_context {\n const struct htp_ops_context * octx;\n\n struct fastdiv_values src0_div21;\n struct fastdiv_values src0_div1;\n\n struct fastdiv_values broadcast_rk2;\n struct fastdiv_values broadcast_rk3;\n struct fastdiv_values broadcast_rv2;\n struct fastdiv_values broadcast_rv3;\n\n struct fastdiv_values src3_div2;\n struct fastdiv_values src3_div3;\n\n float scale;\n float max_bias;\n float logit_softcap;\n\n uint32_t n_head_log2;\n float m0;\n float m1;\n\n uint32_t n_blocks;\n\n size_t size_q_row_padded;\n size_t size_k_row_padded;\n size_t size_v_row_padded;\n\n size_t size_k_block;\n size_t size_v_block;\n size_t size_m_block;\n\n uint32_t qrows;\n uint32_t qrows_per_thread;\n\n bool is_q_fp32;\n\n uint64_t t_start;\n};\n\nstatic inline void hvx_scale_vec_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, HVX_Vector vs) {\n assert((size_t) dst % 128 == 0);\n assert((size_t) src % 128 == 0);\n\n const HVX_Vector * restrict vsrc = (const HVX_Vector * restrict) src;\n HVX_Vector * restrict vdst = (HVX_Vector * restrict) dst;\n\n const uint32_t nvec = n / VLEN_FP32;\n const uint32_t nloe = n % VLEN_FP32;\n\n uint32_t i = 0;\n #pragma unroll(4)\n for (; i < nvec; ++i) {\n vdst[i] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs));\n }\n if (nloe) {\n HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs);\n hvx_vec_store_a(&vdst[i], nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v));\n }\n}\n\nstatic void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) {\n struct htp_fa_context * factx = (struct htp_fa_context *) data;\n const struct htp_ops_context * octx = factx->octx;\n const struct htp_tensor * q = &octx->src0;\n const struct htp_tensor * k = &octx->src1;\n const struct htp_tensor * v = &octx->src2;\n const struct htp_tensor * mask = (octx->src3.data) ? &octx->src3 : NULL;\n const struct htp_tensor * sinks = (octx->src4.data) ? &octx->src4 : NULL;\n const struct htp_tensor * dst = &octx->dst;\n\n const uint32_t neq0 = q->ne[0];\n const uint32_t neq1 = q->ne[1];\n const uint32_t neq2 = q->ne[2];\n const uint32_t neq3 = q->ne[3];\n\n const uint32_t nek0 = k->ne[0];\n const uint32_t nek1 = k->ne[1];\n const uint32_t nek2 = k->ne[2];\n const uint32_t nek3 = k->ne[3];\n\n const uint32_t nev0 = v->ne[0];\n const uint32_t nev1 = v->ne[1];\n const uint32_t nev2 = v->ne[2];\n const uint32_t nev3 = v->ne[3];\n\n const uint32_t nbq1 = q->nb[1];\n const uint32_t nbq2 = q->nb[2];\n const uint32_t nbq3 = q->nb[3];\n\n const uint32_t nbk1 = k->nb[1];\n const uint32_t nbk2 = k->nb[2];\n const uint32_t nbk3 = k->nb[3];\n\n const uint32_t nbv1 = v->nb[1];\n const uint32_t nbv2 = v->nb[2];\n const uint32_t nbv3 = v->nb[3];\n\n const uint32_t ne1 = dst->ne[1];\n const uint32_t ne2 = dst->ne[2];\n const uint32_t ne3 = dst->ne[3];\n\n const uint32_t nb1 = dst->nb[1];\n const uint32_t nb2 = dst->nb[2];\n const uint32_t nb3 = dst->nb[3];\n\n // total rows in q\n const uint32_t nr = factx->qrows;\n const uint32_t dr = factx->qrows_per_thread;\n const uint32_t ir0 = dr * ith;\n const uint32_t ir1 = MIN(ir0 + dr, nr);\n\n if (ir0 >= ir1) return;\n\n dma_queue * dma = octx->ctx->dma[ith];\n\n const uint32_t DK = nek0;\n const uint32_t DV = nev0;\n\n const size_t size_q_row = DK * ((q->type == HTP_TYPE_F32) ? 4 : 2);\n const size_t size_k_row = DK * sizeof(__fp16);\n const size_t size_v_row = DV * sizeof(__fp16);\n\n // Scratchpad buffers for Q, K, V, Mask, and VKQ32 accumulator\n uint8_t * spad_q = octx->src0_spad.data + octx->src0_spad.size_per_thread * ith;\n uint8_t * spad_k = octx->src1_spad.data + octx->src1_spad.size_per_thread * ith;\n uint8_t * spad_v = octx->src2_spad.data + octx->src2_spad.size_per_thread * ith;\n uint8_t * spad_m = octx->src3_spad.data + octx->src3_spad.size_per_thread * ith;\n uint8_t * spad_a = octx->dst_spad.data + octx->dst_spad.size_per_thread * ith;\n\n const HVX_Vector logit_cap = hvx_vec_splat_f32(factx->logit_softcap);\n\n for (uint32_t ir = ir0; ir < ir1; ++ir) {\n const uint32_t iq3 = fastdiv(ir, &factx->src0_div21);\n const uint32_t iq2 = fastdiv(ir - iq3*neq2*neq1, &factx->src0_div1);\n const uint32_t iq1 = (ir - iq3*neq2*neq1 - iq2 * neq1);\n\n const uint32_t ik3 = fastdiv(iq3, &factx->broadcast_rk3);\n const uint32_t ik2 = fastdiv(iq2, &factx->broadcast_rk2);\n\n const uint32_t iv3 = fastdiv(iq3, &factx->broadcast_rv3);\n const uint32_t iv2 = fastdiv(iq2, &factx->broadcast_rv2);\n\n // Fetch Q row\n const uint8_t * q_row_ptr = (const uint8_t *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3);\n dma_queue_push(dma, dma_make_ptr(spad_q, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1);\n\n // FARF(HIGH, \"fa %u: prefetch Q: ir %u iq1 %u iq2 %u iq3 %u q_row_ptr %p size %u : usec %u\", ith, ir, iq1, iq2, iq3, q_row_ptr, size_q_row,\n // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start));\n\n const __fp16 * mp_base = NULL;\n if (mask) {\n const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2);\n const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3);\n mp_base = (const __fp16 *) ((const uint8_t *) mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3]);\n }\n\n // Prefetch first two blocks\n for (uint32_t ib = 0; ib < MIN(factx->n_blocks, 2); ++ib) {\n const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE;\n const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);\n\n // K\n const uint8_t * k_src = (const uint8_t *) k->data + (ic_start*nbk1 + ik2*nbk2 + ik3*nbk3);\n uint8_t * k_dst = spad_k + (ib % 2) * factx->size_k_block;\n dma_queue_push(dma, dma_make_ptr(k_dst, k_src), factx->size_k_row_padded, nbk1, size_k_row, current_block_size);\n\n // V\n const uint8_t * v_src = (const uint8_t *) v->data + (ic_start*nbv1 + iv2*nbv2 + iv3*nbv3);\n uint8_t * v_dst = spad_v + (ib % 2) * factx->size_v_block;\n dma_queue_push(dma, dma_make_ptr(v_dst, v_src), factx->size_v_row_padded, nbv1, size_v_row, current_block_size);\n\n // Mask\n if (mask) {\n const uint8_t * m_src = (const uint8_t *) (mp_base + ic_start);\n uint8_t * m_dst = spad_m + (ib % 2) * factx->size_m_block;\n // Mask is 1D contiguous for this row\n dma_queue_push(dma, dma_make_ptr(m_dst, m_src), current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);\n }\n\n // FARF(HIGH, \"fa %u: prefetch KVM: ir %u ib %u iq1 %u iq2 %u iq3 %u : size_k_row %u size_v_row %u bs %u: usec %u\",\n // ith, ir, ib, iq1, iq2, iq3,\n // size_k_row, size_v_row, current_block_size,\n // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start));\n }\n\n const uint32_t h = iq2; // head index\n const float slope = (factx->max_bias > 0.0f) ? (h < factx->n_head_log2 ? powf(factx->m0, h + 1) : powf(factx->m1, 2*(h - factx->n_head_log2) + 1)) : 1.0f;\n\n HVX_Vector S_vec = hvx_vec_splat_f32(0.0f);\n HVX_Vector M_vec = hvx_vec_splat_f32(-INFINITY);\n\n // Clear accumulator\n hvx_splat_f32_a(spad_a, 0, DV);\n float * VKQ32 = (float *) (spad_a + 0);\n\n uint8_t * q_ptr_vtcm = dma_queue_pop(dma).dst;\n if (factx->is_q_fp32) {\n hvx_copy_f16_f32_aa(q_ptr_vtcm, q_ptr_vtcm, DK); // inplace convert f32 to f16\n }\n\n const HVX_Vector slope_vec = hvx_vec_splat_f16(slope);\n for (uint32_t ib = 0; ib < factx->n_blocks; ++ib) {\n const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE;\n const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);\n\n // Wait for DMA\n uint8_t * k_base = dma_queue_pop(dma).dst; // K\n uint8_t * v_base = dma_queue_pop(dma).dst; // V\n __fp16 * m_base = mask ? dma_queue_pop(dma).dst : NULL; // M\n\n // FARF(HIGH, \"fa %u: process: ir %u ib %u : iq1 %u iq2 %u iq3 %u q_ptr_vtcm %p : usec %u\",\n // ith, ir, ib, iq1, iq2, iq3, q_ptr_vtcm,\n // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start));\n\n // Inner loop processing the block from VTCM\n uint32_t ic = 0;\n\n // Process in sub-blocks of 32 (VLEN_FP32)\n HVX_Vector sb_scores[FLASH_ATTN_BLOCK_SIZE / VLEN_FP32];\n HVX_Vector v_max = hvx_vec_splat_f32(-INFINITY);\n for (uint32_t iv = 0; ic + VLEN_FP32 <= current_block_size; ic += VLEN_FP32, ++iv) {\n // 1. Compute scores\n HVX_Vector scores = hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base + ic * factx->size_k_row_padded, factx->size_k_row_padded, DK, factx->scale);\n\n // 2. Softcap\n if (factx->logit_softcap != 0.0f) {\n scores = hvx_vec_tanh_f32(scores);\n scores = Q6_Vqf32_vmpy_VsfVsf(scores, logit_cap);\n scores = Q6_Vsf_equals_Vqf32(scores);\n }\n\n // 3. Mask\n if (mask) {\n const __fp16 * mp = m_base + ic;\n HVX_Vector m_vals_f16 = *(const HVX_UVector *) mp;\n HVX_VectorPair m_vals_f32_pair = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(m_vals_f16), slope_vec);\n HVX_Vector add_val = Q6_V_lo_W(m_vals_f32_pair);\n scores = Q6_Vqf32_vadd_Vqf32Vsf(add_val, scores);\n scores = Q6_Vsf_equals_Vqf32(scores);\n }\n\n sb_scores[iv] = scores;\n v_max = hvx_vec_reduce_max2_f32(scores, v_max); // All lanes have block max\n }\n\n {\n // 4. Online Softmax Update\n HVX_Vector M_new_vec = Q6_Vsf_vmax_VsfVsf(v_max, M_vec);\n HVX_Vector diff_vec = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(M_vec, M_new_vec));\n HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec);\n M_vec = M_new_vec;\n\n hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec);\n\n HVX_Vector p_sum_vec = hvx_vec_splat_f32(0.0f);\n for (uint32_t ic2 = 0, iv = 0; ic2 + VLEN_FP32 <= current_block_size; ic2 += VLEN_FP32, ++iv) {\n HVX_Vector scores = sb_scores[iv];\n HVX_Vector scores_shifted = Q6_Vqf32_vsub_VsfVsf(scores, M_vec);\n HVX_Vector P = hvx_vec_exp_f32(Q6_Vsf_equals_Vqf32(scores_shifted));\n\n p_sum_vec = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(p_sum_vec, P));\n\n // 5. Accumulate V\n __fp16 __attribute__((aligned(VLEN))) p_arr[VLEN_FP16];\n hvx_vec_f32_to_f16_a(p_arr, P, hvx_vec_splat_f32(0));\n\n for (uint32_t j = 0; j < VLEN_FP32; j += 2) {\n const uint32_t cur_ic = ic2 + j;\n const uint8_t * v_ptr = v_base + cur_ic * factx->size_v_row_padded;\n hvx_mad_f32_f16_aa_rx2(VKQ32, v_ptr, v_ptr + factx->size_v_row_padded, (p_arr + j), (p_arr + j + 1), DV);\n }\n }\n\n p_sum_vec = hvx_vec_reduce_sum_f32(p_sum_vec);\n S_vec = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(S_vec, ms_vec)), p_sum_vec));\n }\n\n if (ic < current_block_size) {\n // Sync scalars for leftover/next block if needed\n float M = hvx_vec_get_f32(M_vec);\n float S = hvx_vec_get_f32(S_vec);\n\n // Leftover\n for (; ic < current_block_size; ++ic) {\n float s_val;\n const uint8_t * k_ptr = k_base + ic * factx->size_k_row_padded;\n hvx_dot_f16_f16_aa(&s_val, q_ptr_vtcm, k_ptr, DK, factx->scale);\n if (factx->logit_softcap != 0.0f) {\n s_val = factx->logit_softcap * tanhf(s_val);\n }\n\n if (mask) {\n const float m_val = m_base[ic];\n s_val += slope * m_val;\n }\n\n const float Mold = M;\n __fp16 vs = 1.0f;\n\n if (s_val > M) {\n M = s_val;\n HVX_Vector diff_vec = hvx_vec_splat_f32(Mold - M);\n HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec);\n hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec);\n\n float ms = hvx_vec_get_f32(ms_vec);\n S = S * ms + vs;\n } else {\n HVX_Vector diff_vec = hvx_vec_splat_f32(s_val - M);\n vs = hvx_vec_get_f32(hvx_vec_exp_f32(diff_vec));\n S += vs;\n }\n\n const uint8_t * v_ptr = v_base + ic * factx->size_v_row_padded;\n\n hvx_mad_f32_f16_aa(VKQ32, v_ptr, &vs, DV);\n }\n\n M_vec = hvx_vec_splat_f32(M);\n S_vec = hvx_vec_splat_f32(S);\n }\n\n // Issue DMA for next+1 block (if exists)\n if (ib + 2 < factx->n_blocks) {\n const uint32_t next_ib = ib + 2;\n const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE;\n const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start);\n\n // K\n const uint8_t * k_src = (const uint8_t *) k->data + (next_ic_start*nbk1 + ik2*nbk2 + ik3*nbk3);\n dma_queue_push(dma, dma_make_ptr(k_base, k_src), factx->size_k_row_padded, nbk1, size_k_row, next_block_size);\n\n // V\n const uint8_t * v_src = (const uint8_t *) v->data + (next_ic_start*nbv1 + iv2*nbv2 + iv3*nbv3);\n dma_queue_push(dma, dma_make_ptr(v_base, v_src), factx->size_v_row_padded, nbv1, size_v_row, next_block_size);\n\n // Mask\n if (mask) {\n const uint8_t * m_src = (const uint8_t *) (mp_base + next_ic_start);\n dma_queue_push(dma, dma_make_ptr(m_base, m_src), next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);\n }\n\n // FARF(HIGH, \"fa %u: prefetch KVM: ir %u ib %u : iq1 %u iq2 %u iq3 %u : size_k_row %u size_v_row %u bs %u: usec %u\",\n // ith, ir, next_ib, iq1, iq2, iq3,\n // size_k_row, size_v_row, next_block_size,\n // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start));\n }\n }\n\n // sinks\n float M = hvx_vec_get_f32(M_vec);\n float S = hvx_vec_get_f32(S_vec);\n\n if (sinks) {\n const float s = ((float *)((char *) sinks->data))[h];\n\n float vs = 1.0f;\n\n if (s > M) {\n HVX_Vector diff_vec = hvx_vec_splat_f32(M - s);\n HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec);\n hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec);\n\n float ms = hvx_vec_get_f32(ms_vec);\n S = S * ms + vs;\n } else {\n HVX_Vector diff_vec = hvx_vec_splat_f32(s - M);\n vs = hvx_vec_get_f32(hvx_vec_exp_f32(diff_vec));\n S += vs;\n }\n }\n\n const float S_inv = S == 0.0f ? 0.0f : 1.0f/S;\n hvx_scale_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, S_inv);\n\n // Store result\n // dst indices\n const int i1 = iq1;\n const int i2 = iq2;\n const int i3 = iq3;\n\n // dst is permuted\n uint8_t * dst_ptr = (uint8_t *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1) * nb1;\n\n if (dst->type == HTP_TYPE_F32) {\n hvx_copy_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV);\n } else if (dst->type == HTP_TYPE_F16) {\n hvx_copy_f16_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV);\n }\n }\n}\n\nint op_flash_attn_ext(struct htp_ops_context * octx) {\n const struct htp_tensor * q = &octx->src0;\n const struct htp_tensor * k = &octx->src1;\n const struct htp_tensor * v = &octx->src2;\n const struct htp_tensor * mask = (octx->src3.data) ? &octx->src3 : NULL;\n const struct htp_tensor * dst = &octx->dst;\n\n // Check support\n if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) || k->type != HTP_TYPE_F16 || v->type != HTP_TYPE_F16) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n struct htp_fa_context factx;\n factx.octx = octx;\n\n factx.t_start = HAP_perf_get_qtimer_count();\n\n factx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]);\n factx.src0_div1 = init_fastdiv_values(q->ne[1]);\n\n factx.broadcast_rk2 = init_fastdiv_values(q->ne[2]/k->ne[2]);\n factx.broadcast_rk3 = init_fastdiv_values(q->ne[3]/k->ne[3]);\n factx.broadcast_rv2 = init_fastdiv_values(q->ne[2]/v->ne[2]);\n factx.broadcast_rv3 = init_fastdiv_values(q->ne[3]/v->ne[3]);\n\n if (mask) {\n factx.src3_div2 = init_fastdiv_values(mask->ne[2]);\n factx.src3_div3 = init_fastdiv_values(mask->ne[3]);\n }\n\n factx.is_q_fp32 = (q->type == HTP_TYPE_F32);\n factx.size_q_row_padded = hex_round_up(q->ne[0] * (factx.is_q_fp32 ? 4 : 2), 128);\n factx.size_k_row_padded = hex_round_up(k->ne[0] * sizeof(__fp16), 128);\n factx.size_v_row_padded = hex_round_up(v->ne[0] * sizeof(__fp16), 128);\n\n size_t size_q_block = factx.size_q_row_padded * 1; // single row for now\n factx.size_k_block = factx.size_k_row_padded * FLASH_ATTN_BLOCK_SIZE;\n factx.size_v_block = factx.size_v_row_padded * FLASH_ATTN_BLOCK_SIZE;\n factx.size_m_block = hex_round_up(FLASH_ATTN_BLOCK_SIZE * sizeof(__fp16), 128);\n\n factx.n_blocks = (k->ne[1] + FLASH_ATTN_BLOCK_SIZE - 1) / FLASH_ATTN_BLOCK_SIZE;\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n float logit_softcap = 0.0f;\n\n memcpy(&scale, (float *) octx->op_params + 0, sizeof(float));\n memcpy(&max_bias, (float *) octx->op_params + 1, sizeof(float));\n memcpy(&logit_softcap, (float *) octx->op_params + 2, sizeof(float));\n\n if (logit_softcap != 0.0f) {\n scale /= logit_softcap;\n }\n\n factx.scale = scale;\n factx.max_bias = max_bias;\n factx.logit_softcap = logit_softcap;\n\n uint32_t n_head = q->ne[2];\n factx.n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n factx.m0 = powf(2.0f, -(max_bias ) / factx.n_head_log2);\n factx.m1 = powf(2.0f, -(max_bias / 2.0f) / factx.n_head_log2);\n\n // total rows in q\n const uint32_t neq0 = q->ne[0];\n const uint32_t neq1 = q->ne[1];\n const uint32_t neq2 = q->ne[2];\n const uint32_t neq3 = q->ne[3];\n\n factx.qrows = neq1*neq2*neq3;\n factx.qrows_per_thread = (factx.qrows + octx->n_threads - 1) / octx->n_threads;\n\n size_t size_vkq_acc = hex_round_up(v->ne[0] * sizeof(float), 128); // VKQ32\n\n octx->src0_spad.size_per_thread = size_q_block * 1;\n octx->src1_spad.size_per_thread = factx.size_k_block * 2;\n octx->src2_spad.size_per_thread = factx.size_v_block * 2;\n octx->src3_spad.size_per_thread = mask ? factx.size_m_block * 2 : 0;\n octx->dst_spad.size_per_thread = size_vkq_acc;\n\n octx->src0_spad.size = octx->src0_spad.size_per_thread * octx->n_threads;\n octx->src1_spad.size = octx->src1_spad.size_per_thread * octx->n_threads;\n octx->src2_spad.size = octx->src2_spad.size_per_thread * octx->n_threads;\n octx->src3_spad.size = octx->src3_spad.size_per_thread * octx->n_threads;\n octx->dst_spad.size = octx->dst_spad.size_per_thread * octx->n_threads;\n\n size_t total_spad = octx->src0_spad.size + octx->src1_spad.size + octx->src2_spad.size + octx->src3_spad.size + octx->dst_spad.size;\n\n if (octx->ctx->vtcm_size < total_spad) {\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n octx->src0_spad.data = octx->ctx->vtcm_base;\n octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;\n octx->src2_spad.data = octx->src1_spad.data + octx->src1_spad.size;\n octx->src3_spad.data = octx->src2_spad.data + octx->src2_spad.size;\n octx->dst_spad.data = octx->src3_spad.data + octx->src3_spad.size;\n\n if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {\n worker_pool_run_func(octx->ctx->worker_pool, flash_attn_ext_f16_thread, &factx, octx->n_threads);\n }\n\n return HTP_STATUS_OK;\n}\n" }, { "path": "src/ggml-hexagon/htp/get-rows-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n#include \"hvx-utils.h\"\n\nstruct get_rows_context {\n struct htp_ops_context * octx;\n uint32_t src1_nrows_per_thread;\n struct fastdiv_values get_rows_div_ne10;\n struct fastdiv_values get_rows_div_ne10_ne11;\n};\n\n#define get_rows_preamble \\\n const uint32_t ne00 = octx->src0.ne[0]; \\\n const uint32_t ne01 = octx->src0.ne[1]; \\\n const uint32_t ne02 = octx->src0.ne[2]; \\\n const uint32_t ne03 = octx->src0.ne[3]; \\\n \\\n const uint32_t ne10 = octx->src1.ne[0]; \\\n const uint32_t ne11 = octx->src1.ne[1]; \\\n const uint32_t ne12 = octx->src1.ne[2]; \\\n \\\n const uint32_t nb01 = octx->src0.nb[1]; \\\n const uint32_t nb02 = octx->src0.nb[2]; \\\n const uint32_t nb03 = octx->src0.nb[3]; \\\n \\\n const uint32_t nb10 = octx->src1.nb[0]; \\\n const uint32_t nb11 = octx->src1.nb[1]; \\\n const uint32_t nb12 = octx->src1.nb[2]; \\\n \\\n const uint32_t nb1 = octx->dst.nb[1]; \\\n const uint32_t nb2 = octx->dst.nb[2]; \\\n const uint32_t nb3 = octx->dst.nb[3]; \\\n \\\n const uint32_t nr = ne10 * ne11 * ne12;\n\nstatic void get_rows_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) {\n struct get_rows_context * grctx = (struct get_rows_context *)data;\n struct htp_ops_context * octx = grctx->octx;\n get_rows_preamble;\n\n // parallelize by src1 elements (which correspond to dst rows)\n const uint32_t dr = grctx->src1_nrows_per_thread;\n const uint32_t ir0 = dr * ith;\n const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr;\n\n const bool is_i32 = (octx->src1.type == HTP_TYPE_I32);\n\n for (uint32_t i = ir0; i < ir1; ++i) {\n const uint32_t i12 = fastdiv(i, &grctx->get_rows_div_ne10_ne11);\n const uint32_t rem = i - i12 * ne11 * ne10;\n const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10);\n const uint32_t i10 = rem - i11 * ne10;\n\n const uintptr_t src1_addr = octx->src1.data + i10*nb10 + i11*nb11 + i12*nb12;\n\n uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;\n\n if (i01 >= ne01) {\n // invalid index, skip for now to avoid crash\n continue;\n }\n\n const uintptr_t src0_ptr = octx->src0.data + i01*nb01 + i11*nb02 + i12*nb03;\n const uintptr_t dst_ptr = octx->dst.data + i10*nb1 + i11*nb2 + i12*nb3;\n hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, ne00);\n }\n}\n\nint op_get_rows(struct htp_ops_context * octx) {\n get_rows_preamble;\n\n const uint32_t n_threads = MIN(nr, octx->n_threads);\n\n if (octx->src0.type != HTP_TYPE_F32) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n if (octx->dst.type != HTP_TYPE_F32) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n if (octx->src1.type != HTP_TYPE_I32 && octx->src1.type != HTP_TYPE_I64) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {\n return HTP_STATUS_OK;\n }\n\n struct get_rows_context grctx;\n grctx.octx = octx;\n grctx.get_rows_div_ne10 = init_fastdiv_values(octx->src1.ne[0]);\n grctx.get_rows_div_ne10_ne11 = init_fastdiv_values(octx->src1.ne[0] * octx->src1.ne[1]);\n\n grctx.src1_nrows_per_thread = (nr + n_threads - 1) / n_threads;\n\n worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32, &grctx, n_threads);\n return HTP_STATUS_OK;\n}\n" }, { "path": "src/ggml-hexagon/htp/hex-dma.c", "content": "#include \"hex-dma.h\"\n\n#include \n#include \n#include \n\n#pragma clang diagnostic ignored \"-Wunused-function\"\n\nstatic inline uint32_t pow2_ceil(uint32_t x) {\n if (x <= 1) {\n return 1;\n }\n int p = 2;\n x--;\n while (x >>= 1) {\n p <<= 1;\n }\n return p;\n}\n\ndma_queue * dma_queue_create(size_t capacity) {\n dma_queue * q = (dma_queue *) memalign(32, sizeof(dma_queue));\n if (q == NULL) {\n FARF(ERROR, \"%s: failed to allocate DMA queue\\n\", __FUNCTION__);\n return NULL;\n }\n\n capacity = pow2_ceil(capacity);\n\n memset(q, 0, sizeof(dma_queue));\n q->capacity = capacity;\n q->idx_mask = capacity - 1;\n\n q->desc = (hexagon_udma_descriptor_type1_t *) memalign(64, capacity * sizeof(hexagon_udma_descriptor_type1_t));\n memset(q->desc, 0, capacity * sizeof(hexagon_udma_descriptor_type1_t));\n\n q->dptr = (dma_ptr *) memalign(4, capacity * sizeof(dma_ptr));\n memset(q->dptr, 0, capacity * sizeof(dma_ptr));\n\n q->tail = &q->desc[capacity - 1];\n\n if (!q->desc && !q->dptr) {\n FARF(ERROR, \"%s: failed to allocate DMA queue items\\n\", __FUNCTION__);\n return NULL;\n }\n\n FARF(HIGH, \"dma-queue: capacity %u\\n\", capacity);\n\n return q;\n}\n\nvoid dma_queue_delete(dma_queue * q) {\n if (!q) {\n return;\n }\n free(q->desc);\n free(q->dptr);\n free(q);\n}\n\nvoid dma_queue_flush(dma_queue * q) {\n while (dma_queue_pop(q).dst != NULL) ;\n}\n" }, { "path": "src/ggml-hexagon/htp/hex-dma.h", "content": "#ifndef HTP_DMA_H\n#define HTP_DMA_H\n\n#include \n#include \n#include \n#include \n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\ntypedef struct {\n void *dst;\n const void *src;\n} dma_ptr;\n\ntypedef struct {\n hexagon_udma_descriptor_type1_t * desc; // descriptor pointers\n hexagon_udma_descriptor_type1_t * tail; // tail pointer\n dma_ptr * dptr; // dst/src pointers\n uint32_t push_idx;\n uint32_t pop_idx;\n uint32_t capacity;\n uint32_t idx_mask;\n} dma_queue;\n\ndma_queue * dma_queue_create(size_t capacity);\nvoid dma_queue_delete(dma_queue * q);\nvoid dma_queue_flush(dma_queue * q);\n\n// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead\n// but those do not seem to always compiler properly.\nstatic inline void dmstart(void * next) {\n asm volatile(\" release(%0):at\" : : \"r\"(next));\n asm volatile(\" dmstart(%0)\" : : \"r\"(next));\n}\n\nstatic inline void dmlink(void * cur, void * next) {\n asm volatile(\" release(%0):at\" : : \"r\"(next));\n asm volatile(\" dmlink(%0, %1)\" : : \"r\"(cur), \"r\"(next));\n}\n\nstatic inline unsigned int dmpoll(void) {\n unsigned int ret = 0;\n asm volatile(\" %0 = dmpoll\" : \"=r\"(ret) : : \"memory\");\n return ret;\n}\n\nstatic inline unsigned int dmwait(void) {\n unsigned int ret = 0;\n asm volatile(\" %0 = dmwait\" : \"=r\"(ret) : : \"memory\");\n return ret;\n}\n\nstatic inline dma_ptr dma_make_ptr(void *dst, const void *src)\n{\n dma_ptr p = { dst, src };\n return p;\n}\n\nstatic inline bool dma_queue_push(dma_queue * q,\n dma_ptr dptr,\n size_t dst_row_size,\n size_t src_row_size,\n size_t width, // width in bytes. number of bytes to transfer per row\n size_t nrows) {\n if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) {\n FARF(ERROR, \"dma-push: queue full\\n\");\n return false;\n }\n\n hexagon_udma_descriptor_type1_t * desc = &q->desc[q->push_idx];\n\n desc->next = NULL;\n desc->length = 0;\n desc->desctype = HEXAGON_UDMA_DESC_DESCTYPE_TYPE1;\n desc->dstbypass = 1;\n desc->srcbypass = 1;\n#if __HVX_ARCH__ >= 73\n desc->dstbypass = 1;\n desc->srcbypass = 1;\n#else\n desc->dstbypass = 0;\n desc->srcbypass = 1;\n#endif\n desc->order = 0;\n desc->dstate = HEXAGON_UDMA_DESC_DSTATE_INCOMPLETE;\n desc->src = (void *) dptr.src;\n desc->dst = (void *) dptr.dst;\n desc->allocation = 0;\n desc->padding = 0;\n desc->roiwidth = width;\n desc->roiheight = nrows;\n desc->srcstride = src_row_size;\n desc->dststride = dst_row_size;\n desc->srcwidthoffset = 0;\n desc->dstwidthoffset = 0;\n\n q->dptr[q->push_idx] = dptr;\n\n dmlink(q->tail, desc);\n q->tail = desc;\n\n // FARF(ERROR, \"dma-push: i %u width %u nrows %d dst %p src %p\\n\", q->push_idx, width, nrows, dptr.dst, dptr.src);\n q->push_idx = (q->push_idx + 1) & q->idx_mask;\n return true;\n}\n\nstatic inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q,\n dma_ptr dptr,\n size_t dst_row_size,\n size_t src_row_size,\n size_t nrows) {\n return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows);\n}\n\n\nstatic inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q,\n dma_ptr dptr,\n size_t dst_row_size,\n size_t src_row_size,\n size_t nrows) {\n return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows);\n}\n\nstatic inline dma_ptr dma_queue_pop(dma_queue * q) {\n dma_ptr dptr = { NULL };\n\n if (q->push_idx == q->pop_idx) {\n return dptr;\n }\n\n hexagon_udma_descriptor_type1_t * desc = &q->desc[q->pop_idx];\n\n // Wait for desc to complete\n while (1) {\n dmpoll();\n if (desc->dstate == HEXAGON_UDMA_DESC_DSTATE_COMPLETE) {\n break;\n }\n // FARF(ERROR, \"dma-pop: waiting for DMA : %u\\n\", q->pop_idx);\n }\n\n dptr = q->dptr[q->pop_idx];\n\n // FARF(ERROR, \"dma-pop: i %u dst %p src %p\\n\", q->pop_idx, dptr.dst, dptr.src);\n q->pop_idx = (q->pop_idx + 1) & q->idx_mask;\n return dptr;\n}\n\nstatic inline dma_ptr dma_queue_pop_nowait(dma_queue * q) {\n dma_ptr dptr = { NULL };\n\n if (q->push_idx == q->pop_idx) {\n return dptr;\n }\n\n dptr = q->dptr[q->pop_idx];\n\n // FARF(ERROR, \"dma-pop-nowait: i %u dst %p src %p\\n\", q->pop_idx, dptr.dst, dptr.src);\n q->pop_idx = (q->pop_idx + 1) & q->idx_mask;\n return dptr;\n}\n\nstatic inline bool dma_queue_empty(dma_queue * q) {\n return q->push_idx == q->pop_idx;\n}\n\nstatic inline uint32_t dma_queue_depth(dma_queue * q) {\n return (q->push_idx - q->pop_idx) & q->idx_mask;\n}\n\nstatic inline uint32_t dma_queue_capacity(dma_queue * q) {\n return q->capacity;\n}\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif\n\n#endif /* HTP_DMA_H */\n" }, { "path": "src/ggml-hexagon/htp/hex-dump.h", "content": "#ifndef HEX_DUMP_H\n#define HEX_DUMP_H\n\n#include \n\nstatic inline void hex_dump_int8_line(char * pref, const int8_t * x, int n) {\n char str[1024], *p = str, *p_end = str + sizeof(str);\n p += snprintf(p, p_end - p, \"%s: \", pref);\n for (int i = 0; i < n && p < p_end; i++) {\n p += snprintf(p, p_end - p, \"%d, \", x[i]);\n }\n FARF(HIGH, \"%s\\n\", str);\n}\n\nstatic inline void hex_dump_uint8_line(char * pref, const uint8_t * x, uint32_t n) {\n char str[1024], *p = str, *p_end = str + sizeof(str);\n p += snprintf(p, p_end - p, \"%s: \", pref);\n for (int i = 0; i < n && p < p_end; i++) {\n p += snprintf(p, p_end - p, \"%d, \", x[i]);\n }\n FARF(HIGH, \"%s\\n\", str);\n}\n\nstatic inline void hex_dump_int32_line(char * pref, const int32_t * x, uint32_t n) {\n char str[1024], *p = str, *p_end = str + sizeof(str);\n p += snprintf(p, p_end - p, \"%s: \", pref);\n for (int i = 0; i < n; i++) {\n p += snprintf(p, p_end - p, \"%d, \", (int) x[i]);\n }\n FARF(HIGH, \"%s\\n\", str);\n}\n\nstatic inline void hex_dump_f16_line(char * pref, const __fp16 * x, uint32_t n) {\n char str[1024], *p = str, *p_end = str + sizeof(str);\n p += snprintf(p, p_end - p, \"%s: \", pref);\n for (int i = 0; i < n; i++) {\n p += snprintf(p, p_end - p, \"%.6f, \", (float) x[i]);\n }\n FARF(HIGH, \"%s\\n\", str);\n}\n\nstatic inline void hex_dump_f32_line(char * pref, const float * x, uint32_t n) {\n char str[1024], *p = str, *p_end = str + sizeof(str);\n p += snprintf(p, p_end - p, \"%s: \", pref);\n for (int i = 0; i < n; i++) {\n p += snprintf(p, p_end - p, \"%.6f, \", x[i]);\n }\n FARF(HIGH, \"%s\\n\", str);\n}\n\nstatic inline void hex_dump_f32(char * pref, const float * x, uint32_t n) {\n uint32_t n0 = n / 16;\n uint32_t n1 = n % 16;\n\n uint32_t i = 0;\n for (; i < n0; i++) {\n hex_dump_f32_line(pref, x + (16 * i), 16);\n }\n if (n1) {\n hex_dump_f32_line(pref, x + (16 * i), n1);\n }\n}\n\nstatic inline void hex_dump_f16(char * pref, const __fp16 * x, uint32_t n) {\n uint32_t n0 = n / 16;\n uint32_t n1 = n % 16;\n\n uint32_t i = 0;\n for (; i < n0; i++) {\n hex_dump_f16_line(pref, x + (16 * i), 16);\n }\n if (n1) {\n hex_dump_f16_line(pref, x + (16 * i), n1);\n }\n}\n\n#endif /* HEX_DUMP_H */\n" }, { "path": "src/ggml-hexagon/htp/hex-fastdiv.h", "content": "#ifndef HEX_FASTDIV_H\n#define HEX_FASTDIV_H\n\n// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.\n// Precompute mp (m' in the paper) and L such that division\n// can be computed using a multiply (high 32b of 64b result)\n// and a shift:\n//\n// n/d = (mulhi(n, mp) + n) >> L;\nstruct fastdiv_values {\n uint32_t mp;\n uint32_t l;\n};\n\nstatic inline struct fastdiv_values init_fastdiv_values(uint32_t d) {\n struct fastdiv_values result = { 0, 0 };\n // compute L = ceil(log2(d));\n while (result.l < 32 && ((uint32_t) 1 << result.l) < d) {\n ++(result.l);\n }\n\n result.mp = (uint32_t) (((uint64_t) 1 << 32) * (((uint64_t) 1 << result.l) - d) / d + 1);\n return result;\n}\n\nstatic inline uint32_t fastdiv(uint32_t n, const struct fastdiv_values * vals) {\n // Compute high 32 bits of n * mp\n const uint32_t hi = (uint32_t) (((uint64_t) n * vals->mp) >> 32); // mulhi(n, mp)\n // add n, apply bit shift\n return (hi + n) >> vals->l;\n}\n\nstatic inline uint32_t fastmodulo(uint32_t n, uint32_t d, const struct fastdiv_values * vals) {\n return n - fastdiv(n, vals) * d;\n}\n\n#endif /* HEX_FASTDIV_H */\n" }, { "path": "src/ggml-hexagon/htp/hex-utils.h", "content": "#ifndef HEX_UTILS_H\n#define HEX_UTILS_H\n\n#include \n#include \n\n#include \"hexagon_types.h\"\n\n#include \"hex-fastdiv.h\"\n#include \"hex-dump.h\"\n\n#ifndef MAX\n#define MAX(a, b) ((a) > (b) ? (a) : (b))\n#endif\n\n#ifndef MIN\n#define MIN(a, b) ((a) < (b) ? (a) : (b))\n#endif\n\nstatic inline uint64_t hex_get_cycles() {\n uint64_t cycles = 0;\n asm volatile(\" %0 = c15:14\\n\" : \"=r\"(cycles));\n return cycles;\n}\n\nstatic inline uint64_t hex_get_pktcnt() {\n uint64_t pktcnt;\n asm volatile(\" %0 = c19:18\\n\" : \"=r\"(pktcnt));\n return pktcnt;\n}\n\nstatic inline int32_t hex_is_aligned(void * addr, uint32_t align) {\n return ((size_t) addr & (align - 1)) == 0;\n}\n\nstatic inline int32_t hex_is_one_chunk(void * addr, uint32_t n, uint32_t chunk_size) {\n uint32_t left_off = (size_t) addr & (chunk_size - 1);\n uint32_t right_off = left_off + n;\n return right_off <= chunk_size;\n}\n\nstatic inline uint32_t hex_round_up(uint32_t n, uint32_t m) {\n return m * ((n + m - 1) / m);\n}\n\nstatic inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride, uint32_t height) {\n const uint64_t control = Q6_P_combine_RR(stride, Q6_R_combine_RlRl(width, height));\n Q6_l2fetch_AP((void *) p, control);\n}\n\n#endif /* HEX_UTILS_H */\n" }, { "path": "src/ggml-hexagon/htp/htp-ctx.h", "content": "#ifndef HTP_CTX_H\n#define HTP_CTX_H\n\n#include \"hex-dma.h\"\n#include \"worker-pool.h\"\n\n#include \n#include \n#include \n#include \n\n#define HTP_MAX_NTHREADS 10\n\n// Main context for htp DSP backend\nstruct htp_context {\n dspqueue_t queue;\n dma_queue * dma[HTP_MAX_NTHREADS];\n worker_pool_context_t worker_pool;\n uint32_t n_threads;\n\n int thread_id;\n int thread_prio;\n\n uint8_t * vtcm_base;\n size_t vtcm_size;\n uint32_t vtcm_rctx;\n\n atomic_bool vtcm_valid;\n atomic_bool vtcm_inuse;\n atomic_bool vtcm_needs_release;\n\n uint32_t opmask;\n};\n\n#endif /* HTP_CTX_H */\n" }, { "path": "src/ggml-hexagon/htp/htp-msg.h", "content": "#ifndef HTP_MSG_H\n#define HTP_MSG_H\n\n#include \n\n// ggml-common.h must be included prio to this header\n\n// Mask to enable various stages of the Ops.\n// Used for debugging and profiling.\nenum {\n HTP_OPMASK_QUEUE = (1 << 0), // Enable Queueing (ie calls into the DSP)\n HTP_OPMASK_QUANTIZE = (1 << 1), // Enable Quantize\n HTP_OPMASK_COMPUTE = (1 << 2), // Enable Compute\n};\n\n// Op flags\nenum {\n HTP_OPFLAGS_SKIP_QUANTIZE = (1 << 0), // Skip dynamic quantization (reuse quantized tensors)\n HTP_OPFLAGS_SKIP_COMPUTE = (1 << 1), // Skip actual computation (used for profiling)\n HTP_OPFLAGS_EARLY_WAKEUP = (1 << 2) // Send early wakeup notification\n};\n\nenum htp_status {\n HTP_STATUS_OK = 1,\n HTP_STATUS_INTERNAL_ERR = 2,\n HTP_STATUS_NO_SUPPORT = 3,\n HTP_STATUS_INVAL_PARAMS = 4,\n HTP_STATUS_VTCM_TOO_SMALL = 5,\n};\n\n// The values must match the ggml_type.\n// Duplicated here because we can't include full ggml.h in the htp build.\n// We have some static_asserts in the cpp code to ensure things are in sync.\nenum htp_data_type {\n HTP_TYPE_F32 = 0,\n HTP_TYPE_F16 = 1,\n HTP_TYPE_Q4_0 = 2,\n HTP_TYPE_Q8_0 = 8,\n HTP_TYPE_I32 = 26,\n HTP_TYPE_I64 = 27,\n HTP_TYPE_MXFP4 = 39,\n HTP_TYPE_COUNT\n};\n\n// Do not reorder first 4 (used as an index)\nenum htp_op {\n HTP_OP_MUL = 0,\n HTP_OP_ADD = 1,\n HTP_OP_SUB = 2,\n HTP_OP_DIV = 3,\n HTP_OP_MUL_MAT,\n HTP_OP_MUL_MAT_ID,\n HTP_OP_RMS_NORM,\n HTP_OP_UNARY_SILU,\n HTP_OP_UNARY_GELU,\n HTP_OP_GLU_SWIGLU,\n HTP_OP_GLU_SWIGLU_OAI,\n HTP_OP_GLU_GEGLU,\n HTP_OP_SOFTMAX,\n HTP_OP_ADD_ID,\n HTP_OP_ROPE,\n HTP_OP_FLASH_ATTN_EXT,\n HTP_OP_SET_ROWS,\n HTP_OP_GET_ROWS,\n HTP_OP_SCALE,\n HTP_OP_CPY,\n HTP_OP_ARGSORT,\n HTP_OP_SQR,\n HTP_OP_SQRT,\n HTP_OP_SUM_ROWS,\n HTP_OP_SSM_CONV,\n INVALID\n};\n\nstatic inline size_t htp_t_block_size(uint32_t t) {\n switch (t) {\n case HTP_TYPE_F32:\n return 1;\n case HTP_TYPE_F16:\n return 1;\n case HTP_TYPE_Q4_0:\n return QK4_0;\n case HTP_TYPE_Q8_0:\n return QK8_0;\n case HTP_TYPE_MXFP4:\n return QK_MXFP4;\n default:\n assert(0 && \"unsupported HTP data type\");\n }\n return 0;\n}\n\nstatic inline size_t htp_type_nbytes(uint32_t t) {\n switch (t) {\n case HTP_TYPE_F32:\n return 4;\n case HTP_TYPE_F16:\n return 2;\n case HTP_TYPE_Q4_0:\n return sizeof(block_q4_0);\n case HTP_TYPE_Q8_0:\n return sizeof(block_q8_0);\n case HTP_TYPE_MXFP4:\n return sizeof(block_mxfp4);\n default:\n assert(0 && \"unsupported HTP data type\");\n }\n return 0;\n}\n\n// Internal types\n#define QK_Q4_0x4x2 256 // 4x Q4_0 blocks packed with next 4x Q4_0 blocks (size in bytes 128)\n#define QK_Q8_0x4x2 256 // 4x Q8_0 blocks concat with next 4x Q8_0 blocks\n#define QK_MXFP4x4x2 256 // 4x MXFP4 blocks concat with next 4x MXFP4 blocks\n\n#define HTP_MAX_DIMS 4\n\nstruct htp_tensor {\n uint32_t data; // Buffer offset in the messages, and data pointer on the NSP\n uint32_t type; // Data type\n uint32_t ne[HTP_MAX_DIMS]; // Number of elements\n uint32_t nb[HTP_MAX_DIMS]; // Stride in bytes (see ggml.h ggml_tensor)\n};\n\n#define HTP_MAX_OP_PARAMS 64\n\nstruct htp_general_req {\n uint32_t op; // GGML/HTP Op\n int32_t op_params[HTP_MAX_OP_PARAMS / sizeof(int32_t)];\n // Params for the op, e.g. epsilon of RMS norm\n uint32_t flags; // Request flags\n\n struct htp_tensor src0; // Input0 tensor\n struct htp_tensor src1; // Input1 tensor\n struct htp_tensor src2; // Input2 tensor\n struct htp_tensor src3; // Input3 tensor\n struct htp_tensor src4; // Input4 tensor\n struct htp_tensor dst; // Output tensor\n\n // should be multiple of 64 bytes (cacheline)\n};\n\nstruct htp_general_rsp {\n uint32_t op; // GGML/HTP Op\n uint32_t status; // HTP_STATUS_...\n uint32_t prof_usecs; // Number of usec per request\n uint32_t prof_cycles; // Number of cycles per request\n uint32_t prof_pkts; // Number of instruction packets per request\n uint8_t unused[44]; // Pad to 64 bytes\n};\n\n#define HTP_MAX_MESSAGE_SIZE sizeof(struct htp_general_req)\n#define HTP_MAX_PACKET_BUFFERS 8\n\n#endif /* HTP_MSG_H */\n" }, { "path": "src/ggml-hexagon/htp/htp-ops.h", "content": "#ifndef HTP_OPS_H\n#define HTP_OPS_H\n\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"worker-pool.h\"\n\n#include \n#include \n\n#include \n\n// ggml-common.h must be included prior to this header\n\nstruct htp_spad {\n uint8_t * data;\n size_t stride;\n size_t size;\n size_t size_per_thread;\n};\n\nstruct htp_ops_context {\n struct htp_context * ctx;\n\n enum htp_op op;\n int32_t op_params[HTP_MAX_OP_PARAMS / sizeof(int32_t)];\n\n struct htp_tensor src0;\n struct htp_tensor src1;\n struct htp_tensor src2;\n struct htp_tensor src3;\n struct htp_tensor src4;\n struct htp_tensor dst;\n\n struct htp_spad src0_spad;\n struct htp_spad src1_spad;\n struct htp_spad src2_spad;\n struct htp_spad src3_spad;\n struct htp_spad dst_spad;\n\n worker_pool_context_t * wpool; // worker pool\n uint32_t n_threads; // num threads\n\n uint32_t flags;\n};\n\nint op_matmul(struct htp_ops_context * octx);\nint op_matmul_id(struct htp_ops_context * octx);\nint op_binary(struct htp_ops_context * octx);\nint op_unary(struct htp_ops_context * octx);\nint op_sum_rows(struct htp_ops_context * octx);\nint op_activations(struct htp_ops_context * octx);\nint op_softmax(struct htp_ops_context * octx);\nint op_add_id(struct htp_ops_context * octx);\nint op_rope(struct htp_ops_context * octx);\nint op_flash_attn_ext(struct htp_ops_context * octx);\nint op_set_rows(struct htp_ops_context * octx);\nint op_get_rows(struct htp_ops_context * octx);\nint op_cpy(struct htp_ops_context * octx);\nint op_argsort(struct htp_ops_context * octx);\nint op_ssm_conv(struct htp_ops_context * octx);\n\n#endif /* HTP_OPS_H */\n" }, { "path": "src/ggml-hexagon/htp/htp_iface.idl", "content": "// FastRPC IDL interface for GGML HTP\n\n#ifndef HTP_IDL\n#define HTP_IDL\n\n#include \"AEEStdDef.idl\"\n#include \"remote.idl\"\n\ninterface htp_iface : remote_handle64 {\n AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx);\n AEEResult stop();\n AEEResult enable_etm();\n AEEResult disable_etm();\n};\n\n#endif /* HTP_IDL */\n" }, { "path": "src/ggml-hexagon/htp/hvx-arith.h", "content": "#ifndef HVX_ARITH_H\n#define HVX_ARITH_H\n\n#include \n#include \n#include \n#include \n\n#include \"hvx-base.h\"\n#include \"hex-utils.h\"\n\n//\n// Binary operations (add, mul, sub)\n//\n\n#define UNUSED(x) (void)(x)\n\n#define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src0_type * restrict vsrc0 = (src0_type *) src0; \\\n src1_type * restrict vsrc1 = (src1_type *) src1; \\\n \\\n const uint32_t epv = 128 / (elem_size); \\\n const uint32_t nvec = n / epv; \\\n const uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \\\n } \\\n if (nloe) { \\\n HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \\\n vec_store((void *) &vdst[i], nloe * (elem_size), v); \\\n } \\\n } while(0)\n\n#if __HVX_ARCH__ < 79\n\n#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b))\n#define HVX_OP_SUB_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a, b))\n#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b))\n\n#else\n\n#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b)\n#define HVX_OP_SUB_F32(a, b) Q6_Vsf_vsub_VsfVsf(a, b)\n#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b)\n\n#endif\n\n#define HVX_OP_ADD_F16(a, b) hvx_vec_add_f16_f16(a, b)\n#define HVX_OP_SUB_F16(a, b) hvx_vec_sub_f16_f16(a, b)\n#define HVX_OP_MUL_F16(a, b) hvx_vec_mul_f16_f16(a, b)\n\n// Generic macro to define alignment permutations for an op\n#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \\\nstatic inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n assert((uintptr_t) src0 % 128 == 0); \\\n assert((uintptr_t) src1 % 128 == 0); \\\n hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n assert((uintptr_t) src0 % 128 == 0); \\\n hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n assert((uintptr_t) src1 % 128 == 0); \\\n hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) src0 % 128 == 0); \\\n assert((uintptr_t) src1 % 128 == 0); \\\n hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) src0 % 128 == 0); \\\n hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) src1 % 128 == 0); \\\n hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \\\n} \\\n\nDEFINE_HVX_BINARY_OP_VARIANTS(hvx_add_f32, HVX_OP_ADD_F32, float)\nDEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f32, HVX_OP_SUB_F32, float)\nDEFINE_HVX_BINARY_OP_VARIANTS(hvx_mul_f32, HVX_OP_MUL_F32, float)\n\nDEFINE_HVX_BINARY_OP_VARIANTS(hvx_add_f16, HVX_OP_ADD_F16, _Float16)\nDEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f16, HVX_OP_SUB_F16, _Float16)\nDEFINE_HVX_BINARY_OP_VARIANTS(hvx_mul_f16, HVX_OP_MUL_F16, _Float16)\n\n// Dispatcher logic\n#define HVX_BINARY_DISPATCHER(OP_NAME) \\\nstatic inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \\\n if (hex_is_aligned((void *) dst, 128)) { \\\n if (hex_is_aligned((void *) src0, 128)) { \\\n if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \\\n else OP_NAME##_aau(dst, src0, src1, num_elems); \\\n } else { \\\n if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \\\n else OP_NAME##_auu(dst, src0, src1, num_elems); \\\n } \\\n } else { \\\n if (hex_is_aligned((void *) src0, 128)) { \\\n if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \\\n else OP_NAME##_uau(dst, src0, src1, num_elems); \\\n } else { \\\n if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \\\n else OP_NAME##_uuu(dst, src0, src1, num_elems); \\\n } \\\n } \\\n}\n\nHVX_BINARY_DISPATCHER(hvx_add_f32)\nHVX_BINARY_DISPATCHER(hvx_sub_f32)\nHVX_BINARY_DISPATCHER(hvx_mul_f32)\n\nHVX_BINARY_DISPATCHER(hvx_add_f16)\nHVX_BINARY_DISPATCHER(hvx_sub_f16)\nHVX_BINARY_DISPATCHER(hvx_mul_f16)\n\n// Mul-Mul Optimized\nstatic inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint8_t * restrict src2, const uint32_t num_elems) {\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src0 % 128 == 0);\n assert((unsigned long) src1 % 128 == 0);\n assert((unsigned long) src2 % 128 == 0);\n\n HVX_Vector * restrict vdst = (HVX_Vector *) dst;\n HVX_Vector * restrict vsrc0 = (HVX_Vector *) src0;\n HVX_Vector * restrict vsrc1 = (HVX_Vector *) src1;\n HVX_Vector * restrict vsrc2 = (HVX_Vector *) src2;\n\n const uint32_t elem_size = sizeof(float);\n const uint32_t epv = 128 / elem_size;\n const uint32_t nvec = num_elems / epv;\n const uint32_t nloe = num_elems % epv;\n\n uint32_t i = 0;\n\n _Pragma(\"unroll(4)\")\n for (; i < nvec; i++) {\n HVX_Vector v1 = HVX_OP_MUL_F32(vsrc0[i], vsrc1[i]);\n vdst[i] = HVX_OP_MUL(v1, vsrc2[i]);\n }\n\n if (nloe) {\n HVX_Vector v1 = HVX_OP_MUL_F32(vsrc0[i], vsrc1[i]);\n HVX_Vector v2 = HVX_OP_MUL_F32(v1, vsrc2[i]);\n hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v2);\n }\n}\n\n// Scalar Operations\n\n#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const uint32_t epv = 128 / (elem_size); \\\n const uint32_t nvec = n / epv; \\\n const uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n HVX_Vector v = vsrc[i]; \\\n vdst[i] = scalar_op_macro(v); \\\n } \\\n if (nloe) { \\\n HVX_Vector v = vsrc[i]; \\\n v = scalar_op_macro(v); \\\n vec_store((void *) &vdst[i], nloe * (elem_size), v); \\\n } \\\n } while(0)\n\n#define HVX_OP_ADD_SCALAR_F32(v) \\\n ({ \\\n const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VwVw(inf, v); \\\n HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \\\n Q6_V_vmux_QVV(pred_inf, inf, out); \\\n })\n\n#define HVX_OP_MUL_SCALAR_F32(v) HVX_OP_MUL_F32(v, val_vec)\n#define HVX_OP_SUB_SCALAR_F32(v) HVX_OP_SUB_F32(v, val_vec)\n\n#define HVX_OP_ADD_SCALAR_F16(v) \\\n ({ \\\n const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VhVh(inf, v); \\\n HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \\\n Q6_V_vmux_QVV(pred_inf, inf, out); \\\n })\n\n#define HVX_OP_MUL_SCALAR_F16(v) HVX_OP_MUL_F16(v, val_vec)\n#define HVX_OP_SUB_SCALAR_F16(v) HVX_OP_SUB_F16(v, val_vec)\n\n// Scalar Variants\n\n// Generic macro to define alignment permutations for an op\n#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \\\nstatic inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \\\n const HVX_Vector val_vec = SPLAT_MACRO(val); \\\n const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \\\n assert((uintptr_t) dst % 128 == 0); \\\n assert((uintptr_t) src % 128 == 0); \\\n hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \\\n const HVX_Vector val_vec = SPLAT_MACRO(val); \\\n const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \\\n assert((uintptr_t) dst % 128 == 0); \\\n hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \\\n const HVX_Vector val_vec = SPLAT_MACRO(val); \\\n const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \\\n assert((uintptr_t) src % 128 == 0); \\\n hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \\\n} \\\nstatic inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \\\n const HVX_Vector val_vec = SPLAT_MACRO(val); \\\n const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \\\n hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \\\n} \\\n\nDEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_add_scalar_f32, HVX_OP_ADD_SCALAR_F32, hvx_vec_splat_f32, float)\nDEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f32, HVX_OP_SUB_SCALAR_F32, hvx_vec_splat_f32, float)\nDEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_mul_scalar_f32, HVX_OP_MUL_SCALAR_F32, hvx_vec_splat_f32, float)\n\nDEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_add_scalar_f16, HVX_OP_ADD_SCALAR_F16, hvx_vec_splat_f16, _Float16)\nDEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f16, HVX_OP_SUB_SCALAR_F16, hvx_vec_splat_f16, _Float16)\nDEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_mul_scalar_f16, HVX_OP_MUL_SCALAR_F16, hvx_vec_splat_f16, _Float16)\n\n// Dispatcher logic\n#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \\\nstatic inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, const uint32_t num_elems) { \\\n if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \\\n OP_NAME##_aa(dst, src, val, num_elems); \\\n } else if (hex_is_aligned((void *) dst, 128)) { \\\n OP_NAME##_au(dst, src, val, num_elems); \\\n } else if (hex_is_aligned((void *) src, 128)) { \\\n OP_NAME##_ua(dst, src, val, num_elems); \\\n } else { \\\n OP_NAME##_uu(dst, src, val, num_elems); \\\n } \\\n}\n\nHVX_BINARY_SCALAR_DISPATCHER(hvx_add_scalar_f32, float)\nHVX_BINARY_SCALAR_DISPATCHER(hvx_sub_scalar_f32, float)\nHVX_BINARY_SCALAR_DISPATCHER(hvx_mul_scalar_f32, float)\n\nHVX_BINARY_SCALAR_DISPATCHER(hvx_add_scalar_f16, _Float16)\nHVX_BINARY_SCALAR_DISPATCHER(hvx_sub_scalar_f16, _Float16)\nHVX_BINARY_SCALAR_DISPATCHER(hvx_mul_scalar_f16, _Float16)\n\n// MIN Scalar variants\n\n#define HVX_OP_MIN_SCALAR(v) Q6_Vsf_vmin_VsfVsf(val_vec, v)\n\nstatic inline void hvx_min_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {\n const HVX_Vector val_vec = hvx_vec_splat_f32(val);\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src % 128 == 0);\n hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_MIN_SCALAR);\n}\n\nstatic inline void hvx_min_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {\n const HVX_Vector val_vec = hvx_vec_splat_f32(val);\n assert((unsigned long) dst % 128 == 0);\n hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_MIN_SCALAR);\n}\n\nstatic inline void hvx_min_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {\n const HVX_Vector val_vec = hvx_vec_splat_f32(val);\n assert((unsigned long) src % 128 == 0);\n hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_MIN_SCALAR);\n}\n\nstatic inline void hvx_min_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {\n const HVX_Vector val_vec = hvx_vec_splat_f32(val);\n hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_MIN_SCALAR);\n}\n\nstatic inline void hvx_min_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float val, const int num_elems) {\n if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {\n hvx_min_scalar_f32_aa(dst, src, val, num_elems);\n } else if (hex_is_aligned((void *) dst, 128)) {\n hvx_min_scalar_f32_au(dst, src, val, num_elems);\n } else if (hex_is_aligned((void *) src, 128)) {\n hvx_min_scalar_f32_ua(dst, src, val, num_elems);\n } else {\n hvx_min_scalar_f32_uu(dst, src, val, num_elems);\n }\n}\n\n// CLAMP Scalar variants\n\n#define HVX_OP_CLAMP_SCALAR(v) \\\n ({ \\\n HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VsfVsf(v, max_vec); \\\n HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VsfVsf(min_vec, v); \\\n HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \\\n Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \\\n })\n\nstatic inline void hvx_clamp_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) {\n const HVX_Vector min_vec = hvx_vec_splat_f32(min);\n const HVX_Vector max_vec = hvx_vec_splat_f32(max);\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src % 128 == 0);\n hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR);\n}\n\nstatic inline void hvx_clamp_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) {\n const HVX_Vector min_vec = hvx_vec_splat_f32(min);\n const HVX_Vector max_vec = hvx_vec_splat_f32(max);\n assert((unsigned long) dst % 128 == 0);\n hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR);\n}\n\nstatic inline void hvx_clamp_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) {\n const HVX_Vector min_vec = hvx_vec_splat_f32(min);\n const HVX_Vector max_vec = hvx_vec_splat_f32(max);\n assert((unsigned long) src % 128 == 0);\n hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR);\n}\n\nstatic inline void hvx_clamp_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) {\n const HVX_Vector min_vec = hvx_vec_splat_f32(min);\n const HVX_Vector max_vec = hvx_vec_splat_f32(max);\n hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR);\n}\n\nstatic inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, const int num_elems) {\n if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {\n hvx_clamp_scalar_f32_aa(dst, src, min, max, num_elems);\n } else if (hex_is_aligned((void *) dst, 128)) {\n hvx_clamp_scalar_f32_au(dst, src, min, max, num_elems);\n } else if (hex_is_aligned((void *) src, 128)) {\n hvx_clamp_scalar_f32_ua(dst, src, min, max, num_elems);\n } else {\n hvx_clamp_scalar_f32_uu(dst, src, min, max, num_elems);\n }\n}\n\n//\n// Square\n//\n\n#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const uint32_t elem_size = sizeof(float); \\\n const uint32_t epv = 128 / elem_size; \\\n const uint32_t nvec = n / epv; \\\n const uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \\\n } \\\n if (nloe) { \\\n HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \\\n vec_store((void *) &vdst[i], nloe * elem_size, v); \\\n } \\\n } while(0)\n\nstatic inline void hvx_sqr_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src % 128 == 0);\n hvx_sqr_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_sqr_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n hvx_sqr_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_sqr_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) src % 128 == 0);\n hvx_sqr_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_sqr_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_sqr_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) {\n if (hex_is_aligned((void *) dst, 128)) {\n if (hex_is_aligned((void *) src, 128)) {\n hvx_sqr_f32_aa(dst, src, num_elems);\n } else {\n hvx_sqr_f32_au(dst, src, num_elems);\n }\n } else {\n if (hex_is_aligned((void *) src, 128)) {\n hvx_sqr_f32_ua(dst, src, num_elems);\n } else {\n hvx_sqr_f32_uu(dst, src, num_elems);\n }\n }\n}\n\n#undef HVX_OP_ADD_F32\n#undef HVX_OP_SUB_F32\n#undef HVX_OP_MUL_F32\n#undef HVX_OP_ADD_F16\n#undef HVX_OP_SUB_F16\n#undef HVX_OP_MUL_F16\n#undef hvx_arith_loop_body\n#undef HVX_OP_ADD_SCALAR_F32\n#undef HVX_OP_SUB_SCALAR_F32\n#undef HVX_OP_MUL_SCALAR_F32\n#undef HVX_OP_ADD_SCALAR_F16\n#undef HVX_OP_SUB_SCALAR_F16\n#undef HVX_OP_MUL_SCALAR_F16\n#undef hvx_scalar_loop_body\n#undef HVX_OP_MIN_SCALAR\n#undef HVX_OP_CLAMP_SCALAR\n#undef DEFINE_HVX_BINARY_OP_VARIANTS\n#undef HVX_BINARY_DISPATCHER\n#undef UNUSED\n\n#endif // HVX_ARITH_H\n" }, { "path": "src/ggml-hexagon/htp/hvx-base.h", "content": "#ifndef HVX_BASE_H\n#define HVX_BASE_H\n\n#include \n#include \n\n#include \"hex-utils.h\"\n#include \"hvx-types.h\"\n\nstatic inline void hvx_vec_store_u(void * restrict dst, uint32_t n, HVX_Vector v) {\n // Rotate as needed.\n v = Q6_V_vlalign_VVR(v, v, (size_t) dst);\n\n uint32_t left_off = (size_t) dst & 127;\n uint32_t right_off = left_off + n;\n\n HVX_VectorPred ql_not = Q6_Q_vsetq_R((size_t) dst);\n HVX_VectorPred qr = Q6_Q_vsetq2_R(right_off);\n\n if (right_off > 128) {\n Q6_vmem_QRIV(qr, (HVX_Vector *) dst + 1, v);\n // all 1's\n qr = Q6_Q_vcmp_eq_VbVb(v, v);\n }\n\n ql_not = Q6_Q_or_QQn(ql_not, qr);\n Q6_vmem_QnRIV(ql_not, (HVX_Vector *) dst, v);\n}\n\nstatic inline void hvx_vec_store_a(void * restrict dst, uint32_t n, HVX_Vector v) {\n assert((unsigned long) dst % 128 == 0);\n HVX_VectorPred m = Q6_Q_or_QQn(Q6_Q_vsetq_R((unsigned long) dst), Q6_Q_vsetq2_R(n));\n Q6_vmem_QnRIV(m, (HVX_Vector *) dst, v);\n}\n\nstatic inline HVX_Vector hvx_vec_splat_f32(float v) {\n union { float f; uint32_t i; } u = { .f = v };\n return Q6_V_vsplat_R(u.i);\n}\n\nstatic inline HVX_Vector hvx_vec_splat_f16(_Float16 v) {\n union { __fp16 f; uint16_t i; } u = { .f = v };\n return Q6_Vh_vsplat_R(u.i);\n}\n\nstatic inline HVX_Vector hvx_vec_repl4(HVX_Vector v) {\n // vdelta control to replicate first 4 bytes across all elements\n static const uint8_t __attribute__((aligned(128))) repl[128] = {\n 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,\n 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,\n 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,\n 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,\n 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,\n 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,\n 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,\n 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,\n };\n\n HVX_Vector ctrl = *(HVX_Vector *) repl;\n return Q6_V_vdelta_VV(v, ctrl);\n}\n\nstatic inline float hvx_vec_get_f32(HVX_Vector v) {\n float __attribute__((aligned(128))) x;\n hvx_vec_store_a(&x, 4, v);\n return x;\n}\n\nstatic inline int32_t hvx_vec_get_i32(HVX_Vector v) {\n int32_t __attribute__((aligned(128))) x;\n hvx_vec_store_a(&x, 4, v);\n return x;\n}\n\nstatic inline HVX_Vector hvx_vec_abs_f16(HVX_Vector v) {\n // abs by clearing the fp16 sign bit\n HVX_Vector mask = Q6_Vh_vsplat_R(0x7fff);\n return Q6_V_vand_VV(v, mask);\n}\n\nstatic inline HVX_Vector hvx_vec_neg_f16(HVX_Vector v) {\n // neg by setting the fp16 sign bit\n HVX_Vector mask = Q6_Vh_vsplat_R(0x8000);\n return Q6_V_vxor_VV(v, mask);\n}\n\nstatic inline HVX_Vector hvx_vec_abs_f32(HVX_Vector v) {\n // abs by clearing the fp32 sign bit\n HVX_Vector mask = Q6_V_vsplat_R(0x7fffffff);\n return Q6_V_vand_VV(v, mask);\n}\n\nstatic inline HVX_Vector hvx_vec_neg_f32(HVX_Vector v) {\n#if __HVX_ARCH__ > 75\n return Q6_Vsf_vfneg_Vsf(v);\n#else\n // neg by setting the fp32 sign bit\n HVX_Vector mask = Q6_V_vsplat_R(0x80000000);\n return Q6_V_vxor_VV(v, mask);\n#endif // __HVX_ARCH__ > 75\n}\n\nstatic inline HVX_VectorPred hvx_vec_is_nan_f16(HVX_Vector v) {\n const HVX_Vector vnan_exp = Q6_Vh_vsplat_R(0x7C00);\n const HVX_Vector vnan_frac = Q6_Vh_vsplat_R(0x7FFF);\n\n // get pred of which are NaN, i.e., exponent bits all 1s and fraction bits non 0s\n HVX_VectorPred p_exp = Q6_Q_vcmp_eq_VhVh(Q6_V_vand_VV(v, vnan_exp), vnan_exp);\n HVX_VectorPred p_frac = Q6_Q_not_Q(Q6_Q_vcmp_eq_VhVh(Q6_V_vand_VV(v, vnan_frac), vnan_exp));\n return Q6_Q_and_QQ(p_exp, p_frac);\n}\n\nstatic inline HVX_Vector hvx_vec_f32_to_f16(HVX_Vector v0, HVX_Vector v1) {\n const HVX_Vector zero = Q6_V_vsplat_R(0);\n HVX_Vector q0 = Q6_Vqf32_vadd_VsfVsf(v0, zero);\n HVX_Vector q1 = Q6_Vqf32_vadd_VsfVsf(v1, zero);\n HVX_Vector v = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(q1, q0)));\n\n#if __HVX_ARCH__ < 79\n // replace NaNs with -INF, older arches produce NaNs for (-INF + 0.0)\n const HVX_Vector neg_inf = hvx_vec_splat_f16(-INFINITY);\n HVX_VectorPred nan = hvx_vec_is_nan_f16(v);\n v = Q6_V_vmux_QVV(nan, neg_inf, v);\n#endif\n\n return v;\n}\n\n/* Q6_Vsf_equals_Vw is only available on v73+.*/\n#if __HVX_ARCH__ < 73\nstatic inline HVX_Vector hvx_vec_i32_to_qf32(HVX_Vector const in)\n{\n HVX_Vector const vzero = Q6_V_vzero();\n HVX_VectorPred is_zero = Q6_Q_vcmp_eq_VwVw(in, vzero);\n HVX_Vector lshift = Q6_Vw_vnormamt_Vw(in);\n HVX_Vector normalized = Q6_Vw_vasl_VwVw(in, lshift);\n HVX_Vector vexp = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7f + 30), lshift);\n HVX_Vector mant = Q6_V_vand_VV(Q6_V_vsplat_R(0xFFFFFF00), normalized);\n HVX_Vector ret = Q6_V_vmux_QVV(is_zero, vzero, Q6_Vw_vadd_VwVw(mant, vexp));\n return ret;\n}\n\nstatic inline HVX_Vector Q6_Vsf_equals_Vw(HVX_Vector const in)\n{\n return Q6_Vsf_equals_Vqf32(hvx_vec_i32_to_qf32(in));\n}\n#endif\n\nstatic inline HVX_Vector hvx_vec_i16_from_hf_rnd_sat(HVX_Vector vin) {\n // This looks complicated.\n // Ideally should just be Q6_Vh_equals_Vhf(vin)\n // but that instruction does not do proper rounding.\n\n // convert to qf32, multiplying by 1.0 in the process.\n HVX_VectorPair v32 = Q6_Wqf32_vmpy_VhfVhf(vin, Q6_Vh_vsplat_R(0x3C00));\n\n // 'in-range' values are +/32752.\n // add 192K to it, convert to sf\n HVX_Vector v192K = Q6_V_vsplat_R(0x48400000);\n HVX_Vector vsf_0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_V_lo_W(v32), v192K));\n HVX_Vector vsf_1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_V_hi_W(v32), v192K));\n\n // for in-range cases, result is {163858... 229360} so the exponent is always 144.\n // if we extract bits 21..0 as a signed quantity, and round 6 bits off, that will be the answer.\n // Start by <<10 to get the final 'sign' bit in bit 15...\n vsf_0 = Q6_Vw_vasl_VwR(vsf_0, 10);\n vsf_1 = Q6_Vw_vasl_VwR(vsf_1, 10);\n\n // now round down to 16\n return Q6_Vh_vround_VwVw_sat(vsf_1, vsf_0);\n}\n\n#if __HVX_ARCH__ < 79\n\nstatic inline HVX_VectorPair hvx_vec_mpyacc_f32_f16(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y)\n{\n HVX_VectorPair m = Q6_Wqf32_vmpy_VhfVhf(x, y);\n HVX_Vector a0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_V_lo_W(m), Q6_V_lo_W(acc)));\n HVX_Vector a1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_V_hi_W(m), Q6_V_hi_W(acc)));\n return Q6_W_vcombine_VV(a1, a0);\n}\n\n#else\n\nstatic inline HVX_VectorPair hvx_vec_mpyacc_f32_f16(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y)\n{\n return Q6_Wsf_vmpyacc_WsfVhfVhf(acc, x, y);\n}\n\n#endif\n\n#if __HVX_ARCH__ < 79\n\nstatic inline HVX_Vector hvx_vec_add_f16_f16(HVX_Vector a, HVX_Vector b)\n{\n const HVX_Vector negone = Q6_Vh_vsplat_R(0xBC00); // -1.0 in IEEE FP16\n const HVX_Vector one = Q6_Vh_vsplat_R(0x3C00); // 1.0 in IEEE FP16\n HVX_VectorPair a_p = Q6_Wqf32_vmpy_VhfVhf(a, one);\n HVX_VectorPair b_p = Q6_Wqf32_vmpy_VhfVhf(b, negone);\n HVX_Vector a0 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_lo_W(a_p), Q6_V_lo_W(b_p));\n HVX_Vector a1 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_hi_W(a_p), Q6_V_hi_W(b_p));\n return Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(a1, a0));\n}\n\nstatic inline HVX_Vector hvx_vec_sub_f16_f16(HVX_Vector a, HVX_Vector b)\n{\n const HVX_Vector negone = Q6_Vh_vsplat_R(0xBC00); // -1.0 in IEEE FP16\n const HVX_Vector one = Q6_Vh_vsplat_R(0x3C00); // 1.0 in IEEE FP16\n HVX_VectorPair a_p = Q6_Wqf32_vmpy_VhfVhf(a, one);\n HVX_VectorPair b_p = Q6_Wqf32_vmpy_VhfVhf(b, negone);\n HVX_Vector a0 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(a_p), Q6_V_lo_W(b_p));\n HVX_Vector a1 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_hi_W(a_p), Q6_V_hi_W(b_p));\n return Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(a1, a0));\n}\n\nstatic inline HVX_Vector hvx_vec_mul_f16_f16(HVX_Vector a, HVX_Vector b)\n{\n return Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(a, b));\n}\n\n#else\n\nstatic inline HVX_Vector hvx_vec_add_f16_f16(HVX_Vector a, HVX_Vector b)\n{\n return Q6_Vhf_vadd_VhfVhf(a, b);\n}\n\nstatic inline HVX_Vector hvx_vec_sub_f16_f16(HVX_Vector a, HVX_Vector b)\n{\n return Q6_Vhf_vsub_VhfVhf(a, b);\n}\n\nstatic inline HVX_Vector hvx_vec_mul_f16_f16(HVX_Vector a, HVX_Vector b)\n{\n return Q6_Vhf_vmpy_VhfVhf(a, b);\n}\n\n#endif // __HVX_ARCH__ < 79\n\n#endif /* HVX_BASE_H */\n" }, { "path": "src/ggml-hexagon/htp/hvx-copy.h", "content": "#ifndef HVX_COPY_H\n#define HVX_COPY_H\n\n#include \n#include \n#include \n\n#include \"hvx-base.h\"\n\n#define hvx_splat_loop_body(dst_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n \\\n uint32_t nvec = n / (128 / elem_size); \\\n uint32_t nloe = n % (128 / elem_size); \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n vdst[i] = src; \\\n } \\\n if (nloe) { \\\n vec_store((void *) &vdst[i], nloe * elem_size, src); \\\n } \\\n } while(0)\n\nstatic inline void hvx_splat_a(uint8_t * restrict dst, HVX_Vector src, uint32_t n, uint32_t elem_size) {\n assert((unsigned long) dst % 128 == 0);\n hvx_splat_loop_body(HVX_Vector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_splat_u(uint8_t * restrict dst, HVX_Vector src, uint32_t n, uint32_t elem_size) {\n hvx_splat_loop_body(HVX_UVector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_splat_f32_a(uint8_t * restrict dst, float v, uint32_t n) {\n hvx_splat_a(dst, hvx_vec_splat_f32(v), n, sizeof(float));\n}\n\nstatic inline void hvx_splat_f32_u(uint8_t * restrict dst, float v, uint32_t n) {\n hvx_splat_u(dst, hvx_vec_splat_f32(v), n, sizeof(float));\n}\n\nstatic inline void hvx_splat_f16_a(uint8_t * restrict dst, _Float16 v, uint32_t n) {\n hvx_splat_u(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16));\n}\n\nstatic inline void hvx_splat_f16_u(uint8_t * restrict dst, _Float16 v, uint32_t n) {\n hvx_splat_u(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16));\n}\n\n#define hvx_copy_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const uint32_t epv = 128 / elem_size; \\\n const uint32_t nvec = n / epv; \\\n const uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { vdst[i] = vsrc[i]; } \\\n if (nloe) { \\\n vec_store((void *) &vdst[i], nloe * elem_size, vsrc[i]); \\\n } \\\n } while(0)\n\n// Generic copy routines\nstatic inline void hvx_copy_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n, uint32_t elem_size) {\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src % 128 == 0);\n hvx_copy_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_copy_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n, uint32_t elem_size) {\n assert((unsigned long) dst % 128 == 0);\n hvx_copy_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_copy_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n, uint32_t elem_size) {\n assert((unsigned long) src % 128 == 0);\n hvx_copy_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_copy_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n, uint32_t elem_size) {\n hvx_copy_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);\n}\n\n// copy n fp16 elements : source and destination are aligned to HVX Vector (128)\nstatic inline void hvx_copy_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_aa(dst, src, n, sizeof(__fp16));\n}\n\n// copy n fp16 elements : source is aligned, destination is potentially unaligned\nstatic inline void hvx_copy_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_au(dst, src, n, sizeof(__fp16));\n}\n\n// copy n fp16 elements : source is aligned, destination is potentially unaligned\nstatic inline void hvx_copy_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_ua(dst, src, n, sizeof(__fp16));\n}\n\n// copy n fp16 elements : source is aligned, destination is potentially unaligned\nstatic inline void hvx_copy_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_uu(dst, src, n, sizeof(__fp16));\n}\n\n// copy n fp32 elements : source and destination are aligned to HVX Vector (128)\nstatic inline void hvx_copy_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_aa(dst, src, n, sizeof(float));\n}\n\n// copy n fp32 elements : source is aligned, destination is unaligned\nstatic inline void hvx_copy_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_ua(dst, src, n, sizeof(float));\n}\n\n// copy n fp32 elements : source is unaligned, destination is aligned\nstatic inline void hvx_copy_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_au(dst, src, n, sizeof(float));\n}\n\n// copy n fp32 elements : source is unaligned, destination unaligned\nstatic inline void hvx_copy_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_uu(dst, src, n, sizeof(float));\n}\n\n//// fp32 -> fp16\n\n#define hvx_copy_f16_f32_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const uint32_t elem_size = sizeof(__fp16); \\\n const uint32_t epv = 128 / elem_size; \\\n const uint32_t nvec = n / epv; \\\n const uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n vdst[i] = hvx_vec_f32_to_f16(vsrc[i*2+0], vsrc[i*2+1]); \\\n } \\\n if (nloe) { \\\n HVX_Vector v = hvx_vec_f32_to_f16(vsrc[i*2+0], vsrc[i*2+1]); \\\n vec_store((void *) &vdst[i], nloe * elem_size, v); \\\n } \\\n } while(0)\n\n// copy/convert n fp32 elements into n fp16 elements : source is aligned, destination is aligned\nstatic inline void hvx_copy_f16_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src % 128 == 0);\n hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\n\n// copy/convert n fp32 elements into n fp16 elements : source is unaligned, destination is aligned\nstatic inline void hvx_copy_f16_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);\n}\n\n// copy/convert n fp32 elements into n fp16 elements : source is aligned, destination is unaligned\nstatic inline void hvx_copy_f16_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) src % 128 == 0);\n hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);\n}\n\n// copy/convert n fp32 elements into n fp16 elements : source is unaligned, destination is unaligned\nstatic inline void hvx_copy_f16_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);\n}\n\n//// fp16 -> fp32\n\n#define hvx_copy_f32_f16_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const HVX_Vector one = hvx_vec_splat_f16(1.0); \\\n \\\n const uint32_t elem_size = sizeof(__fp16); \\\n const uint32_t epv = 128 / elem_size; \\\n const uint32_t nvec = n / epv; \\\n uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (i = 0; i < nvec; ++i) { \\\n HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vsrc[i]), one); \\\n vdst[i*2] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); \\\n vdst[i*2+1] = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); \\\n } \\\n \\\n if (nloe) { \\\n HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vsrc[i]), one); \\\n \\\n HVX_Vector vd = Q6_V_lo_W(p); \\\n i = 2 * i; \\\n \\\n if (nloe >= 32) { \\\n vdst[i] = Q6_Vsf_equals_Vqf32(vd); \\\n nloe -= 32; ++i; vd = Q6_V_hi_W(p); \\\n } \\\n \\\n if (nloe) { \\\n vd = Q6_Vsf_equals_Vqf32(vd); \\\n hvx_vec_store_u(&vdst[i], nloe * sizeof(float), vd); \\\n } \\\n } \\\n } while(0)\n\n// copy/convert n fp16 elements into n fp32 elements : source is aligned, destination is aligned\nstatic inline void hvx_copy_f32_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src % 128 == 0);\n hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\n\n// copy/convert n fp16 elements into n fp32 elements : source is unaligned, destination is aligned\nstatic inline void hvx_copy_f32_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);\n}\n\n// copy/convert n fp16 elements into n fp32 elements : source is aligned, destination is unaligned\nstatic inline void hvx_copy_f32_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) src % 128 == 0);\n hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);\n}\n\n// copy/convert n fp16 elements into n fp32 elements : source is unaligned, destination is unaligned\nstatic inline void hvx_copy_f32_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);\n}\n\n#endif // HVX_COPY_H\n" }, { "path": "src/ggml-hexagon/htp/hvx-div.h", "content": "#ifndef HVX_DIV_H\n#define HVX_DIV_H\n\n#include \n\n#include \n#include \n#include \n#include \n#include \n\n#include \"hvx-base.h\"\n#include \"hex-utils.h\"\n#include \"hvx-inverse.h\"\n#include \"hvx-arith.h\"\n\n#if __HVX_ARCH__ < 79\n#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b))\n#else\n#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b)\n#endif\n\n// Compute div by scaler in f32. Requires first by expanding fp32 to fp16 and converting the result back to fp32.\nstatic inline HVX_Vector hvx_div_mul_f16_const_using_f32(HVX_Vector vec1_hf, HVX_Vector vec2_sf_const, HVX_Vector vec_hf_one_1_0) {\n#if __HVX_ARCH__ < 79\n HVX_VectorPair src_to_f32 = Q6_Wqf32_vmpy_VhfVhf(vec1_hf, vec_hf_one_1_0);\n HVX_Vector src_to_f32_0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(src_to_f32));\n HVX_Vector src_to_f32_1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(src_to_f32));\n#else\n HVX_VectorPair src_to_f32 = Q6_Wsf_vmpy_VhfVhf(vec1_hf, vec_hf_one_1_0);\n HVX_Vector src_to_f32_0 = Q6_V_lo_W(src_to_f32);\n HVX_Vector src_to_f32_1 = Q6_V_hi_W(src_to_f32);\n#endif\n\n HVX_Vector div_f32_0 = HVX_OP_MUL_F32(src_to_f32_0, vec2_sf_const);\n HVX_Vector div_f32_1 = HVX_OP_MUL_F32(src_to_f32_1, vec2_sf_const);\n\n#if __HVX_ARCH__ < 79\n HVX_Vector res = hvx_vec_f32_to_f16(div_f32_0, div_f32_1);\n#else\n HVX_Vector res = Q6_Vhf_vcvt_VsfVsf(div_f32_0, div_f32_1);\n#endif\n return res;\n}\n\n#define hvx_div_scaler_f16_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n HVX_Vector hf_one = Q6_Vh_vsplat_R(0x3C00); \\\n \\\n const uint32_t nvec = n / VLEN_FP16; \\\n const uint32_t nloe = n % VLEN_FP16; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n HVX_Vector res = hvx_div_mul_f16_const_using_f32(vsrc[i], val_vec_f32, hf_one); \\\n vdst[i] = res; \\\n } \\\n if (nloe) { \\\n HVX_Vector res = hvx_div_mul_f16_const_using_f32(vsrc[i], val_vec_f32, hf_one); \\\n vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, res); \\\n } \\\n } while(0)\n\nstatic inline void hvx_div_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {\n const HVX_Vector val_vec_f32 = hvx_vec_splat_f32(1.0f/((float)val));\n assert((uintptr_t) dst % 128 == 0);\n assert((uintptr_t) src % 128 == 0);\n hvx_div_scaler_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\nstatic inline void hvx_div_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {\n const HVX_Vector val_vec_f32 = hvx_vec_splat_f32(1.0f/((float)val));\n assert((uintptr_t) dst % 128 == 0);\n hvx_div_scaler_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);\n}\nstatic inline void hvx_div_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {\n const HVX_Vector val_vec_f32 = hvx_vec_splat_f32(1.0f/((float)val));\n assert((uintptr_t) src % 128 == 0);\n hvx_div_scaler_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);\n}\nstatic inline void hvx_div_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {\n const HVX_Vector val_vec_f32 = hvx_vec_splat_f32(1.0f/((float)val));\n hvx_div_scaler_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);\n}\n\n// Compute div by using hvx_vec_inverse_f32_guard. Requires first by exapnding fp32 to fp16 and convert the result back to fp32.\nstatic inline HVX_Vector hvx_vec_div_f16_using_f32(HVX_Vector vec1, HVX_Vector vec2, HVX_Vector f32_nan_inf_mask, HVX_Vector vec_hf_one_1_0) {\n#if __HVX_ARCH__ < 79\n // Convert first input to fp32\n HVX_VectorPair vec1_to_f32 = Q6_Wqf32_vmpy_VhfVhf(vec1, vec_hf_one_1_0); // *1.0\n HVX_Vector vec1_to_f32_0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vec1_to_f32));\n HVX_Vector vec1_to_f32_1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vec1_to_f32));\n\n // Convert second input to fp32\n HVX_VectorPair vec2_to_f32 = Q6_Wqf32_vmpy_VhfVhf(vec2, vec_hf_one_1_0); // *1.0\n HVX_Vector vec2_to_f32_0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vec2_to_f32));\n HVX_Vector vec2_to_f32_1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vec2_to_f32));\n#else\n // Convert first input to fp32\n HVX_VectorPair vec1_to_f32 = Q6_Wsf_vmpy_VhfVhf(vec1, vec_hf_one_1_0); // *1.0\n HVX_Vector vec1_to_f32_0 = Q6_V_lo_W(vec1_to_f32);\n HVX_Vector vec1_to_f32_1 = Q6_V_hi_W(vec1_to_f32);\n\n // Convert second input to fp32\n HVX_VectorPair vec2_to_f32 = Q6_Wsf_vmpy_VhfVhf(vec2, vec_hf_one_1_0); // *1.0\n HVX_Vector vec2_to_f32_0 = Q6_V_lo_W(vec2_to_f32);\n HVX_Vector vec2_to_f32_1 = Q6_V_hi_W(vec2_to_f32);\n#endif\n\n // Inverse second input in fp32\n HVX_Vector vec2_inv_f32_0 = hvx_vec_inverse_f32_guard(vec2_to_f32_0, f32_nan_inf_mask);\n HVX_Vector vec2_inv_f32_1 = hvx_vec_inverse_f32_guard(vec2_to_f32_1, f32_nan_inf_mask);\n\n // Multiply first input by inverse of second, in fp32\n HVX_Vector div_f32_0 = HVX_OP_MUL_F32(vec1_to_f32_0, vec2_inv_f32_0);\n HVX_Vector div_f32_1 = HVX_OP_MUL_F32(vec1_to_f32_1, vec2_inv_f32_1);\n\n // Convert back to fp16\n#if __HVX_ARCH__ < 79\n HVX_Vector recip = hvx_vec_f32_to_f16(div_f32_0, div_f32_1);\n#else\n HVX_Vector recip = Q6_Vhf_vcvt_VsfVsf(div_f32_0, div_f32_1);\n#endif\n\n return recip;\n}\n\n#define hvx_div_f16_loop_body(dst_type, src0_type, src1_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src0_type * restrict vsrc0 = (src0_type *) src0; \\\n src1_type * restrict vsrc1 = (src1_type *) src1; \\\n \\\n const HVX_Vector nan_inf_mask = Q6_V_vsplat_R(0x7f800000); \\\n const HVX_Vector hf_one = Q6_Vh_vsplat_R(0x3C00); \\\n \\\n const uint32_t nvec = n / VLEN_FP16; \\\n const uint32_t nloe = n % VLEN_FP16; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n HVX_Vector res = hvx_vec_div_f16_using_f32(vsrc0[i], vsrc1[i], nan_inf_mask, hf_one); \\\n vdst[i] = res; \\\n } \\\n if (nloe) { \\\n HVX_Vector res = hvx_vec_div_f16_using_f32(vsrc0[i], vsrc1[i], nan_inf_mask, hf_one); \\\n vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, res); \\\n } \\\n } while(0)\n\n#define hvx_div_f32_loop_body(dst_type, src0_type, src1_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src0_type * restrict vsrc0 = (src0_type *) src0; \\\n src1_type * restrict vsrc1 = (src1_type *) src1; \\\n \\\n const HVX_Vector nan_inf_mask = Q6_V_vsplat_R(0x7f800000); \\\n \\\n const uint32_t nvec = n / VLEN_FP32; \\\n const uint32_t nloe = n % VLEN_FP32; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n HVX_Vector inv_src1 = hvx_vec_inverse_f32_guard(vsrc1[i], nan_inf_mask); \\\n HVX_Vector res = HVX_OP_MUL_F32(vsrc0[i], inv_src1); \\\n vdst[i] = res; \\\n } \\\n if (nloe) { \\\n HVX_Vector inv_src1 = hvx_vec_inverse_f32_guard(vsrc1[i], nan_inf_mask); \\\n HVX_Vector res = HVX_OP_MUL_F32(vsrc0[i], inv_src1); \\\n vec_store((void *) &vdst[i], nloe * SIZEOF_FP32, res); \\\n } \\\n } while(0)\n\n// Generic macro to define alignment permutations for an op\n#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \\\nstatic inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n assert((uintptr_t) src0 % 128 == 0); \\\n assert((uintptr_t) src1 % 128 == 0); \\\n OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \\\n} \\\nstatic inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n assert((uintptr_t) src0 % 128 == 0); \\\n OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \\\n} \\\nstatic inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n assert((uintptr_t) src1 % 128 == 0); \\\n OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \\\n} \\\nstatic inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \\\n} \\\nstatic inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) src0 % 128 == 0); \\\n assert((uintptr_t) src1 % 128 == 0); \\\n OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \\\n} \\\nstatic inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) src0 % 128 == 0); \\\n OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \\\n} \\\nstatic inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n assert((uintptr_t) src1 % 128 == 0); \\\n OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \\\n} \\\nstatic inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \\\n OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \\\n} \\\n\n// Dispatcher logic\n#define HVX_DIV_DISPATCHER(OP_NAME) \\\nstatic inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \\\n if (hex_is_aligned((void *) dst, 128)) { \\\n if (hex_is_aligned((void *) src0, 128)) { \\\n if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \\\n else OP_NAME##_aau(dst, src0, src1, num_elems); \\\n } else { \\\n if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \\\n else OP_NAME##_auu(dst, src0, src1, num_elems); \\\n } \\\n } else { \\\n if (hex_is_aligned((void *) src0, 128)) { \\\n if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \\\n else OP_NAME##_uau(dst, src0, src1, num_elems); \\\n } else { \\\n if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \\\n else OP_NAME##_uuu(dst, src0, src1, num_elems); \\\n } \\\n } \\\n}\n\nDEFINE_HVX_DIV_OP_VARIANTS(hvx_div_f32, hvx_div_f32_loop_body)\nDEFINE_HVX_DIV_OP_VARIANTS(hvx_div_f16, hvx_div_f16_loop_body)\n\nHVX_DIV_DISPATCHER(hvx_div_f32)\nHVX_DIV_DISPATCHER(hvx_div_f16)\n\n#undef HVX_OP_MUL_F32\n\n#endif // HVX_DIV_H\n" }, { "path": "src/ggml-hexagon/htp/hvx-dump.h", "content": "#ifndef HVX_DUMP_H\n#define HVX_DUMP_H\n\n#include \n\n#include \n#include \n\n#include \"hex-utils.h\"\n#include \"hvx-types.h\"\n\nstatic void hvx_vec_dump_f16_n(char * pref, HVX_Vector v, uint32_t n) {\n HVX_VectorAlias u = { .v = v };\n\n const uint32_t n0 = n / 16;\n const uint32_t n1 = n % 16;\n int i = 0;\n for (; i < n0; i++) {\n hex_dump_f16_line(pref, u.fp16 + (16 * i), 16);\n }\n if (n1) {\n hex_dump_f16_line(pref, u.fp16 + (16 * i), n1);\n }\n}\n\nstatic void hvx_vec_dump_f16(char * pref, HVX_Vector v) {\n hvx_vec_dump_f16_n(pref, v, 64);\n}\n\nstatic void hvx_vec_dump_f32_n(char * pref, HVX_Vector v, uint32_t n) {\n HVX_VectorAlias u = { .v = v };\n\n const uint32_t n0 = n / 16;\n const uint32_t n1 = n % 16;\n int i = 0;\n for (; i < n0; i++) {\n hex_dump_f32_line(pref, u.fp32 + (16 * i), 16);\n }\n if (n1) {\n hex_dump_f32_line(pref, u.fp32 + (16 * i), n1);\n }\n}\n\nstatic void hvx_vec_dump_f32_hmt(char * pref, HVX_Vector v) {\n union {\n HVX_Vector v;\n float d[32];\n } u = { .v = v };\n\n FARF(HIGH, \"%s: %.6f %.6f %.6f %.6f ... %.6f %.6f %.6f %.6f ... %.6f %.6f %.6f %.6f\\n\", pref, u.d[0], u.d[1],\n u.d[2], u.d[3], u.d[12], u.d[13], u.d[14], u.d[15], u.d[28], u.d[29], u.d[30], u.d[31]);\n}\n\nstatic void hvx_vec_dump_f32(char * pref, HVX_Vector v) {\n hvx_vec_dump_f32_n(pref, v, 32);\n}\n\nstatic void hvx_vec_dump_int32(char * pref, HVX_Vector v) {\n union {\n HVX_Vector v;\n int32_t d[32];\n } u = { .v = v };\n\n for (int i = 0; i < 32 / 16; i++) {\n hex_dump_int32_line(pref, u.d + (16 * i), 16);\n }\n}\n\nstatic void hvx_vec_dump_int32_hmt(char * pref, HVX_Vector v) {\n union {\n HVX_Vector v;\n int32_t d[32];\n } u = { .v = v };\n\n FARF(HIGH, \"%s: %d %d %d %d ... %d %d %d %d ... %d %d %d %d\\n\", pref, u.d[0], u.d[1], u.d[2], u.d[3], u.d[12],\n u.d[13], u.d[14], u.d[15], u.d[28], u.d[29], u.d[30], u.d[31]);\n}\n\nstatic void hvx_vec_dump_int8_hmt(char * pref, HVX_Vector v) {\n union {\n HVX_Vector v;\n int8_t d[128];\n } u = { .v = v };\n\n FARF(HIGH, \"%s: %d %d %d %d ... %d %d %d %d ... %d %d %d %d\\n\", pref, u.d[0], u.d[1], u.d[2], u.d[3], u.d[60],\n u.d[61], u.d[62], u.d[63], u.d[124], u.d[125], u.d[126], u.d[127]);\n}\n\nstatic void hvx_vec_dump_int8(char * pref, HVX_Vector v) {\n union {\n HVX_Vector v;\n int8_t d[128];\n } u = { .v = v };\n\n for (int i = 0; i < 128 / 16; i++) {\n hex_dump_int8_line(pref, u.d + (16 * i), 16);\n }\n}\n\nstatic void hvx_vec_dump_uint8(char * pref, HVX_Vector v) {\n union {\n HVX_Vector v;\n uint8_t d[128];\n } u = { .v = v };\n\n for (int i = 0; i < 128 / 16; i++) {\n hex_dump_uint8_line(pref, u.d + (16 * i), 16);\n }\n}\n\nstatic bool hvx_vec_eq(HVX_Vector v0, HVX_Vector v1, size_t n) {\n typedef union {\n HVX_Vector v;\n int8_t d[128];\n } U;\n\n U u0 = { .v = v0 };\n U u1 = { .v = v1 };\n\n for (int i = 0; i < n; i++) {\n if (u0.d[i] != u1.d[i]) {\n return false;\n }\n }\n\n return true;\n}\n\n#endif /* HVX_DUMP_H */\n" }, { "path": "src/ggml-hexagon/htp/hvx-exp.h", "content": "#ifndef HVX_EXP_H\n#define HVX_EXP_H\n\n#include \n#include \n\n#include \"hvx-base.h\"\n#include \"hvx-floor.h\"\n\n#define EXP_COEFF_5 (0x39506967) // 0.000198757 = 1/(7!)\n#define EXP_COEFF_4 (0x3AB743CE) // 0.0013982 = 1/(6!)\n#define EXP_COEFF_3 (0x3C088908) // 0.00833345 = 1/(5!)\n#define EXP_COEFF_2 (0x3D2AA9C1) // 0.416658 = 1/(4!)\n#define EXP_COEFF_1 (0x3E2AAAAA) // 0.16666667 = 1/(3!)\n#define EXP_COEFF_0 (0x3F000000) // 0.5 = 1/(2!)\n#define EXP_LOGN2 (0x3F317218) // ln(2) = 0.6931471805\n#define EXP_LOG2E (0x3FB8AA3B) // log2(e) = 1/ln(2) = 1.4426950408\n#define EXP_ONE (0x3f800000) // 1.0\n#define EXP_RANGE_R (0x41a00000) // 20.0\n#define EXP_RANGE_L (0xc1a00000) // -20.0\n\nstatic inline HVX_Vector hvx_vec_exp_f32(HVX_Vector in_vec) {\n HVX_Vector z_qf32_v;\n HVX_Vector x_v;\n HVX_Vector x_qf32_v;\n HVX_Vector y_v;\n HVX_Vector k_v;\n HVX_Vector f_v;\n HVX_Vector epsilon_v;\n HVX_Vector log2e = Q6_V_vsplat_R(EXP_LOG2E);\n HVX_Vector logn2 = Q6_V_vsplat_R(EXP_LOGN2);\n HVX_Vector E_const;\n HVX_Vector zero_v = Q6_V_vzero();\n\n // exp(x) is approximated as follows:\n // f = floor(x/ln(2)) = floor(x*log2(e))\n // epsilon = x - f*ln(2)\n // exp(x) = exp(epsilon+f*ln(2))\n // = exp(epsilon)*exp(f*ln(2))\n // = exp(epsilon)*2^f\n //\n // Since epsilon is close to zero, it can be approximated with its Taylor series:\n // exp(x) ~= 1+x+x^2/2!+x^3/3!+...+x^n/n!+...\n // Preserving the first eight elements, we get:\n // exp(x) ~= 1+x+e0*x^2+e1*x^3+e2*x^4+e3*x^5+e4*x^6+e5*x^7\n // = 1+x+(E0+(E1+(E2+(E3+(E4+E5*x)*x)*x)*x)*x)*x^2\n\n HVX_Vector temp_v = in_vec;\n\n // Clamp inputs to (-20.0, 20.0)\n HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VsfVsf(in_vec, Q6_V_vsplat_R(EXP_RANGE_R));\n HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VsfVsf(Q6_V_vsplat_R(EXP_RANGE_L), in_vec);\n\n in_vec = Q6_V_vmux_QVV(pred_cap_right, Q6_V_vsplat_R(EXP_RANGE_R), temp_v);\n in_vec = Q6_V_vmux_QVV(pred_cap_left, Q6_V_vsplat_R(EXP_RANGE_L), temp_v);\n\n epsilon_v = Q6_Vqf32_vmpy_VsfVsf(log2e, in_vec);\n epsilon_v = Q6_Vsf_equals_Vqf32(epsilon_v);\n\n // f_v is the floating point result and k_v is the integer result\n f_v = hvx_vec_floor_f32(epsilon_v);\n k_v = hvx_vec_truncate_f32(f_v);\n\n x_qf32_v = Q6_Vqf32_vadd_VsfVsf(in_vec, zero_v);\n\n // x = x - f_v * logn2;\n epsilon_v = Q6_Vqf32_vmpy_VsfVsf(f_v, logn2);\n x_qf32_v = Q6_Vqf32_vsub_Vqf32Vqf32(x_qf32_v, epsilon_v);\n // normalize before every QFloat's vmpy\n x_qf32_v = Q6_Vqf32_vadd_Vqf32Vsf(x_qf32_v, zero_v);\n\n // z = x * x;\n z_qf32_v = Q6_Vqf32_vmpy_Vqf32Vqf32(x_qf32_v, x_qf32_v);\n z_qf32_v = Q6_Vqf32_vadd_Vqf32Vsf(z_qf32_v, zero_v);\n\n x_v = Q6_Vsf_equals_Vqf32(x_qf32_v);\n\n // y = E4 + E5 * x;\n E_const = Q6_V_vsplat_R(EXP_COEFF_5);\n y_v = Q6_Vqf32_vmpy_VsfVsf(E_const, x_v);\n E_const = Q6_V_vsplat_R(EXP_COEFF_4);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);\n\n // y = E3 + y * x;\n E_const = Q6_V_vsplat_R(EXP_COEFF_3);\n y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, x_qf32_v);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);\n\n // y = E2 + y * x;\n E_const = Q6_V_vsplat_R(EXP_COEFF_2);\n y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, x_qf32_v);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);\n\n // y = E1 + y * x;\n E_const = Q6_V_vsplat_R(EXP_COEFF_1);\n y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, x_qf32_v);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);\n\n // y = E0 + y * x;\n E_const = Q6_V_vsplat_R(EXP_COEFF_0);\n y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, x_qf32_v);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);\n\n // y = x + y * z;\n y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, z_qf32_v);\n y_v = Q6_Vqf32_vadd_Vqf32Vqf32(y_v, x_qf32_v);\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);\n\n // y = y + 1.0;\n y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, Q6_V_vsplat_R(EXP_ONE));\n\n // insert exponents\n // y = ldexpf(y, k);\n // y_v += k_v; // qf32\n // modify exponent\n\n y_v = Q6_Vsf_equals_Vqf32(y_v);\n\n // add k_v to the exponent of y_v\n HVX_Vector y_v_exponent = Q6_Vw_vasl_VwR(y_v, 1);\n\n y_v_exponent = Q6_Vuw_vlsr_VuwR(y_v_exponent, IEEE_VSF_MANTLEN + 1);\n y_v_exponent = Q6_Vw_vadd_VwVw(k_v, y_v_exponent);\n\n // exponent cannot be negative; if overflow is detected, result is set to zero\n HVX_VectorPred qy_v_negative_exponent = Q6_Q_vcmp_gt_VwVw(zero_v, y_v_exponent);\n\n y_v = Q6_Vw_vaslacc_VwVwR(y_v, k_v, IEEE_VSF_MANTLEN);\n\n y_v = Q6_V_vmux_QVV(qy_v_negative_exponent, zero_v, y_v);\n\n return y_v;\n}\n\nstatic inline HVX_Vector hvx_vec_exp_f32_guard(HVX_Vector in_vec, HVX_Vector max_exp, HVX_Vector inf) {\n const HVX_VectorPred pred0 = Q6_Q_vcmp_gt_VsfVsf(in_vec, max_exp);\n\n HVX_Vector out = hvx_vec_exp_f32(in_vec);\n\n return Q6_V_vmux_QVV(pred0, inf, out);\n}\n\nstatic inline void hvx_exp_f32(const uint8_t * restrict src, uint8_t * restrict dst, const int num_elems, bool negate) {\n int left_over = num_elems & (VLEN_FP32 - 1);\n int num_elems_whole = num_elems - left_over;\n\n int unaligned_addr = 0;\n int unaligned_loop = 0;\n if ((0 == hex_is_aligned((void *) src, VLEN)) || (0 == hex_is_aligned((void *) dst, VLEN))) {\n unaligned_addr = 1;\n }\n // assert((0 == unaligned_addr) || (0 == num_elems_whole));\n if ((1 == unaligned_addr) && (num_elems_whole != 0)) {\n unaligned_loop = 1;\n }\n\n HVX_Vector vec_out = Q6_V_vzero();\n\n static const float kInf = INFINITY;\n static const float kMaxExp = 88.02f; // log(INF)\n\n const HVX_Vector max_exp = hvx_vec_splat_f32(kMaxExp);\n const HVX_Vector inf = hvx_vec_splat_f32(kInf);\n\n if (0 == unaligned_loop) {\n HVX_Vector * p_vec_in1 = (HVX_Vector *) src;\n HVX_Vector * p_vec_out = (HVX_Vector *) dst;\n\n #pragma unroll(4)\n for (int i = 0; i < num_elems_whole; i += VLEN_FP32) {\n if (true == negate) {\n HVX_Vector neg_vec_in = hvx_vec_neg_f32(*p_vec_in1++);\n *p_vec_out++ = hvx_vec_exp_f32_guard(neg_vec_in, max_exp, inf);\n } else {\n *p_vec_out++ = hvx_vec_exp_f32_guard(*p_vec_in1++, max_exp, inf);\n }\n }\n } else {\n #pragma unroll(4)\n for (int i = 0; i < num_elems_whole; i += VLEN_FP32) {\n HVX_Vector in = *(HVX_UVector *) (src + i * SIZEOF_FP32);\n\n if (true == negate) {\n HVX_Vector neg_vec_in = hvx_vec_neg_f32(in);\n *(HVX_UVector *) (dst + i * SIZEOF_FP32) = hvx_vec_exp_f32_guard(neg_vec_in, max_exp, inf);\n } else {\n *(HVX_UVector *) (dst + i * SIZEOF_FP32) = hvx_vec_exp_f32_guard(in, max_exp, inf);\n }\n }\n }\n\n if (left_over > 0) {\n const float * srcf = (float *) src + num_elems_whole;\n float * dstf = (float *) dst + num_elems_whole;\n\n HVX_Vector in = *(HVX_UVector *) srcf;\n\n if (true == negate) {\n HVX_Vector neg_vec_in = hvx_vec_neg_f32(in);\n\n vec_out = hvx_vec_exp_f32_guard(neg_vec_in, max_exp, inf);\n } else {\n vec_out = hvx_vec_exp_f32_guard(in, max_exp, inf);\n }\n\n hvx_vec_store_u((void *) dstf, left_over * SIZEOF_FP32, vec_out);\n }\n}\n\n#endif /* HVX_EXP_H */\n" }, { "path": "src/ggml-hexagon/htp/hvx-floor.h", "content": "#ifndef HVX_FLOOR_H\n#define HVX_FLOOR_H\n\n#include \n#include \n\n#include \"hvx-base.h\"\n\n#define IEEE_VSF_EXPLEN (8)\n#define IEEE_VSF_EXPBIAS (127)\n#define IEEE_VSF_EXPMASK (0xFF)\n#define IEEE_VSF_MANTLEN (23)\n#define IEEE_VSF_MANTMASK (0x7FFFFF)\n#define IEEE_VSF_MIMPMASK (0x800000)\n\nstatic inline HVX_Vector hvx_vec_truncate_f32(HVX_Vector in_vec) {\n HVX_Vector mask_mant_v = Q6_V_vsplat_R(IEEE_VSF_MANTMASK);\n HVX_Vector mask_impl_v = Q6_V_vsplat_R(IEEE_VSF_MIMPMASK);\n HVX_Vector const_zero_v = Q6_V_vzero();\n\n HVX_VectorPred q_negative = Q6_Q_vcmp_gt_VwVw(const_zero_v, in_vec);\n\n HVX_Vector expval_v = in_vec >> IEEE_VSF_MANTLEN;\n expval_v &= IEEE_VSF_EXPMASK;\n expval_v -= IEEE_VSF_EXPBIAS;\n\n // negative exp == fractional value\n HVX_VectorPred q_negexp = Q6_Q_vcmp_gt_VwVw(const_zero_v, expval_v);\n\n HVX_Vector rshift_v = IEEE_VSF_MANTLEN - expval_v; // fractional bits - exp shift\n\n HVX_Vector mant_v = in_vec & mask_mant_v; // obtain mantissa\n HVX_Vector vout = Q6_Vw_vadd_VwVw(mant_v, mask_impl_v); // add implicit 1.0\n\n vout = Q6_Vw_vasr_VwVw(vout, rshift_v); // shift to obtain truncated integer\n vout = Q6_V_vmux_QVV(q_negexp, const_zero_v, vout); // expval<0 -> 0\n\n HVX_Vector neg_vout = -vout;\n\n vout = Q6_V_vmux_QVV(q_negative, neg_vout, vout); // handle negatives\n\n return (vout);\n}\n\nstatic inline HVX_Vector hvx_vec_floor_f32(HVX_Vector in_vec) {\n HVX_Vector mask_mant_v = Q6_V_vsplat_R(IEEE_VSF_MANTMASK);\n HVX_Vector mask_impl_v = Q6_V_vsplat_R(IEEE_VSF_MIMPMASK);\n HVX_Vector const_mnlen_v = Q6_V_vsplat_R(IEEE_VSF_MANTLEN);\n HVX_Vector const_zero_v = Q6_V_vzero();\n HVX_Vector const_negone_v = Q6_V_vsplat_R(0xbf800000); // -1 IEEE vsf\n\n HVX_VectorPred q_negative = Q6_Q_vcmp_gt_VwVw(const_zero_v, in_vec);\n\n HVX_Vector expval_v = in_vec >> IEEE_VSF_MANTLEN;\n expval_v &= IEEE_VSF_EXPMASK;\n expval_v -= IEEE_VSF_EXPBIAS;\n\n HVX_VectorPred q_negexp = Q6_Q_vcmp_gt_VwVw(const_zero_v, expval_v);\n HVX_VectorPred q_expltmn = Q6_Q_vcmp_gt_VwVw(const_mnlen_v, expval_v);\n HVX_VectorPred q_negexp_pos = Q6_Q_vcmp_gtand_QVwVw(q_negexp, in_vec, const_zero_v);\n HVX_VectorPred q_negexp_neg = Q6_Q_vcmp_gtand_QVwVw(q_negexp, const_zero_v, in_vec);\n\n // if expval < 0 (q_negexp) // <0, floor is 0\n // if vin > 0\n // floor = 0\n // if vin < 0\n // floor = -1\n // if expval < mant_len (q_expltmn) // >0, but fraction may exist\n // get sign (q_negative)\n // mask >> expval // fraction bits to mask off\n // vout = ~(mask) // apply mask to remove fraction\n // if (qneg) // negative floor is one less (more, sign bit for neg)\n // vout += ((impl_mask) >> expval)\n // if (mask && vin)\n // vout = vin\n // else // already an integer\n // ; // no change\n\n // compute floor\n mask_mant_v >>= expval_v;\n HVX_Vector neg_addin_v = mask_impl_v >> expval_v;\n HVX_Vector vout_neg_addin = Q6_Vw_vadd_VwVw(in_vec, neg_addin_v);\n HVX_Vector vout = Q6_V_vmux_QVV(q_negative, vout_neg_addin, in_vec);\n\n HVX_Vector mask_chk_v = Q6_V_vand_VV(in_vec, mask_mant_v); // chk if bits set\n HVX_VectorPred q_integral = Q6_Q_vcmp_eq_VwVw(const_zero_v, mask_chk_v);\n\n HVX_Vector not_mask_v = Q6_V_vnot_V(mask_mant_v); // frac bits to clear\n HVX_Vector vfrfloor_v = Q6_V_vand_VV(vout, not_mask_v); // clear frac bits\n\n vout = in_vec;\n vout = Q6_V_vmux_QVV(q_expltmn, vfrfloor_v, vout); // expval0 -> 0\n vout = Q6_V_vmux_QVV(q_negexp_neg, const_negone_v, vout); // expval<0 x<0 -> -1\n\n return vout;\n}\n\n#endif /* HVX_FLOOR_H */\n" }, { "path": "src/ggml-hexagon/htp/hvx-inverse.h", "content": "#ifndef HVX_INVERSE_H\n#define HVX_INVERSE_H\n\n#include \n\n#include \n#include \n#include \n#include \n#include \n\n#include \"hvx-base.h\"\n\n// ====================================================\n// FUNCTION: 1/(x+1) y(0) = 1, y(0.5) = 0.6667, y(1) = 0.5\n// Order:3; continuity: True; Ends forced: True\n// Mode: unsigned; Result fractional bits: 14\n// Peak Error: 1.1295e-04 Rms Error: 2.8410e-05 Mean Error: 1.1370e-05\n// 32769 -32706 31252 -10589\n// 32590 -30635 22793 -4493\n// 32066 -27505 16481 -2348\n// 31205 -24054 11849 -1306\n\nstatic inline HVX_Vector hvx_vec_recip_xp1_O3_unsigned(HVX_Vector vx) {\n // input is 0..0xffff representing 0.0 .. 1.0\n HVX_Vector p;\n p = Q6_Vh_vlut4_VuhPh(vx, 0xFAE6F6D4EE73D6A3ull);\n p = Q6_Vh_vmpa_VhVhVuhPuh_sat(p, vx, 0x2E49406159097A14ull);\n p = Q6_Vh_vmps_VhVhVuhPuh_sat(p, vx, 0x5DF66B7177AB7FC2ull);\n p = Q6_Vh_vmpa_VhVhVuhPuh_sat(p, vx, 0x79E57D427F4E8001ull);\n return p; // signed result, 14 fractional bits\n}\n\n// Find reciprocal of fp16.\n// (1) first, convert to fp32, multiplying by 1.0; this is done to\n// handle denormals. Ignoring sign and zero, result should be at\n// least 5.9604645e-08 (32-bit code 0x33800000) and at most 131008 (0x47ffe000)\n// (exponent in range [103,143])\n// (2) extract the mantissa into 16-bit unsigned; find reciprocal using a fitted poly\n// (3) put this, along with '253-exp' (exp from (1)) together to make an qf32\n// (4) convert that to fp16\n// (5) put sign back in. Also, if the original value (w/o sign) was <0x81, replace\n// the result with the max value.\nstatic inline HVX_Vector hvx_vec_inverse_f16(HVX_Vector vals) {\n HVX_Vector em_mask = Q6_Vh_vsplat_R(0x7FFF);\n HVX_Vector avals = Q6_V_vand_VV(vals, em_mask);\n HVX_VectorPred is_neg = Q6_Q_vcmp_gt_VhVh(avals, vals);\n // is too small to 1/x ? for 'standard' fp16, this would be 0x101\n HVX_VectorPred is_small = Q6_Q_vcmp_gt_VhVh(Q6_Vh_vsplat_R(0x101), avals);\n\n HVX_VectorPair to_qf32 = Q6_Wqf32_vmpy_VhfVhf(avals, Q6_Vh_vsplat_R(0x3C00)); // *1.0\n HVX_Vector to_f32_0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(to_qf32));\n HVX_Vector to_f32_1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(to_qf32));\n\n // bits 22..13 contain the mantissa now (w/o hidden bit); move to bit 14..5 of a 16-bit vector\n HVX_Vector mant_u16 = Q6_Vh_vshuffo_VhVh(Q6_Vw_vasl_VwR(to_f32_1, 9), Q6_Vw_vasl_VwR(to_f32_0, 9));\n // likewise extract the upper 16 from each, containing the exponents in range 103..142\n HVX_Vector exp_u16 = Q6_Vh_vshuffo_VhVh(to_f32_1, to_f32_0);\n //Get exponent in IEEE 32-bit representation\n exp_u16 = Q6_Vuh_vlsr_VuhR(exp_u16, 7);\n\n // so, mant_u16 contains an unbiased mantissa in upper 10 bits of each u16 lane\n // We can consider it to be x-1.0, with 16 fractional bits, where 'x' is in range [1.0,2.0)\n // Use poly to transform to 1/x, with 14 fractional bits\n //\n HVX_Vector rm = hvx_vec_recip_xp1_O3_unsigned(mant_u16);\n\n HVX_Vector vcl0 = Q6_Vuh_vcl0_Vuh(rm); //count leading zeros\n\n // Get mantissa for 16-bit representation\n HVX_Vector mant_recip = Q6_V_vand_VV(Q6_Vh_vasr_VhR(Q6_Vh_vasl_VhVh(rm, vcl0), 5), Q6_Vh_vsplat_R(0x03FF));\n\n //Compute Reciprocal Exponent\n HVX_Vector exp_recip =\n Q6_Vh_vsub_VhVh(Q6_Vh_vsub_VhVh(Q6_Vh_vsplat_R(254), exp_u16), Q6_Vh_vsub_VhVh(vcl0, Q6_Vh_vsplat_R(1)));\n //Convert it for 16-bit representation\n exp_recip = Q6_Vh_vadd_VhVh_sat(Q6_Vh_vsub_VhVh(exp_recip, Q6_Vh_vsplat_R(127)), Q6_Vh_vsplat_R(15));\n exp_recip = Q6_Vh_vasl_VhR(exp_recip, 10);\n\n //Merge exponent and mantissa for reciprocal\n HVX_Vector recip = Q6_V_vor_VV(exp_recip, mant_recip);\n // map 'small' inputs to standard largest value 0x7bff\n recip = Q6_V_vmux_QVV(is_small, Q6_Vh_vsplat_R(0x7bff), recip);\n // add sign back\n recip = Q6_V_vandor_VQR(recip, is_neg, 0x80008000);\n return recip;\n}\n\nstatic inline HVX_Vector hvx_vec_inverse_f32(HVX_Vector v_sf) {\n HVX_Vector inv_aprox_sf = Q6_V_vsplat_R(0x7EEEEBB3);\n HVX_Vector two_sf = hvx_vec_splat_f32(2.0);\n\n // First approximation\n HVX_Vector i_sf = Q6_Vw_vsub_VwVw(inv_aprox_sf, v_sf);\n\n HVX_Vector r_qf;\n\n // Refine\n r_qf = Q6_Vqf32_vmpy_VsfVsf(\n i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v_sf)))));\n r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(\n r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v_sf))));\n r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(\n r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v_sf))));\n\n return Q6_Vsf_equals_Vqf32(r_qf);\n}\n\nstatic inline HVX_Vector hvx_vec_inverse_f32_guard(HVX_Vector v_sf, HVX_Vector nan_inf_mask) {\n HVX_Vector out = hvx_vec_inverse_f32(v_sf);\n\n HVX_Vector masked_out = Q6_V_vand_VV(out, nan_inf_mask);\n const HVX_VectorPred pred = Q6_Q_vcmp_eq_VwVw(nan_inf_mask, masked_out);\n\n return Q6_V_vmux_QVV(pred, Q6_V_vzero(), out);\n}\n\n#define hvx_inverse_f32_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const HVX_Vector nan_inf_mask = Q6_V_vsplat_R(0x7f800000); \\\n \\\n const uint32_t nvec = n / VLEN_FP32; \\\n const uint32_t nloe = n % VLEN_FP32; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n vdst[i] = hvx_vec_inverse_f32_guard(vsrc[i], nan_inf_mask); \\\n } \\\n if (nloe) { \\\n HVX_Vector v = hvx_vec_inverse_f32_guard(vsrc[i], nan_inf_mask); \\\n vec_store((void *) &vdst[i], nloe * SIZEOF_FP32, v); \\\n } \\\n } while(0)\n\nstatic inline HVX_Vector hvx_vec_inverse_f16_guard(HVX_Vector v_sf, HVX_Vector nan_inf_mask) {\n HVX_Vector out = hvx_vec_inverse_f16(v_sf);\n\n HVX_Vector masked_out = Q6_V_vand_VV(out, nan_inf_mask);\n const HVX_VectorPred pred = Q6_Q_vcmp_eq_VhVh(nan_inf_mask, masked_out);\n\n return Q6_V_vmux_QVV(pred, Q6_V_vzero(), out);\n}\n\n#define hvx_inverse_f16_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const HVX_Vector nan_inf_mask = Q6_Vh_vsplat_R(0x7c00); \\\n \\\n const uint32_t nvec = n / VLEN_FP16; \\\n const uint32_t nloe = n % VLEN_FP16; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n vdst[i] = hvx_vec_inverse_f16_guard(vsrc[i], nan_inf_mask); \\\n } \\\n if (nloe) { \\\n HVX_Vector v = hvx_vec_inverse_f16_guard(vsrc[i], nan_inf_mask); \\\n vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \\\n } \\\n } while(0)\n\n// Generic macro to define alignment permutations for an op\n#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \\\nstatic inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n assert((uintptr_t) src % 128 == 0); \\\n OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \\\n} \\\nstatic inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \\\n assert((uintptr_t) dst % 128 == 0); \\\n OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \\\n} \\\nstatic inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \\\n assert((uintptr_t) src % 128 == 0); \\\n OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \\\n} \\\nstatic inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \\\n OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \\\n} \\\n\n// Dispatcher logic\n#define HVX_INV_DISPATCHER(OP_NAME) \\\nstatic inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { \\\n if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \\\n OP_NAME##_aa(dst, src, num_elems); \\\n } else if (hex_is_aligned((void *) dst, 128)) { \\\n OP_NAME##_au(dst, src, num_elems); \\\n } else if (hex_is_aligned((void *) src, 128)) { \\\n OP_NAME##_ua(dst, src, num_elems); \\\n } else { \\\n OP_NAME##_uu(dst, src, num_elems); \\\n } \\\n}\n\nDEFINE_HVX_INV_OP_VARIANTS(hvx_inverse_f32, hvx_inverse_f32_loop_body)\nDEFINE_HVX_INV_OP_VARIANTS(hvx_inverse_f16, hvx_inverse_f16_loop_body)\n\nHVX_INV_DISPATCHER(hvx_inverse_f32)\nHVX_INV_DISPATCHER(hvx_inverse_f16)\n\n#endif // HVX_INVERSE_H\n" }, { "path": "src/ggml-hexagon/htp/hvx-reduce.h", "content": "#ifndef HVX_REDUCE_H\n#define HVX_REDUCE_H\n\n#include \n#include \n#include \n#include \n\n#include \"hex-utils.h\"\n#include \"hvx-base.h\"\n#include \"hvx-types.h\"\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_n_i32(HVX_Vector in, unsigned int n) {\n unsigned int total = n * 4; // total vec nbytes\n unsigned int width = 4; // int32\n\n HVX_Vector sum = in, sum_t;\n while (width < total) {\n sum_t = Q6_V_vror_VR(sum, width); // rotate right\n sum = Q6_Vw_vadd_VwVw(sum_t, sum); // elementwise sum\n width = width << 1;\n }\n return sum;\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_i32(HVX_Vector in) {\n return hvx_vec_reduce_sum_n_i32(in, 32);\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_n_qf32(HVX_Vector in, unsigned int n) {\n unsigned int total = n * 4; // total vec nbytes\n unsigned int width = 4; // fp32 nbytes\n\n HVX_Vector sum = in, sum_t;\n while (width < total) {\n sum_t = Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum), width); // rotate right\n sum = Q6_Vqf32_vadd_Vqf32Vsf(sum, sum_t); // elementwise sum\n width = width << 1;\n }\n return sum;\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_qf32(HVX_Vector in) {\n return hvx_vec_reduce_sum_n_qf32(in, 32);\n}\n\n#if __HVX_ARCH__ > 75\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_f32x4(HVX_Vector_x4 in) {\n HVX_VectorPair sum_p01 = Q6_W_vshuff_VVR(in.v[1], in.v[0], 4);\n HVX_VectorPair sum_p23 = Q6_W_vshuff_VVR(in.v[3], in.v[2], 4);\n HVX_Vector sum_sf01 = Q6_Vsf_vadd_VsfVsf(Q6_V_lo_W(sum_p01), Q6_V_hi_W(sum_p01));\n HVX_Vector sum_sf23 = Q6_Vsf_vadd_VsfVsf(Q6_V_lo_W(sum_p23), Q6_V_hi_W(sum_p23));\n\n HVX_VectorPair sum_p0123 = Q6_W_vshuff_VVR(sum_sf23, sum_sf01, 8);\n HVX_Vector sum_sf = Q6_Vsf_vadd_VsfVsf(Q6_V_lo_W(sum_p0123), Q6_V_hi_W(sum_p0123));\n\n sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 2));\n sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 4));\n sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 8));\n return sum_sf;\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_f32x2(HVX_Vector in0, HVX_Vector in1) {\n HVX_VectorPair sump = Q6_W_vshuff_VVR(in1, in0, 4);\n HVX_Vector sum_sf = Q6_Vsf_vadd_VsfVsf(Q6_V_lo_W(sump), Q6_V_hi_W(sump));\n\n sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 2));\n sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 4));\n sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 8));\n sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 16));\n return sum_sf;\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_n_f32(HVX_Vector in, unsigned int n) {\n unsigned int total = n * 4; // total vec nbytes\n unsigned int width = 4; // fp32 nbytes\n\n HVX_Vector sum = in, sum_t;\n while (width < total) {\n sum_t = Q6_V_vror_VR(sum, width); // rotate right\n sum = Q6_Vsf_vadd_VsfVsf(sum, sum_t); // elementwise sum\n width = width << 1;\n }\n return sum;\n}\n\n#else\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_f32x4(HVX_Vector_x4 in) {\n HVX_VectorPair sum_p01 = Q6_W_vshuff_VVR(in.v[1], in.v[0], 4);\n HVX_VectorPair sum_p23 = Q6_W_vshuff_VVR(in.v[3], in.v[2], 4);\n HVX_Vector sum_qf01 = Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(sum_p01), Q6_V_hi_W(sum_p01));\n HVX_Vector sum_qf23 = Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(sum_p23), Q6_V_hi_W(sum_p23));\n\n HVX_VectorPair sum_p0123 = Q6_W_vshuff_VVR(Q6_Vsf_equals_Vqf32(sum_qf23), Q6_Vsf_equals_Vqf32(sum_qf01), 8);\n HVX_Vector sum_qf = Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(sum_p0123), Q6_V_hi_W(sum_p0123));\n\n sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 2));\n sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 4));\n sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 8));\n return Q6_Vsf_equals_Vqf32(sum_qf);\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_f32x2(HVX_Vector in0, HVX_Vector in1) {\n HVX_VectorPair sump = Q6_W_vshuff_VVR(in1, in0, 4);\n HVX_Vector sum_qf = Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(sump), Q6_V_hi_W(sump));\n\n sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 2));\n sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 4));\n sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 8));\n sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 16));\n return Q6_Vsf_equals_Vqf32(sum_qf);\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_n_f32(HVX_Vector in, unsigned int n) {\n unsigned int total = n * 4; // total vec nbytes\n unsigned int width = 4; // fp32 nbytes\n\n HVX_Vector sum = in, sum_t;\n while (width < total) {\n sum_t = Q6_V_vror_VR(sum, width); // rotate right\n sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(sum, sum_t)); // elementwise sum\n width = width << 1;\n }\n return sum;\n}\n\n#endif\n\nstatic inline HVX_Vector hvx_vec_reduce_sum_f32(HVX_Vector in) {\n return hvx_vec_reduce_sum_n_f32(in, 32);\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_max_f16(HVX_Vector in) {\n unsigned total = 128; // total vec nbytes\n unsigned width = 2; // fp16 nbytes\n\n HVX_Vector _max = in, _max_t;\n while (width < total) {\n _max_t = Q6_V_vror_VR(_max, width); // rotate right\n _max = Q6_Vhf_vmax_VhfVhf(_max_t, _max); // elementwise max\n width = width << 1;\n }\n\n return _max;\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_max2_f16(HVX_Vector in, HVX_Vector _max) {\n unsigned total = 128; // total vec nbytes\n unsigned width = 2; // fp32 nbytes\n\n HVX_Vector _max_t;\n\n _max = Q6_Vhf_vmax_VhfVhf(in, _max);\n while (width < total) {\n _max_t = Q6_V_vror_VR(_max, width); // rotate right\n _max = Q6_Vhf_vmax_VhfVhf(_max_t, _max); // elementwise max\n width = width << 1;\n }\n\n return _max;\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_max_f32(HVX_Vector in) {\n unsigned total = 128; // total vec nbytes\n unsigned width = 4; // fp32 nbytes\n\n HVX_Vector _max = in, _max_t;\n while (width < total) {\n _max_t = Q6_V_vror_VR(_max, width); // rotate right\n _max = Q6_Vsf_vmax_VsfVsf(_max_t, _max); // elementwise max\n width = width << 1;\n }\n\n return _max;\n}\n\nstatic inline HVX_Vector hvx_vec_reduce_max2_f32(HVX_Vector in, HVX_Vector _max) {\n unsigned total = 128; // total vec nbytes\n unsigned width = 4; // fp32 nbytes\n\n HVX_Vector _max_t;\n\n _max = Q6_Vsf_vmax_VsfVsf(in, _max);\n while (width < total) {\n _max_t = Q6_V_vror_VR(_max, width); // rotate right\n _max = Q6_Vsf_vmax_VsfVsf(_max_t, _max); // elementwise max\n width = width << 1;\n }\n\n return _max;\n}\n\n#define hvx_reduce_loop_body(src_type, init_vec, pad_vec, vec_op, reduce_op, scalar_reduce) \\\n do { \\\n src_type * restrict vsrc = (src_type *) src; \\\n HVX_Vector acc = init_vec; \\\n \\\n const uint32_t elem_size = sizeof(float); \\\n const uint32_t epv = 128 / elem_size; \\\n const uint32_t nvec = num_elems / epv; \\\n const uint32_t nloe = num_elems % epv; \\\n \\\n uint32_t i = 0; \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n acc = vec_op(acc, vsrc[i]); \\\n } \\\n if (nloe) { \\\n const float * srcf = (const float *) src + i * epv; \\\n HVX_Vector in = *(HVX_UVector *) srcf; \\\n HVX_Vector temp = Q6_V_valign_VVR(in, pad_vec, nloe * elem_size); \\\n acc = vec_op(acc, temp); \\\n } \\\n HVX_Vector v = reduce_op(acc); \\\n return scalar_reduce(v); \\\n } while(0)\n\n#define HVX_REDUCE_MAX_OP(acc, val) Q6_Vsf_vmax_VsfVsf(acc, val)\n#define HVX_REDUCE_SUM_OP(acc, val) Q6_Vqf32_vadd_VsfVsf(Q6_Vsf_equals_Vqf32(acc), val)\n#define HVX_SUM_SQ_OP(acc, val) Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(val, val))\n#define HVX_REDUCE_MAX_SCALAR(v) hvx_vec_get_f32(v)\n#define HVX_REDUCE_SUM_SCALAR(v) hvx_vec_get_f32(Q6_Vsf_equals_Vqf32(v))\n\n// Max variants\n\nstatic inline float hvx_reduce_max_f32_a(const uint8_t * restrict src, const int num_elems) {\n HVX_Vector init_vec = hvx_vec_splat_f32(((const float *) src)[0]);\n assert((unsigned long) src % 128 == 0);\n hvx_reduce_loop_body(HVX_Vector, init_vec, init_vec, HVX_REDUCE_MAX_OP, hvx_vec_reduce_max_f32, HVX_REDUCE_MAX_SCALAR);\n}\n\nstatic inline float hvx_reduce_max_f32_u(const uint8_t * restrict src, const int num_elems) {\n HVX_Vector init_vec = hvx_vec_splat_f32(((const float *) src)[0]);\n hvx_reduce_loop_body(HVX_UVector, init_vec, init_vec, HVX_REDUCE_MAX_OP, hvx_vec_reduce_max_f32, HVX_REDUCE_MAX_SCALAR);\n}\n\nstatic inline float hvx_reduce_max_f32(const uint8_t * restrict src, const int num_elems) {\n if (hex_is_aligned((void *) src, 128)) {\n return hvx_reduce_max_f32_a(src, num_elems);\n } else {\n return hvx_reduce_max_f32_u(src, num_elems);\n }\n}\n\n// Sum variants\n\nstatic inline float hvx_reduce_sum_f32_a(const uint8_t * restrict src, const int num_elems) {\n HVX_Vector init_vec = Q6_V_vsplat_R(0);\n assert((unsigned long) src % 128 == 0);\n hvx_reduce_loop_body(HVX_Vector, init_vec, init_vec, HVX_REDUCE_SUM_OP, hvx_vec_reduce_sum_qf32, HVX_REDUCE_SUM_SCALAR);\n}\n\nstatic inline float hvx_reduce_sum_f32_u(const uint8_t * restrict src, const int num_elems) {\n HVX_Vector init_vec = Q6_V_vsplat_R(0);\n hvx_reduce_loop_body(HVX_UVector, init_vec, init_vec, HVX_REDUCE_SUM_OP, hvx_vec_reduce_sum_qf32, HVX_REDUCE_SUM_SCALAR);\n}\n\nstatic inline float hvx_reduce_sum_f32(const uint8_t * restrict src, const int num_elems) {\n if (hex_is_aligned((void *) src, 128)) {\n return hvx_reduce_sum_f32_a(src, num_elems);\n } else {\n return hvx_reduce_sum_f32_u(src, num_elems);\n }\n}\n\n// Sum of squares variants\n\nstatic inline float hvx_sum_of_squares_f32_a(const uint8_t * restrict src, const int num_elems) {\n HVX_Vector init_vec = Q6_V_vsplat_R(0);\n assert((uintptr_t) src % 128 == 0);\n hvx_reduce_loop_body(HVX_Vector, init_vec, init_vec, HVX_SUM_SQ_OP, hvx_vec_reduce_sum_qf32, HVX_REDUCE_SUM_SCALAR);\n}\n\nstatic inline float hvx_sum_of_squares_f32_u(const uint8_t * restrict src, const int num_elems) {\n HVX_Vector init_vec = Q6_V_vsplat_R(0);\n hvx_reduce_loop_body(HVX_UVector, init_vec, init_vec, HVX_SUM_SQ_OP, hvx_vec_reduce_sum_qf32, HVX_REDUCE_SUM_SCALAR);\n}\n\nstatic inline float hvx_sum_of_squares_f32(const uint8_t * restrict src, const int num_elems) {\n if (hex_is_aligned((void *) src, 128)) {\n return hvx_sum_of_squares_f32_a(src, num_elems);\n } else {\n return hvx_sum_of_squares_f32_u(src, num_elems);\n }\n}\n\n#undef hvx_reduce_loop_body\n#undef HVX_REDUCE_MAX_OP\n#undef HVX_REDUCE_SUM_OP\n#undef HVX_REDUCE_MAX_SCALAR\n#undef HVX_REDUCE_SUM_SCALAR\n#undef HVX_SUM_SQ_OP\n\n#endif /* HVX_REDUCE_H */\n" }, { "path": "src/ggml-hexagon/htp/hvx-scale.h", "content": "#ifndef HVX_SCALE_H\n#define HVX_SCALE_H\n\n#include \n#include \n#include \n\n#include \"hvx-base.h\"\n\n#define hvx_scale_f32_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n HVX_Vector vs = hvx_vec_splat_f32(scale); \\\n \\\n const uint32_t elem_size = sizeof(float); \\\n const uint32_t epv = 128 / elem_size; \\\n const uint32_t nvec = n / epv; \\\n const uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; ++i) { \\\n HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs); \\\n vdst[i] = Q6_Vsf_equals_Vqf32(v); \\\n } \\\n if (nloe) { \\\n HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs); \\\n vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \\\n } \\\n } while(0)\n\nstatic inline void hvx_scale_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {\n assert((size_t) dst % 128 == 0);\n assert((size_t) src % 128 == 0);\n hvx_scale_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_scale_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {\n assert((size_t) dst % 128 == 0);\n hvx_scale_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_scale_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {\n assert((size_t) src % 128 == 0);\n hvx_scale_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_scale_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {\n hvx_scale_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_scale_f32(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {\n if (((size_t) dst & 127) == 0) {\n if (((size_t) src & 127) == 0) {\n hvx_scale_f32_aa(dst, src, n, scale);\n } else {\n hvx_scale_f32_au(dst, src, n, scale);\n }\n } else {\n if (((size_t) src & 127) == 0) {\n hvx_scale_f32_ua(dst, src, n, scale);\n } else {\n hvx_scale_f32_uu(dst, src, n, scale);\n }\n }\n}\n\n#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n HVX_Vector vs = hvx_vec_splat_f32(scale); \\\n HVX_Vector vo = hvx_vec_splat_f32(offset); \\\n \\\n const uint32_t elem_size = sizeof(float); \\\n const uint32_t epv = 128 / elem_size; \\\n const uint32_t nvec = n / epv; \\\n const uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; ++i) { \\\n HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \\\n vdst[i] = Q6_Vsf_equals_Vqf32(v); \\\n } \\\n if (nloe) { \\\n HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \\\n vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \\\n } \\\n } while(0)\n\nstatic inline void hvx_scale_offset_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {\n assert((size_t) dst % 128 == 0);\n assert((size_t) src % 128 == 0);\n hvx_scale_offset_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_scale_offset_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {\n assert((size_t) dst % 128 == 0);\n hvx_scale_offset_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_scale_offset_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {\n assert((size_t) src % 128 == 0);\n hvx_scale_offset_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_scale_offset_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {\n hvx_scale_offset_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_scale_offset_f32(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {\n if (((size_t) dst & 127) == 0) {\n if (((size_t) src & 127) == 0) {\n hvx_scale_offset_f32_aa(dst, src, n, scale, offset);\n } else {\n hvx_scale_offset_f32_au(dst, src, n, scale, offset);\n }\n } else {\n if (((size_t) src & 127) == 0) {\n hvx_scale_offset_f32_ua(dst, src, n, scale, offset);\n } else {\n hvx_scale_offset_f32_uu(dst, src, n, scale, offset);\n }\n }\n}\n\n#endif // HVX_SCALE_H\n" }, { "path": "src/ggml-hexagon/htp/hvx-sigmoid.h", "content": "#ifndef HVX_SIGMOID_H\n#define HVX_SIGMOID_H\n\n#include \"hvx-base.h\"\n\n#define FAST_SIGMOID_LOG2F (0x3fb8aa3b) // 1.442695022\n#define FAST_SIGMOID_C1 (0x3d009076) // 0.03138777\n#define FAST_SIGMOID_C2 (0x3e8d74bd) // 0.276281267\n#define FAST_SIGMOID_C3 (0x3f000000) // 0.5\n\nstatic inline HVX_Vector hvx_vec_fast_sigmoid_f32(HVX_Vector v) {\n v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));\n v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3));\n\n HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));\n HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));\n HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x);\n\n HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2));\n v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));\n\n HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1));\n v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx);\n v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x);\n\n HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));\n HVX_Vector v3_exponent = Q6_Vw_vasl_VwR(v3, 1);\n v3_exponent = Q6_Vuw_vlsr_VuwR(v3_exponent, 24);\n v3_exponent = Q6_Vw_vadd_VwVw(in_int, v3_exponent);\n v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);\n\n HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));\n HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));\n\n HVX_Vector res = hvx_vec_inverse_f32(v5);\n res = Q6_Vqf32_vmpy_VsfVsf(v3, res);\n\n return Q6_Vsf_equals_Vqf32(res);\n}\n\nstatic inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard(HVX_Vector v,\n HVX_Vector one,\n HVX_Vector max_exp,\n HVX_Vector min_exp) {\n const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v);\n const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp);\n\n HVX_Vector out = hvx_vec_fast_sigmoid_f32(v);\n out = Q6_V_vmux_QVV(pred_max, out, one);\n return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero());\n}\n\nstatic inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) {\n // tanh(x) = 2 * sigmoid(2x) - 1\n HVX_Vector two = hvx_vec_splat_f32(2.0f);\n HVX_Vector one = hvx_vec_splat_f32(1.0f);\n HVX_Vector x2 = Q6_Vqf32_vmpy_VsfVsf(x, two);\n\n HVX_Vector max_exp = hvx_vec_splat_f32(87.f);\n HVX_Vector min_exp = hvx_vec_splat_f32(-87.f);\n\n HVX_Vector sig2x = hvx_vec_fast_sigmoid_f32_guard(Q6_Vsf_equals_Vqf32(x2), one, max_exp, min_exp);\n\n HVX_Vector res = Q6_Vqf32_vmpy_VsfVsf(sig2x, two);\n res = Q6_Vqf32_vsub_Vqf32Vsf(res, one);\n return Q6_Vsf_equals_Vqf32(res);\n}\n\n#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const HVX_Vector one = hvx_vec_splat_f32(1.f); \\\n const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \\\n const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \\\n \\\n const uint32_t epv = 128 / sizeof(float); \\\n const uint32_t nvec = n / epv; \\\n const uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \\\n } \\\n if (nloe) { \\\n HVX_Vector tmp = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \\\n vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \\\n } \\\n } while(0)\n\n#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const uint32_t epv = 128 / sizeof(float); \\\n const uint32_t nvec = n / epv; \\\n const uint32_t nloe = n % epv; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \\\n } \\\n if (nloe) { \\\n HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \\\n vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \\\n } \\\n } while(0)\n\nstatic inline void hvx_sigmoid_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src % 128 == 0);\n hvx_sigmoid_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_sigmoid_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n hvx_sigmoid_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_sigmoid_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) src % 128 == 0);\n hvx_sigmoid_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_sigmoid_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_sigmoid_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_tanh_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src % 128 == 0);\n hvx_tanh_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\n\n#endif /* HVX_SIGMOID_H */\n" }, { "path": "src/ggml-hexagon/htp/hvx-sqrt.h", "content": "#ifndef HVX_SQRT_H\n#define HVX_SQRT_H\n\n#include \n#include \n\n#include \"hex-utils.h\"\n\n#include \"hvx-base.h\"\n\n#define RSQRT_CONST 0x5f3759df // Constant for fast inverse square root calculation\n#define RSQRT_ONE_HALF 0x3f000000 // 0.5\n#define RSQRT_THREE_HALVES 0x3fc00000 // 1.5\n\n#if __HVX_ARCH__ < 79\n#define HVX_OP_MUL(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b))\n#else\n#define HVX_OP_MUL(a, b) Q6_Vsf_vmpy_VsfVsf(a, b)\n#endif\n\nstatic inline HVX_Vector hvx_vec_rsqrt_f32(HVX_Vector in_vec) {\n //Algorithm :\n // x2 = input*0.5\n // y = * (long *) &input\n // y = 0x5f3759df - (y>>1)\n // y = y*(threehalfs - x2*y*y)\n\n HVX_Vector rsqrtconst = Q6_V_vsplat_R(RSQRT_CONST);\n HVX_Vector onehalf = Q6_V_vsplat_R(RSQRT_ONE_HALF);\n HVX_Vector threehalfs = Q6_V_vsplat_R(RSQRT_THREE_HALVES);\n\n HVX_Vector x2, y, ypower2, temp;\n\n x2 = Q6_Vqf32_vmpy_VsfVsf(in_vec, onehalf);\n x2 = Q6_Vqf32_vadd_Vqf32Vsf(x2, Q6_V_vzero());\n\n y = Q6_Vw_vasr_VwR(in_vec, 1);\n y = Q6_Vw_vsub_VwVw(rsqrtconst, y);\n\n // 1st iteration\n ypower2 = Q6_Vqf32_vmpy_VsfVsf(y, y);\n ypower2 = Q6_Vqf32_vadd_Vqf32Vsf(ypower2, Q6_V_vzero());\n temp = Q6_Vqf32_vmpy_Vqf32Vqf32(x2, ypower2);\n temp = Q6_Vqf32_vsub_VsfVsf(threehalfs, Q6_Vsf_equals_Vqf32(temp));\n temp = Q6_Vqf32_vmpy_VsfVsf(y, Q6_Vsf_equals_Vqf32(temp));\n\n // 2nd iteration\n y = Q6_Vqf32_vadd_Vqf32Vsf(temp, Q6_V_vzero());\n ypower2 = Q6_Vqf32_vmpy_Vqf32Vqf32(y, y);\n ypower2 = Q6_Vqf32_vadd_Vqf32Vsf(ypower2, Q6_V_vzero());\n temp = Q6_Vqf32_vmpy_Vqf32Vqf32(x2, ypower2);\n temp = Q6_Vqf32_vsub_VsfVsf(threehalfs, Q6_Vsf_equals_Vqf32(temp));\n temp = Q6_Vqf32_vmpy_Vqf32Vqf32(y, temp);\n\n // 3rd iteration\n y = Q6_Vqf32_vadd_Vqf32Vsf(temp, Q6_V_vzero());\n ypower2 = Q6_Vqf32_vmpy_Vqf32Vqf32(y, y);\n ypower2 = Q6_Vqf32_vadd_Vqf32Vsf(ypower2, Q6_V_vzero());\n temp = Q6_Vqf32_vmpy_Vqf32Vqf32(x2, ypower2);\n temp = Q6_Vqf32_vsub_VsfVsf(threehalfs, Q6_Vsf_equals_Vqf32(temp));\n temp = Q6_Vqf32_vmpy_Vqf32Vqf32(y, temp);\n\n return Q6_Vsf_equals_Vqf32(temp);\n}\n\n// Compute sqrt(x) as x*inv_sqrt(x)\n#define hvx_sqrt_f32_loop_body(dst_type, src_type, vec_store) \\\n do { \\\n dst_type * restrict vdst = (dst_type *) dst; \\\n src_type * restrict vsrc = (src_type *) src; \\\n \\\n const uint32_t nvec = n / VLEN_FP32; \\\n const uint32_t nloe = n % VLEN_FP32; \\\n \\\n uint32_t i = 0; \\\n \\\n _Pragma(\"unroll(4)\") \\\n for (; i < nvec; i++) { \\\n HVX_Vector inv_sqrt = hvx_vec_rsqrt_f32(vsrc[i]); \\\n HVX_Vector sqrt_res = HVX_OP_MUL(inv_sqrt, vsrc[i]); \\\n vdst[i] = sqrt_res; \\\n } \\\n if (nloe) { \\\n HVX_Vector inv_sqrt = hvx_vec_rsqrt_f32(vsrc[i]); \\\n HVX_Vector sqrt_res = HVX_OP_MUL(inv_sqrt, vsrc[i]); \\\n vec_store((void *) &vdst[i], nloe * SIZEOF_FP32, sqrt_res); \\\n } \\\n } while(0)\n\nstatic inline void hvx_sqrt_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n assert((unsigned long) src % 128 == 0);\n hvx_sqrt_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_sqrt_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) dst % 128 == 0);\n hvx_sqrt_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);\n}\n\nstatic inline void hvx_sqrt_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n assert((unsigned long) src % 128 == 0);\n hvx_sqrt_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_sqrt_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {\n hvx_sqrt_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);\n}\n\nstatic inline void hvx_sqrt_f32(uint8_t * restrict dst, const uint8_t * restrict src, const int num_elems) {\n if ((unsigned long) dst % 128 == 0) {\n if ((unsigned long) src % 128 == 0) {\n hvx_sqrt_f32_aa(dst, src, num_elems);\n } else {\n hvx_sqrt_f32_au(dst, src, num_elems);\n }\n } else {\n if ((unsigned long) src % 128 == 0) {\n hvx_sqrt_f32_ua(dst, src, num_elems);\n } else {\n hvx_sqrt_f32_uu(dst, src, num_elems);\n }\n }\n}\n\n#endif /* HVX_SQRT_H */\n" }, { "path": "src/ggml-hexagon/htp/hvx-types.h", "content": "#ifndef HVX_TYPES_H\n#define HVX_TYPES_H\n\n#include \n#include \n\n#include \n\n#define SIZEOF_FP32 (4)\n#define SIZEOF_FP16 (2)\n#define VLEN (128)\n#define VLEN_FP32 (VLEN / SIZEOF_FP32)\n#define VLEN_FP16 (VLEN / SIZEOF_FP16)\n\ntypedef union {\n HVX_Vector v;\n uint8_t b[VLEN];\n uint16_t h[VLEN_FP16];\n uint32_t w[VLEN_FP32];\n __fp16 fp16[VLEN_FP16];\n float fp32[VLEN_FP32];\n} __attribute__((aligned(VLEN), packed)) HVX_VectorAlias;\n\ntypedef struct {\n HVX_Vector v[2];\n} HVX_Vector_x2;\n\ntypedef struct {\n HVX_Vector v[4];\n} HVX_Vector_x4;\n\ntypedef struct {\n HVX_Vector v[8];\n} HVX_Vector_x8;\n\n#endif /* HVX_TYPES_H */\n" }, { "path": "src/ggml-hexagon/htp/hvx-utils.h", "content": "#ifndef HVX_UTILS_H\n#define HVX_UTILS_H\n\n#include \"hex-utils.h\"\n\n#include \"hvx-types.h\"\n#include \"hvx-copy.h\"\n#include \"hvx-scale.h\"\n#include \"hvx-exp.h\"\n#include \"hvx-inverse.h\"\n#include \"hvx-reduce.h\"\n#include \"hvx-sigmoid.h\"\n#include \"hvx-sqrt.h\"\n#include \"hvx-arith.h\"\n#include \"hvx-div.h\"\n#include \"hvx-base.h\"\n\n#ifndef GATHER_TYPE\n# if defined(__hexagon__)\n# define GATHER_TYPE(_a) (intptr_t) _a\n# else\n# define GATHER_TYPE(_a) (HVX_Vector *) _a\n# endif\n#endif\n\n#endif /* HVX_UTILS_H */\n" }, { "path": "src/ggml-hexagon/htp/main.c", "content": "#pragma clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hex-utils.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n#include \"worker-pool.h\"\n\nAEEResult htp_iface_open(const char * uri, remote_handle64 * handle) {\n struct htp_context * ctx;\n int err = 0;\n\n ctx = calloc(1, sizeof(*ctx));\n if (ctx == NULL) {\n return AEE_ENOMEMORY;\n }\n\n // Use the context structure as a handle\n *handle = (remote_handle64) ctx;\n\n // Enable FARF logs\n HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0);\n\n // Set client class\n {\n HAP_power_request_t request;\n memset(&request, 0, sizeof(HAP_power_request_t));\n request.type = HAP_power_set_apptype;\n request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS;\n\n if ((err = HAP_power_set((void *) ctx, &request)) != 0) {\n return err;\n }\n }\n\n {\n HAP_power_request_t request;\n memset(&request, 0, sizeof(request));\n\n request.type = HAP_power_set_DCVS_v3;\n request.dcvs_v3.set_dcvs_enable = TRUE;\n request.dcvs_v3.dcvs_enable = TRUE;\n request.dcvs_v3.dcvs_option = HAP_DCVS_V2_PERFORMANCE_MODE;\n request.dcvs_v3.set_bus_params = TRUE;\n request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX;\n request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX;\n request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX;\n request.dcvs_v3.set_core_params = TRUE;\n request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX;\n request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX;\n request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX;\n request.dcvs_v3.set_sleep_disable = TRUE;\n request.dcvs_v3.sleep_disable = TRUE;\n if ((err = HAP_power_set((void *) ctx, &request)) != 0) {\n return err;\n }\n\n memset(&request, 0, sizeof(request));\n request.type = HAP_power_set_HVX;\n request.hvx.power_up = TRUE;\n if ((err = HAP_power_set((void *) ctx, &request)) != 0) {\n return err;\n }\n }\n\n {\n // Power on HMX\n HAP_power_request_t request;\n memset(&request, 0, sizeof(HAP_power_request_t));\n request.type = HAP_power_set_HMX;\n request.hmx.power_up = TRUE;\n FARF(ALWAYS, \"Powering HMX on\\n\");\n err = HAP_power_set((void *) &ctx, &request);\n if (err != AEE_SUCCESS) {\n FARF(ERROR, \"Error powering on HMX.\");\n return err;\n }\n }\n\n return AEE_SUCCESS;\n}\n\nAEEResult htp_iface_close(remote_handle64 handle) {\n struct htp_context * ctx = (struct htp_context *) handle;\n\n if (!ctx) {\n return AEE_EBADPARM;\n }\n\n if (ctx->queue) {\n FARF(ERROR, \"Closing handle with queue still open\");\n return AEE_EITEMBUSY;\n }\n\n free(ctx);\n return AEE_SUCCESS;\n}\n\nAEEResult htp_iface_enable_etm(remote_handle64 handle) {\n int err = HAP_user_etm_enable();\n if (err) {\n if (err == AEE_EVERSIONNOTSUPPORT) {\n FARF(ERROR, \"API HAP_user_etm_enable is not supported\\n\");\n } else {\n FARF(ERROR, \"Error executing HAP_user_etm_enable with error code : 0x%x\\n\", err);\n }\n }\n return err;\n}\n\nAEEResult htp_iface_disable_etm(remote_handle64 handle) {\n int err = HAP_user_etm_disable();\n if (err) {\n if (err == AEE_EVERSIONNOTSUPPORT) {\n FARF(ERROR, \"API HAP_user_etm_disable is not supported\\n\");\n } else {\n FARF(ERROR, \"Error executing HAP_user_etm_disable with error code : 0x%x\\n\", err);\n }\n }\n return err;\n}\n\nstatic int vtcm_acquire(struct htp_context * ctx) {\n int err;\n if (!ctx->vtcm_valid) {\n // Temporarily bump thread priority to make sure it's higher than other sessions.\n // This way the resource manager will notify the other thread to release VTCM.\n // Note that we need to reaquire VTCM at normal priority for this to work next time.\n qurt_thread_set_priority(qurt_thread_get_id(), ctx->thread_prio - 10);\n err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 1000000);\n if (err != 0) {\n FARF(ERROR, \"Failed to acquire VTCM: 0x%08x\", (unsigned)err);\n abort();\n }\n HAP_compute_res_release_cached(ctx->vtcm_rctx);\n qurt_thread_set_priority(qurt_thread_get_id(), ctx->thread_prio);\n\n err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 1000000);\n if (err != 0) {\n FARF(ERROR, \"Failed to acquire VTCM: 0x%08x\", (unsigned)err);\n abort();\n }\n ctx->vtcm_valid = true;\n }\n\n ctx->vtcm_inuse = true;\n return 0;\n}\n\nstatic int vtcm_release(struct htp_context * ctx) {\n ctx->vtcm_inuse = false;\n\n if (ctx->vtcm_valid && ctx->vtcm_needs_release) {\n ctx->vtcm_valid = false;\n ctx->vtcm_needs_release = false;\n HAP_compute_res_release_cached(ctx->vtcm_rctx);\n }\n\n return 0;\n}\n\nstatic int vtcm_release_callback(unsigned int rctx, void * state) {\n struct htp_context * ctx = (struct htp_context *) state;\n\n if (!ctx || ctx->vtcm_rctx != rctx) {\n return AEE_EBADPARM;\n }\n\n // If VTCM is not inuse (not processing Ops) release it right here\n // otherwise we'll release it once we're done with the current Op.\n\n if (ctx->vtcm_inuse) {\n ctx->vtcm_needs_release = true;\n return 0;\n }\n\n ctx->vtcm_valid = false;\n HAP_compute_res_release_cached(ctx->vtcm_rctx);\n\n return 0;\n}\n\nstatic int vtcm_alloc(struct htp_context * ctx) {\n unsigned int vtcm_size = 8 * 1024 * 1024; // 8MB default\n HAP_compute_res_query_VTCM(0, &vtcm_size, NULL, NULL, NULL);\n\n compute_res_attr_t attr;\n HAP_compute_res_attr_init(&attr);\n HAP_compute_res_attr_set_serialize(&attr, 0);\n HAP_compute_res_attr_set_cache_mode(&attr, 1);\n HAP_compute_res_attr_set_vtcm_param_v2(&attr, vtcm_size, 0, vtcm_size);\n HAP_compute_res_attr_set_release_callback(&attr, vtcm_release_callback, (void *) ctx);\n HAP_compute_res_attr_set_hmx_param(&attr, 1);\n\n // Allocate VTCM for scratch pads\n uint32_t rctx = HAP_compute_res_acquire(&attr, 1000000 /* timeout */);\n if (!rctx) {\n FARF(ERROR, \"failed to allocate %zu bytes VTCM\\n\", ctx->vtcm_size);\n return AEE_ENOMEMORY;\n }\n\n void * vtcm_ptr;\n if (HAP_compute_res_attr_get_vtcm_ptr_v2(&attr, &vtcm_ptr, &vtcm_size) != 0) {\n HAP_compute_res_release(rctx);\n FARF(ERROR, \"failed to allocate %zu bytes VTCM (new)\\n\", ctx->vtcm_size);\n return AEE_ENOMEMORY;\n }\n\n ctx->vtcm_base = (uint8_t *) vtcm_ptr;\n ctx->vtcm_size = vtcm_size;\n ctx->vtcm_rctx = rctx;\n ctx->vtcm_valid = false;\n ctx->vtcm_inuse = false;\n ctx->vtcm_needs_release = false;\n\n return 0;\n}\n\nstatic void vtcm_free(struct htp_context * ctx) {\n if (ctx->vtcm_rctx) {\n HAP_compute_res_release(ctx->vtcm_rctx);\n ctx->vtcm_base = 0;\n ctx->vtcm_rctx = 0;\n }\n}\n\nstatic void htp_packet_callback(dspqueue_t queue, int error, void * context);\nstatic void htp_error_callback(dspqueue_t queue, int error, void * context);\n\nAEEResult htp_iface_start(remote_handle64 handle, uint32 sess_id, uint64 dsp_queue_id, uint32 n_hvx) {\n struct htp_context * ctx = (struct htp_context *) handle;\n\n if (!ctx) {\n return AEE_EBADPARM;\n }\n\n if (ctx->queue) {\n FARF(ERROR, \"Queue already open\");\n return AEE_EITEMBUSY;\n }\n\n // Import queue created on the CPU\n int err = dspqueue_import(dsp_queue_id, // Queue ID from dspqueue_export\n htp_packet_callback, // Packet callback\n htp_error_callback, // Error callback; no errors expected on the DSP\n (void *) ctx, // Callback context\n &ctx->queue);\n\n if (err) {\n FARF(ERROR, \"Queue import failed with 0x%08x\", (unsigned) err);\n return err;\n }\n\n ctx->thread_id = qurt_thread_get_id();\n ctx->thread_prio = qurt_thread_get_priority(ctx->thread_id);\n\n // allocate VTCM\n err = vtcm_alloc(ctx);\n if (err != AEE_SUCCESS) {\n FARF(ERROR, \"Unable to allocate VTCM\");\n return AEE_ENOMEMORY;\n }\n\n qurt_sysenv_max_hthreads_t hw_threads;\n qurt_sysenv_get_max_hw_threads(&hw_threads);\n uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF;\n\n if (n_hvx == 0) {\n n_hvx = hw_nhvx;\n }\n if (n_hvx > hw_threads.max_hthreads) {\n n_hvx = hw_threads.max_hthreads;\n }\n if (n_hvx > HTP_MAX_NTHREADS) {\n n_hvx = HTP_MAX_NTHREADS;\n }\n\n ctx->n_threads = n_hvx;\n for (int i = 0; i < ctx->n_threads; i++) {\n // see discussion https://github.com/ggml-org/llama.cpp/pull/18151#discussion_r2632388541\n ctx->dma[i] = dma_queue_create(64);\n }\n\n // init worker pool\n err = worker_pool_init(&ctx->worker_pool, n_hvx);\n if (err != AEE_SUCCESS) {\n FARF(ERROR, \"Unable to create worker pool\");\n return err;\n }\n\n FARF(HIGH, \"session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \\n\",\n sess_id, hw_nhvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio);\n\n return AEE_SUCCESS;\n}\n\nAEEResult htp_iface_stop(remote_handle64 handle) {\n struct htp_context * ctx = (struct htp_context *) handle;\n if (!ctx) {\n return AEE_EBADPARM;\n }\n\n if (!ctx->queue) {\n FARF(ERROR, \"Queue not open\");\n return AEE_EBADSTATE;\n }\n\n // Close queue. dspqueue_close() will also wait for callbacks to finish.\n int err = dspqueue_close(ctx->queue);\n ctx->queue = NULL;\n if (err != 0) {\n FARF(ERROR, \"Queue close failed with 0x%08x\", (unsigned) err);\n return err;\n }\n\n if (ctx->worker_pool) {\n // Release worker pool\n worker_pool_release(&ctx->worker_pool);\n }\n\n for (int i = 0; i < ctx->n_threads; i++) {\n dma_queue_delete(ctx->dma[i]);\n }\n\n vtcm_free(ctx);\n\n return AEE_SUCCESS;\n}\n\nstatic void htp_error_callback(dspqueue_t queue, int error, void * context) {\n // No errors expected on the DSP.\n FARF(ERROR, \"Error callback: 0x%08x\", (unsigned) error);\n}\n\nstruct profile_data {\n uint64_t usecs;\n uint64_t cycles;\n uint64_t pkts;\n};\n\nstatic inline void profile_start(struct profile_data * d) {\n d->usecs = HAP_perf_get_qtimer_count();\n d->cycles = hex_get_cycles();\n d->pkts = hex_get_pktcnt();\n}\n\nstatic inline void profile_stop(struct profile_data * d) {\n d->usecs = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - d->usecs);\n d->cycles = hex_get_cycles() - d->cycles;\n d->pkts = hex_get_pktcnt() - d->pkts;\n}\n\nstatic int send_htp_rsp(struct htp_context * c,\n uint32_t op,\n uint32_t status,\n struct dspqueue_buffer * bufs,\n size_t n_bufs,\n struct profile_data * prof) {\n // Prep response struct\n struct htp_general_rsp rsp;\n rsp.op = op;\n rsp.status = status;\n rsp.prof_usecs = prof->usecs;\n rsp.prof_cycles = prof->cycles;\n rsp.prof_pkts = prof->pkts;\n\n int err = dspqueue_write(c->queue,\n 0, // Flags\n n_bufs,\n bufs, // Buffer references\n sizeof(rsp),\n (const uint8_t *) &rsp, // Message\n DSPQUEUE_TIMEOUT_NONE);\n\n if (err != 0) {\n FARF(ERROR, \"dspqueue_write failed: 0x%08x\", (unsigned) err);\n }\n\n return err;\n}\n\nstatic void proc_matmul_req(struct htp_context * ctx,\n struct htp_general_req * req,\n struct dspqueue_buffer * bufs,\n size_t n_bufs) {\n struct dspqueue_buffer rsp_bufs[1];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[2].fd;\n rsp_bufs[0].ptr = bufs[2].ptr;\n rsp_bufs[0].size = bufs[2].size;\n rsp_bufs[0].offset = bufs[2].offset;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.src1 = req->src1;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.src1.data = (uint32_t) bufs[1].ptr;\n octx.dst.data = (uint32_t) bufs[2].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_matmul(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_argsort_req(struct htp_context * ctx, struct htp_general_req * req, struct dspqueue_buffer * bufs) {\n struct dspqueue_buffer rsp_bufs[1];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[1].fd;\n rsp_bufs[0].ptr = bufs[1].ptr;\n rsp_bufs[0].offset = bufs[1].offset;\n rsp_bufs[0].size = bufs[1].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n memcpy(octx.op_params, req->op_params, sizeof(octx.op_params));\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.dst.data = (uint32_t) bufs[1].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_argsort(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_cpy_req(struct htp_context * ctx, struct htp_general_req * req, struct dspqueue_buffer * bufs) {\n struct dspqueue_buffer rsp_bufs[1];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[1].fd;\n rsp_bufs[0].ptr = bufs[1].ptr;\n rsp_bufs[0].offset = bufs[1].offset;\n rsp_bufs[0].size = bufs[1].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.dst.data = (uint32_t) bufs[1].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_cpy(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_get_rows_req(struct htp_context * ctx, struct htp_general_req * req, struct dspqueue_buffer * bufs) {\n struct dspqueue_buffer rsp_bufs[1];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[2].fd;\n rsp_bufs[0].ptr = bufs[2].ptr;\n rsp_bufs[0].offset = bufs[2].offset;\n rsp_bufs[0].size = bufs[2].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.src1 = req->src1;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.src1.data = (uint32_t) bufs[1].ptr;\n octx.dst.data = (uint32_t) bufs[2].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_get_rows(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_matmul_id_req(struct htp_context * ctx,\n struct htp_general_req * req,\n struct dspqueue_buffer * bufs,\n size_t n_bufs) {\n struct dspqueue_buffer rsp_bufs[1];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[3].fd;\n rsp_bufs[0].ptr = bufs[3].ptr;\n rsp_bufs[0].size = bufs[3].size;\n rsp_bufs[0].offset = bufs[3].offset;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.src1 = req->src1;\n octx.src2 = req->src2;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.src1.data = (uint32_t) bufs[1].ptr;\n octx.src2.data = (uint32_t) bufs[2].ptr;\n octx.dst.data = (uint32_t) bufs[3].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_matmul_id(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_binary_req(struct htp_context * ctx, struct htp_general_req * req, struct dspqueue_buffer * bufs) {\n struct dspqueue_buffer rsp_bufs[1];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[2].fd;\n rsp_bufs[0].ptr = bufs[2].ptr;\n rsp_bufs[0].offset = bufs[2].offset;\n rsp_bufs[0].size = bufs[2].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.src1 = req->src1;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.src1.data = (uint32_t) bufs[1].ptr;\n octx.dst.data = (uint32_t) bufs[2].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_binary(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_add_id_req(struct htp_context * ctx, struct htp_general_req * req, struct dspqueue_buffer * bufs) {\n struct dspqueue_buffer rsp_bufs[1];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[3].fd;\n rsp_bufs[0].ptr = bufs[3].ptr;\n rsp_bufs[0].offset = bufs[3].offset;\n rsp_bufs[0].size = bufs[3].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.src1 = req->src1;\n octx.src2 = req->src2;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.src1.data = (uint32_t) bufs[1].ptr;\n octx.src2.data = (uint32_t) bufs[2].ptr;\n octx.dst.data = (uint32_t) bufs[3].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_binary(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_unary_req(struct htp_context * ctx, struct htp_general_req * req, struct dspqueue_buffer * bufs) {\n struct dspqueue_buffer rsp_bufs[HTP_MAX_PACKET_BUFFERS];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[1].fd;\n rsp_bufs[0].ptr = bufs[1].ptr;\n rsp_bufs[0].offset = bufs[1].offset;\n rsp_bufs[0].size = bufs[1].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n memcpy(octx.op_params, req->op_params, sizeof(octx.op_params));\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.dst.data = (uint32_t) bufs[1].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_unary(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_sum_rows_req(struct htp_context * ctx, struct htp_general_req * req, struct dspqueue_buffer * bufs) {\n struct dspqueue_buffer rsp_bufs[HTP_MAX_PACKET_BUFFERS];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[1].fd;\n rsp_bufs[0].ptr = bufs[1].ptr;\n rsp_bufs[0].offset = bufs[1].offset;\n rsp_bufs[0].size = bufs[1].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n memcpy(octx.op_params, req->op_params, sizeof(octx.op_params));\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.dst.data = (uint32_t) bufs[1].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_sum_rows(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_ssm_conv_req(struct htp_context * ctx, struct htp_general_req * req, struct dspqueue_buffer * bufs) {\n struct dspqueue_buffer rsp_bufs[HTP_MAX_PACKET_BUFFERS];\n\n // We've written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[2].fd;\n rsp_bufs[0].ptr = bufs[2].ptr;\n rsp_bufs[0].offset = bufs[2].offset;\n rsp_bufs[0].size = bufs[2].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup OP context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.src1 = req->src1;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n memcpy(octx.op_params, req->op_params, sizeof(octx.op_params));\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.src1.data = (uint32_t) bufs[1].ptr;\n octx.dst.data = (uint32_t) bufs[2].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_ssm_conv(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_activations_req(struct htp_context * ctx,\n struct htp_general_req * req,\n struct dspqueue_buffer * bufs,\n uint32_t n_bufs) {\n struct dspqueue_buffer rsp_bufs[HTP_MAX_PACKET_BUFFERS];\n\n int write_idx = (n_bufs == 3) ? 2 : 1;\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[write_idx].fd;\n rsp_bufs[0].ptr = bufs[write_idx].ptr;\n rsp_bufs[0].offset = bufs[write_idx].offset;\n rsp_bufs[0].size = bufs[write_idx].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n if (3 == n_bufs) {\n octx.src1 = req->src1;\n }\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n memcpy(octx.op_params, req->op_params, sizeof(octx.op_params));\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n if (3 == n_bufs) {\n octx.src1.data = (uint32_t) bufs[1].ptr;\n octx.dst.data = (uint32_t) bufs[2].ptr;\n } else {\n octx.dst.data = (uint32_t) bufs[1].ptr;\n }\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n if (octx.op == HTP_OP_SOFTMAX) {\n rsp_status = op_softmax(&octx);\n } else {\n rsp_status = op_activations(&octx);\n }\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_rope_req(struct htp_context * ctx,\n struct htp_general_req * req,\n struct dspqueue_buffer * bufs,\n uint32_t n_bufs) {\n struct dspqueue_buffer rsp_bufs[HTP_MAX_PACKET_BUFFERS];\n\n int write_idx = n_bufs - 1;\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[write_idx].fd;\n rsp_bufs[0].ptr = bufs[write_idx].ptr;\n rsp_bufs[0].offset = bufs[write_idx].offset;\n rsp_bufs[0].size = bufs[write_idx].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.src1 = req->src1;\n if (4 == n_bufs) {\n octx.src2 = req->src2;\n }\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n memcpy(octx.op_params, req->op_params, sizeof(octx.op_params));\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.src1.data = (uint32_t) bufs[1].ptr;\n if (4 == n_bufs) {\n octx.src2.data = (uint32_t) bufs[2].ptr;\n octx.dst.data = (uint32_t) bufs[3].ptr;\n } else {\n octx.dst.data = (uint32_t) bufs[2].ptr;\n }\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_rope(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_set_rows_req(struct htp_context * ctx, struct htp_general_req * req, struct dspqueue_buffer * bufs) {\n struct dspqueue_buffer rsp_bufs[1];\n\n // We had written to the output buffer, we'd also need to flush it\n rsp_bufs[0].fd = bufs[2].fd;\n rsp_bufs[0].ptr = bufs[2].ptr;\n rsp_bufs[0].offset = bufs[2].offset;\n rsp_bufs[0].size = bufs[2].size;\n rsp_bufs[0].flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n // Setup Op context\n struct htp_ops_context octx = { 0 };\n octx.ctx = ctx;\n octx.src0 = req->src0;\n octx.src1 = req->src1;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.src1.data = (uint32_t) bufs[1].ptr;\n octx.dst.data = (uint32_t) bufs[2].ptr;\n octx.n_threads = ctx->n_threads;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_set_rows(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n send_htp_rsp(ctx, req->op, rsp_status, rsp_bufs, 1, &prof);\n}\n\nstatic void proc_flash_attn_ext_req(struct htp_context * ctx,\n struct htp_general_req * req,\n struct dspqueue_buffer * bufs,\n uint32_t n_bufs) {\n // Setup Op context\n struct htp_ops_context octx;\n memset(&octx, 0, sizeof(octx));\n\n octx.ctx = ctx;\n octx.n_threads = ctx->n_threads;\n\n octx.src0 = req->src0;\n octx.src1 = req->src1;\n octx.src2 = req->src2;\n octx.src3 = req->src3;\n octx.src4 = req->src4;\n octx.dst = req->dst;\n octx.flags = req->flags;\n octx.op = req->op;\n\n memcpy(octx.op_params, req->op_params, sizeof(octx.op_params));\n\n // Update data pointers\n octx.src0.data = (uint32_t) bufs[0].ptr;\n octx.src1.data = (uint32_t) bufs[1].ptr;\n octx.src2.data = (uint32_t) bufs[2].ptr;\n\n int last_buf = 3;\n\n if (octx.src3.ne[0]) {\n octx.src3.data = (uint32_t) bufs[last_buf++].ptr; // mask is valid\n }\n\n if (octx.src4.ne[0]) {\n octx.src4.data = (uint32_t) bufs[last_buf++].ptr; // sinks is valid\n }\n\n octx.dst.data = (uint32_t) bufs[last_buf].ptr;\n\n struct profile_data prof;\n profile_start(&prof);\n\n uint32_t rsp_status = HTP_STATUS_INTERNAL_ERR;\n if (vtcm_acquire(ctx) == AEE_SUCCESS) {\n rsp_status = op_flash_attn_ext(&octx);\n vtcm_release(ctx);\n }\n\n profile_stop(&prof);\n\n struct dspqueue_buffer rsp_buf = bufs[last_buf];\n rsp_buf.flags = (DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | // Flush HTP\n DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT); // Invalidate CPU\n\n send_htp_rsp(ctx, req->op, rsp_status, &bufs[last_buf], 1, &prof);\n}\n\nstatic void htp_packet_callback(dspqueue_t queue, int error, void * context) {\n struct htp_context * ctx = (struct htp_context *) context;\n\n // Repeatedly read packets from the queue until it's empty. We don't\n // necessarily get a separate callback for each packet, and new packets\n // may arrive while we're processing the previous one. This ensures we\n // keep the DSP busy as much as possible and avoid waiting for the CPU.\n\n while (1) {\n struct htp_general_req req;\n uint32_t req_size;\n\n struct dspqueue_buffer bufs[HTP_MAX_PACKET_BUFFERS];\n uint32_t n_bufs;\n uint32_t flags;\n\n // Read packet from queue\n int err = dspqueue_read_noblock(queue, &flags,\n HTP_MAX_PACKET_BUFFERS, // Maximum number of buffer references\n &n_bufs, // Number of buffer references\n bufs, // Buffer references\n sizeof(req), // Max message length\n &req_size, // Message length\n (uint8_t *) &req); // Message\n\n if (err == AEE_EWOULDBLOCK) {\n // Consumed all packets available for now\n return;\n }\n\n if (err != 0) {\n FARF(ERROR, \"dspqueue_read_noblock failed: 0x%08x\", (unsigned) err);\n return;\n }\n\n if (req_size != sizeof(req)) {\n FARF(ERROR, \"Invalid request size\");\n continue;\n }\n\n if (req.flags & HTP_OPFLAGS_EARLY_WAKEUP) {\n // Host wants early notification\n dspqueue_write_early_wakeup_noblock(ctx->queue, 10, 0);\n }\n\n // Process packet based on its message type\n switch (req.op) {\n case HTP_OP_MUL_MAT:\n if (n_bufs != 3) {\n FARF(ERROR, \"Bad matmul-req buffer list\");\n continue;\n }\n proc_matmul_req(ctx, &req, bufs, n_bufs);\n break;\n\n case HTP_OP_MUL_MAT_ID:\n if (n_bufs != 4) {\n FARF(ERROR, \"Bad matmul-id-req buffer list\");\n continue;\n }\n proc_matmul_id_req(ctx, &req, bufs, n_bufs);\n break;\n\n case HTP_OP_MUL:\n case HTP_OP_ADD:\n case HTP_OP_SUB:\n case HTP_OP_DIV:\n if (n_bufs != 3) {\n FARF(ERROR, \"Bad binary-req buffer list\");\n continue;\n }\n proc_binary_req(ctx, &req, bufs);\n break;\n\n case HTP_OP_RMS_NORM:\n case HTP_OP_SCALE:\n if (n_bufs != 2) {\n FARF(ERROR, \"Bad unary-req buffer list\");\n continue;\n }\n\n proc_unary_req(ctx, &req, bufs);\n break;\n\n case HTP_OP_SQR:\n case HTP_OP_SQRT:\n if (n_bufs != 2) {\n FARF(ERROR, \"Bad unary-req buffer list\");\n continue;\n }\n\n proc_unary_req(ctx, &req, bufs);\n break;\n\n case HTP_OP_SUM_ROWS:\n if (n_bufs != 2) {\n FARF(ERROR, \"Bad unary-req buffer list\");\n continue;\n }\n\n proc_sum_rows_req(ctx, &req, bufs);\n break;\n\n case HTP_OP_UNARY_SILU:\n case HTP_OP_UNARY_GELU:\n if (n_bufs != 2) {\n FARF(ERROR, \"Bad act-req buffer list\");\n continue;\n }\n proc_activations_req(ctx, &req, bufs, n_bufs);\n break;\n\n case HTP_OP_GLU_SWIGLU:\n case HTP_OP_GLU_SWIGLU_OAI:\n case HTP_OP_SOFTMAX:\n case HTP_OP_GLU_GEGLU:\n if ((n_bufs != 2) && (n_bufs != 3)) {\n FARF(ERROR, \"Bad act-req buffer list\");\n continue;\n }\n proc_activations_req(ctx, &req, bufs, n_bufs);\n break;\n\n case HTP_OP_ADD_ID:\n if (n_bufs != 4) {\n FARF(ERROR, \"Bad add-id-req buffer list\");\n continue;\n }\n proc_add_id_req(ctx, &req, bufs);\n break;\n\n case HTP_OP_ROPE:\n if ((n_bufs != 3) && (n_bufs != 4)) {\n FARF(ERROR, \"Bad rope-req buffer list\");\n continue;\n }\n proc_rope_req(ctx, &req, bufs, n_bufs);\n break;\n\n case HTP_OP_FLASH_ATTN_EXT:\n if (!(n_bufs >= 4 && n_bufs <= 6)) {\n FARF(ERROR, \"Bad flash-attn-ext-req buffer list\");\n continue;\n }\n proc_flash_attn_ext_req(ctx, &req, bufs, n_bufs);\n break;\n\n case HTP_OP_SET_ROWS:\n if (n_bufs != 3) {\n FARF(ERROR, \"Bad set-rows-req buffer list\");\n continue;\n }\n proc_set_rows_req(ctx, &req, bufs);\n break;\n\n case HTP_OP_GET_ROWS:\n if (n_bufs != 3) {\n FARF(ERROR, \"Bad get-rows-req buffer list\");\n continue;\n }\n proc_get_rows_req(ctx, &req, bufs);\n break;\n\n case HTP_OP_CPY:\n if (n_bufs != 2) {\n FARF(ERROR, \"Bad cpy-req buffer list\");\n continue;\n }\n proc_cpy_req(ctx, &req, bufs);\n break;\n\n case HTP_OP_ARGSORT:\n if (n_bufs != 2) {\n FARF(ERROR, \"Bad argsort-req buffer list\");\n continue;\n }\n proc_argsort_req(ctx, &req, bufs);\n break;\n\n case HTP_OP_SSM_CONV:\n if (n_bufs != 3) {\n FARF(ERROR, \"Bad ssm-conv-req buffer list\");\n continue;\n }\n proc_ssm_conv_req(ctx, &req, bufs);\n break;\n\n default:\n FARF(ERROR, \"Unknown Op %u\", req.op);\n break;\n }\n }\n}\n" }, { "path": "src/ggml-hexagon/htp/matmul-ops.c", "content": "#pragma clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hvx-utils.h\"\n#include \"hvx-dump.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\n#define MM_SPAD_SRC0_NROWS 16\n#define MM_SPAD_SRC1_NROWS 16\n#define MM_SPAD_DST_NROWS 2\n\nstruct htp_matmul_context {\n const char * type;\n struct htp_ops_context * octx;\n\n void (*vec_dot_1x1)(const int n, float * restrict s0,\n const void * restrict vx0,\n const void * restrict vy0);\n\n void (*vec_dot_2x1)(const int n, float * restrict s0,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0);\n\n void (*vec_dot_2x2)(const int n, float * restrict s0, float * restrict s1,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0, const void * restrict vy1);\n\n // Precomputed values\n uint32_t src0_nrows_per_thread;\n uint32_t src1_nrows_per_thread;\n\n struct fastdiv_values mm_div_ne12_ne1;\n struct fastdiv_values mm_div_ne1;\n struct fastdiv_values mm_div_r2;\n struct fastdiv_values mm_div_r3;\n};\n\n// vdelta control to expand first 32 e8m0 values into 32 uint32 elements\nstatic const uint8_t __attribute__((aligned(128))) expand_x32_e8m0[128] = {\n 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x02, 0x00, 0x08, 0x08, 0x01, 0x02, 0x00, 0x04, 0x04, 0x00, 0x00,\n 0x00, 0x11, 0x10, 0x10, 0x10, 0x02, 0x00, 0x04, 0x00, 0x01, 0x02, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x01, 0x04,\n 0x00, 0x00, 0x22, 0x20, 0x20, 0x20, 0x21, 0x22, 0x20, 0x24, 0x04, 0x00, 0x00, 0x00, 0x09, 0x08, 0x00, 0x00, 0x02,\n 0x00, 0x04, 0x00, 0x11, 0x12, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x01, 0x04, 0x00, 0x00, 0x02, 0x00, 0x08, 0x08,\n 0x01, 0x02, 0x00, 0x04, 0x44, 0x40, 0x40, 0x40, 0x41, 0x40, 0x40, 0x40, 0x42, 0x40, 0x44, 0x40, 0x41, 0x42, 0x48,\n 0x48, 0x08, 0x08, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x12, 0x10, 0x10, 0x10, 0x01, 0x02, 0x00, 0x04, 0x04, 0x00,\n 0x00, 0x00, 0x09, 0x08, 0x00, 0x00, 0x22, 0x20, 0x24, 0x20, 0x21, 0x22, 0x20, 0x20,\n};\n\nstatic const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = {\n 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 6, 0, 8, 0, 12, 0, 0, 0, 0xff, 0, 0xfe, 0, 0xfd, 0, 0xfc, 0,\n 0xfa, 0, 0xf8, 0, 0xf4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n};\n\n// q4x4x2 and q8x4x2 are the flat q4/8_0 formats where all quants are stored first followed by all scales\n\nstatic inline size_t q8x4x2_row_size(uint32_t ne) {\n // ensures perfect alignment of quants and full row\n const uint32_t qk = QK_Q8_0x4x2;\n const uint32_t nb = (ne + qk - 1) / qk;\n return hex_round_up(ne + nb * 8 * sizeof(__fp16), 128);\n}\n\nstatic inline HVX_Vector_x8 hvx_vec_load_q4x4x8_full(const uint8_t * restrict ptr) {\n const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr;\n\n HVX_Vector v0_1 = vptr[0]; // first 256 elements (128 bytes)\n HVX_Vector v2_3 = vptr[1]; // ...\n HVX_Vector v4_5 = vptr[2]; // ...\n HVX_Vector v6_7 = vptr[3]; // ...\n\n const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F);\n const HVX_Vector i8 = Q6_Vb_vsplat_R(8);\n\n HVX_Vector v0 = Q6_V_vand_VV(v0_1, mask_h4); // & 0x0F : first 128 elements\n HVX_Vector v1 = Q6_Vub_vlsr_VubR(v0_1, 4); // >> 4 : second 128 elements\n HVX_Vector v2 = Q6_V_vand_VV(v2_3, mask_h4); // & 0x0F ...\n HVX_Vector v3 = Q6_Vub_vlsr_VubR(v2_3, 4); // >> 4\n HVX_Vector v4 = Q6_V_vand_VV(v4_5, mask_h4); // & 0x0F\n HVX_Vector v5 = Q6_Vub_vlsr_VubR(v4_5, 4); // >> 4\n HVX_Vector v6 = Q6_V_vand_VV(v6_7, mask_h4); // & 0x0F\n HVX_Vector v7 = Q6_Vub_vlsr_VubR(v6_7, 4); // >> 4\n\n // Convert uint4 to int4 (i.e. x - 8)\n v0 = Q6_Vb_vsub_VbVb(v0, i8);\n v1 = Q6_Vb_vsub_VbVb(v1, i8);\n v2 = Q6_Vb_vsub_VbVb(v2, i8);\n v3 = Q6_Vb_vsub_VbVb(v3, i8);\n v4 = Q6_Vb_vsub_VbVb(v4, i8);\n v5 = Q6_Vb_vsub_VbVb(v5, i8);\n v6 = Q6_Vb_vsub_VbVb(v6, i8);\n v7 = Q6_Vb_vsub_VbVb(v7, i8);\n\n HVX_Vector_x8 r = { v0, v1, v2, v3, v4, v5, v6, v7 };\n return r;\n}\n\nstatic HVX_Vector_x8 hvx_vec_load_q4x4x8_partial(const uint8_t * restrict ptr, uint32_t n) {\n const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr;\n\n const uint32_t qk = QK_Q4_0x4x2; // 256\n const uint32_t nb = n / qk;\n const uint32_t nloe = n % qk;\n\n const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F);\n const HVX_Vector i8 = Q6_Vb_vsplat_R(8);\n\n HVX_Vector_x8 r;\n uint32_t i = 0;\n\n #pragma unroll(2)\n for (i=0; i < nb; i++) {\n HVX_Vector v = vptr[i]; // 256 elements (128 bytes)\n HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : first 128 elements\n HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : second 128 elements\n r.v[i*2+0] = Q6_Vb_vsub_VbVb(v0, i8);\n r.v[i*2+1] = Q6_Vb_vsub_VbVb(v1, i8);\n }\n\n if (nloe) {\n HVX_Vector v = vptr[i]; // 256 elements (128 bytes)\n HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : even 128 elements\n HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : odd 128 elements\n HVX_VectorPair v0_1_p = Q6_W_vshuff_VVR(v1, v0, -1); // zip even:odd:...\n r.v[i*2+0] = Q6_Vb_vsub_VbVb(Q6_V_lo_W(v0_1_p), i8);\n r.v[i*2+1] = Q6_Vb_vsub_VbVb(Q6_V_hi_W(v0_1_p), i8);\n }\n\n return r;\n}\n\nstatic inline HVX_Vector_x8 hvx_vec_load_mxfp4x4x8_full(const uint8_t * restrict ptr) {\n const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr;\n\n HVX_Vector v0_1 = vptr[0]; // first 256 elements (128 bytes)\n HVX_Vector v2_3 = vptr[1]; // ...\n HVX_Vector v4_5 = vptr[2]; // ...\n HVX_Vector v6_7 = vptr[3]; // ...\n\n const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F);\n const HVX_Vector lut = *(const HVX_Vector *) kvalues_mxfp4_lut;\n\n HVX_Vector v0 = Q6_V_vand_VV(v0_1, mask_h4); // & 0x0F\n HVX_Vector v1 = Q6_Vub_vlsr_VubR(v0_1, 4); // >> 4\n HVX_Vector v2 = Q6_V_vand_VV(v2_3, mask_h4); // & 0x0F\n HVX_Vector v3 = Q6_Vub_vlsr_VubR(v2_3, 4); // >> 4\n HVX_Vector v4 = Q6_V_vand_VV(v4_5, mask_h4); // & 0x0F\n HVX_Vector v5 = Q6_Vub_vlsr_VubR(v4_5, 4); // >> 4\n HVX_Vector v6 = Q6_V_vand_VV(v6_7, mask_h4); // & 0x0F\n HVX_Vector v7 = Q6_Vub_vlsr_VubR(v6_7, 4); // >> 4\n\n v0 = Q6_Vb_vlut32_VbVbI(v0, lut, 0);\n v1 = Q6_Vb_vlut32_VbVbI(v1, lut, 0);\n v2 = Q6_Vb_vlut32_VbVbI(v2, lut, 0);\n v3 = Q6_Vb_vlut32_VbVbI(v3, lut, 0);\n v4 = Q6_Vb_vlut32_VbVbI(v4, lut, 0);\n v5 = Q6_Vb_vlut32_VbVbI(v5, lut, 0);\n v6 = Q6_Vb_vlut32_VbVbI(v6, lut, 0);\n v7 = Q6_Vb_vlut32_VbVbI(v7, lut, 0);\n\n HVX_Vector_x8 r = { v0, v1, v2, v3, v4, v5, v6, v7 };\n return r;\n}\n\nstatic inline HVX_Vector_x8 hvx_vec_load_mxfp4x4x8_partial(const uint8_t * restrict ptr, uint32_t n) {\n const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr;\n\n const uint32_t qk = QK_Q4_0x4x2; // 256\n const uint32_t nb = n / qk;\n const uint32_t nloe = n % qk;\n\n const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F);\n const HVX_Vector lut = *(const HVX_Vector *) kvalues_mxfp4_lut;\n\n HVX_Vector_x8 r;\n uint32_t i = 0;\n\n #pragma unroll(2)\n for (i=0; i < nb; i++) {\n HVX_Vector v = vptr[i]; // 256 elements (128 bytes)\n HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : first 128 elements\n HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : second 128 elements\n r.v[i*2+0] = Q6_Vb_vlut32_VbVbI(v0, lut, 0);\n r.v[i*2+1] = Q6_Vb_vlut32_VbVbI(v1, lut, 0);\n }\n\n if (nloe) {\n HVX_Vector v = vptr[i]; // 256 elements (128 bytes)\n HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : even 128 elements\n HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : odd 128 elements\n HVX_VectorPair v0_1_p = Q6_W_vshuff_VVR(v1, v0, -1); // zip even:odd:...\n r.v[i*2+0] = Q6_Vb_vlut32_VbVbI(Q6_V_lo_W(v0_1_p), lut, 0);\n r.v[i*2+1] = Q6_Vb_vlut32_VbVbI(Q6_V_hi_W(v0_1_p), lut, 0);\n }\n\n return r;\n}\n\nstatic inline HVX_Vector_x8 hvx_vec_load_q8x4x8_full(const uint8_t * restrict ptr) {\n const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr;\n\n HVX_Vector v0 = vptr[0]; // first 128 vals\n HVX_Vector v1 = vptr[1]; // ...\n HVX_Vector v2 = vptr[2]; // ...\n HVX_Vector v3 = vptr[3]; // ...\n HVX_Vector v4 = vptr[4]; // ...\n HVX_Vector v5 = vptr[5]; // ...\n HVX_Vector v6 = vptr[6]; // ...\n HVX_Vector v7 = vptr[7]; // ...\n\n HVX_Vector_x8 r = { v0, v1, v2, v3, v4, v5, v6, v7 };\n return r;\n}\n\nstatic inline HVX_Vector_x8 hvx_vec_load_q8x4x8_partial(const uint8_t * restrict ptr, uint32_t nloe) {\n return hvx_vec_load_q8x4x8_full(ptr);\n}\n\n// Reduce multiply 1024 x 1024 int8 elements (32x q4/8 blocks in 8x HVX vectors).\n// Accumulate each block into a single int32 value.\n// Return a single HVX vector with 32x int32 accumulators.\n// This version is parameterized to support less than 1024 elements.\n// if() checks are optimized out at compile time -- make sure to pass N as a constexpr.\n\nstatic inline HVX_Vector hvx_vec_rmpy_x8_n(HVX_Vector_x8 x, HVX_Vector_x8 y, unsigned int n) {\n HVX_Vector r0 = Q6_V_vzero();\n HVX_Vector r1 = Q6_V_vzero();\n HVX_Vector r2 = Q6_V_vzero();\n HVX_Vector r3 = Q6_V_vzero();\n HVX_Vector r4 = Q6_V_vzero();\n HVX_Vector r5 = Q6_V_vzero();\n HVX_Vector r6 = Q6_V_vzero();\n HVX_Vector r7 = Q6_V_vzero();\n\n HVX_VectorPair p3;\n HVX_VectorPair p2;\n HVX_VectorPair p1;\n HVX_VectorPair p0;\n\n if (n >= 128) { r0 = Q6_Vw_vrmpy_VbVb(x.v[0], y.v[0]); }\n if (n >= 256) { r1 = Q6_Vw_vrmpy_VbVb(x.v[1], y.v[1]); }\n if (n >= 384) { r2 = Q6_Vw_vrmpy_VbVb(x.v[2], y.v[2]); }\n if (n >= 512) { r3 = Q6_Vw_vrmpy_VbVb(x.v[3], y.v[3]); }\n if (n >= 640) { r4 = Q6_Vw_vrmpy_VbVb(x.v[4], y.v[4]); }\n if (n >= 768) { r5 = Q6_Vw_vrmpy_VbVb(x.v[5], y.v[5]); }\n if (n >= 896) { r6 = Q6_Vw_vrmpy_VbVb(x.v[6], y.v[6]); }\n if (n >= 1024) { r7 = Q6_Vw_vrmpy_VbVb(x.v[7], y.v[7]); }\n\n if (n >= 128) { p0 = Q6_W_vdeal_VVR(r1, r0, -4); }\n if (n >= 384) { p1 = Q6_W_vdeal_VVR(r3, r2, -4); }\n if (n >= 640) { p2 = Q6_W_vdeal_VVR(r5, r4, -4); }\n if (n >= 896) { p3 = Q6_W_vdeal_VVR(r7, r6, -4); }\n\n if (n >= 128) { r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0)); }\n if (n >= 384) { r1 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p1), Q6_V_hi_W(p1)); }\n if (n >= 640) { r2 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p2), Q6_V_hi_W(p2)); }\n if (n >= 896) { r3 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p3), Q6_V_hi_W(p3)); }\n\n if (n >= 128) { p0 = Q6_W_vdeal_VVR(r1, r0, -4); }\n if (n >= 640) { p1 = Q6_W_vdeal_VVR(r3, r2, -4); }\n\n if (n >= 128) { r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0)); }\n if (n >= 640) { r1 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p1), Q6_V_hi_W(p1)); }\n\n if (n >= 128) { p0 = Q6_W_vdeal_VVR(r1, r0, -4); }\n if (n >= 128) { r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0)); }\n\n return r0;\n}\n\nstatic inline HVX_Vector hvx_vec_rmpy_x8_full(HVX_Vector_x8 x, HVX_Vector_x8 y) {\n HVX_Vector r0 = Q6_Vw_vrmpy_VbVb(x.v[0], y.v[0]);\n HVX_Vector r1 = Q6_Vw_vrmpy_VbVb(x.v[1], y.v[1]);\n HVX_Vector r2 = Q6_Vw_vrmpy_VbVb(x.v[2], y.v[2]);\n HVX_Vector r3 = Q6_Vw_vrmpy_VbVb(x.v[3], y.v[3]);\n HVX_Vector r4 = Q6_Vw_vrmpy_VbVb(x.v[4], y.v[4]);\n HVX_Vector r5 = Q6_Vw_vrmpy_VbVb(x.v[5], y.v[5]);\n HVX_Vector r6 = Q6_Vw_vrmpy_VbVb(x.v[6], y.v[6]);\n HVX_Vector r7 = Q6_Vw_vrmpy_VbVb(x.v[7], y.v[7]);\n\n HVX_VectorPair p0 = Q6_W_vdeal_VVR(r1, r0, -4);\n HVX_VectorPair p1 = Q6_W_vdeal_VVR(r3, r2, -4);\n HVX_VectorPair p2 = Q6_W_vdeal_VVR(r5, r4, -4);\n HVX_VectorPair p3 = Q6_W_vdeal_VVR(r7, r6, -4);\n\n r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0));\n r1 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p1), Q6_V_hi_W(p1));\n r2 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p2), Q6_V_hi_W(p2));\n r3 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p3), Q6_V_hi_W(p3));\n\n p0 = Q6_W_vdeal_VVR(r1, r0, -4);\n p1 = Q6_W_vdeal_VVR(r3, r2, -4);\n\n r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0));\n r1 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p1), Q6_V_hi_W(p1));\n\n p0 = Q6_W_vdeal_VVR(r1, r0, -4);\n r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0));\n\n return r0;\n}\n\nstatic inline HVX_Vector hvx_vec_rmpy_x8_partial(HVX_Vector_x8 x, HVX_Vector_x8 y, unsigned int n) {\n if (n >= 512)\n return hvx_vec_rmpy_x8_full(x, y);\n\n return hvx_vec_rmpy_x8_partial(x, y, 512);\n}\n\nstatic void vec_dot_q4x4x2_q8x4x2_1x1(const int n, float * restrict s0, const void * restrict vx0, const void * restrict vy0) {\n assert(n % 32 == 0); // min sub-block size\n assert((unsigned long) vx0 % 128 == 0);\n assert((unsigned long) vy0 % 128 == 0);\n\n const uint32_t qk = QK_Q4_0x4x2 * 4;\n\n const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t x_qblk_size = qk / 2; // int4\n const uint32_t x_qrow_size = n / 2; // int4 (not padded)\n\n const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t y_qblk_size = qk; // int8\n const uint32_t y_qrow_size = n; // int8 (not padded)\n\n const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0 + 0); // quants first\n const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0 + x_qrow_size); // then scales\n\n const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first\n const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales\n\n // Row sum (sf)\n HVX_Vector r0_sum = Q6_V_vzero();\n\n // Multiply and accumulate into int32.\n // Compute combined scale (fp32).\n // Apply scale to acc and accumulate into the row sum (qf32).\n\n const uint32_t nb = n / qk; // num full blocks\n const uint32_t nloe = n % qk; // num leftover elemements\n\n uint32_t i = 0;\n for (; i < nb; i++) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size);\n HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_full(r0_x_q + i * x_qblk_size);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q));\n\n HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d)));\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n }\n\n // Process leftovers\n if (nloe) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_partial(r0_x_q + i * x_qblk_size, nloe);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe));\n\n HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d)));\n\n // Zero out unused elements\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8);\n r0_dd = Q6_V_vand_QV(bmask, r0_dd);\n r0_ia = Q6_V_vand_QV(bmask, r0_ia);\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n }\n\n r0_sum = hvx_vec_reduce_sum_f32(r0_sum);\n\n hvx_vec_store_u(s0, 4, r0_sum);\n}\n\nstatic void vec_dot_q4x4x2_q8x4x2_2x1(const int n, float * restrict s0,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0) {\n assert(n % 32 == 0); // min sub-block size\n assert((unsigned long) vx0 % 128 == 0);\n assert((unsigned long) vx1 % 128 == 0);\n assert((unsigned long) vy0 % 128 == 0);\n\n const uint32_t qk = QK_Q4_0x4x2 * 4;\n\n const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t x_qblk_size = qk / 2; // int4\n const uint32_t x_qrow_size = n / 2; // int4 (not padded)\n\n const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t y_qblk_size = qk; // int8\n const uint32_t y_qrow_size = n; // int8 (not padded)\n\n const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first\n const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales\n const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first\n const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales\n\n const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first\n const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales\n\n // Row sum (sf)\n HVX_Vector r0_sum = Q6_V_vzero();\n HVX_Vector r1_sum = Q6_V_vzero();\n\n // Multiply and accumulate into int32.\n // Compute combined scale (fp32).\n // Apply scale to acc and accumulate into the row sum (qf32).\n\n const uint32_t nb = n / qk; // num full blocks\n const uint32_t nloe = n % qk; // num leftover elemements\n\n uint32_t i = 0;\n for (; i < nb; i++) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size);\n HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_full(r0_x_q + i * x_qblk_size);\n HVX_Vector_x8 r1_q = hvx_vec_load_q4x4x8_full(r1_x_q + i * x_qblk_size);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q));\n HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy_q));\n\n HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size));\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d)));\n HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d)));\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum));\n }\n\n // Process leftovers\n if (nloe) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_partial(r0_x_q + i * x_qblk_size, nloe);\n HVX_Vector_x8 r1_q = hvx_vec_load_q4x4x8_partial(r1_x_q + i * x_qblk_size, nloe);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe));\n HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy_q, nloe));\n\n HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size));\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d)));\n HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d)));\n\n // Zero out unused elements\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8);\n r0_dd = Q6_V_vand_QV(bmask, r0_dd);\n r1_dd = Q6_V_vand_QV(bmask, r1_dd);\n r0_ia = Q6_V_vand_QV(bmask, r0_ia);\n r1_ia = Q6_V_vand_QV(bmask, r1_ia);\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum));\n }\n\n HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(r0_sum, r1_sum);\n hvx_vec_store_u(s0, 8, rsum);\n}\n\nstatic void vec_dot_q4x4x2_q8x4x2_2x2(const int n, float * restrict s0, float * restrict s1,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0, const void * restrict vy1) {\n assert(n % 32 == 0);\n assert((unsigned long) vx0 % 128 == 0);\n assert((unsigned long) vx1 % 128 == 0);\n assert((unsigned long) vy0 % 128 == 0);\n assert((unsigned long) vy1 % 128 == 0);\n\n const uint32_t qk = QK_Q4_0x4x2 * 4;\n\n const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t x_qblk_size = qk / 2; // int4\n const uint32_t x_qrow_size = n / 2; // int4 (not padded)\n\n const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t y_qblk_size = qk; // int8\n const uint32_t y_qrow_size = n; // int8 (not padded)\n\n const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first\n const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales\n const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first\n const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales\n\n const uint8_t * restrict y0_q = ((const uint8_t *) vy0) + 0; // quants first\n const uint8_t * restrict y0_d = ((const uint8_t *) vy0) + y_qrow_size; // then scales\n const uint8_t * restrict y1_q = ((const uint8_t *) vy1) + 0; // quants first\n const uint8_t * restrict y1_d = ((const uint8_t *) vy1) + y_qrow_size; // then scales\n\n // Row sums (sf) - 4 accumulators for 2×2 tile\n HVX_Vector r0_c0_sum = Q6_V_vzero();\n HVX_Vector r0_c1_sum = Q6_V_vzero();\n HVX_Vector r1_c0_sum = Q6_V_vzero();\n HVX_Vector r1_c1_sum = Q6_V_vzero();\n\n const uint32_t nb = n / qk; // num full blocks\n const uint32_t nloe = n % qk; // num leftover elements\n\n uint32_t i = 0;\n for (; i < nb; i++) {\n // Load src1 columns (reused across both src0 rows)\n HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_full(y0_q + i * y_qblk_size);\n HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_full(y1_q + i * y_qblk_size);\n\n // Load src0 rows (reused across both src1 columns)\n HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_full(r0_x_q + i * x_qblk_size);\n HVX_Vector_x8 r1_q = hvx_vec_load_q4x4x8_full(r1_x_q + i * x_qblk_size);\n\n // Compute 4 dot products: r0×c0, r0×c1, r1×c0, r1×c1\n HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy0_q));\n HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy1_q));\n HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy0_q));\n HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy1_q));\n\n // Load scales\n HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size));\n HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size));\n\n // Compute combined scales\n HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d)));\n HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d)));\n HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d)));\n HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d)));\n\n // Apply scales and accumulate\n HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd);\n HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd);\n HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd);\n HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd);\n\n r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum));\n r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum));\n r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum));\n r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum));\n }\n\n // Process leftovers\n if (nloe) {\n HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_partial(y0_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_partial(y1_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_partial(r0_x_q + i * x_qblk_size, nloe);\n HVX_Vector_x8 r1_q = hvx_vec_load_q4x4x8_partial(r1_x_q + i * x_qblk_size, nloe);\n\n HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy0_q, nloe));\n HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy1_q, nloe));\n HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy0_q, nloe));\n HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy1_q, nloe));\n\n HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size));\n HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size));\n\n HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d)));\n HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d)));\n HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d)));\n HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d)));\n\n // Zero out unused scales\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8);\n r0_c0_dd = Q6_V_vand_QV(bmask, r0_c0_dd);\n r0_c1_dd = Q6_V_vand_QV(bmask, r0_c1_dd);\n r1_c0_dd = Q6_V_vand_QV(bmask, r1_c0_dd);\n r1_c1_dd = Q6_V_vand_QV(bmask, r1_c1_dd);\n r0_c0_ia = Q6_V_vand_QV(bmask, r0_c0_ia);\n r0_c1_ia = Q6_V_vand_QV(bmask, r0_c1_ia);\n r1_c0_ia = Q6_V_vand_QV(bmask, r1_c0_ia);\n r1_c1_ia = Q6_V_vand_QV(bmask, r1_c1_ia);\n\n HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd);\n HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd);\n HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd);\n HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd);\n\n r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum));\n r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum));\n r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum));\n r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum));\n }\n\n // Reduce and store results\n HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum);\n HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum);\n\n hvx_vec_store_u(s0, 8, r0_r1_c0_sum); // row0,col0 row1,col0\n hvx_vec_store_u(s1, 8, r0_r1_c1_sum); // row0,col1 row1,col1\n}\n\nstatic void vec_dot_q8x4x2_q8x4x2_1x1(const int n, float * restrict s0, const void * restrict vx0, const void * restrict vy0) {\n assert(n % 32 == 0); // min sub-block size\n assert((unsigned long) vx0 % 128 == 0);\n assert((unsigned long) vy0 % 128 == 0);\n\n const uint32_t qk = QK_Q4_0x4x2 * 4;\n\n const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t x_qblk_size = qk; // int8\n const uint32_t x_qrow_size = n; // int8 (not padded)\n\n const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t y_qblk_size = qk; // int8\n const uint32_t y_qrow_size = n; // int8 (not padded)\n\n const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0 + 0); // quants first\n const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0 + x_qrow_size); // then scales\n\n const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first\n const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales\n\n // Row sum (sf)\n HVX_Vector r0_sum = Q6_V_vzero();\n\n // Multiply and accumulate into int32.\n // Compute combined scale (fp32).\n // Apply scale to acc and accumulate into the row sum (qf32).\n\n const uint32_t nb = n / qk; // num full blocks\n int32_t nloe = n % qk; // num leftover elemements (must be signed)\n\n uint32_t i = 0;\n for (; i < nb; i++) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size);\n HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_full(r0_x_q + i * x_qblk_size);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q));\n\n HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d)));\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n }\n\n // Process leftovers\n if (nloe) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_partial(r0_x_q + i * x_qblk_size, nloe);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe));\n\n HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d)));\n\n // Zero out unused elements\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8);\n r0_dd = Q6_V_vand_QV(bmask, r0_dd);\n r0_ia = Q6_V_vand_QV(bmask, r0_ia);\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n }\n\n r0_sum = hvx_vec_reduce_sum_f32(r0_sum);\n\n hvx_vec_store_u(s0, 4, r0_sum);\n}\n\nstatic void vec_dot_q8x4x2_q8x4x2_2x1(const int n, float * restrict s0,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0) {\n assert(n % 32 == 0); // min sub-block size\n assert((unsigned long) vx0 % 128 == 0);\n assert((unsigned long) vx1 % 128 == 0);\n assert((unsigned long) vy0 % 128 == 0);\n\n const uint32_t qk = QK_Q4_0x4x2 * 4;\n\n const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t x_qblk_size = qk; // int8\n const uint32_t x_qrow_size = n; // int8 (not padded)\n\n const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t y_qblk_size = qk; // int8\n const uint32_t y_qrow_size = n; // int8 (not padded)\n\n const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first\n const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales\n const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first\n const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales\n\n const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first\n const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales\n\n // Row sum (qf32)\n HVX_Vector r0_sum = Q6_V_vzero();\n HVX_Vector r1_sum = Q6_V_vzero();\n\n // Multiply and accumulate into int32.\n // Compute combined scale (fp32).\n // Apply scale to acc and accumulate into the row sum (qf32).\n\n const uint32_t nb = n / qk; // num full blocks\n int32_t nloe = n % qk; // num leftover elemements (must be signed)\n\n uint32_t i = 0;\n for (; i < nb; i++) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size);\n HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_full(r0_x_q + i * x_qblk_size);\n HVX_Vector_x8 r1_q = hvx_vec_load_q8x4x8_full(r1_x_q + i * x_qblk_size);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q));\n HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy_q));\n\n HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size));\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d)));\n HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d)));\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum));\n }\n\n // Process leftovers\n if (nloe) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_partial(r0_x_q + i * x_qblk_size, nloe);\n HVX_Vector_x8 r1_q = hvx_vec_load_q8x4x8_partial(r1_x_q + i * x_qblk_size, nloe);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe));\n HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy_q, nloe));\n\n HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size));\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d)));\n HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d)));\n\n // Zero out unused elements\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8);\n r0_dd = Q6_V_vand_QV(bmask, r0_dd);\n r1_dd = Q6_V_vand_QV(bmask, r1_dd);\n r0_ia = Q6_V_vand_QV(bmask, r0_ia);\n r1_ia = Q6_V_vand_QV(bmask, r1_ia);\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum));\n }\n\n HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(r0_sum, r1_sum);\n hvx_vec_store_u(s0, 8, rsum);\n}\n\nstatic void vec_dot_q8x4x2_q8x4x2_2x2(const int n, float * restrict s0, float * restrict s1,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0, const void * restrict vy1) {\n assert(n % 32 == 0);\n assert((unsigned long) vx0 % 128 == 0);\n assert((unsigned long) vx1 % 128 == 0);\n assert((unsigned long) vy0 % 128 == 0);\n assert((unsigned long) vy1 % 128 == 0);\n\n const uint32_t qk = QK_Q8_0x4x2 * 4;\n\n const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t x_qblk_size = qk; // int8\n const uint32_t x_qrow_size = n; // int8 (not padded)\n\n const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t y_qblk_size = qk; // int8\n const uint32_t y_qrow_size = n; // int8 (not padded)\n\n const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first\n const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales\n const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first\n const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales\n\n const uint8_t * restrict y0_q = ((const uint8_t *) vy0) + 0; // quants first\n const uint8_t * restrict y0_d = ((const uint8_t *) vy0) + y_qrow_size; // then scales\n const uint8_t * restrict y1_q = ((const uint8_t *) vy1) + 0; // quants first\n const uint8_t * restrict y1_d = ((const uint8_t *) vy1) + y_qrow_size; // then scales\n\n // Row sums (sf) - 4 accumulators for 2×2 tile\n HVX_Vector r0_c0_sum = Q6_V_vzero();\n HVX_Vector r0_c1_sum = Q6_V_vzero();\n HVX_Vector r1_c0_sum = Q6_V_vzero();\n HVX_Vector r1_c1_sum = Q6_V_vzero();\n\n const uint32_t nb = n / qk; // num full blocks\n const uint32_t nloe = n % qk; // num leftover elements\n\n uint32_t i = 0;\n for (; i < nb; i++) {\n // Load src1 columns (reused across both src0 rows)\n HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_full(y0_q + i * y_qblk_size);\n HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_full(y1_q + i * y_qblk_size);\n\n // Load src0 rows (reused across both src1 columns)\n HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_full(r0_x_q + i * x_qblk_size);\n HVX_Vector_x8 r1_q = hvx_vec_load_q8x4x8_full(r1_x_q + i * x_qblk_size);\n\n // Compute 4 dot products: r0×c0, r0×c1, r1×c0, r1×c1\n HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy0_q));\n HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy1_q));\n HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy0_q));\n HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy1_q));\n\n // Load scales\n HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size));\n HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size));\n\n // Compute combined scales\n HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d)));\n HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d)));\n HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d)));\n HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d)));\n\n // Apply scales and accumulate\n HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd);\n HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd);\n HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd);\n HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd);\n\n r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum));\n r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum));\n r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum));\n r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum));\n }\n\n // Process leftovers\n if (nloe) {\n HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_partial(y0_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_partial(y1_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_partial(r0_x_q + i * x_qblk_size, nloe);\n HVX_Vector_x8 r1_q = hvx_vec_load_q8x4x8_partial(r1_x_q + i * x_qblk_size, nloe);\n\n HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy0_q, nloe));\n HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy1_q, nloe));\n HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy0_q, nloe));\n HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy1_q, nloe));\n\n HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size));\n HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size));\n HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size));\n HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size));\n\n HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d)));\n HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d)));\n HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d)));\n HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d)));\n\n // Zero out unused elements\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8);\n r0_c0_dd = Q6_V_vand_QV(bmask, r0_c0_dd);\n r0_c1_dd = Q6_V_vand_QV(bmask, r0_c1_dd);\n r1_c0_dd = Q6_V_vand_QV(bmask, r1_c0_dd);\n r1_c1_dd = Q6_V_vand_QV(bmask, r1_c1_dd);\n r0_c0_ia = Q6_V_vand_QV(bmask, r0_c0_ia);\n r0_c1_ia = Q6_V_vand_QV(bmask, r0_c1_ia);\n r1_c0_ia = Q6_V_vand_QV(bmask, r1_c0_ia);\n r1_c1_ia = Q6_V_vand_QV(bmask, r1_c1_ia);\n\n HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd);\n HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd);\n HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd);\n HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd);\n\n r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum));\n r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum));\n r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum));\n r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum));\n }\n\n // Reduce and store results\n HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum);\n HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum);\n\n hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0\n hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1\n}\n\nstatic void vec_dot_mxfp4x4x2_q8x4x2_1x1(const int n, float * restrict s0, const void * restrict vx0, const void * restrict vy0) {\n assert(n % 32 == 0); // min sub-block size\n assert((unsigned long) vx0 % 128 == 0);\n assert((unsigned long) vy0 % 128 == 0);\n\n const uint32_t qk = QK_MXFP4x4x2 * 4;\n\n const uint32_t x_dblk_size = 8 * 4 * 1; // 32x e8m0\n const uint32_t x_qblk_size = qk / 2; // fp4\n const uint32_t x_qrow_size = n / 2; // fp4 (not padded)\n\n const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t y_qblk_size = qk; // int8\n const uint32_t y_qrow_size = n; // int8 (not padded)\n\n const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0 + 0); // quants first\n const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0 + x_qrow_size); // then scales\n\n const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first\n const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales\n\n // Row sum (sf)\n HVX_Vector r0_sum = Q6_V_vzero();\n\n // Multiply and accumulate into int32.\n // Compute combined scale (fp32).\n // Apply scale to acc and accumulate into the row sum (qf32).\n\n const uint32_t nb = n / qk; // num full blocks\n int32_t nloe = n % qk; // num leftover elemements (must be signed)\n\n uint32_t i = 0;\n for (; i < nb; i++) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full( y_q + i * y_qblk_size);\n HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_full(r0_x_q + i * x_qblk_size);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q));\n\n HVX_Vector vy_d = *(const HVX_UVector *) (y_d + i * y_dblk_size);\n HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size);\n\n // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving\n HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16\n vy_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy_d), half));\n vy_d = Q6_Vsf_equals_Vqf32(vy_d);\n\n // Convert rX_d scales from e8m0 to fp32\n // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ...\n // Left shift with zero fill to create FP32\n // FIXME: might need to handle zero as a special case (see ggml-cpu code)\n HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0;\n HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff);\n r0_d = Q6_V_vdelta_VV(r0_d, expand);\n r0_d = Q6_V_vand_VV(r0_d, e8m0_mask);\n r0_d = Q6_Vw_vasl_VwR(r0_d, 23);\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy_d));\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n }\n\n // Process leftovers\n if (nloe) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial( y_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_partial(r0_x_q + i * x_qblk_size, nloe);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe));\n\n HVX_Vector vy_d = *(const HVX_UVector *) (y_d + i * y_dblk_size);\n HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size);\n\n // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving\n HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16\n vy_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy_d), half));\n vy_d = Q6_Vsf_equals_Vqf32(vy_d);\n\n // Convert rX_d scales from e8m0 to fp32\n // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ...\n // Left shift with zero fill to create FP32\n // FIXME: might need to handle zero as a special case (see ggml-cpu code)\n HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0;\n HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff);\n r0_d = Q6_V_vdelta_VV(r0_d, expand);\n r0_d = Q6_V_vand_VV(r0_d, e8m0_mask);\n r0_d = Q6_Vw_vasl_VwR(r0_d, 23);\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy_d));\n\n // Zero-out unused scales\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8);\n r0_dd = Q6_V_vand_QV(bmask, r0_dd);\n r0_ia = Q6_V_vand_QV(bmask, r0_ia);\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n }\n\n r0_sum = hvx_vec_reduce_sum_f32(r0_sum);\n\n hvx_vec_store_u(s0, 4, r0_sum);\n}\n\nstatic void vec_dot_mxfp4x4x2_q8x4x2_2x1(const int n, float * restrict s0,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0) {\n assert(n % 32 == 0); // min sub-block size\n assert((unsigned long) vx0 % 128 == 0);\n assert((unsigned long) vx1 % 128 == 0);\n assert((unsigned long) vy0 % 128 == 0);\n\n const uint32_t qk = QK_MXFP4x4x2 * 4;\n\n const uint32_t x_dblk_size = 8 * 4 * 1; // 32x e8m0\n const uint32_t x_qblk_size = qk / 2; // fp4\n const uint32_t x_qrow_size = n / 2; // fp4 (not padded)\n\n const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t y_qblk_size = qk; // int8\n const uint32_t y_qrow_size = n; // int8 (not padded)\n\n const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first\n const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales\n const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first\n const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales\n\n const uint8_t * restrict y_q = ((const uint8_t *) vy0) + 0; // quants first\n const uint8_t * restrict y_d = ((const uint8_t *) vy0) + y_qrow_size; // then scales\n\n // Row sum (sf)\n HVX_Vector r0_sum = Q6_V_vzero();\n HVX_Vector r1_sum = Q6_V_vzero();\n\n // Multiply and accumulate into int32.\n // Compute combined scale (fp32).\n // Apply scale to acc and accumulate into the row sum (f32).\n\n const uint32_t nb = n / qk; // num full blocks\n int32_t nloe = n % qk; // num leftover elemements (must be signed)\n\n uint32_t i = 0;\n for (; i < nb; i++) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full( y_q + i * y_qblk_size);\n HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_full(r0_x_q + i * x_qblk_size);\n HVX_Vector_x8 r1_q = hvx_vec_load_mxfp4x4x8_full(r1_x_q + i * x_qblk_size);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q));\n HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy_q));\n\n HVX_Vector vy_d = *(const HVX_UVector *) (y_d + i * y_dblk_size);\n HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size);\n HVX_Vector r1_d = *(const HVX_UVector *) (r1_x_d + i * x_dblk_size);\n\n // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving\n HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16\n vy_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy_d), half));\n vy_d = Q6_Vsf_equals_Vqf32(vy_d);\n\n // Convert rX_d scales from e8m0 to fp32\n // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ...\n // Left shift with zero fill to create FP32\n // FIXME: might need to handle zero as a special case (see ggml-cpu code)\n HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0;\n HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff);\n r0_d = Q6_V_vdelta_VV(r0_d, expand);\n r0_d = Q6_V_vand_VV(r0_d, e8m0_mask);\n r0_d = Q6_Vw_vasl_VwR(r0_d, 23);\n r1_d = Q6_V_vdelta_VV(r1_d, expand);\n r1_d = Q6_V_vand_VV(r1_d, e8m0_mask);\n r1_d = Q6_Vw_vasl_VwR(r1_d, 23);\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy_d));\n HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy_d));\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum));\n }\n\n // Process leftovers\n if (nloe) {\n HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial( y_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_partial(r0_x_q + i * x_qblk_size, nloe);\n HVX_Vector_x8 r1_q = hvx_vec_load_mxfp4x4x8_partial(r1_x_q + i * x_qblk_size, nloe);\n\n HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q));\n HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy_q));\n\n HVX_Vector vy_d = *(const HVX_UVector *) (y_d + i * y_dblk_size);\n HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size);\n HVX_Vector r1_d = *(const HVX_UVector *) (r1_x_d + i * x_dblk_size);\n\n // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving\n HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16\n vy_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy_d), half));\n vy_d = Q6_Vsf_equals_Vqf32(vy_d);\n\n // Convert rX_d scales from e8m0 to fp32\n // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ...\n // Left shift with zero fill to create FP32\n // FIXME: might need to handle zero as a special case (see ggml-cpu code)\n HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0;\n HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff);\n r0_d = Q6_V_vdelta_VV(r0_d, expand);\n r0_d = Q6_V_vand_VV(r0_d, e8m0_mask);\n r0_d = Q6_Vw_vasl_VwR(r0_d, 23);\n r1_d = Q6_V_vdelta_VV(r1_d, expand);\n r1_d = Q6_V_vand_VV(r1_d, e8m0_mask);\n r1_d = Q6_Vw_vasl_VwR(r1_d, 23);\n\n HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy_d));\n HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy_d));\n\n // Zero-out unused values\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8);\n r0_dd = Q6_V_vand_QV(bmask, r0_dd);\n r1_dd = Q6_V_vand_QV(bmask, r1_dd);\n r0_ia = Q6_V_vand_QV(bmask, r0_ia);\n r1_ia = Q6_V_vand_QV(bmask, r1_ia);\n\n HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd);\n HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd);\n\n r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum));\n r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum));\n }\n\n HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(r0_sum, r1_sum);\n hvx_vec_store_u(s0, 8, rsum);\n}\n\nstatic void vec_dot_mxfp4x4x2_q8x4x2_2x2(const int n, float * restrict s0, float * restrict s1,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0, const void * restrict vy1) {\n assert(n % 32 == 0);\n assert((unsigned long) vx0 % 128 == 0);\n assert((unsigned long) vx1 % 128 == 0);\n assert((unsigned long) vy0 % 128 == 0);\n assert((unsigned long) vy1 % 128 == 0);\n\n const uint32_t qk = QK_MXFP4x4x2 * 4;\n\n const uint32_t x_dblk_size = 8 * 4 * 1; // 32x e8m0\n const uint32_t x_qblk_size = qk / 2; // fp4\n const uint32_t x_qrow_size = n / 2; // fp4 (not padded)\n\n const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16\n const uint32_t y_qblk_size = qk; // int8\n const uint32_t y_qrow_size = n; // int8 (not padded)\n\n const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first\n const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales\n const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first\n const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales\n\n const uint8_t * restrict y0_q = ((const uint8_t *) vy0) + 0; // quants first\n const uint8_t * restrict y0_d = ((const uint8_t *) vy0) + y_qrow_size; // then scales\n const uint8_t * restrict y1_q = ((const uint8_t *) vy1) + 0; // quants first\n const uint8_t * restrict y1_d = ((const uint8_t *) vy1) + y_qrow_size; // then scales\n\n // Row sums (sf) - 4 accumulators for 2×2 tile\n HVX_Vector r0_c0_sum = Q6_V_vzero();\n HVX_Vector r0_c1_sum = Q6_V_vzero();\n HVX_Vector r1_c0_sum = Q6_V_vzero();\n HVX_Vector r1_c1_sum = Q6_V_vzero();\n\n const uint32_t nb = n / qk; // num full blocks\n const uint32_t nloe = n % qk; // num leftover elements\n\n uint32_t i = 0;\n for (; i < nb; i++) {\n // Load src1 columns (reused across both src0 rows)\n HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_full(y0_q + i * y_qblk_size);\n HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_full(y1_q + i * y_qblk_size);\n\n // Load src0 rows (reused across both src1 columns)\n HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_full(r0_x_q + i * x_qblk_size);\n HVX_Vector_x8 r1_q = hvx_vec_load_mxfp4x4x8_full(r1_x_q + i * x_qblk_size);\n\n // Compute 4 dot products: r0×c0, r0×c1, r1×c0, r1×c1\n HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy0_q));\n HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy1_q));\n HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy0_q));\n HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy1_q));\n\n // Load scales\n HVX_Vector vy0_d = *(const HVX_UVector *) (y0_d + i * y_dblk_size);\n HVX_Vector vy1_d = *(const HVX_UVector *) (y1_d + i * y_dblk_size);\n HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size);\n HVX_Vector r1_d = *(const HVX_UVector *) (r1_x_d + i * x_dblk_size);\n\n // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving\n HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16\n vy0_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy0_d), half));\n vy0_d = Q6_Vsf_equals_Vqf32(vy0_d);\n vy1_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy1_d), half));\n vy1_d = Q6_Vsf_equals_Vqf32(vy1_d);\n\n // Convert rX_d scales from e8m0 to fp32\n // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ...\n // Left shift with zero fill to create FP32\n // FIXME: might need to handle zero as a special case (see ggml-cpu code)\n HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0;\n HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff);\n r0_d = Q6_V_vdelta_VV(r0_d, expand);\n r0_d = Q6_V_vand_VV(r0_d, e8m0_mask);\n r0_d = Q6_Vw_vasl_VwR(r0_d, 23);\n r1_d = Q6_V_vdelta_VV(r1_d, expand);\n r1_d = Q6_V_vand_VV(r1_d, e8m0_mask);\n r1_d = Q6_Vw_vasl_VwR(r1_d, 23);\n\n // Compute combined scales\n HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy0_d));\n HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy1_d));\n HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy0_d));\n HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy1_d));\n\n // Apply scales and accumulate\n HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd);\n HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd);\n HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd);\n HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd);\n\n r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum));\n r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum));\n r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum));\n r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum));\n }\n\n // Process leftovers\n if (nloe) {\n HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_partial( y0_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_partial( y1_q + i * y_qblk_size, nloe);\n HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_partial(r0_x_q + i * x_qblk_size, nloe);\n HVX_Vector_x8 r1_q = hvx_vec_load_mxfp4x4x8_partial(r1_x_q + i * x_qblk_size, nloe);\n\n HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy0_q, nloe));\n HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy1_q, nloe));\n HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy0_q, nloe));\n HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy1_q, nloe));\n\n HVX_Vector vy0_d = *(const HVX_UVector *) (y0_d + i * y_dblk_size);\n HVX_Vector vy1_d = *(const HVX_UVector *) (y1_d + i * y_dblk_size);\n HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size);\n HVX_Vector r1_d = *(const HVX_UVector *) (r1_x_d + i * x_dblk_size);\n\n // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving\n HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16\n vy0_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy0_d), half));\n vy0_d = Q6_Vsf_equals_Vqf32(vy0_d);\n vy1_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy1_d), half));\n vy1_d = Q6_Vsf_equals_Vqf32(vy1_d);\n\n // Convert rX_d scales from e8m0 to fp32\n // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ...\n // Left shift with zero fill to create FP32\n // FIXME: might need to handle zero as a special case (see ggml-cpu code)\n HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0;\n HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff);\n r0_d = Q6_V_vdelta_VV(r0_d, expand);\n r0_d = Q6_V_vand_VV(r0_d, e8m0_mask);\n r0_d = Q6_Vw_vasl_VwR(r0_d, 23);\n r1_d = Q6_V_vdelta_VV(r1_d, expand);\n r1_d = Q6_V_vand_VV(r1_d, e8m0_mask);\n r1_d = Q6_Vw_vasl_VwR(r1_d, 23);\n\n HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy0_d));\n HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy1_d));\n HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy0_d));\n HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy1_d));\n\n // Zero out unused scales\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8);\n r0_c0_dd = Q6_V_vand_QV(bmask, r0_c0_dd);\n r0_c1_dd = Q6_V_vand_QV(bmask, r0_c1_dd);\n r1_c0_dd = Q6_V_vand_QV(bmask, r1_c0_dd);\n r1_c1_dd = Q6_V_vand_QV(bmask, r1_c1_dd);\n r0_c0_ia = Q6_V_vand_QV(bmask, r0_c0_ia);\n r0_c1_ia = Q6_V_vand_QV(bmask, r0_c1_ia);\n r1_c0_ia = Q6_V_vand_QV(bmask, r1_c0_ia);\n r1_c1_ia = Q6_V_vand_QV(bmask, r1_c1_ia);\n\n HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd);\n HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd);\n HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd);\n HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd);\n\n r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum));\n r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum));\n r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum));\n r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum));\n }\n\n // Reduce and store results\n HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum);\n HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum);\n\n hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0\n hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1\n}\n\nstatic void vec_dot_f16_f16_aa_1x1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {\n const HVX_Vector * restrict x = (const HVX_Vector *) vx;\n const HVX_Vector * restrict y = (const HVX_Vector *) vy;\n\n uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors\n uint32_t nloe = n % VLEN_FP16; // leftover elements\n\n HVX_VectorPair rsum_p = Q6_W_vzero();\n\n uint32_t i = 0;\n\n #pragma unroll(4)\n for (i = 0; i < nvec; i++) {\n rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x[i], y[i]);\n }\n\n if (nloe) {\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);\n HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]);\n HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]);\n rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf);\n }\n\n HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p)));\n hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum));\n}\n\nstatic void vec_dot_f16_f16_aa_2x1(const int n, float * restrict s0,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0) {\n const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;\n const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;\n const HVX_Vector * restrict y = (const HVX_Vector *) vy0;\n\n uint32_t nvec = n / VLEN_FP16;\n uint32_t nloe = n % VLEN_FP16;\n\n HVX_VectorPair rsum0_p = Q6_W_vzero();\n HVX_VectorPair rsum1_p = Q6_W_vzero();\n\n uint32_t i = 0;\n\n #pragma unroll(2)\n for (i = 0; i < nvec; i++) {\n HVX_Vector y_hf = y[i];\n rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0[i], y_hf);\n rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1[i], y_hf);\n }\n\n if (nloe) {\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);\n HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]);\n HVX_Vector x0_hf = Q6_V_vand_QV(bmask, x0[i]);\n HVX_Vector x1_hf = Q6_V_vand_QV(bmask, x1[i]);\n rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf);\n rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf);\n }\n\n HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p)));\n HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p)));\n HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1);\n hvx_vec_store_u(s0, 8, rsum);\n}\n\nstatic void vec_dot_f16_f16_aa_2x2(const int n, float * restrict s0, float * restrict s1,\n const void * restrict vx0, const void * restrict vx1,\n const void * restrict vy0, const void * restrict vy1) {\n const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;\n const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;\n const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0;\n const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1;\n\n uint32_t nvec = n / VLEN_FP16;\n uint32_t nloe = n % VLEN_FP16;\n\n // Row sums (sf) - 4 accumulators for 2×2 tile\n HVX_VectorPair r0_c0_sum_p = Q6_W_vzero();\n HVX_VectorPair r0_c1_sum_p = Q6_W_vzero();\n HVX_VectorPair r1_c0_sum_p = Q6_W_vzero();\n HVX_VectorPair r1_c1_sum_p = Q6_W_vzero();\n\n uint32_t i = 0;\n\n #pragma unroll(2)\n for (i = 0; i < nvec; i++) {\n HVX_Vector r0_hf = x0[i];\n HVX_Vector r1_hf = x1[i];\n HVX_Vector c0_hf = y0[i];\n HVX_Vector c1_hf = y1[i];\n\n // Compute 4 dot products: r0×c0, r0×c1, r1×c0, r1×c1\n r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf);\n r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf);\n r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf);\n r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf);\n }\n\n if (nloe) {\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);\n\n HVX_Vector r0_hf = Q6_V_vand_QV(bmask, x0[i]);\n HVX_Vector r1_hf = Q6_V_vand_QV(bmask, x1[i]);\n HVX_Vector c0_hf = Q6_V_vand_QV(bmask, y0[i]);\n HVX_Vector c1_hf = Q6_V_vand_QV(bmask, y1[i]);\n\n r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf);\n r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf);\n r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf);\n r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf);\n }\n\n HVX_Vector r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c0_sum_p), Q6_V_hi_W(r0_c0_sum_p)));\n HVX_Vector r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c1_sum_p), Q6_V_hi_W(r0_c1_sum_p)));\n HVX_Vector r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c0_sum_p), Q6_V_hi_W(r1_c0_sum_p)));\n HVX_Vector r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c1_sum_p), Q6_V_hi_W(r1_c1_sum_p)));\n\n // Reduce and store results\n HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum);\n HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum);\n\n hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0\n hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1\n}\n\nstatic void vec_dot_f16_f16_uu_1x1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {\n const HVX_UVector * restrict x = (const HVX_UVector *) vx;\n const HVX_UVector * restrict y = (const HVX_UVector *) vy;\n\n uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors\n uint32_t nloe = n % VLEN_FP16; // leftover elements\n\n HVX_Vector rsum = Q6_V_vzero();\n\n uint32_t i = 0;\n\n #pragma unroll(4)\n for (i = 0; i < nvec; i++) {\n HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x[i], y[i]);\n rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf)));\n }\n\n if (nloe) {\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);\n HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]);\n HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]);\n\n HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf);\n rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf)));\n }\n\n rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum));\n hvx_vec_store_u(&s[0], 4, rsum);\n}\n\nstatic void vec_dot_f16_f32_uu_1x1(const int n, float * restrict s, const void * restrict x, const void * restrict y) {\n const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x;\n const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y;\n\n uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors\n uint32_t nloe = n % VLEN_FP16; // leftover elements\n\n const HVX_Vector zero = Q6_V_vzero();\n\n HVX_Vector rsum = Q6_V_vzero();\n\n uint32_t i = 0;\n\n #pragma unroll(2)\n for (i = 0; i < nvec; i++) {\n // Load y (fp32) and convert into fp16\n HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements\n HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements\n HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf)));\n\n // Load x (fp16)\n HVX_Vector x_hf = vx[i];\n\n HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf);\n\n rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf)));\n }\n\n if (nloe) {\n // Load y (fp32) and convert into fp16\n HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements\n HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements\n HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf)));\n\n // Load x (fp16)\n HVX_Vector x_hf = vx[i];\n\n // Zero-out unused elements\n // Note that we need to clear both x and y because they may contain NANs\n HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);\n x_hf = Q6_V_vand_QV(bmask, x_hf);\n y_hf = Q6_V_vand_QV(bmask, y_hf);\n\n HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf);\n\n rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf)));\n }\n\n // Convert into fp32 and reduce\n rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum));\n hvx_vec_store_u(&s[0], 4, rsum);\n}\n\n#define htp_matmul_tensors_preamble \\\n struct htp_tensor * restrict src0 = &octx->src0; \\\n struct htp_tensor * restrict src1 = &octx->src1; \\\n struct htp_tensor * restrict src2 = &octx->src2; \\\n struct htp_tensor * restrict dst = &octx->dst; \\\n struct htp_spad * restrict src0_spad = &octx->src0_spad; \\\n struct htp_spad * restrict src1_spad = &octx->src1_spad; \\\n struct htp_spad * restrict dst_spad = &octx->dst_spad; \\\n \\\n const uint32_t ne00 = src0->ne[0]; \\\n const uint32_t ne01 = src0->ne[1]; \\\n const uint32_t ne02 = src0->ne[2]; \\\n const uint32_t ne03 = src0->ne[3]; \\\n \\\n const uint32_t ne10 = src1->ne[0]; \\\n const uint32_t ne11 = src1->ne[1]; \\\n const uint32_t ne12 = src1->ne[2]; \\\n const uint32_t ne13 = src1->ne[3]; \\\n \\\n const uint32_t ne20 = src2->ne[0]; \\\n const uint32_t ne21 = src2->ne[1]; \\\n const uint32_t ne22 = src2->ne[2]; \\\n const uint32_t ne23 = src2->ne[3]; \\\n \\\n const uint32_t ne0 = dst->ne[0]; \\\n const uint32_t ne1 = dst->ne[1]; \\\n const uint32_t ne2 = dst->ne[2]; \\\n const uint32_t ne3 = dst->ne[3]; \\\n \\\n const uint32_t nb00 = src0->nb[0]; \\\n const uint32_t nb01 = src0->nb[1]; \\\n const uint32_t nb02 = src0->nb[2]; \\\n const uint32_t nb03 = src0->nb[3]; \\\n \\\n const uint32_t nb10 = src1->nb[0]; \\\n const uint32_t nb11 = src1->nb[1]; \\\n const uint32_t nb12 = src1->nb[2]; \\\n const uint32_t nb13 = src1->nb[3]; \\\n \\\n const uint32_t nb0 = dst->nb[0]; \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3];\n\n#define htp_matmul_preamble \\\n struct htp_matmul_context * mmctx = data; \\\n struct htp_ops_context * octx = mmctx->octx; \\\n htp_matmul_tensors_preamble; \\\n dma_queue *dma_queue = octx->ctx->dma[ith]; \\\n uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread;\n\n// *** matmul with support for 4d tensors and full broadcasting\n\nstatic void matmul_4d(unsigned int nth, unsigned int ith, void * data) {\n htp_matmul_preamble;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n assert(ne12 % ne02 == 0);\n assert(ne13 % ne03 == 0);\n\n // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers)\n const uint32_t nr0 = ne0;\n\n // This is the size of the rest of the dimensions of the result\n const uint32_t nr1 = ne1 * ne2 * ne3;\n\n // distribute the thread work across the inner or outer loop based on which one is larger\n uint32_t nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows\n uint32_t nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows\n\n // The number of elements in each chunk\n const uint32_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;\n const uint32_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;\n\n uint32_t current_chunk = ith;\n\n const uint32_t ith0 = current_chunk % nchunk0;\n const uint32_t ith1 = current_chunk / nchunk0;\n\n const uint32_t ir0_start = dr0 * ith0;\n const uint32_t ir0_end = MIN(ir0_start + dr0, nr0);\n\n const uint32_t ir1_start = dr1 * ith1;\n const uint32_t ir1_end = MIN(ir1_start + dr1, nr1);\n\n // no work for this thread\n if (ir0_start >= ir0_end || ir1_start >= ir1_end) {\n return;\n }\n\n // block-tiling attempt\n const uint32_t blck_0 = 64;\n const uint32_t blck_1 = 64;\n\n for (uint32_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {\n for (uint32_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {\n for (uint32_t ir1 = iir1; ir1 < MIN(iir1 + blck_1, ir1_end); ir1++) {\n const uint32_t i13 = fastdiv(ir1, &mmctx->mm_div_ne12_ne1);\n const uint32_t i12 = fastdiv(ir1 - i13 * ne12 * ne1, &mmctx->mm_div_ne1);\n const uint32_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1);\n\n // broadcast src0 into src1\n const uint32_t i03 = fastdiv(i13, &mmctx->mm_div_r3);\n const uint32_t i02 = fastdiv(i12, &mmctx->mm_div_r2);\n\n const uint32_t i1 = i11;\n const uint32_t i2 = i12;\n const uint32_t i3 = i13;\n\n const uint8_t * restrict src0_base = (const uint8_t *) src0->data + (0 + i02 * nb02 + i03 * nb03);\n const uint8_t * restrict src1_col = (const uint8_t *) src1->data + (i11 * nb11 + i12 * nb12 + i13 * nb13);\n float * dst_col = (float *) ((uint8_t * restrict) dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3));\n\n const uint32_t ir0_block_end = MIN(iir0 + blck_0, ir0_end);\n for (uint32_t ir0 = iir0; ir0 < ir0_block_end; ir0++) {\n const uint8_t * restrict src0_row = src0_base + ir0 * nb01;\n mmctx->vec_dot_1x1(ne00, &dst_col[ir0], src0_row, src1_col);\n }\n }\n }\n }\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"matmul-4d %d/%d: %ux%ux%ux%u (%u:%u %u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\\n\", ith, nth,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0_start, ir0_end, ir1_start, ir1_end, src1->ne[0],\n src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n// src1 tensor is already in VTCM spad\nstatic void matmul_2d(unsigned int nth, unsigned int ith, void * data) {\n htp_matmul_preamble;\n\n const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows\n const uint32_t src1_nrows = ne11 * ne12 * ne13; // src1 rows\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n const size_t dst_row_size = nb1;\n const size_t src0_row_size = nb01;\n const size_t src1_row_size = nb11;\n\n const size_t src0_stride = src0_spad->stride;\n const size_t src1_stride = src1_spad->stride;\n\n // Per-thread VTCM scratchpads for all tensors\n // Note that the entire src1 tensor is already in VTCM\n // For other tensors we allocate N rows per thread, padded to HVX vector size\n uint8_t * restrict spad_dst = dst_spad->data + dst_spad->size_per_thread * ith;\n uint8_t * restrict spad_src0 = src0_spad->data + src0_spad->size_per_thread * ith;\n uint8_t * restrict src1_data = src1_spad->data;\n\n volatile uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const uint8_t * restrict src0_row = (const uint8_t *) src0->data;\n\n // Prefill spad with src0 rows\n #pragma unroll(4)\n for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {\n const int is0 = (ir0 - src0_start_row);\n if (is0 >= MM_SPAD_SRC0_NROWS) {\n break;\n }\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + ir0 * src0_row_size),\n src0_stride, src0_row_size, 2);\n }\n\n // Process src0 rows\n for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {\n const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;\n\n // Process src1 columns in pairs (2×2 tiling)\n uint32_t ir1 = 0;\n for (; ir1 + 1 < src1_nrows; ir1 += 2) {\n const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride);\n const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride);\n float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size));\n float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size));\n mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1);\n }\n\n // Handle remaining src1 rows (fallback to 2×1)\n for (; ir1 < src1_nrows; ++ir1) {\n const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride);\n float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size));\n mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, src1_col);\n }\n\n // Prefetch next (n + spad_nrows) row\n const int pr0 = (ir0 + MM_SPAD_SRC0_NROWS);\n const int is0 = (pr0 - src0_start_row) % MM_SPAD_SRC0_NROWS;\n if (pr0 < src0_end_row_x2) {\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + pr0 * src0_row_size),\n src0_stride, src0_row_size, 2);\n }\n }\n\n // Process the last row (if any)\n if (src0_end_row != src0_end_row_x2) {\n uint32_t ir0 = src0_end_row_x2;\n const int is0 = (ir0 - src0_start_row);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + ir0 * src0_row_size),\n src0_stride, src0_row_size, 1);\n const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;\n\n #pragma unroll(2)\n for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) {\n const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride);\n float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size));\n mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col);\n }\n }\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"matmul-%s %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\\n\", mmctx->type, ith, nth,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], src1->ne[1],\n src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n// q8x4x2 src1 tensor is already in VTCM spad\nstatic void matvec_2d(unsigned int nth, unsigned int ith, void * data) {\n htp_matmul_preamble;\n\n const uint32_t src0_nrows = ne01;\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n const size_t dst_row_size = nb1;\n const size_t src0_row_size = nb01;\n const size_t src1_row_size = nb11;\n\n const size_t src0_stride = src0_spad->stride;\n const size_t src1_stride = src1_spad->stride;\n\n // Per-thread VTCM scratchpads for all tensors\n // Note that the entire src1 tensor is already in VTCM\n // For other tensors we allocate N rows per thread, padded to HVX vector size\n uint8_t * spad_dst = dst_spad->data + dst_spad->size_per_thread * ith;\n uint8_t * spad_src0 = src0_spad->data + src0_spad->size_per_thread * ith;\n uint8_t * src1_data = src1_spad->data;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n float * tmp = (float *) spad_dst;\n\n const uint8_t * restrict src0_row = (const uint8_t *) src0->data;\n const uint8_t * restrict src1_col = (const uint8_t *) src1_data;\n float * restrict dst_col = (float *) dst->data;\n\n // Prefill spad with 2x src0 rows\n #pragma unroll(2)\n for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {\n const uint32_t is0 = (ir0 - src0_start_row);\n if (is0 >= MM_SPAD_SRC0_NROWS) {\n break;\n }\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + ir0 * src0_row_size),\n src0_stride, src0_row_size, 2);\n }\n\n // Process src0 rows\n for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {\n const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;\n mmctx->vec_dot_2x1(ne00, &tmp[ir0 - src0_start_row], ss0, ss0 + src0_stride, src1_col);\n\n // Prefetch next (n + spad_nrows) row\n const uint32_t pr0 = (ir0 + MM_SPAD_SRC0_NROWS);\n const uint32_t is0 = (pr0 - src0_start_row) % MM_SPAD_SRC0_NROWS;\n if (pr0 < src0_end_row_x2) {\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + pr0 * src0_row_size),\n src0_stride, src0_row_size, 2);\n }\n }\n\n // Process the last row (if any)\n if (src0_end_row != src0_end_row_x2) {\n const uint32_t ir0 = src0_end_row_x2;\n const uint32_t is0 = (ir0 - src0_start_row);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + ir0 * src0_row_size),\n src0_stride, src0_row_size, 1);\n const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;\n mmctx->vec_dot_1x1(ne00, &tmp[ir0 - src0_start_row], ss0, src1_col);\n }\n\n hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, src0_end_row - src0_start_row);\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"matvec-%s %u/%u: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\\n\", mmctx->type, ith, nth,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], src1->ne[1],\n src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * ids->ne[0] * ids->ne[1] + (i1)]\n\nstruct mmid_row_mapping {\n uint32_t i1;\n uint32_t i2;\n};\n\n// src1 tensor is already in VTCM spad\nstatic void matmul_id(unsigned int nth, unsigned int ith, void * data) {\n htp_matmul_preamble;\n\n struct htp_tensor * restrict ids = &octx->src2;\n struct htp_spad * restrict src2_spad = &octx->src2_spad;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const uint32_t src0_nrows = ne01; // src0 rows per expert\n const uint32_t src1_nrows = ne11;\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n const uint32_t n_ids = ids->ne[0]; // n_expert_used\n const uint32_t n_as = ne02; // n_expert\n\n const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);\n const size_t matrix_row_map_size = n_as * ids->ne[0] * ids->ne[1] * sizeof(struct mmid_row_mapping);\n\n const uint32_t * matrix_row_counts = (const uint32_t *) src2_spad->data + 0;\n const struct mmid_row_mapping * matrix_rows = (const void *) src2_spad->data + matrix_row_counts_size;\n\n const size_t dst_row_size = nb1;\n const size_t src0_row_size = nb01;\n const size_t src1_row_size = q8x4x2_row_size(ne10);\n\n const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);\n\n // Per-thread VTCM scratchpads for all tensors\n // Note that the entire src1 tensor is already in VTCM\n // For other tensors we allocate N rows per thread, padded to HVX vector size\n uint8_t * restrict spad_dst = dst_spad->data + dst_spad->size_per_thread * ith;\n uint8_t * restrict spad_src0 = src0_spad->data + src0_spad->size_per_thread * ith;\n uint8_t * restrict src1_data = src1_spad->data;\n\n for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) {\n const int32_t cne1 = matrix_row_counts[cur_a];\n\n if (cne1 == 0) {\n continue;\n }\n\n const uint8_t * src0_row = (const uint8_t *) src0->data + (0 + cur_a * nb02 + 0);\n\n // Prefill spad with src0 rows\n #pragma unroll(4)\n for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {\n const int is0 = (ir0 - src0_start_row);\n if (is0 >= MM_SPAD_SRC0_NROWS) {\n break;\n }\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + ir0 * src0_row_size),\n src0_row_size_padded, src0_row_size, 2);\n }\n\n // Process src0 rows\n for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {\n const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;\n\n for (uint32_t cid = 0; cid < cne1; ++cid) {\n struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, cid);\n const int rm1 = row_mapping.i1; // expert idx\n const int rm2 = row_mapping.i2; // token idx\n\n const uint32_t ir1 = src1_nrows == 1 ? 0 : rm1; // src1 row idx\n const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (ir1 + rm2 * ne11 + 0) * src1_row_size);\n float * dst_row = (float *) (dst->data + (rm1 * nb1 + rm2 * nb2 + 0));\n\n mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_row_size_padded, src1_col);\n }\n\n // Prefetch next (n + spad_nrows) row\n const int pr0 = (ir0 + MM_SPAD_SRC0_NROWS);\n const int is0 = (pr0 - src0_start_row) % MM_SPAD_SRC0_NROWS;\n if (pr0 < src0_end_row_x2) {\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + pr0 * src0_row_size),\n src0_row_size_padded, src0_row_size, 2);\n }\n }\n\n // Process the last row (if any)\n if (src0_end_row != src0_end_row_x2) {\n uint32_t ir0 = src0_end_row_x2;\n const uint32_t is0 = (ir0 - src0_start_row);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + ir0 * src0_row_size),\n src0_row_size_padded, src0_row_size, 1);\n const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;\n\n for (uint32_t cid = 0; cid < cne1; ++cid) {\n struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, cid);\n const int rm1 = row_mapping.i1; // expert idx\n const int rm2 = row_mapping.i2; // token idx\n\n const uint32_t ir1 = src1_nrows == 1 ? 0 : rm1; // src1 row idx\n const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (ir1 + rm2 * ne11 + 0) * src1_row_size);\n float * dst_row = (float *) (dst->data + (rm1 * nb1 + rm2 * nb2 + 0));\n\n mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col);\n }\n }\n }\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"matmul-id-%s %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u (%ux%ux%ux%u) -> %ux%ux%ux%u usec %u\\n\", mmctx->type,\n ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0],\n src1->ne[1], src1->ne[2], src1->ne[3], ids->ne[0], ids->ne[1], ids->ne[2], ids->ne[3], dst->ne[0], dst->ne[1],\n dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n// src1 tensor is already in VTCM spad\nstatic void matvec_id(unsigned int nth, unsigned int ith, void * data) {\n htp_matmul_preamble;\n\n struct htp_tensor * restrict ids = &octx->src2;\n struct htp_spad * restrict src2_spad = &octx->src2_spad;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const uint32_t src0_nrows = ne01; // src0 rows per expert\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n assert(ne13 % ne03 == 0);\n\n const size_t dst_row_size = nb1;\n const size_t src0_row_size = nb01;\n const size_t src1_row_size = q8x4x2_row_size(ne10);\n\n const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);\n\n const uint32_t n_aids = src2->ne[0]; // num activated experts\n const uint32_t n_ids = ne02; // num experts\n\n // Per-thread VTCM scratchpads for all tensors\n // Note that the entire src1 tensor is already in VTCM\n // For other tensors we allocate N rows per thread, padded to HVX vector size\n uint8_t * restrict spad_dst = dst_spad->data + dst_spad->size_per_thread * ith;\n uint8_t * restrict spad_src0 = src0_spad->data + src0_spad->size_per_thread * ith;\n uint8_t * restrict src1_data = src1_spad->data;\n\n for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert\n const uint32_t eid = *(const int32_t *) ((const uint8_t *) src2->data + ie1 * src2->nb[0]);\n assert(eid < n_ids);\n\n const uint8_t * restrict src0_row = (const uint8_t *) src0->data + eid * nb02;\n const uint8_t * restrict src1_col = (const uint8_t *) src1_data;\n float * restrict dst_row = (float *) (dst->data + ie1 * nb1);\n\n // Prefill spad with src0 rows\n #pragma unroll(4)\n for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {\n const int is0 = (ir0 - src0_start_row);\n if (is0 >= MM_SPAD_SRC0_NROWS) {\n break;\n }\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + ir0 * src0_row_size),\n src0_row_size_padded, src0_row_size, 2);\n }\n\n // Process src0 rows\n for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {\n const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;\n mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_row_size_padded, src1_col);\n\n // Prefetch next (n + spad_nrows) row\n const int pr0 = (ir0 + MM_SPAD_SRC0_NROWS);\n const int is0 = (pr0 - src0_start_row) % MM_SPAD_SRC0_NROWS;\n if (pr0 < src0_end_row_x2) {\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + pr0 * src0_row_size),\n src0_row_size_padded, src0_row_size, 2);\n }\n }\n\n // Process the last row (if any)\n if (src0_end_row != src0_end_row_x2) {\n uint32_t ir0 = src0_end_row_x2;\n const uint32_t is0 = (ir0 - src0_start_row);\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + ir0 * src0_row_size),\n src0_row_size_padded, src0_row_size, 1);\n const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;\n mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col);\n }\n }\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"matvec-id-%s %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u (%ux%ux%ux%u) -> %ux%ux%ux%u usec %u\\n\", mmctx->type,\n ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0],\n src1->ne[1], src1->ne[2], src1->ne[3], src2->ne[0], src2->ne[1], src2->ne[2], src2->ne[3], dst->ne[0],\n dst->ne[1], dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n// *** dynamic quant\n\nstatic inline void quantize_block_f32_q8x1(float * restrict x, uint8_t * restrict y_q, uint8_t * restrict y_d) {\n assert((unsigned long) x % 128 == 0);\n assert((unsigned long) y_q % 128 == 0);\n\n HVX_Vector * vx = (HVX_Vector *) x;\n HVX_Vector zero = Q6_V_vzero();\n\n // Use reduce max fp32 to find max(abs(e)) first\n HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0]));\n HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1]));\n HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2]));\n HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3]));\n // Load and convert into QF32\n HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); // 32 elements\n HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); // 32 elements\n HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); // 32 elements\n HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); // 32 elements\n\n // Convert to QF32\n HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); // replicated over all lanes\n HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); // replicated over all lanes\n HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); // replicated over all lanes\n HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); // replicated over all lanes\n\n // Combine and convert to fp16\n HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));\n HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));\n\n // Convert into fp16\n HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));\n HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));\n\n HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0\n HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0\n HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16);\n HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16);\n\n hvx_vec_store_u(y_d + 0, 2, vd01_hf);\n HVX_Vector rotated_vd_hf = Q6_V_vror_VR(vd01_hf, 64);\n hvx_vec_store_u(y_d + 2, 2, rotated_vd_hf);\n\n hvx_vec_store_u(y_d + 4, 2, vd23_hf);\n rotated_vd_hf = Q6_V_vror_VR(vd23_hf, 64);\n hvx_vec_store_u(y_d + 6, 2, rotated_vd_hf);\n\n // Divide input by the scale\n HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);\n HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf);\n vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));\n vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));\n\n // Convert to int8\n HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);\n HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);\n HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16);\n\n *(HVX_Vector *) y_q = vx_i8;\n}\n\nstatic inline void quantize_block_f32_q8x2(float * restrict x, uint8_t * restrict y_q, uint8_t * restrict y_d) {\n assert((unsigned long) x % 128 == 0);\n assert((unsigned long) y_q % 128 == 0);\n\n HVX_Vector * vx = (HVX_Vector *) x;\n\n // Load and convert into QF32\n HVX_Vector zero = Q6_V_vzero();\n HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); // 32 elements\n HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); // 32 elements\n HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); // 32 elements\n HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); // 32 elements\n\n // Convert into fp16\n HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));\n HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));\n\n // Compute max and scale\n HVX_Vector vmax01_hf = hvx_vec_reduce_max_f16(hvx_vec_abs_f16(vx01_hf)); // replicated over all lanes\n HVX_Vector vmax23_hf = hvx_vec_reduce_max_f16(hvx_vec_abs_f16(vx23_hf)); // replicated over all lanes\n\n HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0\n HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0\n HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16);\n HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16);\n\n hvx_vec_store_u(y_d + 0, 4, vd01_hf);\n hvx_vec_store_u(y_d + 4, 4, vd23_hf);\n\n // Divide input by the scale\n HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);\n HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf);\n vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));\n vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));\n\n // Convert to int8\n HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);\n HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);\n HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16);\n\n *(HVX_Vector *) y_q = vx_i8;\n}\n\nstatic inline void quantize_block_f32_q8x4(float * restrict x, uint8_t * restrict y_q, uint8_t * restrict y_d) {\n assert((unsigned long) x % 128 == 0);\n assert((unsigned long) y_q % 128 == 0);\n\n HVX_Vector * vx = (HVX_Vector *) x;\n\n // Load and convert into QF32\n HVX_Vector zero = Q6_V_vzero();\n HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); // 32 elements\n HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); // 32 elements\n HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); // 32 elements\n HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); // 32 elements\n\n // Convert into fp16\n HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));\n HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));\n\n // Compute max and scale\n HVX_Vector vmax_hf = hvx_vec_reduce_max_f16(hvx_vec_abs_f16(vx01_hf));\n vmax_hf = hvx_vec_reduce_max2_f16(hvx_vec_abs_f16(vx23_hf), vmax_hf); // replicated over all lanes\n\n HVX_Vector vd_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0\n HVX_Vector vd_hf = Q6_Vhf_equals_Vqf16(vd_qf16);\n\n *(HVX_UVector *) y_d = vd_hf;\n\n // Divide input by the scale\n HVX_Vector vd_inv_hf = hvx_vec_inverse_f16(vd_hf);\n vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd_inv_hf));\n vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd_inv_hf));\n\n // Convert to int8\n HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);\n HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);\n HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16);\n\n *(HVX_Vector *) y_q = vx_i8;\n}\n\n// Overrides input x\nstatic void quantize_row_f32_q8x4x2(float * restrict x, uint8_t * restrict y, uint32_t k) {\n assert(k % 32 == 0);\n const uint32_t qk = QK_Q8_0x4x2;\n const uint32_t nb = (k + qk - 1) / qk;\n\n const uint32_t qrow_size = k; // int8\n\n const uint32_t dblk_size = 8 * 2; // 8x __fp16\n const uint32_t qblk_size = QK_Q8_0x4x2; // int8\n\n uint8_t * restrict y_q = (y + 0); // quants first\n uint8_t * restrict y_d = (y + qrow_size); // then scales\n\n // Temp scales override input since we're working off of the aligned temp buffer in VTCM\n uint8_t * restrict t_d = (uint8_t *) x;\n\n for (uint32_t i = 0; i < nb; i++) {\n#if FP32_QUANTIZE_GROUP_SIZE == 32\n quantize_block_f32_q8x1(x + (i*2 + 0) * qk/2, y_q + (i*2 + 0) * qblk_size/2, t_d + (i*2 + 0) * dblk_size/2);\n quantize_block_f32_q8x1(x + (i*2 + 1) * qk/2, y_q + (i*2 + 1) * qblk_size/2, t_d + (i*2 + 1) * dblk_size/2);\n#elif FP32_QUANTIZE_GROUP_SIZE == 64\n quantize_block_f32_q8x2(x + (i*2 + 0) * qk/2, y_q + (i*2 + 0) * qblk_size/2, t_d + (i*2 + 0) * dblk_size/2);\n quantize_block_f32_q8x2(x + (i*2 + 1) * qk/2, y_q + (i*2 + 1) * qblk_size/2, t_d + (i*2 + 1) * dblk_size/2);\n#elif FP32_QUANTIZE_GROUP_SIZE == 128\n quantize_block_f32_q8x4(x + (i*2 + 0) * qk/2, y_q + (i*2 + 0) * qblk_size/2, t_d + (i*2 + 0) * dblk_size/2);\n quantize_block_f32_q8x4(x + (i*2 + 1) * qk/2, y_q + (i*2 + 1) * qblk_size/2, t_d + (i*2 + 1) * dblk_size/2);\n#else\n#error \"FP32_QUANTIZE_GROUP_SIZE must be 32, 64, or 128\"\n#endif\n }\n\n // now copy the scales into final location\n hvx_copy_f16_ua(y_d, t_d, nb * 8);\n}\n\nstatic void quantize_f32_q8x4x2(unsigned int nth, unsigned int ith, void * data) {\n struct htp_matmul_context * mmctx = data;\n struct htp_ops_context * octx = mmctx->octx;\n\n const struct htp_tensor * src = &octx->src1;\n uint8_t * restrict dst = octx->src1_spad.data;\n struct htp_spad * spad = &octx->src0_spad;\n uint32_t nrows_per_thread = mmctx->src1_nrows_per_thread;\n\n uint64_t t1 = HAP_perf_get_qtimer_count();\n\n const uint32_t ne0 = src->ne[0];\n const uint32_t ne1 = src->ne[1];\n const uint32_t ne2 = src->ne[2];\n const uint32_t ne3 = src->ne[3];\n\n const uint32_t nrows = ne1 * ne2 * ne3; // total n_rows\n\n const uint32_t ir_first = nrows_per_thread * ith; // first row\n const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); // last row\n\n const size_t src_row_size = src->nb[1];\n const size_t dst_row_size = q8x4x2_row_size(ne0);\n\n uint8_t * restrict src_data = (uint8_t *) src->data + (src_row_size * ir_first);\n uint8_t * restrict dst_data = (uint8_t *) dst + (dst_row_size * ir_first);\n uint8_t * restrict tmp_data = (uint8_t *) spad->data + (spad->size_per_thread * ith);\n\n const size_t src_row_size_padded = hex_round_up(src_row_size, QK_Q8_0x4x2 * sizeof(float));\n memset(tmp_data, 0, src_row_size_padded); // zero-out temp row data for padding\n\n for (uint32_t i = ir_first; i < ir_last; ++i) {\n hex_l2fetch(src_data, src_row_size, src_row_size, 2);\n hvx_copy_f32_aa(tmp_data, src_data, ne0);\n\n // FARF(HIGH, \"quantize-q8x4-row: %u\\n\", i);\n quantize_row_f32_q8x4x2((float *) tmp_data, dst_data, ne0);\n dst_data += dst_row_size;\n src_data += src_row_size;\n }\n\n uint64_t t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"quantize-f32-q8x4: %u/%u : n-rows %u (%u:%u) row-size %u -> %u usec %u\\n\", ith, nth, nrows, ir_first,\n ir_last, src_row_size, dst_row_size, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\nstatic void quantize_f32_f16(unsigned int nth, unsigned int ith, void * data) {\n struct htp_matmul_context * mmctx = data;\n struct htp_ops_context * octx = mmctx->octx;\n\n const struct htp_tensor * src = &octx->src1;\n uint8_t * restrict dst = octx->src1_spad.data;\n uint32_t nrows_per_thread = mmctx->src1_nrows_per_thread;\n uint32_t dst_stride = octx->src1_spad.stride;\n\n uint64_t t1 = HAP_perf_get_qtimer_count();\n\n const uint32_t ne0 = src->ne[0];\n const uint32_t ne1 = src->ne[1];\n const uint32_t ne2 = src->ne[2];\n const uint32_t ne3 = src->ne[3];\n\n const uint32_t nrows = ne1 * ne2 * ne3; // total n_rows\n\n const uint32_t ir_first = nrows_per_thread * ith; // first row\n const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); // last row\n\n const size_t src_row_size = ne0 * sizeof(float);\n const size_t src_stride = src->nb[1];\n\n uint8_t * restrict src_data = (uint8_t *) src->data + (src_stride * ir_first);\n uint8_t * restrict dst_data = (uint8_t *) dst + (dst_stride * ir_first);\n\n for (uint32_t i = ir_first; i < ir_last; ++i) {\n hex_l2fetch(src_data, src_row_size, src_stride, 2);\n hvx_copy_f16_f32_au(dst_data, src_data, ne0);\n\n dst_data += dst_stride;\n src_data += src_stride;\n }\n\n uint64_t t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"quantize-f32-f16: %u/%u : n-rows %u (%u:%u) row-size %u (%u) -> %u usec %u\\n\", ith, nth, nrows, ir_first,\n ir_last, src_row_size, src_stride, dst_stride, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n// TODO just a plain copy that should be done via the DMA during the Op setup\nstatic void quantize_f16_f16(unsigned int nth, unsigned int ith, void * data) {\n struct htp_matmul_context * mmctx = data;\n struct htp_ops_context * octx = mmctx->octx;\n\n const struct htp_tensor * src = &octx->src1;\n uint8_t * restrict dst = octx->src1_spad.data;\n uint32_t nrows_per_thread = mmctx->src1_nrows_per_thread;\n uint32_t dst_stride = octx->src1_spad.stride;\n\n uint64_t t1 = HAP_perf_get_qtimer_count();\n\n const uint32_t ne0 = src->ne[0];\n const uint32_t ne1 = src->ne[1];\n const uint32_t ne2 = src->ne[2];\n const uint32_t ne3 = src->ne[3];\n\n const uint32_t nrows = ne1 * ne2 * ne3; // total n_rows\n\n const uint32_t ir_first = nrows_per_thread * ith; // first row\n const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); // last row\n\n const size_t src_row_size = ne0 * sizeof(float);\n const size_t src_stride = src->nb[1];\n\n uint8_t * restrict src_data = (uint8_t *) src->data + (src_stride * ir_first);\n uint8_t * restrict dst_data = (uint8_t *) dst + (dst_stride * ir_first);\n\n for (uint32_t i = ir_first; i < ir_last; ++i) {\n hex_l2fetch(src_data, src_row_size, src_stride, 2);\n hvx_copy_f16_au(dst_data, src_data, ne0);\n\n dst_data += dst_stride;\n src_data += src_stride;\n }\n\n uint64_t t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"quantize-f16-f16: %u/%u : n-rows %u (%u:%u) row-size %u (%u) -> %u usec %u\\n\", ith, nth, nrows, ir_first,\n ir_last, src_row_size, src_stride, dst_stride, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n\nstatic inline bool htp_is_permuted(const struct htp_tensor * t) {\n return t->nb[0] > t->nb[1] || t->nb[1] > t->nb[2] || t->nb[2] > t->nb[3];\n}\n\nstatic int htp_mminit_vec_dot(struct htp_matmul_context * mmctx, enum htp_data_type type) {\n switch (type) {\n case HTP_TYPE_Q4_0:\n mmctx->type = \"q4x4x2-f32\";\n mmctx->vec_dot_1x1 = vec_dot_q4x4x2_q8x4x2_1x1;\n mmctx->vec_dot_2x1 = vec_dot_q4x4x2_q8x4x2_2x1;\n mmctx->vec_dot_2x2 = vec_dot_q4x4x2_q8x4x2_2x2;\n return 0;\n case HTP_TYPE_Q8_0:\n mmctx->type = \"q8x4x2-f32\";\n mmctx->vec_dot_1x1 = vec_dot_q8x4x2_q8x4x2_1x1;\n mmctx->vec_dot_2x1 = vec_dot_q8x4x2_q8x4x2_2x1;\n mmctx->vec_dot_2x2 = vec_dot_q8x4x2_q8x4x2_2x2;\n return 0;\n case HTP_TYPE_MXFP4:\n mmctx->type = \"mxfp4x4x2-f32\";\n mmctx->vec_dot_1x1 = vec_dot_mxfp4x4x2_q8x4x2_1x1;\n mmctx->vec_dot_2x1 = vec_dot_mxfp4x4x2_q8x4x2_2x1;\n mmctx->vec_dot_2x2 = vec_dot_mxfp4x4x2_q8x4x2_2x2;\n return 0;\n default:\n return -1;\n }\n}\n\nstatic void htp_mminit_spad(struct htp_ops_context * octx,\n size_t dst_row_size,\n size_t src0_row_size_padded,\n size_t src1_row_size,\n uint32_t src1_nrows,\n size_t src2_spad_size_per_thread) {\n octx->dst_spad.size_per_thread = hex_round_up(MM_SPAD_DST_NROWS * dst_row_size, 256);\n octx->src0_spad.size_per_thread = hex_round_up(MM_SPAD_SRC0_NROWS * src0_row_size_padded, 256);\n octx->src1_spad.size_per_thread = hex_round_up(src1_row_size * src1_nrows, 256);\n\n if (src2_spad_size_per_thread > 0) {\n octx->src2_spad.size_per_thread = src2_spad_size_per_thread;\n octx->src2_spad.size = octx->src2_spad.size_per_thread;\n }\n\n // src0 spad is also used in dynamic quantizer to store padded src1 rows\n size_t src1_row_size_padded = hex_round_up(src1_row_size, QK_Q8_0x4x2 * sizeof(float));\n if (octx->src0_spad.size_per_thread < src1_row_size_padded) {\n octx->src0_spad.size_per_thread = src1_row_size_padded;\n }\n\n octx->src1_spad.size = octx->src1_spad.size_per_thread;\n octx->src0_spad.size = octx->src0_spad.size_per_thread * octx->n_threads;\n octx->dst_spad.size = octx->dst_spad.size_per_thread * octx->n_threads;\n}\n\nint op_matmul(struct htp_ops_context * octx) {\n htp_matmul_tensors_preamble;\n\n struct htp_matmul_context mmctx_struct = {0};\n struct htp_matmul_context * mmctx = &mmctx_struct;\n mmctx->octx = octx;\n\n const uint32_t src0_nrows = ne01 * ne02 * ne03;\n const uint32_t src1_nrows = ne11 * ne12 * ne13;\n\n // Compute src0_nrows_per_thread\n mmctx->src0_nrows_per_thread = (src0_nrows + octx->n_threads - 1) / octx->n_threads;\n mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even\n\n const size_t src0_row_size = nb01;\n const size_t dst_row_size = nb1;\n size_t src1_row_size = nb11;\n\n const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);\n size_t src1_row_size_padded;\n\n worker_callback_t quant_job_func;\n worker_callback_t matmul_job_func = src1_nrows > 1 ? matmul_2d : matvec_2d;\n\n bool need_quant = !(octx->flags & HTP_OPFLAGS_SKIP_QUANTIZE);\n\n if (src0->type == HTP_TYPE_F16) {\n // Try optimized f16-f16 path first (src1 in VTCM)\n const size_t f16_src1_row_size = hex_round_up(ne10 * 2, 128);\n const size_t f16_src1_spad_size = hex_round_up(f16_src1_row_size * src1_nrows, 256);\n const size_t f16_src0_spad_size = hex_round_up(MM_SPAD_SRC0_NROWS * src0_row_size_padded, 256) * octx->n_threads;\n const size_t f16_dst_spad_size = hex_round_up(MM_SPAD_DST_NROWS * dst_row_size, 256) * octx->n_threads;\n\n const size_t f16_total_size = f16_src1_spad_size + f16_src0_spad_size + f16_dst_spad_size;\n\n // Default matmul implementation does not support multi-batch src0 (N-vs-N broadcasting).\n // It only supports 1-vs-N broadcasting (src0 is 2D) or standard 2D matmul.\n const bool is_batched = (ne02 > 1) || (ne03 > 1);\n const bool is_permuted = htp_is_permuted(&octx->src0) || htp_is_permuted(&octx->src1);\n\n if (!is_batched && !is_permuted && f16_total_size <= octx->ctx->vtcm_size) {\n // Optimized path\n quant_job_func = (src1->type == HTP_TYPE_F32) ? quantize_f32_f16 : quantize_f16_f16;\n mmctx->type = \"f16-f16\";\n mmctx->vec_dot_1x1 = vec_dot_f16_f16_aa_1x1;\n mmctx->vec_dot_2x1 = vec_dot_f16_f16_aa_2x1;\n mmctx->vec_dot_2x2 = vec_dot_f16_f16_aa_2x2;\n\n src1_row_size = f16_src1_row_size; // row size post quantization\n\n octx->dst_spad.size_per_thread = hex_round_up(MM_SPAD_DST_NROWS * dst_row_size, 256);\n octx->src0_spad.size_per_thread = hex_round_up(MM_SPAD_SRC0_NROWS * src0_row_size_padded, 256);\n octx->src1_spad.size_per_thread = hex_round_up(src1_row_size * src1_nrows, 256);\n\n octx->src1_spad.size = octx->src1_spad.size_per_thread;\n octx->src0_spad.size = octx->src0_spad.size_per_thread * octx->n_threads;\n octx->dst_spad.size = octx->dst_spad.size_per_thread * octx->n_threads;\n } else {\n // Fallback to f16/f32 (DDR) if src1 doesn't fit in VTCM or broadcasting is required\n quant_job_func = NULL;\n if (src1->type == HTP_TYPE_F32) {\n mmctx->type = \"f16-f32\";\n mmctx->vec_dot_1x1 = vec_dot_f16_f32_uu_1x1;\n matmul_job_func = matmul_4d;\n } else {\n mmctx->type = \"f16-f16\";\n mmctx->vec_dot_1x1 = vec_dot_f16_f16_uu_1x1;\n matmul_job_func = matmul_4d;\n }\n\n src1_row_size = nb11; // original row size in DDR\n\n octx->dst_spad.size_per_thread = hex_round_up(MM_SPAD_DST_NROWS * dst_row_size, 256);\n octx->src0_spad.size_per_thread = hex_round_up(MM_SPAD_SRC0_NROWS * src0_row_size, 256);\n octx->src1_spad.size_per_thread = hex_round_up(MM_SPAD_SRC1_NROWS * src1_row_size, 256);\n\n octx->src0_spad.size = octx->src0_spad.size_per_thread * octx->n_threads;\n octx->src1_spad.size = octx->src1_spad.size_per_thread * octx->n_threads;\n octx->dst_spad.size = octx->dst_spad.size_per_thread * octx->n_threads;\n\n // Init fastdiv for matmul_4d (supports broadcasting)\n mmctx->mm_div_ne12_ne1 = init_fastdiv_values(src1->ne[2] * dst->ne[1]);\n mmctx->mm_div_ne1 = init_fastdiv_values(dst->ne[1]);\n mmctx->mm_div_r2 = init_fastdiv_values(src1->ne[2] / src0->ne[2]);\n mmctx->mm_div_r3 = init_fastdiv_values(src1->ne[3] / src0->ne[3]);\n\n need_quant = false;\n }\n } else {\n if (htp_mminit_vec_dot(mmctx, src0->type) != 0) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n quant_job_func = quantize_f32_q8x4x2;\n src1_row_size = q8x4x2_row_size(ne10);\n htp_mminit_spad(octx, dst_row_size, src0_row_size_padded, src1_row_size, src1_nrows, 0);\n }\n\n // VTCM scratchpads for all tensors\n size_t spad_size = octx->src1_spad.size + octx->src0_spad.size + octx->dst_spad.size;\n\n FARF(HIGH, \"matmul-%s : src0-spad-size %u src1-spad-size %u dst-spad-size %u (%zu)\\n\", mmctx->type,\n octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size, spad_size);\n\n FARF(HIGH, \"matmul-%s : %ux%ux%ux%u * %ux%ux%ux%u-> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\\n\", mmctx->type, src0->ne[0],\n src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0],\n dst->ne[1], dst->ne[2], dst->ne[3], src0->data, src1->data, dst->data);\n\n // Make sure the reserved vtcm size is sufficient\n if (octx->ctx->vtcm_size < spad_size) {\n FARF(ERROR, \"matmul-%s : current VTCM reservation %zu is too small, needed %zu\\n\", mmctx->type,\n octx->ctx->vtcm_size, spad_size);\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n octx->src0_spad.data = octx->ctx->vtcm_base;\n octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;\n octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size;\n\n octx->src0_spad.stride = src0_row_size_padded;\n octx->src1_spad.stride = src1_row_size;\n\n if (need_quant) {\n const uint32_t n_quant_jobs = MIN(src1_nrows, octx->n_threads);\n mmctx->src1_nrows_per_thread = (src1_nrows + n_quant_jobs - 1) / n_quant_jobs;\n worker_pool_run_func(octx->ctx->worker_pool, quant_job_func, mmctx, n_quant_jobs);\n }\n\n if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {\n const uint32_t n_matmul_jobs = octx->n_threads;\n worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs);\n }\n\n return HTP_STATUS_OK;\n}\n\nint op_matmul_id(struct htp_ops_context * octx) {\n htp_matmul_tensors_preamble;\n\n struct htp_matmul_context mmctx_struct = {0};\n struct htp_matmul_context * mmctx = &mmctx_struct;\n mmctx->octx = octx;\n\n struct htp_tensor * restrict ids = &octx->src2;\n\n const size_t src0_row_size = nb01;\n const size_t dst_row_size = nb1;\n\n const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);\n\n const uint32_t src0_nrows = ne01; // per expert\n const uint32_t src1_nrows = ne11 * ne12 * ne13;\n\n worker_callback_t quant_job_func;\n worker_callback_t matmul_id_job_func = src1_nrows > 1 ? matmul_id : matvec_id;\n\n // Compute src0_nrows_per_thread\n mmctx->src0_nrows_per_thread = (src0_nrows + octx->n_threads - 1) / octx->n_threads;\n mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even\n\n size_t src1_row_size;\n size_t src1_row_size_padded;\n\n // row groups\n const int n_ids = ids->ne[0]; // n_expert_used\n const int n_as = ne02; // n_expert\n\n size_t matrix_row_counts_size = n_as * sizeof(uint32_t);\n size_t matrix_row_map_size = n_as * ids->ne[0] * ids->ne[1] * sizeof(struct mmid_row_mapping);\n\n if (htp_mminit_vec_dot(mmctx, src0->type) != 0) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n quant_job_func = quantize_f32_q8x4x2;\n src1_row_size = q8x4x2_row_size(ne10);\n\n const size_t src2_spad_size_per_thread = hex_round_up(matrix_row_counts_size + matrix_row_map_size, 256);\n htp_mminit_spad(octx, dst_row_size, src0_row_size_padded, src1_row_size, src1_nrows, src2_spad_size_per_thread);\n\n size_t spad_size = octx->src2_spad.size + octx->src1_spad.size + octx->src0_spad.size + octx->dst_spad.size;\n\n FARF(HIGH, \"matmul-id-%s : src0-spad-size %u src1-spad-size %u src2-spad-size %u dst-spad-size %u (%zu)\\n\", mmctx->type,\n octx->src0_spad.size, octx->src1_spad.size, octx->src2_spad.size, octx->dst_spad.size, spad_size);\n\n FARF(HIGH, \"matmul-id-%s : %ux%ux%ux%u * %ux%ux%ux%u (%ux%ux%ux%u) -> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\\n\", mmctx->type,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],\n ids->ne[0], ids->ne[1], ids->ne[2], ids->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], src0->data,\n src1->data, dst->data);\n\n // Make sure the reserved vtcm size is sufficient\n if (octx->ctx->vtcm_size < spad_size) {\n FARF(ERROR, \"matmul-id-%s : current VTCM reservation %zu is too small, needed %zu\\n\", mmctx->type, octx->ctx->vtcm_size, spad_size);\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n octx->src0_spad.data = octx->ctx->vtcm_base;\n octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;\n octx->src2_spad.data = octx->src1_spad.data + octx->src1_spad.size;\n octx->dst_spad.data = octx->src2_spad.data + octx->src2_spad.size;\n\n octx->src0_spad.stride = src0_row_size_padded;\n octx->src1_spad.stride = src1_row_size;\n\n if (src1_nrows > 1) {\n // initialize matrix_row_counts and map\n uint32_t * matrix_row_counts = (uint32_t *) octx->src2_spad.data + 0;\n struct mmid_row_mapping * matrix_rows = (void *) octx->src2_spad.data + matrix_row_counts_size;\n\n memset(matrix_row_counts, 0, n_as * sizeof(uint32_t));\n\n // group rows by src0 matrix\n for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { // token idx\n for (uint32_t id = 0; id < n_ids; ++id) { // expert idx\n const uint32_t i02 = *(const uint32_t *) ((const uint8_t *) ids->data + iid1 * ids->nb[1] + id * ids->nb[0]);\n\n assert(i02 >= 0 && i02 < n_as);\n\n MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) { id, iid1 };\n matrix_row_counts[i02] += 1;\n }\n }\n }\n\n // Setup worker pool callbacks\n if (!(octx->flags & HTP_OPFLAGS_SKIP_QUANTIZE)) {\n const uint32_t n_quant_jobs = MIN(src1_nrows, octx->n_threads);\n mmctx->src1_nrows_per_thread = (src1_nrows + n_quant_jobs - 1) / n_quant_jobs;\n worker_pool_run_func(octx->ctx->worker_pool, quant_job_func, mmctx, n_quant_jobs);\n }\n\n if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {\n const uint32_t n_matmul_jobs = octx->n_threads;\n worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, n_matmul_jobs);\n }\n\n return HTP_STATUS_OK;\n}\n" }, { "path": "src/ggml-hexagon/htp/rope-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hvx-utils.h\"\n#include \"hex-fastdiv.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\n// Redefined the types GGML_ROPE_TYPE_NORMAL & GGML_ROPE_TYPE_NEOX as we can't include ggml.h\n#define HTP_ROPE_TYPE_NORMAL 0\n#define HTP_ROPE_TYPE_NEOX 2\n\n#define HTP_ROPE_SPAD_NROWS 16\n#define HTP_ROPE_SPAD_BLOCK (HTP_ROPE_SPAD_NROWS/2)\n\n#define htp_rope_preamble \\\n const uint32_t ne00 = src0->ne[0]; \\\n const uint32_t ne01 = src0->ne[1]; \\\n const uint32_t ne02 = src0->ne[2]; \\\n const uint32_t ne03 = src0->ne[3]; \\\n \\\n const uint32_t ne0 = dst->ne[0]; \\\n const uint32_t ne1 = dst->ne[1]; \\\n const uint32_t ne2 = dst->ne[2]; \\\n const uint32_t ne3 = dst->ne[3]; \\\n \\\n const uint32_t nb00 = src0->nb[0]; \\\n const uint32_t nb01 = src0->nb[1]; \\\n const uint32_t nb02 = src0->nb[2]; \\\n const uint32_t nb03 = src0->nb[3]; \\\n \\\n const uint32_t nb0 = dst->nb[0]; \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3];\n\nstruct htp_rope_context {\n int32_t n_dims;\n int32_t mode;\n int32_t n_ctx_orig;\n int32_t sections[4];\n\n float freq_base;\n float freq_scale;\n float ext_factor;\n float attn_factor;\n float beta_fast;\n float beta_slow;\n float theta_scale;\n float corr_dims[2];\n\n uint32_t src0_nrows_per_thread;\n size_t spad_stride;\n\n struct htp_ops_context * octx;\n\n size_t src0_row_size;\n size_t dst_row_size;\n size_t src0_row_size_aligned;\n size_t dst_row_size_aligned;\n size_t theta_cache_offset;\n uint32_t src0_nrows;\n\n uint64_t t_start;\n};\n\nstatic float rope_yarn_ramp(const float low, const float high, const int i0) {\n const float y = (i0 / 2 - low) / MAX(0.001f, high - low);\n\n return (1 - MIN(1, MAX(0, y)));\n}\n\nstatic void rope_cache_init(const float theta_base,\n const float freq_scale,\n const float * freq_factors,\n float * corr_dims,\n const uint32_t ne0,\n const float ext_factor,\n const float mscale,\n float * cache,\n const float theta_scale) {\n // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py\n float theta = theta_base;\n\n for (uint32_t i0 = 0; i0 < ne0; i0 += 2) {\n const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;\n\n float theta_extrap = theta / ff;\n\n // Get n-d rotational scaling corrected for extrapolation\n float theta_interp = freq_scale * theta_extrap;\n float theta_final = theta_interp;\n float mscale_final = mscale;\n\n if (ext_factor != 0.0f) {\n float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;\n theta_final = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;\n\n // Get n-d magnitude scaling corrected for interpolation\n mscale_final *= 1.0f + 0.1f * logf(1.0f / freq_scale);\n }\n\n cache[i0 + 0] = cosf(theta_final) * mscale_final;\n cache[i0 + 1] = sinf(theta_final) * mscale_final;\n\n theta *= theta_scale;\n }\n}\n\n#define M_PI 3.1415926535897932384626433\n\nstatic void rope_corr_dims(int n_dims,\n int n_ctx_orig,\n float freq_base,\n float beta_fast,\n float beta_slow,\n float * dims) {\n float start = floorf(n_dims * logf(n_ctx_orig / (beta_fast * 2 * (float) M_PI)) / (2 * logf(freq_base)));\n float end = ceilf(n_dims * logf(n_ctx_orig / (beta_slow * 2 * (float) M_PI)) / (2 * logf(freq_base)));\n dims[0] = MAX(0, start);\n dims[1] = MIN(n_dims - 1, end);\n}\n\nstatic inline void hvx_rope_neox_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) {\n const HVX_Vector * restrict vsrc = (const HVX_Vector *) src0;\n const HVX_Vector * restrict vtheta = (const HVX_Vector *) theta_cache;\n HVX_Vector * restrict vdst = (HVX_Vector *) dst;\n\n uint32_t nvec = (ne / (VLEN_FP32 * 2) * 2); // 2 vecs per loop, step of 2\n\n uint32_t he = ne / 2; // half_dims offset in elements\n uint32_t hv = he / VLEN_FP32; // half_dims offset in vectors\n\n #pragma unroll(2)\n for (uint32_t i = 0; i < nvec; i += 2) {\n HVX_Vector v0 = vsrc[i/2+0];\n HVX_Vector v1 = vsrc[i/2+hv];\n\n HVX_Vector v2 = vtheta[i+0];\n HVX_Vector v3 = vtheta[i+1];\n\n HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); // vcos_sin[0] = cos_theta, vcos_sin[1] = sin_theta\n\n HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin));\n HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin));\n HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin));\n HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin));\n\n HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);\n HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);\n\n vdst[i/2+0] = Q6_Vsf_equals_Vqf32(v4);\n vdst[i/2+hv] = Q6_Vsf_equals_Vqf32(v5);\n }\n\n for (uint32_t i = nvec * VLEN_FP32; i < ne; i += 2) {\n const float cos_theta = theta_cache[i+0];\n const float sin_theta = theta_cache[i+1];\n float x0 = src0[i/2];\n float x1 = src0[i/2 + he];\n dst[i/2] = x0 * cos_theta - x1 * sin_theta;\n dst[i/2 + he] = x0 * sin_theta + x1 * cos_theta;\n }\n}\n\nstatic inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) {\n const HVX_Vector * restrict vsrc = (const HVX_Vector *) src0;\n const HVX_Vector * restrict vtheta = (const HVX_Vector *) theta_cache;\n HVX_Vector * restrict vdst = (HVX_Vector *) dst;\n\n uint32_t nvec = (ne / (VLEN_FP32 * 2)) * 2; // 2 vecs per loop, step of two\n\n #pragma unroll(2)\n for (uint32_t i = 0; i < nvec; i+=2) {\n HVX_Vector v0 = vsrc[i+0];\n HVX_Vector v1 = vsrc[i+1];\n\n HVX_Vector v2 = vtheta[i+0];\n HVX_Vector v3 = vtheta[i+1];\n\n HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); // vx0_x1[0] = x0, vx0_x1[1] = x1\n HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); // vcos_sin[0] = cos_theta, vcos_sin[1] = sin_theta\n\n HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin));\n HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin));\n HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin));\n HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin));\n\n HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);\n HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);\n\n HVX_VectorPair vstore = Q6_W_vshuff_VVR(Q6_Vsf_equals_Vqf32(v5), Q6_Vsf_equals_Vqf32(v4), -4);\n\n vdst[i+0] = Q6_V_lo_W(vstore);\n vdst[i+1] = Q6_V_hi_W(vstore);\n }\n\n for (uint32_t i = nvec * VLEN_FP32; i < ne; i += 2) {\n const float cos_theta = theta_cache[i+0];\n const float sin_theta = theta_cache[i+1];\n float x0 = src0[i+0];\n float x1 = src0[i+1];\n dst[i+0] = x0 * cos_theta - x1 * sin_theta;\n dst[i+1] = x0 * sin_theta + x1 * cos_theta;\n }\n}\n\nstatic void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,\n uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {\n #pragma unroll(4)\n for (uint32_t i = 0; i < nr; i++) {\n float * d = (float *) (dst + i * rctx->dst_row_size_aligned);\n float * s = (float *) (src + i * rctx->src0_row_size_aligned);\n\n hvx_rope_f32_aa(d, s, rctx->n_dims, theta_cache);\n\n // fill the remain channels with data from src tensor\n if (rctx->n_dims < ne0) {\n hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims);\n }\n }\n}\n\nstatic void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,\n uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {\n #pragma unroll(4)\n for (uint32_t i = 0; i < nr; i++) {\n float * d = (float *) (dst + i * rctx->dst_row_size_aligned);\n float * s = (float *) (src + i * rctx->src0_row_size_aligned);\n\n hvx_rope_neox_f32_aa(d, s, rctx->n_dims, theta_cache);\n\n // fill the remain channels with data from src tensor\n if (rctx->n_dims < ne0) {\n hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims);\n }\n }\n}\n\nstatic void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {\n struct htp_rope_context * rctx = (struct htp_rope_context *) data;\n struct htp_ops_context * octx = rctx->octx;\n\n const struct htp_tensor * src0 = &octx->src0;\n const struct htp_tensor * src1 = &octx->src1;\n const struct htp_tensor * src2 = &octx->src2;\n struct htp_tensor * dst = &octx->dst;\n\n htp_rope_preamble;\n\n const uint32_t src0_nrows = rctx->src0_nrows;\n const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread;\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n uint64_t tt = HAP_perf_get_qtimer_count();\n\n const int32_t mode = rctx->mode;\n const bool is_neox = mode & HTP_ROPE_TYPE_NEOX;\n\n // VTCM setup\n uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);\n float * theta_cache = (float *) (src0_spad_base);\n src0_spad_base = src0_spad_base + rctx->theta_cache_offset;\n uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);\n\n dma_queue * dma_queue = octx->ctx->dma[ith];\n const int32_t * pos = (const int32_t *) src1->data;\n const float * freq_factors = src2->data ? (const float *) src2->data : NULL;\n\n uint32_t ir = 0;\n uint32_t prev_i2 = (uint32_t) -1;\n\n for (uint32_t i3 = 0; i3 < ne3; i3++) { // batch\n for (uint32_t i2 = 0; i2 < ne2; i2++) { // seq-len\n for (uint32_t i1 = 0; i1 < ne1; ) { // attn-heads\n if (ir < src0_start_row) { ir++; i1++; continue; }\n if (ir >= src0_end_row) goto done;\n\n // Rows in this block\n const uint32_t nrows = MIN(src0_end_row - ir, ne1 - i1);\n\n // Depth before prefetch\n uint32_t dma_depth = dma_queue_depth(dma_queue);\n\n // FARF(HIGH, \"rope-block %u: ir %u n-rows %u dma-depth %u : usec %u\", ith, ir, nrows, dma_depth,\n // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start));\n\n // Prefetch loop\n for (uint32_t pnr = 0, pr = 0; pr < nrows && pr < HTP_ROPE_SPAD_NROWS; pr += pnr) {\n pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK);\n\n uint32_t pi1 = i1 + pr;\n uint32_t pir = ir + pr;\n\n // Dummy DMA transaction for sequencing (interleaving dst,src,dst,...)\n dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr((void *) dst->data, dst_spad_base + pr * rctx->dst_row_size_aligned), 0, 0, 0);\n\n const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;\n uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned;\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr),\n rctx->src0_row_size_aligned, rctx->src0_row_size, pnr);\n\n // FARF(HIGH, \"rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u\", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);\n }\n\n // Update theta cache\n if (i2 != prev_i2) {\n prev_i2 = i2;\n\n const int32_t p = pos[i2];\n rope_cache_init(p, rctx->freq_scale, freq_factors, rctx->corr_dims, ne0, rctx->ext_factor, rctx->attn_factor, theta_cache, rctx->theta_scale);\n\n // FARF(HIGH, \"rope-theta %u: ir %u i1 %u i2 %u i3 %u cache %p : usec %u\", ith, ir, i1, i2, i3, theta_cache,\n // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start));\n }\n\n // Skip DMA transactions from prev block (if any)\n // No need to wait for these since the DMA is setup for in-order processing\n for (uint32_t d=0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); }\n\n // Compute loop\n for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr, ir += cnr, i1 += cnr) {\n // Number of rows to compute\n cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK);\n\n uint8_t * dst_spad = (uint8_t *) dma_queue_pop(dma_queue).src;\n uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_queue).dst;\n\n // FARF(HIGH, \"rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u\", ith, ir, i1, i2, i3, src_spad, cnr,\n // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start));\n\n if (is_neox) {\n rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);\n } else {\n rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);\n }\n\n uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1;\n dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(dst_addr, dst_spad), rctx->dst_row_size, rctx->dst_row_size_aligned, cnr);\n\n // Prefetch more rows (if any)\n if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) {\n uint32_t pnr = MIN(nrows - (cr + HTP_ROPE_SPAD_NROWS), HTP_ROPE_SPAD_BLOCK);\n uint32_t pi1 = i1 + HTP_ROPE_SPAD_NROWS;\n uint32_t pir = ir + HTP_ROPE_SPAD_NROWS;\n\n const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr),\n rctx->src0_row_size_aligned, rctx->src0_row_size, pnr);\n\n // FARF(HIGH, \"rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u\", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);\n }\n }\n }\n }\n }\n\ndone:\n dma_queue_flush(dma_queue);\n tt = HAP_perf_get_qtimer_count() - tt;\n\n FARF(HIGH, \"rope-f32: %d/%d: (%u:%u) usec %u\\n\", ith, nth, src0_start_row, src0_end_row, (unsigned) HAP_perf_qtimer_count_to_us(tt));\n}\n\nstatic int execute_op_rope_f32(struct htp_ops_context * octx) {\n int err = HTP_STATUS_OK;\n\n const struct htp_tensor * src0 = &octx->src0;\n const struct htp_tensor * src1 = &octx->src1;\n const struct htp_tensor * src2 = &octx->src2;\n struct htp_tensor * dst = &octx->dst;\n\n const char * op_type = \"rope-f32\";\n\n switch (octx->op) {\n case HTP_OP_ROPE:\n break;\n\n default:\n FARF(ERROR, \"Unsupported Op %u\\n\", octx->op);\n return HTP_STATUS_NO_SUPPORT;\n }\n\n const uint32_t ne0 = dst->ne[0];\n const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];\n const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);\n\n const size_t src0_row_size = src0->nb[1];\n const size_t dst_row_size = dst->nb[1];\n\n // Aligned row sizes for VTCM\n const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);\n const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);\n const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 128);\n\n // Calculate spad sizes per thread\n size_t src0_spad_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;\n size_t dst_spad_per_thread = HTP_ROPE_SPAD_NROWS * dst_row_size_aligned;\n size_t spad_per_thread = src0_spad_per_thread + dst_spad_per_thread;\n\n // Check if we fit in VTCM\n size_t total_vtcm_needed = spad_per_thread * n_threads;\n if (octx->ctx->vtcm_size < total_vtcm_needed) {\n FARF(ERROR, \"%s : current VTCM reservation %zu is too small, needed %zu\\n\", op_type, octx->ctx->vtcm_size, total_vtcm_needed);\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n // Assign sizes\n octx->src0_spad.size_per_thread = src0_spad_per_thread;\n octx->dst_spad.size_per_thread = dst_spad_per_thread;\n octx->src0_spad.size = n_threads * src0_spad_per_thread;\n octx->dst_spad.size = n_threads * dst_spad_per_thread;\n octx->src1_spad.size = 0;\n\n // Assign pointers\n octx->src0_spad.data = octx->ctx->vtcm_base;\n octx->src1_spad.data = NULL;\n octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size;\n\n // Fill context\n struct htp_rope_context rctx;\n memset(&rctx, 0, sizeof(struct htp_rope_context));\n\n rctx.t_start = HAP_perf_get_qtimer_count();\n\n rctx.octx = octx;\n\n const int32_t * op_params = &octx->op_params[0];\n rctx.n_dims = ((const int32_t *) op_params)[1];\n rctx.mode = ((const int32_t *) op_params)[2];\n rctx.n_ctx_orig = ((const int32_t *) op_params)[4];\n\n memcpy(&rctx.freq_base, (int32_t *) op_params + 5, sizeof(float));\n memcpy(&rctx.freq_scale, (int32_t *) op_params + 6, sizeof(float));\n memcpy(&rctx.ext_factor, (int32_t *) op_params + 7, sizeof(float));\n memcpy(&rctx.attn_factor, (int32_t *) op_params + 8, sizeof(float));\n memcpy(&rctx.beta_fast, (int32_t *) op_params + 9, sizeof(float));\n memcpy(&rctx.beta_slow, (int32_t *) op_params + 10, sizeof(float));\n memcpy(&rctx.sections, (int32_t *) op_params + 11, sizeof(int) * 4);\n\n rctx.theta_scale = powf(rctx.freq_base, -2.0f / rctx.n_dims);\n\n rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims);\n\n rctx.src0_row_size = src0_row_size;\n rctx.dst_row_size = dst_row_size;\n rctx.src0_row_size_aligned = src0_row_size_aligned;\n rctx.dst_row_size_aligned = dst_row_size_aligned;\n rctx.theta_cache_offset = theta_cache_size_aligned;\n\n rctx.src0_nrows = src0_nrows;\n rctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;\n\n FARF(HIGH, \"rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\\n\", rctx.src0_nrows, rctx.n_dims, ne0,\n rctx.ext_factor, rctx.theta_scale, rctx.attn_factor);\n\n if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {\n worker_pool_run_func(octx->ctx->worker_pool, rope_job_f32, &rctx, n_threads);\n }\n\n return err;\n}\n\nint op_rope(struct htp_ops_context * octx) {\n int err = HTP_STATUS_OK;\n\n switch (octx->src0.type) {\n case HTP_TYPE_F32:\n err = execute_op_rope_f32(octx);\n break;\n\n default:\n err = HTP_STATUS_NO_SUPPORT;\n break;\n }\n\n return err;\n}\n" }, { "path": "src/ggml-hexagon/htp/set-rows-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hvx-utils.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\n#define set_rows_preamble \\\n const uint32_t ne00 = octx->src0.ne[0]; \\\n const uint32_t ne01 = octx->src0.ne[1]; \\\n const uint32_t ne02 = octx->src0.ne[2]; \\\n const uint32_t ne03 = octx->src0.ne[3]; \\\n \\\n const uint32_t ne10 = octx->src1.ne[0]; \\\n const uint32_t ne11 = octx->src1.ne[1]; \\\n const uint32_t ne12 = octx->src1.ne[2]; \\\n \\\n const uint32_t nb01 = octx->src0.nb[1]; \\\n const uint32_t nb02 = octx->src0.nb[2]; \\\n const uint32_t nb03 = octx->src0.nb[3]; \\\n \\\n const uint32_t nb10 = octx->src1.nb[0]; \\\n const uint32_t nb11 = octx->src1.nb[1]; \\\n const uint32_t nb12 = octx->src1.nb[2]; \\\n \\\n const uint32_t nb1 = octx->dst.nb[1]; \\\n const uint32_t nb2 = octx->dst.nb[2]; \\\n const uint32_t nb3 = octx->dst.nb[3]; \\\n \\\n const uint32_t ne1 = octx->dst.ne[1]; \\\n \\\n const uint32_t nr = ne01;\n\nstruct htp_set_rows_context {\n struct htp_ops_context * octx;\n struct fastdiv_values div_ne12;\n struct fastdiv_values div_ne11;\n uint32_t src0_nrows_per_thread;\n};\n\nstatic void set_rows_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) {\n struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data;\n struct htp_ops_context * octx = srctx->octx;\n\n set_rows_preamble;\n\n // parallelize by rows of src0\n const uint32_t dr = srctx->src0_nrows_per_thread;\n const uint32_t ir0 = dr * ith;\n const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr;\n\n const bool is_i32 = (octx->src1.type == HTP_TYPE_I32);\n\n for (uint32_t i03 = 0; i03 < ne03; ++i03) {\n for (uint32_t i02 = 0; i02 < ne02; ++i02) {\n for (uint32_t i = ir0; i < ir1; ++i) {\n const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12);\n const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11);\n const uint32_t i10 = i;\n\n const uintptr_t src1_addr = octx->src1.data + i10*nb10 + i11*nb11 + i12*nb12;\n\n uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;\n if (i1 >= ne1) {\n // ignore invalid indices\n continue;\n }\n\n const uintptr_t src0_ptr = octx->src0.data + i*nb01 + i02*nb02 + i03*nb03;\n const uintptr_t dst_ptr = octx->dst.data + i1*nb1 + i02*nb2 + i03*nb3;\n\n // copy row\n hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, ne00);\n }\n }\n }\n}\n\nstatic void set_rows_thread_f16_f32(unsigned int nth, unsigned int ith, void *data) {\n struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data;\n struct htp_ops_context * octx = srctx->octx;\n\n set_rows_preamble;\n\n // parallelize by rows of src0\n const uint32_t dr = srctx->src0_nrows_per_thread;\n const uint32_t ir0 = dr * ith;\n const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr;\n\n const bool is_i32 = (octx->src1.type == HTP_TYPE_I32);\n\n for (uint32_t i03 = 0; i03 < ne03; ++i03) {\n for (uint32_t i02 = 0; i02 < ne02; ++i02) {\n for (uint32_t i = ir0; i < ir1; ++i) {\n const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12);\n const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11);\n const uint32_t i10 = i;\n\n const uintptr_t src1_addr = octx->src1.data + i10*nb10 + i11*nb11 + i12*nb12;\n\n uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;\n if (i1 >= ne1) {\n // ignore invalid indices\n continue;\n }\n\n const uint8_t* src0_ptr = (const uint8_t *) octx->src0.data + i*nb01 + i02*nb02 + i03*nb03;\n uint8_t* dst_ptr = (uint8_t *) octx->dst.data + i1*nb1 + i02*nb2 + i03*nb3;\n\n hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00);\n }\n }\n }\n}\n\nint op_set_rows(struct htp_ops_context * octx) {\n set_rows_preamble;\n\n const uint32_t n_threads = MIN(nr, octx->n_threads);\n\n if (octx->src0.type != HTP_TYPE_F32) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n if (octx->dst.type != HTP_TYPE_F32 && octx->dst.type != HTP_TYPE_F16) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n if (octx->src1.type != HTP_TYPE_I32 && octx->src1.type != HTP_TYPE_I64) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {\n return HTP_STATUS_OK;\n }\n\n struct htp_set_rows_context srctx;\n srctx.octx = octx;\n srctx.div_ne12 = init_fastdiv_values(ne12);\n srctx.div_ne11 = init_fastdiv_values(ne11);\n\n srctx.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads;\n\n switch(octx->dst.type) {\n case HTP_TYPE_F32:\n worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f32_f32, &srctx, n_threads);\n break;\n case HTP_TYPE_F16:\n worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f16_f32, &srctx, n_threads);\n break;\n default:\n return HTP_STATUS_NO_SUPPORT;\n }\n\n return HTP_STATUS_OK;\n}\n" }, { "path": "src/ggml-hexagon/htp/softmax-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hvx-utils.h\"\n#include \"hex-fastdiv.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\n#define htp_softmax_preamble3 \\\n const uint32_t ne00 = src0->ne[0]; \\\n const uint32_t ne01 = src0->ne[1]; \\\n const uint32_t ne02 = src0->ne[2]; \\\n const uint32_t ne03 = src0->ne[3]; \\\n \\\n const uint32_t nb00 = src0->nb[0]; \\\n const uint32_t nb01 = src0->nb[1]; \\\n const uint32_t nb02 = src0->nb[2]; \\\n const uint32_t nb03 = src0->nb[3]; \\\n \\\n const uint32_t ne10 = (src1->ne[0]) ? src1->ne[0] : 1; \\\n const uint32_t ne11 = (src1->ne[0]) ? src1->ne[1] : 1; \\\n const uint32_t ne12 = (src1->ne[0]) ? src1->ne[2] : 1; \\\n const uint32_t ne13 = (src1->ne[0]) ? src1->ne[3] : 1; \\\n \\\n const uint32_t nb10 = (src1->ne[0]) ? src1->nb[0] : 1; \\\n const uint32_t nb11 = (src1->ne[0]) ? src1->nb[1] : 1; \\\n const uint32_t nb12 = (src1->ne[0]) ? src1->nb[2] : 1; \\\n const uint32_t nb13 = (src1->ne[0]) ? src1->nb[3] : 1; \\\n \\\n const uint32_t ne0 = dst->ne[0]; \\\n const uint32_t ne1 = dst->ne[1]; \\\n const uint32_t ne2 = dst->ne[2]; \\\n const uint32_t ne3 = dst->ne[3]; \\\n \\\n const uint32_t nb0 = dst->nb[0]; \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3];\n\nstruct htp_softmax_context {\n bool use_f16;\n bool use_src1;\n uint32_t n_head;\n uint32_t n_head_log2;\n\n float scale;\n float max_bias;\n float m0;\n float m1;\n\n uint32_t src0_nrows_per_thread;\n struct fastdiv_values fastdiv_ne01;\n struct fastdiv_values fastdiv_ne02;\n struct fastdiv_values fastdiv_ne12; // For mask broadcasting\n struct fastdiv_values fastdiv_ne13; // For mask broadcasting\n size_t spad_stride;\n\n struct htp_ops_context * octx;\n};\n\nstatic void init_softmax_ctx(struct htp_softmax_context * smctx, struct htp_ops_context * octx) {\n const struct htp_tensor * src0 = &octx->src0;\n const struct htp_tensor * src1 = &octx->src1;\n\n memset(smctx, 0, sizeof(struct htp_softmax_context));\n\n memcpy(&smctx->scale, (float *) octx->op_params, sizeof(float));\n memcpy(&smctx->max_bias, (float *) octx->op_params + 1, sizeof(float));\n\n smctx->n_head = src0->ne[2];\n smctx->n_head_log2 = 1u << (uint32_t) floor(log2(smctx->n_head));\n\n smctx->m0 = powf(2.0f, -(smctx->max_bias) / smctx->n_head_log2);\n smctx->m1 = powf(2.0f, -(smctx->max_bias / 2.0f) / smctx->n_head_log2);\n\n smctx->use_src1 = (src1->ne[0] != 0);\n smctx->use_f16 = (src1->ne[0] != 0) && (src1->type == HTP_TYPE_F16);\n\n smctx->octx = octx;\n\n // Initialize fastdiv values\n const uint32_t ne01 = src0->ne[1];\n const uint32_t ne02 = src0->ne[2];\n\n if (ne01 > 0) smctx->fastdiv_ne01 = init_fastdiv_values(ne01);\n if (ne02 > 0) smctx->fastdiv_ne02 = init_fastdiv_values(ne02);\n\n const uint32_t ne12 = (src1->ne[0]) ? src1->ne[2] : 1;\n const uint32_t ne13 = (src1->ne[0]) ? src1->ne[3] : 1;\n\n if (ne12 > 0) smctx->fastdiv_ne12 = init_fastdiv_values(ne12);\n if (ne13 > 0) smctx->fastdiv_ne13 = init_fastdiv_values(ne13);\n}\n\nstatic void hvx_fast_softmax_prep_f32(const uint8_t * restrict src,\n uint8_t * restrict dst,\n const int num_elems,\n float scale,\n const uint8_t * restrict mask,\n float slope) {\n const uint8_t * restrict src_curr = src;\n uint8_t * restrict dst_curr = dst;\n const uint8_t * restrict mask_curr = mask;\n\n HVX_Vector scale_vec = hvx_vec_splat_f32(scale);\n HVX_Vector slope_vec = hvx_vec_splat_f32(slope);\n\n int step_of_1 = num_elems >> 5;\n\n #pragma unroll(4)\n for (int i = 0; i < step_of_1; i++) {\n HVX_Vector v1 = *(HVX_Vector *) src_curr;\n\n HVX_Vector v3 = *(HVX_Vector *) mask_curr;\n\n HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec);\n\n HVX_Vector v4 = Q6_Vqf32_vmpy_VsfVsf(v3, slope_vec);\n\n HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, v4);\n\n *(HVX_Vector *) dst_curr = Q6_Vsf_equals_Vqf32(v5);\n\n src_curr += VLEN;\n dst_curr += VLEN;\n mask_curr += VLEN;\n }\n}\n\nstatic void hvx_fast_softmax_f32(const uint8_t * restrict src,\n uint8_t * restrict dst,\n uint8_t * restrict pad,\n const int num_elems) {\n const HVX_Vector * restrict v_src = (HVX_Vector *) src;\n HVX_Vector * restrict v_pad = (HVX_Vector *) pad;\n HVX_Vector * restrict v_dst = (HVX_Vector *) dst;\n\n HVX_Vector sum_vec = Q6_V_vsplat_R(0x00000000);\n HVX_Vector max_vec = hvx_vec_splat_f32(((const float *) src)[0]);\n HVX_Vector zero_v = Q6_V_vzero();\n HVX_Vector one_v = hvx_vec_splat_f32(1.0);\n\n int step_of_1 = num_elems >> 5;\n\n #pragma unroll(4)\n for (int i = 0; i < step_of_1; i++) {\n HVX_Vector v1 = v_src[i];\n max_vec = Q6_Vsf_vmax_VsfVsf(max_vec, v1);\n }\n\n max_vec = hvx_vec_reduce_max_f32(max_vec); // replicated over all lanes\n\n #pragma unroll(4)\n for (int i = 0; i < step_of_1; i++) {\n HVX_Vector v1 = v_src[i];\n HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, max_vec);\n\n HVX_Vector v3 = hvx_vec_exp_f32(Q6_Vsf_equals_Vqf32(v2));\n\n sum_vec = Q6_Vqf32_vadd_VsfVsf(Q6_Vsf_equals_Vqf32(sum_vec), v3);\n\n v_pad[i] = v3;\n }\n\n sum_vec = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_vec)); // replicated over all lanes\n\n HVX_VectorPred pos_sum = Q6_Q_vcmp_gt_VwVw(sum_vec, zero_v);\n HVX_Vector v4 = hvx_vec_inverse_f32(sum_vec);\n HVX_Vector scale_vec = Q6_V_vmux_QVV(pos_sum, v4, one_v);\n\n #pragma unroll(4)\n for (int i = 0; i < step_of_1; i++) {\n HVX_Vector v1 = v_pad[i];\n HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec);\n v_dst[i] = Q6_Vsf_equals_Vqf32(v2);\n }\n}\n\nstatic float hvx_softmax_f32(const uint8_t * restrict src,\n uint8_t * restrict dst,\n uint8_t * restrict spad,\n const int num_elems,\n const float max) {\n hvx_sub_scalar_f32(spad, src, max, num_elems);\n\n hvx_exp_f32(spad, dst, num_elems, false);\n\n float sum = hvx_reduce_sum_f32(dst, num_elems);\n\n return sum;\n}\n\nstatic void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) {\n struct htp_softmax_context * smctx = (struct htp_softmax_context *) data;\n struct htp_ops_context * octx = smctx->octx;\n\n const struct htp_tensor * src0 = &octx->src0;\n const struct htp_tensor * src1 = &octx->src1;\n struct htp_tensor * dst = &octx->dst;\n\n htp_softmax_preamble3;\n\n const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows\n const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread;\n\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n int is_aligned = 1;\n int opt_path = 0;\n if (!hex_is_aligned((void *) src0->data, VLEN) || !hex_is_aligned((void *) dst->data, VLEN)) {\n is_aligned = 0;\n FARF(HIGH, \"softmax-f32: unaligned addresses in elementwise op, possibly slower execution\\n\");\n }\n if ((1 == is_aligned) && !(nb01 & (VLEN - 1))) {\n opt_path = 1;\n }\n\n uint8_t * src0_spad_data = octx->src0_spad.data + (ith * smctx->spad_stride);\n uint8_t * src1_spad_data = octx->src1_spad.data + (ith * smctx->spad_stride);\n uint8_t * dst_spad_data = octx->dst_spad.data + (ith * smctx->spad_stride);\n\n float * wp0 = (float *) src0_spad_data;\n float * wp1 = (float *) src1_spad_data;\n float * wp2 = (float *) dst_spad_data;\n\n uint32_t prev_i2 = (uint32_t)-1;\n float slope = 1.0f;\n\n for (uint32_t r = src0_start_row; r < src0_end_row; ++r) {\n uint32_t i1 = fastmodulo(r, ne01, &smctx->fastdiv_ne01);\n uint32_t r_div_ne01 = fastdiv(r, &smctx->fastdiv_ne01);\n uint32_t i2 = fastmodulo(r_div_ne01, ne02, &smctx->fastdiv_ne02);\n uint32_t i3 = fastdiv(r_div_ne01, &smctx->fastdiv_ne02);\n\n // Map to original logic indices\n // i01 = i1\n // i02 = i2\n // i03 = i3\n\n const uint32_t i11 = i1;\n // const uint32_t i12 = i2 % ne12;\n // const uint32_t i13 = i3 % ne13;\n\n uint32_t i12, i13;\n if (ne12 == ne02) {\n i12 = i2;\n } else {\n i12 = fastmodulo(i2, ne12, &smctx->fastdiv_ne12);\n }\n\n if (ne13 == ne03) {\n i13 = i3;\n } else {\n i13 = fastmodulo(i3, ne13, &smctx->fastdiv_ne13);\n }\n\n // ALiBi\n if (i2 != prev_i2) {\n const uint32_t h = i2; // head\n\n slope = (smctx->max_bias > 0.0f) ?\n h < smctx->n_head_log2 ?\n powf(smctx->m0, h + 1) :\n powf(smctx->m1, 2 * (h - smctx->n_head_log2) + 1) :\n 1.0f;\n prev_i2 = i2;\n }\n\n float * sp = (float *) ((char *) octx->src0.data + i1 * nb01 + i2 * nb02 + i3 * nb03);\n float * dp = (float *) ((char *) octx->dst.data + i1 * nb1 + i2 * nb2 + i3 * nb3);\n\n // broadcast the mask across rows\n __fp16 * mp_f16 = (smctx->use_src1) ?\n (__fp16 *) ((char *) octx->src1.data + i11 * nb11 + i12 * nb12 + i13 * nb13) :\n NULL;\n float * mp_f32 = (smctx->use_src1) ?\n (float *) ((char *) octx->src1.data + i11 * nb11 + i12 * nb12 + i13 * nb13) :\n NULL;\n\n if ((1 == opt_path) && (mp_f32) && !(smctx->use_f16)) {\n hvx_fast_softmax_prep_f32((const uint8_t *) sp, (uint8_t *) wp0, ne00, smctx->scale,\n (const uint8_t *) mp_f32, slope);\n } else {\n hvx_scale_f32((uint8_t *) wp0, (const uint8_t *) sp, ne00, smctx->scale);\n if (mp_f32) {\n if (smctx->use_f16) {\n for (int i = 0; i < ne00; ++i) {\n wp0[i] += slope * (float) mp_f16[i];\n }\n } else {\n for (int i = 0; i < ne00; ++i) {\n wp0[i] += slope * mp_f32[i];\n }\n }\n }\n }\n\n if (1 == opt_path) {\n hvx_fast_softmax_f32((const uint8_t *) wp0, (uint8_t *) dp, (uint8_t *) wp1, ne00);\n } else {\n float max = hvx_reduce_max_f32((const uint8_t *) wp0, ne00);\n float sum = hvx_softmax_f32((const uint8_t *) wp0, (uint8_t *) wp2, (uint8_t *) wp1, ne00, max);\n sum = sum > 0.0 ? (1.0 / sum) : 1;\n hvx_scale_f32((uint8_t *) dp, (const uint8_t *) wp2, ne00, sum);\n }\n }\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"softmax-f32 %d/%d/%d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\\n\", ith, nth,\n smctx->use_f16, opt_path, ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13,\n ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\nstatic int execute_op_softmax_f32(struct htp_ops_context * octx) {\n int err = HTP_STATUS_OK;\n\n const struct htp_tensor * src0 = &octx->src0;\n const struct htp_tensor * src1 = &octx->src1;\n struct htp_tensor * dst = &octx->dst;\n\n struct htp_softmax_context smctx;\n const char * op_type = \"softmax-f32\";\n\n switch (octx->op) {\n case HTP_OP_SOFTMAX:\n init_softmax_ctx(&smctx, octx);\n break;\n\n default:\n FARF(ERROR, \"Unsupported Op %u\\n\", octx->op);\n return HTP_STATUS_NO_SUPPORT;\n }\n\n const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];\n const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);\n\n const size_t src0_row_size = src0->nb[1];\n const size_t src1_row_size = src0_row_size;\n const size_t dst_row_size = dst->nb[1];\n\n // VTCM scratchpads for all tensors\n // N rows per thread, padded to HVX vector size\n octx->dst_spad.size = hex_round_up(dst_row_size, 128) * n_threads;\n octx->src0_spad.size = hex_round_up(src0_row_size, 128) * n_threads;\n octx->src1_spad.size = hex_round_up(src1_row_size, 128) * n_threads;\n\n // Use stride for calculating offset\n smctx.spad_stride = hex_round_up(src0_row_size, 128);\n\n size_t spad_size = octx->src0_spad.size + octx->src1_spad.size + octx->dst_spad.size;\n\n if (src1->ne[0]) {\n FARF(HIGH,\n \"%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\\n\",\n op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2],\n src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], octx->src0_spad.size, octx->src1_spad.size,\n octx->dst_spad.size);\n } else {\n FARF(HIGH, \"%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\\n\", op_type,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size);\n }\n\n // Make sure the reserved vtcm size is sufficient\n if (octx->ctx->vtcm_size < spad_size) {\n FARF(ERROR, \"%s : current VTCM reservation %zu is too small, needed %zu\\n\", op_type, octx->ctx->vtcm_size,\n spad_size);\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n octx->src0_spad.data = octx->ctx->vtcm_base;\n octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;\n octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size;\n\n if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {\n smctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;\n worker_pool_run_func(octx->ctx->worker_pool, softmax_job_f32, &smctx, n_threads);\n }\n\n return err;\n}\n\nint op_softmax(struct htp_ops_context * octx) {\n int err = HTP_STATUS_OK;\n\n switch (octx->src0.type) {\n case HTP_TYPE_F32:\n err = execute_op_softmax_f32(octx);\n break;\n\n default:\n err = HTP_STATUS_NO_SUPPORT;\n break;\n }\n\n return err;\n}\n" }, { "path": "src/ggml-hexagon/htp/ssm-conv.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"hex-dma.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n#include \"hvx-utils.h\"\n\n#define htp_ssm_conv_tensors_preamble \\\n struct htp_tensor * restrict src0 = &octx->src0; \\\n struct htp_tensor * restrict src1 = &octx->src1; \\\n struct htp_tensor * restrict dst = &octx->dst; \\\n struct htp_spad * restrict src0_spad = &octx->src0_spad; \\\n struct htp_spad * restrict src1_spad = &octx->src1_spad; \\\n struct htp_spad * restrict dst_spad = &octx->dst_spad; \\\n \\\n const uint32_t ne00 = src0->ne[0]; \\\n const uint32_t ne01 = src0->ne[1]; \\\n const uint32_t ne02 = src0->ne[2]; \\\n const uint32_t ne03 = src0->ne[3]; \\\n \\\n const uint32_t ne10 = src1->ne[0]; \\\n const uint32_t ne11 = src1->ne[1]; \\\n const uint32_t ne12 = src1->ne[2]; \\\n const uint32_t ne13 = src1->ne[3]; \\\n \\\n const uint32_t ne0 = dst->ne[0]; \\\n const uint32_t ne1 = dst->ne[1]; \\\n const uint32_t ne2 = dst->ne[2]; \\\n const uint32_t ne3 = dst->ne[3]; \\\n \\\n const uint32_t nb00 = src0->nb[0]; \\\n const uint32_t nb01 = src0->nb[1]; \\\n const uint32_t nb02 = src0->nb[2]; \\\n const uint32_t nb03 = src0->nb[3]; \\\n \\\n const uint32_t nb10 = src1->nb[0]; \\\n const uint32_t nb11 = src1->nb[1]; \\\n const uint32_t nb12 = src1->nb[2]; \\\n const uint32_t nb13 = src1->nb[3]; \\\n \\\n const uint32_t nb0 = dst->nb[0]; \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3];\n\nstruct htp_ssm_conv_context {\n struct htp_ops_context * octx;\n uint32_t nrows_per_thread;\n uint64_t t_start;\n};\n\n#define htp_ssm_conv_preamble \\\n struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; \\\n struct htp_ops_context * octx = scctx->octx; \\\n htp_ssm_conv_tensors_preamble; \\\n dma_queue * dma_queue = octx->ctx->dma[ith];\n\n// Scalar FP32 SSM_CONV implementation\nstatic void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) {\n htp_ssm_conv_preamble;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const uint32_t d_conv = src1->ne[0];\n const uint32_t d_inner = src0->ne[1];\n const uint32_t n_t = dst->ne[1];\n const uint32_t n_s = dst->ne[2];\n\n const uint32_t src0_stride_inner = src0->nb[1] / sizeof(float); // stride for inner dimension\n const uint32_t src0_stride_seq = src0->nb[2] / sizeof(float); // stride for sequence dimension\n const uint32_t src1_stride_inner = src1->nb[1] / sizeof(float); // stride for inner dimension\n const uint32_t dst_stride_token = dst->nb[1] / sizeof(float); // stride for token dimension\n const uint32_t dst_stride_seq = dst->nb[2] / sizeof(float); // stride for sequence dimension\n\n const float * src0_data = (const float *) src0->data;\n const float * src1_data = (const float *) src1->data;\n float * dst_data = (float *) dst->data;\n\n // Calculate row range for this thread\n const uint32_t d_inner_per_thread = scctx->nrows_per_thread;\n const uint32_t d_inner_start = d_inner_per_thread * ith;\n const uint32_t d_inner_end = MIN(d_inner_start + d_inner_per_thread, d_inner);\n\n // No work for this thread\n if (d_inner_start >= d_inner_end) {\n return;\n }\n\n for (uint32_t i3 = 0; i3 < n_s; ++i3) {\n for (uint32_t i2 = 0; i2 < n_t; ++i2) {\n for (uint32_t i1 = d_inner_start; i1 < d_inner_end; ++i1) {\n float sumf = 0.0f;\n\n for (uint32_t i0 = 0; i0 < d_conv; ++i0) {\n const uint32_t src0_idx = (i2 + i0) + i1 * src0_stride_inner + i3 * src0_stride_seq;\n const uint32_t src1_idx = i0 + i1 * src1_stride_inner;\n\n sumf += src0_data[src0_idx] * src1_data[src1_idx];\n }\n\n const uint32_t dst_idx = i1 + i2 * dst_stride_token + i3 * dst_stride_seq;\n dst_data[dst_idx] = sumf;\n }\n }\n }\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"ssm-conv-f32 %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\\n\",\n ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], d_inner_start, d_inner_end,\n src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1],\n dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\n// HVX FP32 SSM_CONV implementation - vectorizes across d_inner dimension\nstatic void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) {\n htp_ssm_conv_preamble;\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const int nc = src1->ne[0]; // d_conv\n const int ncs = src0->ne[0]; // d_conv - 1 + n_t\n\n const uint32_t d_conv = src1->ne[0];\n const uint32_t d_inner = src0->ne[1];\n const uint32_t n_t = dst->ne[1];\n const uint32_t n_s = dst->ne[2];\n\n const float * src0_data = (const float *) src0->data;\n const float * src1_data = (const float *) src1->data;\n float * dst_data = (float *) dst->data;\n\n // Calculate row range for this thread\n const int dr = scctx->nrows_per_thread;\n const uint32_t ir0 = dr * ith;\n const uint32_t ir1 = MIN(ir0 + dr, d_inner);\n const int ir = ir1 - ir0;\n\n if (ir0 >= ir1) {\n return; // No work for this thread\n }\n\n // src0 and src1 gather offsets\n uint32_t __attribute__((aligned(VLEN))) src0_offsets[VLEN_FP32] = { 0 };\n uint32_t __attribute__((aligned(VLEN))) src1_offsets[VLEN_FP32] = { 0 };\n\n for (uint32_t i = 0; i < VLEN_FP32; ++i) {\n src0_offsets[i] = i * (ncs) * sizeof(float);\n src1_offsets[i] = i * (d_conv) * sizeof(float);\n }\n\n const uint32_t src0_gather_len = VLEN * ncs;\n const uint32_t src1_gather_len = VLEN * d_conv;\n\n // gather scratchpads\n HVX_Vector * src0_vec = (HVX_Vector *) (octx->ctx->vtcm_base + ith * VLEN*2 + 0);\n HVX_Vector * src1_vec = (HVX_Vector *) (octx->ctx->vtcm_base + ith * VLEN*2 + VLEN);\n\n float * data_src0 = (float *) ((char *) src0->data + ir0 * src0->nb[1]);\n float * data_src1 = (float *) ((char *) src1->data + ir0 * src1->nb[1]);\n\n uint8_t * spad_src0 = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread;\n uint8_t * spad_src1 = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread;\n\n // copy src1 workload to VTCM\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src1, data_src1), nb11, nb11, ir);\n\n // FARF(HIGH, \"ssm-conv-src1-fetch %d: ir0 %u size %u\\n\", ith, ir0, nb11 * ir);\n\n for (uint32_t i3 = 0; i3 < n_s; ++i3) {\n float * src0_data_ptr = (float *) ((char *) data_src0 + i3 * (src0->nb[2]));\n\n // copy src0 workload to VTCM\n dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0, src0_data_ptr), nb01, nb01, ir);\n\n // FARF(HIGH, \"ssm-conv-src0-fetch %d: ir0 %u i3 %u size %u\\n\", ith, ir0, i3, nb01 * ir);\n\n dma_queue_flush(dma_queue);\n\n for (uint32_t i2 = 0; i2 < n_t; ++i2) {\n float * dst_ptr = (float *) ((char *) dst->data + ir0 * (dst->nb[0]) + i2 * (dst->nb[1]) + i3 * (dst->nb[2]));\n\n const uint32_t nvec = ir / VLEN_FP32;\n const uint32_t nloe = ir % VLEN_FP32;\n uint32_t i1 = 0;\n\n for (uint32_t vi1 = 0; vi1 < nvec; vi1++) {\n HVX_Vector acc_vec = Q6_V_vsplat_R(0);\n\n for (uint32_t i0 = 0; i0 < d_conv; ++i0) {\n Q6_vgather_ARMVw(src0_vec, GATHER_TYPE(spad_src0 + (i0 + i1 * ncs) * sizeof(float) + i2 * (src0->nb[0])),\n src0_gather_len, (*(const HVX_Vector *) src0_offsets));\n Q6_vgather_ARMVw(src1_vec, GATHER_TYPE(spad_src1 + (i0 + i1 * nc) * sizeof(float)),\n src1_gather_len, (*(const HVX_Vector *) src1_offsets));\n\n HVX_Vector prod = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *) src0_vec, *(const HVX_Vector *) src1_vec);\n acc_vec = Q6_Vqf32_vadd_Vqf32Vqf32(acc_vec, prod);\n }\n\n *(HVX_UVector *) (dst_ptr + i1) = Q6_Vsf_equals_Vqf32(acc_vec);\n i1 += VLEN_FP32;\n }\n\n if (nloe) {\n HVX_Vector acc_vec = Q6_V_vsplat_R(0);\n\n for (uint32_t i0 = 0; i0 < d_conv; ++i0) {\n Q6_vgather_ARMVw(src0_vec, GATHER_TYPE(spad_src0 + (i0 + i1 * ncs) * sizeof(float) + i2 * (src0->nb[0])),\n src0_gather_len, (*(const HVX_Vector *) src0_offsets));\n Q6_vgather_ARMVw(src1_vec, GATHER_TYPE(spad_src1 + (i0 + i1 * nc) * sizeof(float)),\n src1_gather_len, (*(const HVX_Vector *) src1_offsets));\n\n HVX_Vector prod = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *) src0_vec, *(const HVX_Vector *) src1_vec);\n acc_vec = Q6_Vqf32_vadd_Vqf32Vqf32(acc_vec, prod);\n }\n\n hvx_vec_store_u(dst_ptr + i1, (ir - i1) * 4, Q6_Vsf_equals_Vqf32(acc_vec));\n }\n }\n }\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"ssm-conv-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\\n\",\n ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1,\n src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1],\n dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\nint op_ssm_conv_f32(struct htp_ops_context * octx) {\n htp_ssm_conv_tensors_preamble;\n\n if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) {\n FARF(ERROR, \"ssm_conv: only (F32 x F32 -> F32) OPs supported\");\n return HTP_STATUS_NO_SUPPORT;\n }\n\n struct htp_ssm_conv_context scctx = { 0 };\n scctx.octx = octx;\n\n const uint32_t d_conv = src1->ne[0];\n const uint32_t d_inner = src0->ne[1];\n const uint32_t n_t = dst->ne[1]; // tokens per sequence\n const uint32_t n_s = dst->ne[2]; // number of sequences in the batch\n\n const uint32_t n_threads = MIN(octx->n_threads, d_inner);\n\n if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {\n uint32_t use_hvx = 0;\n if (d_inner >= VLEN_FP32 && d_inner % VLEN_FP32 == 0) {\n int is_aligned = hex_is_aligned((void *) src0->data, VLEN) &&\n hex_is_aligned((void *) src1->data, VLEN) &&\n hex_is_aligned((void *) dst->data, VLEN);\n\n if (is_aligned) {\n use_hvx = 1;\n }\n }\n\n if (use_hvx) {\n scctx.nrows_per_thread = (d_inner + n_threads - 1) / n_threads; // d_inner chunks per thread\n scctx.nrows_per_thread += (scctx.nrows_per_thread & 1); // round up to even\n\n octx->src0_spad.size_per_thread = hex_round_up(scctx.nrows_per_thread * nb01, 256);\n octx->src1_spad.size_per_thread = hex_round_up(scctx.nrows_per_thread * nb11, 256);\n octx->dst_spad.size_per_thread = hex_round_up(scctx.nrows_per_thread * sizeof(float), 256);\n\n octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads;\n octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads;\n octx->dst_spad.size = octx->dst_spad.size_per_thread * n_threads;\n\n // Compute gather scratchpad size for src0 and src1\n const size_t gather_spad_size = n_threads * VLEN * 2;\n\n octx->src0_spad.data = octx->ctx->vtcm_base + gather_spad_size;\n octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;\n octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size;\n\n FARF(HIGH, \"ssm_conv-f32: gather-spad:%zu spad-per-thread:(%u:%u:%u) spad-sizes:(%u:%u:%u) spad-data:(%p:%p:%p)\\n\",\n gather_spad_size, octx->src0_spad.size_per_thread, octx->src1_spad.size_per_thread,\n octx->dst_spad.size_per_thread, octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size,\n octx->src0_spad.data, octx->src1_spad.data, octx->dst_spad.data);\n\n const size_t total_spad_size =\n gather_spad_size + octx->src0_spad.size + octx->src1_spad.size + octx->dst_spad.size;\n\n if (total_spad_size > octx->ctx->vtcm_size) {\n FARF(HIGH, \"ssm_conv-f32: HVX scratchpad size %zu exceeds VTCM size %zu\", total_spad_size,\n octx->ctx->vtcm_size);\n use_hvx = 0;\n }\n }\n\n FARF(HIGH, \"ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_hvx %d\\n\", src0->ne[0],\n src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0],\n dst->ne[1], dst->ne[2], dst->ne[3], use_hvx);\n\n if (use_hvx) {\n worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32_hvx, &scctx, n_threads);\n } else {\n worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32, &scctx, n_threads);\n }\n }\n\n return HTP_STATUS_OK;\n}\n\nint op_ssm_conv(struct htp_ops_context * octx) {\n int err = HTP_STATUS_OK;\n struct htp_tensor * dst = &octx->dst;\n\n switch (dst->type) {\n case HTP_TYPE_F32:\n err = op_ssm_conv_f32(octx);\n break;\n default:\n err = HTP_STATUS_NO_SUPPORT;\n break;\n }\n\n return err;\n}\n" }, { "path": "src/ggml-hexagon/htp/sum-rows-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hvx-utils.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\n#define sum_rows_preamble \\\n struct htp_tensor *src0 = &octx->src0;\\\n struct htp_tensor *dst = &octx->dst; \\\n \\\n const uint32_t ne00 = src0->ne[0]; \\\n const uint32_t ne01 = src0->ne[1]; \\\n const uint32_t ne02 = src0->ne[2]; \\\n const uint32_t ne03 = src0->ne[3]; \\\n \\\n const uint32_t nb00 = src0->nb[0]; \\\n const uint32_t nb01 = src0->nb[1]; \\\n const uint32_t nb02 = src0->nb[2]; \\\n const uint32_t nb03 = src0->nb[3]; \\\n \\\n const uint32_t ne0 = dst->ne[0]; \\\n const uint32_t ne1 = dst->ne[1]; \\\n const uint32_t ne2 = dst->ne[2]; \\\n const uint32_t ne3 = dst->ne[3]; \\\n \\\n const uint32_t nb0 = dst->nb[0]; \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3]; \\\n\nstruct sum_rows_context {\n const uint8_t * src_data;\n uint8_t * dst_data;\n uint32_t ne00;\n size_t src_stride;\n size_t dst_stride;\n uint32_t rows_per_thread;\n uint32_t total_rows;\n bool opt_path;\n};\n\nstatic void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) {\n const struct sum_rows_context * smctx = (const struct sum_rows_context *) data;\n\n const uint32_t rows_per_thread = smctx->rows_per_thread;\n const uint32_t total_rows = smctx->total_rows;\n\n const uint32_t start_row = rows_per_thread * ith;\n const uint32_t end_row = MIN(start_row + rows_per_thread, total_rows);\n\n if (start_row >= end_row) {\n return;\n }\n\n const size_t src_stride = smctx->src_stride;\n const size_t dst_stride = smctx->dst_stride;\n const uint32_t ne00 = smctx->ne00;\n const bool opt_path = smctx->opt_path;\n\n const float * restrict src_th = (const float *) (smctx->src_data + (start_row * src_stride));\n float * restrict dst_th = (float *) (smctx->dst_data + (start_row * dst_stride));\n\n // Calculate actual number of rows for this thread\n const uint32_t n_rows = end_row - start_row;\n\n for (uint32_t ir = 0; ir < n_rows; ir++) {\n const float * restrict src_local = src_th + (ir * (src_stride / sizeof(float)));\n\n if (ir + 1 < n_rows) {\n hex_l2fetch(src_local + (src_stride / sizeof(float)), src_stride, src_stride, 1);\n }\n\n if (opt_path) {\n dst_th[ir] = hvx_reduce_sum_f32_a((const uint8_t *) src_local, ne00);\n } else {\n dst_th[ir] = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00);\n }\n }\n}\n\nint op_sum_rows(struct htp_ops_context * octx) {\n sum_rows_preamble;\n\n if (octx->src0.type != HTP_TYPE_F32) {\n return HTP_STATUS_NO_SUPPORT;\n }\n\n if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {\n return HTP_STATUS_OK;\n }\n\n const uint32_t src0_nrows = ne01 * ne02 * ne03;\n const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);\n const uint32_t rows_per_thread = (src0_nrows + n_threads - 1) / n_threads;\n\n bool opt_path = false;\n if ((0 == hex_is_aligned((void *) src0->data, VLEN)) && !(nb01 & (VLEN - 1))) {\n opt_path = true;\n }\n\n struct sum_rows_context smctx = {\n .src_data = (const uint8_t *) src0->data,\n .dst_data = (uint8_t *) dst->data,\n .ne00 = ne00,\n .src_stride = nb01,\n .dst_stride = nb1,\n .rows_per_thread = rows_per_thread,\n .total_rows = src0_nrows,\n .opt_path = opt_path,\n };\n\n worker_pool_run_func(octx->ctx->worker_pool, sum_rows_thread_f32, &smctx, n_threads);\n\n return HTP_STATUS_OK;\n}\n" }, { "path": "src/ggml-hexagon/htp/unary-ops.c", "content": "#pragma clang diagnostic ignored \"-Wunused-variable\"\n#pragma clang diagnostic ignored \"-Wunused-function\"\n#pragma clang diagnostic ignored \"-Wunused-but-set-variable\"\n\n#include \n#include \n\n#include \n#include \n\n#include \"hex-dma.h\"\n#include \"hvx-utils.h\"\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n#include \"htp-ctx.h\"\n#include \"htp-msg.h\"\n#include \"htp-ops.h\"\n\nstruct htp_unary_context {\n struct htp_ops_context * octx;\n\n // Precomputed values\n const uint8_t * data_src0;\n uint8_t * data_dst;\n\n size_t src0_row_size;\n size_t dst_row_size;\n\n size_t src0_row_size_aligned;\n size_t dst_row_size_aligned;\n\n size_t src0_spad_half_size;\n size_t dst_spad_half_size;\n\n uint32_t block;\n uint32_t src0_nrows;\n uint32_t src0_nrows_per_thread;\n uint32_t nc;\n};\n\n#define htp_unary_preamble \\\n const uint32_t ne00 = src->ne[0]; \\\n const uint32_t ne01 = src->ne[1]; \\\n const uint32_t ne02 = src->ne[2]; \\\n const uint32_t ne03 = src->ne[3]; \\\n \\\n const uint32_t ne0 = dst->ne[0]; \\\n const uint32_t ne1 = dst->ne[1]; \\\n const uint32_t ne2 = dst->ne[2]; \\\n const uint32_t ne3 = dst->ne[3]; \\\n \\\n const uint32_t nb00 = src->nb[0]; \\\n const uint32_t nb01 = src->nb[1]; \\\n const uint32_t nb02 = src->nb[2]; \\\n const uint32_t nb03 = src->nb[3]; \\\n \\\n const uint32_t nb0 = dst->nb[0]; \\\n const uint32_t nb1 = dst->nb[1]; \\\n const uint32_t nb2 = dst->nb[2]; \\\n const uint32_t nb3 = dst->nb[3];\n\nstatic void hvx_fast_rms_norm_f32(const uint8_t * restrict src,\n uint8_t * restrict dst,\n uint8_t * restrict pad,\n const int num_elems,\n float epsilon) {\n const HVX_Vector * restrict v_src = (HVX_Vector *) src;\n HVX_Vector * restrict v_dst = (HVX_Vector *) dst;\n\n HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);\n HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);\n\n int step_of_1 = num_elems >> 5;\n #pragma unroll(4)\n for (int i = 0; i < step_of_1; i++) {\n HVX_Vector v1 = v_src[i];\n HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);\n sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);\n }\n\n sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); // replicated over all lanes\n\n HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);\n HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);\n HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);\n HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);\n\n HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));\n\n #pragma unroll(4)\n for (int i = 0; i < step_of_1; i++) {\n HVX_Vector v1 = v_src[i];\n HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);\n v_dst[i] = Q6_Vsf_equals_Vqf32(v2);\n }\n}\n\nstatic void scale_f32(const float * restrict src,\n float * restrict dst,\n uint8_t * restrict spad,\n const uint32_t num_rows,\n const uint32_t row_elems,\n const size_t row_size,\n int32_t * op_params) {\n float scale = 0.f;\n float bias = 0.f;\n memcpy(&scale, &op_params[0], sizeof(float));\n memcpy(&bias, &op_params[1], sizeof(float));\n\n for (uint32_t ir = 0; ir < num_rows; ir++) {\n const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);\n uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);\n\n hvx_scale_offset_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, row_elems, scale, bias);\n }\n}\n\nstatic void rms_norm_f32(const float * restrict src,\n float * restrict dst,\n uint8_t * restrict spad,\n const uint32_t num_rows,\n const uint32_t row_elems,\n const size_t row_size,\n int32_t * op_params) {\n float epsilon = 0.f;\n memcpy(&epsilon, op_params, sizeof(float));\n\n for (uint32_t ir = 0; ir < num_rows; ir++) {\n const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);\n uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);\n\n hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, spad, row_elems, epsilon);\n }\n}\n\nstatic void sqr_f32(const float * restrict src,\n float * restrict dst,\n uint8_t * restrict spad,\n const uint32_t num_rows,\n const uint32_t row_elems,\n const size_t row_size,\n int32_t * op_params) {\n\n for (uint32_t ir = 0; ir < num_rows; ir++) {\n const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);\n uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);\n\n hvx_sqr_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, row_elems);\n }\n}\n\nstatic void sqrt_f32(const float * restrict src,\n float * restrict dst,\n uint8_t * restrict spad,\n const uint32_t num_rows,\n const uint32_t row_elems,\n const size_t row_size,\n int32_t * op_params) {\n\n for (uint32_t ir = 0; ir < num_rows; ir++) {\n const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);\n uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);\n\n hvx_sqrt_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, row_elems);\n }\n}\n\nstatic void unary_job_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {\n const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;\n struct htp_ops_context * octx = uctx->octx;\n const struct htp_tensor * src = &octx->src0;\n const struct htp_tensor * dst = &octx->dst;\n\n htp_unary_preamble;\n\n int htp_op = octx->op;\n int32_t * op_params = octx->op_params;\n uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread;\n\n const size_t src0_row_size = uctx->src0_row_size;\n const size_t dst_row_size = uctx->dst_row_size;\n\n const size_t src0_row_size_aligned = uctx->src0_row_size_aligned;\n const size_t dst_row_size_aligned = uctx->dst_row_size_aligned;\n\n const uint32_t src0_nrows = uctx->src0_nrows;\n const uint32_t src0_start_row = src0_nrows_per_thread * ith;\n const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);\n\n // no work for this thread\n if (src0_start_row >= src0_end_row) {\n return;\n }\n\n uint64_t t1, t2;\n t1 = HAP_perf_get_qtimer_count();\n\n const uint8_t * restrict data_src = uctx->data_src0;\n uint8_t * restrict data_dst = uctx->data_dst;\n\n uint8_t * src0_spad_data = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);\n uint8_t * dst_spad_data = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);\n\n size_t src0_spad_half_size = uctx->src0_spad_half_size;\n size_t dst_spad_half_size = uctx->dst_spad_half_size;\n\n const int BLOCK = uctx->block;\n if (BLOCK == 0) {\n FARF(ERROR, \"unary-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\\n\",\n octx->src0_spad.size_per_thread, src0_row_size_aligned);\n return;\n }\n\n dma_queue * dma_queue = octx->ctx->dma[ith];\n\n for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)\n dma_queue_push_vtcm_to_ddr(dma_queue,\n dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),\n dst_row_size, dst_row_size_aligned, 0);\n\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src + (ir * src0_row_size)),\n src0_row_size_aligned, src0_row_size, block_size);\n }\n\n for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {\n const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);\n\n float * dst_spad = (float *) dma_queue_pop(dma_queue).src;\n float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;\n\n // Process block in VTCM\n switch (htp_op) {\n case HTP_OP_RMS_NORM:\n rms_norm_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);\n break;\n case HTP_OP_SCALE:\n scale_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);\n break;\n case HTP_OP_SQR:\n sqr_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);\n break;\n case HTP_OP_SQRT:\n sqrt_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);\n break;\n default:\n break;\n }\n\n dma_queue_push_vtcm_to_ddr(dma_queue,\n dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad),\n dst_row_size, dst_row_size_aligned, block_size);\n\n // prefetch N+2 loop iteration if any\n const uint32_t pref_block = (ir + BLOCK * 2);\n if (pref_block < src0_end_row) {\n const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);\n dma_queue_push_ddr_to_vtcm(dma_queue,\n dma_make_ptr(src0_spad, data_src + (pref_block * src0_row_size)),\n src0_row_size_aligned, src0_row_size, pref_block_size);\n }\n }\n\n dma_queue_flush(dma_queue);\n\n t2 = HAP_perf_get_qtimer_count();\n\n FARF(HIGH, \"unary-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\\n\", ith, nth, src->ne[0],\n src->ne[1], src->ne[2], src->ne[3], src0_start_row, src0_end_row, dst->ne[0], dst->ne[1], dst->ne[2],\n dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));\n}\n\nstatic int execute_op_unary_f32(struct htp_ops_context * octx) {\n int err = HTP_STATUS_OK;\n\n const struct htp_tensor * src0 = &octx->src0;\n struct htp_tensor * dst = &octx->dst;\n\n const char * op_type = NULL;\n\n switch (octx->op) {\n case HTP_OP_RMS_NORM:\n op_type = \"rmsnorm-f32\";\n break;\n case HTP_OP_SCALE:\n op_type = \"scale-f32\";\n break;\n case HTP_OP_SQR:\n op_type = \"sqr-f32\";\n break;\n case HTP_OP_SQRT:\n op_type = \"sqrt-f32\";\n break;\n\n default:\n FARF(ERROR, \"Unsupported unary Op %u\\n\", octx->op);\n return HTP_STATUS_NO_SUPPORT;\n }\n\n const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];\n const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);\n\n const size_t src0_row_size = src0->nb[1];\n const size_t dst_row_size = dst->nb[1];\n\n const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);\n const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);\n\n // VTCM scratchpads for all tensors\n // N rows per thread, padded to HVX vector size\n // Double buffering requires 2x size per buffer\n\n size_t spad_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned);\n size_t vtcm_row_per_thread = (octx->ctx->vtcm_size)/ (n_threads * spad_size_per_row);\n\n // Make sure the reserved vtcm size is sufficient\n if (vtcm_row_per_thread == 0) {\n FARF(ERROR, \"unary-%s : current VTCM reservation %zu is too small, needed %zu\\n\", op_type, octx->ctx->vtcm_size,\n spad_size_per_row * n_threads);\n return HTP_STATUS_VTCM_TOO_SMALL;\n }\n\n octx->src0_spad.size_per_thread = src0_row_size_aligned * vtcm_row_per_thread * 2;\n octx->dst_spad.size_per_thread = dst_row_size_aligned * vtcm_row_per_thread * 2;\n\n octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;\n octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;\n\n octx->src0_spad.data = octx->ctx->vtcm_base;\n octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size;\n\n FARF(HIGH, \"%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-spad-size %u src1-spad-size %u dst-spad-size %u\\n\", op_type,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size);\n\n if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {\n struct htp_unary_context uctx = {\n .octx = octx,\n .src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads,\n .src0_nrows = src0_nrows,\n\n .data_src0 = (const uint8_t *)src0->data,\n .data_dst = (uint8_t *)dst->data,\n\n .src0_row_size = src0_row_size,\n .dst_row_size = dst_row_size,\n\n .src0_row_size_aligned = src0_row_size_aligned,\n .dst_row_size_aligned = dst_row_size_aligned,\n\n .src0_spad_half_size = octx->src0_spad.size_per_thread / 2,\n .dst_spad_half_size = octx->dst_spad.size_per_thread / 2,\n\n .block = (octx->src0_spad.size_per_thread / 2) / src0_row_size_aligned,\n .nc = src0->ne[0],\n };\n\n worker_pool_run_func(octx->ctx->worker_pool, unary_job_f32_per_thread, &uctx, n_threads);\n }\n\n return err;\n}\n\nint op_unary(struct htp_ops_context * octx) {\n int err = HTP_STATUS_OK;\n\n switch (octx->src0.type) {\n case HTP_TYPE_F32:\n err = execute_op_unary_f32(octx);\n break;\n\n default:\n err = HTP_STATUS_NO_SUPPORT;\n break;\n }\n\n return err;\n}\n" }, { "path": "src/ggml-hexagon/htp/worker-pool.c", "content": "#include \"worker-pool.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"HAP_farf.h\"\n\n#define WORKER_THREAD_STACK_SZ (2 * 16384)\n#define LOWEST_USABLE_QURT_PRIO (254)\n\nstruct worker_pool_s;\n\n// internal structure kept in thread-local storage per instance of worker pool\ntypedef struct {\n struct worker_pool_s * pool;\n unsigned int id;\n} worker_context_t;\n\n// internal structure kept in thread-local storage per instance of worker pool\ntypedef struct worker_pool_s {\n worker_pool_job_t job[MAX_NUM_WORKERS]; // list of job descriptors\n qurt_thread_t thread[MAX_NUM_WORKERS]; // thread ID's of the workers\n worker_context_t context[MAX_NUM_WORKERS]; // worker contexts\n void * stack[MAX_NUM_WORKERS]; // thread stack pointers\n unsigned int n_threads; // number of workers in this pool\n\n atomic_uint seqn; // seqno used to detect new jobs\n atomic_uint next_job; // next job index\n atomic_uint n_pending; // number of pending jobs\n atomic_uint n_jobs; // number of current jobs\n atomic_bool killed; // threads need to exit\n} worker_pool_t;\n\nstatic void worker_pool_main(void * context) {\n worker_context_t * me = (worker_context_t *) context;\n worker_pool_t * pool = me->pool;\n\n FARF(HIGH, \"worker-pool: thread %u started\", me->id);\n\n unsigned int prev_seqn = 0;\n while (!atomic_load(&pool->killed)) {\n unsigned int seqn = atomic_load(&pool->seqn);\n if (seqn == prev_seqn) {\n // Nothing to do\n qurt_futex_wait(&pool->seqn, prev_seqn);\n continue;\n }\n\n // New job\n prev_seqn = seqn;\n\n unsigned int n = atomic_load(&pool->n_jobs);\n unsigned int i = atomic_fetch_add(&pool->next_job, 1);\n if (i >= n) {\n // Spurious wakeup\n continue;\n }\n\n pool->job[i].func(n, i, pool->job[i].data);\n\n atomic_fetch_sub(&pool->n_pending, 1);\n }\n\n FARF(HIGH, \"worker-pool: thread %u stopped\", me->id);\n}\n\nAEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context, uint32_t n_threads, uint32_t stack_size) {\n int err = 0;\n\n if (NULL == context) {\n FARF(ERROR, \"NULL context passed to worker_pool_init().\");\n return AEE_EBADPARM;\n }\n\n // Allocations\n int size = (stack_size * n_threads) + (sizeof(worker_pool_t));\n\n unsigned char * mem_blob = (unsigned char *) malloc(size);\n if (!mem_blob) {\n FARF(ERROR, \"Could not allocate memory for worker pool!!\");\n return AEE_ENOMEMORY;\n }\n\n worker_pool_t * me = (worker_pool_t *) (mem_blob + stack_size * n_threads);\n\n // name for the first worker, useful in debugging threads\n char name[19];\n snprintf(name, 12, \"0x%8x:\", (int) me);\n strcat(name, \"worker0\");\n me->n_threads = n_threads;\n\n // initializations\n for (unsigned int i = 0; i < me->n_threads; i++) {\n me->stack[i] = NULL;\n me->thread[i] = 0;\n\n me->context[i].id = i;\n me->context[i].pool = me;\n }\n\n // initialize job queue\n me->n_pending = 0;\n me->n_jobs = 0;\n me->next_job = 0;\n me->seqn = 0;\n me->killed = 0;\n\n // launch the workers\n qurt_thread_attr_t attr;\n qurt_thread_attr_init(&attr);\n\n for (unsigned int i = 0; i < me->n_threads; i++) {\n // set up stack\n me->stack[i] = mem_blob;\n mem_blob += stack_size;\n qurt_thread_attr_set_stack_addr(&attr, me->stack[i]);\n qurt_thread_attr_set_stack_size(&attr, stack_size);\n\n // set up name\n qurt_thread_attr_set_name(&attr, name);\n name[17] = (name[17] + 1);\n // name threads context:worker0, context:worker1, .. (recycle at 9, but num threads should be less than that anyway)\n if (name[17] > '9') {\n name[17] = '0';\n }\n\n // set up priority - by default, match the creating thread's prio\n int prio = qurt_thread_get_priority(qurt_thread_get_id());\n\n if (prio < 1) {\n prio = 1;\n }\n if (prio > LOWEST_USABLE_QURT_PRIO) {\n prio = LOWEST_USABLE_QURT_PRIO;\n }\n\n qurt_thread_attr_set_priority(&attr, prio);\n\n // launch\n err = qurt_thread_create(&me->thread[i], &attr, worker_pool_main, (void *) &me->context[i]);\n if (err) {\n FARF(ERROR, \"Could not launch worker threads!\");\n worker_pool_release((worker_pool_context_t *) &me);\n return AEE_EQURTTHREADCREATE;\n }\n }\n *context = (worker_pool_context_t *) me;\n return AEE_SUCCESS;\n}\n\nAEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads) {\n return worker_pool_init_with_stack_size(context, n_threads, WORKER_THREAD_STACK_SZ);\n}\n\n// clean up worker pool\nvoid worker_pool_release(worker_pool_context_t * context) {\n worker_pool_t * me = (worker_pool_t *) *context;\n\n // if no worker pool exists, return error.\n if (NULL == me) {\n return;\n }\n\n atomic_store(&me->killed, 1);\n atomic_fetch_add(&me->seqn, 1);\n qurt_futex_wake(&me->seqn, me->n_threads);\n\n // de-initializations\n for (unsigned int i = 0; i < me->n_threads; i++) {\n if (me->thread[i]) {\n int status;\n (void) qurt_thread_join(me->thread[i], &status);\n }\n }\n\n // free allocated memory (were allocated as a single buffer starting at stack[0])\n if (me->stack[0]) {\n free(me->stack[0]);\n }\n\n *context = NULL;\n}\n\n// run jobs\nAEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n) {\n worker_pool_t * me = (worker_pool_t *) context;\n if (NULL == me) {\n FARF(ERROR, \"worker-pool: invalid context\");\n return AEE_EBADPARM;\n }\n\n if (n > me->n_threads) {\n FARF(ERROR, \"worker-pool: invalid number of jobs %u for n-threads %u\", n, me->n_threads);\n return AEE_EBADPARM;\n }\n\n memcpy(me->job, job, sizeof(worker_pool_job_t) * n);\n\n if (n > 1) {\n atomic_store(&me->next_job, 1);\n atomic_store(&me->n_jobs, n);\n atomic_store(&me->n_pending, n - 1);\n\n // wake up workers\n atomic_fetch_add(&me->seqn, 1);\n qurt_futex_wake(&me->seqn, n - 1);\n }\n\n // main thread runs job #0\n me->job[0].func(n, 0, me->job[0].data);\n\n if (n > 1) {\n while (atomic_load(&me->n_pending))\n ;\n }\n\n return 0;\n}\n\n// run func\nAEEResult worker_pool_run_func(worker_pool_context_t context, worker_callback_t func, void * data, unsigned int n) {\n worker_pool_job_t job[n];\n\n for (unsigned int i = 0; i < n; i++) {\n job[i].func = func;\n job[i].data = data;\n }\n\n return worker_pool_run_jobs(context, job, n);\n}\n\nAEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio) {\n worker_pool_t * me = (worker_pool_t *) context;\n\n // if no worker pool exists, return error.\n if (!me) {\n return AEE_ENOMORE;\n }\n\n int result = AEE_SUCCESS;\n if (prio < 1) {\n prio = 1;\n }\n if (prio > LOWEST_USABLE_QURT_PRIO) {\n prio = LOWEST_USABLE_QURT_PRIO;\n }\n\n for (unsigned int i = 0; i < me->n_threads; i++) {\n int res = qurt_thread_set_priority(me->thread[i], (unsigned short) prio);\n if (0 != res) {\n result = AEE_EBADPARM;\n FARF(ERROR, \"QURT failed to set priority of thread %d, ERROR = %d\", me->thread[i], res);\n }\n }\n\n return result;\n}\n\nAEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids) {\n worker_pool_t * me = (worker_pool_t *) context;\n if (!me) {\n FARF(ERROR, \"worker-pool: invalid context\");\n return AEE_EBADPARM;\n ;\n }\n\n for (int i = 0; i < me->n_threads; i++) {\n tids[i] = me->thread[i];\n }\n\n return AEE_SUCCESS;\n}\n\nAEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio) {\n worker_pool_t * me = (worker_pool_t *) context;\n if (!me) {\n FARF(ERROR, \"worker-pool: invalid context\");\n return AEE_EBADPARM;\n }\n\n int priority = qurt_thread_get_priority(me->thread[0]);\n if (priority > 0) {\n *prio = priority;\n return 0;\n } else {\n *prio = 0;\n return AEE_EBADSTATE;\n }\n}\n" }, { "path": "src/ggml-hexagon/htp/worker-pool.h", "content": "#ifndef HTP_WORKER_POOL_H\n#define HTP_WORKER_POOL_H\n\n// MACRO enables function to be visible in shared-library case.\n#define WORKERPOOL_API __attribute__((visibility(\"default\")))\n\n#include \n#include \n#include \n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n/// signature of callbacks to be invoked by worker threads\ntypedef void (*worker_callback_t)(unsigned int n, unsigned int i, void *);\n\n/// Typedef of worker_pool context\ntypedef void * worker_pool_context_t;\n\n/// descriptor for requested callback\ntypedef struct {\n worker_callback_t func;\n void * data;\n} worker_pool_job_t;\n\n/// Maximum supported number of worker threads.\n#define MAX_NUM_WORKERS 10\n\n// Initialize worker pool.\nWORKERPOOL_API AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads);\n\n// Initialize worker pool with custom stack size\nWORKERPOOL_API AEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context,\n uint32_t n_threads,\n uint32_t stack_size);\n\n// Kill worker threads and release worker pool resources\nWORKERPOOL_API void worker_pool_release(worker_pool_context_t * context);\n\n// Run jobs with the worker pool.\nWORKERPOOL_API AEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n);\n\nWORKERPOOL_API AEEResult worker_pool_run_func(worker_pool_context_t context,\n worker_callback_t func,\n void * data,\n unsigned int n);\n\nWORKERPOOL_API AEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio);\nWORKERPOOL_API AEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio);\nWORKERPOOL_API AEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif // #ifndef HTP_WORKER_POOL_H\n" }, { "path": "src/ggml-hexagon/htp-drv.cpp", "content": "// sample drv interface\n\n#pragma clang diagnostic ignored \"-Wgnu-anonymous-struct\"\n#pragma clang diagnostic ignored \"-Wmissing-prototypes\"\n#pragma clang diagnostic ignored \"-Wsign-compare\"\n\n#include \n#include \n#include \n#include \n#ifdef _WIN32\n# define WIN32_LEAN_AND_MEAN\n# ifndef NOMINMAX\n# define NOMINMAX\n# endif\n# include \n# include \n#else\n# include \n# include \n#endif\n#include \"ggml-impl.h\"\n#include \"htp-drv.h\"\n#include \"libdl.h\"\n\n#include \n\n//\n// Driver API types\n//\n\ntypedef void * (*rpcmem_alloc_pfn_t)(int heapid, uint32_t flags, int size);\ntypedef void * (*rpcmem_alloc2_pfn_t)(int heapid, uint32_t flags, size_t size);\ntypedef void (*rpcmem_free_pfn_t)(void * po);\ntypedef int (*rpcmem_to_fd_pfn_t)(void * po);\n\ntypedef AEEResult (*dspqueue_create_pfn_t)(int domain,\n uint32_t flags,\n uint32_t req_queue_size,\n uint32_t resp_queue_size,\n dspqueue_callback_t packet_callback,\n dspqueue_callback_t error_callback,\n void * callback_context,\n dspqueue_t * queue);\ntypedef AEEResult (*dspqueue_close_pfn_t)(dspqueue_t queue);\ntypedef AEEResult (*dspqueue_export_pfn_t)(dspqueue_t queue, uint64_t *queue_id);\ntypedef AEEResult (*dspqueue_write_pfn_t)(dspqueue_t queue, uint32_t flags,\n uint32_t num_buffers,\n struct dspqueue_buffer *buffers,\n uint32_t message_length,\n const uint8_t *message,\n uint32_t timeout_us);\ntypedef AEEResult (*dspqueue_read_pfn_t)(dspqueue_t queue, uint32_t *flags,\n uint32_t max_buffers, uint32_t *num_buffers,\n struct dspqueue_buffer *buffers,\n uint32_t max_message_length,\n uint32_t *message_length, uint8_t *message,\n uint32_t timeout_us);\n\ntypedef int (*fastrpc_mmap_pfn_t)(int domain, int fd, void *addr, int offset, size_t length, enum fastrpc_map_flags flags);\ntypedef int (*fastrpc_munmap_pfn_t)(int domain, int fd, void *addr, size_t length);\n\ntypedef int (*remote_handle64_open_pfn_t)(const char* name, remote_handle64 *ph);\ntypedef int (*remote_handle64_invoke_pfn_t)(remote_handle64 h, uint32_t dwScalars, remote_arg *pra);\ntypedef int (*remote_handle64_close_pfn_t)(remote_handle h);\ntypedef int (*remote_handle_control_pfn_t)(uint32_t req, void* data, uint32_t datalen);\ntypedef int (*remote_handle64_control_pfn_t)(remote_handle64 h, uint32_t req, void* data, uint32_t datalen);\ntypedef int (*remote_session_control_pfn_t)(uint32_t req, void *data, uint32_t datalen);\n\n//\n// Driver API pfns\n//\n\nrpcmem_alloc_pfn_t rpcmem_alloc_pfn = nullptr;\nrpcmem_alloc2_pfn_t rpcmem_alloc2_pfn = nullptr;\nrpcmem_free_pfn_t rpcmem_free_pfn = nullptr;\nrpcmem_to_fd_pfn_t rpcmem_to_fd_pfn = nullptr;\n\nfastrpc_mmap_pfn_t fastrpc_mmap_pfn = nullptr;\nfastrpc_munmap_pfn_t fastrpc_munmap_pfn = nullptr;\n\ndspqueue_create_pfn_t dspqueue_create_pfn = nullptr;\ndspqueue_close_pfn_t dspqueue_close_pfn = nullptr;\ndspqueue_export_pfn_t dspqueue_export_pfn = nullptr;\ndspqueue_write_pfn_t dspqueue_write_pfn = nullptr;\ndspqueue_read_pfn_t dspqueue_read_pfn = nullptr;\n\nremote_handle64_open_pfn_t remote_handle64_open_pfn = nullptr;\nremote_handle64_invoke_pfn_t remote_handle64_invoke_pfn = nullptr;\nremote_handle64_close_pfn_t remote_handle64_close_pfn = nullptr;\nremote_handle_control_pfn_t remote_handle_control_pfn = nullptr;\nremote_handle64_control_pfn_t remote_handle64_control_pfn = nullptr;\nremote_session_control_pfn_t remote_session_control_pfn = nullptr;\n\n//\n// Driver API\n//\n\nvoid * rpcmem_alloc(int heapid, uint32_t flags, int size) {\n return rpcmem_alloc_pfn(heapid, flags, size);\n}\n\nvoid * rpcmem_alloc2(int heapid, uint32_t flags, size_t size) {\n if (rpcmem_alloc2_pfn) {\n return rpcmem_alloc2_pfn(heapid, flags, size);\n } else {\n GGML_LOG_INFO(\"ggml-hex: rpcmem_alloc2 not found, falling back to rpcmem_alloc\\n\");\n return rpcmem_alloc_pfn(heapid, flags, size);\n }\n}\n\nvoid rpcmem_free(void * po) {\n return rpcmem_free_pfn(po);\n}\n\nint rpcmem_to_fd(void * po) {\n return rpcmem_to_fd_pfn(po);\n}\n\nHTPDRV_API int fastrpc_mmap(int domain, int fd, void * addr, int offset, size_t length, enum fastrpc_map_flags flags) {\n return fastrpc_mmap_pfn(domain, fd, addr, offset, length, flags);\n}\n\nHTPDRV_API int fastrpc_munmap(int domain, int fd, void * addr, size_t length) {\n return fastrpc_munmap_pfn(domain, fd, addr, length);\n}\n\nAEEResult dspqueue_create(int domain,\n uint32_t flags,\n uint32_t req_queue_size,\n uint32_t resp_queue_size,\n dspqueue_callback_t packet_callback,\n dspqueue_callback_t error_callback,\n void * callback_context,\n dspqueue_t * queue) {\n return dspqueue_create_pfn(domain, flags, req_queue_size, resp_queue_size, packet_callback, error_callback,\n callback_context, queue);\n}\n\nAEEResult dspqueue_close(dspqueue_t queue) {\n return dspqueue_close_pfn(queue);\n}\n\nAEEResult dspqueue_export(dspqueue_t queue, uint64_t * queue_id) {\n return dspqueue_export_pfn(queue, queue_id);\n}\n\nAEEResult dspqueue_write(dspqueue_t queue,\n uint32_t flags,\n uint32_t num_buffers,\n struct dspqueue_buffer * buffers,\n uint32_t message_length,\n const uint8_t * message,\n uint32_t timeout_us) {\n return dspqueue_write_pfn(queue, flags, num_buffers, buffers, message_length, message, timeout_us);\n}\n\nAEEResult dspqueue_read(dspqueue_t queue,\n uint32_t * flags,\n uint32_t max_buffers,\n uint32_t * num_buffers,\n struct dspqueue_buffer * buffers,\n uint32_t max_message_length,\n uint32_t * message_length,\n uint8_t * message,\n uint32_t timeout_us) {\n return dspqueue_read_pfn(queue, flags, max_buffers, num_buffers, buffers, max_message_length, message_length,\n message, timeout_us);\n}\n\nHTPDRV_API int remote_handle64_open(const char * name, remote_handle64 * ph) {\n return remote_handle64_open_pfn(name, ph);\n}\n\nHTPDRV_API int remote_handle64_invoke(remote_handle64 h, uint32_t dwScalars, remote_arg * pra) {\n return remote_handle64_invoke_pfn(h, dwScalars, pra);\n}\n\nHTPDRV_API int remote_handle64_close(remote_handle64 h) {\n return remote_handle64_close_pfn(h);\n}\n\nHTPDRV_API int remote_handle_control(uint32_t req, void * data, uint32_t datalen) {\n return remote_handle_control_pfn(req, data, datalen);\n}\n\nHTPDRV_API int remote_handle64_control(remote_handle64 h, uint32_t req, void * data, uint32_t datalen) {\n return remote_handle64_control_pfn(h, req, data, datalen);\n}\n\nHTPDRV_API int remote_session_control(uint32_t req, void * data, uint32_t datalen) {\n return remote_session_control_pfn(req, data, datalen);\n}\n\n#ifdef _WIN32\n\nstatic std::string wstr_to_str(std::wstring_view wstr) {\n std::string result;\n if (wstr.empty()) {\n return result;\n }\n auto bytes_needed = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS,\n wstr.data(), (int) wstr.size(),\n nullptr, 0, nullptr, nullptr);\n if (bytes_needed == 0) {\n GGML_LOG_ERROR(\"ggml-hex: WideCharToMultiByte failed. Error %lu\\n\", GetLastError());\n throw std::runtime_error(\"Invalid wstring input\");\n }\n\n result.resize(bytes_needed, '\\0');\n int bytes_written = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS,\n wstr.data(), (int) wstr.size(),\n result.data(), bytes_needed,\n nullptr, nullptr);\n if (bytes_written == 0) {\n GGML_LOG_ERROR(\"ggml-hex: WideCharToMultiByte failed. Error %lu\\n\", GetLastError());\n throw std::runtime_error(\"Wstring conversion failed\");\n }\n return result;\n}\n\nstatic std::string get_driver_path() {\n std::wstring serviceName = L\"qcnspmcdm\";\n std::string result;\n\n // Get a handle to the SCM database.\n SC_HANDLE schSCManager = OpenSCManagerW(NULL, NULL, STANDARD_RIGHTS_READ);\n if (nullptr == schSCManager) {\n GGML_LOG_ERROR(\"ggml-hex: Failed to open SCManager. Error: %lu\\n\", GetLastError());\n return result;\n }\n\n // Get a handle to the service.\n SC_HANDLE schService = OpenServiceW(schSCManager, // SCM database\n serviceName.c_str(), // name of service\n SERVICE_QUERY_CONFIG); // need query config access\n\n if (nullptr == schService) {\n GGML_LOG_ERROR(\"ggml-hex: Failed to open qcnspmcdm service. Error: %lu\\n\", GetLastError());\n CloseServiceHandle(schSCManager);\n return result;\n }\n\n // Store the size of buffer used as an output.\n DWORD bufferSize;\n if (!QueryServiceConfigW(schService, NULL, 0, &bufferSize) &&\n (GetLastError() != ERROR_INSUFFICIENT_BUFFER)) {\n GGML_LOG_ERROR(\"ggml-hex: Failed to query service config. Error: %lu\\n\", GetLastError());\n CloseServiceHandle(schService);\n CloseServiceHandle(schSCManager);\n return result;\n }\n // Get the configuration of the service.\n LPQUERY_SERVICE_CONFIGW serviceConfig =\n static_cast(LocalAlloc(LMEM_FIXED, bufferSize));\n if (!QueryServiceConfigW(schService, serviceConfig, bufferSize, &bufferSize)) {\n fprintf(stderr, \"ggml-hex: Failed to query service config. Error: %lu\\n\", GetLastError());\n LocalFree(serviceConfig);\n CloseServiceHandle(schService);\n CloseServiceHandle(schSCManager);\n return result;\n }\n\n // Read the driver file path get its parent directory\n std::wstring driverPath = std::wstring(serviceConfig->lpBinaryPathName);\n driverPath = driverPath.substr(0, driverPath.find_last_of(L\"\\\\\"));\n\n // Clean up resources\n LocalFree(serviceConfig);\n CloseServiceHandle(schService);\n CloseServiceHandle(schSCManager);\n\n // Driver path would contain invalid path string, like:\n // \\SystemRoot\\System32\\DriverStore\\FileRepository\\qcadsprpc8280.inf_arm64_c2b9460c9a072f37\n // \"\\SystemRoot\" should be replace with a correct one (e.g. C:\\Windows)\n const std::wstring systemRootPlaceholder = L\"\\\\SystemRoot\";\n if (0 != driverPath.compare(0, systemRootPlaceholder.length(), systemRootPlaceholder)) {\n GGML_LOG_ERROR(\"ggml-hex: String pattern not found in driver path.\\n\");\n return result;\n }\n\n // Replace \\SystemRoot with an absolute path from system ENV windir\n const std::wstring systemRootEnv = L\"windir\";\n\n // Query the number of wide characters this variable requires\n DWORD numWords = GetEnvironmentVariableW(systemRootEnv.c_str(), NULL, 0);\n if (numWords == 0) {\n GGML_LOG_ERROR(\"ggml-hex: Failed get systemRoot environment variable\\n\");\n return result;\n }\n\n // Query the actual system root name from environment variable\n std::vector systemRoot(numWords + 1);\n numWords = GetEnvironmentVariableW(systemRootEnv.c_str(), systemRoot.data(), numWords + 1);\n if (numWords == 0) {\n GGML_LOG_ERROR(\"ggml-hex: Failed to read windir environment variable\\n\");\n return result;\n }\n driverPath.replace(0, systemRootPlaceholder.length(), std::wstring(systemRoot.data()));\n\n return wstr_to_str(driverPath);\n}\n\n#endif\n\nusing dl_handle_ptr = std::unique_ptr;\n\nint htpdrv_init() {\n static dl_handle_ptr lib_cdsp_rpc_handle = nullptr;\n static bool initialized = false;\n#ifdef _WIN32\n std::string drv_path = get_driver_path() + \"\\\\\" + \"libcdsprpc.dll\";\n#else\n std::string drv_path = \"libcdsprpc.so\";\n#endif\n if (initialized) {\n GGML_LOG_INFO(\"ggml-hex: Driver already loaded\\n\");\n return AEE_SUCCESS;\n }\n GGML_LOG_INFO(\"ggml-hex: Loading driver %s\\n\", drv_path.c_str());\n\n fs::path path{ drv_path.c_str() };\n dl_handle_ptr handle { dl_load_library(path) };\n if (!handle) {\n GGML_LOG_ERROR(\"ggml-hex: failed to load %s: %s\\n\", path.u8string().c_str(), dl_error());\n return AEE_EUNABLETOLOAD;\n }\n\n#define dlsym(drv, type, pfn, symbol, ignore) \\\n do { \\\n pfn = (type) dl_get_sym(drv, #symbol); \\\n if (!ignore && nullptr == pfn) { \\\n GGML_LOG_ERROR(\"ggml-hex: failed to dlsym %s\\n\", #symbol); \\\n return AEE_EUNABLETOLOAD; \\\n } \\\n } while (0)\n\n dlsym(handle.get(), rpcmem_alloc_pfn_t, rpcmem_alloc_pfn, rpcmem_alloc, false);\n dlsym(handle.get(), rpcmem_alloc2_pfn_t, rpcmem_alloc2_pfn, rpcmem_alloc2, true);\n dlsym(handle.get(), rpcmem_free_pfn_t, rpcmem_free_pfn, rpcmem_free, false);\n dlsym(handle.get(), rpcmem_to_fd_pfn_t, rpcmem_to_fd_pfn, rpcmem_to_fd, false);\n dlsym(handle.get(), fastrpc_mmap_pfn_t, fastrpc_mmap_pfn, fastrpc_mmap, false);\n dlsym(handle.get(), fastrpc_munmap_pfn_t, fastrpc_munmap_pfn, fastrpc_munmap, false);\n dlsym(handle.get(), dspqueue_create_pfn_t, dspqueue_create_pfn, dspqueue_create, false);\n dlsym(handle.get(), dspqueue_close_pfn_t, dspqueue_close_pfn, dspqueue_close, false);\n dlsym(handle.get(), dspqueue_export_pfn_t, dspqueue_export_pfn, dspqueue_export, false);\n dlsym(handle.get(), dspqueue_write_pfn_t, dspqueue_write_pfn, dspqueue_write, false);\n dlsym(handle.get(), dspqueue_read_pfn_t, dspqueue_read_pfn, dspqueue_read, false);\n dlsym(handle.get(), remote_handle64_open_pfn_t, remote_handle64_open_pfn, remote_handle64_open, false);\n dlsym(handle.get(), remote_handle64_invoke_pfn_t, remote_handle64_invoke_pfn, remote_handle64_invoke, false);\n dlsym(handle.get(), remote_handle_control_pfn_t, remote_handle_control_pfn, remote_handle_control, false);\n dlsym(handle.get(), remote_handle64_control_pfn_t, remote_handle64_control_pfn, remote_handle64_control, false);\n dlsym(handle.get(), remote_session_control_pfn_t, remote_session_control_pfn, remote_session_control, false);\n dlsym(handle.get(), remote_handle64_close_pfn_t, remote_handle64_close_pfn, remote_handle64_close, false);\n\n lib_cdsp_rpc_handle = std::move(handle);\n initialized = true;\n\n return AEE_SUCCESS;\n}\n\ndomain * get_domain(int domain_id) {\n int i = 0;\n int size = sizeof(supported_domains) / sizeof(domain);\n\n for (i = 0; i < size; i++) {\n if (supported_domains[i].id == domain_id) {\n return &supported_domains[i];\n }\n }\n\n return NULL;\n}\n\nint get_hex_arch_ver(int domain, int * arch) {\n if (!remote_handle_control_pfn) {\n GGML_LOG_ERROR(\"ggml-hex: remote_handle_control is not supported on this device\\n\");\n return AEE_EUNSUPPORTEDAPI;\n }\n\n struct remote_dsp_capability arch_ver;\n arch_ver.domain = (uint32_t) domain;\n arch_ver.attribute_ID = ARCH_VER;\n arch_ver.capability = (uint32_t) 0;\n\n int err = remote_handle_control(DSPRPC_GET_DSP_INFO, &arch_ver, sizeof(arch_ver));\n if ((err & 0xff) == (AEE_EUNSUPPORTEDAPI & 0xff)) {\n GGML_LOG_ERROR(\"ggml-hex: FastRPC capability API is not supported on this device\\n\");\n return AEE_EUNSUPPORTEDAPI;\n }\n\n if (err != AEE_SUCCESS) {\n GGML_LOG_ERROR(\"ggml-hex: FastRPC capability query failed (err %d)\\n\", err);\n return err;\n }\n\n switch (arch_ver.capability & 0xff) {\n case 0x68:\n *arch = 68;\n return 0;\n case 0x69:\n *arch = 69;\n return 0;\n case 0x73:\n *arch = 73;\n return 0;\n case 0x75:\n *arch = 75;\n return 0;\n case 0x79:\n *arch = 79;\n return 0;\n case 0x81:\n *arch = 81;\n return 0;\n }\n return -1;\n}\n" }, { "path": "src/ggml-hexagon/htp-drv.h", "content": "#pragma once\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#ifdef _WIN32\n# pragma clang diagnostic ignored \"-Wignored-attributes\"\n#endif\n\n#include \n#include \n#include \n#include \n\n#if defined(_WIN32) && !defined(__MINGW32__)\n# ifdef GGML_BACKEND_BUILD\n# define HTPDRV_API __declspec(dllexport) extern\n# else\n# define HTPDRV_API __declspec(dllimport) extern\n# endif\n#else\n# define HTPDRV_API __attribute__ ((visibility (\"default\"))) extern\n#endif\n\n/* Offset to differentiate HLOS and Hexagon error codes.\n Stores the value of AEE_EOFFSET for Hexagon. */\n#ifndef DSP_OFFSET\n# define DSP_OFFSET 0x80000400\n#endif\n\n/* Errno for connection reset by peer. */\n#ifndef ECONNRESET\n# ifdef __hexagon__\n# define ECONNRESET 104\n# endif\n#endif\n\n/* Abstraction of different OS specific sleep APIs.\n SLEEP accepts input in seconds. */\n#ifndef SLEEP\n# ifdef __hexagon__\n# define SLEEP(x) \\\n { /* Do nothing for simulator. */ \\\n }\n# else\n# ifdef _WIN32\n# define SLEEP(x) Sleep(1000 * x) /* Sleep accepts input in milliseconds. */\n# else\n# define SLEEP(x) sleep(x) /* sleep accepts input in seconds. */\n# endif\n# endif\n#endif\n\n/* Include windows specific header files. */\n#ifdef _WIN32\n# include \n# include \n# define _CRT_SECURE_NO_WARNINGS 1\n# define _WINSOCK_DEPRECATED_NO_WARNINGS 1\n#endif\n\n/* Includes and defines for all HLOS except windows */\n#if !defined(__hexagon__) && !defined(_WIN32)\n# include \"unistd.h\"\n\n# include \n#endif\n\n/* Includes and defines for Hexagon and all HLOS except Windows. */\n#if !defined(_WIN32)\n/* Weak reference to remote symbol for compilation. */\n# pragma weak remote_session_control\n# pragma weak remote_handle_control\n# pragma weak remote_handle64_control\n# pragma weak fastrpc_mmap\n# pragma weak fastrpc_munmap\n# pragma weak rpcmem_alloc2\n#endif\n\n#if !defined(_WIN32)\n# pragma weak remote_system_request\n#endif\n\n#ifdef _WIN32\n# define DSPQUEUE_TIMEOUT DSPQUEUE_TIMEOUT_NONE\n#else\n# define DSPQUEUE_TIMEOUT 1000000\n#endif\n\n/**\n * htpdrv_init API: driver interface entry point\n *\n * @return Return AEE error codes as defined in Hexagon SDK.\n */\nHTPDRV_API int htpdrv_init(void);\n\n/**\n * get_domain API: get domain struct from domain value.\n *\n * @param[in] domain value of a domain\n * @return Returns domain struct of the domain if it is supported or else\n * returns NULL.\n *\n */\nHTPDRV_API domain * get_domain(int domain_id);\n\n/**\n * get_hex_arch_ver API: query the Hexagon processor architecture version information\n *\n * @param[in] domain_id value of a domain\n * @param[out] Arch version (73, 75, ...)\n * @return 0 if query is successful.\n * non-zero if error, return value points to the error.\n *\n */\nHTPDRV_API int get_hex_arch_ver(int domain, int * arch);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-hexagon/libdl.h", "content": "#pragma once\n\n#ifdef _WIN32\n# define WIN32_LEAN_AND_MEAN\n# ifndef NOMINMAX\n# define NOMINMAX\n# endif\n# include \n# include \n#else\n# include \n# include \n#endif\n#include \n\nnamespace fs = std::filesystem;\n\n#ifdef _WIN32\n\nusing dl_handle = std::remove_pointer_t;\n\nstruct dl_handle_deleter {\n void operator()(HMODULE handle) {\n FreeLibrary(handle);\n }\n};\n\nstatic inline dl_handle * dl_load_library(const fs::path & path) {\n // suppress error dialogs for missing DLLs\n DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);\n SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);\n\n HMODULE handle = LoadLibraryW(path.wstring().c_str());\n\n SetErrorMode(old_mode);\n\n return handle;\n}\n\nstatic inline void * dl_get_sym(dl_handle * handle, const char * name) {\n DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);\n SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);\n\n void * p = (void *) GetProcAddress(handle, name);\n\n SetErrorMode(old_mode);\n\n return p;\n}\n\nstatic inline const char * dl_error() {\n return \"\";\n}\n\n#else\n\nusing dl_handle = void;\n\nstruct dl_handle_deleter {\n void operator()(void * handle) {\n dlclose(handle);\n }\n};\n\nstatic inline dl_handle * dl_load_library(const fs::path & path) {\n dl_handle * handle = dlopen(path.string().c_str(), RTLD_NOW | RTLD_LOCAL);\n return handle;\n}\n\nstatic inline void * dl_get_sym(dl_handle * handle, const char * name) {\n return dlsym(handle, name);\n}\n\nstatic inline const char * dl_error() {\n const char *rslt = dlerror();\n return rslt != nullptr ? rslt : \"\";\n}\n\n#endif\n" }, { "path": "src/ggml-hexagon/libggml-htp.inf", "content": "[Version]\nSignature = \"$WINDOWS NT$\"\nClass = ComputeAccelerator\nClassGuid = {F01A9D53-3FF6-48D2-9F97-C8A7004BE10C}\nProvider = %GGML%\nDriverVer = 01/01/2026,1.0.0.0\nCatalogFile = libggml-htp.cat\nPnpLockDown = 1\n\n[DestinationDirs]\nDrivers_Dir = 6\n\n[SourceDisksNames]\n1 = %DiskId%\n\n[SourceDisksFiles]\nlibggml-htp-v68.so = 1\nlibggml-htp-v69.so = 1\nlibggml-htp-v73.so = 1\nlibggml-htp-v75.so = 1\nlibggml-htp-v81.so = 1\n\n[ControlFlags]\nExcludeFromSelect = *\n\n[DefaultInstall.NTarm64]\nCopyFiles=Drivers_Dir\n\n[Drivers_Dir]\nlibggml-htp-v68.so,,,0x10 ;COPYFLG_NO_OVERWRITE\nlibggml-htp-v69.so,,,0x10 ;COPYFLG_NO_OVERWRITE\nlibggml-htp-v73.so,,,0x10 ;COPYFLG_NO_OVERWRITE\nlibggml-htp-v75.so,,,0x10 ;COPYFLG_NO_OVERWRITE\nlibggml-htp-v81.so,,,0x10 ;COPYFLG_NO_OVERWRITE\n\n[Strings]\nGGML = 'GGML'\nDiskId = 'GGML HTP library'\n" }, { "path": "src/ggml-hexagon/op-desc.h", "content": "#ifndef OP_DESC_H\n#define OP_DESC_H\n\n#define GGML_COMMON_IMPL_CPP\n#include \"ggml-backend-impl.h\"\n#include \"ggml-common.h\"\n\n#include \n#include \n\nstruct op_desc {\n char strides[64 * GGML_MAX_SRC];\n char dims[64 * GGML_MAX_SRC];\n char types[16 * GGML_MAX_SRC];\n char buffs[64 * GGML_MAX_SRC];\n char names[64 * GGML_MAX_SRC];\n\n int format_tensor_dims(char * str, const struct ggml_tensor * t) {\n if (t->ne[2] == 1 && t->ne[3] == 1) {\n return sprintf(str, \"%d:%d\", (int) t->ne[0], (int) t->ne[1]);\n } else {\n return sprintf(str, \"%d:%d:%d:%d\", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]);\n }\n }\n\n void format_op_dims(char * str, const struct ggml_tensor * t) {\n char * p = str;\n\n // append src0 and src1 (if any)\n if (t->src[0]) {\n p += format_tensor_dims(p, t->src[0]);\n\n for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) {\n p += sprintf(p, \" x \");\n p += format_tensor_dims(p, t->src[i]);\n }\n\n p += sprintf(p, \" -> \");\n }\n\n // format self dims separately for better visual alignment\n char self[64];\n format_tensor_dims(self, t);\n\n p += sprintf(p, \"%s\", self);\n }\n\n int format_tensor_strides(char * str, const struct ggml_tensor * t) {\n const char * c = ggml_is_contiguous(t) ? \"\" : \"!\";\n\n if (t->ne[2] == 1 && t->ne[3] == 1) {\n return sprintf(str, \"%zu:%zu%s\", (size_t) t->nb[0], (size_t) t->nb[1], c);\n } else {\n return sprintf(str, \"%zu:%zu:%zu:%zu%s\", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c);\n }\n }\n\n void format_op_strides(char * str, const struct ggml_tensor * t) {\n char * p = str;\n\n // append src0 and src1 (if any)\n if (t->src[0]) {\n p += format_tensor_strides(p, t->src[0]);\n\n for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) {\n p += sprintf(p, \" x \");\n p += format_tensor_strides(p, t->src[i]);\n }\n\n p += sprintf(p, \" -> \");\n }\n\n // format self dims separately for better visual alignment\n char self[64];\n format_tensor_strides(self, t);\n\n p += sprintf(p, \"%s\", self);\n }\n\n void format_op_types(char * str, const struct ggml_tensor * t) {\n char * p = str;\n\n // append src0 and src1 (if any)\n if (t->src[0]) {\n p += sprintf(p, \"%s\", ggml_type_name(t->src[0]->type));\n\n for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) {\n p += sprintf(p, \" x \");\n p += sprintf(p, \"%s\", ggml_type_name(t->src[i]->type));\n }\n\n p += sprintf(p, \" -> \");\n }\n\n p += sprintf(p, \"%s\", ggml_type_name(t->type));\n }\n\n const char * tensor_buff_name(const struct ggml_tensor * t) {\n if (t->buffer) {\n return ggml_backend_buffer_name(t->buffer);\n }\n return \"NONE\";\n }\n\n void format_op_buffs(char * str, const struct ggml_tensor * t) {\n char * p = str;\n\n // append src0 and src1 (if any)\n if (t->src[0]) {\n p += sprintf(p, \"%s\", tensor_buff_name(t->src[0]));\n\n for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) {\n p += sprintf(p, \" x \");\n p += sprintf(p, \"%s\", tensor_buff_name(t->src[i]));\n }\n\n p += sprintf(p, \" -> \");\n }\n\n p += sprintf(p, \"%s\", tensor_buff_name(t));\n }\n\n void format_op_names(char * str, const struct ggml_tensor * t) {\n char * p = str;\n\n // append src0 and src1 (if any)\n if (t->src[0]) {\n p += sprintf(p, \"%s\", t->src[0]->name);\n\n for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) {\n p += sprintf(p, \" x \");\n p += sprintf(p, \"%s\", t->src[i]->name);\n }\n\n p += sprintf(p, \" -> \");\n }\n\n p += sprintf(p, \"%s\", t->name);\n }\n\n void format(const ggml_tensor * op) {\n format_op_dims(dims, op);\n format_op_strides(strides, op);\n format_op_types(types, op);\n format_op_buffs(buffs, op);\n format_op_names(names, op);\n }\n\n op_desc() {}\n op_desc(const ggml_tensor * op) { format(op); }\n};\n\n#endif // OP_DESC_H\n" }, { "path": "src/ggml-hip/CMakeLists.txt", "content": "if (NOT EXISTS $ENV{ROCM_PATH})\n if (NOT EXISTS /opt/rocm)\n set(ROCM_PATH /usr)\n else()\n set(ROCM_PATH /opt/rocm)\n endif()\nelse()\n set(ROCM_PATH $ENV{ROCM_PATH})\nendif()\n\nlist(APPEND CMAKE_PREFIX_PATH ${ROCM_PATH})\nlist(APPEND CMAKE_PREFIX_PATH \"${ROCM_PATH}/lib64/cmake\")\n\nif (NOT DEFINED CMAKE_HIP_FLAGS_DEBUG)\n set(CMAKE_HIP_FLAGS_DEBUG \"-g -O2\")\nendif()\n\n# CMake on Windows doesn't support the HIP language yet\nif (WIN32)\n set(CXX_IS_HIPCC TRUE)\nelse()\n string(REGEX MATCH \"hipcc(\\.bat)?$\" CXX_IS_HIPCC \"${CMAKE_CXX_COMPILER}\")\nendif()\n\nif (CXX_IS_HIPCC)\n if (LINUX)\n if (NOT ${CMAKE_CXX_COMPILER_ID} MATCHES \"Clang\")\n message(WARNING \"Only LLVM is supported for HIP, hint: CXX=/opt/rocm/llvm/bin/clang++\")\n endif()\n\n message(WARNING \"Setting hipcc as the C++ compiler is legacy behavior.\"\n \" Prefer setting the HIP compiler directly. See README for details.\")\n endif()\nelse()\n # Forward (AMD)GPU_TARGETS to CMAKE_HIP_ARCHITECTURES.\n if(AMDGPU_TARGETS AND NOT GPU_TARGETS)\n set(GPU_TARGETS ${AMDGPU_TARGETS})\n endif()\n if(GPU_TARGETS AND NOT CMAKE_HIP_ARCHITECTURES)\n set(CMAKE_HIP_ARCHITECTURES ${GPU_TARGETS})\n endif()\n cmake_minimum_required(VERSION 3.21)\n enable_language(HIP)\nendif()\n\nfind_package(hip REQUIRED)\nfind_package(hipblas REQUIRED)\nfind_package(rocblas REQUIRED)\n\nif (${hip_VERSION} VERSION_LESS 6.1)\n message(FATAL_ERROR \"At least ROCM/HIP V6.1 is required\")\nendif()\n\nmessage(STATUS \"HIP and hipBLAS found\")\n\n# Workaround old compilers\nset(CMAKE_HIP_FLAGS \"${CMAKE_HIP_FLAGS} --gpu-max-threads-per-block=1024\")\n\nfile(GLOB GGML_HEADERS_ROCM \"../ggml-cuda/*.cuh\")\nlist(APPEND GGML_HEADERS_ROCM \"../../include/ggml-cuda.h\")\n\nfile(GLOB GGML_SOURCES_ROCM \"../ggml-cuda/*.cu\")\nfile(GLOB SRCS \"../ggml-cuda/template-instances/fattn-tile*.cu\")\nlist(APPEND GGML_SOURCES_ROCM ${SRCS})\nfile(GLOB SRCS \"../ggml-cuda/template-instances/fattn-mma*.cu\")\nlist(APPEND GGML_SOURCES_ROCM ${SRCS})\nfile(GLOB SRCS \"../ggml-cuda/template-instances/mmq*.cu\")\nlist(APPEND GGML_SOURCES_ROCM ${SRCS})\nfile(GLOB SRCS \"../ggml-cuda/template-instances/mmf*.cu\")\nlist(APPEND GGML_SOURCES_ROCM ${SRCS})\n\nif (GGML_CUDA_FA_ALL_QUANTS)\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-vec*.cu\")\n list(APPEND GGML_SOURCES_ROCM ${SRCS})\n add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS)\nelse()\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-vec*q4_0-q4_0.cu\")\n list(APPEND GGML_SOURCES_ROCM ${SRCS})\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-vec*q8_0-q8_0.cu\")\n list(APPEND GGML_SOURCES_ROCM ${SRCS})\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-vec*f16-f16.cu\")\n list(APPEND GGML_SOURCES_ROCM ${SRCS})\nendif()\n\nggml_add_backend_library(ggml-hip\n ${GGML_HEADERS_ROCM}\n ${GGML_SOURCES_ROCM}\n )\n\n# TODO: do not use CUDA definitions for HIP\nif (NOT GGML_BACKEND_DL)\n target_compile_definitions(ggml PUBLIC GGML_USE_CUDA)\nendif()\n\nadd_compile_definitions(GGML_USE_HIP)\n\nif (GGML_CUDA_FORCE_MMQ)\n add_compile_definitions(GGML_CUDA_FORCE_MMQ)\nendif()\n\nif (GGML_CUDA_FORCE_CUBLAS)\n add_compile_definitions(GGML_CUDA_FORCE_CUBLAS)\nendif()\n\nif (GGML_CUDA_NO_PEER_COPY)\n add_compile_definitions(GGML_CUDA_NO_PEER_COPY)\nendif()\n\nif (GGML_HIP_GRAPHS)\n add_compile_definitions(GGML_HIP_GRAPHS)\nendif()\n\nif (GGML_HIP_NO_VMM)\n add_compile_definitions(GGML_HIP_NO_VMM)\nendif()\n\nif (GGML_HIP_ROCWMMA_FATTN)\n add_compile_definitions(GGML_HIP_ROCWMMA_FATTN)\nendif()\n\nif (NOT GGML_HIP_MMQ_MFMA)\n add_compile_definitions(GGML_HIP_NO_MMQ_MFMA)\nendif()\n\nif (GGML_HIP_EXPORT_METRICS)\n set(CMAKE_HIP_FLAGS \"${CMAKE_HIP_FLAGS} -Rpass-analysis=kernel-resource-usage --save-temps\")\nendif()\n\nif (NOT GGML_CUDA_FA)\n add_compile_definitions(GGML_CUDA_NO_FA)\nendif()\n\nif (CXX_IS_HIPCC)\n set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE CXX)\n target_link_libraries(ggml-hip PRIVATE hip::device)\nelse()\n set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE HIP)\nendif()\n\nif (GGML_STATIC)\n message(FATAL_ERROR \"Static linking not supported for HIP/ROCm\")\nendif()\n\ntarget_link_libraries(ggml-hip PRIVATE ggml-base hip::host roc::rocblas roc::hipblas)\n" }, { "path": "src/ggml-impl.h", "content": "#pragma once\n\n// GGML internal header\n\n#include \"ggml.h\"\n#include \"gguf.h\"\n\n#include \n#include \n#include // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/\n#include \n#include \n#include \n\n#ifdef __ARM_FEATURE_SVE\n#include \n#endif // __ARM_FEATURE_SVE\n\n#if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__)\n// if YCM cannot find , make a symbolic link to it, for example:\n//\n// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/\n//\n#include \n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nvoid ggml_print_backtrace(void);\n\n#ifndef MIN\n# define MIN(a, b) ((a) < (b) ? (a) : (b))\n#endif\n\n#ifndef MAX\n# define MAX(a, b) ((a) > (b) ? (a) : (b))\n#endif\n\n// required for mmap as gguf only guarantees 32-byte alignment\n#define TENSOR_ALIGNMENT 32\n\n// static_assert should be a #define, but if it's not,\n// fall back to the _Static_assert C11 keyword.\n// if C99 - static_assert is noop\n// ref: https://stackoverflow.com/a/53923785/4039976\n#ifndef __cplusplus\n #ifndef static_assert\n #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)\n #define static_assert(cond, msg) _Static_assert(cond, msg)\n #else\n #define static_assert(cond, msg) struct global_scope_noop_trick\n #endif\n #endif\n#endif\n\nstatic inline int ggml_up32(int n) {\n return (n + 31) & ~31;\n}\n\n//static inline int ggml_up64(int n) {\n// return (n + 63) & ~63;\n//}\n\nstatic inline int ggml_up(int n, int m) {\n // assert m is a power of 2\n GGML_ASSERT((m & (m - 1)) == 0);\n return (n + m - 1) & ~(m - 1);\n}\n\n// TODO: move to ggml.h? (won't be able to inline)\nstatic bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {\n if (a->type != b->type) {\n return false;\n }\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n if (a->ne[i] != b->ne[i]) {\n return false;\n }\n if (a->nb[i] != b->nb[i]) {\n return false;\n }\n }\n return true;\n}\n\nstatic bool ggml_op_is_empty(enum ggml_op op) {\n switch (op) {\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_TRANSPOSE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n return true;\n default:\n return false;\n }\n}\n\nstatic inline bool ggml_impl_is_view(const struct ggml_tensor * t) {\n return t->view_src != NULL;\n}\n\nstatic inline float ggml_compute_softplus_f32(float input) {\n return (input > 20.0f) ? input : logf(1 + expf(input));\n}\n//\n// logging\n//\n\nGGML_ATTRIBUTE_FORMAT(2, 3)\nGGML_API void ggml_log_internal (enum ggml_log_level level, const char * format, ...);\nGGML_API void ggml_log_callback_default(enum ggml_log_level level, const char * text, void * user_data);\n\n#define GGML_LOG(...) ggml_log_internal(GGML_LOG_LEVEL_NONE , __VA_ARGS__)\n#define GGML_LOG_INFO(...) ggml_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)\n#define GGML_LOG_WARN(...) ggml_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)\n#define GGML_LOG_ERROR(...) ggml_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)\n#define GGML_LOG_DEBUG(...) ggml_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)\n#define GGML_LOG_CONT(...) ggml_log_internal(GGML_LOG_LEVEL_CONT , __VA_ARGS__)\n\n#define GGML_DEBUG 0\n\n#if (GGML_DEBUG >= 1)\n#define GGML_PRINT_DEBUG(...) GGML_LOG_DEBUG(__VA_ARGS__)\n#else\n#define GGML_PRINT_DEBUG(...)\n#endif\n\n#if (GGML_DEBUG >= 5)\n#define GGML_PRINT_DEBUG_5(...) GGML_LOG_DEBUG(__VA_ARGS__)\n#else\n#define GGML_PRINT_DEBUG_5(...)\n#endif\n\n#if (GGML_DEBUG >= 10)\n#define GGML_PRINT_DEBUG_10(...) GGML_LOG_DEBUG(__VA_ARGS__)\n#else\n#define GGML_PRINT_DEBUG_10(...)\n#endif\n\n// tensor params\n\nstatic void ggml_set_op_params(struct ggml_tensor * tensor, const void * params, size_t params_size) {\n GGML_ASSERT(tensor != NULL); // silence -Warray-bounds warnings\n assert(params_size <= GGML_MAX_OP_PARAMS);\n memcpy(tensor->op_params, params, params_size);\n}\n\nstatic int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) {\n assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));\n return ((const int32_t *)(tensor->op_params))[i];\n}\n\nstatic float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t i) {\n assert(i < GGML_MAX_OP_PARAMS / sizeof(float));\n return ((const float *)(tensor->op_params))[i];\n}\n\nstatic void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) {\n assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));\n ((int32_t *)(tensor->op_params))[i] = value;\n}\n\nstatic void ggml_set_op_params_f32(struct ggml_tensor * tensor, uint32_t i, float value) {\n assert(i < GGML_MAX_OP_PARAMS / sizeof(float));\n ((float *)(tensor->op_params))[i] = value;\n}\n\nstruct ggml_map_custom1_op_params {\n ggml_custom1_op_t fun;\n int n_tasks;\n void * userdata;\n};\n\nstruct ggml_map_custom2_op_params {\n ggml_custom2_op_t fun;\n int n_tasks;\n void * userdata;\n};\n\nstruct ggml_map_custom3_op_params {\n ggml_custom3_op_t fun;\n int n_tasks;\n void * userdata;\n};\n\nstruct ggml_custom_op_params {\n ggml_custom_op_t fun;\n int n_tasks;\n void * userdata;\n};\n\n// bitset\n\ntypedef uint32_t ggml_bitset_t;\n\nstatic_assert(sizeof(ggml_bitset_t) == 4, \"bitset_t constants must be updated\");\n#define BITSET_SHR 5 // log2(sizeof(ggml_bitset_t)*8)\n#define BITSET_MASK (sizeof(ggml_bitset_t)*8 - 1)\n\nstatic size_t ggml_bitset_size(size_t n) {\n return (n + BITSET_MASK) >> BITSET_SHR;\n}\n\nstatic inline bool ggml_bitset_get(const ggml_bitset_t * bitset, size_t i) {\n return !!(bitset[i >> BITSET_SHR] & (1u << (i & BITSET_MASK)));\n}\n\nstatic inline void ggml_bitset_set(ggml_bitset_t * bitset, size_t i) {\n bitset[i >> BITSET_SHR] |= (1u << (i & BITSET_MASK));\n}\n\nstatic inline void ggml_bitset_clear(ggml_bitset_t * bitset, size_t i) {\n bitset[i >> BITSET_SHR] &= ~(1u << (i & BITSET_MASK));\n}\n\n// hash set\n\n#define GGML_HASHSET_FULL ((size_t)-1)\n#define GGML_HASHSET_ALREADY_EXISTS ((size_t)-2)\n\nstruct ggml_hash_set {\n size_t size;\n ggml_bitset_t * used; // whether or not the keys are in use i.e. set\n struct ggml_tensor ** keys; // actual tensors in the set, keys[i] is only defined if ggml_bitset_get(used, i)\n};\n\nstruct ggml_hash_set ggml_hash_set_new(size_t size);\nvoid ggml_hash_set_free(struct ggml_hash_set * hash_set);\n\n// returns the minimum size for a hash set that can hold min_sz elements\nsize_t ggml_hash_size(size_t min_sz);\n\n// remove all elements from the hash set\nvoid ggml_hash_set_reset(struct ggml_hash_set * hash_set);\n\n// returns true if key is in the hash set\nstatic bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key);\n\n// returns GGML_HASHSET_FULL if table is full, otherwise the current index of the key or where it should be inserted\nstatic size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key);\n\n// returns GGML_HASHSET_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full\nstatic size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);\n\n// return index, asserts if table is full\nstatic size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);\n\n// hash function for ggml_tensor\nstatic inline size_t ggml_hash(const struct ggml_tensor * p) {\n // the last 4 bits are always zero due to alignment\n return (size_t)(uintptr_t)p >> 4;\n}\n\nstatic size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key) {\n size_t h = ggml_hash(key) % hash_set->size;\n\n // linear probing\n size_t i = h;\n while (ggml_bitset_get(hash_set->used, i) && hash_set->keys[i] != key) {\n i = (i + 1) % hash_set->size;\n if (i == h) {\n // visited all hash table entries -> not found\n return GGML_HASHSET_FULL;\n }\n }\n return i;\n}\n\nstatic bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key) {\n size_t i = ggml_hash_find(hash_set, key);\n return i != GGML_HASHSET_FULL && ggml_bitset_get(hash_set->used, i);\n}\n\nstatic size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {\n size_t h = ggml_hash(key) % hash_set->size;\n\n // linear probing\n size_t i = h;\n do {\n if (!ggml_bitset_get(hash_set->used, i)) {\n ggml_bitset_set(hash_set->used, i);\n hash_set->keys[i] = key;\n return i;\n }\n if (hash_set->keys[i] == key) {\n return GGML_HASHSET_ALREADY_EXISTS;\n }\n i = (i + 1) % hash_set->size;\n } while (i != h);\n\n // visited all hash table entries -> not found\n GGML_ABORT(\"fatal error\");\n}\n\nstatic size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {\n size_t h = ggml_hash(key) % hash_set->size;\n\n // linear probing\n size_t i = h;\n do {\n if (!ggml_bitset_get(hash_set->used, i)) {\n ggml_bitset_set(hash_set->used, i);\n hash_set->keys[i] = key;\n return i;\n }\n if (hash_set->keys[i] == key) {\n return i;\n }\n i = (i + 1) % hash_set->size;\n } while (i != h);\n\n // visited all hash table entries -> not found\n GGML_ABORT(\"fatal error\");\n}\n\n// computation graph\n\nenum ggml_cgraph_eval_order {\n GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,\n GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,\n GGML_CGRAPH_EVAL_ORDER_COUNT\n};\n\nstruct ggml_cgraph {\n int size; // maximum number of nodes/leafs/grads/grad_accs\n int n_nodes; // number of nodes currently in use\n int n_leafs; // number of leafs currently in use\n\n struct ggml_tensor ** nodes; // tensors with data that can change if the graph is evaluated\n struct ggml_tensor ** grads; // the outputs of these tensors are the gradients of the nodes\n struct ggml_tensor ** grad_accs; // accumulators for node gradients\n struct ggml_tensor ** leafs; // tensors with constant data\n int32_t * use_counts;// number of uses of each tensor, indexed by hash table slot\n\n struct ggml_hash_set visited_hash_set;\n\n enum ggml_cgraph_eval_order order;\n};\n\n// returns a slice of cgraph with nodes [i0, i1)\n// the slice does not have leafs or gradients\n// if you need the gradients, get them from the original graph\nstruct ggml_cgraph ggml_graph_view(struct ggml_cgraph * cgraph, int i0, int i1);\n\n// ggml-alloc.c: true if the operation can reuse memory from its sources\nGGML_API bool ggml_op_can_inplace(enum ggml_op op);\n\n\n// Memory allocation\n\nGGML_API void * ggml_aligned_malloc(size_t size);\nGGML_API void ggml_aligned_free(void * ptr, size_t size);\n\n// FP16 <-> FP32\n// ref: https://github.com/Maratyszcza/FP16\n\nstatic inline float fp32_from_bits(uint32_t w) {\n union {\n uint32_t as_bits;\n float as_value;\n } fp32;\n fp32.as_bits = w;\n return fp32.as_value;\n}\n\nstatic inline uint32_t fp32_to_bits(float f) {\n union {\n float as_value;\n uint32_t as_bits;\n } fp32;\n fp32.as_value = f;\n return fp32.as_bits;\n}\n\nstatic inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {\n const uint32_t w = (uint32_t) h << 16;\n const uint32_t sign = w & UINT32_C(0x80000000);\n const uint32_t two_w = w + w;\n\n const uint32_t exp_offset = UINT32_C(0xE0) << 23;\n#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L)\n const float exp_scale = 0x1.0p-112f;\n#else\n const float exp_scale = fp32_from_bits(UINT32_C(0x7800000));\n#endif\n const float normalized_value = fp32_from_bits((two_w >> 4) + exp_offset) * exp_scale;\n\n const uint32_t magic_mask = UINT32_C(126) << 23;\n const float magic_bias = 0.5f;\n const float denormalized_value = fp32_from_bits((two_w >> 17) | magic_mask) - magic_bias;\n\n const uint32_t denormalized_cutoff = UINT32_C(1) << 27;\n const uint32_t result = sign |\n (two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value));\n return fp32_from_bits(result);\n}\n\nstatic inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {\n#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L)\n const float scale_to_inf = 0x1.0p+112f;\n const float scale_to_zero = 0x1.0p-110f;\n#else\n const float scale_to_inf = fp32_from_bits(UINT32_C(0x77800000));\n const float scale_to_zero = fp32_from_bits(UINT32_C(0x08800000));\n#endif\n float base = (fabsf(f) * scale_to_inf) * scale_to_zero;\n\n const uint32_t w = fp32_to_bits(f);\n const uint32_t shl1_w = w + w;\n const uint32_t sign = w & UINT32_C(0x80000000);\n uint32_t bias = shl1_w & UINT32_C(0xFF000000);\n if (bias < UINT32_C(0x71000000)) {\n bias = UINT32_C(0x71000000);\n }\n\n base = fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base;\n const uint32_t bits = fp32_to_bits(base);\n const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00);\n const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF);\n const uint32_t nonsign = exp_bits + mantissa_bits;\n return (sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign);\n}\n\n#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)\n#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)\n\n#define GGML_FP16_TO_FP32(x) GGML_COMPUTE_FP16_TO_FP32(x)\n#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)\n\nstatic inline float ggml_e8m0_to_fp32(uint8_t x) {\n uint32_t bits; // Stores the raw bit representation of the float\n\n // Handle special case for minimum exponent (denormalized float)\n if (x == 0) {\n // Bit pattern for 2^(-127):\n // - Sign bit: 0 (positive)\n // - Exponent: 0 (denormalized number)\n // - Mantissa: 0x400000 (0.5 in fractional form)\n // Value = 0.5 * 2^(-126) = 2^(-127)\n bits = 0x00400000;\n }\n // note: disabled as we don't need to handle NaNs\n //// Handle special case for NaN (all bits set)\n //else if (x == 0xFF) {\n // // Standard quiet NaN pattern:\n // // - Sign bit: 0\n // // - Exponent: all 1s (0xFF)\n // // - Mantissa: 0x400000 (quiet NaN flag)\n // bits = 0x7FC00000;\n //}\n // Normalized values (most common case)\n else {\n // Construct normalized float by shifting exponent into position:\n // - Exponent field: 8 bits (positions 30-23)\n // - Mantissa: 0 (implicit leading 1)\n // Value = 2^(x - 127)\n bits = (uint32_t) x << 23;\n }\n\n float result; // Final float value\n // Safely reinterpret bit pattern as float without type-punning issues\n memcpy(&result, &bits, sizeof(float));\n return result;\n}\n\n// Equal to ggml_e8m0_to_fp32/2\n// Useful with MXFP4 quantization since the E0M2 values are doubled\nstatic inline float ggml_e8m0_to_fp32_half(uint8_t x) {\n uint32_t bits;\n\n // For x < 2: use precomputed denormal patterns\n if (x < 2) {\n // 0x00200000 = 2^(-128), 0x00400000 = 2^(-127)\n bits = 0x00200000 << x;\n }\n // For x >= 2: normalized exponent adjustment\n else {\n // 0.5 * 2^(x-127) = 2^(x-128) = normalized with exponent (x-1)\n bits = (uint32_t)(x - 1) << 23;\n }\n // Note: NaNs are not handled here\n\n float result;\n memcpy(&result, &bits, sizeof(float));\n return result;\n}\n\n#define GGML_E8M0_TO_FP32(x) ggml_e8m0_to_fp32(x)\n#define GGML_E8M0_TO_FP32_HALF(x) ggml_e8m0_to_fp32_half(x)\n\n// UE4M3: unsigned, 4 exp bits (bias=7), 3 mantissa bits\n// Returns value * 0.5 to match kvalues_mxfp4 convention (kvalues = 2 * E2M1_float)\nstatic inline float ggml_ue4m3_to_fp32(uint8_t x) {\n if (x == 0 || x == 0x7F) {\n return 0.0f;\n }\n int exp = (x >> 3) & 0xF;\n int man = x & 0x7;\n float raw;\n if (exp == 0) {\n raw = ldexpf((float) man, -9);\n } else {\n raw = ldexpf(1.0f + (float) man / 8.0f, exp - 7);\n }\n return raw * 0.5f;\n}\n\nstatic inline uint8_t ggml_fp32_to_ue4m3(float x) {\n if (!(x > 0.0f)) {\n return 0;\n }\n if (x > 448.0f) {\n x = 448.0f;\n }\n uint32_t bits;\n memcpy(&bits, &x, 4);\n int fp32_exp = ((bits >> 23) & 0xFF) - 127;\n int fp32_man = (bits >> 20) & 0x7;\n int ue4m3_exp = fp32_exp + 7;\n if (ue4m3_exp <= 0) {\n // subnormal: value = man * 2^-9, man = round(x * 2^9)\n int man = (int) (x * 512.0f + 0.5f);\n if (man > 7) {\n man = 7;\n }\n if (man < 1) {\n return 0;\n }\n return (uint8_t) man;\n }\n if (ue4m3_exp >= 15) {\n return 0x7E;\n }\n int round_bit = (bits >> 19) & 1;\n int ue4m3_man = fp32_man + round_bit;\n if (ue4m3_man > 7) {\n ue4m3_man = 0;\n ue4m3_exp++;\n if (ue4m3_exp >= 15) {\n return 0x7E;\n }\n }\n return (uint8_t) ((ue4m3_exp << 3) | ue4m3_man);\n}\n\n/**\n * Converts brain16 to float32.\n *\n * The bfloat16 floating point format has the following structure:\n *\n * ┌sign\n * │\n * │ ┌exponent\n * │ │\n * │ │ ┌mantissa\n * │ │ │\n * │┌──┴───┐┌─┴───┐\n * 0b0000000000000000 brain16\n *\n * Since bf16 has the same number of exponent bits as a 32bit float,\n * encoding and decoding numbers becomes relatively straightforward.\n *\n * ┌sign\n * │\n * │ ┌exponent\n * │ │\n * │ │ ┌mantissa\n * │ │ │\n * │┌──┴───┐┌─┴───────────────────┐\n * 0b00000000000000000000000000000000 IEEE binary32\n *\n * For comparison, the standard fp16 format has fewer exponent bits.\n *\n * ┌sign\n * │\n * │ ┌exponent\n * │ │\n * │ │ ┌mantissa\n * │ │ │\n * │┌─┴─┐┌─┴──────┐\n * 0b0000000000000000 IEEE binary16\n *\n * @see IEEE 754-2008\n */\nstatic inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) {\n union {\n float f;\n uint32_t i;\n } u;\n u.i = (uint32_t)h.bits << 16;\n return u.f;\n}\n\n/**\n * Converts float32 to brain16.\n *\n * This is binary identical with Google Brain float conversion.\n * Floats shall round to nearest even, and NANs shall be quiet.\n * Subnormals aren't flushed to zero, except perhaps when used.\n * This code should vectorize nicely if using modern compilers.\n */\nstatic inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {\n ggml_bf16_t h;\n union {\n float f;\n uint32_t i;\n } u;\n u.f = s;\n if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */\n h.bits = (u.i >> 16) | 64; /* force to quiet */\n return h;\n }\n h.bits = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;\n return h;\n}\n\n#define GGML_FP32_TO_BF16(x) ggml_compute_fp32_to_bf16(x)\n#define GGML_BF16_TO_FP32(x) ggml_compute_bf16_to_fp32(x)\n\nstatic inline int32_t ggml_node_get_use_count(const struct ggml_cgraph * cgraph, int node_idx) {\n const struct ggml_tensor * node = cgraph->nodes[node_idx];\n\n size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, node);\n if (!ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) {\n return 0;\n }\n return cgraph->use_counts[hash_pos];\n}\n\n// return true if the node's results are only used by N other nodes\n// and can be fused into their calculations.\nstatic inline bool ggml_node_has_n_uses(const struct ggml_cgraph * cgraph, int node_idx, int32_t n_uses) {\n const struct ggml_tensor * node = cgraph->nodes[node_idx];\n\n // check the use count against how many we're replacing\n if (ggml_node_get_use_count(cgraph, node_idx) != n_uses) {\n return false;\n }\n\n // if node is a view, some other node might be using the intermediate result\n // via the view source.\n if (node->view_src) {\n return false;\n }\n\n // If the user requested output for the node, can't fuse\n if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {\n return false;\n }\n\n return true;\n}\n\n// Returns true if nodes with indices { node_idxs } are the sequence of ggml_ops in ops[]\n// and are fusable. Nodes are considered fusable according to this function if:\n// - all nodes except the last have only one use and are not views/outputs (see ggml_node_has_N_uses).\n// - all nodes except the last are a src of the following node.\n// - all nodes are the same shape.\n// TODO: Consider allowing GGML_OP_NONE nodes in between\nstatic inline bool ggml_can_fuse_ext(const struct ggml_cgraph * cgraph, const int * node_idxs, const enum ggml_op * ops, int num_ops) {\n for (int i = 0; i < num_ops; ++i) {\n if (node_idxs[i] >= cgraph->n_nodes) {\n return false;\n }\n\n struct ggml_tensor * node = cgraph->nodes[node_idxs[i]];\n if (node->op != ops[i]) {\n return false;\n }\n if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n return false;\n }\n if (i < num_ops - 1 && !ggml_node_has_n_uses(cgraph, node_idxs[i], 1)) {\n return false;\n }\n if (i > 0) {\n struct ggml_tensor * prev = cgraph->nodes[node_idxs[i - 1]];\n if (node->src[0] != prev && node->src[1] != prev) {\n return false;\n }\n if (!ggml_are_same_shape(node, prev)) {\n return false;\n }\n }\n }\n return true;\n}\n\n// same as above, for sequential indices starting at node_idx\nstatic inline bool ggml_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, const enum ggml_op * ops, int num_ops) {\n assert(num_ops < 32);\n\n if (node_idx + num_ops > cgraph->n_nodes) {\n return false;\n }\n\n int idxs[32];\n for (int i = 0; i < num_ops; ++i) {\n idxs[i] = node_idx + i;\n }\n\n return ggml_can_fuse_ext(cgraph, idxs, ops, num_ops);\n}\n\nGGML_API bool ggml_can_fuse_subgraph_ext(const struct ggml_cgraph * cgraph,\n const int * node_idxs,\n int count,\n const enum ggml_op * ops,\n const int * outputs,\n int num_outputs);\n\n// Returns true if the subgraph formed by {node_idxs} can be fused\n// checks whethers all nodes which are not part of outputs can be elided\n// by checking if their num_uses are confined to the subgraph\nstatic inline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph,\n int node_idx,\n int count,\n const enum ggml_op * ops,\n const int * outputs,\n int num_outputs) {\n GGML_ASSERT(count < 32);\n if (node_idx + count > cgraph->n_nodes) {\n return false;\n }\n\n int idxs[32];\n\n for (int i = 0; i < count; ++i) {\n idxs[i] = node_idx + i;\n }\n\n return ggml_can_fuse_subgraph_ext(cgraph, idxs, count, ops, outputs, num_outputs);\n}\n\n#ifdef __cplusplus\n}\n#endif\n\n#ifdef __cplusplus\n#include \n#include \n#include \n\n// nicer C++ syntax for ggml_can_fuse\ninline bool ggml_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) {\n return ggml_can_fuse(cgraph, node_idx, ops.begin(), (int)ops.size());\n}\n\ninline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph,\n int start_idx,\n std::initializer_list ops,\n std::initializer_list outputs = {}) {\n return ggml_can_fuse_subgraph(cgraph, start_idx, ops.size(), ops.begin(), outputs.begin(), outputs.size());\n}\n\n// Return true if the edges in the graph match expectations.\ninline bool ggml_check_edges(const struct ggml_cgraph * cgraph,\n int start_idx,\n std::initializer_list> edges) {\n for (const auto & edge : edges) {\n int dst_node = edge[0];\n int src_idx = edge[1];\n int src_node = edge[2];\n if (cgraph->nodes[start_idx + dst_node]->src[src_idx] != cgraph->nodes[start_idx + src_node]) {\n return false;\n }\n }\n return true;\n}\n\n// expose GGUF internals for test code\nGGML_API size_t gguf_type_size(enum gguf_type type);\nGGML_API struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_params params);\nGGML_API void gguf_write_to_buf(const struct gguf_context * ctx, std::vector & buf, bool only_meta);\n#endif // __cplusplus\n" }, { "path": "src/ggml-metal/CMakeLists.txt", "content": "find_library(FOUNDATION_LIBRARY Foundation REQUIRED)\nfind_library(METAL_FRAMEWORK Metal REQUIRED)\nfind_library(METALKIT_FRAMEWORK MetalKit REQUIRED)\n\nmessage(STATUS \"Metal framework found\")\n\nggml_add_backend_library(ggml-metal\n ggml-metal.cpp\n ggml-metal-device.m\n ggml-metal-device.cpp\n ggml-metal-common.cpp\n ggml-metal-context.m\n ggml-metal-ops.cpp\n )\n\ntarget_link_libraries(ggml-metal PRIVATE\n ${FOUNDATION_LIBRARY}\n ${METAL_FRAMEWORK}\n ${METALKIT_FRAMEWORK}\n )\n\nif (GGML_METAL_NDEBUG)\n add_compile_definitions(GGML_METAL_NDEBUG)\nendif()\n\nset(METALLIB_COMMON \"${CMAKE_CURRENT_SOURCE_DIR}/../ggml-common.h\")\nif (GGML_METAL_EMBED_LIBRARY)\n enable_language(ASM)\n\n add_compile_definitions(GGML_METAL_EMBED_LIBRARY)\n\n set(METALLIB_SOURCE \"${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal\")\n set(METALLIB_IMPL \"${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal-impl.h\")\n\n file(MAKE_DIRECTORY \"${CMAKE_CURRENT_BINARY_DIR}/autogenerated\")\n\n # merge ggml-common.h and ggml-metal.metal into a single file\n set(METALLIB_EMBED_ASM \"${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed.s\")\n set(METALLIB_SOURCE_EMBED \"${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed.metal\")\n set(METALLIB_SOURCE_EMBED_TMP \"${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed.metal.tmp\")\n\n add_custom_command(\n OUTPUT \"${METALLIB_EMBED_ASM}\"\n COMMAND echo \"Embedding Metal library\"\n COMMAND sed -e \"/__embed_ggml-common.h__/r ${METALLIB_COMMON}\" -e \"/__embed_ggml-common.h__/d\" < \"${METALLIB_SOURCE}\" > \"${METALLIB_SOURCE_EMBED_TMP}\"\n COMMAND sed -e \"/\\#include \\\"ggml-metal-impl.h\\\"/r ${METALLIB_IMPL}\" -e \"/\\#include \\\"ggml-metal-impl.h\\\"/d\" < \"${METALLIB_SOURCE_EMBED_TMP}\" > \"${METALLIB_SOURCE_EMBED}\"\n COMMAND echo \".section __DATA,__ggml_metallib\" > \"${METALLIB_EMBED_ASM}\"\n COMMAND echo \".globl _ggml_metallib_start\" >> \"${METALLIB_EMBED_ASM}\"\n COMMAND echo \"_ggml_metallib_start:\" >> \"${METALLIB_EMBED_ASM}\"\n COMMAND echo .incbin \"\\\"${METALLIB_SOURCE_EMBED}\\\"\" >> \"${METALLIB_EMBED_ASM}\"\n COMMAND echo \".globl _ggml_metallib_end\" >> \"${METALLIB_EMBED_ASM}\"\n COMMAND echo \"_ggml_metallib_end:\" >> \"${METALLIB_EMBED_ASM}\"\n DEPENDS ../ggml-common.h ggml-metal.metal ggml-metal-impl.h\n COMMENT \"Generate assembly for embedded Metal library\"\n VERBATIM\n )\n\n target_sources(ggml-metal PRIVATE \"${METALLIB_EMBED_ASM}\")\nelse()\n # copy metal files to bin directory\n configure_file(../ggml-common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COPYONLY)\n configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)\n configure_file(ggml-metal-impl.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h COPYONLY)\n\n if (GGML_METAL_SHADER_DEBUG)\n # custom command to do the following:\n # xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air\n # xcrun -sdk macosx metallib ggml-metal.air -o default.metallib\n #\n # note: this is the only way I found to disable fast-math in Metal. it's ugly, but at least it works\n # disabling fast math is needed in order to pass tests/test-backend-ops\n # note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1\n # note: unfortunately, we have to call it default.metallib instead of ggml.metallib\n # ref: https://github.com/ggml-org/whisper.cpp/issues/1720\n # note: adding -g causes segmentation fault during compile\n #set(XC_FLAGS -fno-fast-math -fno-inline -g)\n set(XC_FLAGS -fno-fast-math -fno-inline)\n else()\n set(XC_FLAGS -O3)\n endif()\n\n # Append macOS metal versioning flags\n if (GGML_METAL_MACOSX_VERSION_MIN)\n message(STATUS \"Adding -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN} flag to metal compilation\")\n list (APPEND XC_FLAGS -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN})\n endif()\n\n if (GGML_METAL_STD)\n message(STATUS \"Adding -std=${GGML_METAL_STD} flag to metal compilation\")\n list (APPEND XC_FLAGS -std=${GGML_METAL_STD})\n endif()\n\n add_custom_command(\n OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib\n COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o - |\n xcrun -sdk macosx metallib - -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib\n COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h\n COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal\n DEPENDS ggml-metal.metal ${METALLIB_COMMON}\n COMMENT \"Compiling Metal kernels\"\n )\n\n # FIXME: only add to the ggml-metal target?\n add_custom_target(\n ggml-metal-lib ALL\n DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib\n )\nendif() # GGML_METAL_EMBED_LIBRARY\n\nif (NOT GGML_METAL_EMBED_LIBRARY)\n install(\n FILES src/ggml-metal/ggml-metal.metal\n PERMISSIONS\n OWNER_READ\n OWNER_WRITE\n GROUP_READ\n WORLD_READ\n DESTINATION ${CMAKE_INSTALL_BINDIR})\n\n install(\n FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib\n DESTINATION ${CMAKE_INSTALL_BINDIR}\n )\nendif()\n" }, { "path": "src/ggml-metal/ggml-metal-common.cpp", "content": "#include \"ggml-metal-common.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \n\n// represents a memory range (i.e. an interval from a starting address p0 to an ending address p1 in a given buffer pb)\n// the type indicates whether it is a source range (i.e. ops read data from it) or a destination range (i.e. ops write data to it)\nstruct ggml_mem_range {\n uint64_t pb; // buffer id\n\n uint64_t p0; // begin\n uint64_t p1; // end\n\n ggml_mem_range_type pt;\n};\n\nstruct ggml_mem_ranges {\n std::vector ranges;\n\n int debug = 0;\n};\n\nggml_mem_ranges_t ggml_mem_ranges_init(int debug) {\n auto * res = new ggml_mem_ranges;\n\n res->ranges.reserve(256);\n res->debug = debug;\n\n return res;\n}\n\nvoid ggml_mem_ranges_free(ggml_mem_ranges_t mrs) {\n delete mrs;\n}\n\nvoid ggml_mem_ranges_reset(ggml_mem_ranges_t mrs) {\n mrs->ranges.clear();\n}\n\nstatic bool ggml_mem_ranges_add(ggml_mem_ranges_t mrs, ggml_mem_range mr) {\n mrs->ranges.push_back(mr);\n\n return true;\n}\n\nstatic ggml_mem_range ggml_mem_range_from_tensor(const ggml_tensor * tensor, ggml_mem_range_type pt) {\n // always use the base tensor\n tensor = tensor->view_src ? tensor->view_src : tensor;\n\n GGML_ASSERT(!tensor->view_src);\n\n ggml_mem_range mr;\n\n if (tensor->buffer) {\n // when the tensor is allocated, use the actual memory address range in the buffer\n //\n // take the actual allocated size with ggml_backend_buft_get_alloc_size()\n // this can be larger than the tensor size if the buffer type allocates extra memory\n // ref: https://github.com/ggml-org/llama.cpp/pull/15966\n mr = {\n /*.pb =*/ (uint64_t) tensor->buffer,\n /*.p0 =*/ (uint64_t) tensor->data,\n /*.p1 =*/ (uint64_t) tensor->data + ggml_backend_buft_get_alloc_size(tensor->buffer->buft, tensor),\n /*.pt =*/ pt,\n };\n } else {\n // otherwise, the pointer address is used as an unique id of the memory ranges\n // that the tensor will be using when it is allocated\n mr = {\n /*.pb =*/ (uint64_t) tensor,\n /*.p0 =*/ 0, //\n /*.p1 =*/ 1024, // [0, 1024) is a dummy range, not used\n /*.pt =*/ pt,\n };\n };\n\n return mr;\n}\n\nstatic ggml_mem_range ggml_mem_range_from_tensor_src(const ggml_tensor * tensor) {\n return ggml_mem_range_from_tensor(tensor, MEM_RANGE_TYPE_SRC);\n}\n\nstatic ggml_mem_range ggml_mem_range_from_tensor_dst(const ggml_tensor * tensor) {\n return ggml_mem_range_from_tensor(tensor, MEM_RANGE_TYPE_DST);\n}\n\nstatic bool ggml_mem_ranges_add_src(ggml_mem_ranges_t mrs, const ggml_tensor * tensor) {\n GGML_ASSERT(tensor);\n\n ggml_mem_range mr = ggml_mem_range_from_tensor_src(tensor);\n\n if (mrs->debug > 2) {\n GGML_LOG_DEBUG(\"%s: add src range buf=%lld, [%lld, %lld)\\n\", __func__, mr.pb, mr.p0, mr.p1);\n }\n\n return ggml_mem_ranges_add(mrs, mr);\n}\n\nstatic bool ggml_mem_ranges_add_dst(ggml_mem_ranges_t mrs, const ggml_tensor * tensor) {\n GGML_ASSERT(tensor);\n\n ggml_mem_range mr = ggml_mem_range_from_tensor_dst(tensor);\n\n if (mrs->debug > 2) {\n GGML_LOG_DEBUG(\"%s: add dst range buf=%lld, [%lld, %lld)\\n\", __func__, mr.pb, mr.p0, mr.p1);\n }\n\n return ggml_mem_ranges_add(mrs, mr);\n}\n\nbool ggml_mem_ranges_add(ggml_mem_ranges_t mrs, const ggml_tensor * tensor) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (tensor->src[i]) {\n ggml_mem_ranges_add_src(mrs, tensor->src[i]);\n }\n }\n\n return ggml_mem_ranges_add_dst(mrs, tensor);\n}\n\nstatic bool ggml_mem_ranges_check(ggml_mem_ranges_t mrs, ggml_mem_range mr) {\n for (size_t i = 0; i < mrs->ranges.size(); i++) {\n const auto & cmp = mrs->ranges[i];\n\n // two memory ranges cannot intersect if they are in different buffers\n if (mr.pb != cmp.pb) {\n continue;\n }\n\n // intersecting source ranges are allowed\n if (mr.pt == MEM_RANGE_TYPE_SRC && cmp.pt == MEM_RANGE_TYPE_SRC) {\n continue;\n }\n\n if (mr.p0 < cmp.p1 && mr.p1 >= cmp.p0) {\n if (mrs->debug > 2) {\n GGML_LOG_DEBUG(\"%s: the %s range buf=%lld, [%lld, %lld) overlaps with a previous %s range buf=%lld, [%lld, %lld)\\n\",\n __func__,\n mr.pt == MEM_RANGE_TYPE_SRC ? \"src\" : \"dst\",\n mr.pb, mr.p0, mr.p1,\n cmp.pt == MEM_RANGE_TYPE_SRC ? \"src\" : \"dst\",\n cmp.pb, cmp.p0, cmp.p1);\n }\n\n return false;\n }\n }\n\n return true;\n}\n\nstatic bool ggml_mem_ranges_check_src(ggml_mem_ranges_t mrs, const ggml_tensor * tensor) {\n GGML_ASSERT(tensor);\n\n ggml_mem_range mr = ggml_mem_range_from_tensor_src(tensor);\n\n const bool res = ggml_mem_ranges_check(mrs, mr);\n\n return res;\n}\n\nstatic bool ggml_mem_ranges_check_dst(ggml_mem_ranges_t mrs, const ggml_tensor * tensor) {\n GGML_ASSERT(tensor);\n\n ggml_mem_range mr = ggml_mem_range_from_tensor_dst(tensor);\n\n const bool res = ggml_mem_ranges_check(mrs, mr);\n\n return res;\n}\n\nbool ggml_mem_ranges_check(ggml_mem_ranges_t mrs, const ggml_tensor * tensor) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (tensor->src[i]) {\n if (!ggml_mem_ranges_check_src(mrs, tensor->src[i])) {\n return false;\n }\n }\n }\n\n return ggml_mem_ranges_check_dst(mrs, tensor);\n}\n\nstruct node_info {\n ggml_tensor * node;\n\n std::vector fused;\n\n ggml_op op() const {\n return node->op;\n }\n\n const ggml_tensor * dst() const {\n return fused.empty() ? node : fused.back();\n }\n\n bool is_empty() const {\n return ggml_op_is_empty(node->op);\n }\n\n void add_fused(ggml_tensor * t) {\n fused.push_back(t);\n }\n};\n\nstatic std::vector ggml_metal_graph_optimize_reorder(const std::vector & nodes) {\n // helper to add node src and dst ranges\n const auto & h_add = [](ggml_mem_ranges_t mrs, const node_info & node) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (node.node->src[i]) {\n if (!ggml_mem_ranges_add_src(mrs, node.node->src[i])) {\n return false;\n }\n }\n }\n\n // keep track of the sources of the fused nodes as well\n for (const auto * fused : node.fused) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (fused->src[i]) {\n if (!ggml_mem_ranges_add_src(mrs, fused->src[i])) {\n return false;\n }\n }\n }\n }\n\n return ggml_mem_ranges_add_dst(mrs, node.dst());\n };\n\n // helper to check if a node can run concurrently with the existing set of nodes\n const auto & h_check = [](ggml_mem_ranges_t mrs, const node_info & node) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (node.node->src[i]) {\n if (!ggml_mem_ranges_check_src(mrs, node.node->src[i])) {\n return false;\n }\n }\n }\n\n for (const auto * fused : node.fused) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (fused->src[i]) {\n if (!ggml_mem_ranges_check_src(mrs, fused->src[i])) {\n return false;\n }\n }\n }\n }\n\n return ggml_mem_ranges_check_dst(mrs, node.dst());\n };\n\n // perform reorders only across these types of ops\n // can be expanded when needed\n const auto & h_safe = [](ggml_op op) {\n switch (op) {\n case GGML_OP_MUL_MAT:\n case GGML_OP_MUL_MAT_ID:\n case GGML_OP_ROPE:\n case GGML_OP_NORM:\n case GGML_OP_RMS_NORM:\n case GGML_OP_GROUP_NORM:\n case GGML_OP_L2_NORM:\n case GGML_OP_SUM_ROWS:\n case GGML_OP_SSM_CONV:\n case GGML_OP_SSM_SCAN:\n case GGML_OP_CLAMP:\n case GGML_OP_TRI:\n case GGML_OP_DIAG:\n case GGML_OP_MUL:\n case GGML_OP_ADD:\n case GGML_OP_SUB:\n case GGML_OP_DIV:\n case GGML_OP_GLU:\n case GGML_OP_SCALE:\n case GGML_OP_UNARY:\n case GGML_OP_GET_ROWS:\n case GGML_OP_SET_ROWS:\n case GGML_OP_SET:\n case GGML_OP_CPY:\n case GGML_OP_CONT:\n case GGML_OP_REPEAT:\n return true;\n default:\n return ggml_op_is_empty(op);\n }\n };\n\n const int n = nodes.size();\n\n std::vector res;\n res.reserve(n);\n\n std::vector used(n, false);\n\n // the memory ranges for the set of currently concurrent nodes\n ggml_mem_ranges_t mrs0 = ggml_mem_ranges_init(0);\n\n // the memory ranges for the set of nodes that haven't been processed yet, when looking forward for a node to reorder\n ggml_mem_ranges_t mrs1 = ggml_mem_ranges_init(0);\n\n for (int i0 = 0; i0 < n; i0++) {\n if (used[i0]) {\n continue;\n }\n\n const auto & node0 = nodes[i0];\n\n // the node is not concurrent with the existing concurrent set, so we have to \"put a barrier\" (i.e reset mrs0)\n // but before we do that, look forward for some other nodes that can be added to the concurrent set mrs0\n //\n // note: we can always add empty nodes to the concurrent set as they don't read nor write anything\n if (!node0.is_empty() && !h_check(mrs0, node0)) {\n // this will hold the set of memory ranges from the nodes that haven't been processed yet\n // if a node is not concurrent with this set, we cannot reorder it\n ggml_mem_ranges_reset(mrs1);\n\n // initialize it with the current node\n h_add(mrs1, node0);\n\n // that many nodes forward to search for a concurrent node\n constexpr int N_FORWARD = 64;\n\n for (int i1 = i0 + 1; i1 < i0 + N_FORWARD && i1 < n; i1++) {\n if (used[i1]) {\n continue;\n }\n\n const auto & node1 = nodes[i1];\n\n // disallow reordering of certain ops\n if (!h_safe(node1.op())) {\n break;\n }\n\n const bool is_empty = node1.is_empty();\n\n // to reorder a node and add it to the concurrent set, it has to be:\n // + empty or concurrent with all nodes in the existing concurrent set (mrs0)\n // + concurrent with all nodes prior to it that haven't been processed yet (mrs1)\n if ((is_empty || h_check(mrs0, node1)) && h_check(mrs1, node1)) {\n // add the node to the existing concurrent set (i.e. reorder it for early execution)\n h_add(mrs0, node1);\n res.push_back(i1);\n\n // mark as used, so we skip re-processing it later\n used[i1] = true;\n } else {\n // expand the set of nodes that haven't been processed yet\n h_add(mrs1, node1);\n }\n }\n\n // finalize the concurrent set and begin a new one\n ggml_mem_ranges_reset(mrs0);\n }\n\n // expand the concurrent set with the current node\n {\n h_add(mrs0, node0);\n res.push_back(i0);\n }\n }\n\n ggml_mem_ranges_free(mrs0);\n ggml_mem_ranges_free(mrs1);\n\n return res;\n}\n\nvoid ggml_graph_optimize(ggml_cgraph * gf) {\n constexpr int MAX_FUSE = 16;\n\n const int n = gf->n_nodes;\n\n enum ggml_op ops[MAX_FUSE];\n\n std::vector nodes;\n nodes.reserve(gf->n_nodes);\n\n // fuse nodes:\n // we don't want to make reorders that break fusing, so we first pack all fusable tensors\n // and perform the reorder over the fused nodes. after the reorder is done, we unfuse\n for (int i = 0; i < n; i++) {\n node_info node = {\n /*.node =*/ gf->nodes[i],\n /*.fused =*/ {},\n };\n\n // fuse only ops that start with these operations\n // can be expanded when needed\n if (node.op() == GGML_OP_ADD ||\n node.op() == GGML_OP_NORM ||\n node.op() == GGML_OP_RMS_NORM) {\n ops[0] = node.op();\n\n int f = i + 1;\n while (f < n && f < i + MAX_FUSE) {\n // conservatively allow fusing only these ops\n // can be expanded when needed\n if (gf->nodes[f]->op != GGML_OP_ADD &&\n gf->nodes[f]->op != GGML_OP_MUL &&\n gf->nodes[f]->op != GGML_OP_NORM &&\n gf->nodes[f]->op != GGML_OP_RMS_NORM) {\n break;\n }\n ops[f - i] = gf->nodes[f]->op;\n f++;\n }\n\n f -= i;\n for (; f > 1; f--) {\n if (ggml_can_fuse(gf, i, ops, f)) {\n break;\n }\n }\n\n // add the fused tensors into the node info so we can unfuse them later\n for (int k = 1; k < f; k++) {\n ++i;\n\n // the .dst() becomes the last fused tensor\n node.add_fused(gf->nodes[i]);\n }\n }\n\n nodes.push_back(std::move(node));\n }\n\n#if 1\n // reorder to improve concurrency\n const auto order = ggml_metal_graph_optimize_reorder(nodes);\n#else\n std::vector order(nodes.size());\n for (size_t i = 0; i < nodes.size(); i++) {\n order[i] = i;\n }\n#endif\n\n // unfuse\n {\n int j = 0;\n for (const auto i : order) {\n const auto & node = nodes[i];\n\n gf->nodes[j++] = node.node;\n\n for (auto * fused : node.fused) {\n gf->nodes[j++] = fused;\n }\n }\n }\n}\n" }, { "path": "src/ggml-metal/ggml-metal-common.h", "content": "// helper functions for ggml-metal that are too difficult to implement in Objective-C\n\n#pragma once\n\n#include \n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nstruct ggml_tensor;\nstruct ggml_cgraph;\n\nenum ggml_mem_range_type {\n MEM_RANGE_TYPE_SRC = 0,\n MEM_RANGE_TYPE_DST = 1,\n};\n\n// a helper object that can be used for reordering operations to improve concurrency\n//\n// the fundamental idea is that a set of tasks (either ggml ops, or something else) can run concurrently if they\n// don't write to a memory that is being read by another task or written to by another task in the set\n//\n// with this structure, we can add tasks to the set, setting memory constraints. we can also check if a new task\n// can be added to the set without violating the constraints (i.e. if it can be executed concurrently with the\n// tasks already in the set)\n//\ntypedef struct ggml_mem_ranges * ggml_mem_ranges_t;\n\nggml_mem_ranges_t ggml_mem_ranges_init(int debug);\nvoid ggml_mem_ranges_free(ggml_mem_ranges_t mrs);\n\n// remove all ranges from the set\nvoid ggml_mem_ranges_reset(ggml_mem_ranges_t mrs);\n\n// add src or dst ranges to track\nbool ggml_mem_ranges_add(ggml_mem_ranges_t mrs, const struct ggml_tensor * tensor);\n\n// return false if:\n// - new src range overlaps with any existing dst range\n// - new dst range overlaps with any existing range (src or dst)\nbool ggml_mem_ranges_check(ggml_mem_ranges_t mrs, const struct ggml_tensor * tensor);\n\n// reorder the nodes in the graph to improve concurrency, while respecting fusion\n//\n// note: this implementation is generic and not specific to metal\n// if it proves to work well, we can start using it for other backends in the future\nvoid ggml_graph_optimize(struct ggml_cgraph * gf);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-metal/ggml-metal-context.h", "content": "#pragma once\n\n#include \"ggml-metal-device.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n//\n// backend context\n//\n\ntypedef struct ggml_metal * ggml_metal_t;\n\nggml_metal_t ggml_metal_init(ggml_metal_device_t dev);\nvoid ggml_metal_free(ggml_metal_t ctx);\n\nconst char * ggml_metal_get_name(ggml_metal_t ctx);\n\nvoid ggml_metal_synchronize(ggml_metal_t ctx);\n\nvoid ggml_metal_set_tensor_async(ggml_metal_t ctx, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\nvoid ggml_metal_get_tensor_async(ggml_metal_t ctx, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);\nbool ggml_metal_cpy_tensor_async(ggml_metal_t ctx_src, ggml_metal_t ctx_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);\n\nenum ggml_status ggml_metal_graph_compute (ggml_metal_t ctx, struct ggml_cgraph * gf);\nvoid ggml_metal_graph_optimize(ggml_metal_t ctx, struct ggml_cgraph * gf);\n\nvoid ggml_metal_event_record(ggml_metal_t ctx, ggml_metal_event_t ev);\nvoid ggml_metal_event_wait (ggml_metal_t ctx, ggml_metal_event_t ev);\n\nggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx);\n\nvoid ggml_metal_set_n_cb (ggml_metal_t ctx, int n_cb);\nvoid ggml_metal_set_abort_callback (ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data);\nbool ggml_metal_supports_family (ggml_metal_t ctx, int family);\nvoid ggml_metal_capture_next_compute(ggml_metal_t ctx);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-metal/ggml-metal-context.m", "content": "#import \"ggml-metal-context.h\"\n\n#import \"ggml-impl.h\"\n#import \"ggml-backend-impl.h\"\n\n#import \"ggml-metal-impl.h\"\n#import \"ggml-metal-common.h\"\n#import \"ggml-metal-ops.h\"\n\n#import \n\n#import \n\n#undef MIN\n#undef MAX\n#define MIN(a, b) ((a) < (b) ? (a) : (b))\n#define MAX(a, b) ((a) > (b) ? (a) : (b))\n\n// max number of MTLCommandBuffer used to submit a graph for processing\n#define GGML_METAL_MAX_COMMAND_BUFFERS 8\n\nstruct ggml_metal_command_buffer {\n id obj;\n};\n\nstruct ggml_metal {\n char name[128];\n\n ggml_metal_device_t dev;\n ggml_metal_library_t lib;\n\n ggml_metal_event_t ev_cpy; // for async copies\n\n dispatch_queue_t d_queue;\n\n // additional, inference-time compiled pipelines\n ggml_metal_pipelines_t pipelines_ext;\n\n bool use_fusion;\n bool use_concurrency;\n bool use_graph_optimize;\n\n int debug_graph;\n int debug_fusion;\n\n // how many times a given op was fused\n uint64_t fuse_cnt[GGML_OP_COUNT];\n\n // capture state\n int capture_compute;\n bool capture_started;\n\n id capture_scope;\n\n // command buffer state\n int n_cb; // number of extra threads used to submit the command buffers\n int n_nodes_0; // number of nodes submitted by the main thread\n int n_nodes_1; // remaining number of nodes submitted by the n_cb threads\n int n_nodes_per_cb;\n\n struct ggml_cgraph * gf;\n\n // the callback given to the thread pool\n void (^encode_async)(size_t ith);\n\n // n_cb command buffers + 1 used by the main thread\n struct ggml_metal_command_buffer cmd_bufs[GGML_METAL_MAX_COMMAND_BUFFERS + 1];\n\n // extra command buffers for things like getting, setting and copying tensors\n NSMutableArray * cmd_bufs_ext;\n\n // the last command buffer queued into the Metal queue with operations relevant to the current Metal backend\n id cmd_buf_last;\n\n // abort ggml_metal_graph_compute if callback returns true\n ggml_abort_callback abort_callback;\n void * abort_callback_data;\n\n // error state - set when a command buffer fails during synchronize\n // once set, graph_compute will return GGML_STATUS_FAILED until the backend is recreated\n bool has_error;\n};\n\nggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {\n GGML_LOG_INFO(\"%s: allocating\\n\", __func__);\n\n#if TARGET_OS_OSX && !GGML_METAL_NDEBUG\n // Show all the Metal device instances in the system\n NSArray * devices = MTLCopyAllDevices();\n for (id device in devices) {\n GGML_LOG_INFO(\"%s: found device: %s\\n\", __func__, [[device name] UTF8String]);\n }\n [devices release]; // since it was created by a *Copy* C method\n#endif\n\n // init context\n ggml_metal_t res = calloc(1, sizeof(struct ggml_metal));\n\n id device = ggml_metal_device_get_obj(dev);\n\n GGML_LOG_INFO(\"%s: picking default device: %s\\n\", __func__, [[device name] UTF8String]);\n\n // TODO: would it be better to have one queue for the backend and one queue for the device?\n // the graph encoders and async ops would use the backend queue while the sync ops would use the device queue?\n //res->queue = [device newCommandQueue]; [TAG_QUEUE_PER_BACKEND]\n id queue = ggml_metal_device_get_queue(dev);\n if (queue == nil) {\n GGML_LOG_ERROR(\"%s: error: failed to create command queue\\n\", __func__);\n return NULL;\n }\n\n res->dev = dev;\n res->lib = ggml_metal_device_get_library(dev);\n if (res->lib == NULL) {\n GGML_LOG_WARN(\"%s: the device does not have a precompiled Metal library - this is unexpected\\n\", __func__);\n GGML_LOG_WARN(\"%s: will try to compile it on the fly\\n\", __func__);\n\n res->lib = ggml_metal_library_init(dev);\n if (res->lib == NULL) {\n GGML_LOG_ERROR(\"%s: error: failed to initialize the Metal library\\n\", __func__);\n\n free(res);\n\n return NULL;\n }\n }\n\n res->ev_cpy = ggml_metal_device_event_init(dev);\n\n const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);\n\n snprintf(res->name, sizeof(res->name), \"%s\", props_dev->name);\n\n res->d_queue = dispatch_queue_create(\"ggml-metal\", DISPATCH_QUEUE_CONCURRENT);\n\n res->use_fusion = getenv(\"GGML_METAL_FUSION_DISABLE\") == nil;\n res->use_concurrency = getenv(\"GGML_METAL_CONCURRENCY_DISABLE\") == nil;\n\n {\n const char * val = getenv(\"GGML_METAL_GRAPH_DEBUG\");\n res->debug_graph = val ? atoi(val) : 0;\n }\n\n {\n const char * val = getenv(\"GGML_METAL_FUSION_DEBUG\");\n res->debug_fusion = val ? atoi(val) : 0;\n }\n\n res->use_graph_optimize = true;\n\n if (getenv(\"GGML_METAL_GRAPH_OPTIMIZE_DISABLE\") != NULL) {\n res->use_graph_optimize = false;\n }\n\n memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt));\n\n GGML_LOG_INFO(\"%s: use fusion = %s\\n\", __func__, res->use_fusion ? \"true\" : \"false\");\n GGML_LOG_INFO(\"%s: use concurrency = %s\\n\", __func__, res->use_concurrency ? \"true\" : \"false\");\n GGML_LOG_INFO(\"%s: use graph optimize = %s\\n\", __func__, res->use_graph_optimize ? \"true\" : \"false\");\n\n res->capture_compute = 0;\n res->capture_started = false;\n res->capture_scope = nil;\n\n {\n const char * val = getenv(\"GGML_METAL_CAPTURE_COMPUTE\");\n if (val) {\n res->capture_compute = atoi(val);\n }\n }\n\n res->has_error = false;\n\n res->gf = nil;\n res->encode_async = nil;\n for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {\n res->cmd_bufs[i].obj = nil;\n }\n\n res->cmd_bufs_ext = [[NSMutableArray alloc] init];\n\n res->cmd_buf_last = nil;\n\n res->pipelines_ext = ggml_metal_pipelines_init();\n\n return res;\n}\n\nvoid ggml_metal_free(ggml_metal_t ctx) {\n GGML_LOG_INFO(\"%s: deallocating\\n\", __func__);\n\n for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {\n if (ctx->cmd_bufs[i].obj) {\n [ctx->cmd_bufs[i].obj release];\n }\n }\n\n for (int i = 0; i < (int) ctx->cmd_bufs_ext.count; ++i) {\n if (ctx->cmd_bufs_ext[i]) {\n [ctx->cmd_bufs_ext[i] release];\n }\n }\n\n [ctx->cmd_bufs_ext removeAllObjects];\n [ctx->cmd_bufs_ext release];\n\n if (ctx->pipelines_ext) {\n ggml_metal_pipelines_free(ctx->pipelines_ext);\n ctx->pipelines_ext = nil;\n }\n\n if (ctx->debug_fusion > 0) {\n GGML_LOG_DEBUG(\"%s: fusion stats:\\n\", __func__);\n for (int i = 0; i < GGML_OP_COUNT; i++) {\n if (ctx->fuse_cnt[i] == 0) {\n continue;\n }\n\n // note: cannot use ggml_log here\n GGML_LOG_DEBUG(\"%s: - %s: %\" PRIu64 \"\\n\", __func__, ggml_op_name((enum ggml_op) i), ctx->fuse_cnt[i]);\n }\n }\n\n Block_release(ctx->encode_async);\n\n //[ctx->queue release]; // [TAG_QUEUE_PER_BACKEND]\n\n dispatch_release(ctx->d_queue);\n\n ggml_metal_device_event_free(ctx->dev, ctx->ev_cpy);\n\n free(ctx);\n}\n\nconst char * ggml_metal_get_name(ggml_metal_t ctx) {\n return ctx->name;\n}\n\nvoid ggml_metal_synchronize(ggml_metal_t ctx) {\n // wait for any backend operations to finish\n if (ctx->cmd_buf_last) {\n [ctx->cmd_buf_last waitUntilCompleted];\n ctx->cmd_buf_last = nil;\n }\n\n // check status of all command buffers\n {\n const int n_cb = ctx->n_cb;\n\n for (int cb_idx = 0; cb_idx <= n_cb; ++cb_idx) {\n id cmd_buf = ctx->cmd_bufs[cb_idx].obj;\n if (!cmd_buf) {\n continue;\n }\n\n MTLCommandBufferStatus status = [cmd_buf status];\n if (status != MTLCommandBufferStatusCompleted) {\n GGML_LOG_ERROR(\"%s: error: command buffer %d failed with status %d\\n\", __func__, cb_idx, (int) status);\n if (status == MTLCommandBufferStatusError) {\n GGML_LOG_ERROR(\"error: %s\\n\", [[cmd_buf error].localizedDescription UTF8String]);\n }\n ctx->has_error = true;\n return;\n }\n }\n }\n\n // release any completed extra command buffers\n if (ctx->cmd_bufs_ext.count > 0) {\n for (size_t i = 0; i < ctx->cmd_bufs_ext.count; ++i) {\n id cmd_buf = ctx->cmd_bufs_ext[i];\n\n MTLCommandBufferStatus status = [cmd_buf status];\n if (status != MTLCommandBufferStatusCompleted) {\n GGML_LOG_ERROR(\"%s: error: command buffer %d failed with status %d\\n\", __func__, (int) i, (int) status);\n if (status == MTLCommandBufferStatusError) {\n GGML_LOG_ERROR(\"error: %s\\n\", [[cmd_buf error].localizedDescription UTF8String]);\n }\n\n // release this and all remaining command buffers before returning\n for (size_t j = i; j < ctx->cmd_bufs_ext.count; ++j) {\n [ctx->cmd_bufs_ext[j] release];\n }\n [ctx->cmd_bufs_ext removeAllObjects];\n\n ctx->has_error = true;\n return;\n }\n\n [cmd_buf release];\n }\n\n [ctx->cmd_bufs_ext removeAllObjects];\n }\n}\n\nstatic struct ggml_metal_buffer_id ggml_metal_get_buffer_id(const struct ggml_tensor * t) {\n if (!t) {\n return (struct ggml_metal_buffer_id) { nil, 0 };\n }\n\n ggml_backend_buffer_t buffer = t->view_src ? t->view_src->buffer : t->buffer;\n\n return ggml_metal_buffer_get_id(buffer->context, t);\n}\n\nvoid ggml_metal_set_tensor_async(ggml_metal_t ctx, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n @autoreleasepool {\n // wrap the source data into a Metal buffer\n id device = ggml_metal_device_get_obj(ctx->dev);\n id buf_src = [device newBufferWithBytes:data\n length:size\n options:MTLResourceStorageModeShared];\n\n GGML_ASSERT(buf_src);\n\n struct ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(tensor);\n if (bid_dst.metal == nil) {\n GGML_ABORT(\"%s: failed to find buffer for tensor '%s'\\n\", __func__, tensor->name);\n }\n\n bid_dst.offs += offset;\n\n // queue the copy operation into the queue of the Metal context\n // this will be queued at the end, after any currently ongoing GPU operations\n id queue = ggml_metal_device_get_queue(ctx->dev);\n id cmd_buf = [queue commandBuffer];\n id encoder = [cmd_buf blitCommandEncoder];\n\n [encoder copyFromBuffer:buf_src\n sourceOffset:0\n toBuffer:bid_dst.metal\n destinationOffset:bid_dst.offs\n size:size];\n\n [encoder endEncoding];\n [cmd_buf commit];\n [buf_src release];\n\n // do not wait here for completion\n //[cmd_buf waitUntilCompleted];\n\n // instead, remember a reference to the command buffer and wait for it later if needed\n [ctx->cmd_bufs_ext addObject:cmd_buf];\n ctx->cmd_buf_last = cmd_buf;\n\n [cmd_buf retain];\n }\n}\n\nvoid ggml_metal_get_tensor_async(ggml_metal_t ctx, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n @autoreleasepool {\n id device = ggml_metal_device_get_obj(ctx->dev);\n id buf_dst = [device newBufferWithBytesNoCopy:data\n length:size\n options:MTLResourceStorageModeShared\n deallocator:nil];\n\n GGML_ASSERT(buf_dst);\n\n struct ggml_metal_buffer_id bid_src = ggml_metal_get_buffer_id(tensor);\n if (bid_src.metal == nil) {\n GGML_ABORT(\"%s: failed to find buffer for tensor '%s'\\n\", __func__, tensor->name);\n }\n\n bid_src.offs += offset;\n\n // queue the copy operation into the queue of the Metal context\n // this will be queued at the end, after any currently ongoing GPU operations\n id queue = ggml_metal_device_get_queue(ctx->dev);\n id cmd_buf = [queue commandBuffer];\n id encoder = [cmd_buf blitCommandEncoder];\n\n [encoder copyFromBuffer:bid_src.metal\n sourceOffset:bid_src.offs\n toBuffer:buf_dst\n destinationOffset:0\n size:size];\n\n [encoder endEncoding];\n [cmd_buf commit];\n [buf_dst release];\n\n // do not wait here for completion\n //[cmd_buf waitUntilCompleted];\n\n // instead, remember a reference to the command buffer and wait for it later if needed\n [ctx->cmd_bufs_ext addObject:cmd_buf];\n ctx->cmd_buf_last = cmd_buf;\n\n [cmd_buf retain];\n }\n}\n\nbool ggml_metal_cpy_tensor_async(ggml_metal_t ctx_src, ggml_metal_t ctx_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {\n @autoreleasepool {\n struct ggml_metal_buffer_id bid_src = ggml_metal_get_buffer_id(src);\n struct ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(dst);\n\n if (bid_src.metal == nil || bid_dst.metal == nil) {\n return false;\n }\n\n // queue the copy operation into the Metal context\n // this will be queued at the end, after any currently ongoing GPU operations\n id queue = ggml_metal_device_get_queue(ctx_src->dev);\n id cmd_buf = [queue commandBuffer];\n id encoder = [cmd_buf blitCommandEncoder];\n\n [encoder copyFromBuffer:bid_src.metal\n sourceOffset:bid_src.offs\n toBuffer:bid_dst.metal\n destinationOffset:bid_dst.offs\n size:ggml_nbytes(src)];\n\n [encoder endEncoding];\n\n ggml_metal_event_t ev_cpy = ggml_metal_get_ev_cpy(ctx_src);\n ggml_metal_event_encode_signal(ev_cpy, cmd_buf);\n\n [cmd_buf commit];\n\n // do not wait here for completion\n //[cmd_buf waitUntilCompleted];\n\n // instead, remember a reference to the command buffer and wait for it later if needed\n [ctx_src->cmd_bufs_ext addObject:cmd_buf];\n ctx_src->cmd_buf_last = cmd_buf;\n\n [cmd_buf retain];\n\n ggml_metal_event_wait(ctx_dst, ev_cpy);\n\n return true;\n }\n}\n\nenum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * gf) {\n if (ctx->has_error) {\n GGML_LOG_ERROR(\"%s: backend is in error state from a previous command buffer failure - recreate the backend to recover\\n\", __func__);\n return GGML_STATUS_FAILED;\n }\n\n // number of nodes encoded by the main thread (empirically determined)\n const int n_main = MAX(64, 0.1*gf->n_nodes);\n\n // number of threads in addition to the main thread\n const int n_cb = ctx->n_cb;\n\n // keep the memory wired\n ggml_metal_device_rsets_keep_alive(ctx->dev);\n\n // submit the ggml compute graph to the GPU by creating command buffers and encoding the ops in them\n // the first n_nodes_0 are encoded and submitted for processing directly by the calling thread\n // while these nodes are processing, we start n_cb threads to enqueue the rest of the nodes\n // each thread creates it's own command buffer and enqueues the ops in parallel\n //\n // tests on M1 Pro and M2 Ultra using LLaMA models, show that optimal values for n_cb are 1 or 2\n\n @autoreleasepool {\n ctx->gf = gf;\n\n ctx->n_nodes_0 = MIN(n_main, gf->n_nodes);\n ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0;\n\n ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb;\n\n if (ctx->capture_compute >= 0) {\n ctx->capture_compute--;\n }\n\n const bool use_capture = ctx->capture_compute == 0;\n if (use_capture) {\n ctx->capture_compute = -1;\n\n // make sure all previous computations have finished before starting the capture\n if (ctx->cmd_buf_last) {\n [ctx->cmd_buf_last waitUntilCompleted];\n ctx->cmd_buf_last = nil;\n }\n\n if (!ctx->capture_started) {\n NSString * path = [NSString stringWithFormat:@\"/tmp/perf-metal-%d.gputrace\", getpid()];\n\n GGML_LOG_WARN(\"%s: capturing graph in %s\\n\", __func__, [path UTF8String]);\n\n // create capture scope\n id device = ggml_metal_device_get_obj(ctx->dev);\n ctx->capture_scope = [[MTLCaptureManager sharedCaptureManager] newCaptureScopeWithDevice:device];\n\n MTLCaptureDescriptor * descriptor = [MTLCaptureDescriptor new];\n descriptor.captureObject = ctx->capture_scope;\n descriptor.destination = MTLCaptureDestinationGPUTraceDocument;\n descriptor.outputURL = [NSURL fileURLWithPath:path];\n\n NSError * error = nil;\n if (![[MTLCaptureManager sharedCaptureManager] startCaptureWithDescriptor:descriptor error:&error]) {\n GGML_LOG_ERROR(\"%s: error: unable to start capture '%s'\\n\", __func__, [[error localizedDescription] UTF8String]);\n } else {\n [ctx->capture_scope beginScope];\n ctx->capture_started = true;\n }\n }\n }\n\n // short-hand\n id queue = ggml_metal_device_get_queue(ctx->dev);\n\n // the main thread commits the first few commands immediately\n // cmd_buf[n_cb]\n {\n id cmd_buf = [queue commandBufferWithUnretainedReferences];\n [cmd_buf retain];\n\n if (ctx->cmd_bufs[n_cb].obj) {\n [ctx->cmd_bufs[n_cb].obj release];\n }\n ctx->cmd_bufs[n_cb].obj = cmd_buf;\n\n [cmd_buf enqueue];\n\n ctx->encode_async(n_cb);\n }\n\n // remember the command buffer for the next iteration\n ctx->cmd_buf_last = ctx->cmd_bufs[n_cb].obj;\n\n // prepare the rest of the command buffers asynchronously (optional)\n // cmd_buf[0.. n_cb)\n for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) {\n id cmd_buf = [queue commandBufferWithUnretainedReferences];\n [cmd_buf retain];\n\n if (ctx->cmd_bufs[cb_idx].obj) {\n [ctx->cmd_bufs[cb_idx].obj release];\n }\n ctx->cmd_bufs[cb_idx].obj = cmd_buf;\n\n // always enqueue the first two command buffers\n // enqueue all of the command buffers if we don't need to abort\n if (cb_idx < 2 || ctx->abort_callback == NULL) {\n [cmd_buf enqueue];\n\n // update the pointer to the last queued command buffer\n // this is needed to implement synchronize()\n ctx->cmd_buf_last = cmd_buf;\n }\n }\n\n dispatch_apply(n_cb, ctx->d_queue, ctx->encode_async);\n\n // for debugging: block until graph is computed\n //[ctx->cmd_buf_last waitUntilCompleted];\n\n // enter here only when capturing in order to wait for all computation to finish\n // otherwise, we leave the graph to compute asynchronously\n if (use_capture && ctx->capture_started) {\n // wait for completion and check status of each command buffer\n // needed to detect if the device ran out-of-memory for example (#1881)\n {\n id cmd_buf = ctx->cmd_bufs[n_cb].obj;\n [cmd_buf waitUntilCompleted];\n\n MTLCommandBufferStatus status = [cmd_buf status];\n if (status != MTLCommandBufferStatusCompleted) {\n GGML_LOG_INFO(\"%s: command buffer %d failed with status %lu\\n\", __func__, n_cb, status);\n if (status == MTLCommandBufferStatusError) {\n GGML_LOG_INFO(\"error: %s\\n\", [[cmd_buf error].localizedDescription UTF8String]);\n }\n\n return GGML_STATUS_FAILED;\n }\n }\n\n for (int i = 0; i < n_cb; ++i) {\n id cmd_buf = ctx->cmd_bufs[i].obj;\n [cmd_buf waitUntilCompleted];\n\n MTLCommandBufferStatus status = [cmd_buf status];\n if (status != MTLCommandBufferStatusCompleted) {\n GGML_LOG_INFO(\"%s: command buffer %d failed with status %lu\\n\", __func__, i, status);\n if (status == MTLCommandBufferStatusError) {\n GGML_LOG_INFO(\"error: %s\\n\", [[cmd_buf error].localizedDescription UTF8String]);\n }\n\n return GGML_STATUS_FAILED;\n }\n\n id next_buffer = (i + 1 < n_cb ? ctx->cmd_bufs[i + 1].obj : nil);\n if (!next_buffer) {\n continue;\n }\n\n const bool next_queued = ([next_buffer status] != MTLCommandBufferStatusNotEnqueued);\n if (next_queued) {\n continue;\n }\n\n if (ctx->abort_callback && ctx->abort_callback(ctx->abort_callback_data)) {\n GGML_LOG_INFO(\"%s: command buffer %d aborted\", __func__, i);\n return GGML_STATUS_ABORTED;\n }\n\n [next_buffer commit];\n }\n\n [ctx->capture_scope endScope];\n [[MTLCaptureManager sharedCaptureManager] stopCapture];\n\n ctx->capture_started = false;\n }\n }\n\n return GGML_STATUS_SUCCESS;\n}\n\nvoid ggml_metal_graph_optimize(ggml_metal_t ctx, struct ggml_cgraph * gf) {\n //const int64_t t_start = ggml_time_us();\n\n if (ctx->use_graph_optimize) {\n ggml_graph_optimize(gf);\n }\n\n //printf(\"%s: graph optimize took %.3f ms\\n\", __func__, (ggml_time_us() - t_start) / 1000.0);\n}\n\nvoid ggml_metal_event_record(ggml_metal_t ctx, ggml_metal_event_t ev) {\n @autoreleasepool {\n id queue = ggml_metal_device_get_queue(ctx->dev);\n id cmd_buf = [queue commandBuffer];\n\n ggml_metal_event_encode_signal(ev, cmd_buf);\n\n [cmd_buf commit];\n\n [ctx->cmd_bufs_ext addObject:cmd_buf];\n ctx->cmd_buf_last = cmd_buf;\n\n [cmd_buf retain];\n }\n}\n\nvoid ggml_metal_event_wait(ggml_metal_t ctx, ggml_metal_event_t ev) {\n @autoreleasepool {\n id queue = ggml_metal_device_get_queue(ctx->dev);\n id cmd_buf = [queue commandBuffer];\n\n ggml_metal_event_encode_wait(ev, cmd_buf);\n\n [cmd_buf commit];\n\n [ctx->cmd_bufs_ext addObject:cmd_buf];\n ctx->cmd_buf_last = cmd_buf;\n\n [cmd_buf retain];\n }\n}\n\nggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) {\n return ctx->ev_cpy;\n}\n\nvoid ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {\n if (ctx->n_cb != n_cb) {\n ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS);\n\n if (ctx->n_cb > 2) {\n GGML_LOG_WARN(\"%s: n_cb = %d, using n_cb > 2 is not recommended and can degrade the performance in some cases\\n\", __func__, n_cb);\n }\n }\n\n if (ctx->encode_async) {\n Block_release(ctx->encode_async);\n }\n\n ctx->encode_async = Block_copy(^(size_t iter) {\n const int cb_idx = iter;\n const int n_cb_l = ctx->n_cb;\n\n const int n_nodes_0 = ctx->n_nodes_0;\n const int n_nodes_1 = ctx->n_nodes_1;\n\n const int n_nodes_per_cb = ctx->n_nodes_per_cb;\n\n int idx_start = 0;\n int idx_end = n_nodes_0;\n\n if (cb_idx < n_cb_l) {\n idx_start = n_nodes_0 + ( (cb_idx + 0) * n_nodes_per_cb);\n idx_end = n_nodes_0 + (MIN((cb_idx == n_cb_l - 1) ? n_nodes_1 : (cb_idx + 1) * n_nodes_per_cb, n_nodes_1));\n }\n\n id cmd_buf = ctx->cmd_bufs[cb_idx].obj;\n\n ggml_metal_op_t ctx_op = ggml_metal_op_init(\n ctx->dev,\n cmd_buf,\n ctx->gf,\n idx_start,\n idx_end,\n ctx->use_fusion,\n ctx->use_concurrency,\n ctx->capture_compute,\n ctx->debug_graph,\n ctx->debug_fusion);\n\n for (int idx = 0; idx < ggml_metal_op_n_nodes(ctx_op); ++idx) {\n const int res = ggml_metal_op_encode(ctx_op, idx);\n if (res == 0) {\n break;\n }\n\n idx += res - 1;\n }\n\n ggml_metal_op_free(ctx_op);\n\n if (cb_idx < 2 || ctx->abort_callback == NULL) {\n [cmd_buf commit];\n }\n });\n}\n\nvoid ggml_metal_set_abort_callback(ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data) {\n ctx->abort_callback = abort_callback;\n ctx->abort_callback_data = user_data;\n}\n\nbool ggml_metal_supports_family(ggml_metal_t ctx, int family) {\n GGML_ASSERT(ctx->dev != nil);\n\n id device = ggml_metal_device_get_obj(ctx->dev);\n\n return [device supportsFamily:(MTLGPUFamilyApple1 + family - 1)];\n}\n\nvoid ggml_metal_capture_next_compute(ggml_metal_t ctx) {\n ctx->capture_compute = 1;\n}\n" }, { "path": "src/ggml-metal/ggml-metal-device.cpp", "content": "#include \"ggml-metal-device.h\"\n\n#include \"ggml-metal-impl.h\"\n\n#include \"ggml-impl.h\"\n\n#include \n#include \n#include \n#include \n\nstruct ggml_metal_device_deleter {\n void operator()(ggml_metal_device_t ctx) {\n ggml_metal_device_free(ctx);\n }\n};\n\ntypedef std::unique_ptr ggml_metal_device_ptr;\n\nggml_metal_device_t ggml_metal_device_get(int device) {\n static std::vector devs;\n\n devs.emplace_back(ggml_metal_device_init(device));\n\n return devs.back().get();\n}\n\nstruct ggml_metal_pipelines {\n std::unordered_map data;\n};\n\nggml_metal_pipelines_t ggml_metal_pipelines_init(void) {\n ggml_metal_pipelines_t res = new ggml_metal_pipelines();\n\n return res;\n}\n\nvoid ggml_metal_pipelines_free(ggml_metal_pipelines_t ppls) {\n if (!ppls) {\n return;\n }\n\n for (auto it = ppls->data.begin(); it != ppls->data.end(); ++it) {\n ggml_metal_pipeline_free(it->second);\n }\n\n delete ppls;\n}\n\nvoid ggml_metal_pipelines_add(ggml_metal_pipelines_t ppls, const char * name, ggml_metal_pipeline_t pipeline) {\n ppls->data[name] = pipeline;\n}\n\nggml_metal_pipeline_t ggml_metal_pipelines_get(ggml_metal_pipelines_t ppls, const char * name) {\n if (ppls->data.find(name) == ppls->data.end()) {\n return nullptr;\n }\n\n return ppls->data[name];\n}\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_base(ggml_metal_library_t lib, ggml_op op) {\n char base[256];\n char name[256];\n\n const char * op_str = \"undefined\";\n switch (op) {\n case GGML_OP_ADD_ID: op_str = \"add_id\"; break;\n case GGML_OP_CONCAT: op_str = \"concat\"; break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n snprintf(base, 256, \"kernel_%s\", op_str);\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cpy(ggml_metal_library_t lib, ggml_type tsrc, ggml_type tdst) {\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_cpy_%s_%s\", ggml_type_name(tsrc), ggml_type_name(tdst));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_1d(ggml_metal_library_t lib, const ggml_tensor * op, ggml_op_pool op_pool) {\n GGML_ASSERT(ggml_is_contiguous(op->src[0]));\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32 && op->src[0]->type == op->type);\n\n const char * pool_str = \"undefined\";\n switch (op_pool) {\n case GGML_OP_POOL_AVG: pool_str = \"avg\"; break;\n case GGML_OP_POOL_MAX: pool_str = \"max\"; break;\n default: GGML_ASSERT(false && \"not implemented\");\n };\n\n char base[256];\n char name[256];\n\n snprintf(base, sizeof(base), \"kernel_pool_1d_%s_%s\", pool_str, ggml_type_name(op->src[0]->type));\n snprintf(name, sizeof(name), \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_2d(ggml_metal_library_t lib, const ggml_tensor * op, ggml_op_pool op_pool) {\n GGML_ASSERT(ggml_is_contiguous(op->src[0]));\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32 && op->src[0]->type == op->type);\n\n const char * pool_str = \"undefined\";\n switch (op_pool) {\n case GGML_OP_POOL_AVG: pool_str = \"avg\"; break;\n case GGML_OP_POOL_MAX: pool_str = \"max\"; break;\n default: GGML_ASSERT(false && \"not implemented\");\n };\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_pool_2d_%s_%s\", pool_str, ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_get_rows(ggml_metal_library_t lib, ggml_type tsrc) {\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_get_rows_%s\", ggml_type_name(tsrc));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows(ggml_metal_library_t lib, ggml_type tidx, ggml_type tdst) {\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_set_rows_%s_%s\", ggml_type_name(tdst), ggml_type_name(tidx));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diag(ggml_metal_library_t lib, const ggml_tensor * op) {\n char base[256];\n char name[256];\n\n const int n = op->src[0]->ne[0];\n\n snprintf(base, 256, \"kernel_diag_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s_n=%d\", base, n);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n res.nsg = 1;\n res.smem = 0;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat(ggml_metal_library_t lib, ggml_type tsrc) {\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_repeat_%s\", ggml_type_name(tsrc));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary(ggml_metal_library_t lib, const ggml_tensor * op) {\n char base[256];\n char name[256];\n\n int op_num = -1;\n\n switch (op->op) {\n case GGML_OP_SCALE: op_num = OP_UNARY_NUM_SCALE; break;\n case GGML_OP_FILL: op_num = OP_UNARY_NUM_FILL; break;\n case GGML_OP_CLAMP: op_num = OP_UNARY_NUM_CLAMP; break;\n case GGML_OP_SQR: op_num = OP_UNARY_NUM_SQR; break;\n case GGML_OP_SQRT: op_num = OP_UNARY_NUM_SQRT; break;\n case GGML_OP_SIN: op_num = OP_UNARY_NUM_SIN; break;\n case GGML_OP_COS: op_num = OP_UNARY_NUM_COS; break;\n case GGML_OP_LOG: op_num = OP_UNARY_NUM_LOG; break;\n case GGML_OP_LEAKY_RELU: op_num = OP_UNARY_NUM_LEAKY_RELU; break;\n case GGML_OP_UNARY:\n switch (ggml_get_unary_op(op)) {\n case GGML_UNARY_OP_TANH: op_num = OP_UNARY_NUM_TANH; break;\n case GGML_UNARY_OP_RELU: op_num = OP_UNARY_NUM_RELU; break;\n case GGML_UNARY_OP_SIGMOID: op_num = OP_UNARY_NUM_SIGMOID; break;\n case GGML_UNARY_OP_GELU: op_num = OP_UNARY_NUM_GELU; break;\n case GGML_UNARY_OP_GELU_ERF: op_num = OP_UNARY_NUM_GELU_ERF; break;\n case GGML_UNARY_OP_GELU_QUICK: op_num = OP_UNARY_NUM_GELU_QUICK; break;\n case GGML_UNARY_OP_SILU: op_num = OP_UNARY_NUM_SILU; break;\n case GGML_UNARY_OP_ELU: op_num = OP_UNARY_NUM_ELU; break;\n case GGML_UNARY_OP_NEG: op_num = OP_UNARY_NUM_NEG; break;\n case GGML_UNARY_OP_ABS: op_num = OP_UNARY_NUM_ABS; break;\n case GGML_UNARY_OP_SGN: op_num = OP_UNARY_NUM_SGN; break;\n case GGML_UNARY_OP_STEP: op_num = OP_UNARY_NUM_STEP; break;\n case GGML_UNARY_OP_HARDSWISH: op_num = OP_UNARY_NUM_HARDSWISH; break;\n case GGML_UNARY_OP_HARDSIGMOID: op_num = OP_UNARY_NUM_HARDSIGMOID; break;\n case GGML_UNARY_OP_EXP: op_num = OP_UNARY_NUM_EXP; break;\n case GGML_UNARY_OP_SOFTPLUS: op_num = OP_UNARY_NUM_SOFTPLUS; break;\n case GGML_UNARY_OP_EXPM1: op_num = OP_UNARY_NUM_EXPM1; break;\n default: GGML_ABORT(\"fatal error\");\n } break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n const char * t0_str = ggml_type_name(op->src[0]->type);\n const char * t_str = ggml_type_name(op->type);\n\n const bool is_c4 = op->src[0]->ne[0] % 4 == 0;\n const bool is_cnt = ggml_is_contiguous(op->src[0]) && ggml_nelements(op) < 32768;\n\n snprintf(base, 256, \"kernel_unary_%s_%s%s\", t0_str, t_str, is_c4 ? \"_4\" : \"\");\n snprintf(name, 256, \"%s_op=%d_cnt=%d\", base, op_num, is_cnt);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, op_num, FC_UNARY + 0);\n ggml_metal_cv_set_bool (cv, is_cnt, FC_UNARY + 1);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n res.c4 = is_c4;\n res.cnt = is_cnt;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_ASSERT(ggml_is_contiguous_1(op->src[0]));\n\n char base[256];\n char name[256];\n\n const char * op_str = \"undefined\";\n switch (op->op) {\n case GGML_OP_GLU:\n switch (ggml_get_glu_op(op)) {\n case GGML_GLU_OP_REGLU: op_str = \"reglu\"; break;\n case GGML_GLU_OP_GEGLU: op_str = \"geglu\"; break;\n case GGML_GLU_OP_SWIGLU: op_str = \"swiglu\"; break;\n case GGML_GLU_OP_SWIGLU_OAI: op_str = \"swiglu_oai\"; break;\n case GGML_GLU_OP_GEGLU_ERF: op_str = \"geglu_erf\"; break;\n case GGML_GLU_OP_GEGLU_QUICK: op_str = \"geglu_quick\"; break;\n default: GGML_ABORT(\"fatal error\");\n } break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n snprintf(base, 256, \"kernel_%s_%s\", op_str, ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_SUM);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_op_sum_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum_rows(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n\n char base[256];\n char name[256];\n\n int op_num = -1;\n\n switch (op->op) {\n case GGML_OP_SUM_ROWS: op_num = OP_SUM_ROWS_NUM_SUM_ROWS; break;\n case GGML_OP_MEAN: op_num = OP_SUM_ROWS_NUM_MEAN; break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n const char * t0_str = ggml_type_name(op->src[0]->type);\n const char * t_str = ggml_type_name(op->type);\n\n const bool is_c4 = op->src[0]->ne[0] % 4 == 0;\n\n snprintf(base, 256, \"kernel_sum_rows_%s_%s%s\", t0_str, t_str, is_c4 ? \"_4\" : \"\");\n snprintf(name, 256, \"%s_op=%d\", base, op_num);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, op_num, FC_SUM_ROWS + 0);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n res.smem = 32*sizeof(float);\n\n if (is_c4) {\n res.smem *= 4;\n }\n\n res.c4 = is_c4;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_blk(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_ASSERT(op->op == GGML_OP_CUMSUM);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_cumsum_blk_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_add(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_ASSERT(op->op == GGML_OP_CUMSUM);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_cumsum_add_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_ASSERT(op->op == GGML_OP_TRI);\n GGML_ASSERT(op->src[0]->nb[0] == ggml_type_size(op->src[0]->type));\n\n char base[256];\n char name[256];\n\n const char * op_str = \"tri\";\n const int ttype = op->op_params[0];\n\n snprintf(base, 256, \"kernel_%s_%s_%d\", op_str, ggml_type_name(op->src[0]->type), ttype);\n\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_ASSERT(!op->src[1] || op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32);\n\n char base[256];\n char name[256];\n\n const char * suffix = \"\";\n\n if (op->src[0]->ne[0] % 4 == 0) {\n suffix = \"_4\";\n }\n\n const ggml_type tsrc1 = op->src[1] ? op->src[1]->type : GGML_TYPE_F32;\n\n snprintf(base, 256, \"kernel_soft_max_%s%s\", ggml_type_name(tsrc1), suffix);\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n res.smem = 32*sizeof(float);\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous(op->src[0]));\n GGML_ASSERT(ggml_is_contiguous(op->src[1]));\n\n char base[256];\n char name[256];\n\n const char * suffix = \"\";\n\n if (op->src[1]->ne[0] % 4 == 0) {\n suffix = \"_4\";\n }\n\n snprintf(base, 256, \"kernel_ssm_conv_%s_%s%s\", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type), suffix);\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched(ggml_metal_library_t lib, const ggml_tensor * op, int ssm_conv_bs) {\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous(op->src[0]));\n GGML_ASSERT(ggml_is_contiguous(op->src[1]));\n\n char base[256];\n char name[256];\n\n const char * suffix = \"\";\n if (op->src[1]->ne[0] % 4 == 0) {\n suffix = \"_4\";\n }\n\n snprintf(base, 256, \"kernel_ssm_conv_%s_%s_batched%s\", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type), suffix);\n snprintf(name, 256, \"%s_ssm_conv_bs=%d\", base, ssm_conv_bs);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, ssm_conv_bs, FC_SSM_CONV + 0);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n\n char base[256];\n char name[256];\n\n const int nsg = (ne00 + 31)/32;\n\n snprintf(base, 256, \"kernel_ssm_scan_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s_nsg=%d\", base, nsg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n // Shared memory layout:\n // - sgptg * NW floats for partial sums (nsg * 32)\n // - sgptg floats for shared_x_dt (nsg)\n // - sgptg floats for shared_dA (nsg)\n // Total: nsg * (32 + 2) floats\n res.smem = (32 + 2)*sizeof(float)*nsg;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rwkv(ggml_metal_library_t lib, const ggml_tensor * op) {\n char base[256];\n char name[256];\n\n const int64_t C = op->ne[0];\n const int64_t H = op->src[0]->ne[1];\n\n switch (op->op) {\n case GGML_OP_RWKV_WKV6:\n {\n GGML_ASSERT(op->src[5]->type == GGML_TYPE_F32);\n GGML_ASSERT(C % H == 0);\n GGML_ASSERT(C / H == 64);\n\n snprintf(base, 256, \"kernel_rwkv_wkv6_%s\", ggml_type_name(op->src[0]->type));\n } break;\n case GGML_OP_RWKV_WKV7:\n {\n GGML_ASSERT(op->src[6]->type == GGML_TYPE_F32);\n GGML_ASSERT(C % H == 0);\n GGML_ASSERT(C / H == 64);\n\n snprintf(base, 256, \"kernel_rwkv_wkv7_%s\", ggml_type_name(op->src[0]->type));\n } break;\n default:\n GGML_ABORT(\"fatal error\");\n }\n\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_gated_delta_net(ggml_metal_library_t lib, const ggml_tensor * op) {\n char base[256];\n char name[256];\n\n // v is src[2], dimensions: S_v = ne[0], H = ne[1]\n const int ne20 = op->src[2]->ne[0]; // S_v\n const int ne21 = op->src[2]->ne[1]; // H\n const int ne30 = op->src[3]->ne[0]; // G\n\n const int nsg = op->src[2]->ne[0]/32;\n\n GGML_ASSERT(op->src[5]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->ne[0] == ne20 * ne21);\n GGML_ASSERT(ne20 % 32 == 0);\n\n snprintf(base, 256, \"kernel_gated_delta_net_%s_%d\", ggml_type_name(op->src[0]->type), nsg);\n snprintf(name, 256, \"%s_ne20=%d_ne30=%d\", base, ne20, ne30);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, ne20, FC_GATED_DELTA_NET + 0);\n ggml_metal_cv_set_int16(cv, ne30, FC_GATED_DELTA_NET + 1);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n res.nsg = nsg;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_solve_tri(ggml_metal_library_t lib, const ggml_tensor * op) {\n char base[256];\n char name[256];\n\n const int nsg = 8;\n const int n = op->src[1]->ne[1];\n const int k = op->src[1]->ne[0];\n\n snprintf(base, 256, \"kernel_solve_tri_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s_nsg=%d_n=%d_k=%d\", base, nsg, n, k);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, nsg, FC_SOLVE_TRI + 0);\n ggml_metal_cv_set_int16(cv, n, FC_SOLVE_TRI + 1);\n ggml_metal_cv_set_int16(cv, k, FC_SOLVE_TRI + 2);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n res.nsg = nsg;\n res.smem = GGML_PAD(GGML_PAD(n, 32)*nsg*sizeof(float), 16);\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext(ggml_metal_library_t lib, ggml_type tsrc0, ggml_type tsrc1, int nsg, int nxpsg, int r1ptg) {\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_mul_mv_ext_%s_%s_r1_%d\", ggml_type_name(tsrc0), ggml_type_name(tsrc1), r1ptg);\n snprintf(name, 256, \"%s_nsg=%d_nxpsg=%d\", base, nsg, nxpsg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, nsg, FC_MUL_MV + 0);\n ggml_metal_cv_set_int16(cv, nxpsg, FC_MUL_MV + 1);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_metal_library_t lib, const ggml_tensor * op) {\n char base[256];\n char name[256];\n\n const ggml_type tsrc0 = op->src[0]->type;\n const ggml_type tsrc1 = op->src[1]->type;\n\n const bool bc_inp = op->src[0]->ne[0] % 32 != 0;\n const bool bc_out = op->ne[0] % 64 != 0 || op->ne[1] % 32 != 0;\n\n snprintf(base, 256, \"kernel_mul_mm_%s_%s\", ggml_type_name(tsrc0), ggml_type_name(tsrc1));\n snprintf(name, 256, \"%s_bci=%d_bco=%d\", base, bc_inp, bc_out);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_bool(cv, bc_inp, FC_MUL_MM + 0);\n ggml_metal_cv_set_bool(cv, bc_out, FC_MUL_MM + 1);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n // when the output size is not multiple of 64x32, we need extra smem to prevent out-of-bounds writes\n res.smem = bc_out ? 8192 : 4096 + 2048;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n\n char base[256];\n char name[256];\n\n int nsg = 0; // number of simdgroups\n int nr0 = 0; // number of src0 rows per simdgroup\n int nr1 = 1; // number of src1 rows per threadgroup\n\n size_t smem = 0; // shared memory\n\n const ggml_type tsrc0 = op->src[0]->type;\n const ggml_type tsrc1 = op->src[1]->type;\n\n const char * suffix = \"\";\n\n // use custom matrix x vector kernel\n switch (tsrc0) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n {\n if (ne00 < 32) {\n nsg = 1;\n nr0 = 32;\n nr1 = 1;\n suffix = \"_short\";\n } else {\n nsg = std::min(4, (ne00 + 127) / 128);\n nr0 = 2;\n nr1 = 1;\n smem = 32*sizeof(float)*nr0;\n suffix = ne00 % 4 == 0 ? \"_4\" : \"\";\n }\n } break;\n case GGML_TYPE_Q4_0:\n {\n nsg = N_SG_Q4_0;\n nr0 = N_R0_Q4_0;\n } break;\n case GGML_TYPE_Q4_1:\n {\n nsg = N_SG_Q4_1;\n nr0 = N_R0_Q4_1;\n } break;\n case GGML_TYPE_Q5_0:\n {\n nsg = N_SG_Q5_0;\n nr0 = N_R0_Q5_0;\n } break;\n case GGML_TYPE_Q5_1:\n {\n nsg = N_SG_Q5_1;\n nr0 = N_R0_Q5_1;\n } break;\n case GGML_TYPE_Q8_0:\n {\n nsg = N_SG_Q8_0;\n nr0 = N_R0_Q8_0;\n smem = 32*sizeof(float)*N_R0_Q8_0;\n } break;\n case GGML_TYPE_MXFP4:\n {\n nsg = N_SG_MXFP4;\n nr0 = N_R0_MXFP4;\n smem = 32*sizeof(float);\n } break;\n case GGML_TYPE_Q2_K:\n {\n nsg = N_SG_Q2_K;\n nr0 = N_R0_Q2_K;\n } break;\n case GGML_TYPE_Q3_K:\n {\n nsg = N_SG_Q3_K;\n nr0 = N_R0_Q3_K;\n } break;\n case GGML_TYPE_Q4_K:\n {\n nsg = N_SG_Q4_K;\n nr0 = N_R0_Q4_K;\n } break;\n case GGML_TYPE_Q5_K:\n {\n nsg = N_SG_Q5_K;\n nr0 = N_R0_Q5_K;\n } break;\n case GGML_TYPE_Q6_K:\n {\n nsg = N_SG_Q6_K;\n nr0 = N_R0_Q6_K;\n } break;\n case GGML_TYPE_IQ2_XXS:\n {\n nsg = N_SG_IQ2_XXS;\n nr0 = N_R0_IQ2_XXS;\n smem = 256*8+128;\n } break;\n case GGML_TYPE_IQ2_XS:\n {\n nsg = N_SG_IQ2_XS;\n nr0 = N_R0_IQ2_XS;\n smem = 512*8+128;\n } break;\n case GGML_TYPE_IQ3_XXS:\n {\n nsg = N_SG_IQ3_XXS;\n nr0 = N_R0_IQ3_XXS;\n smem = 256*4+128;\n } break;\n case GGML_TYPE_IQ3_S:\n {\n nsg = N_SG_IQ3_S;\n nr0 = N_R0_IQ3_S;\n smem = 512*4;\n } break;\n case GGML_TYPE_IQ2_S:\n {\n nsg = N_SG_IQ2_S;\n nr0 = N_R0_IQ2_S;\n } break;\n case GGML_TYPE_IQ1_S:\n {\n nsg = N_SG_IQ1_S;\n nr0 = N_R0_IQ1_S;\n } break;\n case GGML_TYPE_IQ1_M:\n {\n nsg = N_SG_IQ1_M;\n nr0 = N_R0_IQ1_M;\n } break;\n case GGML_TYPE_IQ4_NL:\n {\n nsg = N_SG_IQ4_NL;\n nr0 = N_R0_IQ4_NL;\n smem = 32*sizeof(float);\n } break;\n case GGML_TYPE_IQ4_XS:\n {\n nsg = N_SG_IQ4_XS;\n nr0 = N_R0_IQ4_XS;\n smem = 32*sizeof(float);\n } break;\n default:\n {\n GGML_LOG_ERROR(\"Asserting on type %d\\n\", (int) tsrc0);\n GGML_ABORT(\"not implemented\");\n }\n };\n\n snprintf(base, 256, \"kernel_mul_mv_%s_%s%s\", ggml_type_name(tsrc0), ggml_type_name(tsrc1), suffix);\n snprintf(name, 256, \"%s_nsg=%d\", base, nsg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, nsg, FC_MUL_MV + 0);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n res.nr0 = nr0;\n res.nr1 = nr1;\n res.nsg = nsg;\n res.smem = smem;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(ggml_metal_library_t lib, int ne02, int ne20) {\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_mul_mm_id_map0_ne20_%d\", ne20);\n snprintf(name, 256, \"%s_ne02=%d\", base, ne02);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n res.smem = (size_t) ne02*ne20*sizeof(uint16_t);\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id(ggml_metal_library_t lib, const ggml_tensor * op) {\n char base[256];\n char name[256];\n\n const ggml_type tsrc0 = op->src[0]->type;\n const ggml_type tsrc1 = op->src[1]->type;\n\n const bool bc_inp = op->src[0]->ne[0] % 32 != 0;\n\n snprintf(base, 256, \"kernel_mul_mm_id_%s_%s\", ggml_type_name(tsrc0), ggml_type_name(tsrc1));\n snprintf(name, 256, \"%s_bci=%d\", base, bc_inp);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_bool(cv, bc_inp, FC_MUL_MM + 0);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n res.smem = 8192;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n\n char base[256];\n char name[256];\n\n int nsg = 0; // number of simdgroups\n int nr0 = 0; // number of src0 rows per simdgroup\n int nr1 = 1; // number of src1 rows per threadgroup\n\n size_t smem = 0; // shared memory\n\n const ggml_type tsrc0 = op->src[0]->type;\n const ggml_type tsrc1 = op->src[1]->type;\n\n const char * suffix = \"\";\n\n // use custom matrix x vector kernel\n switch (tsrc0) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n {\n nsg = std::min(4, (ne00 + 127) / 128);\n nr0 = 2;\n nr1 = 1;\n smem = 32*sizeof(float)*nr0;\n suffix = ne00 % 4 == 0 ? \"_4\" : \"\";\n } break;\n case GGML_TYPE_Q4_0:\n {\n nsg = N_SG_Q4_0;\n nr0 = N_R0_Q4_0;\n } break;\n case GGML_TYPE_Q4_1:\n {\n nsg = N_SG_Q4_1;\n nr0 = N_R0_Q4_1;\n } break;\n case GGML_TYPE_Q5_0:\n {\n nsg = N_SG_Q5_0;\n nr0 = N_R0_Q5_0;\n } break;\n case GGML_TYPE_Q5_1:\n {\n nsg = N_SG_Q5_1;\n nr0 = N_R0_Q5_1;\n } break;\n case GGML_TYPE_Q8_0:\n {\n nsg = N_SG_Q8_0;\n nr0 = N_R0_Q8_0;\n smem = 32*sizeof(float)*N_R0_Q8_0;\n } break;\n case GGML_TYPE_MXFP4:\n {\n nsg = N_SG_MXFP4;\n nr0 = N_R0_MXFP4;\n smem = 32*sizeof(float);\n } break;\n case GGML_TYPE_Q2_K:\n {\n nsg = N_SG_Q2_K;\n nr0 = N_R0_Q2_K;\n } break;\n case GGML_TYPE_Q3_K:\n {\n nsg = N_SG_Q3_K;\n nr0 = N_R0_Q3_K;\n } break;\n case GGML_TYPE_Q4_K:\n {\n nsg = N_SG_Q4_K;\n nr0 = N_R0_Q4_K;\n } break;\n case GGML_TYPE_Q5_K:\n {\n nsg = N_SG_Q5_K;\n nr0 = N_R0_Q5_K;\n } break;\n case GGML_TYPE_Q6_K:\n {\n nsg = N_SG_Q6_K;\n nr0 = N_R0_Q6_K;\n } break;\n case GGML_TYPE_IQ2_XXS:\n {\n nsg = N_SG_IQ2_XXS;\n nr0 = N_R0_IQ2_XXS;\n smem = 256*8+128;\n } break;\n case GGML_TYPE_IQ2_XS:\n {\n nsg = N_SG_IQ2_XS;\n nr0 = N_R0_IQ2_XS;\n smem = 512*8+128;\n } break;\n case GGML_TYPE_IQ3_XXS:\n {\n nsg = N_SG_IQ3_XXS;\n nr0 = N_R0_IQ3_XXS;\n smem = 256*4+128;\n } break;\n case GGML_TYPE_IQ3_S:\n {\n nsg = N_SG_IQ3_S;\n nr0 = N_R0_IQ3_S;\n smem = 512*4;\n } break;\n case GGML_TYPE_IQ2_S:\n {\n nsg = N_SG_IQ2_S;\n nr0 = N_R0_IQ2_S;\n } break;\n case GGML_TYPE_IQ1_S:\n {\n nsg = N_SG_IQ1_S;\n nr0 = N_R0_IQ1_S;\n } break;\n case GGML_TYPE_IQ1_M:\n {\n nsg = N_SG_IQ1_M;\n nr0 = N_R0_IQ1_M;\n } break;\n case GGML_TYPE_IQ4_NL:\n {\n nsg = N_SG_IQ4_NL;\n nr0 = N_R0_IQ4_NL;\n smem = 32*sizeof(float);\n } break;\n case GGML_TYPE_IQ4_XS:\n {\n nsg = N_SG_IQ4_XS;\n nr0 = N_R0_IQ4_XS;\n smem = 32*sizeof(float);\n } break;\n default:\n {\n GGML_LOG_ERROR(\"Asserting on type %d\\n\", (int)op->src[2]->type);\n GGML_ABORT(\"not implemented\");\n }\n };\n\n snprintf(base, 256, \"kernel_mul_mv_id_%s_%s%s\", ggml_type_name(tsrc0), ggml_type_name(tsrc1), suffix);\n snprintf(name, 256, \"%s_nsg=%d\", base, nsg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, nsg, FC_MUL_MV + 0);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n res.nr0 = nr0;\n res.nr1 = nr1;\n res.nsg = nsg;\n res.smem = smem;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(ggml_is_contiguous_1(op->src[0]));\n GGML_ASSERT(op->src[0]->nb[0] == ggml_type_size(op->src[0]->type));\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_argmax_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n res.smem = 32*(sizeof(float) + sizeof(int32_t));\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_ARGSORT);\n\n char base[256];\n char name[256];\n\n ggml_sort_order order = (ggml_sort_order) op->op_params[0];\n\n const char * order_str = \"undefined\";\n switch (order) {\n case GGML_SORT_ORDER_ASC: order_str = \"asc\"; break;\n case GGML_SORT_ORDER_DESC: order_str = \"desc\"; break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n snprintf(base, 256, \"kernel_argsort_%s_%s_%s\", ggml_type_name(op->src[0]->type), ggml_type_name(op->type), order_str);\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_ARGSORT);\n\n char base[256];\n char name[256];\n\n ggml_sort_order order = (ggml_sort_order) op->op_params[0];\n\n const char * order_str = \"undefined\";\n switch (order) {\n case GGML_SORT_ORDER_ASC: order_str = \"asc\"; break;\n case GGML_SORT_ORDER_DESC: order_str = \"desc\"; break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n snprintf(base, 256, \"kernel_argsort_merge_%s_%s_%s\", ggml_type_name(op->src[0]->type), ggml_type_name(op->type), order_str);\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\n// note: reuse the argsort kernel for top_k\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_TOP_K);\n\n char base[256];\n char name[256];\n\n // note: the top_k kernel is always descending order\n ggml_sort_order order = GGML_SORT_ORDER_DESC;\n\n const char * order_str = \"undefined\";\n switch (order) {\n case GGML_SORT_ORDER_ASC: order_str = \"asc\"; break;\n case GGML_SORT_ORDER_DESC: order_str = \"desc\"; break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n snprintf(base, 256, \"kernel_argsort_%s_%s_%s\", ggml_type_name(op->src[0]->type), ggml_type_name(op->type), order_str);\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_TOP_K);\n\n char base[256];\n char name[256];\n\n ggml_sort_order order = GGML_SORT_ORDER_DESC;\n\n const char * order_str = \"undefined\";\n switch (order) {\n case GGML_SORT_ORDER_ASC: order_str = \"asc\"; break;\n case GGML_SORT_ORDER_DESC: order_str = \"desc\"; break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n snprintf(base, 256, \"kernel_argsort_merge_%s_%s_%s\", ggml_type_name(op->src[0]->type), ggml_type_name(op->type), order_str);\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_pad(\n ggml_metal_library_t lib,\n const struct ggml_tensor * op,\n bool has_mask,\n int32_t ncpsg) {\n assert(op->op == GGML_OP_FLASH_ATTN_EXT);\n GGML_UNUSED(op);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_%s\",\n \"flash_attn_ext_pad\");\n\n snprintf(name, 256, \"%s_mask=%d_ncpsg=%d\",\n base,\n has_mask,\n ncpsg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_bool(cv, has_mask, FC_FLASH_ATTN_EXT_PAD + 0);\n //ggml_metal_cv_set_bool(cv, has_sinks, FC_FLASH_ATTN_EXT_PAD + 1);\n //ggml_metal_cv_set_bool(cv, has_bias, FC_FLASH_ATTN_EXT_PAD + 2);\n //ggml_metal_cv_set_bool(cv, has_scap, FC_FLASH_ATTN_EXT_PAD + 3);\n\n //ggml_metal_cv_set_int32(cv, ns10, FC_FLASH_ATTN_EXT_PAD + 20);\n //ggml_metal_cv_set_int32(cv, ns20, FC_FLASH_ATTN_EXT_PAD + 21);\n //ggml_metal_cv_set_int32(cv, nsg, FC_FLASH_ATTN_EXT_PAD + 22);\n //ggml_metal_cv_set_int32(cv, nwg, FC_FLASH_ATTN_EXT_PAD + 23);\n //ggml_metal_cv_set_int32(cv, nqptg, FC_FLASH_ATTN_EXT_PAD + 24);\n ggml_metal_cv_set_int32(cv, ncpsg, FC_FLASH_ATTN_EXT_PAD + 25);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk(\n ggml_metal_library_t lib,\n const struct ggml_tensor * op,\n int32_t nqptg,\n int32_t ncpsg) {\n assert(op->op == GGML_OP_FLASH_ATTN_EXT);\n GGML_UNUSED(op);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_%s\",\n \"flash_attn_ext_blk\");\n\n snprintf(name, 256, \"%s_nqptg=%d_ncpsg=%d\",\n base,\n nqptg,\n ncpsg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n //ggml_metal_cv_set_bool(cv, has_mask, FC_FLASH_ATTN_EXT_BLK + 0);\n //ggml_metal_cv_set_bool(cv, has_sinks, FC_FLASH_ATTN_EXT_BLK + 1);\n //ggml_metal_cv_set_bool(cv, has_bias, FC_FLASH_ATTN_EXT_BLK + 2);\n //ggml_metal_cv_set_bool(cv, has_scap, FC_FLASH_ATTN_EXT_BLK + 3);\n\n //ggml_metal_cv_set_int32(cv, ns10, FC_FLASH_ATTN_EXT_BLK + 20);\n //ggml_metal_cv_set_int32(cv, ns20, FC_FLASH_ATTN_EXT_BLK + 21);\n //ggml_metal_cv_set_int32(cv, nsg, FC_FLASH_ATTN_EXT_BLK + 22);\n //ggml_metal_cv_set_int32(cv, nwg, FC_FLASH_ATTN_EXT_BLK + 23);\n ggml_metal_cv_set_int32(cv, nqptg, FC_FLASH_ATTN_EXT_BLK + 24);\n ggml_metal_cv_set_int32(cv, ncpsg, FC_FLASH_ATTN_EXT_BLK + 25);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(\n ggml_metal_library_t lib,\n const ggml_tensor * op,\n bool has_mask,\n bool has_sinks,\n bool has_bias,\n bool has_scap,\n bool has_kvpad,\n int32_t nsg) {\n assert(op->op == GGML_OP_FLASH_ATTN_EXT);\n\n char base[256];\n char name[256];\n\n const int32_t dk = (int32_t) op->src[1]->ne[0];\n const int32_t dv = (int32_t) op->src[2]->ne[0];\n\n const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0];\n const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0];\n\n // do bounds checks for the mask?\n const bool bc_mask = op->src[3] && (op->src[3]->ne[1] % 8 != 0);\n\n snprintf(base, 256, \"kernel_%s_%s_dk%d_dv%d\",\n \"flash_attn_ext\",\n ggml_type_name(op->src[1]->type),\n dk,\n dv);\n\n snprintf(name, 256, \"%s_mask=%d_sinks=%d_bias=%d_scap=%d_kvpad=%d_bcm=%d_ns10=%d_ns20=%d_nsg=%d\",\n base,\n has_mask,\n has_sinks,\n has_bias,\n has_scap,\n has_kvpad,\n bc_mask,\n ns10,\n ns20,\n nsg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_bool(cv, has_mask, FC_FLASH_ATTN_EXT + 0);\n ggml_metal_cv_set_bool(cv, has_sinks, FC_FLASH_ATTN_EXT + 1);\n ggml_metal_cv_set_bool(cv, has_bias, FC_FLASH_ATTN_EXT + 2);\n ggml_metal_cv_set_bool(cv, has_scap, FC_FLASH_ATTN_EXT + 3);\n ggml_metal_cv_set_bool(cv, has_kvpad, FC_FLASH_ATTN_EXT + 4);\n\n ggml_metal_cv_set_bool(cv, bc_mask, FC_FLASH_ATTN_EXT + 10);\n\n ggml_metal_cv_set_int32(cv, ns10, FC_FLASH_ATTN_EXT + 20);\n ggml_metal_cv_set_int32(cv, ns20, FC_FLASH_ATTN_EXT + 21);\n ggml_metal_cv_set_int32(cv, nsg, FC_FLASH_ATTN_EXT + 22);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec(\n ggml_metal_library_t lib,\n const ggml_tensor * op,\n bool has_mask,\n bool has_sinks,\n bool has_bias,\n bool has_scap,\n bool has_kvpad,\n int32_t nsg,\n int32_t nwg) {\n assert(op->op == GGML_OP_FLASH_ATTN_EXT);\n\n char base[256];\n char name[256];\n\n const int32_t dk = (int32_t) op->src[1]->ne[0];\n const int32_t dv = (int32_t) op->src[2]->ne[0];\n\n const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0];\n const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0];\n\n snprintf(base, 256, \"kernel_%s_%s_dk%d_dv%d\",\n \"flash_attn_ext_vec\",\n ggml_type_name(op->src[1]->type),\n dk,\n dv);\n\n snprintf(name, 256, \"%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d\",\n base,\n has_mask,\n has_sinks,\n has_bias,\n has_scap,\n has_kvpad,\n ns10,\n ns20,\n nsg, nwg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_bool(cv, has_mask, FC_FLASH_ATTN_EXT_VEC + 0);\n ggml_metal_cv_set_bool(cv, has_sinks, FC_FLASH_ATTN_EXT_VEC + 1);\n ggml_metal_cv_set_bool(cv, has_bias, FC_FLASH_ATTN_EXT_VEC + 2);\n ggml_metal_cv_set_bool(cv, has_scap, FC_FLASH_ATTN_EXT_VEC + 3);\n ggml_metal_cv_set_bool(cv, has_kvpad, FC_FLASH_ATTN_EXT_VEC + 4);\n\n ggml_metal_cv_set_int32(cv, ns10, FC_FLASH_ATTN_EXT_VEC + 20);\n ggml_metal_cv_set_int32(cv, ns20, FC_FLASH_ATTN_EXT_VEC + 21);\n ggml_metal_cv_set_int32(cv, nsg, FC_FLASH_ATTN_EXT_VEC + 22);\n ggml_metal_cv_set_int32(cv, nwg, FC_FLASH_ATTN_EXT_VEC + 23);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce(\n ggml_metal_library_t lib,\n const ggml_tensor * op,\n int32_t dv,\n int32_t nwg) {\n assert(op->op == GGML_OP_FLASH_ATTN_EXT);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_flash_attn_ext_vec_reduce\");\n snprintf(name, 256, \"%s_dv=%d_nwg=%d\", base, dv, nwg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int32(cv, dv, FC_FLASH_ATTN_EXT_VEC_REDUCE + 0);\n ggml_metal_cv_set_int32(cv, nwg, FC_FLASH_ATTN_EXT_VEC_REDUCE + 1);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n\n GGML_UNUSED(op);\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin(ggml_metal_library_t lib, const ggml_tensor * op, int32_t n_fuse) {\n char base[256];\n char name[256];\n\n int op_num = -1;\n\n switch (op->op) {\n case GGML_OP_ADD: op_num = 0; break;\n case GGML_OP_SUB: op_num = 1; break;\n case GGML_OP_MUL: op_num = 2; break;\n case GGML_OP_DIV: op_num = 3; break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n const char * t0_str = ggml_type_name(op->src[0]->type);\n const char * t1_str = ggml_type_name(op->src[1]->type);\n const char * t_str = ggml_type_name(op->type);\n\n const bool is_c4 = (op->src[0]->ne[0] % 4 == 0) && (op->src[1]->ne[0] % 4 == 0);\n\n const bool is_cb = op->src[0]->ne[0] != op->src[1]->ne[0];\n const bool is_rb = ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && (ggml_nrows(op->src[1]) == 1) && ggml_nelements(op) < 65536;\n\n snprintf(base, 256, \"kernel_bin_fuse_%s_%s_%s%s\", t0_str, t1_str, t_str, is_c4 ? \"_4\" : \"\");\n snprintf(name, 256, \"%s_op=%d_nf=%d_rb=%d_cb=%d\", base, op_num, n_fuse, is_rb, is_cb);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, op_num, FC_BIN + 0);\n ggml_metal_cv_set_int16(cv, n_fuse, FC_BIN + 1);\n ggml_metal_cv_set_bool (cv, is_rb, FC_BIN + 2);\n ggml_metal_cv_set_bool (cv, is_cb, FC_BIN + 3);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n res.c4 = is_c4;\n res.cnt = is_rb;\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin_one(ggml_metal_library_t lib, ggml_op op) {\n char base[256];\n char name[256];\n\n int op_num = -1;\n\n switch (op) {\n case GGML_OP_ADD: op_num = 0; break;\n case GGML_OP_SUB: op_num = 1; break;\n case GGML_OP_MUL: op_num = 2; break;\n case GGML_OP_DIV: op_num = 3; break;\n default: GGML_ABORT(\"fatal error\");\n };\n\n snprintf(base, 256, \"kernel_bin_fuse_%s_%s_%s\", \"f32\", \"f32\", \"f32\");\n snprintf(name, 256, \"%s_op=%d_nf=%d\", base, op_num, 1);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, op_num, FC_BIN + 0);\n ggml_metal_cv_set_int16(cv, 1, FC_BIN + 1);\n ggml_metal_cv_set_bool (cv, false, FC_BIN + 2);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_l2_norm(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_L2_NORM);\n\n char base[256];\n char name[256];\n\n const bool is_c4 = op->src[0]->ne[0] % 4 == 0;\n\n const char * t0_str = ggml_type_name(op->src[0]->type);\n const char * t_str = ggml_type_name(op->type);\n\n snprintf(base, 256, \"kernel_l2_norm_%s_%s%s\", t0_str, t_str, is_c4 ? \"_4\" : \"\");\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n res.c4 = is_c4;\n res.smem = 32*sizeof(float);\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_group_norm(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_GROUP_NORM);\n\n GGML_ASSERT(ggml_is_contiguous(op->src[0]));\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_group_norm_f32\");\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n res.smem = 32*sizeof(float);\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm(ggml_metal_library_t lib, const ggml_tensor * op, int n_fuse) {\n assert(op->op == GGML_OP_NORM || op->op == GGML_OP_RMS_NORM);\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n\n char base[256];\n char name[256];\n\n const char * suffix = \"\";\n if (op->ne[0] % 4 == 0) {\n suffix = \"_4\";\n }\n\n switch (op->op) {\n case GGML_OP_NORM:\n switch (n_fuse) {\n case 1: snprintf(base, 256, \"kernel_norm_f32%s\", suffix); break;\n case 2: snprintf(base, 256, \"kernel_norm_mul_f32%s\", suffix); break;\n case 3: snprintf(base, 256, \"kernel_norm_mul_add_f32%s\", suffix); break;\n default: GGML_ABORT(\"fatal error\");\n } break;\n case GGML_OP_RMS_NORM:\n switch (n_fuse) {\n case 1: snprintf(base, 256, \"kernel_rms_norm_f32%s\", suffix); break;\n case 2: snprintf(base, 256, \"kernel_rms_norm_mul_f32%s\", suffix); break;\n case 3: snprintf(base, 256, \"kernel_rms_norm_mul_add_f32%s\", suffix); break;\n default: GGML_ABORT(\"fatal error\");\n } break;\n default: GGML_ABORT(\"fatal error\");\n }\n\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n res.smem = 32*sizeof(float);\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_ROPE);\n\n char base[256];\n char name[256];\n\n const int mode = ((const int32_t *) op->op_params)[2];\n\n const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;\n const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;\n const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE;\n const bool is_vision = mode == GGML_ROPE_TYPE_VISION;\n\n if (is_neox) {\n snprintf(base, 256, \"kernel_rope_neox_%s\", ggml_type_name(op->src[0]->type));\n } else if ((is_mrope || is_imrope) && !is_vision) {\n GGML_ASSERT(op->src[1]->ne[0]*4 >= op->src[0]->ne[2]); // need at least 4 pos per token\n snprintf(base, 256, \"kernel_rope_multi_%s\", ggml_type_name(op->src[0]->type));\n } else if (is_vision) {\n GGML_ASSERT(op->src[1]->ne[0]*4 >= op->src[0]->ne[2]); // need at least 4 pos per token\n snprintf(base, 256, \"kernel_rope_vision_%s\", ggml_type_name(op->src[0]->type));\n } else {\n snprintf(base, 256, \"kernel_rope_norm_%s\", ggml_type_name(op->src[0]->type));\n }\n\n snprintf(name, 256, \"%s_imrope=%d\", base, is_imrope ? 1 : 0);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_bool(cv, is_imrope, FC_ROPE + 0);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_im2col(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_IM2COL);\n\n GGML_ASSERT(ggml_is_contiguous(op->src[1]));\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_im2col_%s\", ggml_type_name(op->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_1d(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_CONV_TRANSPOSE_1D);\n\n GGML_ASSERT(ggml_is_contiguous(op->src[0]));\n GGML_ASSERT(ggml_is_contiguous(op->src[1]));\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->type == GGML_TYPE_F32);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_conv_transpose_1d_%s_%s\", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_2d(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_CONV_TRANSPOSE_2D);\n\n GGML_ASSERT(ggml_is_contiguous(op->src[0]));\n GGML_ASSERT(ggml_is_contiguous(op->src[1]));\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->type == GGML_TYPE_F32);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_conv_transpose_2d_%s_%s\", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_CONV_2D);\n\n GGML_ASSERT(ggml_is_contiguous(op->src[0]));\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->type == GGML_TYPE_F32);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_conv_2d_%s_%s\", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_UPSCALE);\n\n char base[256];\n char name[256];\n\n const int32_t mode_flags = ggml_get_op_params_i32(op, 0);\n const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF);\n\n const bool antialias = (mode_flags & GGML_SCALE_FLAG_ANTIALIAS);\n\n if (mode == GGML_SCALE_MODE_BILINEAR) {\n snprintf(base, 256, \"kernel_upscale_bilinear_%s\", ggml_type_name(op->src[0]->type));\n } else if (mode == GGML_SCALE_MODE_BICUBIC) {\n snprintf(base, 256, \"kernel_upscale_bicubic_%s\", ggml_type_name(op->src[0]->type));\n } else {\n snprintf(base, 256, \"kernel_upscale_nearest_%s\", ggml_type_name(op->src[0]->type));\n }\n snprintf(name, 256, \"%s_aa=%d\", base, antialias);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_bool(cv, antialias, FC_UPSCALE + 0);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_PAD);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_pad_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (res.pipeline) {\n return res;\n }\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad_reflect_1d(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_PAD_REFLECT_1D);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_pad_reflect_1d_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_arange(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_ARANGE);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_arange_%s\", ggml_type_name(op->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_timestep_embedding(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_TIMESTEP_EMBEDDING);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_timestep_embedding_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_adamw(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_OPT_STEP_ADAMW);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_opt_step_adamw_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_sgd(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_OPT_STEP_SGD);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_opt_step_sgd_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_memset(ggml_metal_library_t lib, const ggml_tensor * op) {\n GGML_ASSERT(op->type == GGML_TYPE_I64);\n\n char base[256];\n char name[256];\n\n snprintf(base, 256, \"kernel_memset_%s\", ggml_type_name(op->type));\n snprintf(name, 256, \"%s\", base);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);\n }\n\n return res;\n}\n\nggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_count_equal(ggml_metal_library_t lib, const ggml_tensor * op) {\n assert(op->op == GGML_OP_COUNT_EQUAL);\n\n GGML_TENSOR_LOCALS(int64_t, ne0, op->src[0], ne);\n\n GGML_ASSERT(op->src[0]->type == op->src[1]->type);\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_I32);\n GGML_ASSERT(op->type == GGML_TYPE_I64);\n\n // note: the kernel only supports i32 output due to metal atomic add only supporting atomic_int\n GGML_ASSERT(ggml_nelements(op->src[0]) < (1LL << 31));\n\n char base[256];\n char name[256];\n\n int nsg = 1;\n while (32*nsg < ne00 && nsg < 32) {\n nsg *= 2;\n }\n\n snprintf(base, 256, \"kernel_count_equal_%s\", ggml_type_name(op->src[0]->type));\n snprintf(name, 256, \"%s_nsg=%d\", base, nsg);\n\n ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);\n if (!res.pipeline) {\n ggml_metal_cv_t cv = ggml_metal_cv_init();\n\n ggml_metal_cv_set_int16(cv, nsg, FC_COUNT_EQUAL + 0);\n\n res = ggml_metal_library_compile_pipeline(lib, base, name, cv);\n\n ggml_metal_cv_free(cv);\n }\n\n res.smem = 32 * sizeof(int32_t);\n res.nsg = nsg;\n\n return res;\n}\n" }, { "path": "src/ggml-metal/ggml-metal-device.h", "content": "#pragma once\n\n#include \"ggml.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nstruct ggml_metal_buffer_id {\n void * metal; // id\n size_t offs;\n};\n\ntypedef struct ggml_metal_device * ggml_metal_device_t;\n\n//\n// MTLFunctionConstantValues wrapper\n//\n\ntypedef struct ggml_metal_cv * ggml_metal_cv_t;\n\nggml_metal_cv_t ggml_metal_cv_init(void);\nvoid ggml_metal_cv_free(ggml_metal_cv_t cv);\n\nvoid ggml_metal_cv_set_int16(ggml_metal_cv_t cv, int16_t value, int32_t idx);\nvoid ggml_metal_cv_set_int32(ggml_metal_cv_t cv, int32_t value, int32_t idx);\nvoid ggml_metal_cv_set_bool (ggml_metal_cv_t cv, bool value, int32_t idx);\n\n//\n// MTLComputePipelineState wrapper\n//\n\ntypedef struct ggml_metal_pipeline * ggml_metal_pipeline_t;\n\nggml_metal_pipeline_t ggml_metal_pipeline_init(void);\nvoid ggml_metal_pipeline_free(ggml_metal_pipeline_t pipeline);\n\n// a collection of pipelines\ntypedef struct ggml_metal_pipelines * ggml_metal_pipelines_t;\n\nggml_metal_pipelines_t ggml_metal_pipelines_init(void);\nvoid ggml_metal_pipelines_free(ggml_metal_pipelines_t ppls);\n\nvoid ggml_metal_pipelines_add(ggml_metal_pipelines_t ppls, const char * name, ggml_metal_pipeline_t pipeline);\nggml_metal_pipeline_t ggml_metal_pipelines_get(ggml_metal_pipelines_t ppls, const char * name);\n\nstruct ggml_metal_pipeline_with_params {\n ggml_metal_pipeline_t pipeline;\n\n int nsg;\n\n int nr0;\n int nr1;\n\n size_t smem;\n\n bool c4;\n bool cnt;\n};\n\nint ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_with_params pipeline);\n\n//\n// MTLCommandBuffer wrapper\n//\n\ntypedef void * ggml_metal_cmd_buf_t;\n\n//\n// MTLComputeCommandEncoder wrapper\n//\n\ntypedef struct ggml_metal_encoder * ggml_metal_encoder_t;\n\nggml_metal_encoder_t ggml_metal_encoder_init(ggml_metal_cmd_buf_t cmd_buf_raw, bool concurrent);\nvoid ggml_metal_encoder_free(ggml_metal_encoder_t encoder);\n\nvoid ggml_metal_encoder_debug_group_push(ggml_metal_encoder_t encoder, const char * name);\nvoid ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder);\n\nvoid ggml_metal_encoder_set_pipeline(ggml_metal_encoder_t encoder, struct ggml_metal_pipeline_with_params pipeline);\n\nvoid ggml_metal_encoder_set_bytes (ggml_metal_encoder_t encoder, void * data, size_t size, int idx);\nvoid ggml_metal_encoder_set_buffer(ggml_metal_encoder_t encoder, struct ggml_metal_buffer_id buffer, int idx);\n\nvoid ggml_metal_encoder_set_threadgroup_memory_size(ggml_metal_encoder_t encoder, size_t size, int idx);\n\nvoid ggml_metal_encoder_dispatch_threadgroups(ggml_metal_encoder_t encoder, int tg0, int tg1, int tg2, int tptg0, int tptg1, int tptg2);\n\nvoid ggml_metal_encoder_memory_barrier(ggml_metal_encoder_t encoder);\n\nvoid ggml_metal_encoder_end_encoding(ggml_metal_encoder_t encoder);\n\n//\n// MTLLibrary wrapper\n//\n\ntypedef struct ggml_metal_library * ggml_metal_library_t;\n\nggml_metal_library_t ggml_metal_library_init (ggml_metal_device_t dev);\nggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev, const char * source, bool verbose);\n\nvoid ggml_metal_library_free(ggml_metal_library_t lib);\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline (ggml_metal_library_t lib, const char * name);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_metal_library_t lib, const char * base, const char * name, ggml_metal_cv_t cv);\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_base (ggml_metal_library_t lib, enum ggml_op op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cpy (ggml_metal_library_t lib, enum ggml_type tsrc, enum ggml_type tdst);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_1d (ggml_metal_library_t lib, const struct ggml_tensor * op, enum ggml_op_pool op_pool);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_2d (ggml_metal_library_t lib, const struct ggml_tensor * op, enum ggml_op_pool op_pool);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_get_rows (ggml_metal_library_t lib, enum ggml_type tsrc);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows (ggml_metal_library_t lib, enum ggml_type tidx, enum ggml_type tdst);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diag (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat (ggml_metal_library_t lib, enum ggml_type tsrc);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum_rows (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_blk (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_add (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rwkv (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_gated_delta_net (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_solve_tri (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext (ggml_metal_library_t lib, enum ggml_type tsrc0, enum ggml_type tsrc1, int nsg, int nxpsg, int r1ptg);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0 (ggml_metal_library_t lib, int ne02, int ne20);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse );\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin_one (ggml_metal_library_t lib, enum ggml_op op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_l2_norm (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_group_norm (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_im2col (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_1d (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad_reflect_1d (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_arange (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_timestep_embedding(ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_adamw (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_sgd (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_memset (ggml_metal_library_t lib, const struct ggml_tensor * op);\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_count_equal (ggml_metal_library_t lib, const struct ggml_tensor * op);\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_pad(\n ggml_metal_library_t lib,\n const struct ggml_tensor * op,\n bool has_mask,\n int32_t ncpsg);\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk(\n ggml_metal_library_t lib,\n const struct ggml_tensor * op,\n int32_t nqptg,\n int32_t ncpsg);\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(\n ggml_metal_library_t lib,\n const struct ggml_tensor * op,\n bool has_mask,\n bool has_sinks,\n bool has_bias,\n bool has_scap,\n bool has_kvpad,\n int32_t nsg);\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec(\n ggml_metal_library_t lib,\n const struct ggml_tensor * op,\n bool has_mask,\n bool has_sinks,\n bool has_bias,\n bool has_scap,\n bool has_kvpad,\n int32_t nsg,\n int32_t nwg);\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce(\n ggml_metal_library_t lib,\n const struct ggml_tensor * op,\n int32_t dv,\n int32_t nwg);\n\n// MTLResidencySet wrapper\n\ntypedef void * ggml_metal_rset_t;\n\n// a collection of residency sets (non-owning)\ntypedef struct ggml_metal_rsets * ggml_metal_rsets_t;\n\nggml_metal_rsets_t ggml_metal_rsets_init(void);\nvoid ggml_metal_rsets_free(ggml_metal_rsets_t rsets);\n\n//\n// device\n//\n\nstruct ggml_metal_device_props {\n int device;\n char name[128];\n char desc[128];\n\n size_t max_buffer_size;\n size_t max_working_set_size;\n size_t max_theadgroup_memory_size;\n\n bool has_simdgroup_reduction;\n bool has_simdgroup_mm;\n bool has_unified_memory;\n bool has_bfloat;\n bool has_tensor;\n bool use_residency_sets;\n bool use_shared_buffers;\n\n bool supports_gpu_family_apple7;\n\n int op_offload_min_batch_size;\n};\n\ntypedef struct ggml_metal_event * ggml_metal_event_t;\n\nvoid ggml_metal_event_encode_signal(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf);\nvoid ggml_metal_event_encode_wait (ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf);\n\nggml_metal_device_t ggml_metal_device_init(int device);\nvoid ggml_metal_device_free(ggml_metal_device_t dev);\n\nggml_metal_device_t ggml_metal_device_get(int device);\n\nvoid * ggml_metal_device_get_obj (ggml_metal_device_t dev); // id\nvoid * ggml_metal_device_get_queue(ggml_metal_device_t dev); // id\n\nggml_metal_library_t ggml_metal_device_get_library(ggml_metal_device_t dev);\n\nvoid ggml_metal_device_rsets_add(ggml_metal_device_t dev, ggml_metal_rset_t rset);\nvoid ggml_metal_device_rsets_rm (ggml_metal_device_t dev, ggml_metal_rset_t rset);\n\nvoid ggml_metal_device_rsets_keep_alive(ggml_metal_device_t dev);\n\nggml_metal_event_t ggml_metal_device_event_init(ggml_metal_device_t dev);\nvoid ggml_metal_device_event_free(ggml_metal_device_t dev, ggml_metal_event_t ev);\nvoid ggml_metal_device_event_synchronize(ggml_metal_device_t dev, ggml_metal_event_t ev);\n\nvoid ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t * total);\nbool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tensor * op);\n\nconst struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev);\n\n//\n// device buffers\n//\n\ntypedef struct ggml_metal_buffer * ggml_metal_buffer_t;\n\nggml_metal_buffer_t ggml_metal_buffer_init(ggml_metal_device_t dev, size_t size, bool shared);\nggml_metal_buffer_t ggml_metal_buffer_map (ggml_metal_device_t dev, void * ptr, size_t size, size_t max_tensor_size);\n\nvoid ggml_metal_buffer_free (ggml_metal_buffer_t buf);\nvoid * ggml_metal_buffer_get_base (ggml_metal_buffer_t buf);\nbool ggml_metal_buffer_is_shared(ggml_metal_buffer_t buf);\n\nvoid ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);\nvoid ggml_metal_buffer_set_tensor (ggml_metal_buffer_t buf, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\nvoid ggml_metal_buffer_get_tensor (ggml_metal_buffer_t buf, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);\nvoid ggml_metal_buffer_clear (ggml_metal_buffer_t buf, uint8_t value);\n\n// finds the Metal buffer that contains the tensor data on the GPU device\n// the assumption is that there is 1-to-1 mapping between the host and device memory buffers, so we can find the\n// Metal buffer based on the host memory pointer\n//\nstruct ggml_metal_buffer_id ggml_metal_buffer_get_id(ggml_metal_buffer_t buf, const struct ggml_tensor * t);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-metal/ggml-metal-device.m", "content": "#import \"ggml-metal-device.h\"\n\n#import \"ggml-impl.h\"\n\n#include \n\n#include \n\n#include \n\n#ifndef TARGET_OS_VISION\n#define TARGET_OS_VISION 0\n#endif\n\n// create residency sets only on macOS >= 15.0\n#if !TARGET_CPU_X86_64 && TARGET_OS_OSX && __MAC_OS_X_VERSION_MAX_ALLOWED >= 150000 || \\\n TARGET_OS_IOS && __IPHONE_OS_VERSION_MAX_ALLOWED >= 180000 || \\\n TARGET_OS_TV && __TV_OS_VERSION_MAX_ALLOWED >= 180000 || \\\n TARGET_OS_VISION && __VISION_OS_VERSION_MAX_ALLOWED >= 200000\n#define GGML_METAL_HAS_RESIDENCY_SETS 1\n#endif\n\n// overload of MTLGPUFamilyMetalX (not available in some environments)\nstatic const NSInteger MTLGPUFamilyMetal3_GGML = 5001;\nstatic const NSInteger MTLGPUFamilyMetal4_GGML = 5002;\n\n#if !GGML_METAL_EMBED_LIBRARY\n// Here to assist with NSBundle Path Hack\n@interface GGMLMetalClass : NSObject\n@end\n@implementation GGMLMetalClass\n@end\n#endif\n\n//\n// MTLFunctionConstantValues wrapper\n//\n\nstruct ggml_metal_cv {\n MTLFunctionConstantValues * obj;\n};\n\nggml_metal_cv_t ggml_metal_cv_init(void) {\n ggml_metal_cv_t res = calloc(1, sizeof(struct ggml_metal_cv));\n\n res->obj = [[MTLFunctionConstantValues alloc] init];\n\n return res;\n}\n\nvoid ggml_metal_cv_free(ggml_metal_cv_t cv) {\n [cv->obj release];\n free(cv);\n}\n\nvoid ggml_metal_cv_set_int16(ggml_metal_cv_t cv, int16_t value, int32_t idx) {\n [cv->obj setConstantValue:&value type:MTLDataTypeShort atIndex:idx];\n}\n\nvoid ggml_metal_cv_set_int32(ggml_metal_cv_t cv, int32_t value, int32_t idx) {\n [cv->obj setConstantValue:&value type:MTLDataTypeInt atIndex:idx];\n}\n\nvoid ggml_metal_cv_set_bool(ggml_metal_cv_t cv, bool value, int32_t idx) {\n [cv->obj setConstantValue:&value type:MTLDataTypeBool atIndex:idx];\n}\n\n//\n// MTLComputePipelineState wrapper\n//\n\nstruct ggml_metal_pipeline {\n id obj;\n};\n\nggml_metal_pipeline_t ggml_metal_pipeline_init(void) {\n ggml_metal_pipeline_t res = calloc(1, sizeof(struct ggml_metal_pipeline));\n\n *res = (struct ggml_metal_pipeline) {\n /*.obj =*/ nil,\n };\n\n return res;\n}\n\nvoid ggml_metal_pipeline_free(ggml_metal_pipeline_t pipeline) {\n [pipeline->obj release];\n\n free(pipeline);\n}\n\nint ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_with_params pipeline) {\n return pipeline.pipeline->obj.maxTotalThreadsPerThreadgroup;\n}\n\nstruct ggml_metal_library {\n id obj;\n id device;\n\n ggml_metal_pipelines_t pipelines; // cache of compiled pipelines\n\n NSLock * lock;\n};\n\nggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {\n id library = nil;\n id device = ggml_metal_device_get_obj(dev);\n\n // load library\n //\n // - first check if the library is embedded\n // - then check if the library is in the bundle\n // - if not found, load the source and compile it\n // - if that fails, return NULL\n //\n // TODO: move to a function\n {\n const int64_t t_start = ggml_time_us();\n\n NSError * error = nil;\n NSString * src = nil;\n\n#if GGML_METAL_EMBED_LIBRARY\n GGML_LOG_INFO(\"%s: using embedded metal library\\n\", __func__);\n\n extern const char ggml_metallib_start[];\n extern const char ggml_metallib_end[];\n\n src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding];\n#else\n\n#ifdef SWIFT_PACKAGE\n NSBundle * bundle = SWIFTPM_MODULE_BUNDLE;\n#else\n NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];\n#endif\n\n NSString * path_lib = [bundle pathForResource:@\"default\" ofType:@\"metallib\"];\n if (path_lib == nil) {\n // Try to find the resource in the directory where the current binary located.\n NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];\n NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];\n\n NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, @\"default.metallib\"]];\n if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {\n GGML_LOG_INFO(\"%s: found '%s'\\n\", __func__, [path_lib_default UTF8String]);\n\n NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];\n if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {\n // Optionally, if this is a symlink, try to resolve it.\n path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];\n if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@\"/\"]) {\n // It is a relative path, adding the binary directory as directory prefix.\n path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];\n }\n if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {\n // Link to the resource could not be resolved.\n path_lib_default = nil;\n } else {\n GGML_LOG_INFO(\"%s: symlink resolved '%s'\\n\", __func__, [path_lib_default UTF8String]);\n }\n }\n } else {\n // The resource couldn't be found in the binary's directory.\n path_lib_default = nil;\n }\n\n path_lib = path_lib_default;\n }\n\n if (path_lib != nil) {\n // pre-compiled library found\n NSURL * libURL = [NSURL fileURLWithPath:path_lib];\n GGML_LOG_INFO(\"%s: loading '%s'\\n\", __func__, [path_lib UTF8String]);\n\n library = [device newLibraryWithURL:libURL error:&error];\n if (error) {\n GGML_LOG_ERROR(\"%s: error: %s\\n\", __func__, [[error description] UTF8String]);\n return nil;\n }\n } else {\n GGML_LOG_INFO(\"%s: default.metallib not found, loading from source\\n\", __func__);\n\n NSString * path_source;\n NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@\"GGML_METAL_PATH_RESOURCES\"];\n\n GGML_LOG_INFO(\"%s: GGML_METAL_PATH_RESOURCES = %s\\n\", __func__, path_resource ? [path_resource UTF8String] : \"nil\");\n\n if (path_resource) {\n path_source = [path_resource stringByAppendingPathComponent:@\"ggml-metal.metal\"];\n } else {\n path_source = [bundle pathForResource:@\"ggml-metal\" ofType:@\"metal\"];\n }\n\n if (path_source == nil) {\n GGML_LOG_WARN(\"%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\\n\", __func__);\n path_source = @\"ggml-metal.metal\";\n }\n\n GGML_LOG_INFO(\"%s: loading '%s'\\n\", __func__, [path_source UTF8String]);\n\n src = [NSString stringWithContentsOfFile:path_source encoding:NSUTF8StringEncoding error:&error];\n if (error) {\n GGML_LOG_ERROR(\"%s: error: %s\\n\", __func__, [[error description] UTF8String]);\n return nil;\n }\n }\n#endif\n\n if (!library) {\n @autoreleasepool {\n // dictionary of preprocessor macros\n NSMutableDictionary * prep = [NSMutableDictionary dictionary];\n\n if (ggml_metal_device_get_props(dev)->has_bfloat) {\n [prep setObject:@\"1\" forKey:@\"GGML_METAL_HAS_BF16\"];\n }\n\n if (ggml_metal_device_get_props(dev)->has_tensor) {\n [prep setObject:@\"1\" forKey:@\"GGML_METAL_HAS_TENSOR\"];\n }\n\n#if GGML_METAL_EMBED_LIBRARY\n [prep setObject:@\"1\" forKey:@\"GGML_METAL_EMBED_LIBRARY\"];\n#endif\n\n MTLCompileOptions * options = [MTLCompileOptions new];\n options.preprocessorMacros = prep;\n\n //[options setFastMathEnabled:false];\n\n library = [device newLibraryWithSource:src options:options error:&error];\n if (error) {\n GGML_LOG_ERROR(\"%s: error: %s\\n\", __func__, [[error description] UTF8String]);\n return nil;\n }\n\n#if !__has_feature(objc_arc)\n [options release];\n#endif\n }\n }\n\n#if GGML_METAL_EMBED_LIBRARY\n [src release];\n#endif // GGML_METAL_EMBED_LIBRARY\n\n GGML_LOG_INFO(\"%s: loaded in %.3f sec\\n\", __func__, (ggml_time_us() - t_start) / 1e6);\n }\n\n ggml_metal_library_t res = calloc(1, sizeof(struct ggml_metal_library));\n\n res->obj = library;\n res->device = device;\n res->pipelines = ggml_metal_pipelines_init();\n res->lock = [NSLock new];\n\n return res;\n}\n\nggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev, const char * source, bool verbose) {\n if (source == NULL) {\n GGML_LOG_ERROR(\"%s: source is NULL\\n\", __func__);\n return NULL;\n }\n\n id device = ggml_metal_device_get_obj(dev);\n id library = nil;\n NSError * error = nil;\n\n const int64_t t_start = ggml_time_us();\n\n NSString * src = [[NSString alloc] initWithBytes:source\n length:strlen(source)\n encoding:NSUTF8StringEncoding];\n if (!src) {\n GGML_LOG_ERROR(\"%s: failed to create NSString from source\\n\", __func__);\n return NULL;\n }\n\n @autoreleasepool {\n NSMutableDictionary * prep = [NSMutableDictionary dictionary];\n\n MTLCompileOptions * options = [MTLCompileOptions new];\n options.preprocessorMacros = prep;\n\n library = [device newLibraryWithSource:src options:options error:&error];\n if (error) {\n if (verbose) {\n GGML_LOG_ERROR(\"%s: error compiling source: %s\\n\", __func__, [[error description] UTF8String]);\n } else {\n GGML_LOG_ERROR(\"%s: error compiling source\\n\", __func__);\n }\n library = nil;\n }\n\n [options release];\n }\n\n [src release];\n\n if (!library) {\n if (verbose) {\n GGML_LOG_ERROR(\"%s: failed to create Metal library from source\\n\", __func__);\n }\n\n return NULL;\n }\n\n if (verbose) {\n GGML_LOG_INFO(\"%s: compiled in %.3f sec\\n\", __func__, (ggml_time_us() - t_start) / 1e6);\n }\n\n ggml_metal_library_t res = calloc(1, sizeof(struct ggml_metal_library));\n if (!res) {\n GGML_LOG_ERROR(\"%s: calloc failed\\n\", __func__);\n return NULL;\n }\n\n res->obj = library;\n res->device = device;\n res->pipelines = ggml_metal_pipelines_init();\n res->lock = [NSLock new];\n\n return res;\n}\n\nvoid ggml_metal_library_free(ggml_metal_library_t lib) {\n if (!lib) {\n return;\n }\n\n if (lib->obj) {\n [lib->obj release];\n }\n\n ggml_metal_pipelines_free(lib->pipelines);\n\n [lib->lock release];\n\n free(lib);\n}\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline(ggml_metal_library_t lib, const char * name) {\n [lib->lock lock];\n\n struct ggml_metal_pipeline_with_params res = {\n /*.pipeline =*/ nil,\n /*.nsg =*/ 0,\n /*.nr0 =*/ 0,\n /*.nr1 =*/ 0,\n /*.smem =*/ 0,\n /*.c4 =*/ false,\n /*.cnt =*/ false,\n };\n\n res.pipeline = ggml_metal_pipelines_get(lib->pipelines, name);\n\n [lib->lock unlock];\n\n return res;\n}\n\nstruct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_metal_library_t lib, const char * base, const char * name, ggml_metal_cv_t cv) {\n struct ggml_metal_pipeline_with_params res = {\n /*.pipeline =*/ nil,\n /*.nsg =*/ 0,\n /*.nr0 =*/ 0,\n /*.nr1 =*/ 0,\n /*.smem =*/ 0,\n /*.c4 =*/ false,\n /*.cnt =*/ false,\n };\n\n [lib->lock lock];\n\n res.pipeline = ggml_metal_pipelines_get(lib->pipelines, name);\n if (res.pipeline) {\n [lib->lock unlock];\n\n return res;\n }\n\n @autoreleasepool {\n NSError * error = nil;\n\n NSString * base_func = [NSString stringWithUTF8String:base];\n\n GGML_LOG_DEBUG(\"%s: compiling pipeline: base = '%s', name = '%s'\\n\", __func__, base, name);\n\n id mtl_function;\n if (!cv) {\n mtl_function = [lib->obj newFunctionWithName:base_func];\n } else {\n mtl_function = [lib->obj newFunctionWithName:base_func constantValues:cv->obj error:&error];\n }\n if (!mtl_function) {\n [lib->lock unlock];\n\n GGML_LOG_ERROR(\"%s: failed to compile pipeline: base = '%s', name = '%s'\\n\", __func__, base, name);\n if (error) {\n GGML_LOG_ERROR(\"%s: %s\\n\", __func__, [[error description] UTF8String]);\n }\n\n return res;\n }\n\n id obj = [lib->device newComputePipelineStateWithFunction:mtl_function error:&error];\n\n [mtl_function release];\n\n if (!obj) {\n [lib->lock unlock];\n\n GGML_LOG_ERROR(\"%s: failed to create pipeline state: base = '%s', name = '%s'\\n\", __func__, base, name);\n if (error) {\n GGML_LOG_ERROR(\"%s: %s\\n\", __func__, [[error description] UTF8String]);\n }\n\n return res;\n }\n\n GGML_LOG_DEBUG(\"%s: loaded %-40s %16p | th_max = %4d | th_width = %4d\\n\", __func__, name,\n (void *) obj,\n (int) obj.maxTotalThreadsPerThreadgroup,\n (int) obj.threadExecutionWidth);\n\n if (obj.maxTotalThreadsPerThreadgroup == 0 || obj.threadExecutionWidth == 0) {\n [obj release];\n\n [lib->lock unlock];\n\n GGML_LOG_ERROR(\"%s: incompatible pipeline %s\\n\", __func__, name);\n\n return res;\n }\n\n res.pipeline = ggml_metal_pipeline_init();\n res.pipeline->obj = obj;\n\n ggml_metal_pipelines_add(lib->pipelines, name, res.pipeline);\n }\n\n [lib->lock unlock];\n\n return res;\n}\n\n//\n// MTLComputeCommandEncoder wrapper\n//\n\nstruct ggml_metal_encoder {\n id obj;\n};\n\nggml_metal_encoder_t ggml_metal_encoder_init(ggml_metal_cmd_buf_t cmd_buf_raw, bool concurrent) {\n ggml_metal_encoder_t res = calloc(1, sizeof(struct ggml_metal_encoder));\n\n id cmd_buf = (id) cmd_buf_raw;\n\n if (concurrent) {\n res->obj = [cmd_buf computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent];\n } else {\n res->obj = [cmd_buf computeCommandEncoder];\n }\n\n [res->obj retain];\n\n return res;\n}\n\nvoid ggml_metal_encoder_free(ggml_metal_encoder_t encoder) {\n [encoder->obj release];\n free(encoder);\n}\n\nvoid ggml_metal_encoder_debug_group_push(ggml_metal_encoder_t encoder, const char * name) {\n [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]];\n}\n\nvoid ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) {\n [encoder->obj popDebugGroup];\n}\n\nvoid ggml_metal_encoder_set_pipeline(ggml_metal_encoder_t encoder, struct ggml_metal_pipeline_with_params pipeline) {\n [encoder->obj setComputePipelineState:pipeline.pipeline->obj];\n}\n\nvoid ggml_metal_encoder_set_bytes(ggml_metal_encoder_t encoder, void * data, size_t size, int idx) {\n [encoder->obj setBytes:data length:size atIndex:idx];\n}\n\nvoid ggml_metal_encoder_set_buffer(ggml_metal_encoder_t encoder, struct ggml_metal_buffer_id buffer, int idx) {\n [encoder->obj setBuffer:buffer.metal offset:buffer.offs atIndex:idx];\n}\n\nvoid ggml_metal_encoder_set_threadgroup_memory_size(ggml_metal_encoder_t encoder, size_t size, int idx) {\n [encoder->obj setThreadgroupMemoryLength:size atIndex:idx];\n}\n\nvoid ggml_metal_encoder_dispatch_threadgroups(ggml_metal_encoder_t encoder, int tg0, int tg1, int tg2, int tptg0, int tptg1, int tptg2) {\n [encoder->obj dispatchThreadgroups:MTLSizeMake(tg0, tg1, tg2) threadsPerThreadgroup:MTLSizeMake(tptg0, tptg1, tptg2)];\n}\n\nvoid ggml_metal_encoder_memory_barrier(ggml_metal_encoder_t encoder) {\n [encoder->obj memoryBarrierWithScope:MTLBarrierScopeBuffers];\n}\n\nvoid ggml_metal_encoder_end_encoding(ggml_metal_encoder_t encoder) {\n [encoder->obj endEncoding];\n}\n\nstruct ggml_metal_device {\n id mtl_device;\n\n // a single global queue shared by all Metal backends\n // technically not needed for devices with unified memory, but enables discrete GPUs support\n // ref: https://github.com/ggml-org/llama.cpp/pull/15906\n id mtl_queue;\n\n ggml_metal_rsets_t rsets;\n\n ggml_metal_library_t library;\n\n struct ggml_metal_device_props props;\n\n // virtual address for GPU memory allocations\n atomic_uintptr_t addr_virt;\n};\n\n//\n// MTLResidenceSet wrapper\n//\n\nstruct ggml_metal_rsets {\n NSLock * lock;\n\n NSMutableArray * data;\n\n // number of seconds since the last graph computation\n // keep the residency sets wired for that amount of time to avoid being collected by the OS\n int keep_alive_s;\n\n // background heartbeat thread to keep the residency sets alive\n atomic_bool d_stop;\n atomic_int d_loop;\n\n dispatch_group_t d_group;\n};\n\nggml_metal_rsets_t ggml_metal_rsets_init(void) {\n ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets));\n\n res->lock = [[NSLock alloc] init];\n res->data = [[NSMutableArray alloc] init];\n\n // by default keep the memory wired for 3 minutes\n res->keep_alive_s = 3*60;\n\n const char * GGML_METAL_RESIDENCY_KEEP_ALIVE_S = getenv(\"GGML_METAL_RESIDENCY_KEEP_ALIVE_S\");\n if (GGML_METAL_RESIDENCY_KEEP_ALIVE_S) {\n res->keep_alive_s = atoi(GGML_METAL_RESIDENCY_KEEP_ALIVE_S);\n }\n\n if (res->keep_alive_s <= 0) {\n res->keep_alive_s = 3*60;\n }\n\n GGML_LOG_INFO(\"%s: creating a residency set collection (keep_alive = %d s)\\n\", __func__, res->keep_alive_s);\n\n atomic_store_explicit(&res->d_stop, false, memory_order_relaxed);\n atomic_store_explicit(&res->d_loop, 2*res->keep_alive_s, memory_order_relaxed);\n\n res->d_group = dispatch_group_create();\n\n // start a background thread that periodically requests residency for all the currently active sets in the collection\n // the requests stop after a certain amount of time (keep_alive_s) of inactivity\n dispatch_queue_t d_queue = dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0);\n dispatch_group_async(res->d_group, d_queue, ^{\n#if defined(GGML_METAL_HAS_RESIDENCY_SETS)\n if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {\n while (!atomic_load_explicit(&res->d_stop, memory_order_relaxed)) {\n if (atomic_load_explicit(&res->d_loop, memory_order_relaxed) > 0) {\n [res->lock lock];\n\n for (int i = 0; i < (int) res->data.count; ++i) {\n [res->data[i] requestResidency];\n }\n\n atomic_fetch_sub_explicit(&res->d_loop, 1, memory_order_relaxed);\n\n [res->lock unlock];\n }\n\n // half a second\n usleep(500 * 1000);\n }\n }\n#endif\n });\n\n return res;\n}\n\nvoid ggml_metal_rsets_free(ggml_metal_rsets_t rsets) {\n if (rsets == NULL) {\n return;\n }\n\n // note: if you hit this assert, most likely you haven't deallocated all Metal resources before exiting\n GGML_ASSERT([rsets->data count] == 0);\n\n atomic_store_explicit(&rsets->d_stop, true, memory_order_relaxed);\n\n dispatch_group_wait(rsets->d_group, DISPATCH_TIME_FOREVER);\n dispatch_release(rsets->d_group);\n\n [rsets->data release];\n [rsets->lock release];\n\n free(rsets);\n}\n\nggml_metal_device_t ggml_metal_device_init(int device) {\n ggml_metal_device_t dev = calloc(1, sizeof(struct ggml_metal_device));\n\n assert(dev != NULL);\n\n if (dev->mtl_device == nil) {\n dev->mtl_device = MTLCreateSystemDefaultDevice();\n\n if (dev->mtl_device) {\n dev->mtl_queue = [dev->mtl_device newCommandQueue];\n if (dev->mtl_queue == nil) {\n GGML_LOG_ERROR(\"%s: error: failed to create command queue\\n\", __func__);\n }\n\n dev->addr_virt = 0x000000400ULL;\n\n dev->props.device = device;\n dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];\n dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];\n\n dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];\n dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory;\n\n dev->props.has_bfloat = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];\n dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6];\n if (getenv(\"GGML_METAL_BF16_DISABLE\") != NULL) {\n dev->props.has_bfloat = false;\n }\n\n dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML];\n if (getenv(\"GGML_METAL_TENSOR_DISABLE\") != NULL) {\n dev->props.has_tensor = false;\n }\n\n // note: disable the tensor API by default for old chips because with the current implementation it is not useful\n // - M2 Ultra: ~5% slower\n // - M4, M4 Max: no significant difference\n //\n // TODO: try to update the tensor API kernels to at least match the simdgroup performance\n if (getenv(\"GGML_METAL_TENSOR_ENABLE\") == NULL &&\n ![[dev->mtl_device name] containsString:@\"M5\"] &&\n ![[dev->mtl_device name] containsString:@\"M6\"] &&\n ![[dev->mtl_device name] containsString:@\"A19\"] &&\n ![[dev->mtl_device name] containsString:@\"A20\"]) {\n GGML_LOG_WARN(\"%s: tensor API disabled for pre-M5 and pre-A19 devices\\n\", __func__);\n dev->props.has_tensor = false;\n }\n\n // double-check that the tensor API compiles\n if (dev->props.has_tensor) {\n const char * src_tensor_f16 = \"\\n\"\n \"#include \\n\"\n \"#include \\n\"\n \"#include \\n\"\n \" \\n\"\n \"using namespace metal; \\n\"\n \"using namespace mpp::tensor_ops; \\n\"\n \" \\n\"\n \"kernel void dummy_kernel( \\n\"\n \" tensor> A [[buffer(0)]], \\n\"\n \" tensor> B [[buffer(1)]], \\n\"\n \" device float * C [[buffer(2)]], \\n\"\n \" uint2 tgid [[threadgroup_position_in_grid]]) \\n\"\n \"{ \\n\"\n \" auto tA = A.slice(0, (int)tgid.y); \\n\"\n \" auto tB = B.slice((int)tgid.x, 0); \\n\"\n \" \\n\"\n \" matmul2d< \\n\"\n \" matmul2d_descriptor(8, 8, dynamic_extent), \\n\"\n \" execution_simdgroups<4>> mm; \\n\"\n \" \\n\"\n \" auto cT = mm.get_destination_cooperative_tensor(); \\n\"\n \" \\n\"\n \" auto sA = tA.slice(0, 0); \\n\"\n \" auto sB = tB.slice(0, 0); \\n\"\n \" mm.run(sB, sA, cT); \\n\"\n \" \\n\"\n \" auto tC = tensor, tensor_inline>(C, dextents(4, 4)); \\n\"\n \" \\n\"\n \" cT.store(tC); \\n\"\n \"}\";\n\n GGML_LOG_INFO(\"%s: testing tensor API for f16 support\\n\", __func__);\n ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false);\n if (lib == NULL) {\n GGML_LOG_WARN(\"%s: - the tensor API is not supported in this environment - disabling\\n\", __func__);\n dev->props.has_tensor = false;\n } else {\n struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, \"dummy_kernel\", \"dummy_kernel\", nil);\n if (!ppl.pipeline) {\n GGML_LOG_WARN(\"%s: - the tensor API is not supported in this environment - disabling\\n\", __func__);\n dev->props.has_tensor = false;\n }\n\n ggml_metal_library_free(lib);\n }\n }\n\n // try to compile a dummy kernel to determine if the tensor API is supported for bfloat\n if (dev->props.has_tensor && dev->props.has_bfloat) {\n const char * src_tensor_bf16 = \"\\n\"\n \"#include \\n\"\n \"#include \\n\"\n \"#include \\n\"\n \" \\n\"\n \"using namespace metal; \\n\"\n \"using namespace mpp::tensor_ops; \\n\"\n \" \\n\"\n \"kernel void dummy_kernel( \\n\"\n \" tensor> A [[buffer(0)]], \\n\"\n \" tensor> B [[buffer(1)]], \\n\"\n \" device float * C [[buffer(2)]], \\n\"\n \" uint2 tgid [[threadgroup_position_in_grid]]) \\n\"\n \"{ \\n\"\n \" auto tA = A.slice(0, (int)tgid.y); \\n\"\n \" auto tB = B.slice((int)tgid.x, 0); \\n\"\n \" \\n\"\n \" matmul2d< \\n\"\n \" matmul2d_descriptor(8, 8, dynamic_extent), \\n\"\n \" execution_simdgroups<4>> mm; \\n\"\n \" \\n\"\n \" auto cT = mm.get_destination_cooperative_tensor(); \\n\"\n \" \\n\"\n \" auto sA = tA.slice(0, 0); \\n\"\n \" auto sB = tB.slice(0, 0); \\n\"\n \" mm.run(sB, sA, cT); \\n\"\n \" \\n\"\n \" auto tC = tensor, tensor_inline>(C, dextents(4, 4)); \\n\"\n \" \\n\"\n \" cT.store(tC); \\n\"\n \"}\";\n\n GGML_LOG_INFO(\"%s: testing tensor API for bfloat support\\n\", __func__);\n ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false);\n if (lib == NULL) {\n GGML_LOG_WARN(\"%s: - the tensor API does not support bfloat - disabling bfloat support\\n\", __func__);\n dev->props.has_bfloat = false;\n } else {\n struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, \"dummy_kernel\", \"dummy_kernel\", nil);\n if (!ppl.pipeline) {\n GGML_LOG_WARN(\"%s: - the tensor API does not support bfloat - disabling bfloat support\\n\", __func__);\n dev->props.has_bfloat = false;\n }\n\n ggml_metal_library_free(lib);\n }\n }\n\n dev->props.use_residency_sets = true;\n#if defined(GGML_METAL_HAS_RESIDENCY_SETS)\n dev->props.use_residency_sets = getenv(\"GGML_METAL_NO_RESIDENCY\") == nil;\n#endif\n\n dev->props.use_shared_buffers = dev->props.has_unified_memory;\n#if TARGET_OS_OSX\n // In case of eGPU, shared memory may be preferable.\n dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal;\n#endif\n if (getenv(\"GGML_METAL_SHARED_BUFFERS_DISABLE\") != NULL) {\n dev->props.use_shared_buffers = false;\n }\n if (getenv(\"GGML_METAL_SHARED_BUFFERS_ENABLE\") != NULL) {\n dev->props.use_shared_buffers = true;\n }\n\n dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];\n\n dev->props.op_offload_min_batch_size = getenv(\"GGML_OP_OFFLOAD_MIN_BATCH\") ? atoi(getenv(\"GGML_OP_OFFLOAD_MIN_BATCH\")) : 32;\n\n dev->props.max_buffer_size = dev->mtl_device.maxBufferLength;\n dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength;\n if (@available(macOS 10.12, iOS 16.0, *)) {\n dev->props.max_working_set_size = dev->mtl_device.recommendedMaxWorkingSetSize;\n } else {\n dev->props.max_working_set_size = dev->mtl_device.maxBufferLength;\n }\n\n snprintf(dev->props.name, sizeof(dev->props.name), \"%s%d\", \"MTL\", device);\n snprintf(dev->props.desc, sizeof(dev->props.desc), \"%s\", [[dev->mtl_device name] UTF8String]);\n\n dev->library = ggml_metal_library_init(dev);\n if (!dev->library) {\n GGML_LOG_ERROR(\"%s: error: failed to create library\\n\", __func__);\n }\n\n if (dev->props.use_residency_sets) {\n dev->rsets = ggml_metal_rsets_init();\n } else {\n dev->rsets = nil;\n }\n\n // print MTL GPU family:\n GGML_LOG_INFO(\"%s: GPU name: %s\\n\", __func__, dev->props.name);\n\n // determine max supported GPU family\n // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf\n // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf\n {\n for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {\n if ([dev->mtl_device supportsFamily:i]) {\n GGML_LOG_INFO(\"%s: GPU family: MTLGPUFamilyApple%d (%d)\\n\", __func__, i - (int) MTLGPUFamilyApple1 + 1, i);\n break;\n }\n }\n\n for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {\n if ([dev->mtl_device supportsFamily:i]) {\n GGML_LOG_INFO(\"%s: GPU family: MTLGPUFamilyCommon%d (%d)\\n\", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);\n break;\n }\n }\n\n for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) {\n if ([dev->mtl_device supportsFamily:i]) {\n GGML_LOG_INFO(\"%s: GPU family: MTLGPUFamilyMetal%d (%d)\\n\", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i);\n break;\n }\n }\n }\n\n GGML_LOG_INFO(\"%s: simdgroup reduction = %s\\n\", __func__, dev->props.has_simdgroup_reduction ? \"true\" : \"false\");\n GGML_LOG_INFO(\"%s: simdgroup matrix mul. = %s\\n\", __func__, dev->props.has_simdgroup_mm ? \"true\" : \"false\");\n GGML_LOG_INFO(\"%s: has unified memory = %s\\n\", __func__, dev->props.has_unified_memory ? \"true\" : \"false\");\n GGML_LOG_INFO(\"%s: has bfloat = %s\\n\", __func__, dev->props.has_bfloat ? \"true\" : \"false\");\n GGML_LOG_INFO(\"%s: has tensor = %s\\n\", __func__, dev->props.has_tensor ? \"true\" : \"false\");\n GGML_LOG_INFO(\"%s: use residency sets = %s\\n\", __func__, dev->props.use_residency_sets ? \"true\" : \"false\");\n GGML_LOG_INFO(\"%s: use shared buffers = %s\\n\", __func__, dev->props.use_shared_buffers ? \"true\" : \"false\");\n\n#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)\n if (@available(macOS 10.12, iOS 16.0, *)) {\n GGML_LOG_INFO(\"%s: recommendedMaxWorkingSetSize = %8.2f MB\\n\", __func__, dev->props.max_working_set_size / 1e6);\n }\n#endif\n }\n }\n\n return dev;\n}\n\nvoid ggml_metal_device_free(ggml_metal_device_t dev) {\n assert(dev != NULL);\n\n ggml_metal_rsets_free(dev->rsets);\n\n ggml_metal_library_free(dev->library);\n dev->library = NULL;\n\n if (dev->mtl_queue) {\n [dev->mtl_queue release];\n dev->mtl_queue = nil;\n }\n\n if (dev->mtl_device) {\n [dev->mtl_device release];\n dev->mtl_device = nil;\n }\n\n free(dev);\n}\n\nvoid * ggml_metal_device_get_obj(ggml_metal_device_t dev) {\n return dev->mtl_device;\n}\n\nvoid * ggml_metal_device_get_queue(ggml_metal_device_t dev) {\n return dev->mtl_queue;\n}\n\nggml_metal_library_t ggml_metal_device_get_library(ggml_metal_device_t dev) {\n return dev->library;\n}\n\nvoid ggml_metal_device_rsets_add(ggml_metal_device_t dev, ggml_metal_rset_t rset) {\n if (rset == nil) {\n return;\n }\n\n GGML_ASSERT(dev->rsets);\n\n [dev->rsets->lock lock];\n\n [dev->rsets->data addObject:rset];\n\n [dev->rsets->lock unlock];\n}\n\nvoid ggml_metal_device_rsets_rm(ggml_metal_device_t dev, ggml_metal_rset_t rset) {\n if (rset == nil) {\n return;\n }\n\n GGML_ASSERT(dev->rsets);\n\n [dev->rsets->lock lock];\n\n [dev->rsets->data removeObject:rset];\n\n [dev->rsets->lock unlock];\n}\n\nvoid ggml_metal_device_rsets_keep_alive(ggml_metal_device_t dev) {\n if (dev->rsets == NULL) {\n return;\n }\n\n atomic_store_explicit(&dev->rsets->d_loop, 2*dev->rsets->keep_alive_s, memory_order_relaxed);\n}\n\nstruct ggml_metal_event {\n void * obj; // id\n\n atomic_int value;\n};\n\nvoid ggml_metal_event_encode_signal(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf_raw) {\n id event = (id)ev->obj;\n\n id cmd_buf = (id) cmd_buf_raw;\n\n [cmd_buf encodeSignalEvent:event value:atomic_fetch_add_explicit(&ev->value, 1, memory_order_relaxed) + 1];\n}\n\nvoid ggml_metal_event_encode_wait(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf_raw) {\n id event = (id)ev->obj;\n\n id cmd_buf = (id) cmd_buf_raw;\n\n [cmd_buf encodeWaitForEvent:event value:atomic_load_explicit(&ev->value, memory_order_relaxed)];\n}\n\nggml_metal_event_t ggml_metal_device_event_init(ggml_metal_device_t dev) {\n id event = [dev->mtl_device newEvent];\n\n ggml_metal_event_t ev = calloc(1, sizeof(struct ggml_metal_event));\n\n ev->obj = (__bridge void *)event;\n ev->value = 0;\n\n return ev;\n}\n\nvoid ggml_metal_device_event_free(ggml_metal_device_t dev, ggml_metal_event_t ev) {\n id event = ev->obj;\n [event release];\n\n free(ev);\n\n GGML_UNUSED(dev);\n}\n\nvoid ggml_metal_device_event_synchronize(ggml_metal_device_t dev, ggml_metal_event_t ev) {\n @autoreleasepool {\n id event = ev->obj;\n\n id cmd_buf = [dev->mtl_queue commandBuffer];\n [cmd_buf encodeWaitForEvent:event value:atomic_load_explicit(&ev->value, memory_order_relaxed)];\n [cmd_buf commit];\n [cmd_buf waitUntilCompleted];\n }\n}\n\nvoid ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t * total) {\n if (@available(macOS 10.12, iOS 16.0, *)) {\n *total = dev->mtl_device.recommendedMaxWorkingSetSize;\n *free = *total - dev->mtl_device.currentAllocatedSize;\n } else {\n *free = 0;\n *total = 0;\n }\n}\n\nbool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tensor * op) {\n const bool has_simdgroup_mm = dev->props.has_simdgroup_mm;\n const bool has_simdgroup_reduction = dev->props.has_simdgroup_reduction;\n const bool has_bfloat = dev->props.has_bfloat;\n\n if (!has_bfloat) {\n if (op->type == GGML_TYPE_BF16) {\n return false;\n }\n\n for (size_t i = 0, n = 3; i < n; ++i) {\n if (op->src[i] != NULL && op->src[i]->type == GGML_TYPE_BF16) {\n return false;\n }\n }\n }\n\n switch (op->op) {\n case GGML_OP_SCALE:\n case GGML_OP_FILL:\n case GGML_OP_CLAMP:\n case GGML_OP_SQR:\n case GGML_OP_SQRT:\n case GGML_OP_SIN:\n case GGML_OP_COS:\n case GGML_OP_LOG:\n return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);\n case GGML_OP_UNARY:\n switch (ggml_get_unary_op(op)) {\n case GGML_UNARY_OP_TANH:\n case GGML_UNARY_OP_RELU:\n case GGML_UNARY_OP_SIGMOID:\n case GGML_UNARY_OP_GELU:\n case GGML_UNARY_OP_GELU_ERF:\n case GGML_UNARY_OP_GELU_QUICK:\n case GGML_UNARY_OP_SILU:\n case GGML_UNARY_OP_ELU:\n case GGML_UNARY_OP_NEG:\n case GGML_UNARY_OP_ABS:\n case GGML_UNARY_OP_SGN:\n case GGML_UNARY_OP_STEP:\n case GGML_UNARY_OP_HARDSWISH:\n case GGML_UNARY_OP_HARDSIGMOID:\n case GGML_UNARY_OP_EXP:\n case GGML_UNARY_OP_SOFTPLUS:\n case GGML_UNARY_OP_EXPM1:\n return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);\n default:\n return false;\n }\n case GGML_OP_GLU:\n switch (ggml_get_glu_op(op)) {\n case GGML_GLU_OP_REGLU:\n case GGML_GLU_OP_GEGLU:\n case GGML_GLU_OP_SWIGLU:\n case GGML_GLU_OP_SWIGLU_OAI:\n case GGML_GLU_OP_GEGLU_ERF:\n case GGML_GLU_OP_GEGLU_QUICK:\n return ggml_is_contiguous_1(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;\n default:\n return false;\n }\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_TRANSPOSE:\n case GGML_OP_PERMUTE:\n case GGML_OP_CONCAT:\n return true;\n case GGML_OP_ADD:\n case GGML_OP_SUB:\n case GGML_OP_MUL:\n case GGML_OP_DIV:\n case GGML_OP_ADD_ID:\n case GGML_OP_ACC:\n return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && op->src[0]->type == GGML_TYPE_F32;\n case GGML_OP_REPEAT:\n case GGML_OP_CONV_TRANSPOSE_1D:\n return true;\n case GGML_OP_CONV_TRANSPOSE_2D:\n return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) &&\n (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) &&\n op->src[1]->type == GGML_TYPE_F32 &&\n op->type == GGML_TYPE_F32;\n case GGML_OP_SUM:\n return has_simdgroup_reduction && ggml_is_contiguous(op->src[0]);\n case GGML_OP_TRI:\n return ggml_is_contiguous_rows(op->src[0]);\n case GGML_OP_SUM_ROWS:\n case GGML_OP_CUMSUM:\n case GGML_OP_MEAN:\n case GGML_OP_SOFT_MAX:\n case GGML_OP_GROUP_NORM:\n case GGML_OP_L2_NORM:\n return has_simdgroup_reduction && ggml_is_contiguous_rows(op->src[0]);\n case GGML_OP_COUNT_EQUAL:\n return has_simdgroup_reduction &&\n op->src[0]->type == GGML_TYPE_I32 &&\n op->src[1]->type == GGML_TYPE_I32 &&\n op->type == GGML_TYPE_I64;\n case GGML_OP_ARGMAX:\n return has_simdgroup_reduction;\n case GGML_OP_NORM:\n case GGML_OP_RMS_NORM:\n return has_simdgroup_reduction && (ggml_is_contiguous_rows(op->src[0]));\n case GGML_OP_ROPE:\n return true;\n case GGML_OP_IM2COL:\n return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);\n case GGML_OP_CONV_2D:\n return ggml_is_contiguous(op->src[0]) &&\n op->src[1]->type == GGML_TYPE_F32 &&\n op->type == GGML_TYPE_F32 &&\n (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);\n case GGML_OP_UPSCALE:\n return op->src[0]->type == GGML_TYPE_F32;\n case GGML_OP_POOL_1D:\n return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;\n case GGML_OP_POOL_2D:\n return op->src[0]->type == GGML_TYPE_F32;\n case GGML_OP_PAD:\n // TODO: add circular padding support for metal, see https://github.com/ggml-org/llama.cpp/pull/16985\n if (ggml_get_op_params_i32(op, 8) != 0) {\n return false;\n }\n\n return (ggml_get_op_params_i32(op, 0) == 0) && (ggml_get_op_params_i32(op, 2) == 0) &&\n (ggml_get_op_params_i32(op, 4) == 0) && (ggml_get_op_params_i32(op, 6) == 0);\n case GGML_OP_PAD_REFLECT_1D:\n case GGML_OP_TIMESTEP_EMBEDDING:\n case GGML_OP_LEAKY_RELU:\n return op->src[0]->type == GGML_TYPE_F32;\n case GGML_OP_ARGSORT:\n case GGML_OP_TOP_K:\n case GGML_OP_ARANGE:\n return true;\n case GGML_OP_FLASH_ATTN_EXT:\n // for new head sizes, add checks here\n if (op->src[0]->ne[0] != 32 &&\n op->src[0]->ne[0] != 40 &&\n op->src[0]->ne[0] != 48 &&\n op->src[0]->ne[0] != 64 &&\n op->src[0]->ne[0] != 72 &&\n op->src[0]->ne[0] != 80 &&\n op->src[0]->ne[0] != 96 &&\n op->src[0]->ne[0] != 112 &&\n op->src[0]->ne[0] != 128 &&\n op->src[0]->ne[0] != 192 &&\n op->src[0]->ne[0] != 256 &&\n op->src[0]->ne[0] != 320 &&\n op->src[0]->ne[0] != 576) {\n return false;\n }\n if (op->src[1]->type != op->src[2]->type) {\n return false;\n }\n return has_simdgroup_mm; // TODO: over-restricted for vec-kernels\n case GGML_OP_SSM_CONV:\n case GGML_OP_SSM_SCAN:\n return has_simdgroup_reduction;\n case GGML_OP_RWKV_WKV6:\n case GGML_OP_RWKV_WKV7:\n return true;\n case GGML_OP_GATED_DELTA_NET:\n return has_simdgroup_reduction && op->src[2]->ne[0] % 32 == 0;\n case GGML_OP_SOLVE_TRI:\n case GGML_OP_MUL_MAT:\n case GGML_OP_MUL_MAT_ID:\n return has_simdgroup_reduction && op->src[0]->type != GGML_TYPE_NVFP4;\n case GGML_OP_SET:\n case GGML_OP_CPY:\n case GGML_OP_DUP:\n case GGML_OP_CONT:\n {\n switch (op->src[0]->type) {\n case GGML_TYPE_F32:\n switch (op->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_IQ4_NL:\n case GGML_TYPE_I32:\n return true;\n default:\n return false;\n }\n case GGML_TYPE_F16:\n switch (op->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n return true;\n default:\n return false;\n }\n case GGML_TYPE_BF16:\n switch (op->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_BF16:\n return true;\n default:\n return false;\n }\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n switch (op->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n return true;\n default:\n return false;\n }\n case GGML_TYPE_I32:\n return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32;\n default:\n return false;\n };\n }\n case GGML_OP_GET_ROWS:\n return op->src[0]->type != GGML_TYPE_NVFP4;\n case GGML_OP_SET_ROWS:\n {\n if (op->src[0]->type != GGML_TYPE_F32) {\n return false;\n }\n\n switch (op->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n case GGML_TYPE_BF16:\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_IQ4_NL:\n return true;\n default:\n return false;\n };\n }\n case GGML_OP_DIAG:\n return true;\n case GGML_OP_OPT_STEP_ADAMW:\n case GGML_OP_OPT_STEP_SGD:\n return has_simdgroup_reduction;\n default:\n return false;\n }\n}\n\nconst struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev) {\n return &dev->props;\n}\n\n//\n// device buffers\n//\n\n// max memory buffers that can be mapped to the device\n#define GGML_METAL_MAX_BUFFERS 64\n\nstruct ggml_metal_buffer_wrapper {\n void * data;\n size_t size;\n\n id metal;\n};\n\nstruct ggml_metal_buffer {\n void * all_data;\n size_t all_size;\n\n // if false, the Metal buffer data is allocated in private GPU memory and is not shared with the host\n bool is_shared;\n bool owned;\n\n // multiple buffers are used only to avoid the maximum buffer size limitation when using mmap\n int n_buffers;\n struct ggml_metal_buffer_wrapper buffers[GGML_METAL_MAX_BUFFERS];\n\n bool use_residency_sets;\n\n // optional MTLResidencySet\n // note: cannot use explicitly \"id\" here because it is not available on certain OSes\n id rset;\n\n // pointers to global device\n ggml_metal_device_t dev;\n};\n\nstatic void ggml_metal_log_allocated_size(id device, size_t size_aligned) {\n#ifndef GGML_METAL_NDEBUG\n#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)\n if (@available(macOS 10.12, iOS 16.0, *)) {\n GGML_LOG_DEBUG(\"%s: allocated buffer, size = %8.2f MiB, (%8.2f / %8.2f)\\n\",\n __func__,\n size_aligned / 1024.0 / 1024.0,\n device.currentAllocatedSize / 1024.0 / 1024.0,\n device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);\n\n if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {\n GGML_LOG_WARN(\"%s: warning: current allocated size is greater than the recommended max working set size\\n\", __func__);\n }\n } else {\n GGML_LOG_INFO(\"%s: allocated buffer, size = %8.2f MiB, (%8.2f)\\n\",\n __func__,\n size_aligned / 1024.0 / 1024.0,\n device.currentAllocatedSize / 1024.0 / 1024.0);\n }\n#endif\n#endif\n GGML_UNUSED(device);\n GGML_UNUSED(size_aligned);\n}\n\n// rset init\nstatic bool ggml_metal_buffer_rset_init(ggml_metal_buffer_t buf) {\n buf->rset = nil;\n\n if (!buf->use_residency_sets) {\n return true;\n }\n\n#if defined(GGML_METAL_HAS_RESIDENCY_SETS)\n if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {\n MTLResidencySetDescriptor * desc = [[MTLResidencySetDescriptor alloc] init];\n desc.label = @\"ggml_metal\";\n desc.initialCapacity = buf->n_buffers;\n\n NSError * error;\n buf->rset = [buf->dev->mtl_device newResidencySetWithDescriptor:desc error:&error];\n if (error) {\n GGML_LOG_ERROR(\"%s: error: %s\\n\", __func__, [[error description] UTF8String]);\n [desc release];\n return false;\n }\n\n [desc release];\n\n for (int i = 0; i < buf->n_buffers; i++) {\n [buf->rset addAllocation:buf->buffers[i].metal];\n }\n\n [buf->rset commit];\n [buf->rset requestResidency];\n\n return true;\n }\n#endif\n\n return true;\n}\n\n// rset free\nstatic void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) {\n#if defined(GGML_METAL_HAS_RESIDENCY_SETS)\n if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {\n if (buf->rset) {\n [buf->rset endResidency];\n [buf->rset removeAllAllocations];\n [buf->rset release];\n }\n }\n#else\n GGML_UNUSED(buf);\n#endif\n}\n\nstatic void * ggml_metal_host_malloc(size_t n) {\n void * data = NULL;\n\n#if TARGET_OS_OSX\n kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE);\n if (err != KERN_SUCCESS) {\n GGML_LOG_ERROR(\"%s: error: vm_allocate failed\\n\", __func__);\n return NULL;\n }\n#else\n const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);\n if (result != 0) {\n GGML_LOG_ERROR(\"%s: error: posix_memalign failed\\n\", __func__);\n return NULL;\n }\n#endif\n\n return data;\n}\n\nggml_metal_buffer_t ggml_metal_buffer_init(ggml_metal_device_t dev, size_t size, bool shared) {\n ggml_metal_buffer_t res = calloc(1, sizeof(struct ggml_metal_buffer));\n\n res->dev = dev;\n\n const size_t size_page = sysconf(_SC_PAGESIZE);\n\n size_t size_aligned = size;\n if ((size_aligned % size_page) != 0) {\n size_aligned += (size_page - (size_aligned % size_page));\n }\n\n const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);\n\n shared = shared && props_dev->use_shared_buffers;\n\n // allocate shared buffer if the device supports it and it is required by the buffer type\n if (shared) {\n res->all_data = ggml_metal_host_malloc(size_aligned);\n res->is_shared = true;\n } else {\n // use virtual address\n res->all_data = (void *) atomic_fetch_add_explicit(&dev->addr_virt, size_aligned, memory_order_relaxed);\n res->is_shared = false;\n }\n res->all_size = size_aligned;\n\n res->owned = true;\n\n res->n_buffers = 1;\n\n if (res->all_data != NULL) {\n res->buffers[0].size = size;\n res->buffers[0].metal = nil;\n\n if (size_aligned > 0) {\n if (props_dev->use_shared_buffers && shared) {\n res->buffers[0].metal = [res->dev->mtl_device newBufferWithBytesNoCopy:res->all_data\n length:size_aligned\n options:MTLResourceStorageModeShared\n deallocator:nil];\n } else {\n res->buffers[0].metal = [res->dev->mtl_device newBufferWithLength:size_aligned options:MTLResourceStorageModePrivate];\n }\n }\n\n res->buffers[0].data = res->all_data;\n }\n\n if (size_aligned > 0 && (res->all_data == NULL || res->buffers[0].metal == nil)) {\n GGML_LOG_ERROR(\"%s: error: failed to allocate buffer, size = %8.2f MiB\\n\", __func__, size_aligned / 1024.0 / 1024.0);\n free(res);\n return NULL;\n }\n\n res->use_residency_sets = props_dev->use_residency_sets;\n\n if (!ggml_metal_buffer_rset_init(res)) {\n GGML_LOG_ERROR(\"%s: error: failed to initialize residency set\\n\", __func__);\n free(res);\n return NULL;\n }\n\n ggml_metal_device_rsets_add(dev, res->rset);\n\n //ggml_metal_log_allocated_size(device, size_aligned);\n\n return res;\n}\n\nggml_metal_buffer_t ggml_metal_buffer_map(ggml_metal_device_t dev, void * ptr, size_t size, size_t max_tensor_size) {\n ggml_metal_buffer_t res = calloc(1, sizeof(struct ggml_metal_buffer));\n\n res->dev = dev;\n\n res->all_data = ptr;\n res->all_size = size;\n\n res->is_shared = true;\n res->owned = false;\n\n res->n_buffers = 0;\n\n const size_t size_page = sysconf(_SC_PAGESIZE);\n\n // page-align the data ptr\n {\n const uintptr_t offs = (uintptr_t) ptr % size_page;\n ptr = (void *) ((char *) ptr - offs);\n size += offs;\n }\n\n size_t size_aligned = size;\n if ((size_aligned % size_page) != 0) {\n size_aligned += (size_page - (size_aligned % size_page));\n }\n\n const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);\n\n // the buffer fits into the max buffer size allowed by the device\n if (size_aligned <= props_dev->max_buffer_size) {\n res->buffers[res->n_buffers].data = ptr;\n res->buffers[res->n_buffers].size = size;\n res->buffers[res->n_buffers].metal = nil;\n\n if (size_aligned > 0) {\n res->buffers[res->n_buffers].metal = [res->dev->mtl_device newBufferWithBytesNoCopy:ptr length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];\n\n if (res->buffers[res->n_buffers].metal == nil) {\n GGML_LOG_ERROR(\"%s: error: failed to allocate buffer, size = %8.2f MiB\\n\", __func__, size_aligned / 1024.0 / 1024.0);\n free(res);\n return NULL;\n }\n }\n\n ggml_metal_log_allocated_size(res->dev->mtl_device, size_aligned);\n\n ++res->n_buffers;\n } else {\n // this overlap between the views will guarantee that the tensor with the maximum size will fully fit into\n // one of the views\n const size_t size_ovlp = ((max_tensor_size + size_page - 1) / size_page + 1) * size_page; // round-up 2 pages just in case\n const size_t size_step = props_dev->max_buffer_size - size_ovlp;\n const size_t size_view = props_dev->max_buffer_size;\n\n for (size_t i = 0; i < size; i += size_step) {\n const size_t size_step_aligned = (i + size_view <= size) ? size_view : (size_aligned - i);\n\n res->buffers[res->n_buffers].data = (void *) ((uint8_t *) ptr + i);\n res->buffers[res->n_buffers].size = size_step_aligned;\n res->buffers[res->n_buffers].metal = nil;\n\n if (size_step_aligned > 0) {\n res->buffers[res->n_buffers].metal = [res->dev->mtl_device newBufferWithBytesNoCopy:(void *) ((uint8_t *) ptr + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];\n\n if (res->buffers[res->n_buffers].metal == nil) {\n GGML_LOG_ERROR(\"%s: error: failed to allocate buffer, size = %8.2f MiB\\n\", __func__, size_step_aligned / 1024.0 / 1024.0);\n free(res);\n return NULL;\n }\n }\n\n ggml_metal_log_allocated_size(res->dev->mtl_device, size_step_aligned);\n\n if (i + size_step < size) {\n GGML_LOG_INFO(\"\\n\");\n }\n\n ++res->n_buffers;\n }\n }\n\n res->use_residency_sets = props_dev->use_residency_sets;\n\n if (!ggml_metal_buffer_rset_init(res)) {\n GGML_LOG_ERROR(\"%s: error: failed to initialize residency set\\n\", __func__);\n free(res);\n return NULL;\n }\n\n ggml_metal_device_rsets_add(dev, res->rset);\n\n return res;\n}\n\nvoid ggml_metal_buffer_free(ggml_metal_buffer_t buf) {\n ggml_metal_device_rsets_rm(buf->dev, buf->rset);\n\n for (int i = 0; i < buf->n_buffers; i++) {\n [buf->buffers[i].metal release];\n }\n\n ggml_metal_buffer_rset_free(buf);\n\n if (buf->is_shared && buf->owned) {\n#if TARGET_OS_OSX\n vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)buf->all_data, buf->all_size);\n#else\n free(buf->all_data);\n#endif\n }\n\n free(buf);\n}\n\nvoid * ggml_metal_buffer_get_base(ggml_metal_buffer_t buf) {\n return buf->all_data;\n}\n\nbool ggml_metal_buffer_is_shared(ggml_metal_buffer_t buf) {\n return buf->is_shared;\n}\n\nvoid ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {\n if (buf->is_shared) {\n memset((char *) tensor->data + offset, value, size);\n return;\n }\n\n @autoreleasepool {\n // dst\n struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(buf, tensor);\n bid_dst.offs += offset;\n\n id cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];\n\n {\n id encoder = [cmd_buf blitCommandEncoder];\n\n [encoder fillBuffer:bid_dst.metal\n range:NSMakeRange(bid_dst.offs, bid_dst.offs + size)\n value:value];\n\n [encoder endEncoding];\n }\n\n [cmd_buf commit];\n [cmd_buf waitUntilCompleted];\n }\n}\n\nvoid ggml_metal_buffer_set_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n if (buf->is_shared) {\n memcpy((char *) tensor->data + offset, data, size);\n return;\n }\n\n @autoreleasepool {\n // src\n void * data_ptr = (void *)(uintptr_t) data; // \"const cast\" the src data\n id buf_src = [buf->dev->mtl_device newBufferWithBytesNoCopy:data_ptr\n length:size\n options:MTLResourceStorageModeShared\n deallocator:nil];\n\n GGML_ASSERT(buf_src);\n\n // dst\n struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(buf, tensor);\n bid_dst.offs += offset;\n\n // note: for experimentation purposes, here we use a semaphore to wait for the copy to complete\n // this is alternative to waitUntilCompleted, which should be faster, but don't seem to make much difference\n dispatch_semaphore_t completion_semaphore = dispatch_semaphore_create(0);\n\n id cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];\n\n {\n id encoder = [cmd_buf blitCommandEncoder];\n\n [encoder copyFromBuffer:buf_src\n sourceOffset:0\n toBuffer:bid_dst.metal\n destinationOffset:bid_dst.offs\n size:size];\n\n [encoder endEncoding];\n }\n\n [cmd_buf addCompletedHandler:^(id cb) {\n // TODO: can check for errors here\n GGML_UNUSED(cb);\n\n dispatch_semaphore_signal(completion_semaphore);\n }];\n\n [cmd_buf commit];\n\n dispatch_semaphore_wait(completion_semaphore, DISPATCH_TIME_FOREVER);\n dispatch_release(completion_semaphore);\n\n //[cmd_buf waitUntilCompleted];\n }\n}\n\nvoid ggml_metal_buffer_get_tensor(ggml_metal_buffer_t buf, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n if (buf->is_shared) {\n memcpy(data, (const char *) tensor->data + offset, size);\n return;\n }\n\n @autoreleasepool {\n // src\n struct ggml_metal_buffer_id bid_src = ggml_metal_buffer_get_id(buf, tensor);\n bid_src.offs += offset;\n\n // dst\n id buf_dst = [buf->dev->mtl_device newBufferWithBytesNoCopy:data\n length:size\n options:MTLResourceStorageModeShared\n deallocator:nil];\n\n GGML_ASSERT(buf_dst);\n\n id cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];\n\n {\n id encoder = [cmd_buf blitCommandEncoder];\n\n [encoder copyFromBuffer:bid_src.metal\n sourceOffset:bid_src.offs\n toBuffer:buf_dst\n destinationOffset:0\n size:size];\n\n [encoder endEncoding];\n }\n\n [cmd_buf commit];\n [cmd_buf waitUntilCompleted];\n }\n}\n\nvoid ggml_metal_buffer_clear(ggml_metal_buffer_t buf, uint8_t value) {\n if (buf->is_shared) {\n memset(buf->all_data, value, buf->all_size);\n return;\n }\n\n @autoreleasepool {\n id cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];\n\n {\n id encoder = [cmd_buf blitCommandEncoder];\n\n [encoder fillBuffer:buf->buffers[0].metal\n range:NSMakeRange(0, buf->buffers[0].size)\n value:value];\n\n [encoder endEncoding];\n }\n\n [cmd_buf commit];\n [cmd_buf waitUntilCompleted];\n }\n}\n\nstruct ggml_metal_buffer_id ggml_metal_buffer_get_id(ggml_metal_buffer_t buf, const struct ggml_tensor * t) {\n struct ggml_metal_buffer_id res = { nil, 0 };\n\n const int64_t tsize = ggml_nbytes(t);\n\n // find the view that contains the tensor fully\n for (int i = 0; i < buf->n_buffers; ++i) {\n const int64_t ioffs = (int64_t) t->data - (int64_t) buf->buffers[i].data;\n\n //GGML_LOG_INFO(\"ioffs = %10ld, tsize = %10ld, sum = %10ld, buf->buffers[%d].size = %10ld\\n\", ioffs, tsize, ioffs + tsize, i, buf->buffers[i].size);\n if (ioffs >= 0 && ioffs + tsize <= (int64_t) buf->buffers[i].size) {\n res.metal = buf->buffers[i].metal;\n res.offs = (size_t) ioffs;\n\n //GGML_LOG_INFO(\"%s: tensor '%16s', offs = %8ld\\n\", __func__, t->name, *offs);\n\n return res;\n }\n }\n\n GGML_LOG_ERROR(\"%s: error: tensor '%s' buffer is nil\\n\", __func__, t->name);\n\n return res;\n}\n" }, { "path": "src/ggml-metal/ggml-metal-impl.h", "content": "#ifndef GGML_METAL_IMPL\n#define GGML_METAL_IMPL\n\n// kernel parameters for mat-vec threadgroups\n//\n// N_R0: number of src0 rows to process per simdgroup\n// N_SG: number of simdgroups per threadgroup\n//\n// TODO: for optimal performance, become function of the device and work size\n\n#define N_R0_Q4_0 4\n#define N_SG_Q4_0 2\n\n#define N_R0_Q4_1 4\n#define N_SG_Q4_1 2\n\n#define N_R0_Q5_0 4\n#define N_SG_Q5_0 2\n\n#define N_R0_Q5_1 4\n#define N_SG_Q5_1 2\n\n#define N_R0_Q8_0 2\n#define N_SG_Q8_0 4\n\n#define N_R0_MXFP4 2\n#define N_SG_MXFP4 2\n\n#define N_R0_Q2_K 4\n#define N_SG_Q2_K 2\n\n#define N_R0_Q3_K 2\n#define N_SG_Q3_K 2\n\n#define N_R0_Q4_K 2\n#define N_SG_Q4_K 2\n\n#define N_R0_Q5_K 1\n#define N_SG_Q5_K 2\n\n#define N_R0_Q6_K 2\n#define N_SG_Q6_K 2\n\n#define N_R0_IQ1_S 4\n#define N_SG_IQ1_S 2\n\n#define N_R0_IQ1_M 4\n#define N_SG_IQ1_M 2\n\n#define N_R0_IQ2_XXS 4\n#define N_SG_IQ2_XXS 2\n\n#define N_R0_IQ2_XS 4\n#define N_SG_IQ2_XS 2\n\n#define N_R0_IQ2_S 4\n#define N_SG_IQ2_S 2\n\n#define N_R0_IQ3_XXS 4\n#define N_SG_IQ3_XXS 2\n\n#define N_R0_IQ3_S 4\n#define N_SG_IQ3_S 2\n\n#define N_R0_IQ4_NL 2\n#define N_SG_IQ4_NL 2\n\n#define N_R0_IQ4_XS 2\n#define N_SG_IQ4_XS 2\n\n// function constants offsets\n#define FC_FLASH_ATTN_EXT_PAD 100\n#define FC_FLASH_ATTN_EXT_BLK 200\n#define FC_FLASH_ATTN_EXT 300\n#define FC_FLASH_ATTN_EXT_VEC 400\n#define FC_FLASH_ATTN_EXT_VEC_REDUCE 500\n#define FC_MUL_MV 600\n#define FC_MUL_MM 700\n#define FC_ROPE 800\n#define FC_SSM_CONV 900\n#define FC_SOLVE_TRI 1000\n#define FC_COUNT_EQUAL 1100\n#define FC_UNARY 1200\n#define FC_BIN 1300\n#define FC_SUM_ROWS 1400\n#define FC_UPSCALE 1500\n#define FC_GATED_DELTA_NET 1600\n\n// op-specific constants\n#define OP_FLASH_ATTN_EXT_NQPSG 8\n#define OP_FLASH_ATTN_EXT_NCPSG 64\n\n#define OP_FLASH_ATTN_EXT_VEC_NQPSG 1\n#define OP_FLASH_ATTN_EXT_VEC_NCPSG 32\n\n#define OP_UNARY_NUM_SCALE 10\n#define OP_UNARY_NUM_FILL 11\n#define OP_UNARY_NUM_CLAMP 12\n#define OP_UNARY_NUM_SQR 13\n#define OP_UNARY_NUM_SQRT 14\n#define OP_UNARY_NUM_SIN 15\n#define OP_UNARY_NUM_COS 16\n#define OP_UNARY_NUM_LOG 17\n#define OP_UNARY_NUM_LEAKY_RELU 18\n\n#define OP_UNARY_NUM_TANH 100\n#define OP_UNARY_NUM_RELU 101\n#define OP_UNARY_NUM_SIGMOID 102\n#define OP_UNARY_NUM_GELU 103\n#define OP_UNARY_NUM_GELU_ERF 104\n#define OP_UNARY_NUM_GELU_QUICK 105\n#define OP_UNARY_NUM_SILU 106\n#define OP_UNARY_NUM_ELU 107\n#define OP_UNARY_NUM_NEG 108\n#define OP_UNARY_NUM_ABS 109\n#define OP_UNARY_NUM_SGN 110\n#define OP_UNARY_NUM_STEP 111\n#define OP_UNARY_NUM_HARDSWISH 112\n#define OP_UNARY_NUM_HARDSIGMOID 113\n#define OP_UNARY_NUM_EXP 114\n#define OP_UNARY_NUM_SOFTPLUS 115\n#define OP_UNARY_NUM_EXPM1 116\n\n#define OP_SUM_ROWS_NUM_SUM_ROWS 10\n#define OP_SUM_ROWS_NUM_MEAN 11\n\n// kernel argument structs\n//\n// - element counters (e.g. ne00) typically use int32_t to reduce register usage\n// however, be careful from int overflows when using those in the kernel implementation\n//\n// - strides (e.g. nb00) use uint64_t\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne10;\n int32_t ne11;\n int32_t ne12;\n int32_t ne13;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n int32_t dim;\n} ggml_metal_kargs_concat;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n float slope;\n float scale;\n float bias;\n float val;\n float min;\n float max;\n} ggml_metal_kargs_unary;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne10;\n int32_t ne11;\n int32_t ne12;\n int32_t ne13;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n uint64_t offs;\n uint64_t o1[8];\n} ggml_metal_kargs_bin;\n\ntypedef struct {\n int64_t ne0;\n int64_t ne1;\n size_t nb01;\n size_t nb02;\n size_t nb11;\n size_t nb21;\n} ggml_metal_kargs_add_id;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_repeat;\n\ntypedef struct {\n int64_t nk0;\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n int64_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int64_t ne0;\n int64_t ne1;\n int64_t ne2;\n int64_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_cpy;\n\ntypedef struct {\n int64_t ne10;\n int64_t ne11;\n int64_t ne12;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n uint64_t offs;\n bool inplace;\n} ggml_metal_kargs_set;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n int32_t n_past;\n int32_t n_dims;\n int32_t n_ctx_orig;\n float freq_base;\n float freq_scale;\n float ext_factor;\n float attn_factor;\n float beta_fast;\n float beta_slow;\n int32_t sect_0;\n int32_t sect_1;\n int32_t sect_2;\n int32_t sect_3;\n bool src2;\n} ggml_metal_kargs_rope;\n\ntypedef struct {\n int32_t ne11;\n int32_t ne_12_2; // assume K and V are same shape\n int32_t ne_12_3;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n uint64_t nb21;\n uint64_t nb22;\n uint64_t nb23;\n int32_t ne31;\n int32_t ne32;\n int32_t ne33;\n uint64_t nb31;\n uint64_t nb32;\n uint64_t nb33;\n} ggml_metal_kargs_flash_attn_ext_pad;\n\ntypedef struct {\n int32_t ne01;\n int32_t ne30;\n int32_t ne31;\n int32_t ne32;\n int32_t ne33;\n uint64_t nb31;\n uint64_t nb32;\n uint64_t nb33;\n} ggml_metal_kargs_flash_attn_ext_blk;\n\ntypedef struct {\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne11;\n int32_t ne_12_2; // assume K and V are same shape\n int32_t ne_12_3;\n int32_t ns10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ns20;\n uint64_t nb21;\n uint64_t nb22;\n uint64_t nb23;\n int32_t ne31;\n int32_t ne32;\n int32_t ne33;\n uint64_t nb31;\n uint64_t nb32;\n uint64_t nb33;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n float scale;\n float max_bias;\n float m0;\n float m1;\n int32_t n_head_log2;\n float logit_softcap;\n} ggml_metal_kargs_flash_attn_ext;\n\ntypedef struct {\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne11;\n int32_t ne_12_2; // assume K and V are same shape\n int32_t ne_12_3;\n int32_t ns10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ns20;\n uint64_t nb21;\n uint64_t nb22;\n uint64_t nb23;\n int32_t ne31;\n int32_t ne32;\n int32_t ne33;\n uint64_t nb31;\n uint64_t nb32;\n uint64_t nb33;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n float scale;\n float max_bias;\n float m0;\n float m1;\n int32_t n_head_log2;\n float logit_softcap;\n} ggml_metal_kargs_flash_attn_ext_vec;\n\ntypedef struct {\n int32_t nrows;\n} ggml_metal_kargs_flash_attn_ext_vec_reduce;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne02;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne12;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ne0;\n int32_t ne1;\n int16_t r2;\n int16_t r3;\n} ggml_metal_kargs_mul_mm;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne10;\n int32_t ne11;\n int32_t ne12;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ne0;\n int32_t ne1;\n int32_t nr0;\n int16_t r2;\n int16_t r3;\n} ggml_metal_kargs_mul_mv;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne10;\n int32_t ne11;\n int32_t ne12;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ne0;\n int32_t ne1;\n int16_t r2;\n int16_t r3;\n} ggml_metal_kargs_mul_mv_ext;\n\ntypedef struct {\n int32_t ne02;\n int32_t ne10;\n int32_t ne11; // n_expert_used (bcast)\n uint64_t nb11;\n uint64_t nb12;\n int32_t ne21; // n_tokens\n int32_t ne20; // n_expert_used\n uint64_t nb21;\n} ggml_metal_kargs_mul_mm_id_map0;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne02;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne11;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ne20;\n int32_t ne21;\n int32_t ne0;\n int32_t ne1;\n int16_t r2;\n int16_t r3;\n} ggml_metal_kargs_mul_mm_id;\n\ntypedef struct {\n int32_t nei0;\n int32_t nei1;\n uint64_t nbi1;\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n int32_t ne10;\n int32_t ne11;\n int32_t ne12;\n int32_t ne13;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n int32_t ne0;\n int32_t ne1;\n uint64_t nb1;\n int32_t nr0;\n} ggml_metal_kargs_mul_mv_id;\n\n// NORM\n// RMS_NORM\ntypedef struct {\n int32_t ne00;\n int32_t ne00_t;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n float eps;\n int32_t nef1[3];\n int32_t nef2[3];\n int32_t nef3[3];\n uint64_t nbf1[3];\n uint64_t nbf2[3];\n uint64_t nbf3[3];\n} ggml_metal_kargs_norm;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n float eps;\n} ggml_metal_kargs_l2_norm;\n\ntypedef struct {\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n int32_t ngrp;\n float eps;\n} ggml_metal_kargs_group_norm;\n\ntypedef struct {\n int32_t IC;\n int32_t IL;\n int32_t K;\n int32_t s0;\n uint64_t nb0;\n uint64_t nb1;\n} ggml_metal_kargs_conv_transpose_1d;\n\ntypedef struct {\n int32_t IC;\n int32_t IH;\n int32_t IW;\n int32_t KH;\n int32_t KW;\n int32_t OC;\n int32_t s0;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n} ggml_metal_kargs_conv_transpose_2d;\n\ntypedef struct {\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n int32_t IW;\n int32_t IH;\n int32_t KW;\n int32_t KH;\n int32_t IC;\n int32_t OC;\n int32_t OW;\n int32_t OH;\n int32_t N;\n int32_t s0;\n int32_t s1;\n int32_t p0;\n int32_t p1;\n int32_t d0;\n int32_t d1;\n} ggml_metal_kargs_conv_2d;\n\ntypedef struct {\n uint64_t ofs0;\n uint64_t ofs1;\n int32_t IW;\n int32_t IH;\n int32_t CHW;\n int32_t s0;\n int32_t s1;\n int32_t p0;\n int32_t p1;\n int32_t d0;\n int32_t d1;\n int32_t N;\n int32_t KH;\n int32_t KW;\n int32_t KHW; // KH * KW, pre-computed on CPU to save GPU resources\n} ggml_metal_kargs_im2col;\n\ntypedef struct{\n int32_t ne00;\n uint64_t nb01;\n int32_t ne10;\n uint64_t nb11;\n int32_t ne0;\n uint64_t nb1;\n int32_t i00;\n int32_t i10;\n float alpha;\n float limit;\n} ggml_metal_kargs_glu;\n\ntypedef struct {\n uint64_t np;\n} ggml_metal_kargs_sum;\n\ntypedef struct {\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n int64_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int64_t ne0;\n int64_t ne1;\n int64_t ne2;\n int64_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_sum_rows;\n\ntypedef struct {\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n int64_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int64_t net0;\n int64_t net1;\n int64_t net2;\n int64_t net3;\n uint64_t nbt0;\n uint64_t nbt1;\n uint64_t nbt2;\n uint64_t nbt3;\n bool outb;\n} ggml_metal_kargs_cumsum_blk;\n\ntypedef struct {\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n int64_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int64_t net0;\n int64_t net1;\n int64_t net2;\n int64_t net3;\n uint64_t nbt0;\n uint64_t nbt1;\n uint64_t nbt2;\n uint64_t nbt3;\n} ggml_metal_kargs_cumsum_add;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne11;\n int32_t ne12;\n int32_t ne13;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n float scale;\n float max_bias;\n float m0;\n float m1;\n int32_t n_head_log2;\n} ggml_metal_kargs_soft_max;\n\ntypedef struct {\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n int64_t ne10;\n int64_t ne11;\n uint64_t nb10;\n uint64_t nb11;\n int64_t ne0;\n int64_t ne1;\n int64_t ne2;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n} ggml_metal_kargs_ssm_conv;\n\ntypedef struct {\n int64_t d_state;\n int64_t d_inner;\n int64_t n_head;\n int64_t n_group;\n int64_t n_seq_tokens;\n int64_t n_seqs;\n uint64_t s_off;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t ns12;\n uint64_t nb13;\n uint64_t nb20;\n uint64_t nb21;\n uint64_t ns21;\n uint64_t nb22;\n int64_t ne30;\n uint64_t nb31;\n uint64_t nb41;\n uint64_t nb42;\n uint64_t ns42;\n uint64_t nb43;\n uint64_t nb51;\n uint64_t nb52;\n uint64_t ns52;\n uint64_t nb53;\n uint64_t nb0;\n} ggml_metal_kargs_ssm_scan;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne10;\n int32_t ne11;\n int32_t ne12;\n int32_t ne13;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ne20;\n int32_t ne21;\n int32_t ne22;\n int32_t ne23;\n uint64_t nb20;\n uint64_t nb21;\n uint64_t nb22;\n uint64_t nb23;\n int32_t ns02;\n int32_t ns12;\n int32_t ns22;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_gated_delta_net;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne10;\n int32_t ne11;\n int32_t ne12;\n int32_t ne13;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_solve_tri;\n\ntypedef struct {\n int32_t ne00t;\n int32_t ne00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne10;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_get_rows;\n\ntypedef struct {\n int32_t nk0;\n int32_t ne01;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne11;\n int32_t ne12;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_set_rows;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_diag;\n\ntypedef struct {\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n int64_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int64_t ne0;\n int64_t ne1;\n int64_t ne2;\n int64_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n float sf0;\n float sf1;\n float sf2;\n float sf3;\n float poffs;\n} ggml_metal_kargs_upscale;\n\ntypedef struct {\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n int64_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int64_t ne0;\n int64_t ne1;\n int64_t ne2;\n int64_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_pad;\n\ntypedef struct {\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n int64_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int64_t ne0;\n int64_t ne1;\n int64_t ne2;\n int64_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n int32_t p0;\n int32_t p1;\n} ggml_metal_kargs_pad_reflect_1d;\n\ntypedef struct {\n uint64_t nb1;\n int dim;\n int max_period;\n} ggml_metal_kargs_timestep_embedding;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n uint64_t nb0;\n uint64_t nb1;\n uint64_t nb2;\n uint64_t nb3;\n} ggml_metal_kargs_tri;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n int32_t top_k;\n} ggml_metal_kargs_argsort;\n\ntypedef struct {\n int64_t ne00;\n int64_t ne01;\n int64_t ne02;\n int64_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n int32_t ne0;\n int32_t ne1;\n int32_t ne2;\n int32_t ne3;\n int32_t top_k;\n int32_t len;\n} ggml_metal_kargs_argsort_merge;\n\ntypedef struct {\n int64_t ne0;\n float start;\n float step;\n} ggml_metal_kargs_arange;\n\ntypedef struct {\n int64_t val;\n} ggml_metal_kargs_memset;\n\ntypedef struct {\n int32_t ne00;\n int32_t ne01;\n int32_t ne02;\n int32_t ne03;\n uint64_t nb00;\n uint64_t nb01;\n uint64_t nb02;\n uint64_t nb03;\n uint64_t nb10;\n uint64_t nb11;\n uint64_t nb12;\n uint64_t nb13;\n} ggml_metal_kargs_count_equal;\n\ntypedef struct {\n int32_t k0;\n int32_t k1;\n int32_t s0;\n int32_t s1;\n int32_t p0;\n int32_t p1;\n int64_t IH;\n int64_t IW;\n int64_t OH;\n int64_t OW;\n int64_t np;\n} ggml_metal_kargs_pool_2d;\n\ntypedef struct {\n int32_t k0;\n int32_t s0;\n int32_t p0;\n int64_t IW;\n int64_t OW;\n int64_t np;\n} ggml_metal_kargs_pool_1d;\n\ntypedef struct {\n int64_t ne00;\n uint64_t nb01;\n} ggml_metal_kargs_argmax;\n\ntypedef struct {\n int64_t np;\n} ggml_metal_kargs_opt_step_adamw;\n\ntypedef struct {\n int64_t np;\n} ggml_metal_kargs_opt_step_sgd;\n\n#endif // GGML_METAL_IMPL\n" }, { "path": "src/ggml-metal/ggml-metal-ops.cpp", "content": "#include \"ggml-metal-ops.h\"\n\n#include \"ggml.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-metal-impl.h\"\n#include \"ggml-metal-common.h\"\n#include \"ggml-metal-device.h\"\n\n#include \n#include \n#include \n#include \n\nstatic ggml_metal_buffer_id ggml_metal_get_buffer_id(const ggml_tensor * t) {\n if (!t) {\n return { nullptr, 0 };\n }\n\n ggml_backend_buffer_t buffer = t->view_src ? t->view_src->buffer : t->buffer;\n\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t) buffer->context;\n\n return ggml_metal_buffer_get_id(ctx, t);\n}\n\nstruct ggml_metal_op {\n ggml_metal_op(\n ggml_metal_device_t dev,\n ggml_metal_cmd_buf_t cmd_buf,\n ggml_cgraph * gf,\n int idx_start,\n int idx_end,\n bool use_fusion,\n bool use_concurrency,\n bool use_capture,\n int debug_graph,\n int debug_fusion) {\n this->dev = dev;\n this->lib = ggml_metal_device_get_library(dev);\n this->enc = ggml_metal_encoder_init(cmd_buf, use_concurrency);\n this->mem_ranges = ggml_mem_ranges_init(debug_graph);\n this->idx_start = idx_start;\n this->idx_end = idx_end;\n this->use_fusion = use_fusion;\n this->use_concurrency = use_concurrency;\n this->use_capture = use_capture;\n this->debug_graph = debug_graph;\n this->debug_fusion = debug_fusion;\n this->gf = gf;\n\n idxs.reserve(gf->n_nodes);\n\n // filter empty nodes\n // TODO: this can be removed when the allocator starts filtering them earlier\n // https://github.com/ggml-org/llama.cpp/pull/16130#issuecomment-3327905830\n for (int i = idx_start; i < idx_end; i++) {\n if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) {\n idxs.push_back(i);\n }\n }\n }\n\n ~ggml_metal_op() {\n ggml_metal_encoder_end_encoding(this->enc);\n ggml_metal_encoder_free(this->enc);\n ggml_mem_ranges_free(this->mem_ranges);\n }\n\n int n_nodes() const {\n return idxs.size();\n }\n\n ggml_tensor * node(int i) const {\n assert(i >= 0 && i < (int) idxs.size());\n return ggml_graph_node(gf, idxs[i]);\n }\n\n bool can_fuse(int i0, const ggml_op * ops, int n_ops) const {\n assert(use_fusion);\n assert(i0 >= 0 && i0 < n_nodes());\n\n if (i0 + n_ops > n_nodes()) {\n return false;\n }\n\n return ggml_can_fuse_ext(gf, idxs.data() + i0, ops, n_ops);\n }\n\n ggml_metal_device_t dev;\n ggml_metal_library_t lib;\n ggml_metal_encoder_t enc;\n ggml_mem_ranges_t mem_ranges;\n\n bool use_fusion;\n bool use_concurrency;\n bool use_capture;\n\n int debug_graph;\n int debug_fusion;\n\nprivate:\n ggml_cgraph * gf;\n\n int idx_start;\n int idx_end;\n\n // non-empty node indices\n std::vector idxs;\n};\n\nggml_metal_op_t ggml_metal_op_init(\n ggml_metal_device_t dev,\n ggml_metal_cmd_buf_t cmd_buf,\n ggml_cgraph * gf,\n int idx_start,\n int idx_end,\n bool use_fusion,\n bool use_concurrency,\n bool use_capture,\n int debug_graph,\n int debug_fusion) {\n ggml_metal_op_t res = new ggml_metal_op(\n dev,\n cmd_buf,\n gf,\n idx_start,\n idx_end,\n use_fusion,\n use_concurrency,\n use_capture,\n debug_graph,\n debug_fusion);\n\n return res;\n}\n\nvoid ggml_metal_op_free(ggml_metal_op_t ctx) {\n delete ctx;\n}\n\nint ggml_metal_op_n_nodes(ggml_metal_op_t ctx) {\n return ctx->n_nodes();\n}\n\nstatic bool ggml_metal_op_concurrency_reset(ggml_metal_op_t ctx) {\n if (!ctx->mem_ranges) {\n return true;\n }\n\n ggml_metal_encoder_memory_barrier(ctx->enc);\n\n ggml_mem_ranges_reset(ctx->mem_ranges);\n\n return true;\n}\n\nstatic bool ggml_metal_op_concurrency_check(ggml_metal_op_t ctx, const ggml_tensor * node) {\n if (!ctx->mem_ranges) {\n return false;\n }\n\n return ggml_mem_ranges_check(ctx->mem_ranges, node);\n}\n\nstatic bool ggml_metal_op_concurrency_add(ggml_metal_op_t ctx, const ggml_tensor * node) {\n if (!ctx->mem_ranges) {\n return true;\n }\n\n return ggml_mem_ranges_add(ctx->mem_ranges, node);\n}\n\nstatic int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {\n struct ggml_tensor * node = ctx->node(idx);\n\n //GGML_LOG_INFO(\"%s: encoding node %3d, op = %8s\\n\", __func__, idx, ggml_op_name(node->op));\n\n if (ggml_is_empty(node)) {\n return 1;\n }\n\n switch (node->op) {\n case GGML_OP_NONE:\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_TRANSPOSE:\n case GGML_OP_PERMUTE:\n {\n // noop -> next node\n if (ctx->debug_graph > 0) {\n GGML_LOG_DEBUG(\"%s: node[%5d] - %-12s %s\\n\", __func__, idx, ggml_op_name(node->op), \"(noop)\");\n }\n } return 1;\n default:\n {\n } break;\n }\n\n if (!ggml_metal_device_supports_op(ctx->dev, node)) {\n GGML_LOG_ERROR(\"%s: error: unsupported op '%s'\\n\", __func__, ggml_op_desc(node));\n GGML_ABORT(\"unsupported op\");\n }\n\n if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n return 1;\n }\n\n int n_fuse = 1;\n\n // check if the current node can run concurrently with other nodes before it\n // the condition is that:\n // - the current node cannot write to any previous src or dst ranges\n // - the current node cannot read from any previous dst ranges\n //\n // if the condition is not satisfied, we put a memory barrier and clear all ranges\n // otherwise, we add the new ranges to the encoding context and process the node concurrently\n //\n {\n const bool is_concurrent = ggml_metal_op_concurrency_check(ctx, node);\n\n if (!is_concurrent) {\n ggml_metal_op_concurrency_reset(ctx);\n }\n\n if (ctx->debug_graph > 0) {\n GGML_LOG_DEBUG(\"%s: node[%5d] - %-12s %-12s %s\\n\", __func__, idx, ggml_op_name(node->op), ggml_get_name(node), is_concurrent ? \"(concurrent)\" : \"\");\n }\n if (ctx->debug_graph > 1) {\n GGML_TENSOR_LOCALS( int64_t, ne0, node->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, node->src[0], nb);\n GGML_TENSOR_LOCALS( int64_t, ne1, node->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, node->src[1], nb);\n GGML_TENSOR_LOCALS( int64_t, ne2, node->src[2], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb2, node->src[2], nb);\n GGML_TENSOR_LOCALS( int64_t, ne3, node->src[3], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb3, node->src[3], nb);\n GGML_TENSOR_LOCALS( int64_t, ne, node, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, node, nb);\n\n if (node->src[0]) {\n GGML_LOG_DEBUG(\"%s: src0 - %4s [%5lld, %5lld, %5lld, %5lld] [%5lld, %5lld, %5lld, %5lld], %d, %s\\n\", __func__, ggml_type_name(node->src[0]->type), ne00, ne01, ne02, ne03, nb00, nb01, nb02, nb03,\n ggml_is_contiguous(node->src[0]), node->src[0]->name);\n }\n if (node->src[1]) {\n GGML_LOG_DEBUG(\"%s: src1 - %4s [%5lld, %5lld, %5lld, %5lld] [%5lld, %5lld, %5lld, %5lld], %d, %s\\n\", __func__, ggml_type_name(node->src[1]->type), ne10, ne11, ne12, ne13, nb10, nb11, nb12, nb13,\n ggml_is_contiguous(node->src[1]), node->src[1]->name);\n }\n if (node->src[2]) {\n GGML_LOG_DEBUG(\"%s: src2 - %4s [%5lld, %5lld, %5lld, %5lld] [%5lld, %5lld, %5lld, %5lld], %d, %s\\n\", __func__, ggml_type_name(node->src[2]->type), ne20, ne21, ne22, ne23, nb20, nb21, nb22, nb23,\n ggml_is_contiguous(node->src[2]), node->src[2]->name);\n }\n if (node->src[3]) {\n GGML_LOG_DEBUG(\"%s: src3 - %4s [%5lld, %5lld, %5lld, %5lld] [%5lld, %5lld, %5lld, %5lld], %d, %s\\n\", __func__, ggml_type_name(node->src[3]->type), ne30, ne31, ne32, ne33, nb30, nb31, nb32, nb33,\n ggml_is_contiguous(node->src[3]), node->src[3]->name);\n }\n if (node) {\n GGML_LOG_DEBUG(\"%s: node - %4s [%5lld, %5lld, %5lld, %5lld] [%5lld, %5lld, %5lld, %5lld], 1, %s\\n\", __func__, ggml_type_name(node->type), ne0, ne1, ne2, ne3, nb0, nb1, nb2, nb3,\n node->name);\n }\n }\n }\n\n switch (node->op) {\n case GGML_OP_CONCAT:\n {\n n_fuse = ggml_metal_op_concat(ctx, idx);\n } break;\n case GGML_OP_ADD:\n case GGML_OP_SUB:\n case GGML_OP_MUL:\n case GGML_OP_DIV:\n {\n n_fuse = ggml_metal_op_bin(ctx, idx);\n } break;\n case GGML_OP_ADD_ID:\n {\n n_fuse = ggml_metal_op_add_id(ctx, idx);\n } break;\n case GGML_OP_REPEAT:\n {\n n_fuse = ggml_metal_op_repeat(ctx, idx);\n } break;\n case GGML_OP_ACC:\n {\n n_fuse = ggml_metal_op_acc(ctx, idx);\n } break;\n case GGML_OP_SCALE:\n case GGML_OP_FILL:\n case GGML_OP_CLAMP:\n case GGML_OP_LEAKY_RELU:\n case GGML_OP_SQR:\n case GGML_OP_SQRT:\n case GGML_OP_SIN:\n case GGML_OP_COS:\n case GGML_OP_LOG:\n case GGML_OP_UNARY:\n {\n n_fuse = ggml_metal_op_unary(ctx, idx);\n } break;\n case GGML_OP_GLU:\n {\n n_fuse = ggml_metal_op_glu(ctx, idx);\n } break;\n case GGML_OP_SUM:\n {\n n_fuse = ggml_metal_op_sum(ctx, idx);\n } break;\n case GGML_OP_SUM_ROWS:\n case GGML_OP_MEAN:\n {\n n_fuse = ggml_metal_op_sum_rows(ctx, idx);\n } break;\n case GGML_OP_CUMSUM:\n {\n n_fuse = ggml_metal_op_cumsum(ctx, idx);\n } break;\n case GGML_OP_SOFT_MAX:\n {\n n_fuse = ggml_metal_op_soft_max(ctx, idx);\n } break;\n case GGML_OP_SSM_CONV:\n {\n n_fuse = ggml_metal_op_ssm_conv(ctx, idx);\n } break;\n case GGML_OP_SSM_SCAN:\n {\n n_fuse = ggml_metal_op_ssm_scan(ctx, idx);\n } break;\n case GGML_OP_RWKV_WKV6:\n case GGML_OP_RWKV_WKV7:\n {\n n_fuse = ggml_metal_op_rwkv(ctx, idx);\n } break;\n case GGML_OP_GATED_DELTA_NET:\n {\n n_fuse = ggml_metal_op_gated_delta_net(ctx, idx);\n } break;\n case GGML_OP_SOLVE_TRI:\n {\n n_fuse = ggml_metal_op_solve_tri(ctx, idx);\n } break;\n case GGML_OP_MUL_MAT:\n {\n n_fuse = ggml_metal_op_mul_mat(ctx, idx);\n } break;\n case GGML_OP_MUL_MAT_ID:\n {\n n_fuse = ggml_metal_op_mul_mat_id(ctx, idx);\n } break;\n case GGML_OP_GET_ROWS:\n {\n n_fuse = ggml_metal_op_get_rows(ctx, idx);\n } break;\n case GGML_OP_SET_ROWS:\n {\n n_fuse = ggml_metal_op_set_rows(ctx, idx);\n } break;\n case GGML_OP_DIAG:\n {\n n_fuse = ggml_metal_op_diag(ctx, idx);\n } break;\n case GGML_OP_L2_NORM:\n {\n n_fuse = ggml_metal_op_l2_norm(ctx, idx);\n } break;\n case GGML_OP_GROUP_NORM:\n {\n n_fuse = ggml_metal_op_group_norm(ctx, idx);\n } break;\n case GGML_OP_NORM:\n case GGML_OP_RMS_NORM:\n {\n n_fuse = ggml_metal_op_norm(ctx, idx);\n } break;\n case GGML_OP_ROPE:\n {\n n_fuse = ggml_metal_op_rope(ctx, idx);\n } break;\n case GGML_OP_IM2COL:\n {\n n_fuse = ggml_metal_op_im2col(ctx, idx);\n } break;\n case GGML_OP_CONV_2D:\n {\n n_fuse = ggml_metal_op_conv_2d(ctx, idx);\n } break;\n case GGML_OP_CONV_TRANSPOSE_1D:\n {\n n_fuse = ggml_metal_op_conv_transpose_1d(ctx, idx);\n } break;\n case GGML_OP_CONV_TRANSPOSE_2D:\n {\n n_fuse = ggml_metal_op_conv_transpose_2d(ctx, idx);\n } break;\n case GGML_OP_UPSCALE:\n {\n n_fuse = ggml_metal_op_upscale(ctx, idx);\n } break;\n case GGML_OP_PAD:\n {\n n_fuse = ggml_metal_op_pad(ctx, idx);\n } break;\n case GGML_OP_PAD_REFLECT_1D:\n {\n n_fuse = ggml_metal_op_pad_reflect_1d(ctx, idx);\n } break;\n case GGML_OP_ARANGE:\n {\n n_fuse = ggml_metal_op_arange(ctx, idx);\n } break;\n case GGML_OP_TIMESTEP_EMBEDDING:\n {\n n_fuse = ggml_metal_op_timestep_embedding(ctx, idx);\n } break;\n case GGML_OP_ARGSORT:\n {\n n_fuse = ggml_metal_op_argsort(ctx, idx);\n } break;\n case GGML_OP_TOP_K:\n {\n n_fuse = ggml_metal_op_top_k(ctx, idx);\n } break;\n case GGML_OP_TRI:\n {\n n_fuse = ggml_metal_op_tri(ctx, idx);\n } break;\n case GGML_OP_FLASH_ATTN_EXT:\n {\n n_fuse = ggml_metal_op_flash_attn_ext(ctx, idx);\n } break;\n case GGML_OP_SET:\n {\n n_fuse = ggml_metal_op_set(ctx, idx);\n } break;\n case GGML_OP_DUP:\n case GGML_OP_CPY:\n case GGML_OP_CONT:\n {\n n_fuse = ggml_metal_op_cpy(ctx, idx);\n } break;\n case GGML_OP_POOL_1D:\n {\n n_fuse = ggml_metal_op_pool_1d(ctx, idx);\n } break;\n case GGML_OP_POOL_2D:\n {\n n_fuse = ggml_metal_op_pool_2d(ctx, idx);\n } break;\n case GGML_OP_ARGMAX:\n {\n n_fuse = ggml_metal_op_argmax(ctx, idx);\n } break;\n case GGML_OP_OPT_STEP_ADAMW:\n {\n n_fuse = ggml_metal_op_opt_step_adamw(ctx, idx);\n } break;\n case GGML_OP_OPT_STEP_SGD:\n {\n n_fuse = ggml_metal_op_opt_step_sgd(ctx, idx);\n } break;\n case GGML_OP_COUNT_EQUAL:\n {\n n_fuse = ggml_metal_op_count_equal(ctx, idx);\n } break;\n default:\n {\n GGML_LOG_ERROR(\"%s: error: node %3d, op = %8s not implemented\\n\", __func__, idx, ggml_op_name(node->op));\n GGML_ABORT(\"fatal error\");\n }\n }\n\n if (ctx->debug_graph > 0) {\n if (n_fuse > 1) {\n GGML_LOG_DEBUG(\"%s: fuse %d ops\\n\", __func__, n_fuse);\n }\n }\n\n // update the mem ranges in the encoding context\n for (int i = 0; i < n_fuse; ++i) {\n if (!ggml_metal_op_concurrency_add(ctx, ctx->node(idx + i))) {\n ggml_metal_op_concurrency_reset(ctx);\n }\n }\n\n return n_fuse;\n}\n\nint ggml_metal_op_encode(ggml_metal_op_t ctx, int idx) {\n if (ctx->use_capture) {\n ggml_metal_encoder_debug_group_push(ctx->enc, ggml_op_desc(ctx->node(idx)));\n }\n\n int res = ggml_metal_op_encode_impl(ctx, idx);\n if (idx + res > ctx->n_nodes()) {\n GGML_ABORT(\"fusion error: nodes spanning multiple encoders have been fused. this indicates a bug in the fusion logic %s\",\n \"https://github.com/ggml-org/llama.cpp/pull/14849\");\n }\n\n if (ctx->use_capture) {\n ggml_metal_encoder_debug_group_pop(ctx->enc);\n }\n\n return res;\n}\n\nint ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const int32_t dim = ((const int32_t *) op->op_params)[0];\n\n ggml_metal_kargs_concat args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne10 =*/ ne10,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.ne13 =*/ ne13,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.dim =*/ dim,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_base(lib, GGML_OP_CONCAT);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n const int nth = std::min(1024, ne0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_repeat(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline = ggml_metal_library_get_pipeline_repeat(lib, op->type);\n\n ggml_metal_kargs_repeat args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[1]));\n\n const size_t pnb1 = ((const int32_t *) op->op_params)[0];\n const size_t pnb2 = ((const int32_t *) op->op_params)[1];\n const size_t pnb3 = ((const int32_t *) op->op_params)[2];\n const size_t offs = ((const int32_t *) op->op_params)[3];\n\n const bool inplace = (bool) ((const int32_t *) op->op_params)[4];\n\n if (!inplace) {\n // run a separate kernel to cpy src->dst\n // not sure how to avoid this\n // TODO: make a simpler cpy_bytes kernel\n\n //const id pipeline = ctx->pipelines[GGML_METAL_PIPELINE_TYPE_CPY_F32_F32].obj;\n auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[0]->type, op->type);\n\n ggml_metal_kargs_cpy args = {\n /*.nk0 =*/ ne00,\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n\n ggml_metal_op_concurrency_reset(ctx);\n }\n\n ggml_metal_kargs_bin args = {\n /*.ne00 =*/ ne10,\n /*.ne01 =*/ ne11,\n /*.ne02 =*/ ne12,\n /*.ne03 =*/ ne13,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ pnb1,\n /*.nb02 =*/ pnb2,\n /*.nb03 =*/ pnb3,\n /*.ne10 =*/ ne10,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.ne13 =*/ ne13,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ne0 =*/ ne10,\n /*.ne1 =*/ ne11,\n /*.ne2 =*/ ne12,\n /*.ne3 =*/ ne13,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ pnb1,\n /*.nb2 =*/ pnb2,\n /*.nb3 =*/ pnb3,\n /*.offs =*/ offs,\n /*.o1 =*/ { 0 },\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_bin_one(lib, GGML_OP_ADD);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n int nth = 1;\n\n while (2*nth < args.ne0 && nth < nth_max) {\n nth *= 2;\n }\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne11, ne12, ne13, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n ggml_metal_kargs_unary args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.slope =*/ 0.0,\n /*.scale =*/ 0.0,\n /*.bias =*/ 0.0,\n /*.val =*/ 0.0,\n /*.min =*/ 0.0,\n /*.max =*/ 0.0,\n };\n\n if (op->op == GGML_OP_LEAKY_RELU) {\n args.slope = ggml_get_op_params_f32(op, 0);\n }\n\n if (op->op == GGML_OP_SCALE) {\n args.scale = ggml_get_op_params_f32(op, 0);\n args.bias = ggml_get_op_params_f32(op, 1);\n }\n\n if (op->op == GGML_OP_FILL) {\n args.val = ggml_get_op_params_f32(op, 0);\n }\n\n if (op->op == GGML_OP_CLAMP) {\n args.min = ggml_get_op_params_f32(op, 0);\n args.max = ggml_get_op_params_f32(op, 1);\n }\n\n auto pipeline = ggml_metal_library_get_pipeline_unary(lib, op);\n\n if (pipeline.c4) {\n args.ne00 = ne00/4;\n args.ne0 = ne0/4;\n }\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n\n if (pipeline.cnt) {\n const int n = pipeline.c4 ? ggml_nelements(op)/4 : ggml_nelements(op);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, 1, 1, 1);\n } else {\n const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n const int nth = MIN(args.ne00, nth_max);\n\n const int nk0 = (args.ne00 + nth - 1)/nth;\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nk0*ne01, ne02, ne03, nth, 1, 1);\n }\n\n return 1;\n}\n\nint ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n if (op->src[1]) {\n GGML_ASSERT(ggml_are_same_shape(op->src[0], op->src[1]));\n }\n\n auto pipeline = ggml_metal_library_get_pipeline_glu(lib, op);\n\n const int32_t swp = ggml_get_op_params_i32(op, 1);\n const float alpha = ggml_get_op_params_f32(op, 2);\n const float limit = ggml_get_op_params_f32(op, 3);\n\n const int32_t i00 = swp ? ne0 : 0;\n const int32_t i10 = swp ? 0 : ne0;\n\n ggml_metal_kargs_glu args = {\n /*.ne00 =*/ ne00,\n /*.nb01 =*/ nb01,\n /*.ne10 =*/ op->src[1] ? ne10 : ne00,\n /*.nb11 =*/ op->src[1] ? nb11 : nb01,\n /*.ne0 =*/ ne0,\n /*.nb1 =*/ nb1,\n /*.i00 =*/ op->src[1] ? 0 : i00,\n /*.i10 =*/ op->src[1] ? 0 : i10,\n /*.alpha=*/ alpha,\n /*.limit=*/ limit\n };\n\n const int64_t nrows = ggml_nrows(op->src[0]);\n\n const int32_t nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00/2);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n if (op->src[1]) {\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n } else {\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 2);\n }\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_sum(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n const uint64_t n = (uint64_t) ggml_nelements(op->src[0]);\n\n ggml_metal_kargs_sum args = {\n /*.np =*/ n,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_sum(lib, op);\n\n int nth = 32; // SIMD width\n\n while (nth < (int) n && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nth *= 2;\n }\n\n nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n nth = std::min(nth, (int) n);\n\n const int nsg = (nth + 31) / 32;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, nsg * sizeof(float), 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_sum_rows(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n ggml_metal_kargs_sum_rows args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_sum_rows(lib, op);\n\n if (pipeline.c4) {\n args.ne00 = ne00/4;\n args.ne0 = ne0/4;\n }\n\n int nth = 32; // SIMD width\n\n while (nth < args.ne00 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nth *= 2;\n }\n\n nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n nth = std::min(nth, (int) args.ne00);\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_cumsum(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline_blk = ggml_metal_library_get_pipeline_cumsum_blk(lib, op);\n\n int nth = 1;\n while (nth < ne00 && 2*nth <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline_blk)) {\n nth *= 2;\n }\n\n GGML_ASSERT(ne00 <= nth*nth);\n\n const int64_t net0 = (ne00 + nth - 1) / nth;\n const int64_t net1 = ne01;\n const int64_t net2 = ne02;\n const int64_t net3 = ne03;\n\n const uint64_t nbt0 = sizeof(float);\n const uint64_t nbt1 = net0*nbt0;\n const uint64_t nbt2 = net1*nbt1;\n const uint64_t nbt3 = net2*nbt2;\n\n const size_t smem = GGML_PAD(32*sizeof(float), 16);\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n ggml_metal_buffer_id bid_tmp = bid_dst;\n bid_tmp.offs += ggml_nbytes(op);\n\n {\n ggml_metal_kargs_cumsum_blk args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.net0 =*/ net0,\n /*.net1 =*/ net1,\n /*.net2 =*/ net2,\n /*.net3 =*/ net3,\n /*.nbt0 =*/ nbt0,\n /*.nbt1 =*/ nbt1,\n /*.nbt2 =*/ nbt2,\n /*.nbt3 =*/ nbt3,\n /*.outb =*/ ne00 > nth,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline_blk);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_tmp, 2);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 3);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, net0*ne01, ne02, ne03, nth, 1, 1);\n }\n\n if (ne00 > nth) {\n ggml_metal_op_concurrency_reset(ctx);\n\n {\n ggml_metal_kargs_cumsum_blk args = {\n /*.ne00 =*/ net0,\n /*.ne01 =*/ net1,\n /*.ne02 =*/ net2,\n /*.ne03 =*/ net3,\n /*.nb00 =*/ nbt0,\n /*.nb01 =*/ nbt1,\n /*.nb02 =*/ nbt2,\n /*.nb03 =*/ nbt3,\n /*.net0 =*/ net0,\n /*.net1 =*/ net1,\n /*.net2 =*/ net2,\n /*.net3 =*/ net3,\n /*.nbt0 =*/ nbt0,\n /*.nbt1 =*/ nbt1,\n /*.nbt2 =*/ nbt2,\n /*.nbt3 =*/ nbt3,\n /*.outb =*/ false,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline_blk);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);\n ggml_metal_encoder_set_buffer (enc, bid_tmp, 2);\n ggml_metal_encoder_set_buffer (enc, bid_tmp, 3);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, net1, net2, net3, nth, 1, 1);\n }\n\n ggml_metal_op_concurrency_reset(ctx);\n\n {\n auto pipeline_add = ggml_metal_library_get_pipeline_cumsum_add(lib, op);\n\n ggml_metal_kargs_cumsum_add args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.net0 =*/ net0,\n /*.net1 =*/ net1,\n /*.net2 =*/ net2,\n /*.net3 =*/ net3,\n /*.nbt0 =*/ nbt0,\n /*.nbt1 =*/ nbt1,\n /*.nbt2 =*/ nbt2,\n /*.nbt3 =*/ nbt3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline_add);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, net0*ne01, ne02, ne03, nth, 1, 1);\n }\n }\n\n return 1;\n}\n\nint ggml_metal_op_get_rows(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline = ggml_metal_library_get_pipeline_get_rows(lib, op->src[0]->type);\n\n ggml_metal_kargs_get_rows args = {\n /*.ne00t =*/ ggml_is_quantized(op->src[0]->type) ? ne00/16 : ne00,\n /*.ne00 =*/ ne00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne10 =*/ ne10,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n const int nth = std::min(args.ne00t, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n const int nw0 = (args.ne00t + nth - 1)/nth;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nw0*ne10, ne11, ne12, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_set_rows(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op->src[1]->type, op->type);\n\n const int32_t nk0 = ne0/ggml_blck_size(op->type);\n\n int nth = 32; // SIMD width\n\n while (nth < nk0 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nth *= 2;\n }\n\n int nrptg = 1;\n if (nth > nk0) {\n nrptg = (nth + nk0 - 1)/nk0;\n nth = nk0;\n\n if (nrptg*nth > ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nrptg--;\n }\n }\n\n nth = std::min(nth, nk0);\n\n ggml_metal_kargs_set_rows args = {\n /*.nk0 =*/ nk0,\n /*.ne01 =*/ ne01,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nrptg - 1)/nrptg, ne02, ne03, nth, nrptg, 1);\n\n return 1;\n}\n\nint ggml_metal_op_diag(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS(int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS(int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n ggml_metal_kargs_diag args = {\n /*.ne00 =*/ne00,\n /*.ne01 =*/ne01,\n /*.ne02 =*/ne02,\n /*.ne03 =*/ne03,\n /*.nb00 =*/nb00,\n /*.nb01 =*/nb01,\n /*.nb02 =*/nb02,\n /*.nb03 =*/nb03,\n /*.ne0 =*/ne0,\n /*.ne1 =*/ne1,\n /*.ne2 =*/ne2,\n /*.ne3 =*/ne3,\n /*.nb0 =*/nb0,\n /*.nb1 =*/nb1,\n /*.nb2 =*/nb2,\n /*.nb3 =*/nb3,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_diag(lib, op);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, 32, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n float scale;\n float max_bias;\n\n memcpy(&scale, ((const int32_t *) op->op_params) + 0, sizeof(scale));\n memcpy(&max_bias, ((const int32_t *) op->op_params) + 1, sizeof(max_bias));\n\n const uint32_t n_head = op->src[0]->ne[2];\n const int32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));\n\n const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n // softmax\n\n ggml_metal_kargs_soft_max args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.ne13 =*/ ne13,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.scale =*/ scale,\n /*.max_bias =*/ max_bias,\n /*.m0 =*/ m0,\n /*.m1 =*/ m1,\n /*.n_head_log2 =*/ n_head_log2,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_soft_max(lib, op);\n\n int nth = 32; // SIMD width\n\n if (ne00%4 == 0) {\n while (nth < ne00/4 && nth*ne01*ne02*ne03 < 256) {\n nth *= 2;\n }\n } else {\n while (nth < ne00 && nth*ne01*ne02*ne03 < 256) {\n nth *= 2;\n }\n }\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n if (op->src[1]) {\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n } else {\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 2);\n }\n if (op->src[2]) {\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[2]), 3);\n } else {\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 3);\n }\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n ggml_metal_kargs_ssm_conv args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.ne10 =*/ ne10,\n /*.ne11 =*/ ne11,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n };\n\n // Use batched kernel for prefill (ne1 > 1) to reduce threadgroup dispatch overhead\n const bool use_batched = (ne1 > 1);\n\n if (use_batched) {\n // Determine the smallest power of 2 that's >= ne1, but <= 256\n int BATCH_SIZE;\n if (ne1 > 128) BATCH_SIZE = 256;\n else if (ne1 > 64 ) BATCH_SIZE = 128;\n else if (ne1 > 32 ) BATCH_SIZE = 64;\n else if (ne1 > 16 ) BATCH_SIZE = 32;\n else if (ne1 > 8 ) BATCH_SIZE = 16;\n else if (ne1 > 4 ) BATCH_SIZE = 8;\n else BATCH_SIZE = 2;\n\n auto pipeline = ggml_metal_library_get_pipeline_ssm_conv_batched(lib, op, BATCH_SIZE);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3);\n\n // Dispatch: ne01 rows, ceil(ne1/BATCH_SIZE) token batches, ne02 sequences\n // Each threadgroup has BATCH_SIZE threads, each handling one token\n const int n_token_batches = (ne1 + BATCH_SIZE - 1) / BATCH_SIZE;\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, n_token_batches, ne02, BATCH_SIZE, 1, 1);\n } else {\n auto pipeline = ggml_metal_library_get_pipeline_ssm_conv(lib, op);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne1, ne02, 1, 1, 1);\n }\n\n return 1;\n}\n\nint ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb);\n GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb3, op->src[3], nb);\n GGML_TENSOR_LOCALS( int32_t, ne4, op->src[4], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb4, op->src[4], nb);\n GGML_TENSOR_LOCALS( int32_t, ne5, op->src[5], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb5, op->src[5], nb);\n GGML_TENSOR_LOCALS( int32_t, ne6, op->src[6], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb6, op->src[6], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const ggml_tensor * src3 = op->src[3];\n const ggml_tensor * src4 = op->src[4];\n const ggml_tensor * src5 = op->src[5];\n const ggml_tensor * src6 = op->src[6];\n\n GGML_ASSERT(src3);\n GGML_ASSERT(src4);\n GGML_ASSERT(src5);\n GGML_ASSERT(src6);\n\n const int64_t d_state = ne00;\n const int64_t d_inner = ne01;\n const int64_t n_head = ne02;\n const int64_t n_group = ne41;\n const int64_t n_seq_tokens = ne12;\n const int64_t n_seqs = ne13;\n\n ggml_metal_kargs_ssm_scan args = {\n /*.d_state =*/ d_state,\n /*.d_inner =*/ d_inner,\n /*.n_head =*/ n_head,\n /*.n_group =*/ n_group,\n /*.n_seq_tokens =*/ n_seq_tokens,\n /*.n_seqs =*/ n_seqs,\n /*.s_off =*/ ggml_nelements(op->src[1]) * sizeof(float),\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.ns12 =*/ nb12/nb10,\n /*.nb13 =*/ nb13,\n /*.nb20 =*/ nb20,\n /*.nb21 =*/ nb21,\n /*.ns21 =*/ nb21/nb20,\n /*.nb22 =*/ nb22,\n /*.ne30 =*/ ne30,\n /*.nb31 =*/ nb31,\n /*.nb41 =*/ nb41,\n /*.nb42 =*/ nb42,\n /*.ns42 =*/ nb42/nb40,\n /*.nb43 =*/ nb43,\n /*.nb51 =*/ nb51,\n /*.nb52 =*/ nb52,\n /*.ns52 =*/ nb52/nb50,\n /*.nb53 =*/ nb53,\n /*.nb0 =*/ nb0,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_ssm_scan(lib, op);\n\n GGML_ASSERT(d_state <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), 3);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), 4);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), 5);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), 6);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), 7);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 8);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, d_inner, n_head, n_seqs, d_state, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_rwkv(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const int64_t B = op->op == GGML_OP_RWKV_WKV6 ? op->src[5]->ne[1] : op->src[6]->ne[1];\n const int64_t T = op->src[0]->ne[2];\n const int64_t C = op->ne[0];\n const int64_t H = op->src[0]->ne[1];\n\n auto pipeline = ggml_metal_library_get_pipeline_rwkv(lib, op);\n\n int ida = 0;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++);\n if (op->op == GGML_OP_RWKV_WKV7) {\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), ida++);\n }\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++);\n ggml_metal_encoder_set_bytes (enc, (void *) &B, sizeof(B), ida++);\n ggml_metal_encoder_set_bytes (enc, (void *) &T, sizeof(T), ida++);\n ggml_metal_encoder_set_bytes (enc, (void *) &C, sizeof(C), ida++);\n ggml_metal_encoder_set_bytes (enc, (void *) &H, sizeof(H), ida++);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, B * H, 1, 1, C/H, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op);\n\n int ida = 0;\n\n ggml_metal_kargs_gated_delta_net args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne10 =*/ ne10,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.ne13 =*/ ne13,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ne20 =*/ ne20,\n /*.ne21 =*/ ne21,\n /*.ne22 =*/ ne22,\n /*.ne23 =*/ ne23,\n /*.nb20 =*/ nb20,\n /*.nb21 =*/ nb21,\n /*.nb22 =*/ nb22,\n /*.nb23 =*/ nb23,\n /*.ns02 =*/ (int32_t) (nb02/sizeof(float)),\n /*.ns12 =*/ (int32_t) (nb12/sizeof(float)),\n /*.ns22 =*/ (int32_t) (nb22/sizeof(float)),\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); // q\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); // k\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); // v\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); // gate\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // beta\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst\n\n const int nsg = pipeline.nsg;\n\n ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1);\n\n return 1;\n}\n\nint ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n ggml_metal_kargs_solve_tri args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne10 =*/ ne10,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.ne13 =*/ ne13,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_solve_tri(lib, op);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n const int nsg = pipeline.nsg;\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, (ne10 + nsg - 1)/nsg, ne02, ne03, 32, nsg, 1);\n\n return 1;\n}\n\nint ggml_metal_op_set(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_src1 = ggml_metal_get_buffer_id(op->src[1]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n const size_t pnb1 = ((const int32_t *) op->op_params)[0];\n const size_t pnb2 = ((const int32_t *) op->op_params)[1];\n const size_t pnb3 = ((const int32_t *) op->op_params)[2];\n const size_t offs = ((const int32_t *) op->op_params)[3];\n\n const bool inplace = (bool) ((const int32_t *) op->op_params)[4];\n\n if (!inplace) {\n // run a separate kernel to cpy src->dst\n // not sure how to avoid this\n // TODO: make a simpler cpy_bytes kernel\n\n //const id pipeline = ctx->pipelines[GGML_METAL_PIPELINE_TYPE_CPY_F32_F32].obj;\n auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[0]->type, op->type);\n\n ggml_metal_kargs_cpy args = {\n /*.nk0 =*/ ne00,\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n\n const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n\n ggml_metal_op_concurrency_reset(ctx);\n }\n\n auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[1]->type, op->type);\n\n GGML_ASSERT(ne10 % ggml_blck_size(op->src[1]->type) == 0);\n\n int64_t nk0 = ne10;\n if (ggml_is_quantized(op->src[1]->type)) {\n nk0 = ne10/16;\n } else if (ggml_is_quantized(op->type)) {\n nk0 = ne10/ggml_blck_size(op->type);\n }\n\n int nth = std::min(nk0, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n // when rows are small, we can batch them together in a single threadgroup\n int nrptg = 1;\n\n // TODO: relax this constraint in the future\n if (ggml_blck_size(op->src[1]->type) == 1 && ggml_blck_size(op->type) == 1) {\n if (nth > nk0) {\n nrptg = (nth + nk0 - 1)/nk0;\n nth = nk0;\n\n if (nrptg*nth > ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nrptg--;\n }\n }\n }\n\n nth = std::min(nth, nk0);\n\n ggml_metal_kargs_cpy args = {\n /*.nk0 =*/ nk0,\n /*.ne00 =*/ ne10,\n /*.ne01 =*/ ne11,\n /*.ne02 =*/ ne12,\n /*.ne03 =*/ ne13,\n /*.nb00 =*/ nb10,\n /*.nb01 =*/ nb11,\n /*.nb02 =*/ nb12,\n /*.nb03 =*/ nb13,\n /*.ne0 =*/ ne10,\n /*.ne1 =*/ ne11,\n /*.ne2 =*/ ne12,\n /*.ne3 =*/ ne13,\n /*.nb0 =*/ ggml_element_size(op),\n /*.nb1 =*/ pnb1,\n /*.nb2 =*/ pnb2,\n /*.nb3 =*/ pnb3,\n };\n\n const int nw0 = nrptg == 1 ? (nk0 + nth - 1)/nth : 1;\n\n bid_dst.offs += offs;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src1, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nw0*(ne11 + nrptg - 1)/nrptg, ne12, ne13, nth, nrptg, 1);\n\n return 1;\n}\n\nint ggml_metal_op_cpy(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline = ggml_metal_library_get_pipeline_cpy(lib, op->src[0]->type, op->type);\n\n GGML_ASSERT(ne00 % ggml_blck_size(op->src[0]->type) == 0);\n\n int64_t nk0 = ne00;\n if (ggml_is_quantized(op->src[0]->type)) {\n nk0 = ne00/16;\n } else if (ggml_is_quantized(op->type)) {\n nk0 = ne00/ggml_blck_size(op->type);\n }\n\n int nth = std::min(nk0, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n // when rows are small, we can batch them together in a single threadgroup\n int nrptg = 1;\n\n // TODO: relax this constraint in the future\n if (ggml_blck_size(op->src[0]->type) == 1 && ggml_blck_size(op->type) == 1) {\n if (nth > nk0) {\n nrptg = (nth + nk0 - 1)/nk0;\n nth = nk0;\n\n if (nrptg*nth > ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nrptg--;\n }\n }\n }\n\n nth = std::min(nth, nk0);\n\n ggml_metal_kargs_cpy args = {\n /*.nk0 =*/ nk0,\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n const int nw0 = nrptg == 1 ? (nk0 + nth - 1)/nth : 1;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nw0*(ne01 + nrptg - 1)/nrptg, ne02, ne03, nth, nrptg, 1);\n\n return 1;\n}\n\nint ggml_metal_op_pool_1d(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const int32_t * opts = op->op_params;\n ggml_op_pool op_pool = (ggml_op_pool) opts[0];\n\n const int32_t k0 = opts[1];\n const int32_t s0 = opts[2];\n const int32_t p0 = opts[3];\n\n const int64_t IW = op->src[0]->ne[0];\n const int64_t OW = op->ne[0];\n\n const int64_t np = ggml_nelements(op);\n\n ggml_metal_kargs_pool_1d args_pool_1d = {\n /* .k0 = */ k0,\n /* .s0 = */ s0,\n /* .p0 = */ p0,\n /* .IW = */ IW,\n /* .OW = */ OW,\n /* .np = */ np\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_pool_1d(lib, op, op_pool);\n\n const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np);\n const int ntg = (np + nth - 1) / nth;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args_pool_1d, sizeof(args_pool_1d), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\n\nint ggml_metal_op_pool_2d(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const int32_t * opts = op->op_params;\n ggml_op_pool op_pool = (ggml_op_pool) opts[0];\n\n const int32_t k0 = opts[1];\n const int32_t k1 = opts[2];\n const int32_t s0 = opts[3];\n const int32_t s1 = opts[4];\n const int32_t p0 = opts[5];\n const int32_t p1 = opts[6];\n\n const int64_t IH = op->src[0]->ne[1];\n const int64_t IW = op->src[0]->ne[0];\n\n const int64_t N = op->ne[3];\n const int64_t OC = op->ne[2];\n const int64_t OH = op->ne[1];\n const int64_t OW = op->ne[0];\n\n const int64_t np = N * OC * OH * OW;\n\n ggml_metal_kargs_pool_2d args_pool_2d = {\n /* .k0 = */ k0,\n /* .k1 = */ k1,\n /* .s0 = */ s0,\n /* .s1 = */ s1,\n /* .p0 = */ p0,\n /* .p1 = */ p1,\n /* .IH = */ IH,\n /* .IW = */ IW,\n /* .OH = */ OH,\n /* .OW = */ OW,\n /* .np = */ np\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_pool_2d(lib, op, op_pool);\n\n const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np);\n const int ntg = (np + nth - 1) / nth;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args_pool_2d, sizeof(args_pool_2d), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n GGML_ASSERT(ne00 == ne10);\n\n GGML_ASSERT(ne12 % ne02 == 0);\n GGML_ASSERT(ne13 % ne03 == 0);\n\n const int16_t r2 = ne12/ne02;\n const int16_t r3 = ne13/ne03;\n\n // find the break-even point where the matrix-matrix kernel becomes more efficient compared\n // to the matrix-vector kernel\n const int ne11_mm_min = 8;\n\n // first try to use small-batch mat-mv kernels\n // these should be efficient for BS [2, ~8]\n if (op->src[1]->type == GGML_TYPE_F32 && (ne00%128 == 0) &&\n (\n (\n (\n op->src[0]->type == GGML_TYPE_F32 || // TODO: helper function\n op->src[0]->type == GGML_TYPE_F16 ||\n op->src[0]->type == GGML_TYPE_BF16 ||\n op->src[0]->type == GGML_TYPE_Q4_0 ||\n op->src[0]->type == GGML_TYPE_Q4_1 ||\n op->src[0]->type == GGML_TYPE_Q5_0 ||\n op->src[0]->type == GGML_TYPE_Q5_1 ||\n op->src[0]->type == GGML_TYPE_Q8_0 ||\n op->src[0]->type == GGML_TYPE_MXFP4 ||\n op->src[0]->type == GGML_TYPE_IQ4_NL ||\n false) && (ne11 >= 2 && ne11 <= 8)\n ) ||\n (\n (\n op->src[0]->type == GGML_TYPE_Q4_K ||\n op->src[0]->type == GGML_TYPE_Q5_K ||\n op->src[0]->type == GGML_TYPE_Q6_K ||\n op->src[0]->type == GGML_TYPE_Q2_K ||\n op->src[0]->type == GGML_TYPE_Q3_K ||\n false) && (ne11 >= 4 && ne11 <= 8)\n )\n )\n ) {\n // TODO: determine the optimal parameters based on grid utilization\n // I still don't know why we should not always use the maximum available threads:\n //\n // nsg = pipeline.maxTotalThreadsPerThreadgroup / 32\n //\n // my current hypothesis is that the work grid is not evenly divisible for different nsg\n // values and there can be some tail effects when nsg is high. need to confirm this\n //\n const int nsg = 2; // num simdgroups per threadgroup\n\n // num threads along row per simdgroup\n int16_t nxpsg = 0;\n if (ne00 % 256 == 0 && ne11 < 3) {\n nxpsg = 16;\n } else if (ne00 % 128 == 0) {\n nxpsg = 8;\n } else {\n nxpsg = 4;\n }\n\n const int16_t nypsg = 32/nxpsg; // num threads along col per simdgroup (i.e. a simdgroup processes that many src0 rows at a time)\n const int16_t r0ptg = nypsg*nsg; // num src0 rows per threadgroup\n int16_t r1ptg = 4; // num src1 rows per threadgroup\n\n // note: not sure how optimal are those across all different hardware. there might be something cleverer\n switch (ne11) {\n case 2:\n r1ptg = 2; break;\n case 3:\n case 6:\n r1ptg = 3; break;\n case 4:\n case 7:\n case 8:\n r1ptg = 4; break;\n case 5:\n r1ptg = 5; break;\n default:\n GGML_ABORT(\"unsupported ne11\");\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_mul_mv_ext(lib, op->src[0]->type, op->src[1]->type, nsg, nxpsg, r1ptg);\n\n ggml_metal_kargs_mul_mv_ext args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne10 =*/ ne10,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.r2 =*/ r2,\n /*.r3 =*/ r3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + r0ptg - 1)/r0ptg), ((ne11 + r1ptg - 1)/r1ptg), ne12*ne13, 32, nsg, 1);\n } else if (\n !ggml_is_transposed(op->src[0]) &&\n !ggml_is_transposed(op->src[1]) &&\n // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs\n // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel\n props_dev->has_simdgroup_mm && ne00 >= 64 && ne11 > ne11_mm_min) {\n //GGML_LOG_INFO(\"matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\\n\", ne00, ne01, ne02, ne11, ne12);\n\n // some Metal matrix data types require aligned pointers\n // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5)\n //switch (op->src[0]->type) {\n // case GGML_TYPE_F32: GGML_ASSERT(nb01 % 16 == 0); break;\n // case GGML_TYPE_F16: GGML_ASSERT(nb01 % 8 == 0); break;\n // case GGML_TYPE_BF16: GGML_ASSERT(nb01 % 8 == 0); break;\n // default: break;\n //}\n\n auto pipeline = ggml_metal_library_get_pipeline_mul_mm(lib, op);\n\n ggml_metal_kargs_mul_mm args = {\n /*.ne00 =*/ ne00,\n /*.ne02 =*/ ne02,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne12 =*/ ne12,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.r2 =*/ r2,\n /*.r3 =*/ r3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n ggml_metal_encoder_dispatch_threadgroups(enc, ((ne11 + 31)/32), ((ne01 + 63)/64), ne12*ne13, 128, 1, 1);\n } else {\n auto pipeline = ggml_metal_library_get_pipeline_mul_mv(lib, op);\n\n const int nr0 = pipeline.nr0;\n const int nr1 = pipeline.nr1;\n const int nsg = pipeline.nsg;\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_kargs_mul_mv args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne10 =*/ ne10,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.nr0 =*/ nr0,\n /*.r2 =*/ r2,\n /*.r3 =*/ r3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n if (op->src[0]->type == GGML_TYPE_F32 ||\n op->src[0]->type == GGML_TYPE_F16 ||\n op->src[0]->type == GGML_TYPE_BF16 ||\n op->src[0]->type == GGML_TYPE_Q8_0) {\n ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + nr0 - 1)/(nr0)), ((ne11 + nr1 - 1)/nr1), ne12*ne13, 32, nsg, 1);\n } else {\n ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + nr0*nsg - 1)/(nr0*nsg)), ((ne11 + nr1 - 1)/nr1), ne12*ne13, 32, nsg, 1);\n }\n }\n\n return 1;\n}\n\nsize_t ggml_metal_op_mul_mat_id_extra_tpe(const ggml_tensor * op) {\n assert(op->op == GGML_OP_MUL_MAT_ID);\n\n const int64_t ne02 = op->src[0]->ne[2]; // n_expert\n\n return ggml_type_size(GGML_TYPE_I32)*ne02;\n}\n\nsize_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) {\n assert(op->op == GGML_OP_MUL_MAT_ID);\n\n const int64_t ne02 = op->src[0]->ne[2]; // n_expert\n const int64_t ne21 = op->src[2]->ne[1]; // n_token\n\n return ggml_type_size(GGML_TYPE_I32)*ne02*ne21;\n}\n\nint ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n // src2 = ids\n GGML_ASSERT(op->src[2]->type == GGML_TYPE_I32);\n\n GGML_ASSERT(!ggml_is_transposed(op->src[0]));\n GGML_ASSERT(!ggml_is_transposed(op->src[1]));\n\n GGML_ASSERT(ne03 == 1);\n GGML_ASSERT(ne13 == 1);\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_src1 = ggml_metal_get_buffer_id(op->src[1]);\n ggml_metal_buffer_id bid_src2 = ggml_metal_get_buffer_id(op->src[2]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n const uint32_t r2 = 1;\n const uint32_t r3 = 1;\n\n // find the break-even point where the matrix-matrix kernel becomes more efficient compared\n // to the matrix-vector kernel\n // ne20 = n_used_experts\n // ne21 = n_rows (batch size)\n const int ne21_mm_id_min = 32;\n\n if (props_dev->has_simdgroup_mm && ne00 >= 64 && (ne21 >= ne21_mm_id_min)) {\n // some Metal matrix data types require aligned pointers\n // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5)\n //switch (op->src[0]->type) {\n // case GGML_TYPE_F32: GGML_ASSERT(nb01 % 16 == 0); break;\n // case GGML_TYPE_F16: GGML_ASSERT(nb01 % 8 == 0); break;\n // case GGML_TYPE_BF16: GGML_ASSERT(nb01 % 8 == 0); break;\n // default: break;\n //}\n\n // extra buffers for intermediate id mapping\n ggml_metal_buffer_id bid_tpe = bid_dst;\n bid_tpe.offs += ggml_nbytes(op);\n\n ggml_metal_buffer_id bid_ids = bid_tpe;\n bid_ids.offs += ggml_metal_op_mul_mat_id_extra_tpe(op);\n\n {\n ggml_metal_kargs_mul_mm_id_map0 args = {\n ne02,\n ne10,\n ne11, // n_expert_used (bcast)\n nb11,\n nb12,\n ne21, // n_tokens\n ne20, // n_expert_used\n nb21,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_map0(lib, ne02, ne20);\n\n const size_t smem = pipeline.smem;\n\n GGML_ASSERT(ne02 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src2, 1);\n ggml_metal_encoder_set_buffer (enc, bid_tpe, 2);\n ggml_metal_encoder_set_buffer (enc, bid_ids, 3);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, ne02, 1, 1);\n }\n\n // this barrier is always needed because the next kernel has to wait for the id maps to be computed\n ggml_metal_op_concurrency_reset(ctx);\n\n {\n auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id(lib, op);\n\n ggml_metal_kargs_mul_mm_id args = {\n /*.ne00 =*/ ne00,\n /*.ne02 =*/ ne02,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne11 =*/ ne11, // n_expert_used (bcast)\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ne20 =*/ ne20, // n_expert_used\n /*.ne21 =*/ ne21, // n_tokens\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.r2 =*/ r2,\n /*.r3 =*/ r3,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_src1, 2);\n ggml_metal_encoder_set_buffer (enc, bid_tpe, 3);\n ggml_metal_encoder_set_buffer (enc, bid_ids, 4);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 5);\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, (ne21 + 31)/32, (ne01 + 63)/64, ne02, 128, 1, 1);\n }\n } else {\n auto pipeline = ggml_metal_library_get_pipeline_mul_mv_id(lib, op);\n\n const int nr0 = pipeline.nr0;\n const int nr1 = pipeline.nr1;\n const int nsg = pipeline.nsg;\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_kargs_mul_mv_id args = {\n /*.nei0 =*/ ne20,\n /*.nei1 =*/ ne21,\n /*.nbi1 =*/ nb21,\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.ne10 =*/ ne10,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.ne13 =*/ ne13,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.nb1 =*/ nb1,\n /*.nr0 =*/ nr0,\n };\n\n if (ggml_is_quantized(op->src[0]->type)) {\n GGML_ASSERT(ne00 >= nsg*nr0);\n }\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer(enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer(enc, bid_src1, 2);\n ggml_metal_encoder_set_buffer(enc, bid_dst, 3);\n ggml_metal_encoder_set_buffer(enc, bid_src2, 4);\n\n const int64_t _ne1 = 1;\n const int64_t ne123 = ne20*ne21;\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n if (op->src[0]->type == GGML_TYPE_F32 ||\n op->src[0]->type == GGML_TYPE_F16 ||\n op->src[0]->type == GGML_TYPE_BF16 ||\n op->src[0]->type == GGML_TYPE_Q8_0) {\n ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nr0 - 1)/(nr0), (_ne1 + nr1 - 1)/nr1, ne123, 32, nsg, 1);\n } else {\n ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nr0*nsg - 1)/(nr0*nsg), (_ne1 + nr1 - 1)/nr1, ne123, 32, nsg, 1);\n }\n }\n\n return 1;\n}\n\nint ggml_metal_op_add_id(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[2]->type == GGML_TYPE_I32);\n GGML_ASSERT(op->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n\n ggml_metal_kargs_add_id args = {\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb11 =*/ nb11,\n /*.nb21 =*/ nb21,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_base(lib, GGML_OP_ADD_ID);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), 3);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 4);\n\n const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, 1, nth, 1, 1);\n\n return 1;\n}\n\nbool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) {\n assert(op->op == GGML_OP_FLASH_ATTN_EXT);\n\n const int64_t ne00 = op->src[0]->ne[0]; // head size\n const int64_t ne01 = op->src[0]->ne[1]; // batch size\n\n // use vec kernel if the batch size is small and if the head size is supported\n return (ne01 < 20) && (ne00 % 32 == 0);\n}\n\nsize_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) {\n assert(op->op == GGML_OP_FLASH_ATTN_EXT);\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb);\n GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb3, op->src[3], nb);\n\n size_t res = 0;\n\n const bool has_mask = op->src[3] != nullptr;\n\n // note: the non-vec kernel requires more extra memory, so always reserve for it\n GGML_ASSERT(OP_FLASH_ATTN_EXT_NCPSG >= OP_FLASH_ATTN_EXT_VEC_NCPSG);\n\n //if (ggml_metal_op_flash_attn_ext_use_vec(op)) {\n if (false) {\n // note: always reserve the padding space to avoid graph reallocations\n //const bool has_kvpad = ne11 % OP_FLASH_ATTN_EXT_VEC_NCPSG != 0;\n const bool has_kvpad = true;\n\n if (has_kvpad) {\n res += OP_FLASH_ATTN_EXT_VEC_NCPSG*(\n nb11*ne12*ne13 +\n nb21*ne22*ne23 +\n (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0));\n }\n } else {\n //const bool has_kvpad = ne11 % OP_FLASH_ATTN_EXT_NCPSG != 0;\n const bool has_kvpad = true;\n\n if (has_kvpad) {\n res += OP_FLASH_ATTN_EXT_NCPSG*(\n nb11*ne12*ne13 +\n nb21*ne22*ne23 +\n (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0));\n }\n }\n\n return res;\n}\n\nsize_t ggml_metal_op_flash_attn_ext_extra_blk(const ggml_tensor * op) {\n assert(op->op == GGML_OP_FLASH_ATTN_EXT);\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n //GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n //GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n //GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n //GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);\n //GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb);\n GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb3, op->src[3], nb);\n\n size_t res = 0;\n\n const bool has_mask = op->src[3] != nullptr;\n\n if (!has_mask) {\n return res;\n }\n\n const bool is_vec = ggml_metal_op_flash_attn_ext_use_vec(op);\n\n // this optimization is not useful for the vector kernels\n // note: always reserve the blk buffer to avoid graph reallocations\n //if (is_vec) {\n // return res;\n //}\n\n const int nqptg = is_vec ? OP_FLASH_ATTN_EXT_VEC_NQPSG : OP_FLASH_ATTN_EXT_NQPSG;\n const int ncpsg = is_vec ? OP_FLASH_ATTN_EXT_VEC_NCPSG : OP_FLASH_ATTN_EXT_NCPSG;\n\n const int64_t ne1 = (ne01 + nqptg - 1)/nqptg;\n const int64_t ne0 = (ne30 + ncpsg - 1)/ncpsg;\n\n res += GGML_PAD(ggml_type_size(GGML_TYPE_I8)*ne0*ne1*ne32*ne33, 32);\n\n return res;\n}\n\nsize_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) {\n assert(op->op == GGML_OP_FLASH_ATTN_EXT);\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n //GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n //GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb);\n //GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne);\n //GGML_TENSOR_LOCALS(uint64_t, nb3, op->src[3], nb);\n\n size_t res = 0;\n\n // note: always reserve the temp buffer to avoid graph reallocations\n //if (ggml_metal_op_flash_attn_ext_use_vec(op)) {\n if (true) {\n const int64_t nwg = 32;\n const int64_t ne01_max = std::min(ne01, 32);\n\n // temp buffer for writing the results from each workgroup\n // - ne20: the size of the Value head\n // - + 2: the S and M values for each intermediate result\n res += ggml_type_size(GGML_TYPE_F32)*(ne01_max*ne02*ne03*nwg*(ne20 + 2));\n }\n\n return res;\n}\n\nint ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb2, op->src[2], nb);\n GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb3, op->src[3], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS( int32_t, nb, op, nb);\n\n GGML_ASSERT(ne00 % 4 == 0);\n\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[1]->type == op->src[2]->type);\n\n //GGML_ASSERT(ggml_are_same_shape (src1, src2));\n GGML_ASSERT(ne11 == ne21);\n GGML_ASSERT(ne12 == ne22);\n\n GGML_ASSERT(!op->src[3] || op->src[3]->type == GGML_TYPE_F16);\n GGML_ASSERT(!op->src[3] || op->src[3]->ne[1] >= op->src[0]->ne[1] &&\n \"the Flash-Attention Metal kernel requires the mask to be at least n_queries big\");\n\n float scale;\n float max_bias;\n float logit_softcap;\n\n memcpy(&scale, ((const int32_t *) op->op_params) + 0, sizeof(scale));\n memcpy(&max_bias, ((const int32_t *) op->op_params) + 1, sizeof(max_bias));\n memcpy(&logit_softcap, ((const int32_t *) op->op_params) + 2, sizeof(logit_softcap));\n\n if (logit_softcap != 0.0f) {\n scale /= logit_softcap;\n }\n\n const bool has_mask = op->src[3] != NULL;\n const bool has_sinks = op->src[4] != NULL;\n const bool has_bias = max_bias != 0.0f;\n const bool has_scap = logit_softcap != 0.0f;\n\n const uint32_t n_head = op->src[0]->ne[2];\n const int32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));\n\n const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n GGML_ASSERT(ne01 < 65536);\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_src1 = ggml_metal_get_buffer_id(op->src[1]);\n ggml_metal_buffer_id bid_src2 = ggml_metal_get_buffer_id(op->src[2]);\n ggml_metal_buffer_id bid_src3 = has_mask ? ggml_metal_get_buffer_id(op->src[3]) : bid_src0;\n ggml_metal_buffer_id bid_src4 = has_sinks ? ggml_metal_get_buffer_id(op->src[4]) : bid_src0;\n\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n ggml_metal_buffer_id bid_pad = bid_dst;\n bid_pad.offs += ggml_nbytes(op);\n\n ggml_metal_buffer_id bid_blk = bid_pad;\n bid_blk.offs += ggml_metal_op_flash_attn_ext_extra_pad(op);\n\n ggml_metal_buffer_id bid_tmp = bid_blk;\n bid_tmp.offs += ggml_metal_op_flash_attn_ext_extra_blk(op);\n\n if (!ggml_metal_op_flash_attn_ext_use_vec(op)) {\n // half8x8 kernel\n const int nqptg = OP_FLASH_ATTN_EXT_NQPSG; // queries per threadgroup\n const int ncpsg = OP_FLASH_ATTN_EXT_NCPSG; // cache values per simdgroup\n\n GGML_ASSERT(nqptg <= 32);\n GGML_ASSERT(nqptg % 8 == 0);\n GGML_ASSERT(ncpsg % 32 == 0);\n\n bool need_sync = false;\n\n const bool has_kvpad = ne11 % ncpsg != 0;\n\n if (has_kvpad) {\n assert(ggml_metal_op_flash_attn_ext_extra_pad(op) != 0);\n\n ggml_metal_kargs_flash_attn_ext_pad args0 = {\n /*.ne11 =*/ne11,\n /*.ne_12_2 =*/ne12,\n /*.ne_12_3 =*/ne13,\n /*.nb11 =*/nb11,\n /*.nb12 =*/nb12,\n /*.nb13 =*/nb13,\n /*.nb21 =*/nb21,\n /*.nb22 =*/nb22,\n /*.nb23 =*/nb23,\n /*.ne31 =*/ne31,\n /*.ne32 =*/ne32,\n /*.ne33 =*/ne33,\n /*.nb31 =*/nb31,\n /*.nb32 =*/nb32,\n /*.nb33 =*/nb33,\n };\n\n auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_pad(lib, op, has_mask, ncpsg);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline0);\n ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src1, 1);\n ggml_metal_encoder_set_buffer (enc, bid_src2, 2);\n ggml_metal_encoder_set_buffer (enc, bid_src3, 3);\n ggml_metal_encoder_set_buffer (enc, bid_pad, 4);\n\n assert(ne12 == ne22);\n assert(ne13 == ne23);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ncpsg, std::max(ne12, ne32), std::max(ne13, ne33), 32, 1, 1);\n\n need_sync = true;\n }\n\n if (has_mask) {\n assert(ggml_metal_op_flash_attn_ext_extra_blk(op) != 0);\n\n ggml_metal_kargs_flash_attn_ext_blk args0 = {\n /*.ne01 =*/ ne01,\n /*.ne30 =*/ ne30,\n /*.ne31 =*/ ne31,\n /*.ne32 =*/ ne32,\n /*.ne33 =*/ ne33,\n /*.nb31 =*/ nb31,\n /*.nb32 =*/ nb32,\n /*.nb33 =*/ nb33,\n };\n\n auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_blk(lib, op, nqptg, ncpsg);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline0);\n ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src3, 1);\n ggml_metal_encoder_set_buffer (enc, bid_blk, 2);\n\n const int32_t nblk1 = ((ne01 + nqptg - 1)/nqptg);\n const int32_t nblk0 = ((ne30 + ncpsg - 1)/ncpsg);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nblk0, nblk1, ne32*ne33, 32, 1, 1);\n\n need_sync = true;\n }\n\n if (need_sync) {\n ggml_metal_op_concurrency_reset(ctx);\n }\n\n const int is_q = ggml_is_quantized(op->src[1]->type) ? 1 : 0;\n\n // 2*(2*ncpsg)\n // ncpsg soft_max values + ncpsg mask values\n //\n // 16*32*(nsg)\n // the shared memory needed for the simdgroups to load the KV cache\n // each thread loads (dequantizes) 16 head elements, there are 32 threads in th SG\n //\n#define FATTN_SMEM(nsg) (GGML_PAD((nqptg*(ne00 + 2*GGML_PAD(ne20, 64) + 2*(2*ncpsg)) + is_q*(16*32*(nsg)))*(sizeof(float)/2), 16))\n\n //int64_t nsgmax = 4;\n //\n //if (is_q) {\n // nsgmax = 2;\n // while (true) {\n // const size_t smem = FATTN_SMEM(nsgmax);\n // if (smem > props_dev->max_theadgroup_memory_size) {\n // break;\n // }\n // nsgmax *= 2;\n // }\n // nsgmax /= 2;\n //}\n\n // simdgroups per threadgroup (a.k.a. warps)\n //nsg = ne01 <= nqptg ? MAX(4, MIN(nsgmax, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32))) : 4;\n int32_t nsg = ne00 >= 512 ? 8 : 4;\n\n const size_t smem = FATTN_SMEM(nsg);\n\n ggml_metal_kargs_flash_attn_ext args = {\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne11 =*/ ne11,\n /*.ne_12_2 =*/ ne12,\n /*.ne_12_3 =*/ ne13,\n /*.ns10 =*/ int32_t(nb11/nb10),\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ns20 =*/ int32_t(nb21/nb20),\n /*.nb21 =*/ nb21,\n /*.nb22 =*/ nb22,\n /*.nb23 =*/ nb23,\n /*.ne31 =*/ ne31,\n /*.ne32 =*/ ne32,\n /*.ne33 =*/ ne33,\n /*.nb31 =*/ nb31,\n /*.nb32 =*/ nb32,\n /*.nb33 =*/ nb33,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.scale =*/ scale,\n /*.max_bias =*/ max_bias,\n /*.m0 =*/ m0,\n /*.m1 =*/ m1,\n /*.n_head_log2 =*/ n_head_log2,\n /*.logit_softcap =*/ logit_softcap,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_src1, 2);\n ggml_metal_encoder_set_buffer (enc, bid_src2, 3);\n ggml_metal_encoder_set_buffer (enc, bid_src3, 4);\n ggml_metal_encoder_set_buffer (enc, bid_src4, 5);\n ggml_metal_encoder_set_buffer (enc, bid_pad, 6);\n ggml_metal_encoder_set_buffer (enc, bid_blk, 7);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 8);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nqptg - 1)/nqptg, ne02, ne03, 32, nsg, 1);\n#undef FATTN_SMEM\n } else {\n // half4x4 kernel\n const int nqptg = OP_FLASH_ATTN_EXT_VEC_NQPSG; // queries per threadgroup\n const int ncpsg = OP_FLASH_ATTN_EXT_VEC_NCPSG; // cache values per simdgroup !! sync with kernel template arguments !!\n const int nhptg = 1; // heads per threadgroup\n\n GGML_ASSERT(nqptg <= 32);\n GGML_ASSERT(nqptg % 1 == 0);\n GGML_ASSERT(ncpsg % 32 == 0);\n\n bool need_sync = false;\n\n const bool has_kvpad = ne11 % ncpsg != 0;\n\n if (has_kvpad) {\n assert(ggml_metal_op_flash_attn_ext_extra_pad(op) != 0);\n\n ggml_metal_kargs_flash_attn_ext_pad args0 = {\n /*.ne11 =*/ne11,\n /*.ne_12_2 =*/ne12,\n /*.ne_12_3 =*/ne13,\n /*.nb11 =*/nb11,\n /*.nb12 =*/nb12,\n /*.nb13 =*/nb13,\n /*.nb21 =*/nb21,\n /*.nb22 =*/nb22,\n /*.nb23 =*/nb23,\n /*.ne31 =*/ne31,\n /*.ne32 =*/ne32,\n /*.ne33 =*/ne33,\n /*.nb31 =*/nb31,\n /*.nb32 =*/nb32,\n /*.nb33 =*/nb33,\n };\n\n auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_pad(lib, op, has_mask, ncpsg);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline0);\n ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src1, 1);\n ggml_metal_encoder_set_buffer (enc, bid_src2, 2);\n ggml_metal_encoder_set_buffer (enc, bid_src3, 3);\n ggml_metal_encoder_set_buffer (enc, bid_pad, 4);\n\n assert(ne12 == ne22);\n assert(ne13 == ne23);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ncpsg, std::max(ne12, ne32), std::max(ne13, ne33), 32, 1, 1);\n\n need_sync = true;\n }\n\n if (need_sync) {\n ggml_metal_op_concurrency_reset(ctx);\n }\n\n // note: for simplicity assume the K is larger or equal than V\n GGML_ASSERT(ne10 >= ne20);\n\n // ne00 + 2*ncpsg*(nsg)\n // for each query, we load it as f16 in shared memory (ne00)\n // and store the soft_max values and the mask\n //\n // ne20*(nsg)\n // each simdgroup has a full f32 head vector in shared mem to accumulate results\n //\n#define FATTN_SMEM(nsg) (GGML_PAD(((GGML_PAD(ne00, 128) + 4*ncpsg + 2*GGML_PAD(ne20, 128))*(nsg))*(sizeof(float)/2), 16))\n\n int64_t nsg = 1;\n\n // workgroups\n // each workgroup handles nsg*nkpsg cache values\n int32_t nwg = 1;\n if (false) {\n // for small KV caches, we could launch a single workgroup and write the results directly to dst/\n // however, this does not lead to significant improvement, so disabled\n nwg = 1;\n nsg = 4;\n } else {\n nwg = 32;\n nsg = 1;\n while (2*nwg*nsg*ncpsg < ne11 && nsg < 4) {\n nsg *= 2;\n }\n }\n\n ggml_metal_kargs_flash_attn_ext_vec args = {\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne11 =*/ ne11,\n /*.ne_12_2 =*/ ne12,\n /*.ne_12_3 =*/ ne13,\n /*.ns10 =*/ int32_t(nb11/nb10),\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ns20 =*/ int32_t(nb21/nb20),\n /*.nb21 =*/ nb21,\n /*.nb22 =*/ nb22,\n /*.nb23 =*/ nb23,\n /*.ne31 =*/ ne31,\n /*.ne32 =*/ ne32,\n /*.ne33 =*/ ne33,\n /*.nb31 =*/ nb31,\n /*.nb32 =*/ nb32,\n /*.nb33 =*/ nb33,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.scale =*/ scale,\n /*.max_bias =*/ max_bias,\n /*.m0 =*/ m0,\n /*.m1 =*/ m1,\n /*.n_head_log2 =*/ n_head_log2,\n /*.logit_softcap =*/ logit_softcap,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg);\n\n GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_src1, 2);\n ggml_metal_encoder_set_buffer (enc, bid_src2, 3);\n ggml_metal_encoder_set_buffer (enc, bid_src3, 4);\n ggml_metal_encoder_set_buffer (enc, bid_src4, 5);\n\n const size_t smem = FATTN_SMEM(nsg);\n\n //printf(\"smem: %zu, max: %zu, nsg = %d, nsgmax = %d\\n\", smem, props_dev->max_theadgroup_memory_size, (int) nsg, (int) nsgmax);\n GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);\n\n if (nwg == 1) {\n assert(ggml_metal_op_flash_attn_ext_extra_tmp(op) == 0);\n\n // using 1 workgroup -> write the result directly into dst\n ggml_metal_encoder_set_buffer(enc, bid_pad, 6);\n ggml_metal_encoder_set_buffer(enc, bid_dst, 7);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nqptg - 1)/nqptg, (ne02 + nhptg - 1)/nhptg, ne03*nwg, 32, nsg, 1);\n } else {\n // sanity checks\n assert(ggml_metal_op_flash_attn_ext_extra_tmp(op) != 0);\n\n GGML_ASSERT(ne01*ne02*ne03 == ne1*ne2*ne3);\n GGML_ASSERT((uint64_t)ne1*ne2*ne3 <= (1u << 31));\n\n // write the results from each workgroup into a temp buffer\n ggml_metal_encoder_set_buffer(enc, bid_pad, 6);\n ggml_metal_encoder_set_buffer(enc, bid_tmp, 7);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nqptg - 1)/nqptg, (ne02 + nhptg - 1)/nhptg, ne03*nwg, 32, nsg, 1);\n\n // sync the 2 kernels\n ggml_metal_op_concurrency_reset(ctx);\n\n // reduce the results from the workgroups\n {\n const int32_t nrows = ne1*ne2*ne3;\n\n ggml_metal_kargs_flash_attn_ext_vec_reduce args0 = {\n nrows,\n };\n\n auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce(lib, op, ne20, nwg);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline0);\n ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);\n ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, 32*nwg, 1, 1);\n }\n }\n#undef FATTN_SMEM\n }\n\n return 1;\n}\n\nint ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n const bool use_fusion = ctx->use_fusion;\n\n const int debug_fusion = ctx->debug_fusion;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[1]));\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_src1 = ggml_metal_get_buffer_id(op->src[1]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n ggml_metal_kargs_bin args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne10 =*/ ne10,\n /*.ne11 =*/ ne11,\n /*.ne12 =*/ ne12,\n /*.ne13 =*/ ne13,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.offs =*/ 0,\n /*.o1 =*/ { bid_src1.offs },\n };\n\n ggml_op fops[8];\n\n int n_fuse = 1;\n\n // c[0] = add(a, b[0])\n // c[1] = add(c[0], b[1])\n // c[2] = add(c[1], b[2])\n // ...\n if (use_fusion) {\n fops[0] = GGML_OP_ADD;\n fops[1] = GGML_OP_ADD;\n fops[2] = GGML_OP_ADD;\n fops[3] = GGML_OP_ADD;\n fops[4] = GGML_OP_ADD;\n fops[5] = GGML_OP_ADD;\n fops[6] = GGML_OP_ADD;\n fops[7] = GGML_OP_ADD;\n\n // note: in metal, we sometimes encode the graph in parallel so we have to avoid fusing ops\n // across splits. idx_end indicates the last node in the current split\n for (n_fuse = 0; n_fuse <= 6; ++n_fuse) {\n if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) {\n break;\n }\n\n ggml_tensor * f0 = ctx->node(idx + n_fuse);\n ggml_tensor * f1 = ctx->node(idx + n_fuse + 1);\n\n if (f0 != f1->src[0]) {\n break;\n }\n\n // b[0] === b[1] === ...\n if (!ggml_are_same_layout(f0->src[1], f1->src[1])) {\n break;\n }\n\n // only fuse ops if src1 is in the same Metal buffer\n ggml_metal_buffer_id bid_fuse = ggml_metal_get_buffer_id(f1->src[1]);\n if (bid_fuse.metal != bid_src1.metal) {\n break;\n }\n\n //ctx->fuse_cnt[ops[n_fuse + 1]->op]++;\n\n args.o1[n_fuse + 1] = bid_fuse.offs;\n }\n\n ++n_fuse;\n\n if (debug_fusion > 1 && n_fuse > 1) {\n GGML_LOG_DEBUG(\"%s: fuse: ADD x %d\\n\", __func__, n_fuse);\n }\n }\n\n // the offsets of src1 and all fused buffers are relative to the start of the src1 buffer\n bid_src1.offs = 0;\n\n struct ggml_metal_pipeline_with_params pipeline;\n\n pipeline = ggml_metal_library_get_pipeline_bin(lib, op, n_fuse);\n\n if (n_fuse > 1) {\n bid_dst = ggml_metal_get_buffer_id(ctx->node(idx + n_fuse - 1));\n\n for (int i = 1; i < n_fuse; ++i) {\n if (!ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) {\n ggml_metal_op_concurrency_reset(ctx);\n\n break;\n }\n }\n }\n\n if (pipeline.c4) {\n args.ne00 = ne00/4;\n args.ne10 = ne10/4;\n args.ne0 = ne0/4;\n }\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_src1, 2);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 3);\n\n if (pipeline.cnt) {\n ggml_metal_encoder_dispatch_threadgroups(enc, args.ne0, ggml_nrows(op), 1, 1, 1, 1);\n } else {\n const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n int nth = 1;\n\n while (2*nth < args.ne0 && nth < nth_max) {\n nth *= 2;\n }\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n }\n\n return n_fuse;\n}\n\nint ggml_metal_op_l2_norm(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n float eps;\n memcpy(&eps, op->op_params, sizeof(float));\n\n ggml_metal_kargs_l2_norm args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.eps =*/ eps,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_l2_norm(lib, op);\n\n if (pipeline.c4) {\n args.ne00 = ne00/4;\n args.ne0 = ne0/4;\n }\n\n int nth = 32; // SIMD width\n\n while (nth < ne00 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nth *= 2;\n }\n\n nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const int32_t ngrp = ((const int32_t *) op->op_params)[0];\n\n float eps;\n memcpy(&eps, op->op_params + 1, sizeof(float));\n\n ggml_metal_kargs_group_norm args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.ngrp =*/ ngrp,\n /*.eps =*/ eps,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_group_norm(lib, op);\n\n int nth = 32; // SIMD width\n //while (nth < ne00/4 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n // nth *= 2;\n //}\n\n //nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n //nth = std::min(nth, ne00/4);\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ngrp, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n const bool use_fusion = ctx->use_fusion;\n\n const int debug_fusion = ctx->debug_fusion;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n float eps;\n memcpy(&eps, op->op_params, sizeof(float));\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n ggml_metal_kargs_norm args = {\n /*.ne00 =*/ ne00,\n /*.ne00_t =*/ ne00 % 4 == 0 ? ne00/4 : ne00,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.eps =*/ eps,\n /*.nef1 =*/ { ne01 },\n /*.nef2 =*/ { ne02 },\n /*.nef3 =*/ { ne03 },\n /*.nbf1 =*/ { nb01 },\n /*.nbf2 =*/ { nb02 },\n /*.nbf3 =*/ { nb03 },\n };\n\n ggml_op fops[8];\n\n int n_fuse = 1;\n\n ggml_metal_buffer_id bid_fuse[2] = { bid_src0, bid_src0 };\n\n // d[0] = norm(a)\n // d[1] = mul(d[0], b)\n // d[2] = add(d[1], c)\n if (use_fusion) {\n fops[0] = op->op;\n fops[1] = GGML_OP_MUL;\n fops[2] = GGML_OP_ADD;\n\n for (n_fuse = 0; n_fuse <= 1; ++n_fuse) {\n if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) {\n break;\n }\n\n ggml_tensor * f0 = ctx->node(idx + n_fuse);\n ggml_tensor * f1 = ctx->node(idx + n_fuse + 1);\n\n if (f0 != f1->src[0]) {\n break;\n }\n\n if (f1->src[1]->ne[0] != op->ne[0]) {\n break;\n }\n\n if (!ggml_is_contiguous_rows(f1->src[1])) {\n break;\n }\n\n if (f1->type != GGML_TYPE_F32) {\n break;\n }\n\n //ctx->fuse_cnt[f1->op]++;\n\n bid_fuse[n_fuse] = ggml_metal_get_buffer_id(f1->src[1]);\n\n args.nef1[n_fuse + 1] = f1->src[1]->ne[1];\n args.nef2[n_fuse + 1] = f1->src[1]->ne[2];\n args.nef3[n_fuse + 1] = f1->src[1]->ne[3];\n\n args.nbf1[n_fuse + 1] = f1->src[1]->nb[1];\n args.nbf2[n_fuse + 1] = f1->src[1]->nb[2];\n args.nbf3[n_fuse + 1] = f1->src[1]->nb[3];\n }\n\n ++n_fuse;\n\n if (debug_fusion > 1 && n_fuse > 1) {\n if (n_fuse == 2) {\n GGML_LOG_DEBUG(\"%s: fuse: %s + MUL\\n\", __func__, ggml_op_name(op->op));\n }\n if (n_fuse == 3) {\n GGML_LOG_DEBUG(\"%s: fuse: %s + MUL + ADD\\n\", __func__, ggml_op_name(op->op));\n }\n }\n }\n\n if (n_fuse > 1) {\n bid_dst = ggml_metal_get_buffer_id(ctx->node(idx + n_fuse - 1));\n\n for (int i = 1; i < n_fuse; ++i) {\n if (!ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) {\n ggml_metal_op_concurrency_reset(ctx);\n\n break;\n }\n }\n }\n\n auto pipeline = ggml_metal_library_get_pipeline_norm(lib, op, n_fuse);\n\n int nth = 32; // SIMD width\n\n while (nth < args.ne00_t && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nth *= 2;\n }\n\n nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n nth = std::min(nth, args.ne00_t);\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_fuse[0], 2);\n ggml_metal_encoder_set_buffer (enc, bid_fuse[1], 3);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 4);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n\n return n_fuse;\n}\n\nint ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n // make sure we have one or more position id(ne10) per token(ne02)\n GGML_ASSERT(ne10 % ne02 == 0);\n GGML_ASSERT(ne10 >= ne02);\n\n const int nth = std::min(1024, ne00);\n\n const int n_past = ((const int32_t *) op->op_params)[0];\n const int n_dims = ((const int32_t *) op->op_params)[1];\n //const int mode = ((const int32_t *) op->op_params)[2];\n // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal\n const int n_ctx_orig = ((const int32_t *) op->op_params)[4];\n\n float freq_base;\n float freq_scale;\n float ext_factor;\n float attn_factor;\n float beta_fast;\n float beta_slow;\n\n memcpy(&freq_base, (const int32_t *) op->op_params + 5, sizeof(float));\n memcpy(&freq_scale, (const int32_t *) op->op_params + 6, sizeof(float));\n memcpy(&ext_factor, (const int32_t *) op->op_params + 7, sizeof(float));\n memcpy(&attn_factor, (const int32_t *) op->op_params + 8, sizeof(float));\n memcpy(&beta_fast, (const int32_t *) op->op_params + 9, sizeof(float));\n memcpy(&beta_slow, (const int32_t *) op->op_params + 10, sizeof(float));\n\n // mrope\n const int sect_0 = ((const int32_t *) op->op_params)[11];\n const int sect_1 = ((const int32_t *) op->op_params)[12];\n const int sect_2 = ((const int32_t *) op->op_params)[13];\n const int sect_3 = ((const int32_t *) op->op_params)[14];\n\n ggml_metal_kargs_rope args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.n_past =*/ n_past,\n /*.n_dims =*/ n_dims,\n /*.n_ctx_orig =*/ n_ctx_orig,\n /*.freq_base =*/ freq_base,\n /*.freq_scale =*/ freq_scale,\n /*.ext_factor =*/ ext_factor,\n /*.attn_factor =*/ attn_factor,\n /*.beta_fast =*/ beta_fast,\n /*.beta_slow =*/ beta_slow,\n /* sect_0 =*/ sect_0,\n /* sect_1 =*/ sect_1,\n /* sect_2 =*/ sect_2,\n /* sect_3 =*/ sect_3,\n /* src2 =*/ op->src[2] != nullptr,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_rope(lib, op);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n if (op->src[2]) {\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), 3);\n } else {\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 3);\n }\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 4);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const int32_t s0 = ((const int32_t *)(op->op_params))[0];\n const int32_t s1 = ((const int32_t *)(op->op_params))[1];\n const int32_t p0 = ((const int32_t *)(op->op_params))[2];\n const int32_t p1 = ((const int32_t *)(op->op_params))[3];\n const int32_t d0 = ((const int32_t *)(op->op_params))[4];\n const int32_t d1 = ((const int32_t *)(op->op_params))[5];\n\n const bool is_2D = ((const int32_t *)(op->op_params))[6] == 1;\n\n const int32_t N = op->src[1]->ne[is_2D ? 3 : 2];\n const int32_t IC = op->src[1]->ne[is_2D ? 2 : 1];\n const int32_t IH = is_2D ? op->src[1]->ne[1] : 1;\n const int32_t IW = op->src[1]->ne[0];\n\n const int32_t KH = is_2D ? op->src[0]->ne[1] : 1;\n const int32_t KW = op->src[0]->ne[0];\n\n const int32_t OH = is_2D ? op->ne[2] : 1;\n const int32_t OW = op->ne[1];\n\n const int32_t CHW = IC * KH * KW;\n\n const uint64_t ofs0 = op->src[1]->nb[is_2D ? 3 : 2] / 4;\n const uint64_t ofs1 = op->src[1]->nb[is_2D ? 2 : 1] / 4;\n\n ggml_metal_kargs_im2col args = {\n /*.ofs0 =*/ ofs0,\n /*.ofs1 =*/ ofs1,\n /*.IW =*/ IW,\n /*.IH =*/ IH,\n /*.CHW =*/ CHW,\n /*.s0 =*/ s0,\n /*.s1 =*/ s1,\n /*.p0 =*/ p0,\n /*.p1 =*/ p1,\n /*.d0 =*/ d0,\n /*.d1 =*/ d1,\n /*.N =*/ N,\n /*.KH =*/ KH,\n /*.KW =*/ KW,\n /*.KHW =*/ KH * KW,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_im2col(lib, op);\n\n GGML_ASSERT(KH*KW <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n const uint64_t ntptg0 = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)/(KH*KW), N);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, IC, OH, OW, ntptg0, KH, KW);\n\n return 1;\n}\n\nint ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n GGML_ASSERT(ggml_is_contiguous(op->src[0]));\n GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);\n GGML_ASSERT(op->type == GGML_TYPE_F32);\n GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);\n\n const int32_t s0 = ((const int32_t *) op->op_params)[0];\n const int32_t s1 = ((const int32_t *) op->op_params)[1];\n const int32_t p0 = ((const int32_t *) op->op_params)[2];\n const int32_t p1 = ((const int32_t *) op->op_params)[3];\n const int32_t d0 = ((const int32_t *) op->op_params)[4];\n const int32_t d1 = ((const int32_t *) op->op_params)[5];\n\n ggml_metal_kargs_conv_2d args = {\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.IW =*/ ne10,\n /*.IH =*/ ne11,\n /*.KW =*/ ne00,\n /*.KH =*/ ne01,\n /*.IC =*/ ne02,\n /*.OC =*/ ne03,\n /*.OW =*/ ne0,\n /*.OH =*/ ne1,\n /*.N =*/ ne3,\n /*.s0 =*/ s0,\n /*.s1 =*/ s1,\n /*.p0 =*/ p0,\n /*.p1 =*/ p1,\n /*.d0 =*/ d0,\n /*.d1 =*/ d1,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_conv_2d(lib, op);\n\n int nth = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline);\n nth = std::min(nth, 256);\n nth = std::max(nth, 1);\n\n const uint64_t n_out = ggml_nelements(op);\n\n uint64_t tg = (n_out + nth - 1)/nth;\n tg = std::max(tg, 1);\n tg = std::min(tg, (uint64_t) std::numeric_limits::max());\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, tg, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_conv_transpose_1d(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const int32_t s0 = ((const int32_t *)(op->op_params))[0];\n\n const int32_t IC = op->src[1]->ne[1];\n const int32_t IL = op->src[1]->ne[0];\n\n const int32_t K = op->src[0]->ne[0];\n\n const int32_t OL = op->ne[0];\n const int32_t OC = op->ne[1];\n\n ggml_metal_kargs_conv_transpose_1d args = {\n /*.IC =*/ IC,\n /*.IL =*/ IL,\n /*.K =*/ K,\n /*.s0 =*/ s0,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_conv_transpose_1d(lib, op);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, OL, OC, 1, 1, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const int32_t s0 = ((const int32_t *)(op->op_params))[0];\n\n const int32_t IC = op->src[1]->ne[2];\n const int32_t IH = op->src[1]->ne[1];\n const int32_t IW = op->src[1]->ne[0];\n\n const int32_t KH = op->src[0]->ne[1];\n const int32_t KW = op->src[0]->ne[0];\n\n const int32_t OW = op->ne[0];\n const int32_t OH = op->ne[1];\n const int32_t OC = op->ne[2];\n\n ggml_metal_kargs_conv_transpose_2d args = {\n /*.IC =*/ IC,\n /*.IH =*/ IH,\n /*.IW =*/ IW,\n /*.KH =*/ KH,\n /*.KW =*/ KW,\n /*.OC =*/ OC,\n /*.s0 =*/ s0,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_conv_transpose_2d(lib, op);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);\n\n // Metal requires buffer size to be multiple of 16 bytes\n const size_t smem = GGML_PAD(KW * KH * sizeof(float), 16);\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC, KW, KH, 1);\n\n return 1;\n}\n\nint ggml_metal_op_upscale(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n float sf0 = (float)ne0/op->src[0]->ne[0];\n float sf1 = (float)ne1/op->src[0]->ne[1];\n float sf2 = (float)ne2/op->src[0]->ne[2];\n float sf3 = (float)ne3/op->src[0]->ne[3];\n\n const int32_t mode_flags = ggml_get_op_params_i32(op, 0);\n\n float poffs = 0.5f;\n\n if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) {\n poffs = 0.0f;\n sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0;\n sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1;\n }\n\n ggml_metal_kargs_upscale args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.sf0 =*/ sf0,\n /*.sf1 =*/ sf1,\n /*.sf2 =*/ sf2,\n /*.sf3 =*/ sf3,\n /*.poffs =*/ poffs,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_upscale(lib, op);\n\n const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne0);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_pad(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n ggml_metal_kargs_pad args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_pad(lib, op);\n\n const int nth = std::min(1024, ne0);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_pad_reflect_1d(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n ggml_metal_kargs_pad_reflect_1d args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n /*.p0 =*/ ((const int32_t *)(op->op_params))[0],\n /*.p1 =*/ ((const int32_t *)(op->op_params))[1]\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_pad_reflect_1d(lib, op);\n\n const int nth = std::min(1024, ne0);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_arange(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n float start;\n float step;\n\n memcpy(&start, ((const int32_t *) op->op_params) + 0, sizeof(float));\n memcpy(&step, ((const int32_t *) op->op_params) + 2, sizeof(float));\n\n ggml_metal_kargs_arange args = {\n /*.ne0 =*/ ne0,\n /*.start =*/ start,\n /*.step =*/ step\n };\n\n const int nth = std::min(1024, ne0);\n\n auto pipeline = ggml_metal_library_get_pipeline_arange(lib, op);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 1);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_timestep_embedding(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n const int dim = op->op_params[0];\n const int max_period = op->op_params[1];\n\n ggml_metal_kargs_timestep_embedding args = {\n /*.nb1 =*/ nb1,\n /*.dim =*/ dim,\n /*.max_period =*/ max_period,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_timestep_embedding(lib, op);\n\n const int nth = std::max(1, std::min(1024, dim/2));\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne00, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_argmax(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n ggml_metal_kargs_argmax args = {\n /*.ne00 = */ ne00,\n /*.nb01 = */ nb01,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_argmax(lib, op);\n\n const int64_t nrows = ggml_nrows(op->src[0]);\n\n int nth = 32; // SIMD width\n while (nth < ne00 && nth*ne01*ne02*ne03 < 256) {\n nth *= 2;\n }\n\n const size_t smem = pipeline.smem;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_argsort(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline = ggml_metal_library_get_pipeline_argsort(lib, op);\n\n // bitonic sort requires the number of elements to be power of 2\n int nth = 1;\n while (nth < ne00 && 2*nth <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nth *= 2;\n }\n\n const int npr = (ne00 + nth - 1)/nth;\n\n // Metal kernels require the buffer size to be multiple of 16 bytes\n // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/1443142-setthreadgroupmemorylength\n const size_t smem = GGML_PAD(nth*sizeof(int32_t), 16);\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n ggml_metal_buffer_id bid_tmp = bid_dst;\n bid_tmp.offs += ggml_nbytes(op);\n\n if ((int) ceil(std::log(npr) / std::log(2)) % 2 == 1) {\n std::swap(bid_dst, bid_tmp);\n }\n\n ggml_metal_kargs_argsort args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.top_k =*/ nth,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, npr*ne01, ne02, ne03, nth, 1, 1);\n\n auto pipeline_merge = ggml_metal_library_get_pipeline_argsort_merge(lib, op);\n\n int len = nth;\n\n while (len < ne00) {\n ggml_metal_op_concurrency_reset(ctx);\n\n ggml_metal_kargs_argsort_merge args_merge = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.top_k =*/ ne00,\n /*.len =*/ len,\n };\n\n // merges per row\n const int nm = (ne00 + 2*len - 1) / (2*len);\n\n const int nth = std::min(512, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline_merge));\n\n ggml_metal_encoder_set_pipeline(enc, pipeline_merge);\n ggml_metal_encoder_set_bytes (enc, &args_merge, sizeof(args_merge), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n ggml_metal_encoder_set_buffer (enc, bid_tmp, 3);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nm*ne01, ne02, ne03, nth, 1, 1);\n\n std::swap(bid_dst, bid_tmp);\n\n len <<= 1;\n }\n\n return 1;\n}\n\nint ggml_metal_op_top_k(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline = ggml_metal_library_get_pipeline_top_k(lib, op);\n\n // bitonic sort requires the number of elements to be power of 2\n int nth = 1;\n while (nth < ne00 && 2*nth <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nth *= 2;\n }\n\n // blocks per row\n const int npr = (ne00 + nth - 1)/nth;\n\n const size_t smem = GGML_PAD(nth*sizeof(int32_t), 16);\n\n ggml_metal_buffer_id bid_src0 = ggml_metal_get_buffer_id(op->src[0]);\n ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);\n\n ggml_metal_buffer_id bid_tmp = bid_dst;\n bid_tmp.offs += sizeof(int32_t)*ggml_nelements(op->src[0]);\n\n if ((int) ceil(std::log(npr) / std::log(2)) % 2 == 1) {\n std::swap(bid_dst, bid_tmp);\n }\n\n const int top_k = ne0;\n\n ggml_metal_kargs_argsort args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.top_k =*/ std::min(nth, top_k), // for each block, keep just the top_k indices\n };\n\n if (npr > 1) {\n args.ne0 = (npr - 1)*args.top_k + std::min(ne00 - (npr - 1)*nth, args.top_k);\n }\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, npr*ne01, ne02, ne03, nth, 1, 1);\n\n auto pipeline_merge = ggml_metal_library_get_pipeline_top_k_merge(lib, op);\n\n int len = args.top_k;\n\n while (len < args.ne0) {\n ggml_metal_op_concurrency_reset(ctx);\n\n // merges per row\n const int nm = (args.ne0 + 2*len - 1) / (2*len);\n\n const int nth = std::min(512, std::min(len, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline_merge)));\n\n ggml_metal_kargs_argsort_merge args_merge = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ args.ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.top_k =*/ nm == 1 ? top_k : args.ne0, // the final merge outputs top_k elements\n /*.len =*/ len,\n };\n\n ggml_metal_encoder_set_pipeline(enc, pipeline_merge);\n ggml_metal_encoder_set_bytes (enc, &args_merge, sizeof(args_merge), 0);\n ggml_metal_encoder_set_buffer (enc, bid_src0, 1);\n ggml_metal_encoder_set_buffer (enc, bid_dst, 2);\n ggml_metal_encoder_set_buffer (enc, bid_tmp, 3);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, nm*ne01, ne02, ne03, nth, 1, 1);\n\n std::swap(bid_dst, bid_tmp);\n\n len <<= 1;\n }\n\n return 1;\n}\n\nint ggml_metal_op_tri(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n ggml_metal_kargs_tri args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.ne0 =*/ ne0,\n /*.ne1 =*/ ne1,\n /*.ne2 =*/ ne2,\n /*.ne3 =*/ ne3,\n /*.nb0 =*/ nb0,\n /*.nb1 =*/ nb1,\n /*.nb2 =*/ nb2,\n /*.nb3 =*/ nb3,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_tri(lib, op);\n\n int nth = 32; // SIMD width\n\n while (nth < ne00 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {\n nth *= 2;\n }\n\n nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n nth = std::min(nth, ne00);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_opt_step_adamw(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline = ggml_metal_library_get_pipeline_opt_step_adamw(lib, op);\n\n const int64_t np = ggml_nelements(op->src[0]);\n ggml_metal_kargs_opt_step_adamw args = {\n /*.np =*/ np,\n };\n\n int ida = 0;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++);\n\n const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne0);\n const int64_t n = (np + nth - 1) / nth;\n\n ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_opt_step_sgd(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS( int32_t, ne, op, ne);\n GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);\n\n auto pipeline = ggml_metal_library_get_pipeline_opt_step_sgd(lib, op);\n\n const int64_t np = ggml_nelements(op->src[0]);\n ggml_metal_kargs_opt_step_sgd args = {\n /*.np =*/ np,\n };\n\n int ida = 0;\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++);\n ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++);\n\n const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne0);\n const int64_t n = (np + nth - 1) / nth;\n\n ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, nth, 1, 1);\n\n return 1;\n}\n\nint ggml_metal_op_count_equal(ggml_metal_op_t ctx, int idx) {\n ggml_tensor * op = ctx->node(idx);\n\n ggml_metal_library_t lib = ctx->lib;\n ggml_metal_encoder_t enc = ctx->enc;\n\n GGML_TENSOR_LOCALS(int32_t, ne0, op->src[0], ne);\n GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);\n GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);\n\n {\n ggml_metal_kargs_memset args = { /*.val =*/ 0 };\n\n auto pipeline = ggml_metal_library_get_pipeline_memset(lib, op);\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 1);\n\n ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, 1, 1, 1);\n }\n\n ggml_metal_op_concurrency_reset(ctx);\n\n {\n ggml_metal_kargs_count_equal args = {\n /*.ne00 =*/ ne00,\n /*.ne01 =*/ ne01,\n /*.ne02 =*/ ne02,\n /*.ne03 =*/ ne03,\n /*.nb00 =*/ nb00,\n /*.nb01 =*/ nb01,\n /*.nb02 =*/ nb02,\n /*.nb03 =*/ nb03,\n /*.nb10 =*/ nb10,\n /*.nb11 =*/ nb11,\n /*.nb12 =*/ nb12,\n /*.nb13 =*/ nb13,\n };\n\n auto pipeline = ggml_metal_library_get_pipeline_count_equal(lib, op);\n\n const size_t smem = pipeline.smem;\n\n const int nth = 32*pipeline.nsg;\n\n GGML_ASSERT(nth <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));\n\n ggml_metal_encoder_set_pipeline(enc, pipeline);\n ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2);\n ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3);\n\n ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);\n ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);\n }\n\n return 1;\n}\n" }, { "path": "src/ggml-metal/ggml-metal-ops.h", "content": "#pragma once\n\n#include \"ggml-metal-device.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\ntypedef struct ggml_metal_op * ggml_metal_op_t;\n\nggml_metal_op_t ggml_metal_op_init(\n ggml_metal_device_t dev,\n ggml_metal_cmd_buf_t cmd_buf,\n struct ggml_cgraph * gf,\n int idx_start,\n int idx_end,\n bool use_fusion,\n bool use_concurrency,\n bool use_capture,\n int debug_graph,\n int debug_fusion);\n\nvoid ggml_metal_op_free(ggml_metal_op_t ctx);\n\nint ggml_metal_op_n_nodes(ggml_metal_op_t ctx);\n\nint ggml_metal_op_encode(ggml_metal_op_t ctx, int idx);\n\n//\n// available ops:\n//\n\n// tokens per expert\nsize_t ggml_metal_op_mul_mat_id_extra_tpe(const struct ggml_tensor * op);\n\n// id map [n_tokens, n_expert]\nsize_t ggml_metal_op_mul_mat_id_extra_ids(const struct ggml_tensor * op);\n\n// return true if we should use the FA vector kernel for this op\nbool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op);\n\nsize_t ggml_metal_op_flash_attn_ext_extra_pad(const struct ggml_tensor * op);\nsize_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op);\nsize_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op);\n\nint ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_acc (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_unary (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_glu (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_sum (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_sum_rows (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_cumsum (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_get_rows (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_set_rows (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_diag (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_soft_max (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_ssm_conv (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_ssm_scan (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_rwkv (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_gated_delta_net (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_solve_tri (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_set (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_cpy (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_pool_1d (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_pool_2d (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_mul_mat (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_mul_mat_id (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_add_id (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_flash_attn_ext (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_bin (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_l2_norm (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_group_norm (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_norm (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_rope (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_im2col (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_conv_2d (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_conv_transpose_1d (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_conv_transpose_2d (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_upscale (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_pad (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_pad_reflect_1d (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_arange (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_timestep_embedding(ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_argmax (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_argsort (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_top_k (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_tri (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_opt_step_adamw (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_opt_step_sgd (ggml_metal_op_t ctx, int idx);\nint ggml_metal_op_count_equal (ggml_metal_op_t ctx, int idx);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-metal/ggml-metal.cpp", "content": "#include \"ggml-metal.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-metal-device.h\"\n#include \"ggml-metal-context.h\"\n#include \"ggml-metal-ops.h\"\n\n#include \n#include \n\n#define GGML_METAL_NAME \"MTL\"\n#define GGML_METAL_MAX_DEVICES 16\n\n// number of Metal devices\n// note: can be overridden with GGML_METAL_DEVICES env to simulate virtual devices\nstatic int g_devices = 1;\n\n////////////////////////////////////////////////////////////////////////////////\n// backend interface\n////////////////////////////////////////////////////////////////////////////////\n\n// shared buffer\n\nstatic void ggml_backend_metal_buffer_shared_free_buffer(ggml_backend_buffer_t buffer) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_free(ctx);\n}\n\nstatic void * ggml_backend_metal_buffer_shared_get_base(ggml_backend_buffer_t buffer) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));\n\n return ggml_metal_buffer_get_base(ctx);\n}\n\nstatic void ggml_backend_metal_buffer_shared_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_memset_tensor(ctx, tensor, value, offset, size);\n}\n\nstatic void ggml_backend_metal_buffer_shared_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_set_tensor(ctx, tensor, data, offset, size);\n}\n\nstatic void ggml_backend_metal_buffer_shared_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);\n}\n\nstatic bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));\n\n GGML_UNUSED(buffer);\n GGML_UNUSED(src);\n GGML_UNUSED(dst);\n\n return false;\n}\n\nstatic void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_clear(ctx, value);\n}\n\nstatic ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {\n /* .free_buffer = */ ggml_backend_metal_buffer_shared_free_buffer,\n /* .get_base = */ ggml_backend_metal_buffer_shared_get_base,\n /* .init_tensor = */ NULL,\n /* .memset_tensor = */ ggml_backend_metal_buffer_shared_memset_tensor,\n /* .set_tensor = */ ggml_backend_metal_buffer_shared_set_tensor,\n /* .get_tensor = */ ggml_backend_metal_buffer_shared_get_tensor,\n /* .cpy_tensor = */ ggml_backend_metal_buffer_shared_cpy_tensor,\n /* .clear = */ ggml_backend_metal_buffer_shared_clear,\n /* .reset = */ NULL,\n};\n\n// private buffer\n\nstatic void ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_t buffer) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_free(ctx);\n}\n\nstatic void * ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t buffer) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));\n\n return ggml_metal_buffer_get_base(ctx);\n}\n\nstatic void ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_memset_tensor(ctx, tensor, value, offset, size);\n}\n\nstatic void ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_set_tensor(ctx, tensor, data, offset, size);\n}\n\nstatic void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);\n}\n\nstatic bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));\n\n GGML_UNUSED(buffer);\n GGML_UNUSED(src);\n GGML_UNUSED(dst);\n\n return false;\n}\n\nstatic void ggml_backend_metal_buffer_private_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;\n\n GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));\n\n ggml_metal_buffer_clear(ctx, value);\n}\n\nstatic ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {\n /* .free_buffer = */ ggml_backend_metal_buffer_private_free_buffer,\n /* .get_base = */ ggml_backend_metal_buffer_private_get_base,\n /* .init_tensor = */ NULL,\n /* .memset_tensor = */ ggml_backend_metal_buffer_private_memset_tensor,\n /* .set_tensor = */ ggml_backend_metal_buffer_private_set_tensor,\n /* .get_tensor = */ ggml_backend_metal_buffer_private_get_tensor,\n /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor,\n /* .clear = */ ggml_backend_metal_buffer_private_clear,\n /* .reset = */ NULL,\n};\n\nstatic bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) {\n return buffer->iface.free_buffer == ggml_backend_metal_buffer_shared_free_buffer ||\n buffer->iface.free_buffer == ggml_backend_metal_buffer_private_free_buffer;\n}\n\n//\n// buffer types\n//\n\nstruct ggml_backend_metal_buffer_type {\n int device;\n std::string name;\n};\n\nstruct ggml_backend_metal_buffer_type_deleter {\n void operator()(ggml_backend_metal_buffer_type * ctx) const {\n delete ctx;\n }\n};\n\ntypedef std::unique_ptr ggml_backend_metal_buffer_type_ptr;\n\n// common method for allocating shread or private Metal buffers\nstatic ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size, bool shared) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)buft->device->context;\n ggml_metal_buffer_t res = ggml_metal_buffer_init(ctx_dev, size, shared);\n\n ggml_backend_buffer_i buf_i = ggml_metal_buffer_is_shared(res)\n ? ggml_backend_metal_buffer_shared_i\n : ggml_backend_metal_buffer_private_i;\n\n return ggml_backend_buffer_init(buft, buf_i, res, size);\n}\n\nstatic size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n size_t res = ggml_nbytes(tensor);\n\n // some operations require additional memory for fleeting data:\n switch (tensor->op) {\n case GGML_OP_MUL_MAT_ID:\n {\n res += ggml_metal_op_mul_mat_id_extra_tpe(tensor);\n res += ggml_metal_op_mul_mat_id_extra_ids(tensor);\n } break;\n case GGML_OP_FLASH_ATTN_EXT:\n {\n res += ggml_metal_op_flash_attn_ext_extra_pad(tensor);\n res += ggml_metal_op_flash_attn_ext_extra_blk(tensor);\n res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor);\n } break;\n case GGML_OP_CUMSUM:\n case GGML_OP_ARGSORT:\n {\n res *= 2;\n } break;\n case GGML_OP_TOP_K:\n {\n res = 2*sizeof(int32_t)*ggml_nelements(tensor->src[0]);\n } break;\n default:\n break;\n }\n\n return res;\n\n GGML_UNUSED(buft);\n}\n\n// default (shared) buffer type\n\nstatic const char * ggml_backend_metal_buffer_type_shared_get_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_metal_buffer_type * ctx = (ggml_backend_metal_buffer_type *)buft->context;\n\n return ctx->name.c_str();\n}\n\nstatic ggml_backend_buffer_t ggml_backend_metal_buffer_type_shared_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n return ggml_backend_metal_buffer_type_alloc_buffer(buft, size, true);\n}\n\nstatic size_t ggml_backend_metal_buffer_type_shared_get_alignment(ggml_backend_buffer_type_t buft) {\n return 32;\n\n GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_metal_buffer_type_shared_get_max_size(ggml_backend_buffer_type_t buft) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)buft->device->context;\n\n return ggml_metal_device_get_props(ctx_dev)->max_buffer_size;\n}\n\nstatic size_t ggml_backend_metal_buffer_type_shared_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n return ggml_backend_metal_buffer_type_get_alloc_size(buft, tensor);\n}\n\nstatic bool ggml_backend_metal_buffer_type_shared_is_host(ggml_backend_buffer_type_t buft) {\n return false;\n\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int device) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n static std::vector bufts;\n static std::vector ctxs;\n\n static bool initialized = false;\n if (!initialized) {\n bufts.reserve(g_devices);\n ctxs.reserve(g_devices);\n\n for (int i = 0; i < g_devices; ++i) {\n ggml_backend_metal_buffer_type * raw_ctx =\n new ggml_backend_metal_buffer_type {\n /* .device = */ i,\n /* .name = */ GGML_METAL_NAME + std::to_string(i),\n };\n ctxs.emplace_back(raw_ctx);\n\n ggml_backend_buffer_type buft = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_metal_buffer_type_shared_get_name,\n /* .alloc_buffer = */ ggml_backend_metal_buffer_type_shared_alloc_buffer,\n /* .get_alignment = */ ggml_backend_metal_buffer_type_shared_get_alignment,\n /* .get_max_size = */ ggml_backend_metal_buffer_type_shared_get_max_size,\n /* .get_alloc_size = */ ggml_backend_metal_buffer_type_shared_get_alloc_size,\n /* .is_host = */ ggml_backend_metal_buffer_type_shared_is_host,\n },\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_metal_reg(), i),\n /* .context = */ raw_ctx,\n };\n\n bufts.emplace_back(buft);\n }\n\n initialized = true;\n }\n\n return &bufts[device];\n}\n\n// default (private) buffer type\n\nstatic const char * ggml_backend_metal_buffer_type_private_get_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_metal_buffer_type * ctx = (ggml_backend_metal_buffer_type *)buft->context;\n\n return ctx->name.c_str();\n}\n\nstatic ggml_backend_buffer_t ggml_backend_metal_buffer_type_private_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n return ggml_backend_metal_buffer_type_alloc_buffer(buft, size, false);\n}\n\nstatic size_t ggml_backend_metal_buffer_type_private_get_alignment(ggml_backend_buffer_type_t buft) {\n return 32;\n\n GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_metal_buffer_type_private_get_max_size(ggml_backend_buffer_type_t buft) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)buft->device->context;\n\n return ggml_metal_device_get_props(ctx_dev)->max_buffer_size;\n}\n\nstatic size_t ggml_backend_metal_buffer_type_private_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n return ggml_backend_metal_buffer_type_get_alloc_size(buft, tensor);\n}\n\nstatic bool ggml_backend_metal_buffer_type_private_is_host(ggml_backend_buffer_type_t buft) {\n return false;\n\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int device) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n static std::vector bufts;\n static std::vector ctxs;\n\n static bool initialized = false;\n if (!initialized) {\n bufts.reserve(g_devices);\n ctxs.reserve(g_devices);\n\n for (int i = 0; i < g_devices; ++i) {\n ggml_backend_metal_buffer_type * raw_ctx = new ggml_backend_metal_buffer_type{\n /* .device = */ i,\n /* .name = */ GGML_METAL_NAME + std::to_string(i) + \"_Private\"\n };\n ctxs.emplace_back(raw_ctx);\n\n ggml_backend_buffer_type buft = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_metal_buffer_type_private_get_name,\n /* .alloc_buffer = */ ggml_backend_metal_buffer_type_private_alloc_buffer,\n /* .get_alignment = */ ggml_backend_metal_buffer_type_private_get_alignment,\n /* .get_max_size = */ ggml_backend_metal_buffer_type_private_get_max_size,\n /* .get_alloc_size = */ ggml_backend_metal_buffer_type_private_get_alloc_size,\n /* .is_host = */ ggml_backend_metal_buffer_type_private_is_host,\n },\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_metal_reg(), i),\n /* .context = */ raw_ctx,\n };\n\n bufts.emplace_back(buft);\n }\n\n initialized = true;\n }\n\n return &bufts[device];\n}\n\n// mapped buffer type\n\nstatic const char * ggml_backend_metal_buffer_type_mapped_get_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_metal_buffer_type * ctx = (ggml_backend_metal_buffer_type *)buft->context;\n\n return ctx->name.c_str();\n}\n\nstatic ggml_backend_buffer_t ggml_backend_metal_buffer_type_mapped_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n // for mapped buffers, prefer shared memory\n return ggml_backend_metal_buffer_type_alloc_buffer(buft, size, true);\n}\n\nstatic size_t ggml_backend_metal_buffer_type_mapped_get_alignment(ggml_backend_buffer_type_t buft) {\n return 32;\n\n GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_metal_buffer_type_mapped_get_max_size(ggml_backend_buffer_type_t buft) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)buft->device->context;\n\n return ggml_metal_device_get_props(ctx_dev)->max_buffer_size;\n}\n\nstatic size_t ggml_backend_metal_buffer_type_mapped_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n return ggml_backend_metal_buffer_type_get_alloc_size(buft, tensor);\n}\n\nstatic bool ggml_backend_metal_buffer_type_mapped_is_host(ggml_backend_buffer_type_t buft) {\n return false;\n\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int device) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n static std::vector bufts;\n static std::vector ctxs;\n\n static bool initialized = false;\n if (!initialized) {\n bufts.reserve(g_devices);\n ctxs.reserve(g_devices);\n\n for (int i = 0; i < g_devices; ++i) {\n ggml_backend_metal_buffer_type * raw_ctx = new ggml_backend_metal_buffer_type{\n /* .device = */ i,\n /* .name = */ GGML_METAL_NAME + std::to_string(i) + \"_Mapped\"\n };\n ctxs.emplace_back(raw_ctx);\n\n // note: not obvious, but this buffer type still needs to implement .alloc_buffer:\n // https://github.com/ggml-org/llama.cpp/pull/15832#discussion_r2333177099\n ggml_backend_buffer_type buft = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_metal_buffer_type_mapped_get_name,\n /* .alloc_buffer = */ ggml_backend_metal_buffer_type_mapped_alloc_buffer,\n /* .get_alignment = */ ggml_backend_metal_buffer_type_mapped_get_alignment,\n /* .get_max_size = */ ggml_backend_metal_buffer_type_mapped_get_max_size,\n /* .get_alloc_size = */ ggml_backend_metal_buffer_type_mapped_get_alloc_size,\n /* .is_host = */ ggml_backend_metal_buffer_type_mapped_is_host,\n },\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_metal_reg(), i),\n /* .context = */ raw_ctx,\n };\n\n bufts.emplace_back(buft);\n }\n\n initialized = true;\n }\n\n return &bufts[device];\n}\n\n// backend\n\nstatic const char * ggml_backend_metal_name(ggml_backend_t backend) {\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n return ggml_metal_get_name(ctx);\n}\n\nstatic void ggml_backend_metal_free(ggml_backend_t backend) {\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n // wait for any ongoing async operations to finish\n ggml_metal_synchronize(ctx);\n\n ggml_metal_free(ctx);\n\n free(backend);\n}\n\nstatic void ggml_backend_metal_synchronize(ggml_backend_t backend) {\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n ggml_metal_synchronize(ctx);\n}\n\nstatic void ggml_backend_metal_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n ggml_metal_set_tensor_async(ctx, tensor, data, offset, size);\n}\n\nstatic void ggml_backend_metal_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n ggml_metal_get_tensor_async(ctx, tensor, data, offset, size);\n}\n\nstatic bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {\n if (!ggml_backend_is_metal(backend_src) || !ggml_backend_is_metal(backend_dst)) {\n return false;\n }\n\n if (!ggml_backend_buffer_is_metal(src->buffer) || !ggml_backend_buffer_is_metal(dst->buffer)) {\n return false;\n }\n\n ggml_metal_t ctx_src = (ggml_metal_t)backend_src->context;\n ggml_metal_t ctx_dst = (ggml_metal_t)backend_dst->context;\n\n //ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;\n //ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;\n\n //ggml_metal_buffer_t buf_ctx_src = (ggml_metal_buffer_t)buf_src->context;\n //ggml_metal_buffer_t buf_ctx_dst = (ggml_metal_buffer_t)buf_dst->context;\n\n return ggml_metal_cpy_tensor_async(ctx_src, ctx_dst, src, dst);\n}\n\nstatic enum ggml_status ggml_backend_metal_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n return ggml_metal_graph_compute(ctx, cgraph);\n}\n\nstatic void ggml_backend_metal_event_record(ggml_backend_t backend, ggml_backend_event_t event) {\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n ggml_metal_event_t ev = (ggml_metal_event_t)event->context;\n\n ggml_metal_event_record(ctx, ev);\n}\n\nstatic void ggml_backend_metal_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n ggml_metal_event_t ev = (ggml_metal_event_t)event->context;\n\n ggml_metal_event_wait(ctx, ev);\n}\n\nstatic void ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) {\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n ggml_metal_graph_optimize(ctx, cgraph);\n}\n\nstatic void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {\n GGML_ASSERT(ggml_backend_is_metal(backend));\n\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n ggml_metal_set_n_cb(ctx, n_cb);\n}\n\nstatic ggml_backend_i ggml_backend_metal_i = {\n /* .get_name = */ ggml_backend_metal_name,\n /* .free = */ ggml_backend_metal_free,\n /* .set_tensor_async = */ ggml_backend_metal_set_tensor_async,\n /* .get_tensor_async = */ ggml_backend_metal_get_tensor_async,\n /* .cpy_tensor_async = */ ggml_backend_metal_cpy_tensor_async, // only needed for multi-GPU setups\n /* .synchronize = */ ggml_backend_metal_synchronize,\n /* .graph_plan_create = */ NULL,\n /* .graph_plan_free = */ NULL,\n /* .graph_plan_update = */ NULL,\n /* .graph_plan_compute = */ NULL,\n /* .graph_compute = */ ggml_backend_metal_graph_compute,\n /* .event_record = */ ggml_backend_metal_event_record,\n /* .event_wait = */ ggml_backend_metal_event_wait,\n /* .graph_optimize = */ ggml_backend_metal_graph_optimize,\n};\n\nstatic ggml_guid_t ggml_backend_metal_guid(void) {\n static ggml_guid guid = { 0x81, 0xa1, 0x8b, 0x1e, 0x71, 0xec, 0x79, 0xed, 0x2b, 0x85, 0xdc, 0x8a, 0x61, 0x98, 0x30, 0xe6 };\n return &guid;\n}\n\nggml_backend_t ggml_backend_metal_init(void) {\n ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_metal_reg(), 0);\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n ggml_metal_t ctx = ggml_metal_init(ctx_dev);\n if (ctx == NULL) {\n GGML_LOG_ERROR(\"%s: error: failed to allocate context\\n\", __func__);\n return NULL;\n }\n\n ggml_backend_t backend = (ggml_backend_t) malloc(sizeof(ggml_backend));\n\n *backend = {\n /* .guid = */ ggml_backend_metal_guid(),\n /* .interface = */ ggml_backend_metal_i,\n /* .device = */ dev,\n /* .context = */ ctx,\n };\n\n ggml_backend_metal_set_n_cb(backend, 1);\n\n return backend;\n}\n\nbool ggml_backend_is_metal(ggml_backend_t backend) {\n return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_metal_guid());\n}\n\nvoid ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data) {\n GGML_ASSERT(ggml_backend_is_metal(backend));\n\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n ggml_metal_set_abort_callback(ctx, abort_callback, user_data);\n}\n\nbool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {\n GGML_ASSERT(ggml_backend_is_metal(backend));\n\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n return ggml_metal_supports_family(ctx, family);\n}\n\nvoid ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {\n GGML_ASSERT(ggml_backend_is_metal(backend));\n\n ggml_metal_t ctx = (ggml_metal_t)backend->context;\n\n ggml_metal_capture_next_compute(ctx);\n}\n\n// backend device\n\nstatic const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);\n\n return props_dev->name;\n}\n\nstatic const char * ggml_backend_metal_device_get_description(ggml_backend_dev_t dev) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n return ggml_metal_device_get_props(ctx_dev)->desc;\n}\n\nstatic void ggml_backend_metal_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n ggml_metal_device_get_memory(ctx_dev, free, total);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backend_dev_t dev) {\n return GGML_BACKEND_DEVICE_TYPE_GPU;\n\n GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {\n props->name = ggml_backend_metal_device_get_name(dev);\n props->description = ggml_backend_metal_device_get_description(dev);\n props->type = ggml_backend_metal_device_get_type(dev);\n\n ggml_backend_metal_device_get_memory(dev, &props->memory_free, &props->memory_total);\n\n props->caps = {\n /* .async = */ true,\n /* .host_buffer = */ false,\n /* .buffer_from_host_ptr = */ true,\n /* .events = */ true,\n };\n}\n\nstatic ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t dev, const char * params) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n ggml_metal_t ctx = ggml_metal_init(ctx_dev);\n if (ctx == NULL) {\n GGML_LOG_ERROR(\"%s: error: failed to allocate context\\n\", __func__);\n return NULL;\n }\n\n ggml_backend_t backend = (ggml_backend_t) malloc(sizeof(ggml_backend));\n\n *backend = {\n /* .guid = */ ggml_backend_metal_guid(),\n /* .interface = */ ggml_backend_metal_i,\n /* .device = */ dev,\n /* .context = */ ctx,\n };\n\n ggml_backend_metal_set_n_cb(backend, 1);\n\n return backend;\n\n GGML_UNUSED(params);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml_backend_dev_t dev) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);\n\n return props_dev->use_shared_buffers ? ggml_backend_metal_buffer_type_shared(props_dev->device) : ggml_backend_metal_buffer_type_private(props_dev->device);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size);\n\n const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);\n\n return ggml_backend_buffer_init(ggml_backend_metal_buffer_type_mapped(props_dev->device), ggml_backend_metal_buffer_shared_i, res, size);\n}\n\nstatic bool ggml_backend_metal_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n return ggml_metal_device_supports_op(ctx_dev, op);\n}\n\nstatic bool ggml_backend_metal_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n return\n buft->device == dev && (\n buft->iface.get_name == ggml_backend_metal_buffer_type_shared_get_name ||\n buft->iface.get_name == ggml_backend_metal_buffer_type_private_get_name ||\n buft->iface.get_name == ggml_backend_metal_buffer_type_mapped_get_name);\n\n GGML_UNUSED(dev);\n}\n\nstatic int64_t get_op_batch_size(const ggml_tensor * op) {\n switch (op->op) {\n case GGML_OP_MUL_MAT:\n return op->ne[1];\n case GGML_OP_MUL_MAT_ID:\n return op->ne[2];\n default:\n return ggml_nrows(op);\n }\n}\n\nstatic bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n return (op->op == GGML_OP_MUL_MAT ||\n op->op == GGML_OP_MUL_MAT_ID) &&\n get_op_batch_size(op) >= ggml_metal_device_get_props(ctx_dev)->op_offload_min_batch_size;\n}\n\nstatic ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev_t dev) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n ggml_metal_event_t event = ggml_metal_device_event_init(ctx_dev);\n GGML_ASSERT(event);\n\n ggml_backend_event_t ev = new ggml_backend_event {\n /* .device = */ dev,\n /* .context = */ event,\n };\n\n return ev;\n}\n\nstatic void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n ggml_metal_event_t ev = (ggml_metal_event_t)event->context;\n\n ggml_metal_device_event_free(ctx_dev, ev);\n\n delete event;\n}\n\nstatic void ggml_backend_metal_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) {\n ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;\n\n ggml_metal_event_t evt = (ggml_metal_event_t)event->context;\n\n ggml_metal_device_event_synchronize(ctx_dev, evt);\n}\n\nstatic ggml_backend_device_i ggml_backend_metal_device_i = {\n /* .get_name = */ ggml_backend_metal_device_get_name,\n /* .get_description = */ ggml_backend_metal_device_get_description,\n /* .get_memory = */ ggml_backend_metal_device_get_memory,\n /* .get_type = */ ggml_backend_metal_device_get_type,\n /* .get_props = */ ggml_backend_metal_device_get_props,\n /* .init_backend = */ ggml_backend_metal_device_init_backend,\n /* .get_buffer_type = */ ggml_backend_metal_device_get_buffer_type,\n /* .get_host_buffer_type = */ NULL,\n /* .buffer_from_host_ptr = */ ggml_backend_metal_device_buffer_mapped,\n /* .supports_op = */ ggml_backend_metal_device_supports_op,\n /* .supports_buft = */ ggml_backend_metal_device_supports_buft,\n /* .offload_op = */ ggml_backend_metal_device_offload_op,\n /* .event_new = */ ggml_backend_metal_device_event_new,\n /* .event_free = */ ggml_backend_metal_device_event_free,\n /* .event_synchronize = */ ggml_backend_metal_device_event_synchronize,\n};\n\n// backend registry\n\nstruct ggml_backend_metal_reg {\n std::vector devices;\n};\n\ntypedef struct ggml_backend_metal_reg * ggml_backend_metal_reg_t;\n\nstatic ggml_backend_metal_reg_t ggml_backend_metal_reg_init(void) {\n ggml_backend_metal_reg_t ctx = new struct ggml_backend_metal_reg;\n\n return ctx;\n}\n\nstatic void ggml_backend_metal_reg_free(ggml_backend_metal_reg_t ctx) {\n delete ctx;\n}\n\nstruct ggml_backend_metal_reg_deleter {\n void operator()(ggml_backend_metal_reg_t ctx) {\n ggml_backend_metal_reg_free(ctx);\n }\n};\n\ntypedef std::unique_ptr ggml_backend_metal_reg_ptr;\n\nstatic const char * ggml_backend_metal_reg_get_name(ggml_backend_reg_t reg) {\n return GGML_METAL_NAME;\n\n GGML_UNUSED(reg);\n}\n\nstatic size_t ggml_backend_metal_reg_device_count(ggml_backend_reg_t reg) {\n ggml_backend_metal_reg_t ctx = (ggml_backend_metal_reg_t)reg->context;\n return ctx->devices.size();\n}\n\nstatic ggml_backend_dev_t ggml_backend_metal_reg_device_get(ggml_backend_reg_t reg, size_t index) {\n ggml_backend_metal_reg_t ctx = (ggml_backend_metal_reg_t)reg->context;\n GGML_ASSERT(index < ctx->devices.size());\n return ctx->devices[index];\n}\n\nstatic ggml_backend_feature g_ggml_backend_metal_features[] = {\n#if defined(GGML_METAL_EMBED_LIBRARY)\n { \"EMBED_LIBRARY\", \"1\" },\n#endif\n { NULL, NULL },\n};\n\nstatic ggml_backend_feature * ggml_backend_metal_get_features(ggml_backend_reg_t reg) {\n return g_ggml_backend_metal_features;\n\n GGML_UNUSED(reg);\n}\n\nstatic void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n if (strcmp(name, \"ggml_backend_get_features\") == 0) {\n return (void *)ggml_backend_metal_get_features;\n }\n\n return NULL;\n\n GGML_UNUSED(reg);\n}\n\nstatic ggml_backend_reg_i ggml_backend_metal_reg_i = {\n /* .get_name = */ ggml_backend_metal_reg_get_name,\n /* .get_device_count = */ ggml_backend_metal_reg_device_count,\n /* .get_device = */ ggml_backend_metal_reg_device_get,\n /* .get_proc_address = */ ggml_backend_metal_get_proc_address,\n};\n\nstatic ggml_backend_dev_t ggml_backend_metal_device_init(ggml_backend_reg_t reg, int device) {\n return new ggml_backend_device {\n /* .iface = */ ggml_backend_metal_device_i,\n /* .reg = */ reg,\n /* .context = */ ggml_metal_device_get(device),\n };\n}\n\nstatic void ggml_backend_metal_device_free(ggml_backend_dev_t dev) {\n delete dev;\n}\n\nstruct ggml_backend_device_deleter {\n void operator()(ggml_backend_dev_t ctx) {\n ggml_backend_metal_device_free(ctx);\n }\n};\n\ntypedef std::unique_ptr ggml_backend_device_ptr;\n\nggml_backend_reg_t ggml_backend_metal_reg(void) {\n static ggml_backend_reg reg;\n static bool initialized = false;\n\n {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n const char * env = getenv(\"GGML_METAL_DEVICES\");\n if (env) {\n g_devices = atoi(env);\n }\n\n static std::vector devs;\n\n if (!initialized) {\n static ggml_backend_metal_reg_ptr reg_ctx(ggml_backend_metal_reg_init());\n\n for (int i = 0; i < g_devices; ++i) {\n auto * dev = ggml_backend_metal_device_init(®, i);\n devs.emplace_back(dev);\n\n reg_ctx->devices.push_back(dev);\n }\n\n reg = {\n /* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_metal_reg_i,\n /* .context = */ reg_ctx.get(),\n };\n }\n\n initialized = true;\n }\n\n return ®\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_metal_reg)\n" }, { "path": "src/ggml-metal/ggml-metal.metal", "content": "#define GGML_COMMON_DECL_METAL\n#define GGML_COMMON_IMPL_METAL\n#if defined(GGML_METAL_EMBED_LIBRARY)\n__embed_ggml-common.h__\n#else\n#include \"ggml-common.h\"\n#endif\n#include \"ggml-metal-impl.h\"\n\n#include \n\n#ifdef GGML_METAL_HAS_TENSOR\n#include \n\n#include \n#endif\n\nusing namespace metal;\n\n#define MAX(x, y) ((x) > (y) ? (x) : (y))\n#define MIN(x, y) ((x) < (y) ? (x) : (y))\n#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }\n\n#define PAD2(x, n) (((x) + (n) - 1) & ~((n) - 1))\n\n#define FOR_UNROLL(x) _Pragma(\"clang loop unroll(full)\") for (x)\n\n#define N_SIMDWIDTH 32 // assuming SIMD group size is 32\n\n// ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf\n//\n// cmd:\n// .../usr/bin/metal -dM -E -c ggml/src/ggml-metal/ggml-metal.metal\n// .../usr/bin/metal -dM -E -c -target air64-apple-ios14.0 ggml/src/ggml-metal/ggml-metal.metal\n//\n#if __METAL_VERSION__ < 310 && defined(GGML_METAL_HAS_BF16)\n#undef GGML_METAL_HAS_BF16\n#endif\n\n#if defined(GGML_METAL_HAS_BF16)\ntypedef matrix bfloat4x4;\ntypedef matrix bfloat2x4;\n#endif\n\nconstexpr constant static float kvalues_iq4nl_f[16] = {\n -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f\n};\n\nconstexpr constant static float kvalues_mxfp4_f[16] = {\n 0, .5f, 1.f, 1.5f, 2.f, 3.f, 4.f, 6.f, -0, -.5f, -1.f, -1.5f, -2.f, -3.f, -4.f, -6.f\n};\n\nstatic inline int best_index_int8(int n, constant float * val, float x) {\n if (x <= val[0]) return 0;\n if (x >= val[n-1]) return n-1;\n int ml = 0, mu = n-1;\n while (mu-ml > 1) {\n int mav = (ml+mu)/2;\n if (x < val[mav]) mu = mav; else ml = mav;\n }\n return x - val[mu-1] < val[mu] - x ? mu-1 : mu;\n}\n\nstatic inline float e8m0_to_fp32(uint8_t x) {\n uint32_t bits;\n\n if (x == 0) {\n bits = 0x00400000;\n } else {\n bits = (uint32_t) x << 23;\n }\n\n return as_type(bits);\n}\n\nstatic inline float dot(float x, float y) {\n return x*y;\n}\n\nstatic inline float sum(float x) {\n return x;\n}\n\nstatic inline float sum(float4 x) {\n return x[0] + x[1] + x[2] + x[3];\n}\n\n// NOTE: this is not dequantizing - we are simply fitting the template\ntemplate \nvoid dequantize_f32(device const float4x4 * src, short il, thread type4x4 & reg) {\n reg = (type4x4)(*src);\n}\n\ntemplate \nvoid dequantize_f32_t4(device const float4 * src, short il, thread type4 & reg) {\n reg = (type4)(*src);\n}\n\ntemplate \nvoid dequantize_f16(device const half4x4 * src, short il, thread type4x4 & reg) {\n reg = (type4x4)(*src);\n}\n\ntemplate \nvoid dequantize_f16_t4(device const half4 * src, short il, thread type4 & reg) {\n reg = (type4)(*(src));\n}\n\n#if defined(GGML_METAL_HAS_BF16)\ntemplate \nvoid dequantize_bf16(device const bfloat4x4 * src, short il, thread type4x4 & reg) {\n reg = (type4x4)(*src);\n}\n\ntemplate \nvoid dequantize_bf16_t4(device const bfloat4 * src, short il, thread type4 & reg) {\n reg = (type4)(*(src));\n}\n#endif\n\ntemplate \nvoid dequantize_q4_0(device const block_q4_0 * xb, short il, thread type4x4 & reg) {\n device const uint16_t * qs = ((device const uint16_t *)xb + 1);\n const float d1 = il ? (xb->d / 16.h) : xb->d;\n const float d2 = d1 / 256.f;\n const float md = -8.h * xb->d;\n const ushort mask0 = il ? 0x00F0 : 0x000F;\n const ushort mask1 = mask0 << 8;\n\n float4x4 reg_f;\n\n for (int i = 0; i < 8; i++) {\n reg_f[i/2][2*(i%2) + 0] = d1 * (qs[i] & mask0) + md;\n reg_f[i/2][2*(i%2) + 1] = d2 * (qs[i] & mask1) + md;\n }\n\n reg = (type4x4) reg_f;\n}\n\ntemplate \nvoid dequantize_q4_0_t4(device const block_q4_0 * xb, short il, thread type4 & reg) {\n device const uint16_t * qs = ((device const uint16_t *)xb + 1);\n const float d1 = (il/4) ? (xb->d / 16.h) : xb->d;\n const float d2 = d1 / 256.f;\n const float md = -8.h * xb->d;\n const ushort mask0 = (il/4) ? 0x00F0 : 0x000F;\n const ushort mask1 = mask0 << 8;\n\n for (int i = 0; i < 2; i++) {\n reg[2*i + 0] = d1 * (qs[2*(il%4) + i] & mask0) + md;\n reg[2*i + 1] = d2 * (qs[2*(il%4) + i] & mask1) + md;\n }\n}\n\nvoid quantize_q4_0(device const float * src, device block_q4_0 & dst) {\n#pragma METAL fp math_mode(safe)\n float amax = 0.0f; // absolute max\n float max = 0.0f;\n\n for (int j = 0; j < QK4_0; j++) {\n const float v = src[j];\n if (amax < fabs(v)) {\n amax = fabs(v);\n max = v;\n }\n }\n\n const float d = max / -8;\n const float id = d ? 1.0f/d : 0.0f;\n\n dst.d = d;\n\n for (int j = 0; j < QK4_0/2; ++j) {\n const float x0 = src[0 + j]*id;\n const float x1 = src[QK4_0/2 + j]*id;\n\n const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f));\n const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f));\n\n dst.qs[j] = xi0;\n dst.qs[j] |= xi1 << 4;\n }\n}\n\nvoid quantize_q4_1(device const float * src, device block_q4_1 & dst) {\n#pragma METAL fp math_mode(safe)\n float min = FLT_MAX;\n float max = -FLT_MAX;\n\n for (int j = 0; j < QK4_1; j++) {\n const float v = src[j];\n if (min > v) min = v;\n if (max < v) max = v;\n }\n\n const float d = (max - min) / ((1 << 4) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n dst.d = d;\n dst.m = min;\n\n for (int j = 0; j < QK4_1/2; ++j) {\n const float x0 = (src[0 + j] - min)*id;\n const float x1 = (src[QK4_1/2 + j] - min)*id;\n\n const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f));\n const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f));\n\n dst.qs[j] = xi0;\n dst.qs[j] |= xi1 << 4;\n }\n}\n\nvoid quantize_q5_0(device const float * src, device block_q5_0 & dst) {\n#pragma METAL fp math_mode(safe)\n float amax = 0.0f; // absolute max\n float max = 0.0f;\n\n for (int j = 0; j < QK5_0; j++) {\n const float v = src[j];\n if (amax < fabs(v)) {\n amax = fabs(v);\n max = v;\n }\n }\n\n const float d = max / -16;\n const float id = d ? 1.0f/d : 0.0f;\n\n dst.d = d;\n\n uint32_t qh = 0;\n for (int j = 0; j < QK5_0/2; ++j) {\n const float x0 = src[0 + j]*id;\n const float x1 = src[QK5_0/2 + j]*id;\n\n const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f));\n const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f));\n\n dst.qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);\n }\n\n thread const uint8_t * qh8 = (thread const uint8_t *)&qh;\n\n for (int j = 0; j < 4; ++j) {\n dst.qh[j] = qh8[j];\n }\n}\n\nvoid quantize_q5_1(device const float * src, device block_q5_1 & dst) {\n#pragma METAL fp math_mode(safe)\n float max = src[0];\n float min = src[0];\n\n for (int j = 1; j < QK5_1; j++) {\n const float v = src[j];\n min = v < min ? v : min;\n max = v > max ? v : max;\n }\n\n const float d = (max - min) / 31;\n const float id = d ? 1.0f/d : 0.0f;\n\n dst.d = d;\n dst.m = min;\n\n uint32_t qh = 0;\n for (int j = 0; j < QK5_1/2; ++j) {\n const float x0 = (src[0 + j] - min)*id;\n const float x1 = (src[QK5_1/2 + j] - min)*id;\n\n const uint8_t xi0 = (uint8_t)(x0 + 0.5f);\n const uint8_t xi1 = (uint8_t)(x1 + 0.5f);\n\n dst.qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2);\n }\n\n thread const uint8_t * qh8 = (thread const uint8_t *)&qh;\n\n for (int j = 0; j < 4; ++j) {\n dst.qh[j] = qh8[j];\n }\n}\n\nvoid quantize_q8_0(device const float * src, device block_q8_0 & dst) {\n#pragma METAL fp math_mode(safe)\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK8_0; j++) {\n const float v = src[j];\n amax = MAX(amax, fabs(v));\n }\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n dst.d = d;\n\n for (int j = 0; j < QK8_0; ++j) {\n const float x0 = src[j]*id;\n\n dst.qs[j] = round(x0);\n }\n}\n\nvoid quantize_iq4_nl(device const float * src, device block_iq4_nl & dst) {\n#pragma METAL fp math_mode(safe)\n float amax = 0.0f; // absolute max\n float max = 0.0f;\n\n for (int j = 0; j < QK4_NL; j++) {\n const float v = src[j];\n if (amax < fabs(v)) {\n amax = fabs(v);\n max = v;\n }\n }\n\n const float d = max / kvalues_iq4nl_f[0];\n const float id = d ? 1.0f/d : 0.0f;\n\n float sumqx = 0, sumq2 = 0;\n for (int j = 0; j < QK4_NL/2; ++j) {\n const float x0 = src[0 + j]*id;\n const float x1 = src[QK4_NL/2 + j]*id;\n\n const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl_f, x0);\n const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl_f, x1);\n\n dst.qs[j] = xi0 | (xi1 << 4);\n\n const float v0 = kvalues_iq4nl_f[xi0];\n const float v1 = kvalues_iq4nl_f[xi1];\n const float w0 = src[0 + j]*src[0 + j];\n const float w1 = src[QK4_NL/2 + j]*src[QK4_NL/2 + j];\n sumqx += w0*v0*src[j] + w1*v1*src[QK4_NL/2 + j];\n sumq2 += w0*v0*v0 + w1*v1*v1;\n\n }\n\n dst.d = sumq2 > 0 ? sumqx/sumq2 : d;\n}\n\ntemplate \nvoid dequantize_q4_1(device const block_q4_1 * xb, short il, thread type4x4 & reg) {\n device const uint16_t * qs = ((device const uint16_t *)xb + 2);\n const float d1 = il ? (xb->d / 16.h) : xb->d;\n const float d2 = d1 / 256.f;\n const float m = xb->m;\n const ushort mask0 = il ? 0x00F0 : 0x000F;\n const ushort mask1 = mask0 << 8;\n\n float4x4 reg_f;\n\n for (int i = 0; i < 8; i++) {\n reg_f[i/2][2*(i%2) + 0] = ((qs[i] & mask0) * d1) + m;\n reg_f[i/2][2*(i%2) + 1] = ((qs[i] & mask1) * d2) + m;\n }\n\n reg = (type4x4) reg_f;\n}\n\ntemplate \nvoid dequantize_q4_1_t4(device const block_q4_1 * xb, short il, thread type4 & reg) {\n device const uint16_t * qs = ((device const uint16_t *)xb + 2);\n const float d1 = (il/4) ? (xb->d / 16.h) : xb->d;\n const float d2 = d1 / 256.f;\n const float m = xb->m;\n const ushort mask0 = (il/4) ? 0x00F0 : 0x000F;\n const ushort mask1 = mask0 << 8;\n\n for (int i = 0; i < 2; i++) {\n reg[2*i + 0] = d1 * (qs[2*(il%4) + i] & mask0) + m;\n reg[2*i + 1] = d2 * (qs[2*(il%4) + i] & mask1) + m;\n }\n}\n\ntemplate \nvoid dequantize_q5_0(device const block_q5_0 * xb, short il, thread type4x4 & reg) {\n device const uint16_t * qs = ((device const uint16_t *)xb + 3);\n const float d = xb->d;\n const float md = -16.h * xb->d;\n const ushort mask = il ? 0x00F0 : 0x000F;\n\n const uint32_t qh = *((device const uint32_t *)xb->qh);\n\n const int x_mv = il ? 4 : 0;\n\n const int gh_mv = il ? 12 : 0;\n const int gh_bk = il ? 0 : 4;\n\n float4x4 reg_f;\n\n for (int i = 0; i < 8; i++) {\n // extract the 5-th bits for x0 and x1\n const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;\n const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;\n\n // combine the 4-bits from qs with the 5th bit\n const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);\n const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);\n\n reg_f[i/2][2*(i%2) + 0] = d * x0 + md;\n reg_f[i/2][2*(i%2) + 1] = d * x1 + md;\n }\n\n reg = (type4x4) reg_f;\n}\n\ntemplate \nvoid dequantize_q5_0_t4(device const block_q5_0 * xb, short il, thread type4 & reg) {\n device const uint16_t * qs = ((device const uint16_t *)xb + 3);\n const float d = xb->d;\n const float md = -16.h * xb->d;\n const ushort mask = (il/4) ? 0x00F0 : 0x000F;\n\n const uint32_t qh = *((device const uint32_t *)xb->qh);\n\n const int x_mv = (il/4) ? 4 : 0;\n\n const int gh_mv = (il/4) ? 12 : 0;\n const int gh_bk = (il/4) ? 0 : 4;\n\n for (int ii = 0; ii < 2; ii++) {\n int i = 2*(il%4) + ii;\n\n // extract the 5-th bits for x0 and x1\n const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;\n const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;\n\n // combine the 4-bits from qs with the 5th bit\n const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);\n const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);\n\n reg[2*ii + 0] = d * x0 + md;\n reg[2*ii + 1] = d * x1 + md;\n }\n}\n\ntemplate \nvoid dequantize_q5_1(device const block_q5_1 * xb, short il, thread type4x4 & reg) {\n device const uint16_t * qs = ((device const uint16_t *)xb + 4);\n const float d = xb->d;\n const float m = xb->m;\n const ushort mask = il ? 0x00F0 : 0x000F;\n\n const uint32_t qh = *((device const uint32_t *)xb->qh);\n\n const int x_mv = il ? 4 : 0;\n\n const int gh_mv = il ? 12 : 0;\n const int gh_bk = il ? 0 : 4;\n\n float4x4 reg_f;\n\n for (int i = 0; i < 8; i++) {\n // extract the 5-th bits for x0 and x1\n const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;\n const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;\n\n // combine the 4-bits from qs with the 5th bit\n const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);\n const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);\n\n reg_f[i/2][2*(i%2) + 0] = d * x0 + m;\n reg_f[i/2][2*(i%2) + 1] = d * x1 + m;\n }\n\n reg = (type4x4) reg_f;\n}\n\ntemplate \nvoid dequantize_q5_1_t4(device const block_q5_1 * xb, short il, thread type4 & reg) {\n device const uint16_t * qs = ((device const uint16_t *)xb + 4);\n const float d = xb->d;\n const float m = xb->m;\n const ushort mask = (il/4) ? 0x00F0 : 0x000F;\n\n const uint32_t qh = *((device const uint32_t *)xb->qh);\n\n const int x_mv = (il/4) ? 4 : 0;\n\n const int gh_mv = (il/4) ? 12 : 0;\n const int gh_bk = (il/4) ? 0 : 4;\n\n for (int ii = 0; ii < 2; ii++) {\n int i = 2*(il%4) + ii;\n\n // extract the 5-th bits for x0 and x1\n const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;\n const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;\n\n // combine the 4-bits from qs with the 5th bit\n const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);\n const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);\n\n reg[2*ii + 0] = d * x0 + m;\n reg[2*ii + 1] = d * x1 + m;\n }\n}\n\ntemplate \nvoid dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & reg) {\n device const int8_t * qs = ((device const int8_t *)xb->qs);\n const float d = xb->d;\n\n float4x4 reg_f;\n\n for (int i = 0; i < 16; i++) {\n reg_f[i/4][i%4] = (qs[i + 16*il] * d);\n }\n\n reg = (type4x4) reg_f;\n}\n\ntemplate \nvoid dequantize_q8_0_t4(device const block_q8_0 *xb, short il, thread type4 & reg) {\n device const int8_t * qs = ((device const int8_t *)xb->qs);\n const float d = xb->d;\n\n for (int i = 0; i < 4; i++) {\n reg[i] = (qs[4*(il%4) + i + 16*(il/4)] * d);\n }\n}\n\ntemplate \nvoid dequantize_mxfp4(device const block_mxfp4 * xb, short il, thread type4x4 & reg) {\n device const uint8_t * q2 = (device const uint8_t *)xb->qs;\n\n const float d = e8m0_to_fp32(xb->e);\n const uint8_t shr = il >= 1 ? 4 : 0;\n\n for (int i = 0; i < 4; ++i) {\n reg[i][0] = d * kvalues_mxfp4_f[(q2[4*i + 0] >> shr) & 0x0F];\n reg[i][1] = d * kvalues_mxfp4_f[(q2[4*i + 1] >> shr) & 0x0F];\n reg[i][2] = d * kvalues_mxfp4_f[(q2[4*i + 2] >> shr) & 0x0F];\n reg[i][3] = d * kvalues_mxfp4_f[(q2[4*i + 3] >> shr) & 0x0F];\n }\n}\n\ntemplate \nvoid dequantize_mxfp4_t4(device const block_mxfp4 * xb, short il, thread type4 & reg) {\n device const uint8_t * q2 = (device const uint8_t *)xb->qs;\n\n const float d = e8m0_to_fp32(xb->e);\n const short il4 = il%4;\n\n const uint8_t shr = il >= 4 ? 4 : 0;\n\n reg[0] = d * kvalues_mxfp4_f[(q2[4*il4 + 0] >> shr) & 0x0F];\n reg[1] = d * kvalues_mxfp4_f[(q2[4*il4 + 1] >> shr) & 0x0F];\n reg[2] = d * kvalues_mxfp4_f[(q2[4*il4 + 2] >> shr) & 0x0F];\n reg[3] = d * kvalues_mxfp4_f[(q2[4*il4 + 3] >> shr) & 0x0F];\n}\n\ntemplate \nvoid dequantize_q2_K(device const block_q2_K *xb, short il, thread type4x4 & reg) {\n const float d = xb->d;\n const float min = xb->dmin;\n device const uint8_t * q = (device const uint8_t *)xb->qs;\n float dl, ml;\n uint8_t sc = xb->scales[il];\n\n q = q + 32*(il/8) + 16*(il&1);\n il = (il/2)%4;\n\n half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h);\n uchar mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3);\n dl = d * (sc & 0xF) * coef, ml = min * (sc >> 4);\n for (int i = 0; i < 16; ++i) {\n reg[i/4][i%4] = dl * (q[i] & mask) - ml;\n }\n}\n\ntemplate \nvoid dequantize_q3_K(device const block_q3_K *xb, short il, thread type4x4 & reg) {\n const half d_all = xb->d;\n device const uint8_t * q = (device const uint8_t *)xb->qs;\n device const uint8_t * h = (device const uint8_t *)xb->hmask;\n device const int8_t * scales = (device const int8_t *)xb->scales;\n\n q = q + 32 * (il/8) + 16 * (il&1);\n h = h + 16 * (il&1);\n uint8_t m = 1 << (il/2);\n uint16_t kmask1 = (il/4)>1 ? ((il/4)>2 ? 192 : 48) : \\\n ((il/4)>0 ? 12 : 3);\n uint16_t kmask2 = il/8 ? 0xF0 : 0x0F;\n uint16_t scale_2 = scales[il%8], scale_1 = scales[8 + il%4];\n int16_t dl_int = (il/4)&1 ? (scale_2&kmask2) | ((scale_1&kmask1) << 2)\n : (scale_2&kmask2) | ((scale_1&kmask1) << 4);\n float dl = il<8 ? d_all * (dl_int - 32.f) : d_all * (dl_int / 16.f - 32.f);\n const float ml = 4.f * dl;\n\n il = (il/2) & 3;\n const half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h);\n const uint8_t mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3);\n dl *= coef;\n\n for (int i = 0; i < 16; ++i) {\n reg[i/4][i%4] = dl * (q[i] & mask) - (h[i] & m ? 0 : ml);\n }\n}\n\nstatic inline uchar2 get_scale_min_k4_just2(int j, int k, device const uchar * q) {\n return j < 4 ? uchar2{uchar(q[j+0+k] & 63), uchar(q[j+4+k] & 63)}\n : uchar2{uchar((q[j+4+k] & 0xF) | ((q[j-4+k] & 0xc0) >> 2)), uchar((q[j+4+k] >> 4) | ((q[j-0+k] & 0xc0) >> 2))};\n}\n\ntemplate \nvoid dequantize_q4_K(device const block_q4_K * xb, short il, thread type4x4 & reg) {\n device const uchar * q = xb->qs;\n\n short is = (il/4) * 2;\n q = q + (il/4) * 32 + 16 * (il&1);\n il = il & 3;\n const uchar2 sc = get_scale_min_k4_just2(is, il/2, xb->scales);\n const float d = il < 2 ? xb->d : xb->d / 16.h;\n const float min = xb->dmin;\n const float dl = d * sc[0];\n const float ml = min * sc[1];\n\n const ushort mask = il < 2 ? 0x0F : 0xF0;\n for (int i = 0; i < 16; ++i) {\n reg[i/4][i%4] = dl * (q[i] & mask) - ml;\n }\n}\n\ntemplate \nvoid dequantize_q5_K(device const block_q5_K *xb, short il, thread type4x4 & reg) {\n device const uint8_t * q = xb->qs;\n device const uint8_t * qh = xb->qh;\n\n short is = (il/4) * 2;\n q = q + 32 * (il/4) + 16 * (il&1);\n qh = qh + 16 * (il&1);\n uint8_t ul = 1 << (il/2);\n il = il & 3;\n const uchar2 sc = get_scale_min_k4_just2(is, il/2, xb->scales);\n const float d = il < 2 ? xb->d : xb->d / 16.f;\n const float min = xb->dmin;\n const float dl = d * sc[0];\n const float ml = min * sc[1];\n\n const ushort mask = il<2 ? 0x0F : 0xF0;\n const float qh_val = il<2 ? 16.f : 256.f;\n for (int i = 0; i < 16; ++i) {\n reg[i/4][i%4] = dl * ((q[i] & mask) + (qh[i] & ul ? qh_val : 0)) - ml;\n }\n}\n\ntemplate \nvoid dequantize_q6_K(device const block_q6_K *xb, short il, thread type4x4 & reg) {\n const half d_all = xb->d;\n device const uint16_t * ql = (device const uint16_t *)xb->ql;\n device const uint16_t * qh = (device const uint16_t *)xb->qh;\n device const int8_t * scales = (device const int8_t *)xb->scales;\n\n ql = ql + 32*(il/8) + 16*((il/2)&1) + 8*(il&1);\n qh = qh + 16*(il/8) + 8*(il&1);\n float sc = scales[(il%2) + 2 * ((il/2))];\n il = (il/2) & 3;\n\n const uint32_t kmask1 = il>1 ? (il>2 ? 0xC0C0C0C0 : 0x30303030) : (il>0 ? 0x0C0C0C0C : 0x03030303);\n const uint32_t kmask2 = il>1 ? 0xF0F0F0F0 : 0x0F0F0F0F;\n const float ml = d_all * sc * 32.f;\n const float dl0 = d_all * sc;\n const float dl1 = dl0 / 256.f;\n const float dl2 = dl0 / (256.f * 256.f);\n const float dl3 = dl0 / (256.f * 256.f * 256.f);\n const uint8_t shr_h = il>2 ? 2 : 0;\n const uint8_t shl_h = il>1 ? 0 : (il>0 ? 2 : 4);\n const uint8_t shr_l = il>1 ? 4 : 0;\n for (int i = 0; i < 4; ++i) {\n const uint32_t low = (ql[2*i] | (uint32_t)(ql[2*i+1] << 16)) & kmask2;\n const uint32_t high = (qh[2*i] | (uint32_t)(qh[2*i+1] << 16)) & kmask1;\n const uint32_t q = ((high << shl_h) >> shr_h) | (low >> shr_l);\n reg[i][0] = dl0 * ((half)(q & 0xFF)) - ml;\n reg[i][1] = dl1 * ((float)(q & 0xFF00)) - ml;\n reg[i][2] = dl2 * ((float)(q & 0xFF0000)) - ml;\n reg[i][3] = dl3 * ((float)(q & 0xFF000000)) - ml;\n }\n}\n\ntemplate \nvoid dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x4 & reg) {\n // il is 0...15 for QK_K = 256 => index of block of 32 is il/2\n const float d = xb->d;\n const int ib32 = il/2;\n il = il%2;\n // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16\n // each block of 32 needs 2 uint32_t's for the quants & scale, so 4 uint16_t's.\n device const uint16_t * q2 = xb->qs + 4*ib32;\n const uint32_t aux32_g = q2[0] | (q2[1] << 16);\n const uint32_t aux32_s = q2[2] | (q2[3] << 16);\n thread const uint8_t * aux8 = (thread const uint8_t *)&aux32_g;\n const float dl = d * (0.5f + (aux32_s >> 28)) * 0.25f;\n constant uint8_t * grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+0]);\n uint8_t signs = ksigns_iq2xs[(aux32_s >> 14*il) & 127];\n for (int i = 0; i < 8; ++i) {\n reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);\n }\n grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+1]);\n signs = ksigns_iq2xs[(aux32_s >> (14*il+7)) & 127];\n for (int i = 0; i < 8; ++i) {\n reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);\n }\n}\n\ntemplate \nvoid dequantize_iq2_xs(device const block_iq2_xs * xb, short il, thread type4x4 & reg) {\n // il is 0...15 for QK_K = 256 => index of block of 32 is il/2\n const float d = xb->d;\n const int ib32 = il/2;\n il = il%2;\n // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16\n device const uint16_t * q2 = xb->qs + 4*ib32;\n const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f;\n constant uint8_t * grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+0] & 511));\n uint8_t signs = ksigns_iq2xs[q2[2*il+0] >> 9];\n for (int i = 0; i < 8; ++i) {\n reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);\n }\n grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+1] & 511));\n signs = ksigns_iq2xs[q2[2*il+1] >> 9];\n for (int i = 0; i < 8; ++i) {\n reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);\n }\n}\n\ntemplate \nvoid dequantize_iq3_xxs(device const block_iq3_xxs * xb, short il, thread type4x4 & reg) {\n // il is 0...15 for QK_K = 256 => index of block of 32 is il/2\n const float d = xb->d;\n const int ib32 = il/2;\n il = il%2;\n // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16\n device const uint8_t * q3 = xb->qs + 8*ib32;\n device const uint16_t * gas = (device const uint16_t *)(xb->qs + QK_K/4) + 2*ib32;\n const uint32_t aux32 = gas[0] | (gas[1] << 16);\n const float dl = d * (0.5f + (aux32 >> 28)) * 0.5f;\n constant uint8_t * grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+0]);\n constant uint8_t * grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+1]);\n uint8_t signs = ksigns_iq2xs[(aux32 >> 14*il) & 127];\n for (int i = 0; i < 4; ++i) {\n reg[0][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f);\n reg[1][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f);\n }\n grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+2]);\n grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+3]);\n signs = ksigns_iq2xs[(aux32 >> (14*il+7)) & 127];\n for (int i = 0; i < 4; ++i) {\n reg[2][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f);\n reg[3][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f);\n }\n}\n\ntemplate \nvoid dequantize_iq3_s(device const block_iq3_s * xb, short il, thread type4x4 & reg) {\n // il is 0...15 for QK_K = 256 => index of block of 32 is il/2\n const float d = xb->d;\n const int ib32 = il/2;\n il = il%2;\n // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16\n device const uint8_t * qs = xb->qs + 8*ib32;\n device const uint8_t * signs = xb->signs + 4*ib32 + 2*il;\n const uint8_t qh = xb->qh[ib32] >> 4*il;\n const float dl = d * (1 + 2*((xb->scales[ib32/2] >> 4*(ib32%2)) & 0xf));\n constant uint8_t * grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+0] | ((qh << 8) & 256)));\n constant uint8_t * grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+1] | ((qh << 7) & 256)));\n for (int i = 0; i < 4; ++i) {\n reg[0][i] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i+0]);\n reg[1][i] = dl * grid2[i] * select(1, -1, signs[0] & kmask_iq2xs[i+4]);\n }\n grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+2] | ((qh << 6) & 256)));\n grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+3] | ((qh << 5) & 256)));\n for (int i = 0; i < 4; ++i) {\n reg[2][i] = dl * grid1[i] * select(1, -1, signs[1] & kmask_iq2xs[i+0]);\n reg[3][i] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i+4]);\n }\n}\n\ntemplate \nvoid dequantize_iq2_s(device const block_iq2_s * xb, short il, thread type4x4 & reg) {\n // il is 0...15 for QK_K = 256 => index of block of 32 is il/2\n const float d = xb->d;\n const int ib32 = il/2;\n il = il%2;\n // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16\n device const uint8_t * qs = xb->qs + 4*ib32 + 2*il;\n device const uint8_t * signs = qs + QK_K/8;\n const uint8_t qh = xb->qh[ib32] >> 4*il;\n const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f;\n constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[0] | ((qh << 8) & 0x300)));\n constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[1] | ((qh << 6) & 0x300)));\n for (int i = 0; i < 8; ++i) {\n reg[i/4+0][i%4] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i]);\n reg[i/4+2][i%4] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i]);\n }\n}\n\ntemplate \nvoid dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) {\n // il is 0...15 for QK_K = 256 => index of block of 32 is il/2\n const int ib32 = il/2;\n il = il%2;\n const float d = xb->d;\n device const uint8_t * qs = xb->qs + 4*ib32 + 2*il;\n device const uint16_t * qh = xb->qh;\n const float dl = d * (2*((qh[ib32] >> 12) & 7) + 1);\n const float ml = dl * (qh[ib32] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA);\n const uint16_t h = qh[ib32] >> 6*il;\n constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((h << 8) & 0x700)));\n constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((h << 5) & 0x700)));\n for (int i = 0; i < 4; ++i) {\n reg[0][i] = dl * (grid1[i] & 0xf) + ml;\n reg[1][i] = dl * (grid1[i] >> 4) + ml;\n reg[2][i] = dl * (grid2[i] & 0xf) + ml;\n reg[3][i] = dl * (grid2[i] >> 4) + ml;\n }\n}\n\ntemplate \nvoid dequantize_iq1_m(device const block_iq1_m * xb, short il, thread type4x4 & reg) {\n // il is 0...15 for QK_K = 256 => index of block of 32 is il/2\n const int ib32 = il/2;\n il = il%2;\n device const uint16_t * sc = (device const uint16_t *)xb->scales;\n\n iq1m_scale_t scale;\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n const float d = scale.f16;\n\n device const uint8_t * qs = xb->qs + 4*ib32 + 2*il;\n device const uint8_t * qh = xb->qh + 2*ib32 + il;\n\n const float dl = d * (2*((sc[ib32/2] >> (6*(ib32%2)+3*il)) & 7) + 1);\n const float ml1 = dl * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);\n const float ml2 = dl * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);\n constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));\n constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700)));\n for (int i = 0; i < 4; ++i) {\n reg[0][i] = dl * (grid1[i] & 0xf) + ml1;\n reg[1][i] = dl * (grid1[i] >> 4) + ml1;\n reg[2][i] = dl * (grid2[i] & 0xf) + ml2;\n reg[3][i] = dl * (grid2[i] >> 4) + ml2;\n }\n}\n\ntemplate \nvoid dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4 & reg) {\n device const uint16_t * q4 = (device const uint16_t *)xb->qs;\n const float d = xb->d;\n uint32_t aux32;\n thread const uint8_t * q8 = (thread const uint8_t *)&aux32;\n for (int i = 0; i < 4; ++i) {\n aux32 = ((q4[2*i] | (q4[2*i+1] << 16)) >> 4*il) & 0x0f0f0f0f;\n reg[i][0] = d * kvalues_iq4nl_f[q8[0]];\n reg[i][1] = d * kvalues_iq4nl_f[q8[1]];\n reg[i][2] = d * kvalues_iq4nl_f[q8[2]];\n reg[i][3] = d * kvalues_iq4nl_f[q8[3]];\n }\n}\n\ntemplate \nvoid dequantize_iq4_nl_t4(device const block_iq4_nl * xb, short il, thread type4 & reg) {\n device const uint16_t * q4 = (device const uint16_t *)xb->qs;\n const float d = xb->d;\n uint32_t aux32;\n thread const uint8_t * q8 = (thread const uint8_t *)&aux32;\n aux32 = ((q4[2*(il%4)] | (q4[2*(il%4)+1] << 16)) >> 4*(il/4)) & 0x0f0f0f0f;\n reg[0] = d * kvalues_iq4nl_f[q8[0]];\n reg[1] = d * kvalues_iq4nl_f[q8[1]];\n reg[2] = d * kvalues_iq4nl_f[q8[2]];\n reg[3] = d * kvalues_iq4nl_f[q8[3]];\n}\n\ntemplate \nvoid dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 & reg) {\n // il is 0...15 for QK_K = 256 => index of block of 32 is il/2\n const int ib32 = il/2;\n il = il%2;\n // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16\n device const uint32_t * q4 = (device const uint32_t *)xb->qs + 4*ib32;\n const int ls = ((xb->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((xb->scales_h >> 2*ib32) & 3) << 4);\n const float d = (float)xb->d * (ls - 32);\n uint32_t aux32;\n thread const uint8_t * q8 = (thread const uint8_t *)&aux32;\n for (int i = 0; i < 4; ++i) {\n aux32 = (q4[i] >> 4*il) & 0x0f0f0f0f;\n reg[i][0] = d * kvalues_iq4nl_f[q8[0]];\n reg[i][1] = d * kvalues_iq4nl_f[q8[1]];\n reg[i][2] = d * kvalues_iq4nl_f[q8[2]];\n reg[i][3] = d * kvalues_iq4nl_f[q8[3]];\n }\n}\n\nenum ggml_sort_order {\n GGML_SORT_ORDER_ASC,\n GGML_SORT_ORDER_DESC,\n};\n\nconstant float GELU_COEF_A = 0.044715f;\nconstant float GELU_QUICK_COEF = -1.702f;\nconstant float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;\nconstant float SQRT_2_INV = 0.70710678118654752440084436210484f;\n\n// based on Abramowitz and Stegun formula 7.1.26 or similar Hastings' approximation\n// ref: https://www.johndcook.com/blog/python_erf/\nconstant float p_erf = 0.3275911f;\nconstant float a1_erf = 0.254829592f;\nconstant float a2_erf = -0.284496736f;\nconstant float a3_erf = 1.421413741f;\nconstant float a4_erf = -1.453152027f;\nconstant float a5_erf = 1.061405429f;\n\ntemplate\ninline T erf_approx(T x) {\n T sign_x = sign(x);\n x = fabs(x);\n T t = 1.0f / (1.0f + p_erf * x);\n T y = 1.0f - (((((a5_erf * t + a4_erf) * t) + a3_erf) * t + a2_erf) * t + a1_erf) * t * exp(-x * x);\n return sign_x * y;\n}\n\ntemplate T elu_approx(T x);\n\ntemplate<> inline float elu_approx(float x) {\n return (x > 0.f) ? x : (exp(x) - 1);\n}\n\ntemplate<> inline float4 elu_approx(float4 x) {\n float4 res;\n\n res[0] = (x[0] > 0.0f) ? x[0] : (exp(x[0]) - 1.0f);\n res[1] = (x[1] > 0.0f) ? x[1] : (exp(x[1]) - 1.0f);\n res[2] = (x[2] > 0.0f) ? x[2] : (exp(x[2]) - 1.0f);\n res[3] = (x[3] > 0.0f) ? x[3] : (exp(x[3]) - 1.0f);\n\n return res;\n}\n\nconstant short FC_unary_op [[function_constant(FC_UNARY + 0)]];\nconstant bool FC_unary_cnt[[function_constant(FC_UNARY + 1)]];\n\ntemplate \nkernel void kernel_unary_impl(\n constant ggml_metal_kargs_unary & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n#define FC_OP FC_unary_op\n#define FC_CNT FC_unary_cnt\n\n device const T0 * src0_ptr;\n device T * dst_ptr;\n\n int i0;\n\n if (FC_CNT) {\n i0 = tgpig.x;\n\n src0_ptr = (device const T0 *) (src0);\n dst_ptr = (device T *) (dst);\n } else {\n const int i03 = tgpig.z;\n const int i02 = tgpig.y;\n const int k0 = tgpig.x/args.ne01;\n const int i01 = tgpig.x - k0*args.ne01;\n\n i0 = k0*ntg.x + tpitg.x;\n\n src0_ptr = (device const T0 *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01);\n dst_ptr = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1 );\n }\n\n {\n //threadgroup_barrier(mem_flags::mem_none);\n\n if (!FC_CNT) {\n if (i0 >= args.ne0) {\n return;\n }\n }\n\n const TC x = (TC) src0_ptr[i0];\n\n if (FC_OP == OP_UNARY_NUM_SCALE) {\n dst_ptr[i0] = (T) (args.scale * x + args.bias);\n }\n\n if (FC_OP == OP_UNARY_NUM_FILL) {\n dst_ptr[i0] = (T) args.val;\n }\n\n if (FC_OP == OP_UNARY_NUM_CLAMP) {\n dst_ptr[i0] = (T) clamp(x, args.min, args.max);\n }\n\n if (FC_OP == OP_UNARY_NUM_SQR) {\n dst_ptr[i0] = (T) (x * x);\n }\n\n if (FC_OP == OP_UNARY_NUM_SQRT) {\n dst_ptr[i0] = (T) sqrt(x);\n }\n\n if (FC_OP == OP_UNARY_NUM_SIN) {\n dst_ptr[i0] = (T) sin(x);\n }\n\n if (FC_OP == OP_UNARY_NUM_COS) {\n dst_ptr[i0] = (T) cos(x);\n }\n\n if (FC_OP == OP_UNARY_NUM_LOG) {\n dst_ptr[i0] = (T) log(x);\n }\n\n if (FC_OP == OP_UNARY_NUM_LEAKY_RELU) {\n dst_ptr[i0] = (T) (TC(x > 0)*x + TC(x <= 0)*(x * args.slope));\n }\n\n if (FC_OP == OP_UNARY_NUM_TANH) {\n dst_ptr[i0] = (T) precise::tanh(x);\n }\n\n if (FC_OP == OP_UNARY_NUM_RELU) {\n dst_ptr[i0] = (T) fmax(0, x);\n }\n\n if (FC_OP == OP_UNARY_NUM_SIGMOID) {\n dst_ptr[i0] = (T) (1 / (1 + exp(-x)));\n }\n\n if (FC_OP == OP_UNARY_NUM_GELU) {\n dst_ptr[i0] = (T) (0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))));\n }\n\n if (FC_OP == OP_UNARY_NUM_GELU_ERF) {\n dst_ptr[i0] = (T) (0.5*x*(1 + erf_approx(SQRT_2_INV*x)));\n }\n\n if (FC_OP == OP_UNARY_NUM_GELU_QUICK) {\n dst_ptr[i0] = (T) (x * (1/(1 + exp(GELU_QUICK_COEF*x))));\n }\n\n if (FC_OP == OP_UNARY_NUM_SILU) {\n dst_ptr[i0] = (T) (x / (1 + exp(-x)));\n }\n\n if (FC_OP == OP_UNARY_NUM_ELU) {\n dst_ptr[i0] = (T) elu_approx(x);\n }\n\n if (FC_OP == OP_UNARY_NUM_NEG) {\n dst_ptr[i0] = (T) -x;\n }\n\n if (FC_OP == OP_UNARY_NUM_ABS) {\n dst_ptr[i0] = (T) fabs(x);\n }\n\n if (FC_OP == OP_UNARY_NUM_SGN) {\n dst_ptr[i0] = T(x > 0) - T(x < 0);\n }\n\n if (FC_OP == OP_UNARY_NUM_STEP) {\n dst_ptr[i0] = T(x > 0);\n }\n\n if (FC_OP == OP_UNARY_NUM_HARDSWISH) {\n dst_ptr[i0] = (T) (x * fmax(0, fmin(1, x/6 + 0.5)));\n }\n\n if (FC_OP == OP_UNARY_NUM_HARDSIGMOID) {\n dst_ptr[i0] = (T) fmax(0, fmin(1, x/6 + 0.5));\n }\n\n if (FC_OP == OP_UNARY_NUM_EXP) {\n dst_ptr[i0] = (T) exp(x);\n }\n\n if (FC_OP == OP_UNARY_NUM_SOFTPLUS) {\n dst_ptr[i0] = (T) select(log(1 + exp(x)), x, x > 20);\n }\n\n if (FC_OP == OP_UNARY_NUM_EXPM1) {\n // TODO: precise implementation\n dst_ptr[i0] = (T) (exp(x) - 1);\n }\n }\n\n#undef FC_OP\n#undef FC_CNT\n}\n\ntypedef decltype(kernel_unary_impl) kernel_unary_t;\n\ntemplate [[host_name(\"kernel_unary_f32_f32\")]] kernel kernel_unary_t kernel_unary_impl;\ntemplate [[host_name(\"kernel_unary_f32_f32_4\")]] kernel kernel_unary_t kernel_unary_impl;\ntemplate [[host_name(\"kernel_unary_f16_f16\")]] kernel kernel_unary_t kernel_unary_impl;\ntemplate [[host_name(\"kernel_unary_f16_f16_4\")]] kernel kernel_unary_t kernel_unary_impl;\n\n// OP: 0 - add, 1 - sub, 2 - mul, 3 - div\nconstant short FC_bin_op [[function_constant(FC_BIN + 0)]];\nconstant short FC_bin_f [[function_constant(FC_BIN + 1)]];\nconstant bool FC_bin_rb [[function_constant(FC_BIN + 2)]];\nconstant bool FC_bin_cb [[function_constant(FC_BIN + 3)]];\n\ntemplate \nkernel void kernel_bin_fuse_impl(\n constant ggml_metal_kargs_bin & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n#define FC_OP FC_bin_op\n#define FC_F FC_bin_f\n#define FC_RB FC_bin_rb\n#define FC_CB FC_bin_cb\n\n if (FC_RB) {\n // row broadcast\n const uint i0 = tgpig.y*args.ne00 + tgpig.x;\n const uint i1 = FC_CB ? tgpig.x%args.ne10 : tgpig.x;\n\n device const T0 * src0_row = (device const T0 *) (src0);\n device T * dst_row = (device T *) (dst);\n\n if (FC_F == 1) {\n device const T1 * src1_row = (device const T1 *) (src1 + args.o1[0]);\n\n if (FC_OP == 0) {\n dst_row[i0] = src0_row[i0] + src1_row[i1];\n }\n\n if (FC_OP == 1) {\n dst_row[i0] = src0_row[i0] - src1_row[i1];\n }\n\n if (FC_OP == 2) {\n dst_row[i0] = src0_row[i0] * src1_row[i1];\n }\n\n if (FC_OP == 3) {\n dst_row[i0] = src0_row[i0] / src1_row[i1];\n }\n } else {\n T0 res = src0_row[i0];\n\n if (FC_OP == 0) {\n FOR_UNROLL (short j = 0; j < FC_F; ++j) {\n res += ((device const T1 *) (src1 + args.o1[j]))[i1];\n }\n }\n\n if (FC_OP == 1) {\n FOR_UNROLL (short j = 0; j < FC_F; ++j) {\n res -= ((device const T1 *) (src1 + args.o1[j]))[i1];\n }\n }\n\n if (FC_OP == 2) {\n FOR_UNROLL (short j = 0; j < FC_F; ++j) {\n res *= ((device const T1 *) (src1 + args.o1[j]))[i1];\n }\n }\n\n if (FC_OP == 3) {\n FOR_UNROLL (short j = 0; j < FC_F; ++j) {\n res /= ((device const T1 *) (src1 + args.o1[j]))[i1];\n }\n }\n\n dst_row[i0] = res;\n }\n } else {\n const int i03 = tgpig.z;\n const int i02 = tgpig.y;\n const int i01 = tgpig.x;\n\n if (i01 >= args.ne01) {\n return;\n }\n\n const int i13 = i03%args.ne13;\n const int i12 = i02%args.ne12;\n const int i11 = i01%args.ne11;\n\n device const T0 * src0_ptr = (device const T0 *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + args.offs);\n device T * dst_ptr = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1 + args.offs);\n\n if (FC_F == 1) {\n device const T1 * src1_ptr = (device const T1 *) (src1 + args.o1[0] + i13*args.nb13 + i12*args.nb12 + i11*args.nb11);\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n const int i10 = FC_CB ? i0%args.ne10 : i0;\n\n if (FC_OP == 0) {\n dst_ptr[i0] = src0_ptr[i0] + src1_ptr[i10];\n }\n\n if (FC_OP == 1) {\n dst_ptr[i0] = src0_ptr[i0] - src1_ptr[i10];\n }\n\n if (FC_OP == 2) {\n dst_ptr[i0] = src0_ptr[i0] * src1_ptr[i10];\n }\n\n if (FC_OP == 3) {\n dst_ptr[i0] = src0_ptr[i0] / src1_ptr[i10];\n }\n }\n } else {\n device const T1 * src1_ptr[8];\n FOR_UNROLL (short j = 0; j < FC_F; ++j) {\n src1_ptr[j] = (device const T1 *) (src1 + args.o1[j] + i13*args.nb13 + i12*args.nb12 + i11*args.nb11);\n }\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n const int i10 = FC_CB ? i0%args.ne10 : i0;\n\n T res = src0_ptr[i0];\n\n if (FC_OP == 0) {\n FOR_UNROLL (short j = 0; j < FC_F; ++j) {\n res += src1_ptr[j][i10];\n }\n }\n\n if (FC_OP == 1) {\n FOR_UNROLL (short j = 0; j < FC_F; ++j) {\n res -= src1_ptr[j][i10];\n }\n }\n\n if (FC_OP == 2) {\n FOR_UNROLL (short j = 0; j < FC_F; ++j) {\n res *= src1_ptr[j][i10];\n }\n }\n\n if (FC_OP == 3) {\n FOR_UNROLL (short j = 0; j < FC_F; ++j) {\n res /= src1_ptr[j][i10];\n }\n }\n\n dst_ptr[i0] = res;\n }\n }\n }\n\n#undef FC_OP\n#undef FC_F\n#undef FC_RB\n#undef FC_CB\n}\n\ntypedef decltype(kernel_bin_fuse_impl) kernel_bin_fuse_t;\n\ntemplate [[host_name(\"kernel_bin_fuse_f32_f32_f32\")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl;\ntemplate [[host_name(\"kernel_bin_fuse_f32_f32_f32_4\")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl;\n\nkernel void kernel_add_id(\n constant ggml_metal_kargs_add_id & args,\n device const char * src0,\n device const char * src1,\n device const char * src2,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int i1 = tgpig.x;\n const int i2 = tgpig.y;\n\n const int i11 = *((device const int32_t *) (src2 + i1*sizeof(int32_t) + i2*args.nb21));\n\n const size_t nb1 = args.ne0 * sizeof(float);\n const size_t nb2 = args.ne1 * nb1;\n\n device float * dst_row = (device float *)((device char *)dst + i1*nb1 + i2*nb2);\n device const float * src0_row = (device const float *)((device char *)src0 + i1*args.nb01 + i2*args.nb02);\n device const float * src1_row = (device const float *)((device char *)src1 + i11*args.nb11);\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n dst_row[i0] = src0_row[i0] + src1_row[i0];\n }\n}\n\ntemplate\nkernel void kernel_repeat(\n constant ggml_metal_kargs_repeat & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int i3 = tgpig.z;\n const int i2 = tgpig.y;\n const int i1 = tgpig.x;\n\n const int i03 = i3%args.ne03;\n const int i02 = i2%args.ne02;\n const int i01 = i1%args.ne01;\n\n device const char * src0_ptr = src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01;\n device char * dst_ptr = dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1;\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n const int i00 = i0%args.ne00;\n *((device T *)(dst_ptr + i0*args.nb0)) = *((device T *)(src0_ptr + i00*args.nb00));\n }\n}\n\ntypedef decltype(kernel_repeat) kernel_repeat_t;\n\ntemplate [[host_name(\"kernel_repeat_f32\")]] kernel kernel_repeat_t kernel_repeat;\ntemplate [[host_name(\"kernel_repeat_f16\")]] kernel kernel_repeat_t kernel_repeat;\ntemplate [[host_name(\"kernel_repeat_i32\")]] kernel kernel_repeat_t kernel_repeat;\ntemplate [[host_name(\"kernel_repeat_i16\")]] kernel kernel_repeat_t kernel_repeat;\n\nkernel void kernel_reglu_f32(\n constant ggml_metal_kargs_glu & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint tgpig[[threadgroup_position_in_grid]],\n uint tpitg[[thread_position_in_threadgroup]],\n uint ntg[[threads_per_threadgroup]]) {\n device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;\n device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;\n device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);\n\n for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n dst_row[i0] = x0*x1*(x0 > 0.0f);\n }\n}\n\nkernel void kernel_geglu_f32(\n constant ggml_metal_kargs_glu & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint tgpig[[threadgroup_position_in_grid]],\n uint tpitg[[thread_position_in_threadgroup]],\n uint ntg[[threads_per_threadgroup]]) {\n device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;\n device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;\n device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);\n\n for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n const float gelu = 0.5f*x0*(1.0f + precise::tanh(SQRT_2_OVER_PI*x0*(1.0f + GELU_COEF_A*x0*x0)));\n\n dst_row[i0] = gelu*x1;\n }\n}\n\nkernel void kernel_swiglu_f32(\n constant ggml_metal_kargs_glu & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint tgpig[[threadgroup_position_in_grid]],\n uint tpitg[[thread_position_in_threadgroup]],\n uint ntg[[threads_per_threadgroup]]) {\n device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;\n device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;\n device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);\n\n for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n const float silu = x0 / (1.0f + exp(-x0));\n\n dst_row[i0] = silu*x1;\n }\n}\n\nkernel void kernel_swiglu_oai_f32(\n constant ggml_metal_kargs_glu & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint tgpig[[threadgroup_position_in_grid]],\n uint tpitg[[thread_position_in_threadgroup]],\n uint ntg[[threads_per_threadgroup]]) {\n device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;\n device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;\n device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);\n\n for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {\n float x0 = src0_row[i0];\n float x1 = src1_row[i0];\n\n x0 = min(x0, args.limit);\n x1 = max(min(x1, args.limit), -args.limit);\n\n float out_glu = x0 / (1.0f + exp(-x0 * args.alpha));\n out_glu = out_glu * (1.0f + x1);\n\n dst_row[i0] = out_glu;\n }\n}\n\nkernel void kernel_geglu_erf_f32(\n constant ggml_metal_kargs_glu & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint tgpig[[threadgroup_position_in_grid]],\n uint tpitg[[thread_position_in_threadgroup]],\n uint ntg[[threads_per_threadgroup]]) {\n device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;\n device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;\n device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);\n\n for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n const float gelu_erf = 0.5f*x0*(1.0f+erf_approx(x0*SQRT_2_INV));\n\n dst_row[i0] = gelu_erf*x1;\n }\n}\n\nkernel void kernel_geglu_quick_f32(\n constant ggml_metal_kargs_glu & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint tgpig[[threadgroup_position_in_grid]],\n uint tpitg[[thread_position_in_threadgroup]],\n uint ntg[[threads_per_threadgroup]]) {\n device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;\n device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;\n device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);\n\n for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n const float gelu_quick = x0*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x0)));\n\n dst_row[i0] = gelu_quick*x1;\n }\n}\n\nkernel void kernel_op_sum_f32(\n constant ggml_metal_kargs_sum & args,\n device const float * src0,\n device float * dst,\n threadgroup float * shmem_f32 [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n\n if (args.np == 0) {\n return;\n }\n\n // TODO: become function constant\n const uint nsg = (ntg.x + 31) / 32;\n\n float sumf = 0;\n\n for (uint64_t i0 = tpitg.x; i0 < args.np; i0 += ntg.x) {\n sumf += src0[i0];\n }\n\n sumf = simd_sum(sumf);\n\n if (tiisg == 0) {\n shmem_f32[sgitg] = sumf;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n float total = 0;\n\n if (sgitg == 0) {\n float v = 0;\n\n if (tpitg.x < nsg) {\n v = shmem_f32[tpitg.x];\n }\n\n total = simd_sum(v);\n\n if (tpitg.x == 0) {\n dst[0] = total;\n }\n }\n}\n\nconstant short FC_sum_rows_op [[function_constant(FC_SUM_ROWS + 0)]];\n\ntemplate \nkernel void kernel_sum_rows_impl(\n constant ggml_metal_kargs_sum_rows & args,\n device const char * src0,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n#define FC_OP FC_sum_rows_op\n\n const int i3 = tgpig.z;\n const int i2 = tgpig.y;\n const int i1 = tgpig.x;\n\n threadgroup T0 * shmem_t = (threadgroup T0 *) shmem;\n\n if (sgitg == 0) {\n shmem_t[tiisg] = 0.0f;\n }\n\n device const T0 * src_row = (device const T0 *) (src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03);\n device T * dst_row = (device T *) (dst + i1*args.nb1 + i2*args.nb2 + i3*args.nb3);\n\n T0 sumf = T0(0.0f);\n\n for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) {\n sumf += src_row[i0];\n }\n\n sumf = simd_sum(sumf);\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n shmem_t[sgitg] = sumf;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n sumf = shmem_t[tiisg];\n sumf = simd_sum(sumf);\n\n if (tpitg.x == 0) {\n if (FC_OP == OP_SUM_ROWS_NUM_MEAN) {\n if (is_same::value) {\n dst_row[0] = sum(sumf) / (4*args.ne00);\n } else {\n dst_row[0] = sum(sumf) / args.ne00;\n }\n } else {\n dst_row[0] = sum(sumf);\n }\n }\n\n#undef FC_OP\n}\n\ntypedef decltype(kernel_sum_rows_impl) kernel_sum_rows_t;\n\ntemplate [[host_name(\"kernel_sum_rows_f32_f32\")]] kernel kernel_sum_rows_t kernel_sum_rows_impl;\ntemplate [[host_name(\"kernel_sum_rows_f32_f32_4\")]] kernel kernel_sum_rows_t kernel_sum_rows_impl;\n\ntemplate\nkernel void kernel_cumsum_blk(\n constant ggml_metal_kargs_cumsum_blk & args,\n device const char * src0,\n device char * tmp,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int ib = tgpig[0]/args.ne01;\n\n const int i00 = ib*ntg.x;\n const int i01 = tgpig[0]%args.ne01;\n const int i02 = tgpig[1];\n const int i03 = tgpig[2];\n\n device const float * src0_row = (device const float *) (src0 +\n args.nb01*i01 +\n args.nb02*i02 +\n args.nb03*i03);\n\n threadgroup float * shmem_f32 = (threadgroup float *) shmem;\n\n float v = 0.0f;\n\n if (i00 + tpitg.x < args.ne00) {\n v = src0_row[i00 + tpitg.x];\n }\n\n float s = simd_prefix_inclusive_sum(v);\n\n if (tiisg == N_SIMDWIDTH - 1) {\n shmem_f32[sgitg] = s;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (sgitg == 0) {\n shmem_f32[tiisg] = simd_prefix_exclusive_sum(shmem_f32[tiisg]);\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n s += shmem_f32[sgitg];\n\n device float * dst_row = (device float *) dst +\n args.ne00*i01 +\n args.ne00*args.ne01*i02 +\n args.ne00*args.ne01*args.ne02*i03;\n\n if (i00 + tpitg.x < args.ne00) {\n dst_row[i00 + tpitg.x] = s;\n }\n\n if (args.outb && tpitg.x == ntg.x - 1) {\n device float * tmp_row = (device float *) tmp +\n args.net0*i01 +\n args.net0*args.net1*i02 +\n args.net0*args.net1*args.net2*i03;\n\n tmp_row[ib] = s;\n }\n}\n\ntypedef decltype(kernel_cumsum_blk) kernel_cumsum_blk_t;\n\ntemplate [[host_name(\"kernel_cumsum_blk_f32\")]] kernel kernel_cumsum_blk_t kernel_cumsum_blk;\n\ntemplate\nkernel void kernel_cumsum_add(\n constant ggml_metal_kargs_cumsum_add & args,\n device const char * tmp,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int ib = tgpig[0]/args.ne01;\n\n if (ib == 0) {\n return;\n }\n\n const int i00 = ib*ntg.x;\n const int i01 = tgpig[0]%args.ne01;\n const int i02 = tgpig[1];\n const int i03 = tgpig[2];\n\n device const float * tmp_row = (device const float *) (tmp +\n args.nbt1*i01 +\n args.nbt2*i02 +\n args.nbt3*i03);\n\n device float * dst_row = (device float *) dst +\n args.ne00*i01 +\n args.ne00*args.ne01*i02 +\n args.ne00*args.ne01*args.ne02*i03;\n\n if (i00 + tpitg.x < args.ne00) {\n dst_row[i00 + tpitg.x] += tmp_row[ib - 1];\n }\n}\n\ntypedef decltype(kernel_cumsum_add) kernel_cumsum_add_t;\n\ntemplate [[host_name(\"kernel_cumsum_add_f32\")]] kernel kernel_cumsum_add_t kernel_cumsum_add;\n\n\ntemplate\nbool _ggml_vec_tri_cmp(const int i, const int r);\n\ntemplate<>\nbool _ggml_vec_tri_cmp(const int i, const int r) {\n return i < r;\n}\n\ntemplate<>\nbool _ggml_vec_tri_cmp(const int i, const int r) {\n return i <= r;\n}\n\ntemplate<>\nbool _ggml_vec_tri_cmp(const int i, const int r) {\n return i > r;\n}\n\ntemplate<>\nbool _ggml_vec_tri_cmp(const int i, const int r) {\n return i >= r;\n}\n\ntemplate\nkernel void kernel_tri(\n constant ggml_metal_kargs_tri & args,\n device const char * src0,\n device const char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int i3 = tgpig.z;\n const int i2 = tgpig.y;\n const int i1 = tgpig.x;\n\n if (i3 >= args.ne03 || i2 >= args.ne02 || i1 >= args.ne01) {\n return;\n }\n\n device const T * src_row = (device const T *) ((device const char *) src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03);\n device T * dst_row = (device T *) ((device char *) dst + i1*args.nb1 + i2*args.nb2 + i3*args.nb3);\n\n // Each thread is a single element of the row if ne00 < max threads per\n // threadgroup, so this will loop once for each index that this thread is\n // responsible for\n for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) {\n // Use the comparison as a mask for branchless\n dst_row[i0] = static_cast(_ggml_vec_tri_cmp(i0, i1)) * src_row[i0];\n }\n}\n\ntypedef decltype(kernel_tri) kernel_tri_t;\n\ntemplate [[host_name(\"kernel_tri_f32_0\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_f32_1\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_f32_2\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_f32_3\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_f16_0\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_f16_1\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_f16_2\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_f16_3\")]] kernel kernel_tri_t kernel_tri;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_tri_bf16_0\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_bf16_1\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_bf16_2\")]] kernel kernel_tri_t kernel_tri;\ntemplate [[host_name(\"kernel_tri_bf16_3\")]] kernel kernel_tri_t kernel_tri;\n#endif\n\ntemplate\nkernel void kernel_soft_max(\n constant ggml_metal_kargs_soft_max & args,\n device const char * src0,\n device const char * src1,\n device const char * src2,\n device char * dst,\n threadgroup float * buf [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint sgitg[[simdgroup_index_in_threadgroup]],\n uint tiisg[[thread_index_in_simdgroup]],\n uint3 tptg[[threads_per_threadgroup]]) {\n const int32_t i03 = tgpig.z;\n const int32_t i02 = tgpig.y;\n const int32_t i01 = tgpig.x;\n\n const int32_t i13 = i03%args.ne13;\n const int32_t i12 = i02%args.ne12;\n const int32_t i11 = i01;\n\n device const float * psrc0 = (device const float *) (src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);\n device const T * pmask = src1 != src0 ? (device const T * ) (src1 + i11*args.nb11 + i12*args.nb12 + i13*args.nb13) : nullptr;\n device const float * psrc2 = src2 != src0 ? (device const float *) (src2) : nullptr;\n device float * pdst = (device float *) (dst + i01*args.nb1 + i02*args.nb2 + i03*args.nb3);\n\n float slope = 1.0f;\n\n // ALiBi\n if (args.max_bias > 0.0f) {\n const int32_t h = i02;\n\n const float base = h < args.n_head_log2 ? args.m0 : args.m1;\n const int exp = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1;\n\n slope = pow(base, exp);\n }\n\n // parallel max\n float lmax = psrc2 ? psrc2[i02] : -INFINITY;\n\n for (int i00 = tpitg.x; i00 < args.ne00; i00 += tptg.x) {\n lmax = MAX(lmax, psrc0[i00]*args.scale + (pmask ? slope*pmask[i00] : 0.0f));\n }\n\n // find the max value in the block\n float max_val = simd_max(lmax);\n if (tptg.x > N_SIMDWIDTH) {\n if (sgitg == 0) {\n buf[tiisg] = -INFINITY;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n buf[sgitg] = max_val;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n max_val = buf[tiisg];\n max_val = simd_max(max_val);\n }\n\n // parallel sum\n float lsum = 0.0f;\n for (int i00 = tpitg.x; i00 < args.ne00; i00 += tptg.x) {\n const float exp_psrc0 = exp((psrc0[i00]*args.scale + (pmask ? slope*pmask[i00] : 0.0f)) - max_val);\n lsum += exp_psrc0;\n pdst[i00] = exp_psrc0;\n }\n\n // This barrier fixes a failing test\n // ref: https://github.com/ggml-org/ggml/pull/621#discussion_r1425156335\n threadgroup_barrier(mem_flags::mem_none);\n\n float sum = simd_sum(lsum);\n\n if (tptg.x > N_SIMDWIDTH) {\n if (sgitg == 0) {\n buf[tiisg] = 0.0f;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n buf[sgitg] = sum;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n sum = buf[tiisg];\n sum = simd_sum(sum);\n }\n\n if (psrc2) {\n sum += exp(psrc2[i02] - max_val);\n }\n\n const float inv_sum = 1.0f/sum;\n\n for (int i00 = tpitg.x; i00 < args.ne00; i00 += tptg.x) {\n pdst[i00] *= inv_sum;\n }\n}\n\ntemplate\nkernel void kernel_soft_max_4(\n constant ggml_metal_kargs_soft_max & args,\n device const char * src0,\n device const char * src1,\n device const char * src2,\n device char * dst,\n threadgroup float * buf [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint sgitg[[simdgroup_index_in_threadgroup]],\n uint tiisg[[thread_index_in_simdgroup]],\n uint3 tptg[[threads_per_threadgroup]]) {\n const int32_t i03 = tgpig.z;\n const int32_t i02 = tgpig.y;\n const int32_t i01 = tgpig.x;\n\n const int32_t i13 = i03%args.ne13;\n const int32_t i12 = i02%args.ne12;\n const int32_t i11 = i01;\n\n device const float4 * psrc4 = (device const float4 *) (src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);\n device const T * pmask = src1 != src0 ? (device const T * ) (src1 + i11*args.nb11 + i12*args.nb12 + i13*args.nb13) : nullptr;\n device const float * psrc2 = src2 != src0 ? (device const float * ) (src2) : nullptr;\n device float4 * pdst4 = (device float4 *) (dst + i01*args.nb1 + i02*args.nb2 + i03*args.nb3);\n\n float slope = 1.0f;\n\n if (args.max_bias > 0.0f) {\n const int32_t h = i02;\n\n const float base = h < args.n_head_log2 ? args.m0 : args.m1;\n const int exp = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1;\n\n slope = pow(base, exp);\n }\n\n // parallel max\n float4 lmax4 = psrc2 ? psrc2[i02] : -INFINITY;\n\n for (int i00 = tpitg.x; i00 < args.ne00/4; i00 += tptg.x) {\n lmax4 = fmax(lmax4, psrc4[i00]*args.scale + (float4)((pmask ? slope*pmask[i00] : 0.0f)));\n }\n\n const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));\n\n float max_val = simd_max(lmax);\n if (tptg.x > N_SIMDWIDTH) {\n if (sgitg == 0) {\n buf[tiisg] = -INFINITY;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n buf[sgitg] = max_val;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n max_val = buf[tiisg];\n max_val = simd_max(max_val);\n }\n\n // parallel sum\n float4 lsum4 = 0.0f;\n for (int i00 = tpitg.x; i00 < args.ne00/4; i00 += tptg.x) {\n const float4 exp_psrc4 = exp((psrc4[i00]*args.scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))) - max_val);\n lsum4 += exp_psrc4;\n pdst4[i00] = exp_psrc4;\n }\n\n const float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3];\n\n // This barrier fixes a failing test\n // ref: https://github.com/ggml-org/ggml/pull/621#discussion_r1425156335\n threadgroup_barrier(mem_flags::mem_none);\n\n float sum = simd_sum(lsum);\n\n if (tptg.x > N_SIMDWIDTH) {\n if (sgitg == 0) {\n buf[tiisg] = 0.0f;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n buf[sgitg] = sum;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n sum = buf[tiisg];\n sum = simd_sum(sum);\n }\n\n if (psrc2) {\n sum += exp(psrc2[i02] - max_val);\n }\n\n const float inv_sum = 1.0f/sum;\n\n for (int i00 = tpitg.x; i00 < args.ne00/4; i00 += tptg.x) {\n pdst4[i00] *= inv_sum;\n }\n}\n\ntypedef decltype(kernel_soft_max) kernel_soft_max_t;\ntypedef decltype(kernel_soft_max_4) kernel_soft_max_4_t;\n\ntemplate [[host_name(\"kernel_soft_max_f16\")]] kernel kernel_soft_max_t kernel_soft_max;\ntemplate [[host_name(\"kernel_soft_max_f32\")]] kernel kernel_soft_max_t kernel_soft_max;\ntemplate [[host_name(\"kernel_soft_max_f16_4\")]] kernel kernel_soft_max_4_t kernel_soft_max_4;\ntemplate [[host_name(\"kernel_soft_max_f32_4\")]] kernel kernel_soft_max_4_t kernel_soft_max_4;\n\n// ref: ggml.c:ggml_compute_forward_ssm_conv_f32\nkernel void kernel_ssm_conv_f32_f32(\n constant ggml_metal_kargs_ssm_conv & args,\n device const void * src0,\n device const void * src1,\n device float * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n const int64_t ir = tgpig.x;\n const int64_t i2 = tgpig.y;\n const int64_t i3 = tgpig.z;\n\n const int64_t nc = args.ne10;\n //const int64_t ncs = args.ne00;\n //const int64_t nr = args.ne01;\n //const int64_t n_t = args.ne1;\n //const int64_t n_s = args.ne2;\n\n device const float * s = (device const float *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02);\n device const float * c = (device const float *) ((device const char *) src1 + ir*args.nb11);\n device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2);\n\n float sumf = 0.0f;\n\n for (int64_t i0 = 0; i0 < nc; ++i0) {\n sumf += s[i0] * c[i0];\n }\n\n x[0] = sumf;\n}\n\nkernel void kernel_ssm_conv_f32_f32_4(\n constant ggml_metal_kargs_ssm_conv & args,\n device const void * src0,\n device const void * src1,\n device float * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n const int64_t ir = tgpig.x;\n const int64_t i2 = tgpig.y;\n const int64_t i3 = tgpig.z;\n\n const int64_t nc = args.ne10;\n //const int64_t ncs = args.ne00;\n //const int64_t nr = args.ne01;\n //const int64_t n_t = args.ne1;\n //const int64_t n_s = args.ne2;\n\n device const float4 * s = (device const float4 *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02);\n device const float4 * c = (device const float4 *) ((device const char *) src1 + ir*args.nb11);\n device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2);\n\n float sumf = 0.0f;\n\n for (int64_t i0 = 0; i0 < nc/4; ++i0) {\n sumf += dot(s[i0], c[i0]);\n }\n\n x[0] = sumf;\n}\n\nconstant short FC_ssm_conv_bs [[function_constant(FC_SSM_CONV + 0)]];\n\n// Batched version: each threadgroup processes multiple tokens for better efficiency\n// Thread layout: each thread handles one token, threadgroup covers BATCH_SIZE tokens\nkernel void kernel_ssm_conv_f32_f32_batched(\n constant ggml_metal_kargs_ssm_conv & args,\n device const void * src0,\n device const void * src1,\n device float * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n // tgpig.x = row index (ir)\n // tgpig.y = batch of tokens (i2_base / BATCH_SIZE)\n // tgpig.z = sequence index (i3)\n // tpitg.x = thread within batch (0..BATCH_SIZE-1)\n const short BATCH_SIZE = FC_ssm_conv_bs;\n\n const int64_t ir = tgpig.x;\n const int64_t i2_base = tgpig.y * BATCH_SIZE;\n const int64_t i3 = tgpig.z;\n const int64_t i2_off = tpitg.x;\n const int64_t i2 = i2_base + i2_off;\n\n const int64_t nc = args.ne10; // conv kernel size (typically 4)\n const int64_t n_t = args.ne1; // number of tokens\n\n // Bounds check for partial batches at the end\n if (i2 >= n_t) {\n return;\n }\n\n // Load conv weights (shared across all tokens for this row)\n device const float * c = (device const float *) ((device const char *) src1 + ir*args.nb11);\n\n // Load source for this specific token\n device const float * s = (device const float *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02);\n\n // Output location for this token\n device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2);\n\n float sumf = 0.0f;\n for (int64_t i0 = 0; i0 < nc; ++i0) {\n sumf += s[i0] * c[i0];\n }\n\n x[0] = sumf;\n}\n\nkernel void kernel_ssm_conv_f32_f32_batched_4(\n constant ggml_metal_kargs_ssm_conv & args,\n device const void * src0,\n device const void * src1,\n device float * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n // tgpig.x = row index (ir)\n // tgpig.y = batch of tokens (i2_base / BATCH_SIZE)\n // tgpig.z = sequence index (i3)\n // tpitg.x = thread within batch (0..BATCH_SIZE-1)\n const short BATCH_SIZE = FC_ssm_conv_bs;\n\n const int64_t ir = tgpig.x;\n const int64_t i2_base = tgpig.y * BATCH_SIZE;\n const int64_t i3 = tgpig.z;\n const int64_t i2_off = tpitg.x;\n const int64_t i2 = i2_base + i2_off;\n\n const int64_t nc = args.ne10; // conv kernel size (typically 4)\n const int64_t n_t = args.ne1; // number of tokens\n\n // Bounds check for partial batches at the end\n if (i2 >= n_t) {\n return;\n }\n\n // Load conv weights (shared across all tokens for this row)\n device const float4 * c = (device const float4 *) ((device const char *) src1 + ir*args.nb11);\n\n // Load source for this specific token\n device const float4 * s = (device const float4 *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02);\n\n // Output location for this token\n device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2);\n\n float sumf = 0.0f;\n for (int64_t i0 = 0; i0 < nc/4; ++i0) {\n sumf += dot(s[i0], c[i0]);\n }\n\n x[0] = sumf;\n}\n\n// ref: ggml.c:ggml_compute_forward_ssm_scan_f32, Mamba-2 part\n// Optimized version: reduces redundant memory loads by having one thread load shared values\nkernel void kernel_ssm_scan_f32(\n constant ggml_metal_kargs_ssm_scan & args,\n device const void * src0,\n device const void * src1,\n device const void * src2,\n device const void * src3,\n device const void * src4,\n device const void * src5,\n device const void * src6,\n device float * dst,\n threadgroup float * shared [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgptg[[simdgroups_per_threadgroup]],\n uint3 tgpg[[threadgroups_per_grid]]) {\n constexpr short NW = N_SIMDWIDTH;\n\n // Shared memory layout:\n // [0..sgptg*NW-1]: partial sums for reduction (existing)\n // [sgptg*NW..sgptg*NW+sgptg-1]: pre-computed x_dt values for each token in batch\n // [sgptg*NW+sgptg..sgptg*NW+2*sgptg-1]: pre-computed dA values for each token in batch\n threadgroup float * shared_sums = shared;\n threadgroup float * shared_x_dt = shared + sgptg * NW;\n threadgroup float * shared_dA = shared + sgptg * NW + sgptg;\n\n shared_sums[tpitg.x] = 0.0f;\n\n const int32_t i0 = tpitg.x;\n const int32_t i1 = tgpig.x;\n const int32_t ir = tgpig.y; // current head\n const int32_t i3 = tgpig.z; // current seq\n\n const int32_t nc = args.d_state;\n const int32_t nr = args.d_inner;\n const int32_t nh = args.n_head;\n const int32_t ng = args.n_group;\n const int32_t n_t = args.n_seq_tokens;\n\n const int32_t s_off = args.s_off;\n\n device const int32_t * ids = (device const int32_t *) src6;\n\n device const float * s0_buff = (device const float *) ((device const char *) src0 + ir*args.nb02 + ids[i3]*args.nb03);\n device float * s_buff = (device float *) ((device char *) dst + ir*args.nb02 + i3*args.nb03 + s_off);\n\n const int32_t i = i0 + i1*nc;\n const int32_t g = ir / (nh / ng); // repeat_interleave\n\n float s0 = s0_buff[i];\n float s = 0.0f;\n\n device const float * A = (device const float *) ((device const char *) src3 + ir*args.nb31); // {ne30, nh}\n\n const float A0 = A[i0%args.ne30];\n\n device const float * x = (device const float *)((device const char *) src1 + i1*args.nb10 + ir*args.nb11 + i3*args.nb13); // {dim, nh, nt, ns}\n device const float * dt = (device const float *)((device const char *) src2 + ir*args.nb20 + i3*args.nb22); // {nh, nt, ns}\n device const float * B = (device const float *)((device const char *) src4 + g*args.nb41 + i3*args.nb43); // {d_state, ng, nt, ns}\n device const float * C = (device const float *)((device const char *) src5 + g*args.nb51 + i3*args.nb53); // {d_state, ng, nt, ns}\n\n device float * y = dst + (i1 + ir*(nr) + i3*(n_t*nh*nr)); // {dim, nh, nt, ns}\n\n for (int i2 = 0; i2 < n_t; i2 += sgptg) {\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n // Pre-compute x_dt and dA for this batch of tokens\n // Only first sgptg threads do the loads and expensive math\n if (i0 < sgptg && i2 + i0 < n_t) {\n // ns12 and ns21 are element strides (nb12/nb10, nb21/nb20)\n device const float * x_t = x + i0 * args.ns12;\n device const float * dt_t = dt + i0 * args.ns21;\n\n const float dt0 = dt_t[0];\n const float dtsp = dt0 <= 20.0f ? log(1.0f + exp(dt0)) : dt0;\n shared_x_dt[i0] = x_t[0] * dtsp;\n shared_dA[i0] = dtsp; // Store dtsp, compute exp(dtsp * A0) per-thread since A0 varies\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n for (int t = 0; t < sgptg && i2 + t < n_t; t++) {\n const float x_dt = shared_x_dt[t];\n const float dA = exp(shared_dA[t] * A0);\n\n s = (s0 * dA) + (B[i0] * x_dt);\n\n const float sumf = simd_sum(s * C[i0]);\n\n if (tiisg == 0) {\n shared_sums[t*NW + sgitg] = sumf;\n }\n\n // recurse\n s0 = s;\n\n B += args.ns42;\n C += args.ns52;\n }\n\n // Advance pointers for next batch\n x += sgptg * args.ns12;\n dt += sgptg * args.ns21;\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n const float sumf = simd_sum(shared_sums[sgitg*NW + tiisg]);\n\n if (tiisg == 0 && i2 + sgitg < n_t) {\n y[sgitg*nh*nr] = sumf;\n }\n\n y += sgptg*nh*nr;\n }\n\n s_buff[i] = s;\n}\n\nkernel void kernel_rwkv_wkv6_f32(\n device const float * k,\n device const float * v,\n device const float * r,\n device const float * tf,\n device const float * td,\n device const float * state_in,\n device float * dst,\n constant uint & B,\n constant uint & T,\n constant uint & C,\n constant uint & H,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n const uint head_size = 64; // TODO: support head_size = 128\n const uint batch_id = tgpig.x / H;\n const uint head_id = tgpig.x % H;\n const uint tid = tpitg.x;\n\n if (batch_id >= B || head_id >= H) {\n return;\n }\n\n const uint state_size = C * head_size;\n const uint n_seq_tokens = T / B;\n\n threadgroup float _k[head_size];\n threadgroup float _r[head_size];\n threadgroup float _tf[head_size];\n threadgroup float _td[head_size];\n\n float state[head_size];\n\n for (uint i = 0; i < head_size; i++) {\n state[i] = state_in[batch_id * state_size + head_id * head_size * head_size\n + i * head_size + tid];\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n _tf[tid] = tf[head_id * head_size + tid];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid;\n const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid;\n\n for (uint t = start_t; t < end_t; t += C) {\n threadgroup_barrier(mem_flags::mem_threadgroup);\n _k[tid] = k[t];\n _r[tid] = r[t];\n _td[tid] = td[t];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n const float v_val = v[t];\n float y = 0.0;\n\n for (uint j = 0; j < head_size; j += 4) {\n float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]);\n float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]);\n float4 tf_vec = float4(_tf[j], _tf[j+1], _tf[j+2], _tf[j+3]);\n float4 td_vec = float4(_td[j], _td[j+1], _td[j+2], _td[j+3]);\n float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]);\n\n float4 kv = k_vec * v_val;\n\n float4 temp = tf_vec * kv + s_vec;\n y += dot(r_vec, temp);\n\n s_vec = s_vec * td_vec + kv;\n state[j] = s_vec[0];\n state[j+1] = s_vec[1];\n state[j+2] = s_vec[2];\n state[j+3] = s_vec[3];\n }\n\n dst[t] = y;\n }\n\n for (uint i = 0; i < head_size; i++) {\n dst[T * C + batch_id * state_size + head_id * head_size * head_size\n + i * head_size + tid] = state[i];\n }\n}\n\nkernel void kernel_rwkv_wkv7_f32(\n device const float * r,\n device const float * w,\n device const float * k,\n device const float * v,\n device const float * a,\n device const float * b,\n device const float * state_in,\n device float * dst,\n constant uint & B,\n constant uint & T,\n constant uint & C,\n constant uint & H,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n const uint head_size = 64; // TODO: support head_size = 128\n const uint batch_id = tgpig.x / H;\n const uint head_id = tgpig.x % H;\n const uint tid = tpitg.x;\n\n if (batch_id >= B || head_id >= H) {\n return;\n }\n\n const uint state_size = C * head_size;\n const uint n_seq_tokens = T / B;\n\n threadgroup float _r[head_size];\n threadgroup float _w[head_size];\n threadgroup float _k[head_size];\n threadgroup float _a[head_size];\n threadgroup float _b[head_size];\n\n float state[head_size];\n\n for (uint i = 0; i < head_size; i++) {\n state[i] = state_in[batch_id * state_size + head_id * head_size * head_size\n + tid * head_size + i];\n }\n\n const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid;\n const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid;\n\n for (uint t = start_t; t < end_t; t += C) {\n threadgroup_barrier(mem_flags::mem_threadgroup);\n _r[tid] = r[t];\n _w[tid] = w[t];\n _k[tid] = k[t];\n _a[tid] = a[t];\n _b[tid] = b[t];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n const float v_val = v[t];\n float y = 0.0, sa = 0.0;\n\n float4 sa_vec(0.0);\n\n for (uint j = 0; j < head_size; j += 4) {\n float4 a_vec = float4(_a[j], _a[j+1], _a[j+2], _a[j+3]);\n float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]);\n sa_vec += a_vec * s_vec;\n }\n sa = sa_vec[0] + sa_vec[1] + sa_vec[2] + sa_vec[3];\n\n for (uint j = 0; j < head_size; j += 4) {\n float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]);\n float4 w_vec = float4(_w[j], _w[j+1], _w[j+2], _w[j+3]);\n float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]);\n float4 b_vec = float4(_b[j], _b[j+1], _b[j+2], _b[j+3]);\n float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]);\n\n float4 kv = k_vec * v_val;\n\n s_vec = s_vec * w_vec + kv + sa * b_vec;\n y += dot(s_vec, r_vec);\n\n state[j] = s_vec[0];\n state[j+1] = s_vec[1];\n state[j+2] = s_vec[2];\n state[j+3] = s_vec[3];\n }\n\n dst[t] = y;\n }\n\n for (uint i = 0; i < head_size; i++) {\n dst[T * C + batch_id * state_size + head_id * head_size * head_size\n + tid * head_size + i] = state[i];\n }\n}\n\nconstant short FC_gated_delta_net_ne20 [[function_constant(FC_GATED_DELTA_NET + 0)]];\nconstant short FC_gated_delta_net_ne30 [[function_constant(FC_GATED_DELTA_NET + 1)]];\n\n#if 1\ntemplate\nkernel void kernel_gated_delta_net_impl(\n constant ggml_metal_kargs_gated_delta_net & args,\n device const char * q,\n device const char * k,\n device const char * v,\n device const char * g,\n device const char * b,\n device const char * s,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n#define S_v FC_gated_delta_net_ne20\n#define G FC_gated_delta_net_ne30\n\n const uint tx = tpitg.x;\n const uint ty = tpitg.y;\n\n const uint i23 = tgpig.z; // B\n const uint i21 = tgpig.y; // H\n const uint i20 = tgpig.x*NSG + ty;\n\n const uint i01 = i21 % args.ne01;\n const uint i11 = i21 % args.ne11;\n\n const float scale = 1.0f / sqrt((float)S_v);\n\n // state is stored transposed: M[i20][is] = S[is][i20], so row i20 is contiguous\n device const float * s_ptr = (device const float *) (s) + (i23*args.ne21 + i21)*S_v*S_v + i20*S_v;\n\n float ls[NSG];\n\n FOR_UNROLL (short j = 0; j < NSG; j++) {\n const short is = tx*NSG + j;\n ls[j] = s_ptr[is];\n }\n\n device float * dst_attn = (device float *) (dst) + (i23*args.ne22*args.ne21 + i21)*S_v + i20;\n\n device const float * q_ptr = (device const float *) (q + i23*args.nb03 + i01*args.nb01);\n device const float * k_ptr = (device const float *) (k + i23*args.nb13 + i11*args.nb11);\n device const float * v_ptr = (device const float *) (v + i23*args.nb23 + i21*args.nb21);\n\n device const float * b_ptr = (device const float *) (b) + (i23*args.ne22*args.ne21 + i21);\n device const float * g_ptr = (device const float *) (g) + (i23*args.ne22*args.ne21 + i21)*G;\n\n for (short t = 0; t < args.ne22; t++) {\n float s_k = 0.0f;\n\n if (G == 1) {\n const float g_exp = exp(g_ptr[0]);\n\n FOR_UNROLL (short j = 0; j < NSG; j++) {\n const short is = tx*NSG + j;\n ls[j] *= g_exp;\n\n s_k += ls[j]*k_ptr[is];\n }\n } else {\n // KDA\n FOR_UNROLL (short j = 0; j < NSG; j++) {\n const short is = tx*NSG + j;\n ls[j] *= exp(g_ptr[is]);\n\n s_k += ls[j]*k_ptr[is];\n }\n }\n\n s_k = simd_sum(s_k);\n\n const float d = (v_ptr[i20] - s_k)*b_ptr[0];\n\n float y = 0.0f;\n\n FOR_UNROLL (short j = 0; j < NSG; j++) {\n const short is = tx*NSG + j;\n ls[j] += k_ptr[is]*d;\n\n y += ls[j]*q_ptr[is];\n }\n\n y = simd_sum(y);\n\n if (tx == 0) {\n dst_attn[t*args.ne21*S_v] = y*scale;\n }\n\n q_ptr += args.ns02;\n k_ptr += args.ns12;\n v_ptr += args.ns22;\n\n b_ptr += args.ne21;\n g_ptr += args.ne21*G;\n }\n\n device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20*S_v;\n\n FOR_UNROLL (short j = 0; j < NSG; j++) {\n const short is = tx*NSG + j;\n dst_state[is] = ls[j];\n }\n\n#undef S_v\n#undef G\n}\n\ntypedef decltype(kernel_gated_delta_net_impl<4>) kernel_gated_delta_net_t;\n\ntemplate [[host_name(\"kernel_gated_delta_net_f32_1\")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl<1>;\ntemplate [[host_name(\"kernel_gated_delta_net_f32_2\")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl<2>;\ntemplate [[host_name(\"kernel_gated_delta_net_f32_4\")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl<4>;\n\n#else\n// a simplified version of the above\n// no performance improvement, so keep the above version for now\n\ntemplate\nkernel void kernel_gated_delta_net_impl(\n constant ggml_metal_kargs_gated_delta_net & args,\n device const char * q,\n device const char * k,\n device const char * v,\n device const char * g,\n device const char * b,\n device const char * s,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n#define S_v FC_gated_delta_net_ne20\n#define G FC_gated_delta_net_ne30\n\n const uint tx = tpitg.x;\n const uint ty = tpitg.y;\n\n const uint i23 = tgpig.z; // B\n const uint i21 = tgpig.y; // H\n const uint i20 = tgpig.x*NSG + ty;\n\n const uint i01 = i21 % args.ne01;\n const uint i11 = i21 % args.ne11;\n\n const float scale = 1.0f / sqrt((float)S_v);\n\n device const float * s_ptr = (device const float *) (s) + (i23*args.ne21 + i21)*S_v*S_v + i20;\n\n float lsf[NSG];\n\n FOR_UNROLL (short j = 0; j < NSG; j++) {\n const short is = tx*NSG + j;\n lsf[j] = s_ptr[is*S_v];\n }\n\n thread T * ls = (thread T *) (lsf);\n\n device float * dst_attn = (device float *) (dst) + (i23*args.ne22*args.ne21 + i21)*S_v + i20;\n\n device const float * q_ptr = (device const float *) (q + i23*args.nb03 + i01*args.nb01);\n device const float * k_ptr = (device const float *) (k + i23*args.nb13 + i11*args.nb11);\n device const float * v_ptr = (device const float *) (v + i23*args.nb23 + i21*args.nb21);\n\n device const float * b_ptr = (device const float *) (b) + (i23*args.ne22*args.ne21 + i21);\n device const float * g_ptr = (device const float *) (g) + (i23*args.ne22*args.ne21 + i21)*G;\n\n for (short t = 0; t < args.ne22; t++) {\n device const T * qt_ptr = (device const T *) (q_ptr);\n device const T * kt_ptr = (device const T *) (k_ptr);\n device const T * gt_ptr = (device const T *) (g_ptr);\n\n if (G == 1) {\n *ls *= exp(g_ptr[0]);\n } else {\n // KDA\n *ls *= exp(gt_ptr[tx]);\n }\n\n const float s_k = simd_sum(dot(*ls, kt_ptr[tx]));\n\n const float d = (v_ptr[i20] - s_k)*b_ptr[0];\n\n *ls += kt_ptr[tx]*d;\n\n const float y = simd_sum(dot(*ls, qt_ptr[tx]));\n\n if (tx == 0) {\n *dst_attn = y*scale;\n }\n\n q_ptr += args.ns02;\n k_ptr += args.ns12;\n v_ptr += args.ns22;\n\n b_ptr += args.ne21;\n g_ptr += args.ne21*G;\n\n dst_attn += args.ne21*S_v;\n }\n\n device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20;\n device T * dstt_state = (device T *) (dst_state);\n\n FOR_UNROLL (short j = 0; j < NSG; j++) {\n const short is = tx*NSG + j;\n dst_state[is*S_v] = lsf[j];\n }\n\n#undef S_v\n#undef G\n}\n\ntypedef decltype(kernel_gated_delta_net_impl) kernel_gated_delta_net_t;\n\ntemplate [[host_name(\"kernel_gated_delta_net_f32_1\")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl;\ntemplate [[host_name(\"kernel_gated_delta_net_f32_2\")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl;\ntemplate [[host_name(\"kernel_gated_delta_net_f32_4\")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl;\n#endif\n\nconstant short FC_solve_tri_nsg [[function_constant(FC_SOLVE_TRI + 0)]];\nconstant short FC_solve_tri_n [[function_constant(FC_SOLVE_TRI + 1)]];\nconstant short FC_solve_tri_k [[function_constant(FC_SOLVE_TRI + 2)]];\n\nkernel void kernel_solve_tri_f32(\n constant ggml_metal_kargs_solve_tri & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n ushort3 tgpig[[threadgroup_position_in_grid]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n constexpr short NW = N_SIMDWIDTH;\n\n const short NSG = FC_solve_tri_nsg;\n const short N = FC_solve_tri_n;\n const short K = FC_solve_tri_k;\n const short NP = PAD2(N, NW);\n\n const int32_t i03 = tgpig.z;\n const int32_t i02 = tgpig.y;\n const int32_t i01 = tgpig.x*NSG + sgitg;\n\n threadgroup float * sh0 = (threadgroup float *) shmem;\n\n device const float * src0_ptr = (device const float *)(src0 + i02 * args.nb02 + i03 * args.nb03) + sgitg*N;\n device const float * src1_ptr = (device const float *)(src1 + i02 * args.nb12 + i03 * args.nb13) + i01;\n device float * dst_ptr = (device float *)(dst + i02 * args.nb2 + i03 * args.nb3) + i01;\n\n for (short rr = 0; rr < N; rr += NSG) {\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n {\n threadgroup float * sh0_cur = sh0 + sgitg*NP;\n\n for (short t = 0; t*NW < N; ++t) {\n const short idx = t*NW + tiisg;\n sh0_cur[idx] = src0_ptr[idx];\n }\n\n src0_ptr += NSG*N;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (i01 >= args.ne10) {\n continue;\n }\n\n for (short ir = 0; ir < NSG && rr + ir < N; ++ir) {\n const short r = rr + ir;\n\n threadgroup float * sh0_cur = sh0 + ir*NP;\n\n float sum = 0.0f;\n\n for (short t = 0; t*NW < r; ++t) {\n const short idx = t*NW + tiisg;\n sum += sh0_cur[idx] * dst_ptr[idx*K] * (idx < r);\n }\n\n sum = simd_sum(sum);\n\n if (tiisg == 0) {\n const float diag = sh0_cur[r];\n\n dst_ptr[r*K] = (src1_ptr[r*K] - sum) / diag;\n }\n }\n }\n}\n\nkernel void kernel_argmax_f32(\n constant ggml_metal_kargs_argmax & args,\n device const char * src0,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint tgpig[[threadgroup_position_in_grid]],\n uint tpitg[[thread_position_in_threadgroup]],\n uint sgitg[[simdgroup_index_in_threadgroup]],\n uint tiisg[[thread_index_in_simdgroup]],\n uint ntg[[threads_per_threadgroup]]) {\n device const float * x_row = (device const float *) ((device const char *) src0 + tgpig * args.nb01);\n\n float lmax = -INFINITY;\n int32_t larg = -1;\n\n for (int i00 = tpitg; i00 < args.ne00; i00 += ntg) {\n if (x_row[i00] > lmax) {\n lmax = x_row[i00];\n larg = i00;\n }\n }\n\n // find the argmax value in the block\n float max_val = simd_max(lmax);\n int32_t arg_val = simd_max(select(-1, larg, lmax == max_val));\n\n device int32_t * dst_i32 = (device int32_t *) dst;\n\n threadgroup float * shared_maxval = (threadgroup float *) shmem;\n threadgroup int32_t * shared_argmax = (threadgroup int32_t *) shmem + N_SIMDWIDTH;\n\n if (ntg > N_SIMDWIDTH) {\n if (sgitg == 0) {\n shared_maxval[tiisg] = -INFINITY;\n shared_argmax[tiisg] = -1;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n shared_maxval[sgitg] = max_val;\n shared_argmax[sgitg] = arg_val;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n max_val = shared_maxval[tiisg];\n arg_val = shared_argmax[tiisg];\n\n float max_val_reduced = simd_max(max_val);\n int32_t arg_val_reduced = simd_max(select(-1, arg_val, max_val == max_val_reduced));\n\n dst_i32[tgpig] = arg_val_reduced;\n\n return;\n }\n\n dst_i32[tgpig] = arg_val;\n}\n\n// F == 1 : norm (no fuse)\n// F == 2 : norm + mul\n// F == 3 : norm + mul + add\ntemplate \nkernel void kernel_norm_fuse_impl(\n constant ggml_metal_kargs_norm & args,\n device const char * src0,\n device const char * src1_0,\n device const char * src1_1,\n device char * dst,\n threadgroup float * shmem_f32 [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n if (sgitg == 0) {\n shmem_f32[tiisg] = 0.0f;\n }\n\n const int i01 = tgpig.x;\n const int i02 = tgpig.y;\n const int i03 = tgpig.z;\n\n device const T * x = (device const T *) (src0 + i03*args.nbf3[0] + i02*args.nbf2[0] + i01*args.nbf1[0]);\n\n device const T * f0 = (device const T *) (src1_0 + (i03%args.nef3[1])*args.nbf3[1] + (i02%args.nef2[1])*args.nbf2[1] + (i01%args.nef1[1])*args.nbf1[1]);\n device const T * f1 = (device const T *) (src1_1 + (i03%args.nef3[2])*args.nbf3[2] + (i02%args.nef2[2])*args.nbf2[2] + (i01%args.nef1[2])*args.nbf1[2]);\n\n T sumft(0.0f);\n\n float sumf = 0.0f;\n\n for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {\n sumft += x[i00];\n }\n sumf = dot(sumft, T(1.0f));\n sumf = simd_sum(sumf);\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n shmem_f32[sgitg] = sumf;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n sumf = shmem_f32[tiisg];\n sumf = simd_sum(sumf);\n\n const float mean = sumf/args.ne00;\n\n device T * y = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1);\n\n sumf = 0.0f;\n for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {\n y[i00] = x[i00] - mean;\n sumf += dot(y[i00], y[i00]);\n }\n sumf = simd_sum(sumf);\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n shmem_f32[sgitg] = sumf;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n sumf = shmem_f32[tiisg];\n sumf = simd_sum(sumf);\n\n const float variance = sumf/args.ne00;\n\n const float scale = 1.0f/sqrt(variance + args.eps);\n for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {\n if (F == 1) {\n y[i00] = (y[i00]*scale);\n }\n if (F == 2) {\n y[i00] = (y[i00]*scale)*f0[i00];\n }\n if (F == 3) {\n y[i00] = (y[i00]*scale)*f0[i00] + f1[i00];\n }\n }\n}\n\ntypedef decltype(kernel_norm_fuse_impl) kernel_norm_fuse_t;\n\ntemplate [[host_name(\"kernel_norm_f32\")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl;\ntemplate [[host_name(\"kernel_norm_mul_f32\")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl;\ntemplate [[host_name(\"kernel_norm_mul_add_f32\")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl;\n\ntemplate [[host_name(\"kernel_norm_f32_4\")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl;\ntemplate [[host_name(\"kernel_norm_mul_f32_4\")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl;\ntemplate [[host_name(\"kernel_norm_mul_add_f32_4\")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl;\n\n// F == 1 : rms_norm (no fuse)\n// F == 2 : rms_norm + mul\n// F == 3 : rms_norm + mul + add\ntemplate \nkernel void kernel_rms_norm_fuse_impl(\n constant ggml_metal_kargs_norm & args,\n device const char * src0,\n device const char * src1_0,\n device const char * src1_1,\n device char * dst,\n threadgroup float * shmem_f32 [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n if (sgitg == 0) {\n shmem_f32[tiisg] = 0.0f;\n }\n\n const int i01 = tgpig.x;\n const int i02 = tgpig.y;\n const int i03 = tgpig.z;\n\n device const T * x = (device const T *) (src0 + i03*args.nbf3[0] + i02*args.nbf2[0] + i01*args.nbf1[0]);\n\n device const T * f0 = (device const T *) (src1_0 + (i03%args.nef3[1])*args.nbf3[1] + (i02%args.nef2[1])*args.nbf2[1] + (i01%args.nef1[1])*args.nbf1[1]);\n device const T * f1 = (device const T *) (src1_1 + (i03%args.nef3[2])*args.nbf3[2] + (i02%args.nef2[2])*args.nbf2[2] + (i01%args.nef1[2])*args.nbf1[2]);\n\n float sumf = 0.0f;\n\n // parallel sum\n for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {\n sumf += dot(x[i00], x[i00]);\n }\n sumf = simd_sum(sumf);\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n shmem_f32[sgitg] = sumf;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n sumf = shmem_f32[tiisg];\n sumf = simd_sum(sumf);\n\n const float mean = sumf/args.ne00;\n const float scale = 1.0f/sqrt(mean + args.eps);\n\n device T * y = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1);\n for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {\n if (F == 1) {\n y[i00] = (x[i00]*scale);\n }\n if (F == 2) {\n y[i00] = (x[i00]*scale)*f0[i00];\n }\n if (F == 3) {\n y[i00] = (x[i00]*scale)*f0[i00] + f1[i00];\n }\n }\n}\n\ntypedef decltype(kernel_rms_norm_fuse_impl) kernel_rms_norm_fuse_t;\n\ntemplate [[host_name(\"kernel_rms_norm_f32\")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl;\ntemplate [[host_name(\"kernel_rms_norm_mul_f32\")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl;\ntemplate [[host_name(\"kernel_rms_norm_mul_add_f32\")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl;\n\ntemplate [[host_name(\"kernel_rms_norm_f32_4\")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl;\ntemplate [[host_name(\"kernel_rms_norm_mul_f32_4\")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl;\ntemplate [[host_name(\"kernel_rms_norm_mul_add_f32_4\")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl;\n\ntemplate \nkernel void kernel_l2_norm_impl(\n constant ggml_metal_kargs_l2_norm & args,\n device const char * src0,\n device char * dst,\n threadgroup float * shmem_f32 [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int i03 = tgpig.z;\n const int i02 = tgpig.y;\n const int i01 = tgpig.x;\n\n if (sgitg == 0) {\n shmem_f32[tiisg] = 0.0f;\n }\n\n device const T0 * x = (device const T0 *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01);\n device T * y = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1);\n\n float sumf = 0.0f;\n\n // parallel sum\n for (int i00 = tpitg.x; i00 < args.ne00; i00 += ntg.x) {\n sumf += dot(x[i00], x[i00]);\n }\n sumf = simd_sum(sumf);\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n shmem_f32[sgitg] = sumf;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n sumf = shmem_f32[tiisg];\n sumf = simd_sum(sumf);\n\n const float scale = 1.0f/max(sqrt(sumf), args.eps);\n\n for (int i00 = tpitg.x; i00 < args.ne00; i00 += ntg.x) {\n y[i00] = x[i00] * scale;\n }\n}\n\ntypedef decltype(kernel_l2_norm_impl) kernel_l2_norm_t;\n\ntemplate [[host_name(\"kernel_l2_norm_f32_f32\")]] kernel kernel_l2_norm_t kernel_l2_norm_impl;\ntemplate [[host_name(\"kernel_l2_norm_f32_f32_4\")]] kernel kernel_l2_norm_t kernel_l2_norm_impl;\n\nkernel void kernel_group_norm_f32(\n constant ggml_metal_kargs_group_norm & args,\n device const float * src0,\n device float * dst,\n threadgroup float * buf [[threadgroup(0)]],\n uint tgpig[[threadgroup_position_in_grid]],\n uint tpitg[[thread_position_in_threadgroup]],\n uint sgitg[[simdgroup_index_in_threadgroup]],\n uint tiisg[[thread_index_in_simdgroup]],\n uint ntg[[threads_per_threadgroup]]) {\n const int64_t ne = args.ne00*args.ne01*args.ne02;\n const int64_t gs = args.ne00*args.ne01*((args.ne02 + args.ngrp - 1) / args.ngrp);\n\n int start = tgpig * gs;\n int end = start + gs;\n\n start += tpitg;\n\n if (end >= ne) {\n end = ne;\n }\n\n float tmp = 0.0f; // partial sum for thread in warp\n\n for (int j = start; j < end; j += ntg) {\n tmp += src0[j];\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n tmp = simd_sum(tmp);\n if (ntg > N_SIMDWIDTH) {\n if (sgitg == 0) {\n buf[tiisg] = 0.0f;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n buf[sgitg] = tmp;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n tmp = buf[tiisg];\n tmp = simd_sum(tmp);\n }\n\n const float mean = tmp / gs;\n tmp = 0.0f;\n\n for (int j = start; j < end; j += ntg) {\n float xi = src0[j] - mean;\n dst[j] = xi;\n tmp += xi * xi;\n }\n\n tmp = simd_sum(tmp);\n if (ntg > N_SIMDWIDTH) {\n if (sgitg == 0) {\n buf[tiisg] = 0.0f;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tiisg == 0) {\n buf[sgitg] = tmp;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n tmp = buf[tiisg];\n tmp = simd_sum(tmp);\n }\n\n const float variance = tmp / gs;\n const float scale = 1.0f/sqrt(variance + args.eps);\n for (int j = start; j < end; j += ntg) {\n dst[j] *= scale;\n }\n}\n\n// function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i])\n// il indicates where the q4 quants begin (0 or QK4_0/4)\n// we assume that the yl's have been multiplied with the appropriate scale factor\n// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)\ninline float block_q_n_dot_y(device const block_q4_0 * qb_curr, float sumy, thread float * yl, int il) {\n float d = qb_curr->d;\n\n float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f };\n\n device const uint16_t * qs = ((device const uint16_t *) qb_curr + 1 + il/2);\n\n for (int i = 0; i < 8; i += 2) {\n acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F);\n acc[1] += yl[i + 1] * (qs[i / 2] & 0x0F00);\n acc[2] += yl[i + 8] * (qs[i / 2] & 0x00F0);\n acc[3] += yl[i + 9] * (qs[i / 2] & 0xF000);\n }\n\n return d * (sumy * -8.f + acc[0] + acc[1] + acc[2] + acc[3]);\n}\n\n// function for calculate inner product between half a q4_1 block and 16 floats (yl), sumy is SUM(yl[i])\n// il indicates where the q4 quants begin (0 or QK4_0/4)\n// we assume that the yl's have been multiplied with the appropriate scale factor\n// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)\ninline float block_q_n_dot_y(device const block_q4_1 * qb_curr, float sumy, thread float * yl, int il) {\n float d = qb_curr->d;\n float m = qb_curr->m;\n\n float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f };\n\n device const uint16_t * qs = ((device const uint16_t *) qb_curr + 2 + il/2);\n\n for (int i = 0; i < 8; i+=2) {\n acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F);\n acc[1] += yl[i + 1] * (qs[i / 2] & 0x0F00);\n acc[2] += yl[i + 8] * (qs[i / 2] & 0x00F0);\n acc[3] += yl[i + 9] * (qs[i / 2] & 0xF000);\n }\n\n return d * (acc[0] + acc[1] + acc[2] + acc[3]) + sumy * m;\n}\n\n// function for calculate inner product between half a q5_0 block and 16 floats (yl), sumy is SUM(yl[i])\n// il indicates where the q5 quants begin (0 or QK5_0/4)\n// we assume that the yl's have been multiplied with the appropriate scale factor\n// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)\ninline float block_q_n_dot_y(device const block_q5_0 * qb_curr, float sumy, thread float * yl, int il) {\n float d = qb_curr->d;\n\n float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f };\n\n device const uint16_t * qs = ((device const uint16_t *)qb_curr + 3 + il/2);\n const uint32_t qh = *((device const uint32_t *)qb_curr->qh);\n\n for (int i = 0; i < 8; i+=2) {\n acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010));\n acc[1] += yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));\n acc[2] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100));\n acc[3] += yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));\n }\n\n return d * (sumy * -16.f + acc[0] + acc[1] + acc[2] + acc[3]);\n}\n\n// function for calculate inner product between half a q5_1 block and 16 floats (yl), sumy is SUM(yl[i])\n// il indicates where the q5 quants begin (0 or QK5_1/4)\n// we assume that the yl's have been multiplied with the appropriate scale factor\n// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)\ninline float block_q_n_dot_y(device const block_q5_1 * qb_curr, float sumy, thread float * yl, int il) {\n float d = qb_curr->d;\n float m = qb_curr->m;\n\n float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f };\n\n device const uint16_t * qs = ((device const uint16_t *)qb_curr + 4 + il/2);\n const uint32_t qh = *((device const uint32_t *)qb_curr->qh);\n\n for (int i = 0; i < 8; i+=2) {\n acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010));\n acc[1] += yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));\n acc[2] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100));\n acc[3] += yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));\n }\n\n return d * (acc[0] + acc[1] + acc[2] + acc[3]) + sumy * m;\n}\n\ntemplate\nstatic inline void helper_mv_reduce_and_write(\n device float * dst_f32,\n float sumf[NR0],\n const int r0,\n const int ne01,\n ushort tiisg,\n ushort sgitg,\n threadgroup char * shmem) {\n constexpr short NW = N_SIMDWIDTH;\n\n threadgroup float * shmem_f32[NR0];\n\n for (short row = 0; row < NR0; ++row) {\n shmem_f32[row] = (threadgroup float *) shmem + NW*row;\n\n if (sgitg == 0) {\n shmem_f32[row][tiisg] = 0.0f;\n }\n\n sumf[row] = simd_sum(sumf[row]);\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n for (short row = 0; row < NR0; ++row) {\n if (tiisg == 0) {\n shmem_f32[row][sgitg] = sumf[row];\n }\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n for (short row = 0; row < NR0 && r0 + row < ne01; ++row) {\n float tot = simd_sum(shmem_f32[row][tiisg]);\n\n if (tiisg == 0 && sgitg == 0) {\n dst_f32[r0 + row] = tot;\n }\n }\n}\n\nconstant short FC_mul_mv_nsg [[function_constant(FC_MUL_MV + 0)]];\nconstant short FC_mul_mv_nxpsg [[function_constant(FC_MUL_MV + 1)]];\n\ntemplate\nvoid mul_vec_q_n_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n constexpr short NW = N_SIMDWIDTH;\n constexpr short NQ = 16;\n\n const int nb = args.ne00/QK4_0;\n\n const int r0 = (tgpig.x*NSG + sgitg)*NR0;\n //const int r0 = tgpig.x*NR0;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n //const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n //device const block_q_type * x = (device const block_q_type *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n // pointers to src0 rows\n device const block_q_type * ax[NR0];\n FOR_UNROLL (int row = 0; row < NR0; ++row) {\n const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n\n ax[row] = (device const block_q_type *) ((device char *) src0 + offset0);\n }\n\n float sumf[NR0] = {0.f};\n\n const short ix = (tiisg/(NW/NQ));\n const short il = (tiisg%(NW/NQ))*8;\n\n //const int ib0 = sgitg*NQ + ix;\n const int ib0 = ix;\n\n float yl[16]; // src1 vector cache\n\n //device const float * yb = y + ix*QK4_0 + il;\n device const float * yb = y + ib0*QK4_0 + il;\n\n // each thread in a SIMD group deals with half a block.\n //for (int ib = ib0; ib < nb; ib += NSG*NQ) {\n for (int ib = ib0; ib < nb; ib += NQ) {\n float sumy[2] = { 0.f, 0.f };\n\n FOR_UNROLL (short i = 0; i < 8; i += 2) {\n sumy[0] += yb[i + 0] + yb[i + 1];\n yl[i + 0] = yb[i + 0];\n yl[i + 1] = yb[i + 1]/256.f;\n\n sumy[1] += yb[i + 16] + yb[i + 17];\n yl[i + 8] = yb[i + 16]/16.f;\n yl[i + 9] = yb[i + 17]/4096.f;\n }\n\n FOR_UNROLL (short row = 0; row < NR0; row++) {\n sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy[0] + sumy[1], yl, il);\n }\n\n yb += QK4_0 * 16;\n //yb += NSG*NQ*QK4_0;\n }\n\n device float * dst_f32 = (device float *) dst + im*args.ne0*args.ne1 + r1*args.ne0;\n\n //helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem);\n\n for (int row = 0; row < NR0; ++row) {\n const float tot = simd_sum(sumf[row]);\n\n if (tiisg == 0 && r0 + row < args.ne01) {\n dst_f32[r0 + row] = tot;\n }\n }\n}\n\nkernel void kernel_mul_mv_q4_0_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\nkernel void kernel_mul_mv_q4_1_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\nkernel void kernel_mul_mv_q5_0_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\nkernel void kernel_mul_mv_q5_1_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_q8_0_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n constexpr short NW = N_SIMDWIDTH;\n constexpr short NQ = 8;\n\n const int nb = args.ne00/QK8_0;\n\n const int r0 = tgpig.x*NR0;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n //const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n //device const block_q8_0 * x = (device const block_q8_0 *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n // pointers to src0 rows\n device const block_q8_0 * ax[NR0];\n FOR_UNROLL (short row = 0; row < NR0; ++row) {\n const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n\n ax[row] = (device const block_q8_0 *) ((device char *) src0 + offset0);\n }\n\n float sumf[NR0] = { 0.f };\n\n const short ix = tiisg/(NW/NQ);\n const short il = tiisg%(NW/NQ);\n\n const int ib0 = sgitg*NQ + ix;\n\n float yl[NQ];\n\n device const float * yb = y + ib0*QK8_0 + il*NQ;\n\n // each thread in a SIMD group deals with NQ quants at a time\n for (int ib = ib0; ib < nb; ib += NSG*NQ) {\n for (short i = 0; i < NQ; ++i) {\n yl[i] = yb[i];\n }\n\n for (short row = 0; row < NR0; row++) {\n device const int8_t * qs = ax[row][ib].qs + il*NQ;\n\n float sumq = 0.f;\n FOR_UNROLL (short i = 0; i < NQ; ++i) {\n sumq += qs[i] * yl[i];\n }\n\n sumf[row] += sumq*ax[row][ib].d;\n }\n\n yb += NSG*NQ*QK8_0;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem);\n}\n\n[[host_name(\"kernel_mul_mv_q8_0_f32\")]]\nkernel void kernel_mul_mv_q8_0_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n kernel_mul_mv_q8_0_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\n// mat-vec kernel processing in chunks of float4\n// chpb - chunks per quantization block\ntemplate\nvoid kernel_mul_mv_ext_q4_f32_impl(\n constant ggml_metal_kargs_mul_mv_ext & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n const short NSG = FC_mul_mv_nsg;\n const short nxpsg = FC_mul_mv_nxpsg;\n\n const short chpt = 4; // chunks per thread\n\n //const short nxpsg = (32);\n const short nypsg = (32/nxpsg);\n\n const short tx = tiisg%nxpsg;\n const short ty = tiisg/nxpsg;\n\n const int i01 = tgpig.x*(nypsg*NSG) + nypsg*sgitg + ty;\n const int i11 = tgpig.y*r1ptg;\n const int i1m = tgpig.z;\n\n const int i12 = i1m%args.ne12;\n const int i13 = i1m/args.ne12;\n\n const uint64_t offset0 = i01*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = i11*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const q_t * xq = (i01 < args.ne01) ? (device const q_t *) (src0 + offset0) + tx/chpb : (device const q_t *) src0;\n\n device const float4 * y4[r1ptg];\n\n for (int ir1 = 0; ir1 < r1ptg; ++ir1) {\n y4[ir1] = (i11 + ir1 < args.ne11) ? (device const float4 *) (src1 + offset1 + ir1*args.nb11) + tx : (device const float4 *) src1;\n }\n\n float sumf[r1ptg] = { [ 0 ... r1ptg - 1 ] = 0.0f };\n\n short cch = tx%chpb; // current chunk index\n\n for (int ich = tx; 4*ich < args.ne00; ich += chpt*nxpsg) {\n float4 lx[chpt];\n\n#pragma unroll(chpt)\n for (short ch = 0; ch < chpt; ++ch) {\n deq_t4(xq, cch, lx[ch]);\n\n cch += nxpsg;\n if (cch >= chpb) {\n xq += cch/chpb;\n cch %= chpb;\n }\n }\n\n#pragma unroll(chpt)\n for (short ch = 0; ch < chpt; ++ch) {\n#pragma unroll(r1ptg)\n for (short ir1 = 0; ir1 < r1ptg; ++ir1) {\n sumf[ir1] += dot(lx[ch], y4[ir1][ch*nxpsg]);\n }\n }\n\n#pragma unroll(r1ptg)\n for (short ir1 = 0; ir1 < r1ptg; ++ir1) {\n y4[ir1] += chpt*nxpsg;\n }\n }\n\n // reduce only the threads in each row\n for (short ir1 = 0; ir1 < r1ptg; ++ir1) {\n if (nxpsg >= 32) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 16);\n }\n if (nxpsg >= 16) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 8);\n }\n if (nxpsg >= 8) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 4);\n }\n if (nxpsg >= 4) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 2);\n }\n if (nxpsg >= 2) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 1);\n }\n\n //sumf[ir1] = simd_sum(sumf[ir1]);\n }\n\n if (tx == 0) {\n for (short ir1 = 0; ir1 < r1ptg && i11 + ir1 < args.ne11; ++ir1) {\n device float * dst_f32 = (device float *) dst + (uint64_t)i1m*args.ne0*args.ne1 + (uint64_t)(i11 + ir1)*args.ne0;\n\n if (i01 < args.ne01) {\n dst_f32[i01] = sumf[ir1];\n }\n }\n }\n}\n\n// mat-vec kernel processing in chunks of float4x4\ntemplate\nvoid kernel_mul_mv_ext_q4x4_f32_impl(\n constant ggml_metal_kargs_mul_mv_ext & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n const short NSG = FC_mul_mv_nsg;\n const short nxpsg = FC_mul_mv_nxpsg;\n\n const short chpt = 1;\n\n //const short nxpsg = (32);\n const short nypsg = (32/nxpsg);\n\n const short tx = tiisg%nxpsg;\n const short ty = tiisg/nxpsg;\n\n const int i01 = tgpig.x*(nypsg*NSG) + nypsg*sgitg + ty;\n const int i11 = tgpig.y*r1ptg;\n const int i1m = tgpig.z;\n\n const int i12 = i1m%args.ne12;\n const int i13 = i1m/args.ne12;\n\n const uint64_t offset0 = i01*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = i11*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const q_t * xq = (i01 < args.ne01) ? (device const q_t *) (src0 + offset0) + tx/chpb : (device const q_t *) src0;\n\n device const float4x4 * y4x4[r1ptg];\n\n for (int ir1 = 0; ir1 < r1ptg; ++ir1) {\n y4x4[ir1] = (i11 + ir1 < args.ne11) ? (device const float4x4 *) (src1 + offset1 + ir1*args.nb11) + tx : (device const float4x4 *) src1;\n }\n\n float sumf[r1ptg] = { [ 0 ... r1ptg - 1 ] = 0.0f };\n\n short cch = tx%chpb;\n\n for (int ich = tx; 16*ich < args.ne00; ich += chpt*nxpsg) {\n float4x4 lx[chpt];\n\n#pragma unroll(chpt)\n for (short ch = 0; ch < chpt; ++ch) {\n deq_t4x4(xq, cch, lx[ch]);\n\n cch += nxpsg;\n if (cch >= chpb) {\n xq += cch/chpb;\n cch %= chpb;\n }\n }\n\n#pragma unroll(chpt)\n for (short ch = 0; ch < chpt; ++ch) {\n#pragma unroll(r1ptg)\n for (short ir1 = 0; ir1 < r1ptg; ++ir1) {\n sumf[ir1] +=\n dot(lx[ch][0], y4x4[ir1][ch*nxpsg][0]) +\n dot(lx[ch][1], y4x4[ir1][ch*nxpsg][1]) +\n dot(lx[ch][2], y4x4[ir1][ch*nxpsg][2]) +\n dot(lx[ch][3], y4x4[ir1][ch*nxpsg][3]);\n\n }\n }\n\n#pragma unroll(r1ptg)\n for (short ir1 = 0; ir1 < r1ptg; ++ir1) {\n y4x4[ir1] += chpt*nxpsg;\n }\n }\n\n for (short ir1 = 0; ir1 < r1ptg; ++ir1) {\n if (nxpsg >= 32) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 16);\n }\n if (nxpsg >= 16) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 8);\n }\n if (nxpsg >= 8) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 4);\n }\n if (nxpsg >= 4) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 2);\n }\n if (nxpsg >= 2) {\n sumf[ir1] += simd_shuffle_down(sumf[ir1], 1);\n }\n\n //sumf[ir1] = simd_sum(sumf[ir1]);\n }\n\n if (tx == 0) {\n for (short ir1 = 0; ir1 < r1ptg && i11 + ir1 < args.ne11; ++ir1) {\n device float * dst_f32 = (device float *) dst + (uint64_t)i1m*args.ne0*args.ne1 + (uint64_t)(i11 + ir1)*args.ne0;\n\n if (i01 < args.ne01) {\n dst_f32[i01] = sumf[ir1];\n }\n }\n }\n}\n\n// dispatchers needed for compile-time nxpsg\n// epb - elements per quantization block\ntemplate\nkernel void kernel_mul_mv_ext_q4_f32_disp(\n constant ggml_metal_kargs_mul_mv_ext & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n kernel_mul_mv_ext_q4_f32_impl(args, src0, src1, dst, tgpig, tiisg, sgitg);\n}\n\ntemplate\nkernel void kernel_mul_mv_ext_q4x4_f32_disp(\n constant ggml_metal_kargs_mul_mv_ext & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n kernel_mul_mv_ext_q4x4_f32_impl(args, src0, src1, dst, tgpig, tiisg, sgitg);\n}\n\ntypedef decltype(kernel_mul_mv_ext_q4_f32_disp <2, block_q8_0, 32, dequantize_q8_0_t4>) mul_mv_ext_q4_f32_t;\ntypedef decltype(kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>) mul_mv_ext_q4x4_f32_t;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_f32_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, float4, 4, dequantize_f32_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_f32_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, float4, 4, dequantize_f32_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_f32_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, float4, 4, dequantize_f32_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_f32_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, float4, 4, dequantize_f32_t4>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_f16_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, half4, 4, dequantize_f16_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_f16_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, half4, 4, dequantize_f16_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_f16_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, half4, 4, dequantize_f16_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_f16_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, half4, 4, dequantize_f16_t4>;\n\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_mul_mv_ext_bf16_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, bfloat4, 4, dequantize_bf16_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_bf16_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, bfloat4, 4, dequantize_bf16_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_bf16_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, bfloat4, 4, dequantize_bf16_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_bf16_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, bfloat4, 4, dequantize_bf16_t4>;\n#endif\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_0_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_0, 32, dequantize_q4_0_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_0_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_0, 32, dequantize_q4_0_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_0_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_0, 32, dequantize_q4_0_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_0_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q4_0, 32, dequantize_q4_0_t4>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_1_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_1, 32, dequantize_q4_1_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_1_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_1, 32, dequantize_q4_1_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_1_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_1, 32, dequantize_q4_1_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_1_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q4_1, 32, dequantize_q4_1_t4>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_0_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_0, 32, dequantize_q5_0_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_0_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_0, 32, dequantize_q5_0_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_0_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_0, 32, dequantize_q5_0_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_0_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q5_0, 32, dequantize_q5_0_t4>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_1_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_1, 32, dequantize_q5_1_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_1_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_1, 32, dequantize_q5_1_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_1_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_1, 32, dequantize_q5_1_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_1_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q5_1, 32, dequantize_q5_1_t4>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q8_0_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q8_0, 32, dequantize_q8_0_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q8_0_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q8_0, 32, dequantize_q8_0_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q8_0_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q8_0, 32, dequantize_q8_0_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q8_0_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q8_0, 32, dequantize_q8_0_t4>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_mxfp4_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_mxfp4, 32, dequantize_mxfp4_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_mxfp4_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_mxfp4, 32, dequantize_mxfp4_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_mxfp4_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_mxfp4, 32, dequantize_mxfp4_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_mxfp4_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_mxfp4, 32, dequantize_mxfp4_t4>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_iq4_nl_f32_r1_2\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_iq4_nl, 32, dequantize_iq4_nl_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_iq4_nl_f32_r1_3\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_iq4_nl, 32, dequantize_iq4_nl_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_iq4_nl_f32_r1_4\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_iq4_nl, 32, dequantize_iq4_nl_t4>;\ntemplate [[host_name(\"kernel_mul_mv_ext_iq4_nl_f32_r1_5\")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_iq4_nl, 32, dequantize_iq4_nl_t4>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_K_f32_r1_2\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_K_f32_r1_3\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q4_K, 256, dequantize_q4_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_K_f32_r1_4\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q4_K, 256, dequantize_q4_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q4_K_f32_r1_5\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q4_K, 256, dequantize_q4_K>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_K_f32_r1_2\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q5_K, 256, dequantize_q5_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_K_f32_r1_3\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q5_K, 256, dequantize_q5_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_K_f32_r1_4\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q5_K, 256, dequantize_q5_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q5_K_f32_r1_5\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q5_K, 256, dequantize_q5_K>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q6_K_f32_r1_2\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q6_K, 256, dequantize_q6_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q6_K_f32_r1_3\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q6_K, 256, dequantize_q6_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q6_K_f32_r1_4\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q6_K, 256, dequantize_q6_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q6_K_f32_r1_5\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q6_K, 256, dequantize_q6_K>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q2_K_f32_r1_2\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q2_K, 256, dequantize_q2_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q2_K_f32_r1_3\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q2_K, 256, dequantize_q2_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q2_K_f32_r1_4\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q2_K, 256, dequantize_q2_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q2_K_f32_r1_5\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q2_K, 256, dequantize_q2_K>;\n\ntemplate [[host_name(\"kernel_mul_mv_ext_q3_K_f32_r1_2\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q3_K, 256, dequantize_q3_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q3_K_f32_r1_3\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q3_K, 256, dequantize_q3_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q3_K_f32_r1_4\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q3_K, 256, dequantize_q3_K>;\ntemplate [[host_name(\"kernel_mul_mv_ext_q3_K_f32_r1_5\")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q3_K, 256, dequantize_q3_K>;\n\ntemplate\nvoid kernel_mul_mv_t_t_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n constexpr short NW = N_SIMDWIDTH;\n constexpr short NB = 32;\n constexpr short NF = 8;\n\n const int nb = args.ne00/NB;\n\n const int r0 = tgpig.x*NR0;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n //const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n //device const T0 * x = (device const T0 *) (src0 + offset0);\n device const T1 * y = (device const T1 *) (src1 + offset1);\n\n // pointers to src0 rows\n device const T0 * ax [NR0];\n FOR_UNROLL (short row = 0; row < NR0; ++row) {\n const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n\n ax[row] = (device const T0 *) ((device char *) src0 + offset0);\n }\n\n float sumf[NR0] = { 0.f };\n\n const short ix = tiisg/(NW/NF);\n const short il = tiisg%(NW/NF);\n\n const int ib0 = sgitg*NF + ix;\n\n T1 yl[NF];\n\n device const T1 * yb = y + (ib0*NB + il*NF);\n\n for (int ib = ib0; ib < nb; ib += NSG*NF) {\n for (short i = 0; i < NF; ++i) {\n yl[i] = yb[i];\n }\n\n for (short row = 0; row < NR0; row++) {\n device const T0 * xb = ax[row] + (ib*NB + il*NF);\n\n float sumq = 0.f;\n FOR_UNROLL (short i = 0; i < NF; ++i) {\n sumq += xb[i] * yl[i];\n }\n\n sumf[row] += sumq;\n }\n\n yb += NSG*NF*NW;\n }\n\n for (int i = nb*NB + sgitg*NW + tiisg; i < args.ne00; i += NW*NSG) {\n for (short row = 0; row < NR0; row++) {\n sumf[row] += ax[row][i] * y[i];\n }\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem);\n}\n\ntemplate\nvoid kernel_mul_mv_t_t_disp(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n switch (args.nr0) {\n //case 1: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;\n case 2: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;\n //case 3: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;\n //case 4: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;\n }\n}\n\ntemplate\nkernel void kernel_mul_mv_t_t(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n kernel_mul_mv_t_t_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntypedef decltype(kernel_mul_mv_t_t) mul_mv_t_t;\n\ntemplate [[host_name(\"kernel_mul_mv_f32_f32\")]] kernel mul_mv_t_t kernel_mul_mv_t_t;\ntemplate [[host_name(\"kernel_mul_mv_f16_f32\")]] kernel mul_mv_t_t kernel_mul_mv_t_t;\ntemplate [[host_name(\"kernel_mul_mv_f16_f16\")]] kernel mul_mv_t_t kernel_mul_mv_t_t;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_mul_mv_bf16_f32\")]] kernel mul_mv_t_t kernel_mul_mv_t_t;\ntemplate [[host_name(\"kernel_mul_mv_bf16_bf16\")]] kernel mul_mv_t_t kernel_mul_mv_t_t;\n#endif\n\ntemplate\nvoid kernel_mul_mv_t_t_4_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n constexpr short NW = N_SIMDWIDTH;\n constexpr short NB = 32;\n constexpr short NF = 16;\n constexpr short NF4 = NF/4;\n\n const int nb = args.ne00/NB;\n\n const int r0 = tgpig.x*NR0;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n //const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const T1 * y = (device const T1 *) (src1 + offset1);\n device const T14 * y4 = (device const T14 *) (src1 + offset1);\n\n // pointers to src0 rows\n device const T0 * ax [NR0];\n device const T04 * ax4[NR0];\n FOR_UNROLL (short row = 0; row < NR0; ++row) {\n const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n\n ax [row] = (device const T0 *) ((device char *) src0 + offset0);\n ax4[row] = (device const T04 *) ((device char *) src0 + offset0);\n }\n\n float sumf[NR0] = { 0.f };\n\n const short ix = tiisg/(NW/NF);\n const short il = tiisg%(NW/NF);\n\n const int ib0 = sgitg*NF + ix;\n\n T14 yl4[NF4];\n\n device const T14 * yb4 = y4 + (ib0*NB + il*NF)/4;\n\n for (int ib = ib0; ib < nb; ib += NSG*NF) {\n for (short i = 0; i < NF4; ++i) {\n yl4[i] = yb4[i];\n }\n\n for (short row = 0; row < NR0; row++) {\n device const T04 * xb4 = ax4[row] + (ib*NB + il*NF)/4;\n\n float sumq = 0.f;\n FOR_UNROLL (short i = 0; i < NF4; ++i) {\n sumq += dot(float4(xb4[i]), float4(yl4[i]));\n }\n\n sumf[row] += sumq;\n }\n\n yb4 += NSG*NF*NW/4;\n }\n\n for (int i = nb*NB + sgitg*NW + tiisg; i < args.ne00; i += NW*NSG) {\n for (short row = 0; row < NR0; row++) {\n sumf[row] += ax[row][i] * y[i];\n }\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem);\n}\n\ntemplate\nvoid kernel_mul_mv_t_t_4_disp(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n switch (args.nr0) {\n //case 1: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;\n case 2: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;\n //case 3: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;\n //case 4: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;\n };\n}\n\ntemplate\nkernel void kernel_mul_mv_t_t_4(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n kernel_mul_mv_t_t_4_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntypedef decltype(kernel_mul_mv_t_t_4) mul_mv_t_t_4;\n\ntemplate [[host_name(\"kernel_mul_mv_f32_f32_4\")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4;\ntemplate [[host_name(\"kernel_mul_mv_f16_f32_4\")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4;\ntemplate [[host_name(\"kernel_mul_mv_f16_f16_4\")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_mul_mv_bf16_f32_4\")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4;\ntemplate [[host_name(\"kernel_mul_mv_bf16_bf16_4\")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4;\n#endif\n\ntemplate\nvoid kernel_mul_mv_t_t_short_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig,\n ushort tiisg) {\n const int r0 = tgpig.x*32 + tiisg;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n if (r0 >= args.ne01) {\n return;\n }\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n\n device const T0 * x = (device const T0 *) (src0 + offset0);\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1;\n\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const T1 * y = (device const T1 *) (src1 + offset1);\n\n float res = 0.0f;\n\n for (int i = 0; i < args.ne00; ++i) {\n res += (float) x[i] * (float) y[i];\n }\n\n dst_f32[(uint64_t)r1*args.ne0 + r0] = res;\n}\n\ntemplate\nkernel void kernel_mul_mv_t_t_short(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]]) {\n kernel_mul_mv_t_t_short_impl(\n args,\n src0,\n src1,\n dst,\n tgpig,\n tiisg);\n}\n\ntypedef decltype(kernel_mul_mv_t_t_short) mul_mv_t_t_short_t;\n\ntemplate [[host_name(\"kernel_mul_mv_f32_f32_short\")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short;\ntemplate [[host_name(\"kernel_mul_mv_f16_f32_short\")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short;\ntemplate [[host_name(\"kernel_mul_mv_f16_f16_short\")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_mul_mv_bf16_f32_short\")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short;\ntemplate [[host_name(\"kernel_mul_mv_bf16_bf16_short\")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short;\n#endif\n\nconstant bool FC_rope_is_imrope [[function_constant(FC_ROPE + 0)]];\n\nstatic float rope_yarn_ramp(const float low, const float high, const int i0) {\n const float y = (i0 / 2 - low) / max(0.001f, high - low);\n return 1.0f - min(1.0f, max(0.0f, y));\n}\n\n// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn\n// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.\nstatic void rope_yarn(\n float theta_extrap, float freq_scale, float corr_dims[2], int i0, float ext_factor, float mscale,\n thread float * cos_theta, thread float * sin_theta) {\n // Get n-d rotational scaling corrected for extrapolation\n float theta_interp = freq_scale * theta_extrap;\n float theta = theta_interp;\n if (ext_factor != 0.0f) {\n float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;\n theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;\n\n // Get n-d magnitude scaling corrected for interpolation\n mscale *= 1.0f + 0.1f * log(1.0f / freq_scale);\n }\n *cos_theta = cos(theta) * mscale;\n *sin_theta = sin(theta) * mscale;\n}\n\n// Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get\n// `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`\nstatic float rope_yarn_corr_factor(int n_dims, int n_ctx_orig, float n_rot, float base) {\n return n_dims * log(n_ctx_orig / (n_rot * 2 * M_PI_F)) / (2 * log(base));\n}\n\nstatic void rope_yarn_corr_dims(\n int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]\n) {\n // start and end correction dims\n dims[0] = max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_fast, freq_base)));\n dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_slow, freq_base)));\n}\n\ntemplate\nkernel void kernel_rope_norm(\n constant ggml_metal_kargs_rope & args,\n device const char * src0,\n device const char * src1,\n device const char * src2,\n device char * dst,\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 tptg [[threads_per_threadgroup]],\n uint3 tgpig[[threadgroup_position_in_grid]]) {\n const int i3 = tgpig[2];\n const int i2 = tgpig[1];\n const int i1 = tgpig[0];\n\n float corr_dims[2];\n rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims);\n\n device const int32_t * pos = (device const int32_t *) src1;\n\n const float theta_base = (float) pos[i2];\n const float inv_ndims = -1.f/args.n_dims;\n\n float cos_theta;\n float sin_theta;\n\n for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) {\n if (i0 < args.n_dims) {\n const int ic = i0/2;\n\n const float theta = theta_base * pow(args.freq_base, inv_ndims*i0);\n\n const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f;\n\n rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, i0, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta);\n\n device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00);\n device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n const float x0 = src[0];\n const float x1 = src[1];\n\n dst_data[0] = x0*cos_theta - x1*sin_theta;\n dst_data[1] = x0*sin_theta + x1*cos_theta;\n } else {\n device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00);\n device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\ntemplate\nkernel void kernel_rope_neox(\n constant ggml_metal_kargs_rope & args,\n device const char * src0,\n device const char * src1,\n device const char * src2,\n device char * dst,\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 tptg [[threads_per_threadgroup]],\n uint3 tgpig[[threadgroup_position_in_grid]]) {\n const int i3 = tgpig[2];\n const int i2 = tgpig[1];\n const int i1 = tgpig[0];\n\n float corr_dims[2];\n rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims);\n\n device const int32_t * pos = (device const int32_t *) src1;\n\n const float theta_base = (float) pos[i2];\n const float inv_ndims = -1.f/args.n_dims;\n\n float cos_theta;\n float sin_theta;\n\n for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) {\n if (i0 < args.n_dims) {\n const int ic = i0/2;\n\n const float theta = theta_base * pow(args.freq_base, inv_ndims*i0);\n\n const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f;\n\n rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, i0, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta);\n\n device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + ic*args.nb00);\n device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + ic*args.nb0);\n\n const float x0 = src[0];\n const float x1 = src[args.n_dims/2];\n\n dst_data[0] = x0*cos_theta - x1*sin_theta;\n dst_data[args.n_dims/2] = x0*sin_theta + x1*cos_theta;\n } else {\n device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00);\n device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\ntemplate\nkernel void kernel_rope_multi(\n constant ggml_metal_kargs_rope & args,\n device const char * src0,\n device const char * src1,\n device const char * src2,\n device char * dst,\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 tptg [[threads_per_threadgroup]],\n uint3 tgpig[[threadgroup_position_in_grid]]) {\n const int i3 = tgpig[2];\n const int i2 = tgpig[1];\n const int i1 = tgpig[0];\n\n float corr_dims[2];\n rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims);\n\n device const int32_t * pos = (device const int32_t *) src1;\n\n const float inv_ndims = -1.f/args.n_dims;\n\n float cos_theta;\n float sin_theta;\n\n for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) {\n if (i0 < args.n_dims) {\n const int ic = i0/2;\n\n // mrope theta calculations\n // note: the rest is the same as kernel_rope_neox\n const int sect_dims = args.sect_0 + args.sect_1 + args.sect_2 + args.sect_3;\n const int sec_w01 = args.sect_0 + args.sect_1; // end of section 1\n const int sec_w012 = args.sect_0 + args.sect_1 + args.sect_2; // end of section 2\n const int sector = ic % sect_dims;\n\n float theta_base;\n if (FC_rope_is_imrope) {\n if (sector % 3 == 1 && sector < 3 * args.sect_1) { // h\n theta_base = (float) pos[i2 + args.ne02 * 1];\n } else if (sector % 3 == 2 && sector < 3 * args.sect_2) { // w\n theta_base = (float) pos[i2 + args.ne02 * 2];\n } else if (sector % 3 == 0 && sector < 3 * args.sect_0) { // t\n theta_base = (float) pos[i2 + args.ne02 * 0];\n } else { // e\n theta_base = (float) pos[i2 + args.ne02 * 3];\n }\n } else {\n if (sector < args.sect_0) {\n theta_base = (float) pos[i2];\n } else if (sector < sec_w01) {\n theta_base = (float) pos[i2 + args.ne02 * 1];\n } else if (sector < sec_w012) {\n theta_base = (float) pos[i2 + args.ne02 * 2];\n } else {\n theta_base = (float) pos[i2 + args.ne02 * 3];\n }\n }\n // end of mrope\n\n const float theta = theta_base * pow(args.freq_base, inv_ndims*i0);\n\n const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f;\n\n rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, i0, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta);\n\n device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + ic*args.nb00);\n device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + ic*args.nb0);\n\n const float x0 = src[0];\n const float x1 = src[args.n_dims/2];\n\n dst_data[0] = x0*cos_theta - x1*sin_theta;\n dst_data[args.n_dims/2] = x0*sin_theta + x1*cos_theta;\n } else {\n device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00);\n device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\ntemplate\nkernel void kernel_rope_vision(\n constant ggml_metal_kargs_rope & args,\n device const char * src0,\n device const char * src1,\n device const char * src2,\n device char * dst,\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 tptg [[threads_per_threadgroup]],\n uint3 tgpig[[threadgroup_position_in_grid]]) {\n const int i3 = tgpig[2];\n const int i2 = tgpig[1];\n const int i1 = tgpig[0];\n\n float corr_dims[2];\n rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims);\n\n device const int32_t * pos = (device const int32_t *) src1;\n\n const float inv_ndims = -1.f/args.n_dims;\n\n float cos_theta;\n float sin_theta;\n\n for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) {\n if (i0 < 2*args.n_dims) { // different from kernel_rope_multi\n const int ic = i0/2;\n\n // mrope theta calculations (only support 2 dimensions)\n const int sect_dims = args.sect_0 + args.sect_1;\n const int sector = ic % sect_dims;\n\n float p;\n float theta_base;\n if (sector < args.sect_1) {\n p = (float) sector;\n theta_base = (float) pos[i2];\n } else {\n p = (float) sector - args.sect_0;\n theta_base = (float) pos[i2 + args.ne02];\n }\n\n const float theta = theta_base * pow(args.freq_base, 2.0f * inv_ndims * p);\n // end of mrope\n\n const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f;\n\n rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, i0, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta);\n\n device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + ic*args.nb00);\n device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + ic*args.nb0);\n\n const float x0 = src[0];\n const float x1 = src[args.n_dims]; // different from kernel_rope_multi\n\n dst_data[0] = x0*cos_theta - x1*sin_theta;\n dst_data[args.n_dims] = x0*sin_theta + x1*cos_theta; // different from kernel_rope_multi\n } else {\n device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00);\n device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\ntypedef decltype(kernel_rope_norm) kernel_rope_norm_t;\ntypedef decltype(kernel_rope_neox) kernel_rope_neox_t;\ntypedef decltype(kernel_rope_multi) kernel_rope_multi_t;\ntypedef decltype(kernel_rope_vision) kernel_rope_vision_t;\n\ntemplate [[host_name(\"kernel_rope_norm_f32\")]] kernel kernel_rope_norm_t kernel_rope_norm;\ntemplate [[host_name(\"kernel_rope_norm_f16\")]] kernel kernel_rope_norm_t kernel_rope_norm;\n\ntemplate [[host_name(\"kernel_rope_neox_f32\")]] kernel kernel_rope_neox_t kernel_rope_neox;\ntemplate [[host_name(\"kernel_rope_neox_f16\")]] kernel kernel_rope_neox_t kernel_rope_neox;\n\ntemplate [[host_name(\"kernel_rope_multi_f32\")]] kernel kernel_rope_multi_t kernel_rope_multi;\ntemplate [[host_name(\"kernel_rope_multi_f16\")]] kernel kernel_rope_multi_t kernel_rope_multi;\n\ntemplate [[host_name(\"kernel_rope_vision_f32\")]] kernel kernel_rope_vision_t kernel_rope_vision;\ntemplate [[host_name(\"kernel_rope_vision_f16\")]] kernel kernel_rope_vision_t kernel_rope_vision;\n\ntypedef void (im2col_t)(\n constant ggml_metal_kargs_im2col & args,\n device const float * x,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]);\n\ntemplate \nkernel void kernel_im2col(\n constant ggml_metal_kargs_im2col & args,\n device const float * x,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n// const int64_t IC = tgpg[0];\n const int64_t OH = tgpg[1];\n const int64_t OW = tgpg[2];\n\n const int64_t KH = ntg[1];\n const int64_t KW = ntg[2];\n\n int64_t in = tpitg[0];\n const int64_t ikh = tpitg[1];\n const int64_t ikw = tpitg[2];\n\n const int64_t iic = tgpig[0];\n const int64_t ioh = tgpig[1];\n const int64_t iow = tgpig[2];\n\n const int64_t iiw = iow*args.s0 + ikw*args.d0 - args.p0;\n const int64_t iih = ioh*args.s1 + ikh*args.d1 - args.p1;\n\n int64_t offset_dst = (in*OH*OW + ioh*OW + iow)*args.CHW + (iic*(KH*KW) + ikh*KW + ikw);\n\n device T * pdst = (device T *) (dst);\n\n if (iih < 0 || iih >= args.IH || iiw < 0 || iiw >= args.IW) {\n while (in < args.N) {\n pdst[offset_dst] = 0.0f;\n offset_dst += ntg[0]*args.CHW*OH*OW;\n\n in += ntg[0];\n }\n } else {\n int64_t offset_src = in*args.ofs0 + iic*args.ofs1 + iih*args.IW + iiw;\n\n while (in < args.N) {\n pdst[offset_dst] = x[offset_src];\n\n offset_dst += ntg[0]*args.CHW*OH*OW;\n offset_src += ntg[0]*args.ofs0;\n\n in += ntg[0];\n }\n }\n}\n\ntemplate [[host_name(\"kernel_im2col_f32\")]] kernel im2col_t kernel_im2col;\ntemplate [[host_name(\"kernel_im2col_f16\")]] kernel im2col_t kernel_im2col;\n\n// TODO: obsolete -- remove\n//typedef void (im2col_ext_t)(\n// constant ggml_metal_kargs_im2col & args,\n// device const float * x,\n// device char * dst,\n// uint3 tgpig[[threadgroup_position_in_grid]],\n// uint3 tgpg[[threadgroups_per_grid]],\n// uint3 tpitg[[thread_position_in_threadgroup]],\n// uint3 ntg[[threads_per_threadgroup]]);\n//\n//template \n//kernel void kernel_im2col_ext(\n// constant ggml_metal_kargs_im2col & args,\n// device const float * x,\n// device char * dst,\n// uint3 tgpig[[threadgroup_position_in_grid]],\n// uint3 tgpg[[threadgroups_per_grid]], // tgpg[0] = D x IC x KH x KW, CHW = IC x KH x KW\n// uint3 tpitg[[thread_position_in_threadgroup]],\n// uint3 ntg[[threads_per_threadgroup]]) { // [M, 1, 1]\n// const int64_t KHW = (int64_t)args.KHW;\n//\n// const int64_t d = tgpig[0] / args.CHW;\n// const int64_t chw = tgpig[0] % args.CHW;\n// const int64_t tgpig_0 = chw / KHW; // 0 ~ (IC - 1)\n// const int64_t HW = tgpig[0] % KHW;\n//\n// const int64_t tpitg_0 = (d * ntg[0]) + tpitg[0];\n// if (tpitg_0 >= args.N) {\n// return;\n// }\n//\n// const int64_t tpitg_1 = HW / args.KW;\n// const int64_t tpitg_2 = HW % args.KW;\n//\n// const int64_t iiw = tgpig[2] * args.s0 + tpitg_2 * args.d0 - args.p0;\n// const int64_t iih = tgpig[1] * args.s1 + tpitg_1 * args.d1 - args.p1;\n//\n// const int64_t offset_dst =\n// (tpitg_0 * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * args.CHW +\n// (tgpig_0 * KHW + tpitg_1 * args.KW + tpitg_2);\n//\n// device T * pdst = (device T *) (dst);\n//\n// if (iih < 0 || iih >= args.IH || iiw < 0 || iiw >= args.IW) {\n// pdst[offset_dst] = 0.0f;\n// } else {\n// const int64_t offset_src = tpitg_0 * args.ofs0 + tgpig_0 * args.ofs1;\n// pdst[offset_dst] = x[offset_src + iih * args.IW + iiw];\n// }\n//}\n//\n//template [[host_name(\"kernel_im2col_ext_f32\")]] kernel im2col_ext_t kernel_im2col_ext;\n//template [[host_name(\"kernel_im2col_ext_f16\")]] kernel im2col_ext_t kernel_im2col_ext;\n\ntemplate \nkernel void kernel_conv_2d(\n constant ggml_metal_kargs_conv_2d & args,\n device const char * weights,\n device const char * src,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n const uint threads_per_tg = ntg.x * ntg.y * ntg.z;\n const uint tg_index = (tgpig.z * tgpg.y + tgpig.y) * tgpg.x + tgpig.x;\n const uint local_thread = tpitg.z * (ntg.x * ntg.y) + tpitg.y * ntg.x + tpitg.x;\n const uint thread_index = tg_index * threads_per_tg + local_thread;\n const uint64_t total_threads = (uint64_t) threads_per_tg * tgpg.x * tgpg.y * tgpg.z;\n const uint64_t total_outputs = (uint64_t) args.N * args.OC * args.OH * args.OW;\n\n for (uint64_t index = thread_index; index < total_outputs; index += total_threads) {\n uint64_t tmp = index;\n\n const int32_t ow = tmp % args.OW; tmp /= args.OW;\n const int32_t oh = tmp % args.OH; tmp /= args.OH;\n const int32_t oc = tmp % args.OC; tmp /= args.OC;\n const int32_t n = tmp;\n\n float acc = 0.0f;\n\n const int32_t base_x = ow*args.s0 - args.p0;\n const int32_t base_y = oh*args.s1 - args.p1;\n\n int32_t ky_start = 0;\n if (base_y < 0) {\n ky_start = (-base_y + args.d1 - 1)/args.d1;\n }\n int32_t ky_end = args.KH;\n const int32_t y_max = args.IH - 1 - base_y;\n if (y_max < 0) {\n ky_end = ky_start;\n } else if (base_y + (args.KH - 1)*args.d1 >= args.IH) {\n ky_end = min(ky_end, y_max/args.d1 + 1);\n }\n\n int32_t kx_start = 0;\n if (base_x < 0) {\n kx_start = (-base_x + args.d0 - 1)/args.d0;\n }\n int32_t kx_end = args.KW;\n const int32_t x_max = args.IW - 1 - base_x;\n if (x_max < 0) {\n kx_end = kx_start;\n } else if (base_x + (args.KW - 1)*args.d0 >= args.IW) {\n kx_end = min(kx_end, x_max/args.d0 + 1);\n }\n\n if (ky_start < ky_end && kx_start < kx_end) {\n const uint64_t src_base_n = (uint64_t) n * args.nb13;\n const uint64_t w_base_oc = (uint64_t) oc * args.nb03;\n\n for (int32_t ic = 0; ic < args.IC; ++ic) {\n const uint64_t src_base_nc = src_base_n + (uint64_t) ic * args.nb12;\n const uint64_t w_base_ocic = w_base_oc + (uint64_t) ic * args.nb02;\n\n for (int32_t ky = ky_start; ky < ky_end; ++ky) {\n const int32_t iy = base_y + ky*args.d1;\n const uint64_t src_base_row = src_base_nc + (uint64_t) iy * args.nb11;\n const uint64_t w_base_row = w_base_ocic + (uint64_t) ky * args.nb01;\n\n for (int32_t kx = kx_start; kx < kx_end; ++kx) {\n const int32_t ix = base_x + kx*args.d0;\n const uint64_t src_offs = src_base_row + (uint64_t) ix * args.nb10;\n const uint64_t w_offs = w_base_row + (uint64_t) kx * args.nb00;\n\n const float x = *(device const float *)(src + src_offs);\n const float w = (float) (*(device const TK *)(weights + w_offs));\n\n acc += x * w;\n }\n }\n }\n }\n\n const uint64_t dst_offs =\n (uint64_t) n * args.nb3 +\n (uint64_t) oc * args.nb2 +\n (uint64_t) oh * args.nb1 +\n (uint64_t) ow * args.nb0;\n\n *(device float *)(dst + dst_offs) = acc;\n }\n}\n\ntemplate [[host_name(\"kernel_conv_2d_f32_f32\")]]\nkernel void kernel_conv_2d(\n constant ggml_metal_kargs_conv_2d & args,\n device const char * weights,\n device const char * src,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]);\n\ntemplate [[host_name(\"kernel_conv_2d_f16_f32\")]]\nkernel void kernel_conv_2d(\n constant ggml_metal_kargs_conv_2d & args,\n device const char * weights,\n device const char * src,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]);\n\ntypedef void (conv_transpose_1d_t)(\n constant ggml_metal_kargs_conv_transpose_1d & args,\n device const float * src0,\n device const float * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]]);\n\ntemplate \nkernel void kernel_conv_transpose_1d(\n constant ggml_metal_kargs_conv_transpose_1d & args,\n device const T * src0,\n device const float * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]]) {\n\n float v = 0.0f;\n\n for (int64_t c = 0; c < args.IC; c++) {\n const int32_t kernel_offset = c * tgpg[1] * args.K + args.K * tgpig[1];\n const int32_t input_offset = c * args.IL;\n\n for (int64_t i = 0; i < args.IL; i++) {\n if (tgpig[0] >= i * args.s0 && tgpig[0] < i * args.s0 + args.K) {\n v += src0[kernel_offset + tgpig[0] - i * args.s0] * src1[input_offset + i];\n }\n }\n }\n\n device float * dst_ptr = (device float *) (dst + tgpig[0] * args.nb0 + tgpig[1] * args.nb1);\n\n dst_ptr[0] = v;\n}\n\ntemplate [[host_name(\"kernel_conv_transpose_1d_f32_f32\")]]\nkernel void kernel_conv_transpose_1d(\n constant ggml_metal_kargs_conv_transpose_1d & args,\n device const float * src0,\n device const float * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]]);\n\ntemplate [[host_name(\"kernel_conv_transpose_1d_f16_f32\")]]\nkernel void kernel_conv_transpose_1d(\n constant ggml_metal_kargs_conv_transpose_1d & args,\n device const half * src0,\n device const float * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]]);\n\n\ntypedef void (conv_transpose_2d_t)(\n constant ggml_metal_kargs_conv_transpose_2d & args,\n device const float * src0,\n device const float * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]]);\n\ntemplate \nkernel void kernel_conv_transpose_2d(\n constant ggml_metal_kargs_conv_transpose_2d & args,\n device const T * src0,\n device const float * src1,\n device char * dst,\n threadgroup float * shared_sum [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n const int64_t out_x = tgpig[0];\n const int64_t out_y = tgpig[1];\n const int64_t out_c = tgpig[2];\n\n const int64_t kw = tpitg[0];\n const int64_t kh = tpitg[1];\n\n float v = 0.0f;\n\n for (int64_t in_c = 0; in_c < args.IC; in_c++) {\n int64_t in_y = out_y - kh;\n\n if (in_y < 0 || in_y % args.s0) continue;\n\n in_y /= args.s0;\n\n if (in_y >= args.IH) continue;\n\n int64_t in_x = out_x - kw;\n\n if (in_x < 0 || in_x % args.s0) continue;\n\n in_x /= args.s0;\n\n if (in_x >= args.IW) continue;\n\n const int64_t input_idx = (args.IW * args.IH) * in_c + (args.IW) * in_y + in_x;\n const int64_t kernel_idx = (args.KH * args.KW * args.OC) * in_c + (args.KH * args.KW) * out_c + (args.KW) * kh + kw;\n\n v += (float)src0[kernel_idx] * src1[input_idx];\n }\n\n const uint tid = tpitg.y * ntg.x + tpitg.x;\n shared_sum[tid] = v;\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (tid == 0) {\n float total = 0.0f;\n const uint num_threads = ntg.x * ntg.y;\n for (uint i = 0; i < num_threads; i++) {\n total += shared_sum[i];\n }\n\n device float * dst_ptr = (device float *) (dst + out_x*args.nb0 + out_y * args.nb1 + out_c*args.nb2);\n dst_ptr[0] = total;\n }\n}\n\ntemplate [[host_name(\"kernel_conv_transpose_2d_f32_f32\")]]\nkernel void kernel_conv_transpose_2d(\n constant ggml_metal_kargs_conv_transpose_2d & args,\n device const float * src0,\n device const float * src1,\n device char * dst,\n threadgroup float * shared_sum [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]);\n\ntemplate [[host_name(\"kernel_conv_transpose_2d_f16_f32\")]]\nkernel void kernel_conv_transpose_2d(\n constant ggml_metal_kargs_conv_transpose_2d & args,\n device const half * src0,\n device const float * src1,\n device char * dst,\n threadgroup float * shared_sum [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]);\n\nconstant bool FC_upscale_aa [[function_constant(FC_UPSCALE + 0)]];\n\nkernel void kernel_upscale_nearest_f32(\n constant ggml_metal_kargs_upscale & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n const int64_t i3 = tgpig.z;\n const int64_t i2 = tgpig.y;\n const int64_t i1 = tgpig.x;\n\n const int64_t i03 = i3/args.sf3;\n const int64_t i02 = i2/args.sf2;\n const int64_t i01 = i1/args.sf1;\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n const int64_t i00 = i0/args.sf0;\n\n device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00);\n device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n dst_ptr[0] = src0_ptr[0];\n }\n}\n\nstatic inline float bilinear_tri(float x) {\n return MAX(0.0f, 1.0f - fabs(x));\n}\n\nkernel void kernel_upscale_bilinear_f32(\n constant ggml_metal_kargs_upscale & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n const int64_t i3 = tgpig.z;\n const int64_t i2 = tgpig.y;\n const int64_t i1 = tgpig.x;\n\n const int64_t i03 = i3 / args.sf3;\n const int64_t i02 = i2 / args.sf2;\n\n const float f01 = ((float)i1 + args.poffs) / args.sf1 - args.poffs;\n const int64_t i01 = MAX(0, MIN(args.ne01 - 1, (int64_t)floor(f01)));\n const int64_t i01p = MAX(0, MIN(args.ne01 - 1, i01 + 1));\n const float fd1 = MAX(0.0f, MIN(1.0f, f01 - (float)i01));\n\n src0 += i03*args.nb03 + i02*args.nb02;\n\n device float * dst_ptr = (device float *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1);\n\n if (FC_upscale_aa) {\n const float support0 = MAX(1.0f, 1.0f / args.sf0);\n const float invscale0 = 1.0f / support0;\n const float support1 = MAX(1.0f, 1.0f / args.sf1);\n const float invscale1 = 1.0f / support1;\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n const float f00 = ((float)i0 + args.poffs) / args.sf0 - args.poffs;\n\n int64_t x_min = MAX((int64_t)0, (int64_t)floor(f00 - support0 + args.poffs));\n int64_t x_max = MIN(args.ne00, (int64_t)ceil (f00 + support0 + args.poffs));\n\n int64_t y_min = MAX((int64_t)0, (int64_t)floor(f01 - support1 + args.poffs));\n int64_t y_max = MIN(args.ne01, (int64_t)ceil (f01 + support1 + args.poffs));\n\n float sum = 0.0f;\n float wsum = 0.0f;\n\n for (int64_t sy = y_min; sy < y_max; ++sy) {\n const float wy = MAX(0.0f, 1.0f - fabs((float)sy - f01) * invscale1);\n for (int64_t sx = x_min; sx < x_max; ++sx) {\n const float wx = MAX(0.0f, 1.0f - fabs((float)sx - f00) * invscale0);\n const float w = wx * wy;\n const device const float * src_ptr = (device const float *)(src0 + sy*args.nb01 + sx*args.nb00);\n sum += (*src_ptr) * w;\n wsum += w;\n }\n }\n\n const float v = (wsum > 0.0f) ? (sum / wsum) : 0.0f;\n dst_ptr[i0] = v;\n }\n } else {\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n const float f00 = ((float)i0 + args.poffs) / args.sf0 - args.poffs;\n const int64_t i00 = MAX(0, MIN(args.ne00 - 1, (int64_t)floor(f00)));\n const int64_t i00p = MAX(0, MIN(args.ne00 - 1, i00 + 1));\n const float fd0 = MAX(0.0f, MIN(1.0f, f00 - (float)i00));\n\n device const float * src00 = (device const float *)(src0 + i01*args.nb01 + i00*args.nb00);\n device const float * src10 = (device const float *)(src0 + i01*args.nb01 + i00p*args.nb00);\n device const float * src01 = (device const float *)(src0 + i01p*args.nb01 + i00*args.nb00);\n device const float * src11 = (device const float *)(src0 + i01p*args.nb01 + i00p*args.nb00);\n\n const float v =\n (*src00) * (1.0f - fd0) * (1.0f - fd1) +\n (*src10) * fd0 * (1.0f - fd1) +\n (*src01) * (1.0f - fd0) * fd1 +\n (*src11) * fd0 * fd1;\n\n dst_ptr[i0] = v;\n }\n }\n}\n\nstatic inline float bicubic_weight1(float x) {\n const float a = -0.75f;\n return ((a + 2) * x - (a + 3)) * x * x + 1;\n}\n\nstatic inline float bicubic_weight2(float x) {\n const float a = -0.75f;\n return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a;\n}\n\nkernel void kernel_upscale_bicubic_f32(\n constant ggml_metal_kargs_upscale & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n const int64_t i3 = tgpig.z;\n const int64_t i2 = tgpig.y;\n const int64_t i1 = tgpig.x;\n\n const int64_t i03 = i3 / args.sf3;\n const int64_t i02 = i2 / args.sf2;\n\n const float f01 = ((float)i1 + args.poffs) / args.sf1 - args.poffs;\n const int64_t i01 = (int64_t)floor(f01);\n const float fd1 = f01 - (float)i01;\n\n const float w_y0 = bicubic_weight2(fd1 + 1.0f);\n const float w_y1 = bicubic_weight1(fd1);\n const float w_y2 = bicubic_weight1(1.0f - fd1);\n const float w_y3 = bicubic_weight2(2.0f - fd1);\n\n const device const char * src_slice = src0 + i03 * args.nb03 + i02 * args.nb02;\n\n device float * dst_ptr = (device float *)(dst + i3 * args.nb3 + i2 * args.nb2 + i1 * args.nb1);\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n const float f00 = ((float)i0 + args.poffs) / args.sf0 - args.poffs;\n const int64_t i00 = (int64_t)floor(f00);\n const float fd0 = f00 - (float)i00;\n\n const float w_x0 = bicubic_weight2(fd0 + 1.0f);\n const float w_x1 = bicubic_weight1(fd0);\n const float w_x2 = bicubic_weight1(1.0f - fd0);\n const float w_x3 = bicubic_weight2(2.0f - fd0);\n\n float sum = 0.0f;\n\n for (int dy = -1; dy <= 2; ++dy) {\n const int64_t iy = MAX(0, MIN(args.ne01 - 1, i01 + dy));\n const float wy = (dy == -1) ? w_y0 : (dy == 0) ? w_y1 : (dy == 1) ? w_y2 : w_y3;\n\n for (int dx = -1; dx <= 2; ++dx) {\n const int64_t ix = MAX(0, MIN(args.ne00 - 1, i00 + dx));\n const float wx = (dx == -1) ? w_x0 : (dx == 0) ? w_x1 : (dx == 1) ? w_x2 : w_x3;\n\n const device const float * src_ptr = (device const float *)(src_slice + iy * args.nb01 + ix * args.nb00);\n sum += (*src_ptr) * wx * wy;\n }\n }\n\n dst_ptr[i0] = sum;\n }\n}\n\nkernel void kernel_pad_f32(\n constant ggml_metal_kargs_pad & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n const int64_t i3 = tgpig.z;\n const int64_t i2 = tgpig.y;\n const int64_t i1 = tgpig.x;\n\n const int64_t i03 = i3;\n const int64_t i02 = i2;\n const int64_t i01 = i1;\n\n device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01);\n device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1);\n\n if (i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) {\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n if (i0 < args.ne00) {\n dst_ptr[i0] = src0_ptr[i0];\n } else {\n dst_ptr[i0] = 0.0f;\n }\n }\n\n return;\n }\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n dst_ptr[i0] = 0.0f;\n }\n}\n\nkernel void kernel_pad_reflect_1d_f32(\n constant ggml_metal_kargs_pad_reflect_1d & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tgpg[[threadgroups_per_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n const int64_t i3 = tgpig.z;\n const int64_t i2 = tgpig.y;\n const int64_t i1 = tgpig.x;\n\n const int64_t i03 = i3;\n const int64_t i02 = i2;\n const int64_t i01 = i1;\n\n device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01);\n device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1);\n\n if (i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) {\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n if (i0 < args.p0) {\n dst_ptr[i0] = src0_ptr[args.p0 - i0];\n } else if (i0 < args.ne0 - args.p1) {\n dst_ptr[i0] = src0_ptr[i0 - args.p0];\n } else {\n dst_ptr[i0] = src0_ptr[(args.ne0 - args.p1 - args.p0) - (args.p1 + 1 - (args.ne0 - i0)) - 1];\n }\n }\n }\n}\n\nkernel void kernel_arange_f32(\n constant ggml_metal_kargs_arange & args,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n device float * dst_ptr = (device float *) dst;\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n dst_ptr[i0] = args.start + args.step * i0;\n }\n}\n\nkernel void kernel_timestep_embedding_f32(\n constant ggml_metal_kargs_timestep_embedding & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint3 tpitg[[thread_position_in_threadgroup]],\n uint3 ntg[[threads_per_threadgroup]]) {\n\n int i = tgpig.x;\n device float * embed_data = (device float *)(dst + i*args.nb1);\n\n int half_ = args.dim / 2;\n for (int j = tpitg.x; j < half_; j += ntg.x) {\n float timestep = ((device float *)src0)[i];\n float freq = (float)exp(-log((float)args.max_period) * j / half_);\n float arg = timestep * freq;\n embed_data[j ] = cos(arg);\n embed_data[j + half_] = sin(arg);\n }\n\n if (args.dim % 2 != 0 && tpitg.x == 0) {\n embed_data[2 * half_] = 0.f;\n }\n}\n\n// bitonic sort implementation following the CUDA kernels as reference\ntypedef void (argsort_t)(\n constant ggml_metal_kargs_argsort & args,\n device const char * src0,\n device int32_t * dst,\n threadgroup int32_t * shmem_i32 [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]);\n\ntemplate\nkernel void kernel_argsort_f32_i32(\n constant ggml_metal_kargs_argsort & args,\n device const char * src0,\n device int32_t * dst,\n threadgroup int32_t * shmem_i32 [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n // bitonic sort\n const int col = tpitg[0];\n const int ib = tgpig[0] / args.ne01;\n\n const int i00 = ib*ntg.x;\n const int i01 = tgpig[0] % args.ne01;\n const int i02 = tgpig[1];\n const int i03 = tgpig[2];\n\n device const float * src0_row = (device const float *) (src0 + args.nb01*i01 + args.nb02*i02 + args.nb03*i03);\n\n // initialize indices\n shmem_i32[col] = i00 + col;\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n for (int k = 2; k <= ntg.x; k *= 2) {\n for (int j = k / 2; j > 0; j /= 2) {\n int ixj = col ^ j;\n if (ixj > col) {\n if ((col & k) == 0) {\n if (shmem_i32[col] >= args.ne00 ||\n (shmem_i32[ixj] < args.ne00 && (order == GGML_SORT_ORDER_ASC ?\n src0_row[shmem_i32[col]] > src0_row[shmem_i32[ixj]] :\n src0_row[shmem_i32[col]] < src0_row[shmem_i32[ixj]]))\n ) {\n SWAP(shmem_i32[col], shmem_i32[ixj]);\n }\n } else {\n if (shmem_i32[ixj] >= args.ne00 ||\n (shmem_i32[col] < args.ne00 && (order == GGML_SORT_ORDER_ASC ?\n src0_row[shmem_i32[col]] < src0_row[shmem_i32[ixj]] :\n src0_row[shmem_i32[col]] > src0_row[shmem_i32[ixj]]))\n ) {\n SWAP(shmem_i32[col], shmem_i32[ixj]);\n }\n }\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n }\n }\n\n const int64_t i0 = ib*args.top_k;\n\n // copy the result to dst without the padding\n if (i0 + col < args.ne0 && col < args.top_k) {\n dst += i0 + args.ne0*i01 + args.ne0*args.ne1*i02 + args.ne0*args.ne1*args.ne2*i03;\n\n dst[col] = shmem_i32[col];\n }\n}\n\ntemplate [[host_name(\"kernel_argsort_f32_i32_asc\")]] kernel argsort_t kernel_argsort_f32_i32;\ntemplate [[host_name(\"kernel_argsort_f32_i32_desc\")]] kernel argsort_t kernel_argsort_f32_i32;\n\ntypedef void (argsort_merge_t)(\n constant ggml_metal_kargs_argsort_merge & args,\n device const char * src0,\n device const int32_t * tmp,\n device int32_t * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]);\n\ntemplate\nkernel void kernel_argsort_merge_f32_i32(\n constant ggml_metal_kargs_argsort_merge & args,\n device const char * src0,\n device const int32_t * tmp,\n device int32_t * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n\n const int im = tgpig[0] / args.ne01;\n const int i01 = tgpig[0] % args.ne01;\n const int i02 = tgpig[1];\n const int i03 = tgpig[2];\n\n const int start = im * (2 * args.len);\n\n const int len0 = MIN(args.len, MAX(0, args.ne0 - (int)(start)));\n const int len1 = MIN(args.len, MAX(0, args.ne0 - (int)(start + args.len)));\n\n const int total = len0 + len1;\n\n device const int32_t * tmp0 = tmp + start\n + i01*args.ne0\n + i02*args.ne0*args.ne01\n + i03*args.ne0*args.ne01*args.ne02;\n\n device const int32_t * tmp1 = tmp0 + args.len;\n\n dst += start\n + i01*args.top_k\n + i02*args.top_k*args.ne01\n + i03*args.top_k*args.ne01*args.ne02;\n\n device const float * src0_row = (device const float *)(src0\n + args.nb01*i01\n + args.nb02*i02\n + args.nb03*i03);\n\n if (total == 0) {\n return;\n }\n\n const int chunk = (total + ntg.x - 1) / ntg.x;\n\n const int k0 = tpitg.x * chunk;\n const int k1 = MIN(MIN(k0 + chunk, total), args.top_k);\n\n if (k0 >= args.top_k) {\n return;\n }\n\n if (k0 >= total) {\n return;\n }\n\n int low = k0 > len1 ? k0 - len1 : 0;\n int high = MIN(k0, len0);\n\n // binary-search partition (i, j) such that i + j = k\n while (low < high) {\n const int mid = (low + high) >> 1;\n\n const int32_t idx0 = tmp0[mid];\n const int32_t idx1 = tmp1[k0 - mid - 1];\n\n const float val0 = src0_row[idx0];\n const float val1 = src0_row[idx1];\n\n bool take_left;\n if (order == GGML_SORT_ORDER_ASC) {\n take_left = (val0 <= val1);\n } else {\n take_left = (val0 >= val1);\n }\n\n if (take_left) {\n low = mid + 1;\n } else {\n high = mid;\n }\n }\n\n int i = low;\n int j = k0 - i;\n\n // keep the merge fronts into registers\n int32_t idx0 = 0;\n float val0 = 0.0f;\n if (i < len0) {\n idx0 = tmp0[i];\n val0 = src0_row[idx0];\n }\n\n int32_t idx1 = 0;\n float val1 = 0.0f;\n if (j < len1) {\n idx1 = tmp1[j];\n val1 = src0_row[idx1];\n }\n\n for (int k = k0; k < k1; ++k) {\n int32_t out_idx;\n\n if (i >= len0) {\n while (k < k1) {\n dst[k++] = tmp1[j++];\n }\n break;\n } else if (j >= len1) {\n while (k < k1) {\n dst[k++] = tmp0[i++];\n }\n break;\n } else {\n bool take_left;\n\n if (order == GGML_SORT_ORDER_ASC) {\n take_left = (val0 <= val1);\n } else {\n take_left = (val0 >= val1);\n }\n\n if (take_left) {\n out_idx = idx0;\n ++i;\n if (i < len0) {\n idx0 = tmp0[i];\n val0 = src0_row[idx0];\n }\n } else {\n out_idx = idx1;\n ++j;\n if (j < len1) {\n idx1 = tmp1[j];\n val1 = src0_row[idx1];\n }\n }\n }\n\n dst[k] = out_idx;\n }\n}\n\ntemplate [[host_name(\"kernel_argsort_merge_f32_i32_asc\")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32;\ntemplate [[host_name(\"kernel_argsort_merge_f32_i32_desc\")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32;\n\nconstant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]];\n\nconstant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]];\n\n// pad the last chunk of C elements of k and v into a an extra pad buffer\nkernel void kernel_flash_attn_ext_pad(\n constant ggml_metal_kargs_flash_attn_ext_pad & args,\n device const char * k,\n device const char * v,\n device const char * mask,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int32_t C = FC_flash_attn_ext_pad_ncpsg;\n\n device char * k_pad = dst;\n device char * v_pad = k_pad + args.nb11*C*args.ne_12_2*args.ne_12_3;\n device char * mask_pad = v_pad + args.nb21*C*args.ne_12_2*args.ne_12_3;\n\n const int32_t icp = args.ne11 % C;\n const int32_t ic0 = args.ne11 - icp;\n\n const int32_t i1 = tgpig[0];\n const int32_t i2 = tgpig[1];\n const int32_t i3 = tgpig[2];\n\n if (i2 < args.ne_12_2 && i3 < args.ne_12_3) {\n device const char * k_src = k + args.nb11*(ic0 + i1) + args.nb12*i2 + args.nb13*i3;\n device const char * v_src = v + args.nb21*(ic0 + i1) + args.nb22*i2 + args.nb23*i3;\n\n device char * k_dst = k_pad + args.nb11*i1 + args.nb11*C*i2 + args.nb11*C*args.ne_12_2*i3;\n device char * v_dst = v_pad + args.nb21*i1 + args.nb21*C*i2 + args.nb21*C*args.ne_12_2*i3;\n\n if (i1 >= icp) {\n // here it is not important the exact value that will be used as we rely on masking out the scores in the attention\n for (uint64_t i = tiitg; i < args.nb11; i += ntg.x) {\n k_dst[i] = 0;\n }\n for (uint64_t i = tiitg; i < args.nb21; i += ntg.x) {\n v_dst[i] = 0;\n }\n } else {\n for (uint64_t i = tiitg; i < args.nb11; i += ntg.x) {\n k_dst[i] = k_src[i];\n }\n for (uint64_t i = tiitg; i < args.nb21; i += ntg.x) {\n v_dst[i] = v_src[i];\n }\n }\n }\n\n if (FC_flash_attn_ext_pad_has_mask) {\n if (i2 < args.ne32 && i3 < args.ne33) {\n for (int ib = i1; ib < args.ne31; ib += C) {\n device const half * mask_src = (device const half *)(mask + args.nb31*ib + args.nb32*i2 + args.nb33*i3) + ic0;\n device half * mask_dst = (device half *)(mask_pad) + C*ib + C*args.ne31*i2 + C*args.ne31*args.ne32*i3;\n\n for (int i = tiitg; i < C; i += ntg.x) {\n if (i >= icp) {\n mask_dst[i] = -MAXHALF;\n } else {\n mask_dst[i] = mask_src[i];\n }\n }\n }\n }\n }\n}\n\nconstant int32_t FC_flash_attn_ext_blk_nqptg [[function_constant(FC_FLASH_ATTN_EXT_BLK + 24)]];\nconstant int32_t FC_flash_attn_ext_blk_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_BLK + 25)]];\n\n// scan the blocks of the mask that are not masked\n// 0 - masked (i.e. full of -INF, skip)\n// 1 - not masked (i.e. at least one element of the mask is not -INF)\n// 2 - all zero\nkernel void kernel_flash_attn_ext_blk(\n constant ggml_metal_kargs_flash_attn_ext_blk & args,\n device const char * mask,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]]) {\n // block size C x Q\n const int32_t Q = FC_flash_attn_ext_blk_nqptg;\n const int32_t C = FC_flash_attn_ext_blk_ncpsg;\n\n constexpr short NW = N_SIMDWIDTH;\n\n const int32_t i3 = tgpig[2]/args.ne32;\n const int32_t i2 = tgpig[2]%args.ne32;\n const int32_t i1 = tgpig[1];\n const int32_t i0 = tgpig[0];\n\n char res = i0*C + C > args.ne30 ? 1 : 0;\n\n device const half * mask_src = (device const half *) (mask + (i1*Q)*args.nb31 + i2*args.nb32 + i3*args.nb33) + i0*C + tiisg;\n\n // detailed check of the elements of the block\n if ((C > NW || Q > 1) && res == 0) {\n half mmin = MAXHALF;\n half mmax = -MAXHALF;\n\n FOR_UNROLL (short j = 0; j < Q; ++j) {\n FOR_UNROLL (short ii = 0; ii < C/NW; ++ii) {\n mmin = min(mmin, mask_src[ii*NW]);\n mmax = max(mmax, mask_src[ii*NW]);\n }\n\n mask_src += args.nb31/2;\n }\n\n mmin = simd_min(mmin);\n mmax = simd_max(mmax);\n\n if (mmax > -MAXHALF) {\n if (mmin == 0.0 && mmax == 0.0) {\n res = 2;\n } else {\n res = 1;\n }\n }\n }\n\n const int32_t nblk1 = ((args.ne01 + Q - 1)/Q);\n const int32_t nblk0 = ((args.ne30 + C - 1)/C);\n\n if (tiisg == 0) {\n dst[((i3*args.ne32 + i2)*nblk1 + i1)*nblk0 + i0] = res;\n }\n}\n\nconstant bool FC_flash_attn_ext_has_mask [[function_constant(FC_FLASH_ATTN_EXT + 0)]];\nconstant bool FC_flash_attn_ext_has_sinks [[function_constant(FC_FLASH_ATTN_EXT + 1)]];\nconstant bool FC_flash_attn_ext_has_bias [[function_constant(FC_FLASH_ATTN_EXT + 2)]];\nconstant bool FC_flash_attn_ext_has_scap [[function_constant(FC_FLASH_ATTN_EXT + 3)]];\nconstant bool FC_flash_attn_ext_has_kvpad [[function_constant(FC_FLASH_ATTN_EXT + 4)]];\n\nconstant bool FC_flash_attn_ext_bc_mask [[function_constant(FC_FLASH_ATTN_EXT + 10)]];\n\n//constant float FC_flash_attn_ext_scale [[function_constant(FC_FLASH_ATTN_EXT + 10)]];\n//constant float FC_flash_attn_ext_max_bias [[function_constant(FC_FLASH_ATTN_EXT + 11)]];\n//constant float FC_flash_attn_ext_logit_softcap [[function_constant(FC_FLASH_ATTN_EXT + 12)]];\n\nconstant int32_t FC_flash_attn_ext_ns10 [[function_constant(FC_FLASH_ATTN_EXT + 20)]];\nconstant int32_t FC_flash_attn_ext_ns20 [[function_constant(FC_FLASH_ATTN_EXT + 21)]];\nconstant int32_t FC_flash_attn_ext_nsg [[function_constant(FC_FLASH_ATTN_EXT + 22)]];\n\n// ref: https://arxiv.org/pdf/2307.08691.pdf\ntemplate<\n typename q_t, // query types in shared memory\n typename q4_t,\n typename q8x8_t,\n typename k_t, // key types in shared memory\n typename k4x4_t,\n typename k8x8_t,\n typename v_t, // value types in shared memory\n typename v4x4_t,\n typename v8x8_t,\n typename qk_t, // Q*K types\n typename qk8x8_t,\n typename s_t, // soft-max types\n typename s2_t,\n typename s8x8_t,\n typename o_t, // attention accumulation types\n typename o4_t,\n typename o8x8_t,\n typename kd4x4_t, // key type in device memory\n short nl_k,\n void (*deq_k)(device const kd4x4_t *, short, thread k4x4_t &),\n typename vd4x4_t, // value type in device memory\n short nl_v,\n void (*deq_v)(device const vd4x4_t *, short, thread v4x4_t &),\n short DK, // K head size\n short DV, // V head size\n short Q, // queries per threadgroup\n short C, // cache items per threadgroup\n short NSG> // number of simd groups\nvoid kernel_flash_attn_ext_impl(\n constant ggml_metal_kargs_flash_attn_ext & args,\n device const char * q,\n device const char * k,\n device const char * v,\n device const char * mask,\n device const char * sinks,\n device const char * pad,\n device const char * blk,\n device char * dst,\n threadgroup half * shmem_f16,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const ushort iq3 = tgpig[2];\n const ushort iq2 = tgpig[1];\n const ushort iq1 = tgpig[0]*Q;\n\n#define NS10 (FC_flash_attn_ext_ns10)\n#define NS20 (FC_flash_attn_ext_ns20)\n\n // note: I had some concerns that using this instead of the ugly macros above was affecting performance\n // need to re-check carefully and if no regressions are observerd - remove the macros\n // the concerns is that maybe using const variables requires extra registers? but not sure if the compiler\n // is clever enough to avoid this. unfortunately, using constexpr is not possible with FC\n //const short NS10 = FC_flash_attn_ext_ns10;\n //const short NS20 = FC_flash_attn_ext_ns20;\n\n constexpr short KV = 8;\n\n constexpr short DK4 = DK/4;\n constexpr short DK8 = DK/8;\n constexpr short DK16 = DK/16;\n constexpr short DV4 = DV/4;\n //constexpr short DV8 = DV/8;\n constexpr short DV16 = DV/16;\n\n constexpr short PV = PAD2(DV, 64);\n constexpr short PV4 = PV/4;\n constexpr short PV8 = PV/8;\n //constexpr short PV16 = PV/16;\n\n constexpr short NW = N_SIMDWIDTH;\n constexpr short NQ = Q/NSG;\n constexpr short SH = 2*C; // shared memory per simdgroup (s_t == float)\n\n constexpr short TS = 2*SH;\n constexpr short T = DK + 2*PV; // shared memory size per query in (half)\n\n threadgroup q_t * sq = (threadgroup q_t *) (shmem_f16 + 0*T); // holds the query data\n threadgroup q4_t * sq4 = (threadgroup q4_t *) (shmem_f16 + 0*T); // same as above but in q4_t\n threadgroup o_t * so = (threadgroup o_t *) (shmem_f16 + 0*T + Q*DK); // the result for all queries in 8x8 matrices (the O matrix from the paper)\n threadgroup o4_t * so4 = (threadgroup o4_t *) (shmem_f16 + 0*T + Q*DK);\n threadgroup s_t * ss = (threadgroup s_t *) (shmem_f16 + Q*T); // scratch buffer for attention, mask and diagonal matrix\n threadgroup s2_t * ss2 = (threadgroup s2_t *) (shmem_f16 + Q*T); // same as above but in s2_t\n\n threadgroup k_t * sk = (threadgroup k_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // scratch buffer to load K in shared memory\n threadgroup k4x4_t * sk4x4 = (threadgroup k4x4_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // same as above but in k4x4_t\n\n threadgroup v_t * sv = (threadgroup v_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // scratch buffer to load V in shared memory\n threadgroup v4x4_t * sv4x4 = (threadgroup v4x4_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // same as above but in v4x4_t\n\n // mask storage in shared mem\n threadgroup half2 * sm2 = (threadgroup half2 *) (shmem_f16 + Q*T + 2*C);\n\n // per-query mask pointers\n device const half2 * pm2[NQ];\n\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n const short j = jj*NSG + sgitg;\n\n pm2[jj] = (device const half2 *) ((device const char *) mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33);\n }\n\n {\n const int32_t nblk1 = ((args.ne01 + Q - 1)/Q);\n const int32_t nblk0 = ((args.ne11 + C - 1)/C);\n\n blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0;\n }\n\n {\n q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03;\n\n const short ikv2 = iq2/(args.ne02/args.ne_12_2);\n const short ikv3 = iq3/(args.ne03/args.ne_12_3);\n\n k += ikv2*args.nb12 + ikv3*args.nb13;\n v += ikv2*args.nb22 + ikv3*args.nb23;\n }\n\n // load heads from Q to shared memory\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n const short j = jj*NSG + sgitg;\n\n device const float4 * q4 = (device const float4 *) ((device const char *) q + j*args.nb01);\n\n for (short i = tiisg; i < DK4; i += NW) {\n if (iq1 + j < args.ne01) {\n sq4[j*DK4 + i] = (q4_t) q4[i];\n } else {\n sq4[j*DK4 + i] = 0;\n }\n }\n }\n\n // zero out\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n const short j = jj*NSG + sgitg;\n\n for (short i = tiisg; i < DV4; i += NW) {\n so4[j*PV4 + i] = 0;\n }\n\n for (short i = tiisg; i < SH; i += NW) {\n ss[j*SH + i] = 0.0f;\n }\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n float S[NQ] = { [0 ... NQ-1] = 0.0f };\n\n {\n float M[NQ] = { [0 ... NQ-1] = -FLT_MAX/2 };\n\n float slope = 1.0f;\n\n // ALiBi\n if (FC_flash_attn_ext_has_bias) {\n const short h = iq2;\n\n const float base = h < args.n_head_log2 ? args.m0 : args.m1;\n const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1;\n\n slope = pow(base, exph);\n }\n\n // loop over the KV cache\n // each simdgroup handles blocks of Q rows and C columns\n for (int ic0 = 0; ; ++ic0) {\n int ic = ic0*C;\n if (ic >= args.ne11) {\n break;\n }\n\n // the last partial chunk uses the pad buffer as source\n if (FC_flash_attn_ext_has_kvpad && ic + C > args.ne11) {\n k = pad;\n v = k + args.nb11*C*args.ne_12_2*args.ne_12_3;\n mask = v + args.nb21*C*args.ne_12_2*args.ne_12_3;\n\n const short ikv2 = iq2/(args.ne02/args.ne_12_2);\n const short ikv3 = iq3/(args.ne03/args.ne_12_3);\n\n k += (ikv2 + ikv3*args.ne_12_2)*args.nb11*C;\n v += (ikv2 + ikv3*args.ne_12_2)*args.nb21*C;\n\n if (!FC_flash_attn_ext_has_mask) {\n threadgroup half * sm = (threadgroup half *) (sm2);\n\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n const short j = jj*NSG + sgitg;\n\n for (short i = tiisg; i < C; i += NW) {\n if (ic + i >= args.ne11) {\n sm[2*j*SH + i] = -MAXHALF;\n }\n }\n }\n } else {\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n const short j = jj*NSG + sgitg;\n\n pm2[jj] = (device const half2 *) ((device const half *) mask +\n (iq1 + j)*C +\n (iq2%args.ne32)*(C*args.ne31) +\n (iq3%args.ne33)*(C*args.ne31*args.ne32));\n }\n }\n\n ic = 0;\n }\n\n char blk_cur = 1;\n\n // read the mask into shared mem\n if (FC_flash_attn_ext_has_mask) {\n blk_cur = blk[ic0];\n\n if (blk_cur == 0) {\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n pm2[jj] += NW;\n }\n\n continue;\n }\n\n if (blk_cur == 1) {\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n const short j = jj*NSG + sgitg;\n\n if (FC_flash_attn_ext_bc_mask) {\n sm2[j*SH + tiisg] = (iq1 + j) < args.ne31 ? pm2[jj][tiisg] : half2(-MAXHALF, -MAXHALF);\n } else {\n sm2[j*SH + tiisg] = pm2[jj][tiisg];\n }\n\n pm2[jj] += NW;\n }\n } else if (blk_cur == 2) {\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n pm2[jj] += NW;\n }\n }\n\n#if 0\n // note: old -INF block optimization - obsoleted by pre-computing non-masked blocks\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n // used to detect blocks full of -INF\n // skip only when the entire threadgroup is masked\n half2 smax2(-MAXHALF/2, -MAXHALF/2);\n\n FOR_UNROLL (short j = 0; j < Q; ++j) {\n smax2 = max(smax2, sm2[j*SH + tiisg]);\n }\n\n smax2 = simd_max(smax2);\n\n if (max(smax2[0], smax2[1]) <= -MAXHALF/2) {\n // this barrier is important\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n continue;\n }\n#endif\n }\n\n // Q*K^T\n // this is compile-time check, so it does not have runtime overhead\n if (is_same::value) {\n // we can read directly from global memory\n device const k_t * pk = (device const k_t *) (k + ic*args.nb11);\n threadgroup const q_t * pq = sq;\n threadgroup s_t * ps = ss;\n\n pk += sgitg*(8*NS10);\n ps += sgitg*(8*1);\n\n static_assert((C/8) % NSG == 0, \"\");\n\n constexpr short NC = (C/8)/NSG;\n\n FOR_UNROLL (short cc = 0; cc < NC; ++cc) {\n qk8x8_t mqk = make_filled_simdgroup_matrix((qk_t) 0.0f);\n\n if (DK % 16 != 0) {\n k8x8_t mk;\n q8x8_t mq;\n\n FOR_UNROLL (short i = 0; i < DK8; ++i) {\n simdgroup_barrier(mem_flags::mem_none);\n\n simdgroup_load(mk, pk + 8*i, NS10, 0, true);\n simdgroup_load(mq, pq + 8*i, DK);\n\n simdgroup_barrier(mem_flags::mem_none);\n\n simdgroup_multiply_accumulate(mqk, mq, mk, mqk);\n }\n } else {\n k8x8_t mk[2];\n q8x8_t mq[2];\n\n // note: too much unroll can tank the performance for large heads\n #pragma unroll (MIN(DK8/2, 4*NSG))\n for (short i = 0; i < DK8/2; ++i) {\n simdgroup_barrier(mem_flags::mem_none);\n\n simdgroup_load(mq[0], pq + 0*8 + 16*i, DK);\n simdgroup_load(mq[1], pq + 1*8 + 16*i, DK);\n\n simdgroup_load(mk[0], pk + 0*8 + 16*i, NS10, 0, true);\n simdgroup_load(mk[1], pk + 1*8 + 16*i, NS10, 0, true);\n\n simdgroup_barrier(mem_flags::mem_none);\n\n simdgroup_multiply_accumulate(mqk, mq[0], mk[0], mqk);\n simdgroup_multiply_accumulate(mqk, mq[1], mk[1], mqk);\n }\n }\n\n simdgroup_store(mqk, ps, SH, 0, false);\n\n pk += 8*(NSG*NS10);\n ps += 8*(NSG);\n }\n } else {\n // TODO: this is the quantized K cache branch - not optimized yet\n for (short ccc = 0; ccc < (C/8)/NSG; ++ccc) {\n const short cc = ccc*NSG + sgitg;\n\n const short tx = tiisg%4;\n const short ty = tiisg/4;\n\n qk8x8_t mqk = make_filled_simdgroup_matrix((qk_t) 0.0f);\n\n for (short ii = 0; ii < DK16; ii += 4) {\n device const kd4x4_t * pk4x4 = (device const kd4x4_t *) (k + ((ic + 8*cc + ty)*args.nb11));\n\n if (DK16%4 == 0) {\n // the head is evenly divisible by 4*16 = 64, so no need for bound checks\n {\n k4x4_t tmp;\n deq_k(pk4x4 + (ii + tx)/nl_k, (ii + tx)%nl_k, tmp);\n sk4x4[4*ty + tx] = tmp;\n }\n\n simdgroup_barrier(mem_flags::mem_threadgroup);\n\n FOR_UNROLL (short k = 0; k < 4; ++k) {\n k8x8_t mk;\n q8x8_t mq;\n\n simdgroup_load(mk, sk + 16*k + 0*8, 4*16, 0, true); // transpose\n simdgroup_load(mq, sq + (2*(ii + k) + 0)*8, DK);\n simdgroup_multiply_accumulate(mqk, mq, mk, mqk);\n\n simdgroup_load(mk, sk + 16*k + 1*8, 4*16, 0, true); // transpose\n simdgroup_load(mq, sq + (2*(ii + k) + 1)*8, DK);\n simdgroup_multiply_accumulate(mqk, mq, mk, mqk);\n }\n } else {\n if (ii + tx < DK16) {\n k4x4_t tmp;\n deq_k(pk4x4 + (ii + tx)/nl_k, (ii + tx)%nl_k, tmp);\n sk4x4[4*ty + tx] = tmp;\n }\n\n simdgroup_barrier(mem_flags::mem_threadgroup);\n\n for (short k = 0; k < 4 && ii + k < DK16; ++k) {\n k8x8_t mk;\n q8x8_t mq;\n\n simdgroup_load(mk, sk + 16*k + 0*8, 4*16, 0, true); // transpose\n simdgroup_load(mq, sq + (2*(ii + k) + 0)*8, DK);\n simdgroup_multiply_accumulate(mqk, mq, mk, mqk);\n\n simdgroup_load(mk, sk + 16*k + 1*8, 4*16, 0, true); // transpose\n simdgroup_load(mq, sq + (2*(ii + k) + 1)*8, DK);\n simdgroup_multiply_accumulate(mqk, mq, mk, mqk);\n }\n }\n }\n\n simdgroup_store(mqk, ss + 8*cc, SH, 0, false);\n }\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n // online softmax\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n const short j = jj*NSG + sgitg;\n\n const float m = M[jj];\n\n // scale and apply the logitcap / mask\n float2 s2 = ss2[j*SH/2 + tiisg]*args.scale;\n\n if (FC_flash_attn_ext_has_scap) {\n s2 = args.logit_softcap*precise::tanh(s2);\n }\n\n // mqk = mqk + slope*mask\n if (blk_cur != 2) {\n if (FC_flash_attn_ext_has_bias) {\n s2 += s2_t(sm2[j*SH + tiisg])*slope;\n } else {\n s2 += s2_t(sm2[j*SH + tiisg]);\n }\n }\n\n M[jj] = simd_max(max(M[jj], max(s2[0], s2[1])));\n\n const float ms = exp(m - M[jj]);\n const float2 vs2 = exp(s2 - M[jj]);\n\n S[jj] = S[jj]*ms + simd_sum(vs2[0] + vs2[1]);\n\n // the P matrix from the paper (Q rows, C columns)\n ss2[j*SH/2 + tiisg] = vs2;\n\n if (DV4 % NW == 0) {\n FOR_UNROLL (short ii = 0; ii < DV4/NW; ++ii) {\n const short i = ii*NW + tiisg;\n\n so4[j*PV4 + i] *= ms;\n }\n } else {\n for (short i = tiisg; i < DV4; i += NW) {\n so4[j*PV4 + i] *= ms;\n }\n }\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n // O = O + (Q*K^T)*V\n {\n // we can read directly from global memory\n if (is_same::value) {\n static_assert(PV8 % NSG == 0, \"\");\n\n constexpr short NO = PV8/NSG;\n\n o8x8_t lo[NO];\n\n {\n auto sot = so + 8*sgitg;\n\n FOR_UNROLL (short ii = 0; ii < NO; ++ii) {\n simdgroup_load(lo[ii], sot, PV, 0, false);\n\n sot += 8*NSG;\n }\n }\n\n {\n device const v_t * pv = (device const v_t *) (v + ic*args.nb21);\n\n pv += 8*sgitg;\n\n if (DV <= 64) {\n FOR_UNROLL (short cc = 0; cc < C/8; ++cc) {\n s8x8_t vs;\n simdgroup_load(vs, ss + 8*cc, SH, 0, false);\n\n FOR_UNROLL (short ii = 0; ii < NO/2; ++ii) {\n v8x8_t mv[2];\n\n simdgroup_load(mv[0], pv + 0*NSG + 16*ii*NSG, NS20, 0, false);\n simdgroup_load(mv[1], pv + 8*NSG + 16*ii*NSG, NS20, 0, false);\n\n simdgroup_multiply_accumulate(lo[2*ii + 0], vs, mv[0], lo[2*ii + 0]);\n simdgroup_multiply_accumulate(lo[2*ii + 1], vs, mv[1], lo[2*ii + 1]);\n }\n\n pv += 8*NS20;\n }\n } else {\n constexpr short NC = (C/8)/2;\n\n FOR_UNROLL (short cc = 0; cc < NC; ++cc) {\n s8x8_t vs[2];\n\n simdgroup_load(vs[0], ss + 16*cc + 0, SH, 0, false);\n simdgroup_load(vs[1], ss + 16*cc + 8, SH, 0, false);\n\n FOR_UNROLL (short ii = 0; ii < NO/2; ++ii) {\n v8x8_t mv[4];\n\n simdgroup_load(mv[0], pv + 0*NSG + 16*ii*NSG + 0*8*NS20, NS20, 0, false);\n simdgroup_load(mv[1], pv + 8*NSG + 16*ii*NSG + 0*8*NS20, NS20, 0, false);\n simdgroup_load(mv[2], pv + 0*NSG + 16*ii*NSG + 1*8*NS20, NS20, 0, false);\n simdgroup_load(mv[3], pv + 8*NSG + 16*ii*NSG + 1*8*NS20, NS20, 0, false);\n\n simdgroup_multiply_accumulate(lo[2*ii + 0], vs[0], mv[0], lo[2*ii + 0]);\n simdgroup_multiply_accumulate(lo[2*ii + 1], vs[0], mv[1], lo[2*ii + 1]);\n simdgroup_multiply_accumulate(lo[2*ii + 0], vs[1], mv[2], lo[2*ii + 0]);\n simdgroup_multiply_accumulate(lo[2*ii + 1], vs[1], mv[3], lo[2*ii + 1]);\n }\n\n pv += 2*8*NS20;\n }\n }\n }\n\n {\n auto sot = so + 8*sgitg;\n\n FOR_UNROLL (short ii = 0; ii < NO; ++ii) {\n simdgroup_store(lo[ii], sot, PV, 0, false);\n\n sot += 8*NSG;\n }\n }\n } else {\n // TODO: this is the quantized V cache branch - not optimized yet\n\n const short tx = tiisg%4;\n const short ty = tiisg/4;\n\n for (short cc = 0; cc < C/8; ++cc) {\n s8x8_t vs;\n simdgroup_load(vs, ss + 8*cc, SH, 0, false);\n\n for (short ii = 4*sgitg; ii < DV16; ii += 4*NSG) {\n device const vd4x4_t * pv4x4 = (device const vd4x4_t *) (v + ((ic + 8*cc + ty)*args.nb21));\n\n if (DV16%4 == 0) {\n // no need for bound checks\n {\n v4x4_t tmp;\n deq_v(pv4x4 + (ii + tx)/nl_v, (ii + tx)%nl_v, tmp);\n sv4x4[4*ty + tx] = tmp;\n }\n\n simdgroup_barrier(mem_flags::mem_threadgroup);\n\n FOR_UNROLL (short k = 0; k < 4; ++k) {\n v8x8_t mv[2];\n o8x8_t lo[2];\n\n simdgroup_load(mv[0], sv + 16*k + 0*8, 4*16, 0, false);\n simdgroup_load(mv[1], sv + 16*k + 1*8, 4*16, 0, false);\n simdgroup_load(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false);\n simdgroup_load(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false);\n\n simdgroup_multiply_accumulate(lo[0], vs, mv[0], lo[0]);\n simdgroup_multiply_accumulate(lo[1], vs, mv[1], lo[1]);\n\n simdgroup_store(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false);\n simdgroup_store(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false);\n }\n } else {\n if (ii + tx < DV16) {\n v4x4_t tmp;\n deq_v(pv4x4 + (ii + tx)/nl_v, (ii + tx)%nl_v, tmp);\n sv4x4[4*ty + tx] = tmp;\n }\n\n simdgroup_barrier(mem_flags::mem_threadgroup);\n\n for (short k = 0; k < 4 && ii + k < DV16; ++k) {\n v8x8_t mv[2];\n o8x8_t lo[2];\n\n simdgroup_load(mv[0], sv + 16*k + 0*8, 4*16, 0, false);\n simdgroup_load(mv[1], sv + 16*k + 1*8, 4*16, 0, false);\n simdgroup_load(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false);\n simdgroup_load(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false);\n\n simdgroup_multiply_accumulate(lo[0], vs, mv[0], lo[0]);\n simdgroup_multiply_accumulate(lo[1], vs, mv[1], lo[1]);\n\n simdgroup_store(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false);\n simdgroup_store(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false);\n }\n }\n }\n }\n }\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n }\n\n if (FC_flash_attn_ext_has_sinks) {\n FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {\n const short j = jj*NSG + sgitg;\n\n const float m = M[jj];\n const float s = tiisg == 0 ? ((device const float *) sinks)[iq2] : -FLT_MAX/2;\n\n M[jj] = simd_max(max(M[jj], s));\n\n const float ms = exp(m - M[jj]);\n const float vs = exp(s - M[jj]);\n\n S[jj] = S[jj]*ms + simd_sum(vs);\n\n for (short i = tiisg; i < DV4; i += NW) {\n so4[j*PV4 + i] *= ms;\n }\n }\n }\n }\n\n // store to global memory\n for (short jj = 0; jj < NQ; ++jj) {\n const short j = jj*NSG + sgitg;\n if (iq1 + j >= args.ne01) {\n break;\n }\n\n device float4 * dst4 = (device float4 *) dst + ((uint64_t)iq3*args.ne2*args.ne1 + iq2 + (uint64_t)(iq1 + j)*args.ne1)*DV4;\n\n const float scale = S[jj] == 0.0 ? 0.0f : 1.0f/S[jj];\n\n if (DV4 % NW == 0) {\n FOR_UNROLL (short ii = 0; ii < DV4/NW; ++ii) {\n const short i = ii*NW + tiisg;\n\n dst4[i] = (float4) so4[j*PV4 + i]*scale;\n }\n } else {\n for (short i = tiisg; i < DV4; i += NW) {\n dst4[i] = (float4) so4[j*PV4 + i]*scale;\n }\n }\n }\n\n#undef NS10\n#undef NS20\n}\n\ntemplate<\n typename q_t, // query types in shared memory\n typename q4_t,\n typename q8x8_t,\n typename k_t, // key types in shared memory\n typename k4x4_t,\n typename k8x8_t,\n typename v_t, // value types in shared memory\n typename v4x4_t,\n typename v8x8_t,\n typename qk_t, // Q*K types\n typename qk8x8_t,\n typename s_t, // soft-max types\n typename s2_t,\n typename s8x8_t,\n typename o_t, // attention accumulation types\n typename o4_t,\n typename o8x8_t,\n typename kd4x4_t, // key type in device memory\n short nl_k,\n void (*deq_k)(device const kd4x4_t *, short, thread k4x4_t &),\n typename vd4x4_t, // value type in device memory\n short nl_v,\n void (*deq_v)(device const vd4x4_t *, short, thread v4x4_t &),\n short DK, // K head size\n short DV, // V head size\n short Q = OP_FLASH_ATTN_EXT_NQPSG, // queries per threadgroup\n short C = OP_FLASH_ATTN_EXT_NCPSG> // cache items per threadgroup\nkernel void kernel_flash_attn_ext(\n constant ggml_metal_kargs_flash_attn_ext & args,\n device const char * q,\n device const char * k,\n device const char * v,\n device const char * mask,\n device const char * sinks,\n device const char * pad,\n device const char * blk,\n device char * dst,\n threadgroup half * shmem_f16 [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n#define FWD_TMPL q_t, q4_t, q8x8_t, k_t, k4x4_t, k8x8_t, v_t, v4x4_t, v8x8_t, qk_t, qk8x8_t, s_t, s2_t, s8x8_t, o_t, o4_t, o8x8_t, kd4x4_t, nl_k, deq_k, vd4x4_t, nl_v, deq_v, DK, DV, Q, C\n#define FWD_ARGS args, q, k, v, mask, sinks, pad, blk, dst, shmem_f16, tgpig, tiisg, sgitg\n switch (FC_flash_attn_ext_nsg) {\n // note: disabled cases to reduce library load time\n //case 1: kernel_flash_attn_ext_impl(FWD_ARGS); break;\n //case 2: kernel_flash_attn_ext_impl(FWD_ARGS); break;\n case 4: kernel_flash_attn_ext_impl(FWD_ARGS); break;\n case 8: kernel_flash_attn_ext_impl(FWD_ARGS); break;\n }\n#undef FWD_TMPL\n#undef FWD_ARGS\n}\n\n// TODO: this is quite ugly. in the future these types will be hardcoded in the kernel, but for now keep them as\n// template to be able to explore different combinations\n//\n#define FA_TYPES \\\n half, half4, simdgroup_half8x8, \\\n half, half4x4, simdgroup_half8x8, \\\n half, half4x4, simdgroup_half8x8, \\\n float, simdgroup_float8x8, \\\n float, float2, simdgroup_float8x8, \\\n float, float4, simdgroup_float8x8\n //half, half4, simdgroup_half8x8\n\n#define FA_TYPES_BF \\\n bfloat, bfloat4, simdgroup_bfloat8x8, \\\n bfloat, bfloat4x4, simdgroup_bfloat8x8, \\\n bfloat, bfloat4x4, simdgroup_bfloat8x8, \\\n float, simdgroup_float8x8, \\\n float, float2, simdgroup_float8x8, \\\n half, half4, simdgroup_half8x8\n //float, float4, simdgroup_float8x8\n\n#define FA_TYPES_F32 \\\n half, half4, simdgroup_half8x8, \\\n float, float4x4, simdgroup_float8x8, \\\n float, float4x4, simdgroup_float8x8, \\\n float, simdgroup_float8x8, \\\n float, float2, simdgroup_float8x8, \\\n float, float4, simdgroup_float8x8\n //half, half4, simdgroup_half8x8\n\ntypedef decltype(kernel_flash_attn_ext) flash_attn_ext_t;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk32_dv32\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk40_dv40\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk48_dv48\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk64_dv64\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk72_dv72\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk80_dv80\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk96_dv96\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk112_dv112\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk128_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk192_dv192\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk192_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk256_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk320_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f32_dk576_dv512\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk32_dv32\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk40_dv40\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk48_dv48\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk64_dv64\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk72_dv72\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk80_dv80\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk96_dv96\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk112_dv112\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk128_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk192_dv192\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk192_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk256_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk320_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_f16_dk576_dv512\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\n\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk32_dv32\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk40_dv40\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk48_dv48\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk64_dv64\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk72_dv72\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk80_dv80\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk96_dv96\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk112_dv112\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk128_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk192_dv192\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk192_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk256_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk320_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_bf16_dk576_dv512\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\n#endif\n\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk32_dv32\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk40_dv40\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk48_dv48\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk64_dv64\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk72_dv72\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk80_dv80\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk96_dv96\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk112_dv112\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk128_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk192_dv192\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk192_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk256_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk320_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_0_dk576_dv512\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk32_dv32\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk40_dv40\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk48_dv48\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk64_dv64\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk72_dv72\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk80_dv80\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk96_dv96\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk112_dv112\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk128_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk192_dv192\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk192_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk256_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk320_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q4_1_dk576_dv512\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk32_dv32\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk40_dv40\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk48_dv48\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk64_dv64\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk72_dv72\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk80_dv80\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk96_dv96\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk112_dv112\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk128_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk192_dv192\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk192_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk256_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk320_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_0_dk576_dv512\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk32_dv32\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk40_dv40\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk48_dv48\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk64_dv64\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk72_dv72\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk80_dv80\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk96_dv96\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk112_dv112\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk128_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk192_dv192\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk192_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk256_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk320_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q5_1_dk576_dv512\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk32_dv32\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk40_dv40\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk48_dv48\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk64_dv64\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk72_dv72\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk80_dv80\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk96_dv96\" )]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk112_dv112\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk128_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk192_dv192\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk192_dv128\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk256_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk320_dv256\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\ntemplate [[host_name(\"kernel_flash_attn_ext_q8_0_dk576_dv512\")]] kernel flash_attn_ext_t kernel_flash_attn_ext;\n\n#undef FA_TYPES\n#undef FA_TYPES_BF\n#undef FA_TYPES_F32\n\nconstant bool FC_flash_attn_ext_vec_has_mask [[function_constant(FC_FLASH_ATTN_EXT_VEC + 0)]];\nconstant bool FC_flash_attn_ext_vec_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_VEC + 1)]];\nconstant bool FC_flash_attn_ext_vec_has_bias [[function_constant(FC_FLASH_ATTN_EXT_VEC + 2)]];\nconstant bool FC_flash_attn_ext_vec_has_scap [[function_constant(FC_FLASH_ATTN_EXT_VEC + 3)]];\nconstant bool FC_flash_attn_ext_vec_has_kvpad [[function_constant(FC_FLASH_ATTN_EXT_VEC + 4)]];\n\n//constant float FC_flash_attn_ext_vec_scale [[function_constant(FC_FLASH_ATTN_EXT_VEC + 10)]];\n//constant float FC_flash_attn_ext_vec_max_bias [[function_constant(FC_FLASH_ATTN_EXT_VEC + 11)]];\n//constant float FC_flash_attn_ext_vec_logit_softcap [[function_constant(FC_FLASH_ATTN_EXT_VEC + 12)]];\n\nconstant int32_t FC_flash_attn_ext_vec_ns10 [[function_constant(FC_FLASH_ATTN_EXT_VEC + 20)]];\nconstant int32_t FC_flash_attn_ext_vec_ns20 [[function_constant(FC_FLASH_ATTN_EXT_VEC + 21)]];\nconstant int32_t FC_flash_attn_ext_vec_nsg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 22)]];\nconstant int32_t FC_flash_attn_ext_vec_nwg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 23)]];\n\ntemplate<\n typename q4_t, // query types in shared memory\n typename k4_t, // key types in shared memory\n typename v4_t, // value types in shared memory\n typename qk_t, // Q*K types\n typename s_t, // soft-max types\n typename s4_t,\n typename o4_t, // attention accumulation types\n typename kd4_t, // key type in device memory\n short nl_k,\n void (*deq_k_t4)(device const kd4_t *, short, thread k4_t &),\n typename vd4_t, // value type in device memory\n short nl_v,\n void (*deq_v_t4)(device const vd4_t *, short, thread v4_t &),\n short DK, // K head size\n short DV, // V head size\n short NE = 4, // head elements per thread\n short Q = OP_FLASH_ATTN_EXT_VEC_NQPSG, // queries per threadgroup\n short C = OP_FLASH_ATTN_EXT_VEC_NCPSG> // cache items per threadgroup\nkernel void kernel_flash_attn_ext_vec(\n constant ggml_metal_kargs_flash_attn_ext_vec & args,\n device const char * q,\n device const char * k,\n device const char * v,\n device const char * mask,\n device const char * sinks,\n device const char * pad,\n device char * dst,\n threadgroup half * shmem_f16 [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n static_assert(DK % 32 == 0, \"DK must be divisible by 32\");\n static_assert(DV % 32 == 0, \"DV must be divisible by 32\");\n\n#define NWG (FC_flash_attn_ext_vec_nwg)\n#define NSG (FC_flash_attn_ext_vec_nsg)\n\n#define NS10 (FC_flash_attn_ext_vec_ns10)\n#define NS20 (FC_flash_attn_ext_vec_ns20)\n\n const short iwg = tgpig[2]%NWG;\n\n const ushort iq3 = tgpig[2]/NWG;\n const ushort iq2 = tgpig[1];\n const ushort iq1 = tgpig[0];\n\n constexpr short DK4 = DK/4;\n constexpr short DV4 = DV/4;\n\n constexpr short PK = PAD2(DK, 128);\n constexpr short PK4 = PK/4;\n\n constexpr short PV = PAD2(DV, 128);\n constexpr short PV4 = PV/4;\n\n constexpr short NW = N_SIMDWIDTH;\n constexpr short NL = NW/NE; // note: this can be adjusted to support different head sizes and simdgroup work loads\n constexpr short SH = 4*C; // shared memory per simdgroup\n\n static_assert(DK4 % NL == 0, \"DK4 must be divisible by NL\");\n static_assert(DV4 % NL == 0, \"DV4 must be divisible by NL\");\n\n //const short T = PK + NSG*SH; // shared memory size per query in (half)\n\n //threadgroup q_t * sq = (threadgroup q_t *) (shmem_f16 + 0*PK); // holds the query data\n threadgroup q4_t * sq4 = (threadgroup q4_t *) (shmem_f16 + 0*PK); // same as above but in q4_t\n threadgroup s_t * ss = (threadgroup s_t *) (shmem_f16 + sgitg*SH + NSG*PK); // scratch buffer for attention\n threadgroup s4_t * ss4 = (threadgroup s4_t *) (shmem_f16 + sgitg*SH + NSG*PK); // same as above but in s4_t\n threadgroup half * sm = (threadgroup half *) (shmem_f16 + sgitg*SH + 2*C + NSG*PK); // scratch buffer for mask\n threadgroup o4_t * so4 = (threadgroup o4_t *) (shmem_f16 + 2*sgitg*PV + NSG*PK + NSG*SH); // scratch buffer for the results\n\n // store the result for all queries in shared memory (the O matrix from the paper)\n so4 += tiisg;\n\n {\n q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03;\n\n const short ikv2 = iq2/(args.ne02/args.ne_12_2);\n const short ikv3 = iq3/(args.ne03/args.ne_12_3);\n\n k += ikv2*args.nb12 + ikv3*args.nb13;\n v += ikv2*args.nb22 + ikv3*args.nb23;\n }\n\n // load heads from Q to shared memory\n device const float4 * q4 = (device const float4 *) ((device const char *) q);\n\n if (iq1 < args.ne01) {\n for (short i = tiisg; i < PK4; i += NW) {\n if (i < DK4) {\n sq4[i] = (q4_t) q4[i];\n } else {\n sq4[i] = (q4_t) 0.0f;\n }\n }\n }\n\n // zero out so\n for (short i = 0; i < DV4/NL; ++i) {\n so4[i*NL] = (o4_t) 0.0f;\n }\n\n // zero out shared memory SH\n for (short i = tiisg; i < SH/4; i += NW) {\n ss4[i] = (s4_t) 0.0f;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n {\n float S = 0.0f;\n float M = -FLT_MAX/2;\n\n // thread indices inside the simdgroup\n const short tx = tiisg%NL;\n const short ty = tiisg/NL;\n\n // pointer to the mask\n device const half * pm = (device const half *) (mask + iq1*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33);\n\n float slope = 1.0f;\n\n // ALiBi\n if (FC_flash_attn_ext_vec_has_bias) {\n const short h = iq2;\n\n const float base = h < args.n_head_log2 ? args.m0 : args.m1;\n const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1;\n\n slope = pow(base, exph);\n }\n\n // loop over the KV cache\n // each simdgroup handles blocks of Q rows and C columns\n for (int ic0 = iwg*NSG + sgitg; ; ic0 += NWG*NSG) {\n int ic = ic0*C;\n if (ic >= args.ne11) {\n break;\n }\n\n // the last partial chunk uses the pad buffer as source\n if (FC_flash_attn_ext_vec_has_kvpad && ic + C > args.ne11) {\n k = pad;\n v = k + args.nb11*C*args.ne_12_2*args.ne_12_3;\n mask = v + args.nb21*C*args.ne_12_2*args.ne_12_3;\n\n const short ikv2 = iq2/(args.ne02/args.ne_12_2);\n const short ikv3 = iq3/(args.ne03/args.ne_12_3);\n\n k += (ikv2 + ikv3*args.ne_12_2)*args.nb11*C;\n v += (ikv2 + ikv3*args.ne_12_2)*args.nb21*C;\n\n if (!FC_flash_attn_ext_vec_has_mask) {\n if (ic + tiisg >= args.ne11) {\n sm[tiisg] = -MAXHALF;\n }\n } else {\n pm = (device const half *) (mask) +\n iq1*C +\n (iq2%args.ne32)*(C*args.ne31) +\n (iq3%args.ne33)*(C*args.ne31*args.ne32);\n }\n\n ic = 0;\n }\n\n if (FC_flash_attn_ext_vec_has_mask) {\n sm[tiisg] = pm[ic + tiisg];\n }\n\n // skip -INF blocks\n if (simd_max(sm[tiisg]) <= -MAXHALF) {\n continue;\n }\n\n // Q*K^T\n {\n device const k4_t * pk4 = (device const k4_t *) (k + ic*args.nb11);\n threadgroup const q4_t * pq4 = sq4;\n\n pk4 += ty*NS10/4 + tx;\n pq4 += tx;\n\n qk_t mqk[C/NE] = { [ 0 ... C/NE - 1] = 0.0f };\n\n // each simdgroup processes 1 query and NE (NW/NL) cache elements\n FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) {\n if (is_same::value) {\n FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) {\n mqk[cc] += dot((float4) pk4[cc*NE*NS10/4 + ii*NL], (float4) pq4[ii*NL]);\n }\n } else {\n device const kd4_t * pk = (device const kd4_t *) (k + ((ic + NE*cc + ty)*args.nb11));\n\n k4_t mk;\n\n FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) {\n const short i = ii*NL + tx;\n\n deq_k_t4(pk + i/nl_k, i%nl_k, mk);\n\n mqk[cc] += dot((float4) mk, (float4) sq4[i]);\n }\n }\n\n if (NE == 1) {\n mqk[cc] = simd_sum(mqk[cc]);\n } else {\n // simdgroup reduce (NE = 4)\n // [ 0 .. 7] -> [ 0]\n // [ 8 .. 15] -> [ 8]\n // [16 .. 23] -> [16]\n // [24 .. 31] -> [24]\n if (NE <= 1) {\n mqk[cc] += simd_shuffle_down(mqk[cc], 16);\n }\n if (NE <= 2) {\n mqk[cc] += simd_shuffle_down(mqk[cc], 8);\n }\n if (NE <= 4) {\n mqk[cc] += simd_shuffle_down(mqk[cc], 4);\n }\n if (NE <= 8) {\n mqk[cc] += simd_shuffle_down(mqk[cc], 2);\n }\n if (NE <= 16) {\n mqk[cc] += simd_shuffle_down(mqk[cc], 1);\n }\n\n // broadcast\n mqk[cc] = simd_shuffle(mqk[cc], NL*ty);\n }\n }\n\n if (FC_flash_attn_ext_vec_has_mask &&\n !FC_flash_attn_ext_vec_has_scap &&\n !FC_flash_attn_ext_vec_has_bias) {\n ss[NE*tx + ty] = fma(mqk[tx], args.scale, (qk_t) sm[NE*tx + ty]);\n } else {\n mqk[tx] *= args.scale;\n\n if (FC_flash_attn_ext_vec_has_scap) {\n mqk[tx] = args.logit_softcap*precise::tanh(mqk[tx]);\n }\n\n if (FC_flash_attn_ext_vec_has_bias) {\n mqk[tx] += (qk_t) sm[NE*tx + ty]*slope;\n } else {\n mqk[tx] += (qk_t) sm[NE*tx + ty];\n }\n\n ss[NE*tx + ty] = mqk[tx];\n }\n }\n\n simdgroup_barrier(mem_flags::mem_threadgroup);\n\n // online softmax\n {\n const float m = M;\n const float s = ss[tiisg];\n\n M = simd_max(max(M, s));\n\n const float ms = exp(m - M);\n const float vs = exp(s - M);\n\n S = S*ms + simd_sum(vs);\n\n // the P matrix from the paper (Q rows, C columns)\n ss[tiisg] = vs;\n\n // O = diag(ms)*O\n if ((DV4/NL % NW == 0) || ty == 0) {\n FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {\n so4[ii*NL] *= ms;\n }\n }\n }\n\n simdgroup_barrier(mem_flags::mem_threadgroup);\n\n // O = O + (Q*K^T)*V\n {\n o4_t lo[DV4/NL];\n FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {\n lo[ii] = 0.0f;\n }\n\n if (is_same::value) {\n device const v4_t * pv4 = (device const v4_t *) (v + ic*args.nb21);\n\n pv4 += ty*NS20/4 + tx;\n\n const auto sst = ss + ty;\n\n FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) {\n FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {\n lo[ii] += o4_t(float4(pv4[cc*NE*NS20/4 + ii*NL])*float4(sst[cc*NE]));\n }\n }\n } else {\n FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) {\n device const vd4_t * pv4 = (device const vd4_t *) (v + ((ic + NE*cc + ty)*args.nb21));\n\n FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {\n const short i = ii*NL + tx;\n\n v4_t mv;\n deq_v_t4(pv4 + i/nl_v, i%nl_v, mv);\n\n lo[ii] += o4_t(float4(mv)*float4(ss[NE*cc + ty]));\n }\n }\n }\n\n FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {\n if (NE > 1) {\n lo[ii][0] += simd_shuffle_down(lo[ii][0], 16);\n lo[ii][1] += simd_shuffle_down(lo[ii][1], 16);\n lo[ii][2] += simd_shuffle_down(lo[ii][2], 16);\n lo[ii][3] += simd_shuffle_down(lo[ii][3], 16);\n }\n\n if (NE > 2) {\n lo[ii][0] += simd_shuffle_down(lo[ii][0], 8);\n lo[ii][1] += simd_shuffle_down(lo[ii][1], 8);\n lo[ii][2] += simd_shuffle_down(lo[ii][2], 8);\n lo[ii][3] += simd_shuffle_down(lo[ii][3], 8);\n }\n\n if (NE > 4) {\n lo[ii][0] += simd_shuffle_down(lo[ii][0], 4);\n lo[ii][1] += simd_shuffle_down(lo[ii][1], 4);\n lo[ii][2] += simd_shuffle_down(lo[ii][2], 4);\n lo[ii][3] += simd_shuffle_down(lo[ii][3], 4);\n }\n\n if (NE > 8) {\n lo[ii][0] += simd_shuffle_down(lo[ii][0], 2);\n lo[ii][1] += simd_shuffle_down(lo[ii][1], 2);\n lo[ii][2] += simd_shuffle_down(lo[ii][2], 2);\n lo[ii][3] += simd_shuffle_down(lo[ii][3], 2);\n }\n\n if (NE > 16) {\n lo[ii][0] += simd_shuffle_down(lo[ii][0], 1);\n lo[ii][1] += simd_shuffle_down(lo[ii][1], 1);\n lo[ii][2] += simd_shuffle_down(lo[ii][2], 1);\n lo[ii][3] += simd_shuffle_down(lo[ii][3], 1);\n }\n }\n\n if ((DV4/NL % NW == 0) || ty == 0) {\n FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {\n so4[ii*NL] += lo[ii];\n }\n }\n }\n }\n\n if (FC_flash_attn_ext_vec_has_sinks && sgitg == 0 && iwg == 0) {\n const float m = M;\n const float s = tiisg == 0 ? ((device const float *) sinks)[iq2] : -FLT_MAX/2;\n\n M = simd_max(max(M, s));\n\n const float ms = exp(m - M);\n const float vs = exp(s - M);\n\n S = S*ms + simd_sum(vs);\n\n if ((DV4/NL % NW == 0) || ty == 0) {\n FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {\n so4[ii*NL] *= ms;\n }\n }\n }\n\n // these are needed for reducing the results from the simdgroups (reuse the ss buffer)\n if (tiisg == 0) {\n ss[0] = (s_t) S;\n ss[1] = (s_t) M;\n }\n }\n\n so4 -= tiisg;\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n // parallel reduce\n for (short r = NSG/2; r > 0; r >>= 1) {\n if (sgitg < r) {\n const float S0 = ss[ 0];\n const float S1 = ss[r*(SH/2) + 0];\n\n const float M0 = ss[ 1];\n const float M1 = ss[r*(SH/2) + 1];\n\n const float M = max(M0, M1);\n\n const float ms0 = exp(M0 - M);\n const float ms1 = exp(M1 - M);\n\n const float S = S0*ms0 + S1*ms1;\n\n if (tiisg == 0) {\n ss[0] = S;\n ss[1] = M;\n }\n\n // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1\n for (short i = tiisg; i < DV4; i += NW) {\n so4[i] = so4[i]*ms0 + so4[i + r*PV4]*ms1;\n }\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n }\n\n // final rescale with 1/S and store to global memory\n if (sgitg == 0) {\n const int64_t nrows = args.ne3*args.ne2*args.ne1;\n const int64_t rid = iq3*args.ne2*args.ne1 + iq2 + iq1*args.ne1;\n\n device float4 * dst4 = (device float4 *) dst;\n device float * dst1 = (device float *) dst + nrows*DV*NWG; // the S and M are stored after the results\n\n const float S = NWG == 1 ? (ss[0] == 0.0f ? 0.0f : 1.0f/ss[0]) : 1.0f;\n\n // interleave the workgroup data\n for (short i = tiisg; i < DV4; i += NW) {\n dst4[rid*DV4*NWG + NWG*i + iwg] = (float4) so4[i]*S;\n }\n\n // store S and M\n if (NWG > 1) {\n if (tiisg == 0) {\n dst1[rid*(2*NWG) + 2*iwg + 0] = ss[0];\n dst1[rid*(2*NWG) + 2*iwg + 1] = ss[1];\n }\n }\n }\n\n#undef NWG\n#undef NSG\n#undef NS10\n#undef NS20\n}\n\n// note: I think the s_t can be half instead of float, because the Q*K scaling is done before storing to shared mem\n// in the other (non-vec) kernel, we need s_t to also be float because we scale during the soft_max\n//\n#define FA_TYPES \\\n half4, \\\n half4, \\\n half4, \\\n float, \\\n float, float4, \\\n float4\n\n#define FA_TYPES_F32 \\\n half4, \\\n float4, \\\n float4, \\\n float, \\\n float, float4, \\\n float4\n\ntypedef decltype(kernel_flash_attn_ext_vec) flash_attn_ext_vec_t;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f32_dk32_dv32\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f16_dk32_dv32\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_bf16_dk32_dv32\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#endif\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_0_dk32_dv32\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_1_dk32_dv32\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_0_dk32_dv32\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_1_dk32_dv32\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q8_0_dk32_dv32\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f32_dk64_dv64\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f16_dk64_dv64\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_bf16_dk64_dv64\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#endif\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_0_dk64_dv64\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_1_dk64_dv64\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_0_dk64_dv64\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_1_dk64_dv64\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q8_0_dk64_dv64\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f32_dk96_dv96\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f16_dk96_dv96\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_bf16_dk96_dv96\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#endif\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_0_dk96_dv96\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_1_dk96_dv96\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_0_dk96_dv96\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_1_dk96_dv96\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q8_0_dk96_dv96\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f32_dk128_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f16_dk128_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_bf16_dk128_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#endif\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_0_dk128_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_1_dk128_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_0_dk128_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_1_dk128_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q8_0_dk128_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f32_dk192_dv192\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f16_dk192_dv192\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_bf16_dk192_dv192\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#endif\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_0_dk192_dv192\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_1_dk192_dv192\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_0_dk192_dv192\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_1_dk192_dv192\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q8_0_dk192_dv192\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f32_dk192_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f16_dk192_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_bf16_dk192_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#endif\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_0_dk192_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_1_dk192_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_0_dk192_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_1_dk192_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q8_0_dk192_dv128\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f32_dk256_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f16_dk256_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_bf16_dk256_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#endif\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_0_dk256_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_1_dk256_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_0_dk256_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_1_dk256_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q8_0_dk256_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f32_dk320_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f16_dk320_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_bf16_dk320_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#endif\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_0_dk320_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_1_dk320_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_0_dk320_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_1_dk320_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q8_0_dk320_dv256\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f32_dk576_dv512\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_f16_dk576_dv512\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_bf16_dk576_dv512\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n#endif\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_0_dk576_dv512\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q4_1_dk576_dv512\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_0_dk576_dv512\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q5_1_dk576_dv512\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\ntemplate [[host_name(\"kernel_flash_attn_ext_vec_q8_0_dk576_dv512\")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec;\n\n#undef FA_TYPES\n#undef FA_TYPES_F32\n\nconstant int32_t FC_flash_attn_ext_vec_reduce_DV [[function_constant(FC_FLASH_ATTN_EXT_VEC_REDUCE + 0)]];\nconstant int32_t FC_flash_attn_ext_vec_reduce_NWG [[function_constant(FC_FLASH_ATTN_EXT_VEC_REDUCE + 1)]];\n\nkernel void kernel_flash_attn_ext_vec_reduce(\n constant ggml_metal_kargs_flash_attn_ext_vec_reduce & args,\n device const char * htmp,\n device char * dst,\n uint tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n#define NWG (FC_flash_attn_ext_vec_reduce_NWG)\n#define DV (FC_flash_attn_ext_vec_reduce_DV)\n\n const uint64_t rid = tgpig;\n\n const short iwg = tiisg;\n\n device const float * ss = (device const float *) htmp + (uint64_t)args.nrows*DV*NWG;\n\n float S = ss[rid*(2*NWG) + 2*iwg + 0];\n float M = ss[rid*(2*NWG) + 2*iwg + 1];\n\n const float m = simd_max(M);\n const float ms = exp(M - m);\n\n S = simd_sum(S*ms);\n S = S == 0.0f ? 0.0f : 1.0f/S;\n\n const short DV4 = DV/4;\n\n device const float4 * htmp4 = (device const float4 *) htmp + rid*DV4*NWG;\n device float4 * dst4 = (device float4 *) dst + rid*DV4;\n\n for (short i = sgitg; i < DV4; i += NWG) {\n const float4 v = simd_sum(htmp4[i*NWG + iwg]*ms);\n\n if (iwg == 0) {\n dst4[i] = v*S;\n }\n }\n\n#undef NWG\n#undef DV\n}\n\ntemplate\nkernel void kernel_cpy_t_t(\n constant ggml_metal_kargs_cpy & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int i03 = tgpig[2];\n const int i02 = tgpig[1];\n const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0];\n const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0;\n\n const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00;\n\n const int64_t i3 = n/(args.ne2*args.ne1*args.ne0);\n const int64_t i2 = (n - i3*args.ne2*args.ne1*args.ne0)/(args.ne1*args.ne0);\n const int64_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0;\n const int64_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0);\n\n device T1 * dst_data = (device T1 *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n for (int64_t i00 = iw0*ntg[0] + tiitg%ntg[0]; i00 < args.ne00; ) {\n device const T0 * src = (device T0 *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00);\n dst_data[i00] = (T1) src[0];\n break;\n }\n}\n\ntypedef decltype(kernel_cpy_t_t) kernel_cpy_t;\n\ntemplate [[host_name(\"kernel_cpy_f32_f32\")]] kernel kernel_cpy_t kernel_cpy_t_t;\ntemplate [[host_name(\"kernel_cpy_f32_f16\")]] kernel kernel_cpy_t kernel_cpy_t_t;\ntemplate [[host_name(\"kernel_cpy_f32_i32\")]] kernel kernel_cpy_t kernel_cpy_t_t;\ntemplate [[host_name(\"kernel_cpy_i32_f32\")]] kernel kernel_cpy_t kernel_cpy_t_t;\ntemplate [[host_name(\"kernel_cpy_i32_i32\")]] kernel kernel_cpy_t kernel_cpy_t_t;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_cpy_f32_bf16\")]] kernel kernel_cpy_t kernel_cpy_t_t;\n#endif\ntemplate [[host_name(\"kernel_cpy_f16_f32\")]] kernel kernel_cpy_t kernel_cpy_t_t;\ntemplate [[host_name(\"kernel_cpy_f16_f16\")]] kernel kernel_cpy_t kernel_cpy_t_t;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_cpy_bf16_f32\")]] kernel kernel_cpy_t kernel_cpy_t_t;\ntemplate [[host_name(\"kernel_cpy_bf16_bf16\")]] kernel kernel_cpy_t kernel_cpy_t_t;\n#endif\n\ntemplate\nkernel void kernel_cpy_f32_q(\n constant ggml_metal_kargs_cpy & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int i03 = tgpig[2];\n const int i02 = tgpig[1];\n const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0];\n const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0;\n\n const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00;\n\n const int64_t i3 = n / (args.ne2*args.ne1*args.ne0);\n const int64_t i2 = (n - i3*args.ne2*args.ne1*args.ne0) / (args.ne1*args.ne0);\n const int64_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0) / args.ne0;\n const int64_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0)/QK;\n\n device block_q * dst_data = (device block_q *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n for (int64_t i00 = iw0*ntg[0] + tiitg%ntg[0]; i00 < args.nk0; ) {\n device const float * src = (device const float *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + (i00*QK)*args.nb00);\n\n quantize_func(src, dst_data[i00]);\n\n break;\n }\n}\n\ntypedef decltype(kernel_cpy_f32_q) cpy_f_q_t;\n\ntemplate [[host_name(\"kernel_cpy_f32_q8_0\")]] kernel cpy_f_q_t kernel_cpy_f32_q;\ntemplate [[host_name(\"kernel_cpy_f32_q4_0\")]] kernel cpy_f_q_t kernel_cpy_f32_q;\ntemplate [[host_name(\"kernel_cpy_f32_q4_1\")]] kernel cpy_f_q_t kernel_cpy_f32_q;\ntemplate [[host_name(\"kernel_cpy_f32_q5_0\")]] kernel cpy_f_q_t kernel_cpy_f32_q;\ntemplate [[host_name(\"kernel_cpy_f32_q5_1\")]] kernel cpy_f_q_t kernel_cpy_f32_q;\ntemplate [[host_name(\"kernel_cpy_f32_iq4_nl\")]] kernel cpy_f_q_t kernel_cpy_f32_q;\n\ntemplate\nkernel void kernel_cpy_q_f32(\n constant ggml_metal_kargs_cpy & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const int i03 = tgpig[2];\n const int i02 = tgpig[1];\n const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0];\n const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0;\n\n const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00;\n\n const int64_t i3 = n/(args.ne2*args.ne1*args.ne0);\n const int64_t i2 = (n - i3*args.ne2*args.ne1*args.ne0)/(args.ne1*args.ne0);\n const int64_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0;\n const int64_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0);\n\n device const block_q * src_data = (device const block_q *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01);\n device T4x4 * dst_data = (device T4x4 *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n for (int64_t i00 = iw0*ntg[0] + tiitg%ntg[0]; i00 < args.nk0; ) {\n T4x4 temp;\n dequantize_func(src_data + i00/nl, i00%nl, temp);\n dst_data[i00] = temp;\n\n break;\n }\n}\n\ntypedef decltype(kernel_cpy_q_f32) cpy_q_f_t;\n\ntemplate [[host_name(\"kernel_cpy_q4_0_f32\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\ntemplate [[host_name(\"kernel_cpy_q4_1_f32\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\ntemplate [[host_name(\"kernel_cpy_q5_0_f32\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\ntemplate [[host_name(\"kernel_cpy_q5_1_f32\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\ntemplate [[host_name(\"kernel_cpy_q8_0_f32\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\n\ntemplate [[host_name(\"kernel_cpy_q4_0_f16\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\ntemplate [[host_name(\"kernel_cpy_q4_1_f16\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\ntemplate [[host_name(\"kernel_cpy_q5_0_f16\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\ntemplate [[host_name(\"kernel_cpy_q5_1_f16\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\ntemplate [[host_name(\"kernel_cpy_q8_0_f16\")]] kernel cpy_q_f_t kernel_cpy_q_f32;\n\nkernel void kernel_concat(\n constant ggml_metal_kargs_concat & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n\n const int i3 = tgpig.z;\n const int i2 = tgpig.y;\n const int i1 = tgpig.x;\n\n int o[4] = {0, 0, 0, 0};\n o[args.dim] = args.dim == 0 ? args.ne00 : (args.dim == 1 ? args.ne01 : (args.dim == 2 ? args.ne02 : args.ne03));\n\n device const float * x;\n\n for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {\n if (i0 < args.ne00 && i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) {\n x = (device const float *)(src0 + (i3 )*args.nb03 + (i2 )*args.nb02 + (i1 )*args.nb01 + (i0 )*args.nb00);\n } else {\n x = (device const float *)(src1 + (i3 - o[3])*args.nb13 + (i2 - o[2])*args.nb12 + (i1 - o[1])*args.nb11 + (i0 - o[0])*args.nb10);\n }\n\n device float * y = (device float *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);\n\n *y = *x;\n }\n}\n\ntemplate\nvoid kernel_mul_mv_q2_K_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_q2_K * x = (device const block_q2_K *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n float yl[32];\n float sumf[nr0]={0.f};\n\n const short ix = tiisg/8; // 0...3\n const short it = tiisg%8; // 0...7\n const short iq = it/4; // 0 or 1\n const short ir = it%4; // 0...3\n const short is = (8*ir)/16;// 0 or 1\n\n device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;\n\n for (int ib = ix; ib < nb; ib += 4) {\n float4 sumy = {0.f, 0.f, 0.f, 0.f};\n for (short i = 0; i < 8; ++i) {\n yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];\n yl[i+ 8] = y4[i+32]; sumy[1] += yl[i+ 8];\n yl[i+16] = y4[i+64]; sumy[2] += yl[i+16];\n yl[i+24] = y4[i+96]; sumy[3] += yl[i+24];\n }\n\n device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is;\n device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;\n device const half * dh = &x[ib].d;\n\n for (short row = 0; row < nr0; row++) {\n float4 acc1 = {0.f, 0.f, 0.f, 0.f};\n float4 acc2 = {0.f, 0.f, 0.f, 0.f};\n for (int i = 0; i < 8; i += 2) {\n acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003);\n acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300);\n acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c);\n acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00);\n acc1[2] += yl[i+16] * (qs[i/2] & 0x0030);\n acc2[2] += yl[i+17] * (qs[i/2] & 0x3000);\n acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0);\n acc2[3] += yl[i+25] * (qs[i/2] & 0xc000);\n }\n float dall = dh[0];\n float dmin = dh[1] * 1.f/16.f;\n sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f +\n (acc1[1] + 1.f/256.f * acc2[1]) * (sc[2] & 0xF) * 1.f/ 4.f +\n (acc1[2] + 1.f/256.f * acc2[2]) * (sc[4] & 0xF) * 1.f/16.f +\n (acc1[3] + 1.f/256.f * acc2[3]) * (sc[6] & 0xF) * 1.f/64.f) -\n dmin * (sumy[0] * (sc[0] & 0xF0) + sumy[1] * (sc[2] & 0xF0) + sumy[2] * (sc[4] & 0xF0) + sumy[3] * (sc[6] & 0xF0));\n\n qs += args.nb01/2;\n sc += args.nb01;\n dh += args.nb01/2;\n }\n\n y4 += 4 * QK_K;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_q2_K_f32\")]]\nkernel void kernel_mul_mv_q2_K_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_q2_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_q3_K_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_q3_K * x = (device const block_q3_K *) (src0 + offset0);\n device const float * yy = (device const float *) (src1 + offset1);\n\n float yl[32];\n\n //const uint16_t kmask1 = 0x3030;\n //const uint16_t kmask2 = 0x0f0f;\n\n const short tid = tiisg/4;\n const short ix = tiisg%4;\n const short ip = tid/4; // 0 or 1\n const short il = 2*((tid%4)/2); // 0 or 2\n const short ir = tid%2;\n const short l0 = 8*ir;\n\n // One would think that the Metal compiler would figure out that ip and il can only have\n // 4 possible states, and optimize accordingly. Well, no. It needs help, and we do it\n // with these two tales.\n //\n // Possible masks for the high bit\n const ushort4 mm[4] = {{0x0001, 0x0100, 0x0002, 0x0200}, // ip = 0, il = 0\n {0x0004, 0x0400, 0x0008, 0x0800}, // ip = 0, il = 2\n {0x0010, 0x1000, 0x0020, 0x2000}, // ip = 1, il = 0\n {0x0040, 0x4000, 0x0080, 0x8000}}; // ip = 1, il = 2\n\n // Possible masks for the low 2 bits\n const int4 qm[2] = {{0x0003, 0x0300, 0x000c, 0x0c00}, {0x0030, 0x3000, 0x00c0, 0xc000}};\n\n const ushort4 hm = mm[2*ip + il/2];\n\n const short shift = 2*il;\n\n const float v1 = il == 0 ? 4.f : 64.f;\n const float v2 = 4.f * v1;\n\n const uint16_t s_shift1 = 4*ip;\n const uint16_t s_shift2 = s_shift1 + il;\n\n const short q_offset = 32*ip + l0;\n const short y_offset = 128*ip + 32*il + l0;\n\n device const float * y1 = yy + ix*QK_K + y_offset;\n\n uint32_t scales32, aux32;\n thread uint16_t * scales16 = (thread uint16_t *)&scales32;\n thread const int8_t * scales = (thread const int8_t *)&scales32;\n\n float sumf1[nr0] = {0.f};\n float sumf2[nr0] = {0.f};\n\n for (int i = ix; i < nb; i += 4) {\n for (short l = 0; l < 8; ++l) {\n yl[l+ 0] = y1[l+ 0];\n yl[l+ 8] = y1[l+16];\n yl[l+16] = y1[l+32];\n yl[l+24] = y1[l+48];\n }\n\n device const uint16_t * q = (device const uint16_t *)(x[i].qs + q_offset);\n device const uint16_t * h = (device const uint16_t *)(x[i].hmask + l0);\n device const uint16_t * a = (device const uint16_t *)(x[i].scales);\n device const half * dh = &x[i].d;\n\n for (short row = 0; row < nr0; ++row) {\n const float d_all = (float)dh[0];\n\n scales16[0] = a[4];\n scales16[1] = a[5];\n aux32 = ((scales32 >> s_shift2) << 4) & 0x30303030;\n scales16[0] = a[il+0];\n scales16[1] = a[il+1];\n scales32 = ((scales32 >> s_shift1) & 0x0f0f0f0f) | aux32;\n\n float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;\n for (short l = 0; l < 8; l += 2) {\n const int32_t qs = q[l/2];\n s1 += yl[l+0] * (qs & qm[il/2][0]);\n s2 += yl[l+1] * (qs & qm[il/2][1]);\n s3 += ((h[l/2] & hm[0]) ? 0.f : yl[l+0]) + ((h[l/2] & hm[1]) ? 0.f : yl[l+1]);\n s4 += yl[l+16] * (qs & qm[il/2][2]);\n s5 += yl[l+17] * (qs & qm[il/2][3]);\n s6 += ((h[l/2] & hm[2]) ? 0.f : yl[l+16]) + ((h[l/2] & hm[3]) ? 0.f : yl[l+17]);\n }\n float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);\n float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);\n sumf1[row] += d1 * (scales[0] - 32);\n sumf2[row] += d2 * (scales[2] - 32);\n\n s1 = s2 = s3 = s4 = s5 = s6 = 0;\n for (short l = 0; l < 8; l += 2) {\n const int32_t qs = q[l/2+8];\n s1 += yl[l+8] * (qs & qm[il/2][0]);\n s2 += yl[l+9] * (qs & qm[il/2][1]);\n s3 += ((h[l/2+8] & hm[0]) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm[1]) ? 0.f : yl[l+9]);\n s4 += yl[l+24] * (qs & qm[il/2][2]);\n s5 += yl[l+25] * (qs & qm[il/2][3]);\n s6 += ((h[l/2+8] & hm[2]) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm[3]) ? 0.f : yl[l+25]);\n }\n d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);\n d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);\n sumf1[row] += d1 * (scales[1] - 32);\n sumf2[row] += d2 * (scales[3] - 32);\n\n q += args.nb01/2;\n h += args.nb01/2;\n a += args.nb01/2;\n dh += args.nb01/2;\n }\n\n y1 += 4 * QK_K;\n }\n\n for (int row = 0; row < nr0; ++row) {\n const float sumf = (sumf1[row] + 0.25f * sumf2[row]) / (1 << shift);\n sumf1[row] = simd_sum(sumf);\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n if (tiisg == 0) {\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n dst_f32[first_row + row] = sumf1[row];\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_q3_K_f32\")]]\nkernel void kernel_mul_mv_q3_K_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_q3_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_q4_K_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n constexpr uint16_t kmask1 = 0x3f3f;\n constexpr uint16_t kmask2 = 0x0f0f;\n constexpr uint16_t kmask3 = 0xc0c0;\n\n const short ix = tiisg/8; // 0...3\n const short it = tiisg%8; // 0...7\n const short iq = it/4; // 0 or 1\n const short ir = it%4; // 0...3\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_q4_K * x = (device const block_q4_K *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n float yl[16];\n float yh[16];\n\n float sumf[nr0]={0.f};\n\n device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;\n\n uint16_t sc16[4];\n thread const uint8_t * sc8 = (thread const uint8_t *)sc16;\n\n for (int ib = ix; ib < nb; ib += 4) {\n float4 sumy = {0.f, 0.f, 0.f, 0.f};\n\n for (short i = 0; i < 8; ++i) {\n yl[i+0] = y4[i+ 0]; sumy[0] += yl[i+0];\n yl[i+8] = y4[i+ 32]; sumy[1] += yl[i+8];\n yh[i+0] = y4[i+128]; sumy[2] += yh[i+0];\n yh[i+8] = y4[i+160]; sumy[3] += yh[i+8];\n }\n\n device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq;\n device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;\n device const half * dh = &x[ib].d;\n\n for (short row = 0; row < nr0; row++) {\n sc16[0] = sc[0] & kmask1;\n sc16[1] = sc[2] & kmask1;\n sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);\n sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);\n\n device const uint16_t * q2 = q1 + 32;\n\n float4 acc1 = {0.f, 0.f, 0.f, 0.f};\n float4 acc2 = {0.f, 0.f, 0.f, 0.f};\n\n FOR_UNROLL (short i = 0; i < 4; ++i) {\n acc1[0] += yl[2*i + 0] * (q1[i] & 0x000F);\n acc1[1] += yl[2*i + 1] * (q1[i] & 0x0F00);\n acc1[2] += yl[2*i + 8] * (q1[i] & 0x00F0);\n acc1[3] += yl[2*i + 9] * (q1[i] & 0xF000);\n acc2[0] += yh[2*i + 0] * (q2[i] & 0x000F);\n acc2[1] += yh[2*i + 1] * (q2[i] & 0x0F00);\n acc2[2] += yh[2*i + 8] * (q2[i] & 0x00F0);\n acc2[3] += yh[2*i + 9] * (q2[i] & 0xF000);\n }\n\n sumf[row] += dh[0] * ((acc1[0] + 1.f/256.f * acc1[1]) * sc8[0] +\n (acc1[2] + 1.f/256.f * acc1[3]) * sc8[1] * 1.f/16.f +\n (acc2[0] + 1.f/256.f * acc2[1]) * sc8[4] +\n (acc2[2] + 1.f/256.f * acc2[3]) * sc8[5] * 1.f/16.f) -\n dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);\n\n q1 += args.nb01/2;\n sc += args.nb01/2;\n dh += args.nb01/2;\n }\n\n y4 += 4 * QK_K;\n }\n\n device float * dst_f32 = (device float *) dst + (int64_t)im*args.ne0*args.ne1 + (int64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_q4_K_f32\")]]\nkernel void kernel_mul_mv_q4_K_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_q4_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_q5_K_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_q5_K * x = (device const block_q5_K *) (src0 + offset0);\n device const float * yy = (device const float *) (src1 + offset1);\n\n float sumf[nr0]={0.f};\n\n float yl[16], yh[16];\n\n constexpr uint16_t kmask1 = 0x3f3f;\n constexpr uint16_t kmask2 = 0x0f0f;\n constexpr uint16_t kmask3 = 0xc0c0;\n\n const short tid = tiisg/4;\n const short ix = tiisg%4;\n const short iq = tid/4;\n const short ir = tid%4;\n\n const short l0 = 8*ir;\n const short q_offset = 32*iq + l0;\n const short y_offset = 64*iq + l0;\n\n const uint8_t hm1 = 1u << (2*iq);\n const uint8_t hm2 = hm1 << 1;\n const uint8_t hm3 = hm1 << 4;\n const uint8_t hm4 = hm2 << 4;\n\n uint16_t sc16[4];\n thread const uint8_t * sc8 = (thread const uint8_t *)sc16;\n\n device const float * y1 = yy + ix*QK_K + y_offset;\n\n for (int i = ix; i < nb; i += 4) {\n device const uint8_t * q1 = x[i].qs + q_offset;\n device const uint8_t * qh = x[i].qh + l0;\n device const half * dh = &x[i].d;\n device const uint16_t * a = (device const uint16_t *)x[i].scales + iq;\n\n device const float * y2 = y1 + 128;\n float4 sumy = {0.f, 0.f, 0.f, 0.f};\n for (short l = 0; l < 8; ++l) {\n yl[l+0] = y1[l+ 0]; sumy[0] += yl[l+0];\n yl[l+8] = y1[l+32]; sumy[1] += yl[l+8];\n yh[l+0] = y2[l+ 0]; sumy[2] += yh[l+0];\n yh[l+8] = y2[l+32]; sumy[3] += yh[l+8];\n }\n\n for (short row = 0; row < nr0; ++row) {\n device const uint8_t * q2 = q1 + 64;\n\n sc16[0] = a[0] & kmask1;\n sc16[1] = a[2] & kmask1;\n sc16[2] = ((a[4] >> 0) & kmask2) | ((a[0] & kmask3) >> 2);\n sc16[3] = ((a[4] >> 4) & kmask2) | ((a[2] & kmask3) >> 2);\n\n float4 acc1 = {0.f};\n float4 acc2 = {0.f};\n FOR_UNROLL (short l = 0; l < 8; ++l) {\n uint8_t h = qh[l];\n acc1[0] += yl[l+0] * (q1[l] & 0x0F);\n acc1[1] += yl[l+8] * (q1[l] & 0xF0);\n acc1[2] += yh[l+0] * (q2[l] & 0x0F);\n acc1[3] += yh[l+8] * (q2[l] & 0xF0);\n acc2[0] += h & hm1 ? yl[l+0] : 0.f;\n acc2[1] += h & hm2 ? yl[l+8] : 0.f;\n acc2[2] += h & hm3 ? yh[l+0] : 0.f;\n acc2[3] += h & hm4 ? yh[l+8] : 0.f;\n }\n\n sumf[row] += dh[0] * (sc8[0] * (acc1[0] + 16.f*acc2[0]) +\n sc8[1] * (acc1[1]/16.f + 16.f*acc2[1]) +\n sc8[4] * (acc1[2] + 16.f*acc2[2]) +\n sc8[5] * (acc1[3]/16.f + 16.f*acc2[3])) -\n dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);\n\n q1 += args.nb01;\n qh += args.nb01;\n dh += args.nb01/2;\n a += args.nb01/2;\n }\n\n y1 += 4 * QK_K;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n const float tot = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = tot;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_q5_K_f32\")]]\nkernel void kernel_mul_mv_q5_K_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_q5_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_q6_K_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n constexpr uint8_t kmask1 = 0x03;\n constexpr uint8_t kmask2 = 0x0C;\n constexpr uint8_t kmask3 = 0x30;\n constexpr uint8_t kmask4 = 0xC0;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_q6_K * x = (device const block_q6_K *) (src0 + offset0);\n device const float * yy = (device const float *) (src1 + offset1);\n\n float sumf[nr0] = { 0.f };\n\n float yl[16];\n\n const short tid = tiisg/2;\n const short ix = tiisg%2;\n const short ip = tid/8; // 0 or 1\n const short il = tid%8;\n const short l0 = 4*il;\n const short is = 8*ip + l0/16;\n\n const short y_offset = 128*ip + l0;\n const short q_offset_l = 64*ip + l0;\n const short q_offset_h = 32*ip + l0;\n\n for (int i = ix; i < nb; i += 2) {\n device const uint8_t * q1 = x[i].ql + q_offset_l;\n device const uint8_t * q2 = q1 + 32;\n device const uint8_t * qh = x[i].qh + q_offset_h;\n device const int8_t * sc = x[i].scales + is;\n device const half * dh = &x[i].d;\n\n device const float * y = yy + i * QK_K + y_offset;\n\n for (short l = 0; l < 4; ++l) {\n yl[4*l + 0] = y[l + 0];\n yl[4*l + 1] = y[l + 32];\n yl[4*l + 2] = y[l + 64];\n yl[4*l + 3] = y[l + 96];\n }\n\n for (short row = 0; row < nr0; ++row) {\n float4 sums = {0.f, 0.f, 0.f, 0.f};\n\n FOR_UNROLL (short l = 0; l < 4; ++l) {\n sums[0] += yl[4*l + 0] * ((int8_t)((q1[l] & 0xF) | ((qh[l] & kmask1) << 4)) - 32);\n sums[1] += yl[4*l + 1] * ((int8_t)((q2[l] & 0xF) | ((qh[l] & kmask2) << 2)) - 32);\n sums[2] += yl[4*l + 2] * ((int8_t)((q1[l] >> 4) | ((qh[l] & kmask3) << 0)) - 32);\n sums[3] += yl[4*l + 3] * ((int8_t)((q2[l] >> 4) | ((qh[l] & kmask4) >> 2)) - 32);\n }\n\n sumf[row] += dh[0] * (sums[0] * sc[0] + sums[1] * sc[2] + sums[2] * sc[4] + sums[3] * sc[6]);\n\n q1 += args.nb01;\n q2 += args.nb01;\n qh += args.nb01;\n sc += args.nb01;\n dh += args.nb01/2;\n }\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_q6_K_f32\")]]\nkernel void kernel_mul_mv_q6_K_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_q6_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);\n}\n\n// ======================= \"True\" 2-bit\n\ntemplate\nvoid kernel_mul_mv_iq2_xxs_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n float yl[32];\n float sumf[nr0]={0.f};\n\n const int nb32 = nb * (QK_K / 32);\n\n threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem);\n threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256);\n {\n int nval = 4;\n int pos = (32*sgitg + tiisg)*nval;\n for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xxs_grid[pos + i];\n nval = 2;\n pos = (32*sgitg + tiisg)*nval;\n for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n }\n\n const int ix = tiisg;\n\n device const float * y4 = y + 32 * ix;\n\n for (int ib32 = ix; ib32 < nb32; ib32 += 32) {\n for (short i = 0; i < 32; ++i) {\n yl[i] = y4[i];\n }\n\n const int ibl = ib32 / (QK_K / 32);\n const int ib = ib32 % (QK_K / 32);\n\n device const block_iq2_xxs * xr = x + ibl;\n device const uint16_t * q2 = xr->qs + 4 * ib;\n device const half * dh = &xr->d;\n\n for (short row = 0; row < nr0; row++) {\n const float db = dh[0];\n device const uint8_t * aux8 = (device const uint8_t *)q2;\n const uint32_t aux32 = q2[2] | (q2[3] << 16);\n const float d = db * (0.5f + (aux32 >> 28));\n\n float sum = 0;\n for (short l = 0; l < 4; ++l) {\n const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + aux8[l]);\n const uint8_t signs = ssigns[(aux32 >> 7*l) & 127];\n for (short j = 0; j < 8; ++j) {\n sum += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n }\n }\n sumf[row] += d * sum;\n\n dh += args.nb01/2;\n q2 += args.nb01/2;\n }\n\n y4 += 32 * 32;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all * 0.25f;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_iq2_xxs_f32\")]]\nkernel void kernel_mul_mv_iq2_xxs_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_iq2_xs_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n float yl[32];\n float sumf[nr0]={0.f};\n\n const int nb32 = nb * (QK_K / 32);\n\n threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem);\n threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512);\n {\n int nval = 8;\n int pos = (32*sgitg + tiisg)*nval;\n for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xs_grid[pos + i];\n nval = 2;\n pos = (32*sgitg + tiisg)*nval;\n for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n }\n\n const int ix = tiisg;\n\n device const float * y4 = y + 32 * ix;\n\n for (int ib32 = ix; ib32 < nb32; ib32 += 32) {\n for (short i = 0; i < 32; ++i) {\n yl[i] = y4[i];\n }\n\n const int ibl = ib32 / (QK_K / 32);\n const int ib = ib32 % (QK_K / 32);\n\n device const block_iq2_xs * xr = x + ibl;\n device const uint16_t * q2 = xr->qs + 4 * ib;\n device const uint8_t * sc = xr->scales + ib;\n device const half * dh = &xr->d;\n\n for (short row = 0; row < nr0; row++) {\n const float db = dh[0];\n const uint8_t ls1 = sc[0] & 0xf;\n const uint8_t ls2 = sc[0] >> 4;\n const float d1 = db * (0.5f + ls1);\n const float d2 = db * (0.5f + ls2);\n\n float sum1 = 0, sum2 = 0;\n for (short l = 0; l < 2; ++l) {\n const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511));\n const uint8_t signs = ssigns[(q2[l] >> 9)];\n for (short j = 0; j < 8; ++j) {\n sum1 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n }\n }\n for (short l = 2; l < 4; ++l) {\n const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511));\n const uint8_t signs = ssigns[(q2[l] >> 9)];\n for (short j = 0; j < 8; ++j) {\n sum2 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n }\n }\n sumf[row] += d1 * sum1 + d2 * sum2;\n\n dh += args.nb01/2;\n q2 += args.nb01/2;\n sc += args.nb01;\n }\n\n y4 += 32 * 32;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all * 0.25f;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_iq2_xs_f32\")]]\nkernel void kernel_mul_mv_iq2_xs_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_iq3_xxs_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_iq3_xxs * x = (device const block_iq3_xxs *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n float yl[32];\n float sumf[nr0]={0.f};\n\n const int nb32 = nb * (QK_K / 32);\n\n threadgroup uint32_t * svalues = (threadgroup uint32_t *)(shmem);\n threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256);\n {\n int nval = 4;\n int pos = (32*sgitg + tiisg)*nval;\n for (int i = 0; i < nval; ++i) svalues[pos + i] = iq3xxs_grid[pos + i];\n nval = 2;\n pos = (32*sgitg + tiisg)*nval;\n for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n }\n\n const int ix = tiisg;\n\n device const float * y4 = y + 32 * ix;\n\n for (int ib32 = ix; ib32 < nb32; ib32 += 32) {\n for (short i = 0; i < 32; ++i) {\n yl[i] = y4[i];\n }\n\n const int ibl = ib32 / (QK_K / 32);\n const int ib = ib32 % (QK_K / 32);\n\n device const block_iq3_xxs * xr = x + ibl;\n device const uint8_t * q3 = xr->qs + 8 * ib;\n device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib;\n device const half * dh = &xr->d;\n\n for (short row = 0; row < nr0; row++) {\n const float db = dh[0];\n const uint32_t aux32 = gas[0] | (gas[1] << 16);\n const float d = db * (0.5f + (aux32 >> 28));\n\n float2 sum = {0};\n for (short l = 0; l < 4; ++l) {\n const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(svalues + q3[2*l+0]);\n const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(svalues + q3[2*l+1]);\n const uint8_t signs = ssigns[(aux32 >> 7*l) & 127];\n for (short j = 0; j < 4; ++j) {\n sum[0] += yl[8*l + j + 0] * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);\n sum[1] += yl[8*l + j + 4] * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);\n }\n }\n sumf[row] += d * (sum[0] + sum[1]);\n\n dh += args.nb01/2;\n q3 += args.nb01;\n gas += args.nb01/2;\n }\n\n y4 += 32 * 32;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all * 0.5f;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_iq3_xxs_f32\")]]\nkernel void kernel_mul_mv_iq3_xxs_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_iq3_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_iq3_s_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_iq3_s * x = (device const block_iq3_s *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n float yl[32];\n float sumf[nr0]={0.f};\n\n const int nb32 = nb * (QK_K / 32);\n\n threadgroup uint32_t * svalues = (threadgroup uint32_t *) shmem;\n {\n int nval = 8;\n int pos = (32*sgitg + tiisg)*nval;\n for (int i = 0; i < nval; ++i) svalues[pos + i] = iq3s_grid[pos + i];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n }\n\n const int ix = tiisg;\n\n device const float * y4 = y + 32 * ix;\n\n for (int ib32 = ix; ib32 < nb32; ib32 += 32) {\n for (short i = 0; i < 32; ++i) {\n yl[i] = y4[i];\n }\n\n const int ibl = ib32 / (QK_K / 32);\n const int ib = ib32 % (QK_K / 32);\n\n device const block_iq3_s * xr = x + ibl;\n device const uint8_t * qs = xr->qs + 8 * ib;\n device const uint8_t * qh = xr->qh + ib;\n device const uint8_t * sc = xr->scales + (ib/2);\n device const uint8_t * signs = xr->signs + 4 * ib;\n device const half * dh = &xr->d;\n\n for (short row = 0; row < nr0; row++) {\n const float db = dh[0];\n const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf));\n\n float2 sum = {0};\n for (short l = 0; l < 4; ++l) {\n const threadgroup uint32_t * table1 = qh[0] & kmask_iq2xs[2*l+0] ? svalues + 256 : svalues;\n const threadgroup uint32_t * table2 = qh[0] & kmask_iq2xs[2*l+1] ? svalues + 256 : svalues;\n const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(table1 + qs[2*l+0]);\n const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(table2 + qs[2*l+1]);\n for (short j = 0; j < 4; ++j) {\n sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l] & kmask_iq2xs[j+0]);\n sum[1] += yl[8*l + j + 4] * grid2[j] * select(1, -1, signs[l] & kmask_iq2xs[j+4]);\n }\n }\n sumf[row] += d * (sum[0] + sum[1]);\n\n dh += args.nb01/2;\n qs += args.nb01;\n qh += args.nb01;\n sc += args.nb01;\n signs += args.nb01;\n }\n\n y4 += 32 * 32;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_iq3_s_f32\")]]\nkernel void kernel_mul_mv_iq3_s_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_iq2_s_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_iq2_s * x = (device const block_iq2_s *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n float yl[32];\n float sumf[nr0]={0.f};\n\n const int nb32 = nb * (QK_K / 32);\n\n //threadgroup uint64_t * svalues = (threadgroup uint64_t *) shmem;\n //{\n // int nval = 32;\n // int pos = (32*sgitg + tiisg)*nval;\n // for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2s_grid[pos + i];\n // threadgroup_barrier(mem_flags::mem_threadgroup);\n //}\n\n const short ix = tiisg;\n\n device const float * y4 = y + 32 * ix;\n\n for (int ib32 = ix; ib32 < nb32; ib32 += 32) {\n for (short i = 0; i < 32; ++i) {\n yl[i] = y4[i];\n }\n\n const int ibl = ib32 / (QK_K / 32);\n const int ib = ib32 % (QK_K / 32);\n\n device const block_iq2_s * xr = x + ibl;\n device const uint8_t * qs = xr->qs + 4 * ib;\n device const uint8_t * qh = xr->qh + ib;\n device const uint8_t * sc = xr->scales + ib;\n device const uint8_t * signs = qs + QK_K/8;\n device const half * dh = &xr->d;\n\n for (short row = 0; row < nr0; row++) {\n const float db = dh[0];\n const float d1 = db * (0.5f + (sc[0] & 0xf));\n const float d2 = db * (0.5f + (sc[0] >> 4));\n\n float2 sum = {0};\n for (short l = 0; l < 2; ++l) {\n //const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(svalues + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300)));\n //const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(svalues + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300)));\n constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300)));\n constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300)));\n for (short j = 0; j < 8; ++j) {\n sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l+0] & kmask_iq2xs[j]);\n sum[1] += yl[8*l + j + 16] * grid2[j] * select(1, -1, signs[l+2] & kmask_iq2xs[j]);\n }\n }\n sumf[row] += d1 * sum[0] + d2 * sum[1];\n\n dh += args.nb01/2;\n qs += args.nb01;\n qh += args.nb01;\n sc += args.nb01;\n signs += args.nb01;\n }\n\n y4 += 32 * 32;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all * 0.25f;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_iq2_s_f32\")]]\nkernel void kernel_mul_mv_iq2_s_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_iq1_s_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n float yl[32];\n float sumf[nr0]={0.f};\n\n const int nb32 = nb * (QK_K / 32);\n\n const short ix = tiisg;\n\n device const float * y4 = y + 32 * ix;\n\n for (int ib32 = ix; ib32 < nb32; ib32 += 32) {\n float sumy = 0;\n for (short i = 0; i < 32; ++i) {\n yl[i] = y4[i];\n sumy += yl[i];\n }\n\n const int ibl = ib32 / (QK_K / 32);\n const int ib = ib32 % (QK_K / 32);\n\n device const block_iq1_s * xr = x + ibl;\n device const uint8_t * qs = xr->qs + 4 * ib;\n device const uint16_t * qh = xr->qh + ib;\n device const half * dh = &xr->d;\n\n for (short row = 0; row < nr0; row++) {\n constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));\n constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700)));\n constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700)));\n constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[0] >> 1) & 0x700)));\n\n float sum = 0;\n for (short j = 0; j < 4; ++j) {\n sum += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4)\n + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4)\n + yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4)\n + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4);\n }\n sumf[row] += (float)dh[0] * (sum + sumy * (qh[0] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA)) * (2*((qh[0] >> 12) & 7) + 1);\n\n dh += args.nb01/2;\n qs += args.nb01;\n qh += args.nb01/2;\n }\n\n y4 += 32 * 32;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_iq1_s_f32\")]]\nkernel void kernel_mul_mv_iq1_s_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_iq1_m_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n const int nb = args.ne00/QK_K;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * nr0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n float yl[32];\n float sumf[nr0]={0.f};\n\n const int nb32 = nb * (QK_K / 32);\n\n const short ix = tiisg;\n\n device const float * y4 = y + 32 * ix;\n\n iq1m_scale_t scale;\n\n for (int ib32 = ix; ib32 < nb32; ib32 += 32) {\n float4 sumy = {0.f};\n for (short i = 0; i < 8; ++i) {\n yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];\n yl[i+ 8] = y4[i+ 8]; sumy[1] += yl[i+ 8];\n yl[i+16] = y4[i+16]; sumy[2] += yl[i+16];\n yl[i+24] = y4[i+24]; sumy[3] += yl[i+24];\n }\n\n const int ibl = ib32 / (QK_K / 32);\n const int ib = ib32 % (QK_K / 32);\n\n device const block_iq1_m * xr = x + ibl;\n device const uint8_t * qs = xr->qs + 4 * ib;\n device const uint8_t * qh = xr->qh + 2 * ib;\n device const uint16_t * sc = (device const uint16_t *)xr->scales;\n\n for (short row = 0; row < nr0; row++) {\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n\n constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));\n constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700)));\n constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[1] << 8) & 0x700)));\n constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[1] << 4) & 0x700)));\n\n float2 sum = {0.f};\n for (short j = 0; j < 4; ++j) {\n sum[0] += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4)\n + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4);\n sum[1] += yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4)\n + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4);\n }\n const float delta1 = sumy[0] * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[1] * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);\n const float delta2 = sumy[2] * (qh[1] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[3] * (qh[1] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);\n\n sumf[row] += (float)scale.f16 * ((sum[0] + delta1) * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 7) + 1) +\n (sum[1] + delta2) * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 7) + 1));\n\n sc += args.nb01/2;\n qs += args.nb01;\n qh += args.nb01;\n }\n\n y4 += 32 * 32;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_iq1_m_f32\")]]\nkernel void kernel_mul_mv_iq1_m_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_iq4_nl_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n threadgroup float * shmem_f32 = (threadgroup float *) shmem;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * NR0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_iq4_nl * x = (device const block_iq4_nl *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n const int nb = args.ne00/QK4_NL;\n const int ns01 = args.nb01/args.nb00;\n\n const short ix = tiisg/2; // 0...15\n const short it = tiisg%2; // 0 or 1\n\n shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n float4 yl[4];\n float sumf[NR0]={0.f};\n\n device const float * yb = y + ix*QK4_NL + it*8;\n\n uint32_t aux32[2];\n thread const uint8_t * q8 = (thread const uint8_t *)aux32;\n\n float4 qf1, qf2;\n\n // [TAG_MUL_MV_WEIRD]\n for (int ib = ix; ib < nb && ib < ns01; ib += 16) {\n device const float4 * y4 = (device const float4 *)yb;\n yl[0] = y4[0];\n yl[1] = y4[4];\n yl[2] = y4[1];\n yl[3] = y4[5];\n\n for (short row = 0; row < NR0; row++) {\n device const block_iq4_nl & xb = x[row*ns01 + ib];\n device const uint16_t * q4 = (device const uint16_t *)(xb.qs + 8*it);\n\n float4 acc1 = {0.f}, acc2 = {0.f};\n\n aux32[0] = q4[0] | (q4[1] << 16);\n aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f;\n aux32[0] &= 0x0f0f0f0f;\n qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]};\n qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]};\n acc1 += yl[0] * qf1;\n acc2 += yl[1] * qf2;\n\n aux32[0] = q4[2] | (q4[3] << 16);\n aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f;\n aux32[0] &= 0x0f0f0f0f;\n qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]};\n qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]};\n acc1 += yl[2] * qf1;\n acc2 += yl[3] * qf2;\n\n acc1 += acc2;\n\n sumf[row] += (float)xb.d * (acc1[0] + acc1[1] + acc1[2] + acc1[3]);\n }\n\n yb += 16 * QK4_NL;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_iq4_nl_f32\")]]\nkernel void kernel_mul_mv_iq4_nl_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_iq4_nl_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_iq4_xs_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n threadgroup float * shmem_f32 = (threadgroup float *) shmem;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n const int first_row = (r0 * NSG + sgitg) * NR0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_iq4_xs * x = (device const block_iq4_xs *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n const int nb = args.ne00/QK_K;\n const int ns01 = args.nb01/args.nb00;\n\n const short ix = tiisg/16; // 0 or 1\n const short it = tiisg%16; // 0...15\n const short ib = it/2;\n const short il = it%2;\n\n shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n float4 yl[4];\n float sumf[NR0]={0.f};\n\n device const float * yb = y + ix * QK_K + ib * 32 + il * 8;\n\n uint32_t aux32[2];\n thread const uint8_t * q8 = (thread const uint8_t *)aux32;\n\n float4 qf1, qf2;\n\n // [TAG_MUL_MV_WEIRD]\n for (int ibl = ix; ibl < nb && ibl < ns01; ibl += 2) {\n device const float4 * y4 = (device const float4 *)yb;\n yl[0] = y4[0];\n yl[1] = y4[4];\n yl[2] = y4[1];\n yl[3] = y4[5];\n\n for (short row = 0; row < NR0; ++row) {\n device const block_iq4_xs & xb = x[row*ns01 + ibl];\n device const uint32_t * q4 = (device const uint32_t *)(xb.qs + 16*ib + 8*il);\n\n float4 acc1 = {0.f}, acc2 = {0.f};\n\n aux32[0] = (q4[0] ) & 0x0f0f0f0f;\n aux32[1] = (q4[0] >> 4) & 0x0f0f0f0f;\n qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]};\n qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]};\n acc1 += yl[0] * qf1;\n acc2 += yl[1] * qf2;\n\n aux32[0] = (q4[1] ) & 0x0f0f0f0f;\n aux32[1] = (q4[1] >> 4) & 0x0f0f0f0f;\n qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]};\n qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]};\n acc1 += yl[2] * qf1;\n acc2 += yl[3] * qf2;\n\n acc1 += acc2;\n\n const int ls = (((xb.scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((xb.scales_h >> 2*ib) & 3) << 4)) - 32;\n sumf[row] += (float)xb.d * ls * (acc1[0] + acc1[1] + acc1[2] + acc1[3]);\n }\n\n yb += 2 * QK_K;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_iq4_xs_f32\")]]\nkernel void kernel_mul_mv_iq4_xs_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_iq4_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntemplate\nvoid kernel_mul_mv_mxfp4_f32_impl(\n args_t args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg) {\n const short NSG = FC_mul_mv_nsg;\n\n threadgroup float * shmem_f32 = (threadgroup float *) shmem;\n\n const int r0 = tgpig.x;\n const int r1 = tgpig.y;\n const int im = tgpig.z;\n\n const int first_row = (r0 * NSG + sgitg) * NR0;\n\n const uint i12 = im%args.ne12;\n const uint i13 = im/args.ne12;\n\n const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;\n\n device const block_mxfp4 * x = (device const block_mxfp4 *) (src0 + offset0);\n device const float * y = (device const float *) (src1 + offset1);\n\n const int nb = args.ne00/QK_MXFP4;\n const int ns01 = args.nb01/args.nb00; // this can be larger than nb for permuted src0 tensors\n\n const short ix = tiisg/2; // 0...15\n const short it = tiisg%2; // 0 or 1\n\n shmem_f32[tiisg] = kvalues_mxfp4_f[tiisg%16];\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n float4 yl[4];\n float sumf[NR0]={0.f};\n\n device const float * yb = y + ix*QK_MXFP4 + it*8;\n\n // note: just the check `ib < nb` is enough, but adding the redundant `&& ib < ns01` check makes the kernel a bit faster\n // no idea why that is - needs some deeper investigation [TAG_MUL_MV_WEIRD]\n for (int ib = ix; ib < nb && ib < ns01; ib += 16) {\n device const float4 * y4 = (device const float4 *) yb;\n\n yl[0] = y4[0];\n yl[1] = y4[4];\n yl[2] = y4[1];\n yl[3] = y4[5];\n\n FOR_UNROLL (short row = 0; row < NR0; row++) {\n device const block_mxfp4 & xb = x[row*ns01 + ib];\n device const uint8_t * q2 = (device const uint8_t *)(xb.qs + 8*it);\n\n float4 acc1 = yl[0]*float4(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]);\n float4 acc2 = yl[1]*float4(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]);\n float4 acc3 = yl[2]*float4(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]);\n float4 acc4 = yl[3]*float4(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]);\n\n acc1 = (acc1 + acc3) + (acc2 + acc4);\n\n sumf[row] += e8m0_to_fp32(xb.e) * ((acc1[0] + acc1[1]) + (acc1[2] + acc1[3]));\n }\n\n yb += 16 * QK_MXFP4;\n }\n\n device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;\n\n for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) {\n float sum_all = simd_sum(sumf[row]);\n if (tiisg == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n[[host_name(\"kernel_mul_mv_mxfp4_f32\")]]\nkernel void kernel_mul_mv_mxfp4_f32(\n constant ggml_metal_kargs_mul_mv & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n kernel_mul_mv_mxfp4_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntemplate\nkernel void kernel_get_rows_q(\n constant ggml_metal_kargs_get_rows & args,\n device const void * src0,\n device const void * src1,\n device void * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 ntg [[threads_per_threadgroup]]) {\n const int32_t iw0 = tgpig.x/args.ne10;\n const int32_t i10 = tgpig.x%args.ne10;\n const int32_t i11 = tgpig.y;\n const int32_t i12 = tgpig.z;\n\n const int32_t r = ((const device int32_t *) ((const device char *) src1 + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0];\n\n const int32_t i02 = i11;\n const int32_t i03 = i12;\n\n auto psrc = (device const block_q *) ((const device char *) src0 + i03*args.nb03 + i02*args.nb02 + r*args.nb01);\n auto pdst = (device float4x4 *) (( device char *) dst + i12*args.nb3 + i11*args.nb2 + i10*args.nb1);\n\n for (int ind = iw0*ntg.x + tiitg; ind < args.ne00t;) {\n float4x4 temp;\n dequantize_func(psrc + ind/nl, ind%nl, temp);\n pdst[ind] = temp;\n\n break;\n }\n}\n\ntemplate\nkernel void kernel_get_rows_f(\n constant ggml_metal_kargs_get_rows & args,\n device const void * src0,\n device const void * src1,\n device void * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort3 ntg [[threads_per_threadgroup]]) {\n const int32_t iw0 = tgpig.x/args.ne10;\n const int32_t i10 = tgpig.x%args.ne10;\n const int32_t i11 = tgpig.y;\n const int32_t i12 = tgpig.z;\n\n const int32_t r = ((const device int32_t *) ((const device char *) src1 + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0];\n\n const int32_t i02 = i11;\n const int32_t i03 = i12;\n\n auto psrc = (const device T0 *) ((const device char *) src0 + i03*args.nb03 + i02*args.nb02 + r*args.nb01);\n auto pdst = ( device T *) (( device char *) dst + i12*args.nb3 + i11*args.nb2 + i10*args.nb1);\n\n for (int ind = iw0*ntg.x + tiitg; ind < args.ne00t;) {\n pdst[ind] = psrc[ind];\n\n break;\n }\n}\n\ntemplate\nkernel void kernel_set_rows_q32(\n constant ggml_metal_kargs_set_rows & args,\n device const void * src0,\n device const void * src1,\n device float * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint tiitg[[thread_index_in_threadgroup]],\n uint3 tptg [[threads_per_threadgroup]]) {\n const int32_t i03 = tgpig.z;\n const int32_t i02 = tgpig.y;\n\n const int32_t i12 = i03%args.ne12;\n const int32_t i11 = i02%args.ne11;\n\n const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x;\n if (i01 >= args.ne01) {\n return;\n }\n\n const int32_t i10 = i01;\n const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0];\n\n device block_q * dst_row = ( device block_q *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3);\n const device float * src_row = (const device float *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);\n\n for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) {\n quantize_func(src_row + 32*ind, dst_row[ind]);\n }\n}\n\ntemplate\nkernel void kernel_set_rows_f(\n constant ggml_metal_kargs_set_rows & args,\n device const void * src0,\n device const void * src1,\n device float * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n uint tiitg[[thread_index_in_threadgroup]],\n uint3 tptg [[threads_per_threadgroup]]) {\n const int32_t i03 = tgpig.z;\n const int32_t i02 = tgpig.y;\n\n const int32_t i12 = i03%args.ne12;\n const int32_t i11 = i02%args.ne11;\n\n const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x;\n if (i01 >= args.ne01) {\n return;\n }\n\n const int32_t i10 = i01;\n const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0];\n\n device T * dst_row = ( device T *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3);\n const device float * src_row = (const device float *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);\n\n for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) {\n dst_row[ind] = (T) src_row[ind];\n }\n}\n\nkernel void kernel_diag_f32(\n constant ggml_metal_kargs_diag & args,\n device const char * src0,\n device char * dst,\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]]) {\n constexpr short NW = N_SIMDWIDTH;\n\n const int32_t i3 = tgpig.z;\n const int32_t i2 = tgpig.y;\n const int32_t i1 = tgpig.x;\n\n device const float * src0_ptr = (device const float *)(src0 + i2*args.nb02 + i3*args.nb03);\n device float * dst_ptr = (device float *)(dst + i1*args.nb01 + i2*args.nb2 + i3*args.nb3);\n\n for (int i0 = tiitg; i0 < args.ne0; i0 += NW) {\n dst_ptr[i0] = i0 == i1 ? src0_ptr[i0] : 0.0f;\n }\n}\n\nconstant bool FC_mul_mm_bc_inp [[function_constant(FC_MUL_MM + 0)]];\nconstant bool FC_mul_mm_bc_out [[function_constant(FC_MUL_MM + 1)]];\n\n// each block_q contains 16*nl weights\ntemplate\nkernel void kernel_mul_mm(\n constant ggml_metal_kargs_mul_mm & args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n\n threadgroup S0 * sa = (threadgroup S0 *)(shmem);\n threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096);\n\n#ifdef GGML_METAL_HAS_TENSOR\n threadgroup float * sc = (threadgroup float *)(shmem);\n#endif\n\n constexpr int NR0 = 64;\n constexpr int NR1 = 32;\n\n constexpr int NK = 32;\n constexpr int NL0 = NK/16;\n constexpr int NL1 = NK/8;\n\n const int im = tgpig.z;\n const int r0 = tgpig.y*NR0;\n const int r1 = tgpig.x*NR1;\n\n // if this block is of 64x32 shape or smaller\n const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0;\n const short nr1 = (args.ne1 - r1 < NR1) ? (args.ne1 - r1) : NR1;\n\n // a thread shouldn't load data outside of the matrix\n const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63\n const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31\n\n const short il0 = (tiitg % NL0);\n\n short il = il0;\n\n const int i12 = im%args.ne12;\n const int i13 = im/args.ne12;\n\n const uint64_t offset0 = (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;\n const short offset1 = il0/nl;\n\n device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;\n\n const short iy = 8*(tiitg % NL1);\n\n device const T1 * y = (device const T1 *)(src1\n + args.nb13*i13\n + args.nb12*i12\n + args.nb11*(r1 + lr1)\n + args.nb10*iy);\n\n#ifndef GGML_METAL_HAS_TENSOR\n S0_8x8 ma[4];\n S1_8x8 mb[2];\n\n simdgroup_float8x8 mc[8];\n\n for (short i = 0; i < 8; i++){\n mc[i] = make_filled_simdgroup_matrix(0.f);\n }\n#else\n auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0));\n auto tB = tensor, tensor_inline>(sb, dextents(NR1, NK ));\n\n mpp::tensor_ops::matmul2d<\n mpp::tensor_ops::matmul2d_descriptor(NR1, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),\n execution_simdgroups<4>> mm;\n\n auto cT = mm.get_destination_cooperative_tensor();\n#endif\n\n for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) {\n#ifndef GGML_METAL_HAS_TENSOR\n // load data and store to threadgroup memory\n if (is_same::value && FC_mul_mm_bc_inp) {\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n // no need for dequantization\n for (short i = 0; i < 16; i++) {\n const short sx = 2*il0 + i/8;\n const short sy = (tiitg/NL0)/8;\n\n //const short lx = i%8;\n //const short ly = (tiitg/NL0)%8;\n const short lx = (tiitg/NL0)%8;\n const short ly = i%8;\n\n const short ib = 8*sx + sy;\n\n *(sa + 64*ib + 8*ly + lx) = loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0;\n }\n } else {\n S0_4x4 temp_a;\n dequantize_func(x, il, temp_a);\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n FOR_UNROLL (short i = 0; i < 16; i++) {\n const short sx = 2*il0 + i/8;\n const short sy = (tiitg/NL0)/8;\n\n //const short lx = i%8;\n //const short ly = (tiitg/NL0)%8;\n const short lx = (tiitg/NL0)%8;\n const short ly = i%8;\n\n const short ib = 8*sx + sy;\n\n // NOTE: this is massively slower.. WTF?\n //sa[64*ib + 8*ly + lx] = temp_a[i/4][i%4];\n\n *(sa + 64*ib + 8*ly + lx) = temp_a[i/4][i%4];\n }\n }\n\n if (FC_mul_mm_bc_inp) {\n for (short i = 0; i < 8; ++i) {\n const short sx = (tiitg%NL1);\n const short sy = (tiitg/NL1)/8;\n\n const short lx = i;\n const short ly = (tiitg/NL1)%8;\n //const short lx = (tiitg/NL1)%8;\n //const short ly = i;\n\n const short ib = 4*sx + sy;\n\n *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0;\n }\n } else {\n const short sx = (tiitg%NL1);\n const short sy = (tiitg/NL1)/8;\n\n //const short dx = sx;\n //const short dy = sy;\n\n const short ly = (tiitg/NL1)%8;\n\n const short ib = 4*sx + sy;\n\n *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y));\n }\n#else\n // load data and store to threadgroup memory\n if (is_same::value && FC_mul_mm_bc_inp) {\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n // no need for dequantization\n for (short i = 0; i < 16; i++) {\n const short sx = 2*il0 + i/8;\n const short sy = (tiitg/NL0)/8;\n\n const short lx = i%8;\n const short ly = (tiitg/NL0)%8;\n //const short lx = (tiitg/NL0)%8;\n //const short ly = i%8;\n\n *(sa + NK*(8*sy + ly) + 8*sx + lx) = loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0;\n }\n } else {\n S0_4x4 temp_a;\n dequantize_func(x, il, temp_a);\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n FOR_UNROLL (short i = 0; i < 16; i++) {\n const short sx = 2*il0 + i/8;\n const short sy = (tiitg/NL0)/8;\n\n const short lx = i%8;\n const short ly = (tiitg/NL0)%8;\n //const short lx = (tiitg/NL0)%8;\n //const short ly = i%8;\n\n *(sa + NK*(8*sy + ly) + 8*sx + lx) = temp_a[i/4][i%4];\n }\n }\n\n if (FC_mul_mm_bc_inp) {\n for (short i = 0; i < 8; ++i) {\n const short sx = (tiitg%NL1);\n const short sy = (tiitg/NL1)/8;\n\n const short lx = i;\n const short ly = (tiitg/NL1)%8;\n //const short lx = (tiitg/NL1)%8;\n //const short ly = i;\n\n *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0;\n }\n } else {\n const short sx = (tiitg%NL1);\n const short sy = (tiitg/NL1)/8;\n\n //const short lx = i;\n const short ly = (tiitg/NL1)%8;\n //const short lx = (tiitg/NL1)%8;\n //const short ly = i;\n\n *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)(*((device T1_2x4 *) y));\n }\n#endif\n\n il = (il + 2 < nl) ? il + 2 : il % 2;\n x = (il < 2) ? x + (2 + nl - 1)/nl : x;\n\n y += NK;\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n#ifndef GGML_METAL_HAS_TENSOR\n // load matrices from threadgroup memory and conduct outer products\n threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2));\n threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2));\n\n FOR_UNROLL (short ik = 0; ik < NK/8; ik++) {\n simdgroup_barrier(mem_flags::mem_none);\n\n FOR_UNROLL (short i = 0; i < 4; i++) {\n simdgroup_load(ma[i], lsma + 64*i, 8, 0, false);\n }\n\n simdgroup_barrier(mem_flags::mem_none);\n\n FOR_UNROLL (short i = 0; i < 2; i++) {\n simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false);\n }\n\n simdgroup_barrier(mem_flags::mem_none);\n\n FOR_UNROLL (short i = 0; i < 8; i++){\n simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]);\n }\n\n lsma += 8*64;\n lsmb += 4*64;\n }\n#else\n auto sA = tA.slice(0, 0);\n auto sB = tB.slice(0, 0);\n\n mm.run(sB, sA, cT);\n#endif\n }\n\n if (!FC_mul_mm_bc_out || (r0 + NR0 <= args.ne0 && r1 + NR1 <= args.ne1)) {\n // if no bounds checks on the output are needed, we can directly write to device memory\n#ifdef GGML_METAL_HAS_TENSOR\n device float * C = (device float *) dst +\n r0 + \\\n r1 * args.ne0 + im*args.ne1*args.ne0;\n\n auto tC = tensor, tensor_inline>(C, dextents(args.ne0, NR1));\n cT.store(tC);\n#else\n device float * C = (device float *) dst +\n (r0 + 32*(sgitg & 1)) + \\\n (r1 + 16*(sgitg >> 1)) * args.ne0 + im*args.ne1*args.ne0;\n\n for (short i = 0; i < 8; i++) {\n simdgroup_store(mc[i], C + 8*(i%4) + 8*args.ne0*(i/4), args.ne0, 0, false);\n }\n#endif\n } else {\n // block is smaller than 64x32, we should avoid writing data outside of the matrix\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;\n\n#ifdef GGML_METAL_HAS_TENSOR\n auto tC = tensor, tensor_inline>(sc, dextents(NR0, NR1));\n cT.store(tC);\n#else\n for (short i = 0; i < 8; i++) {\n simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);\n }\n#endif\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (sgitg == 0) {\n for (int j = tiitg; j < nr1; j += NR1) {\n device float * D = (device float *) dst + r0 + (r1 + j)*args.ne0 + im*args.ne1*args.ne0;\n device float4 * D4 = (device float4 *) D;\n\n threadgroup float * C = temp_str + (j*NR0);\n threadgroup float4 * C4 = (threadgroup float4 *) C;\n\n int i = 0;\n for (; i < nr0/4; i++) {\n *(D4 + i) = *(C4 + i);\n }\n\n i *= 4;\n for (; i < nr0; i++) {\n *(D + i) = *(C + i);\n }\n }\n }\n }\n}\n\ntemplate // n_expert_used\nkernel void kernel_mul_mm_id_map0(\n constant ggml_metal_kargs_mul_mm_id_map0 & args,\n device const char * src2,\n device char * htpe,\n device char * hids,\n threadgroup char * shmem [[threadgroup(0)]],\n ushort tpitg[[thread_position_in_threadgroup]],\n ushort ntg[[threads_per_threadgroup]]) {\n const short ide = tpitg; // expert id\n\n uint32_t n_all = 0;\n\n device int32_t * ids_i32 = (device int32_t *) hids + ide*args.ne21;\n\n for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens\n if (i21 + tpitg < args.ne21) {\n device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21);\n\n threadgroup uint16_t * sids = (threadgroup uint16_t *) shmem + tpitg*ne20;\n\n #pragma unroll(ne20)\n for (short i20 = 0; i20 < ne20; i20++) {\n sids[i20] = src2_i32[i20];\n }\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n for (short t = 0; t < ntg; t++) {\n if (i21 + t >= args.ne21) {\n break;\n }\n\n threadgroup const uint16_t * sids = (threadgroup const uint16_t *) shmem + t*ne20;\n\n short sel = 0;\n #pragma unroll(ne20)\n for (short i20 = 0; i20 < ne20; i20++) {\n sel += (sids[i20] == ide)*(i20 + 1);\n }\n\n ids_i32[n_all] = (i21 + t)*ne20 + sel - 1;\n\n n_all += sel > 0;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n }\n\n device uint32_t * tpe_u32 = (device uint32_t *) (htpe);\n tpe_u32[ide] = n_all;\n}\n\ntypedef decltype(kernel_mul_mm_id_map0<1>) kernel_mul_mm_id_map0_t;\n\ntemplate [[host_name(\"kernel_mul_mm_id_map0_ne20_1\" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<1>;\ntemplate [[host_name(\"kernel_mul_mm_id_map0_ne20_2\" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<2>;\ntemplate [[host_name(\"kernel_mul_mm_id_map0_ne20_4\" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<4>;\ntemplate [[host_name(\"kernel_mul_mm_id_map0_ne20_5\" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<5>;\ntemplate [[host_name(\"kernel_mul_mm_id_map0_ne20_6\" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<6>;\ntemplate [[host_name(\"kernel_mul_mm_id_map0_ne20_8\" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<8>;\ntemplate [[host_name(\"kernel_mul_mm_id_map0_ne20_10\")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<10>;\ntemplate [[host_name(\"kernel_mul_mm_id_map0_ne20_16\")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<16>;\ntemplate [[host_name(\"kernel_mul_mm_id_map0_ne20_22\")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<22>;\n\ntemplate\nkernel void kernel_mul_mm_id(\n constant ggml_metal_kargs_mul_mm_id & args,\n device const char * src0,\n device const char * src1,\n device const char * htpe,\n device const char * hids,\n device char * dst,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n threadgroup S0 * sa = (threadgroup S0 *)(shmem);\n threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096);\n\n#ifdef GGML_METAL_HAS_TENSOR\n threadgroup float * sc = (threadgroup float *)(shmem);\n#endif\n\n constexpr int NR0 = 64;\n constexpr int NR1 = 32;\n\n constexpr int NK = 32;\n constexpr int NL0 = NK/16;\n constexpr int NL1 = NK/8;\n\n const int im = tgpig.z; // expert\n const int r0 = tgpig.y*NR0;\n const int r1 = tgpig.x*NR1;\n\n device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe);\n device const int32_t * ids_i32 = (device const int32_t *) (hids);\n\n const int32_t neh1 = tpe_u32[im];\n\n if (r1 >= neh1) {\n return;\n }\n\n // if this block is of 64x32 shape or smaller\n const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0;\n const short nr1 = ( neh1 - r1 < NR1) ? ( neh1 - r1) : NR1;\n\n // a thread shouldn't load data outside of the matrix\n const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63\n const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31\n\n const short il0 = (tiitg % NL0);\n\n short il = il0;\n\n const int id = ids_i32[im*args.ne21 + r1 + lr1];\n\n const short i11 = (id % args.ne20) % args.ne11;\n const short i12 = (id / args.ne20);\n const short i13 = 0;\n\n const uint64_t offset0 = im*args.nb02 + i13*args.nb03;\n const short offset1 = il0/nl;\n\n device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;\n\n const short iy = 8*(tiitg % NL1);\n\n device const T1 * y = (device const T1 *)(src1\n + args.nb13*i13\n + args.nb12*i12\n + args.nb11*i11\n + args.nb10*iy);\n\n#ifndef GGML_METAL_HAS_TENSOR\n S0_8x8 ma[4];\n S1_8x8 mb[2];\n\n simdgroup_float8x8 mc[8];\n\n for (short i = 0; i < 8; i++){\n mc[i] = make_filled_simdgroup_matrix(0.f);\n }\n#else\n auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0));\n auto tB = tensor, tensor_inline>(sb, dextents(NR1, NK ));\n\n mpp::tensor_ops::matmul2d<\n mpp::tensor_ops::matmul2d_descriptor(NR1, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),\n execution_simdgroups<4>> mm;\n\n auto cT = mm.get_destination_cooperative_tensor();\n#endif\n\n for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) {\n#ifndef GGML_METAL_HAS_TENSOR\n // load data and store to threadgroup memory\n if (is_same::value && FC_mul_mm_bc_inp) {\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n // no need for dequantization\n for (short i = 0; i < 16; i++) {\n const short sx = 2*il0 + i/8;\n const short sy = (tiitg/NL0)/8;\n\n //const short lx = i%8;\n //const short ly = (tiitg/NL0)%8;\n const short lx = (tiitg/NL0)%8;\n const short ly = i%8;\n\n const short ib = 8*sx + sy;\n\n *(sa + 64*ib + 8*ly + lx) = loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0;\n }\n } else {\n S0_4x4 temp_a;\n dequantize_func(x, il, temp_a);\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n FOR_UNROLL (short i = 0; i < 16; i++) {\n const short sx = 2*il0 + i/8;\n const short sy = (tiitg/NL0)/8;\n\n //const short lx = i%8;\n //const short ly = (tiitg/NL0)%8;\n const short lx = (tiitg/NL0)%8;\n const short ly = i%8;\n\n const short ib = 8*sx + sy;\n\n // NOTE: this is massively slower.. WTF?\n //sa[64*ib + 8*ly + lx] = temp_a[i/4][i%4];\n\n *(sa + 64*ib + 8*ly + lx) = temp_a[i/4][i%4];\n }\n }\n\n if (FC_mul_mm_bc_inp) {\n for (short i = 0; i < 8; ++i) {\n const short sx = (tiitg%NL1);\n const short sy = (tiitg/NL1)/8;\n\n const short lx = i;\n const short ly = (tiitg/NL1)%8;\n //const short lx = (tiitg/NL1)%8;\n //const short ly = i;\n\n const short ib = 4*sx + sy;\n\n *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0;\n }\n } else {\n const short sx = (tiitg%NL1);\n const short sy = (tiitg/NL1)/8;\n\n //const short dx = sx;\n //const short dy = sy;\n\n const short ly = (tiitg/NL1)%8;\n\n const short ib = 4*sx + sy;\n\n *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y));\n }\n#else\n // load data and store to threadgroup memory\n if (is_same::value && FC_mul_mm_bc_inp) {\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n // no need for dequantization\n for (short i = 0; i < 16; i++) {\n const short sx = 2*il0 + i/8;\n const short sy = (tiitg/NL0)/8;\n\n const short lx = i%8;\n const short ly = (tiitg/NL0)%8;\n //const short lx = (tiitg/NL0)%8;\n //const short ly = i%8;\n\n *(sa + NK*(8*sy + ly) + 8*sx + lx) = loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0;\n }\n } else {\n S0_4x4 temp_a;\n dequantize_func(x, il, temp_a);\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n FOR_UNROLL (short i = 0; i < 16; i++) {\n const short sx = 2*il0 + i/8;\n const short sy = (tiitg/NL0)/8;\n\n const short lx = i%8;\n const short ly = (tiitg/NL0)%8;\n //const short lx = (tiitg/NL0)%8;\n //const short ly = i%8;\n\n *(sa + NK*(8*sy + ly) + 8*sx + lx) = temp_a[i/4][i%4];\n }\n }\n\n if (FC_mul_mm_bc_inp) {\n for (short i = 0; i < 8; ++i) {\n const short sx = (tiitg%NL1);\n const short sy = (tiitg/NL1)/8;\n\n const short lx = i;\n const short ly = (tiitg/NL1)%8;\n //const short lx = (tiitg/NL1)%8;\n //const short ly = i;\n\n *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0;\n }\n } else {\n const short sx = (tiitg%NL1);\n const short sy = (tiitg/NL1)/8;\n\n //const short lx = i;\n const short ly = (tiitg/NL1)%8;\n //const short lx = (tiitg/NL1)%8;\n //const short ly = i;\n\n *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)(*((device T1_2x4 *) y));\n }\n#endif\n\n il = (il + 2 < nl) ? il + 2 : il % 2;\n x = (il < 2) ? x + (2 + nl - 1)/nl : x;\n\n y += NK;\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n#ifndef GGML_METAL_HAS_TENSOR\n // load matrices from threadgroup memory and conduct outer products\n threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2));\n threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2));\n\n FOR_UNROLL (short ik = 0; ik < NK/8; ik++) {\n simdgroup_barrier(mem_flags::mem_none);\n\n FOR_UNROLL (short i = 0; i < 4; i++) {\n simdgroup_load(ma[i], lsma + 64*i, 8, 0, false);\n }\n\n simdgroup_barrier(mem_flags::mem_none);\n\n FOR_UNROLL (short i = 0; i < 2; i++) {\n simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false);\n }\n\n simdgroup_barrier(mem_flags::mem_none);\n\n FOR_UNROLL (short i = 0; i < 8; i++){\n simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]);\n }\n\n lsma += 8*64;\n lsmb += 4*64;\n }\n#else\n auto sA = tA.slice(0, 0);\n auto sB = tB.slice(0, 0);\n\n mm.run(sB, sA, cT);\n#endif\n }\n\n // block is smaller than 64x32, we should avoid writing data outside of the matrix\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n#ifdef GGML_METAL_HAS_TENSOR\n auto tC = tensor, tensor_inline>(sc, dextents(NR0, NR1));\n cT.store(tC);\n#else\n threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;\n\n for (short i = 0; i < 8; i++) {\n simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);\n }\n#endif\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n for (short j = sgitg; j < nr1; j += 4) {\n const int id = ids_i32[im*args.ne21 + r1 + j];\n\n const short ide = id % args.ne20;\n const short idt = id / args.ne20;\n\n device float * D = (device float *) dst + r0 + ide*args.ne0 + idt*args.ne1*args.ne0;\n device float4 * D4 = (device float4 *) D;\n\n threadgroup float * C = (threadgroup float *) shmem + j*NR0;\n threadgroup float4 * C4 = (threadgroup float4 *) C;\n\n int i = tiisg;\n for (; i < nr0/4; i += 32) {\n *(D4 + i) = *(C4 + i);\n }\n\n i = (4*(nr0/4)) + tiisg;\n for (; i < nr0; i += 32) {\n *(D + i) = *(C + i);\n }\n }\n}\n\n#define QK_NL 16\n\n//\n// get rows\n//\n\ntypedef decltype(kernel_get_rows_f) get_rows_f_t;\n\ntemplate [[host_name(\"kernel_get_rows_f32\")]] kernel get_rows_f_t kernel_get_rows_f;\ntemplate [[host_name(\"kernel_get_rows_f16\")]] kernel get_rows_f_t kernel_get_rows_f;\ntemplate [[host_name(\"kernel_get_rows_i32\")]] kernel get_rows_f_t kernel_get_rows_f;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_get_rows_bf16\")]] kernel get_rows_f_t kernel_get_rows_f;\n#endif\n\ntypedef decltype(kernel_get_rows_q) get_rows_q_t;\n\ntemplate [[host_name(\"kernel_get_rows_q4_0\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_q4_1\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_q5_0\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_q5_1\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_q8_0\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_mxfp4\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_q2_K\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_q3_K\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_q4_K\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_q5_K\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_q6_K\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_iq2_xxs\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_iq2_xs\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_iq3_xxs\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_iq3_s\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_iq2_s\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_iq1_s\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_iq1_m\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_iq4_nl\")]] kernel get_rows_q_t kernel_get_rows_q;\ntemplate [[host_name(\"kernel_get_rows_iq4_xs\")]] kernel get_rows_q_t kernel_get_rows_q;\n\n//\n// set rows\n//\n\ntypedef decltype(kernel_set_rows_f) set_rows_f_t;\n\ntemplate [[host_name(\"kernel_set_rows_f32_i64\")]] kernel set_rows_f_t kernel_set_rows_f;\ntemplate [[host_name(\"kernel_set_rows_f32_i32\")]] kernel set_rows_f_t kernel_set_rows_f;\ntemplate [[host_name(\"kernel_set_rows_f16_i64\")]] kernel set_rows_f_t kernel_set_rows_f;\ntemplate [[host_name(\"kernel_set_rows_f16_i32\")]] kernel set_rows_f_t kernel_set_rows_f;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_set_rows_bf16_i64\")]] kernel set_rows_f_t kernel_set_rows_f;\ntemplate [[host_name(\"kernel_set_rows_bf16_i32\")]] kernel set_rows_f_t kernel_set_rows_f;\n#endif\n\ntypedef decltype(kernel_set_rows_q32) set_rows_q32_t;\n\ntemplate [[host_name(\"kernel_set_rows_q8_0_i64\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_q8_0_i32\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_q4_0_i64\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_q4_0_i32\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_q4_1_i64\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_q4_1_i32\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_q5_0_i64\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_q5_0_i32\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_q5_1_i64\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_q5_1_i32\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_iq4_nl_i64\")]] kernel set_rows_q32_t kernel_set_rows_q32;\ntemplate [[host_name(\"kernel_set_rows_iq4_nl_i32\")]] kernel set_rows_q32_t kernel_set_rows_q32;\n\n//\n// matrix-matrix multiplication\n//\n\ntypedef decltype(kernel_mul_mm) mul_mm_t;\n\ntemplate [[host_name(\"kernel_mul_mm_f32_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_f16_f32\")]] kernel mul_mm_t kernel_mul_mm;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_mul_mm_bf16_f32\")]] kernel mul_mm_t kernel_mul_mm;\n#endif\ntemplate [[host_name(\"kernel_mul_mm_q4_0_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q4_1_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q5_0_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q5_1_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q8_0_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_mxfp4_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q2_K_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q3_K_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q4_K_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q5_K_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q6_K_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq2_xxs_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq2_xs_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq3_xxs_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq3_s_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq2_s_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq1_s_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq1_m_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq4_nl_f32\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq4_xs_f32\")]] kernel mul_mm_t kernel_mul_mm;\n\ntemplate [[host_name(\"kernel_mul_mm_f32_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_f16_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q4_0_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q4_1_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q5_0_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q5_1_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q8_0_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_mxfp4_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q2_K_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q3_K_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q4_K_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q5_K_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_q6_K_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq2_xxs_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq2_xs_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq3_xxs_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq3_s_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq2_s_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq1_s_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq1_m_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq4_nl_f16\")]] kernel mul_mm_t kernel_mul_mm;\ntemplate [[host_name(\"kernel_mul_mm_iq4_xs_f16\")]] kernel mul_mm_t kernel_mul_mm;\n\n//\n// indirect matrix-matrix multiplication\n//\n\ntypedef decltype(kernel_mul_mm_id) mul_mm_id;\n\ntemplate [[host_name(\"kernel_mul_mm_id_f32_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_f16_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_mul_mm_id_bf16_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\n#endif\ntemplate [[host_name(\"kernel_mul_mm_id_q4_0_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q4_1_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q5_0_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q5_1_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q8_0_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_mxfp4_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q2_K_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q3_K_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q4_K_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q5_K_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q6_K_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq2_xxs_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq2_xs_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq3_xxs_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq3_s_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq2_s_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq1_s_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq1_m_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq4_nl_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq4_xs_f32\")]] kernel mul_mm_id kernel_mul_mm_id;\n\ntemplate [[host_name(\"kernel_mul_mm_id_f32_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_f16_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q4_0_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q4_1_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q5_0_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q5_1_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q8_0_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_mxfp4_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q2_K_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q3_K_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q4_K_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q5_K_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_q6_K_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq2_xxs_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq2_xs_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq3_xxs_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq3_s_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq2_s_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq1_s_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq1_m_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq4_nl_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\ntemplate [[host_name(\"kernel_mul_mm_id_iq4_xs_f16\")]] kernel mul_mm_id kernel_mul_mm_id;\n\n//\n// matrix-vector multiplication\n//\n\ntypedef void (kernel_mul_mv_disp_t)(\n ggml_metal_kargs_mul_mv args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n uint3 tgpig,\n ushort tiisg);\n\ntypedef void (kernel_mul_mv2_disp_t)(\n ggml_metal_kargs_mul_mv args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiisg,\n ushort sgitg);\n\ntemplate\nvoid mmv_fn(\n ggml_metal_kargs_mul_mv args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiitg,\n ushort tiisg,\n ushort sgitg) {\n disp_fn(args, src0, src1, dst, tgpig, tiisg);\n}\n\ntemplate\nvoid mmv_fn(\n ggml_metal_kargs_mul_mv args,\n device const char * src0,\n device const char * src1,\n device char * dst,\n threadgroup char * shmem,\n uint3 tgpig,\n ushort tiitg,\n ushort tiisg,\n ushort sgitg) {\n disp_fn(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);\n}\n\ntypedef decltype(mmv_fn>) mul_mv_disp_fn_t;\n\ntemplate\nkernel void kernel_mul_mv_id(\n constant ggml_metal_kargs_mul_mv_id & args,\n device const char * src0s,\n device const char * src1,\n device char * dst,\n device const char * ids,\n threadgroup char * shmem [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort tiitg[[thread_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]]) {\n const int iid1 = tgpig.z/args.nei0;\n const int idx = tgpig.z%args.nei0;\n\n tgpig.z = 0;\n\n const int32_t i02 = ((device const int32_t *) (ids + iid1*args.nbi1))[idx];\n\n const int64_t i11 = idx % args.ne11;\n const int64_t i12 = iid1;\n\n const int64_t i1 = idx;\n const int64_t i2 = i12;\n\n device const char * src0_cur = src0s + i02*args.nb02;\n device const char * src1_cur = src1 + i11*args.nb11 + i12*args.nb12;\n\n device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float);\n\n ggml_metal_kargs_mul_mv args0 = {\n /*.ne00 =*/ args.ne00,\n /*.ne01 =*/ args.ne01,\n /*.ne02 =*/ 1, // args.ne02,\n /*.nb00 =*/ args.nb00,\n /*.nb01 =*/ args.nb01,\n /*.nb02 =*/ args.nb02,\n /*.nb03 =*/ args.nb02, // args.ne02 == 1\n /*.ne10 =*/ args.ne10,\n /*.ne11 =*/ 1, // args.ne11,\n /*.ne12 =*/ 1, // args.ne12,\n /*.nb10 =*/ args.nb10,\n /*.nb11 =*/ args.nb11,\n /*.nb12 =*/ args.nb12,\n /*.nb13 =*/ args.nb12, // ne12 == 1\n /*.ne0 =*/ args.ne0,\n /*.ne1 =*/ 1, // args.ne1,\n /*.nr0 =*/ args.nr0,\n /*.r2 =*/ 1,\n /*.r3 =*/ 1,\n };\n\n disp_fn(\n args0,\n /* src0 */ src0_cur,\n /* src1 */ src1_cur,\n /* dst */ dst_cur,\n shmem,\n tgpig,\n tiitg,\n tiisg,\n sgitg);\n}\n\ntypedef decltype(kernel_mul_mv_id>>) kernel_mul_mv_id_t;\n\ntypedef decltype(kernel_mul_mv_id>>) kernel_mul_mv_id_4_t;\n\ntemplate [[host_name(\"kernel_mul_mv_id_f32_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_f16_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_mul_mv_id_bf16_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\n#endif\ntemplate [[host_name(\"kernel_mul_mv_id_f32_f32_4\")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_f16_f32_4\")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id>>;\n#if defined(GGML_METAL_HAS_BF16)\ntemplate [[host_name(\"kernel_mul_mv_id_bf16_f32_4\")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id>>;\n#endif\n\ntemplate [[host_name(\"kernel_mul_mv_id_q8_0_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\n\ntemplate [[host_name(\"kernel_mul_mv_id_q4_0_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_q4_1_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_q5_0_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_q5_1_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\n\ntemplate [[host_name(\"kernel_mul_mv_id_mxfp4_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\n\ntemplate [[host_name(\"kernel_mul_mv_id_q2_K_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_q3_K_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_q4_K_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_q5_K_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_q6_K_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_iq1_s_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_iq1_m_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_iq2_xxs_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_iq2_xs_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_iq3_xxs_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_iq3_s_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_iq2_s_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_iq4_nl_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\ntemplate [[host_name(\"kernel_mul_mv_id_iq4_xs_f32\")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>;\n\nkernel void kernel_pool_2d_max_f32(\n constant ggml_metal_kargs_pool_2d & args,\n device const float * src0,\n device float * dst,\n uint gid[[thread_position_in_grid]]) {\n\n if (gid >= args.np) {\n return;\n }\n\n const int idx = gid;\n const int I_HW = args.IH * args.IW;\n const int O_HW = args.OH * args.OW;\n const int nc = idx / O_HW;\n const int cur_oh = idx % O_HW / args.OW;\n const int cur_ow = idx % O_HW % args.OW;\n\n device const float * i_ptr = src0 + nc * I_HW;\n device float * o_ptr = dst + nc * O_HW;\n\n const int start_h = cur_oh * args.s1 - args.p1;\n const int bh = MAX(0, start_h);\n const int eh = MIN(args.IH, start_h + args.k1);\n const int start_w = cur_ow * args.s0 - args.p0;\n const int bw = MAX(0, start_w);\n const int ew = MIN(args.IW, start_w + args.k0);\n\n float res = -INFINITY;\n\n for (int i = bh; i < eh; i += 1) {\n for (int j = bw; j < ew; j += 1) {\n res = MAX(res, i_ptr[i * args.IW + j]);\n }\n }\n\n o_ptr[cur_oh * args.OW + cur_ow] = res;\n}\n\nkernel void kernel_pool_2d_avg_f32(\n constant ggml_metal_kargs_pool_2d & args,\n device const float * src0,\n device float * dst,\n uint gid[[thread_position_in_grid]]) {\n\n if (gid >= args.np) {\n return;\n }\n\n const int idx = gid;\n const int I_HW = args.IH * args.IW;\n const int O_HW = args.OH * args.OW;\n const int nc = idx / O_HW;\n const int cur_oh = idx % O_HW / args.OW;\n const int cur_ow = idx % O_HW % args.OW;\n\n device const float * i_ptr = src0 + nc * I_HW;\n device float * o_ptr = dst + nc * O_HW;\n\n const int start_h = cur_oh * args.s1 - args.p1;\n const int bh = MAX(0, start_h);\n const int eh = MIN(args.IH, start_h + args.k1);\n const int start_w = cur_ow * args.s0 - args.p0;\n const int bw = MAX(0, start_w);\n const int ew = MIN(args.IW, start_w + args.k0);\n // const float scale = 1. / ((eh - bh) * (ew - bw));\n const float scale = 1. / (args.k0 * args.k1);\n\n float res = 0;\n\n for (int i = bh; i < eh; i += 1) {\n for (int j = bw; j < ew; j += 1) {\n float cur = i_ptr[i * args.IW + j];\n res += cur * scale;\n }\n }\n\n o_ptr[cur_oh * args.OW + cur_ow] = res;\n}\n\n\nkernel void kernel_pool_1d_max_f32(\n constant ggml_metal_kargs_pool_1d & args,\n device const float * src,\n device float * dst,\n uint gid [[thread_position_in_grid]]\n) {\n\n if (gid >= args.np) {\n return;\n }\n\n const int ow = (int)gid % args.OW;\n const int row = (int)gid / args.OW;\n\n const int base = ow * args.s0 - args.p0;\n\n float acc = -INFINITY;\n\n const int src_off = row * args.IW;\n const int dst_off = row * args.OW;\n\n for (int ki = 0; ki < args.k0; ++ki) {\n int j = base + ki;\n if (j < 0 || j >= args.IW){\n continue;\n }\n float v = src[src_off + j];\n acc = max(acc, v);\n }\n\n dst[dst_off + ow] = acc;\n}\n\nkernel void kernel_pool_1d_avg_f32(\n constant ggml_metal_kargs_pool_1d & args,\n device const float * src,\n device float * dst,\n uint gid [[thread_position_in_grid]]\n) {\n\n if (gid >= args.np) {\n return;\n }\n\n const int ow = (int)gid % args.OW;\n const int row = (int)gid / args.OW;\n\n const int base = ow * args.s0 - args.p0;\n\n float acc = 0.0f;\n int cnt = 0;\n\n const int src_off = row * args.IW;\n const int dst_off = row * args.OW;\n\n for (int ki = 0; ki < args.k0; ++ki) {\n const int j = base + ki;\n if (j < 0 || j >= args.IW) {\n continue;\n }\n acc += src[src_off + j];\n cnt += 1;\n }\n\n dst[dst_off + ow] = (cnt > 0) ? (acc / (float)cnt) : 0.0f;\n}\n\nkernel void kernel_opt_step_adamw_f32(\n constant ggml_metal_kargs_opt_step_adamw & args,\n device float * x,\n device const float * g,\n device float * g_m,\n device float * g_v,\n device const float * pars,\n uint gid[[thread_position_in_grid]]) {\n\n if (gid >= args.np) {\n return;\n }\n\n const float alpha = pars[0];\n const float beta1 = pars[1];\n const float beta2 = pars[2];\n const float eps = pars[3];\n const float wd = pars[4];\n const float beta1h = pars[5];\n const float beta2h = pars[6];\n\n const float gi = g[gid];\n const float gmi = g_m[gid] * beta1 + gi * (1.0f - beta1);\n const float gvi = g_v[gid] * beta2 + gi * gi * (1.0f - beta2);\n\n g_m[gid] = gmi;\n g_v[gid] = gvi;\n\n const float mh = gmi * beta1h;\n const float vh = sqrt(gvi * beta2h) + eps;\n\n x[gid] = x[gid] * (1.0f - alpha * wd) - alpha * mh / vh;\n}\n\nkernel void kernel_opt_step_sgd_f32(\n constant ggml_metal_kargs_opt_step_sgd & args,\n device float * x,\n device const float * g,\n device const float * pars,\n uint gid[[thread_position_in_grid]]) {\n\n if (gid >= args.np) {\n return;\n }\n\n x[gid] = x[gid] * (1.0f - pars[0] * pars[1]) - pars[0] * g[gid];\n}\n\ntemplate\nkernel void kernel_memset(\n constant ggml_metal_kargs_memset & args,\n device T * dst,\n uint tpig[[thread_position_in_grid]]) {\n dst[tpig] = args.val;\n}\n\ntypedef decltype(kernel_memset) kernel_memset_t;\n\ntemplate [[host_name(\"kernel_memset_i64\")]] kernel kernel_memset_t kernel_memset;\n\nconstant short FC_count_equal_nsg [[function_constant(FC_COUNT_EQUAL + 0)]];\n\ntemplate\nkernel void kernel_count_equal(\n constant ggml_metal_kargs_count_equal & args,\n device const char * src0,\n device const char * src1,\n device atomic_int * dst,\n threadgroup int32_t * shmem_i32 [[threadgroup(0)]],\n uint3 tgpig[[threadgroup_position_in_grid]],\n ushort3 tpitg[[thread_position_in_threadgroup]],\n ushort sgitg[[simdgroup_index_in_threadgroup]],\n ushort tiisg[[thread_index_in_simdgroup]],\n ushort3 ntg[[threads_per_threadgroup]]) {\n const short NSG = FC_count_equal_nsg;\n\n const int i3 = tgpig.z;\n const int i2 = tgpig.y;\n const int i1 = tgpig.x;\n\n if (i3 >= args.ne03 || i2 >= args.ne02 || i1 >= args.ne01) {\n return;\n }\n\n int sum = 0;\n\n device const char * base0 = src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03;\n device const char * base1 = src1 + i1*args.nb11 + i2*args.nb12 + i3*args.nb13;\n\n for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) {\n const T v0 = *(device const T *)(base0 + i0*args.nb00);\n const T v1 = *(device const T *)(base1 + i0*args.nb10);\n sum += (v0 == v1);\n }\n\n sum = simd_sum(sum);\n\n if (tiisg == 0) {\n shmem_i32[sgitg] = sum;\n }\n\n threadgroup_barrier(mem_flags::mem_threadgroup);\n\n if (sgitg == 0) {\n float v = 0.0f;\n if (tpitg.x < NSG) {\n v = shmem_i32[tpitg.x];\n }\n\n float total = simd_sum(v);\n if (tpitg.x == 0) {\n atomic_fetch_add_explicit(dst, (int32_t) total, memory_order_relaxed);\n }\n }\n}\n\ntypedef decltype(kernel_count_equal) kernel_count_equal_t;\n\ntemplate [[host_name(\"kernel_count_equal_i32\")]] kernel kernel_count_equal_t kernel_count_equal;\n" }, { "path": "src/ggml-musa/CMakeLists.txt", "content": "if (NOT EXISTS $ENV{MUSA_PATH})\n if (NOT EXISTS /opt/musa)\n set(MUSA_PATH /usr/local/musa)\n else()\n set(MUSA_PATH /opt/musa)\n endif()\nelse()\n set(MUSA_PATH $ENV{MUSA_PATH})\nendif()\n\nset(CMAKE_C_COMPILER \"${MUSA_PATH}/bin/clang\")\nset(CMAKE_C_EXTENSIONS OFF)\nset(CMAKE_CXX_COMPILER \"${MUSA_PATH}/bin/clang++\")\nset(CMAKE_CXX_EXTENSIONS OFF)\n\nlist(APPEND CMAKE_MODULE_PATH \"${MUSA_PATH}/cmake\")\n\nfind_package(MUSAToolkit)\n\nif (MUSAToolkit_FOUND)\n message(STATUS \"MUSA Toolkit found\")\n\n if (NOT DEFINED MUSA_ARCHITECTURES)\n set(MUSA_ARCHITECTURES \"21;22;31\")\n endif()\n message(STATUS \"Using MUSA architectures: ${MUSA_ARCHITECTURES}\")\n\n file(GLOB GGML_HEADERS_MUSA \"../ggml-cuda/*.cuh\")\n list(APPEND GGML_HEADERS_MUSA \"../../include/ggml-cuda.h\")\n list(APPEND GGML_HEADERS_MUSA \"../ggml-musa/mudnn.cuh\")\n\n file(GLOB GGML_SOURCES_MUSA \"../ggml-cuda/*.cu\")\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-tile*.cu\")\n list(APPEND GGML_SOURCES_MUSA ${SRCS})\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-mma*.cu\")\n list(APPEND GGML_SOURCES_MUSA ${SRCS})\n file(GLOB SRCS \"../ggml-cuda/template-instances/mmq*.cu\")\n list(APPEND GGML_SOURCES_MUSA ${SRCS})\n\n if (GGML_MUSA_MUDNN_COPY)\n file(GLOB SRCS \"../ggml-musa/*.cu\")\n list(APPEND GGML_SOURCES_MUSA ${SRCS})\n add_compile_definitions(GGML_MUSA_MUDNN_COPY)\n endif()\n\n if (GGML_CUDA_FA_ALL_QUANTS)\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-vec*.cu\")\n list(APPEND GGML_SOURCES_MUSA ${SRCS})\n add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS)\n else()\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-vec*q4_0-q4_0.cu\")\n list(APPEND GGML_SOURCES_MUSA ${SRCS})\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-vec*q8_0-q8_0.cu\")\n list(APPEND GGML_SOURCES_MUSA ${SRCS})\n file(GLOB SRCS \"../ggml-cuda/template-instances/fattn-vec*f16-f16.cu\")\n list(APPEND GGML_SOURCES_MUSA ${SRCS})\n endif()\n\n set_source_files_properties(${GGML_SOURCES_MUSA} PROPERTIES LANGUAGE CXX)\n foreach(SOURCE ${GGML_SOURCES_MUSA})\n set(COMPILE_FLAGS \"-Od3 -fno-strict-aliasing -ffast-math -fsigned-char -x musa -mtgpu -fmusa-flush-denormals-to-zero\")\n foreach(ARCH ${MUSA_ARCHITECTURES})\n set(COMPILE_FLAGS \"${COMPILE_FLAGS} --cuda-gpu-arch=mp_${ARCH}\")\n endforeach()\n set_property(SOURCE ${SOURCE} PROPERTY COMPILE_FLAGS ${COMPILE_FLAGS})\n endforeach()\n\n ggml_add_backend_library(ggml-musa\n ${GGML_HEADERS_MUSA}\n ${GGML_SOURCES_MUSA}\n )\n\n # TODO: do not use CUDA definitions for MUSA\n if (NOT GGML_BACKEND_DL)\n target_compile_definitions(ggml PUBLIC GGML_USE_CUDA)\n endif()\n\n add_compile_definitions(GGML_USE_MUSA)\n add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})\n\n if (GGML_MUSA_GRAPHS)\n add_compile_definitions(GGML_MUSA_GRAPHS)\n endif()\n\n if (GGML_CUDA_FORCE_MMQ)\n add_compile_definitions(GGML_CUDA_FORCE_MMQ)\n endif()\n\n if (GGML_CUDA_FORCE_CUBLAS)\n add_compile_definitions(GGML_CUDA_FORCE_CUBLAS)\n endif()\n\n if (GGML_CUDA_NO_VMM)\n add_compile_definitions(GGML_CUDA_NO_VMM)\n endif()\n\n if (NOT GGML_CUDA_FA)\n add_compile_definitions(GGML_CUDA_NO_FA)\n endif()\n\n if (GGML_CUDA_NO_PEER_COPY)\n add_compile_definitions(GGML_CUDA_NO_PEER_COPY)\n endif()\n\n if (GGML_STATIC)\n target_link_libraries(ggml-musa PRIVATE MUSA::musart_static MUSA::mublas_static)\n # TODO: mudnn has not provided static libraries yet\n # if (GGML_MUSA_MUDNN_COPY)\n # target_link_libraries(ggml-musa PRIVATE mudnn_static)\n # endif()\n else()\n target_link_libraries(ggml-musa PRIVATE MUSA::musart MUSA::mublas)\n if (GGML_MUSA_MUDNN_COPY)\n target_link_libraries(ggml-musa PRIVATE mudnn)\n endif()\n endif()\n\n if (GGML_CUDA_NO_VMM)\n # No VMM requested, no need to link directly with the musa driver lib (libmusa.so)\n else()\n target_link_libraries(ggml-musa PRIVATE MUSA::musa_driver)\n endif()\nelse()\n message(FATAL_ERROR \"MUSA Toolkit not found\")\nendif()\n" }, { "path": "src/ggml-musa/mudnn.cu", "content": "#include \n#include \n\n#include \"mudnn.cuh\"\n\nnamespace mudnn = musa::dnn;\n\n// Returns a human-readable error string for mudnn::Status\nconst char* mudnnGetErrorString(mudnn::Status err) {\n switch (err) {\n case mudnn::Status::SUCCESS:\n return \"Success\";\n case mudnn::Status::INVALID_PARAMETER:\n return \"Invalid parameter\";\n case mudnn::Status::NOT_INITIALIZED:\n return \"Not initialized\";\n case mudnn::Status::ALLOC_FAILED:\n return \"Allocation failed\";\n case mudnn::Status::NOT_SUPPORTED:\n return \"Not supported\";\n case mudnn::Status::INTERNAL_ERROR:\n return \"Internal error\";\n case mudnn::Status::ARCH_MISMATCH:\n return \"Architecture mismatch\";\n case mudnn::Status::EXECUTION_FAILED:\n return \"Execution failed\";\n default:\n return \"Unknown mudnn status\";\n }\n}\n\n// Error checking macro for MUDNN calls\n#define MUDNN_CHECK(err) CUDA_CHECK_GEN(err, mudnn::Status::SUCCESS, mudnnGetErrorString)\n\nnamespace {\n // Thread-safe cache for mudnn::Handle objects per device\n std::unordered_map> handle_cache;\n std::mutex handle_cache_mutex;\n\n mudnn::Handle* get_cached_handle(int device_id) {\n std::lock_guard lock(handle_cache_mutex);\n auto it = handle_cache.find(device_id);\n if (it != handle_cache.end()) {\n return it->second.get();\n }\n auto handle = std::make_unique(device_id);\n mudnn::Handle* handle_ptr = handle.get();\n handle_cache[device_id] = std::move(handle);\n return handle_ptr;\n }\n}\n\n// Extracts dimensions and strides from a ggml_tensor\nint get_ggml_dims_and_strides(const ggml_tensor* tensor,\n std::vector& dims,\n std::vector& strides) {\n const int ndims = ggml_n_dims(tensor);\n const size_t element_size = ggml_element_size(tensor);\n\n dims.resize(ndims);\n strides.resize(ndims);\n\n for (int i = 0; i < ndims; ++i) {\n dims[i] = tensor->ne[i];\n strides[i] = tensor->nb[i] / static_cast(element_size);\n }\n return ndims;\n}\n\n// Converts ggml_type to mudnn::Tensor::Type\nmudnn::Tensor::Type ggml_type_to_mudnn_type(ggml_type type) {\n switch (type) {\n case GGML_TYPE_F32:\n return mudnn::Tensor::Type::FLOAT;\n case GGML_TYPE_F16:\n return mudnn::Tensor::Type::HALF;\n\n // TODO: Add support for other types\n\n default:\n MUDNN_CHECK(mudnn::Status::NOT_SUPPORTED);\n }\n\n return mudnn::Tensor::Type::FLOAT; // Default fallback\n}\n\n// Asynchronous memory copy using mudnn::Unary::IDENTITY\nmusaError_t mudnnMemcpyAsync(ggml_backend_cuda_context& ctx, const ggml_tensor* dst, const ggml_tensor* src) {\n mudnn::Tensor tensor_dst, tensor_src;\n\n MUDNN_CHECK(tensor_dst.SetType(ggml_type_to_mudnn_type(dst->type)));\n MUDNN_CHECK(tensor_src.SetType(ggml_type_to_mudnn_type(src->type)));\n\n std::vector dims, strides;\n const int ndims = get_ggml_dims_and_strides(src, dims, strides);\n\n MUDNN_CHECK(tensor_dst.SetNdInfo(ndims, dims.data(), strides.data()));\n MUDNN_CHECK(tensor_src.SetNdInfo(ndims, dims.data(), strides.data()));\n MUDNN_CHECK(tensor_dst.SetAddr(dst->data));\n MUDNN_CHECK(tensor_src.SetAddr(src->data));\n\n mudnn::Unary op;\n MUDNN_CHECK(op.SetMode(mudnn::Unary::Mode::IDENTITY));\n MUDNN_CHECK(op.SetAlpha(0.0f));\n MUDNN_CHECK(op.SetBeta(0.0f));\n\n mudnn::Handle* handle = get_cached_handle(ctx.device);\n MUDNN_CHECK(handle->SetStream(ctx.stream()));\n MUDNN_CHECK(op.Run(*handle, tensor_dst, tensor_src));\n\n return musaSuccess;\n}\n" }, { "path": "src/ggml-musa/mudnn.cuh", "content": "#pragma once\n\n#include \"ggml-cuda/common.cuh\"\n#include \"ggml.h\"\n\n// Asynchronously copies data from src tensor to dst tensor using the provided context.\n// Returns a musaError_t indicating success or failure.\nmusaError_t mudnnMemcpyAsync(\n ggml_backend_cuda_context &ctx,\n const ggml_tensor *dst,\n const ggml_tensor *src\n);\n" }, { "path": "src/ggml-opencl/CMakeLists.txt", "content": "find_package(OpenCL REQUIRED)\nfind_package(Python3 REQUIRED)\n\nset(TARGET_NAME ggml-opencl)\n\nggml_add_backend_library(${TARGET_NAME}\n ggml-opencl.cpp\n ../../include/ggml-opencl.h)\ntarget_link_libraries(${TARGET_NAME} PRIVATE ${OpenCL_LIBRARIES})\ntarget_include_directories(${TARGET_NAME} PRIVATE ${OpenCL_INCLUDE_DIRS})\n\nif (GGML_OPENCL_PROFILING)\n message(STATUS \"OpenCL profiling enabled (increases CPU overhead)\")\n add_compile_definitions(GGML_OPENCL_PROFILING)\nendif ()\n\nadd_compile_definitions(GGML_OPENCL_SOA_Q)\nadd_compile_definitions(GGML_OPENCL_TARGET_VERSION=${GGML_OPENCL_TARGET_VERSION})\n\nif (GGML_OPENCL_USE_ADRENO_KERNELS)\n message(STATUS \"OpenCL will use matmul kernels optimized for Adreno\")\n add_compile_definitions(GGML_OPENCL_USE_ADRENO_KERNELS)\nendif ()\n\nif (GGML_OPENCL_EMBED_KERNELS)\n add_compile_definitions(GGML_OPENCL_EMBED_KERNELS)\n\n set(EMBED_KERNEL_SCRIPT \"${CMAKE_CURRENT_SOURCE_DIR}/kernels/embed_kernel.py\")\n file(MAKE_DIRECTORY \"${CMAKE_CURRENT_BINARY_DIR}/autogenerated\")\n\n target_include_directories(${TARGET_NAME} PRIVATE \"${CMAKE_CURRENT_BINARY_DIR}/autogenerated\")\nendif ()\n\nfunction(ggml_opencl_add_kernel KNAME)\n set(KERN_HDR ${CMAKE_CURRENT_BINARY_DIR}/autogenerated/${KNAME}.cl.h)\n set(KERN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/kernels/${KNAME}.cl)\n\n if (GGML_OPENCL_EMBED_KERNELS)\n message(STATUS \"opencl: embedding kernel ${KNAME}\")\n\n # Python must be accessible from command line\n add_custom_command(\n OUTPUT ${KERN_HDR}\n COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} ${KERN_SRC} ${KERN_HDR}\n DEPENDS ${KERN_SRC} ${EMBED_KERNEL_SCRIPT}\n COMMENT \"Generate ${KERN_HDR}\"\n )\n\n target_sources(${TARGET_NAME} PRIVATE ${KERN_HDR})\n else ()\n message(STATUS \"opencl: adding kernel ${KNAME}\")\n configure_file(${KERN_SRC} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${KNAME}.cl COPYONLY)\n endif ()\nendfunction()\n\nset(GGML_OPENCL_KERNELS\n add\n add_id\n argsort\n tri\n fill\n clamp\n cpy\n cvt\n diag_mask_inf\n diag\n div\n gelu\n gemv_noshuffle_general\n gemv_noshuffle\n get_rows\n glu\n group_norm\n solve_tri\n im2col_f32\n im2col_f16\n mean\n mul_mat_Ab_Bi_8x4\n mul_mv_f16_f16\n mul_mv_f16_f32_1row\n mul_mv_f16_f32_l4\n mul_mv_f16_f32\n mul_mv_f32_f32\n mul_mv_q4_0_f32\n mul_mv_q4_0_f32_v\n mul_mv_q4_0_f32_8x_flat\n mul_mv_q4_0_f32_1d_8x_flat\n mul_mv_q4_0_f32_1d_16x_flat\n mul_mv_q4_1_f32\n mul_mv_q4_1_f32_flat\n mul_mv_q4_k_f32\n mul_mv_q6_k_f32\n mul_mv_q6_k_f32_flat\n mul_mv_q8_0_f32\n mul_mv_q8_0_f32_flat\n mul_mv_mxfp4_f32\n mul_mv_mxfp4_f32_flat\n mul_mv_id_q4_0_f32_8x_flat\n mul_mv_id_q8_0_f32\n mul_mv_id_q8_0_f32_flat\n mul_mv_id_mxfp4_f32\n mul_mv_id_mxfp4_f32_flat\n gemm_moe_mxfp4_f32\n gemv_moe_mxfp4_f32\n mul_mm_f32_f32_l4_lm\n mul_mm_f16_f32_l4_lm\n mul_mm_q4_0_f32_l4_lm\n mul_mm_q4_1_f32_l4_lm\n mul_mm_q8_0_f32_l4_lm\n mul_mm_q6_k_f32_l4_lm\n mul_mm_q8_0_f32_8x4\n gemv_noshuffle_q4_1_f32\n gemm_noshuffle_q4_1_f32\n gemv_noshuffle_general_q8_0_f32\n mul\n neg\n norm\n relu\n l2_norm\n rms_norm\n rope\n scale\n set_rows\n sigmoid\n silu\n softmax_4_f32\n softmax_4_f16\n softmax_f32\n softmax_f16\n sqr\n sqrt\n ssm_conv\n sub\n sum_rows\n cumsum\n transpose\n concat\n tsembd\n upscale\n tanh\n exp\n expm1\n softplus\n pad\n repeat\n mul_mat_f16_f32\n mul_mm_f16_f32_kq_kqv\n conv2d\n conv2d_f16_f32\n flash_attn_f32_f16\n flash_attn_f16\n flash_attn_f32\n)\n\nforeach (K ${GGML_OPENCL_KERNELS})\n ggml_opencl_add_kernel(${K})\nendforeach()\n" }, { "path": "src/ggml-opencl/ggml-opencl.cpp", "content": "#define CL_TARGET_OPENCL_VERSION GGML_OPENCL_TARGET_VERSION\n#define CL_USE_DEPRECATED_OPENCL_1_2_APIS\n\n// suppress warnings in CL headers for GCC and Clang\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#ifdef __clang__\n#pragma GCC diagnostic ignored \"-Wgnu-anonymous-struct\"\n#endif\n\n#include \"ggml-opencl.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml.h\"\n\n#include \n\n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#undef MIN\n#undef MAX\n#define MIN(a, b) ((a) < (b) ? (a) : (b))\n#define MAX(a, b) ((a) > (b) ? (a) : (b))\n#define CEIL_DIV(M, N) (((M) + (N)-1) / (N))\n\n#define UNUSED(x) (void)(x)\n\n#define CL_CHECK(err) \\\n do { \\\n cl_int err_ = (err); \\\n if (err_ != CL_SUCCESS) { \\\n GGML_LOG_ERROR(\"ggml_opencl: %s error %d at %s:%d\\n\", \\\n #err, err_, __FILE__, __LINE__); \\\n GGML_ASSERT(0); \\\n } \\\n } while (0)\n\n//------------------------------------------------------------------------------\n// OpenCL\n//------------------------------------------------------------------------------\n\nbool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor);\n\n// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.\n// Precompute mp (m' in the paper) and L such that division\n// can be computed using a multiply (high 32b of 64b result)\n// and a shift:\n//\n// n/d = (mulhi(n, mp) + n) >> L;\nstruct fastdiv_vals {\n uint32_t mp;\n uint32_t L;\n uint32_t d;\n uint32_t pad;\n};\nstatic_assert(sizeof(fastdiv_vals) == 16, \"fastdiv_vals size incorrect\");\n\nstatic fastdiv_vals init_fastdiv_values(uint64_t d_64) {\n GGML_ASSERT(d_64 != 0);\n GGML_ASSERT(d_64 <= std::numeric_limits::max());\n\n uint32_t d = (uint32_t)d_64;\n\n // compute L = ceil(log2(d));\n uint32_t L = 0;\n while (L < 32 && (uint32_t{ 1 } << L) < d) {\n L++;\n }\n\n uint32_t mp = (uint32_t) ((uint64_t{ 1 } << 32) * ((uint64_t{ 1 } << L) - d) / d + 1);\n // pack divisor as well to reduce error surface\n return { mp, L, d, 0 };\n}\n\nenum GPU_FAMILY {\n ADRENO,\n INTEL,\n UNKNOWN,\n};\n\nenum ADRENO_GPU_GEN {\n ADRENO_UNKNOWN,\n A7X,\n A8X,\n X1E,\n};\n\nenum ADRENO_CL_COMPILER_TYPE {\n E031,\n DX,\n};\n\nstruct ggml_cl_version {\n cl_uint major = 0;\n cl_uint minor = 0;\n};\n\n\nstruct ggml_cl_compiler_version {\n ADRENO_CL_COMPILER_TYPE type;\n int major = -1;\n int minor = -1;\n int patch = -1;\n\n bool same(ADRENO_CL_COMPILER_TYPE t, int x, int y, int z) const {\n return major == x && minor == y && patch == z && type == t;\n }\n bool newer_than(ADRENO_CL_COMPILER_TYPE t, int x, int y, int z) const {\n return major*10000 + minor*100 + patch > x*10000 + y*100 + z && type == t;\n }\n bool newer_than_or_same(ADRENO_CL_COMPILER_TYPE t, int x, int y, int z) const {\n return same(t, x, y, z) || newer_than(t, x, y, z);\n }\n};\n\nstatic size_t align_to(size_t value, size_t to_alignment) {\n GGML_ASSERT(to_alignment && \"Invalid alignment (must be non-zero)\");\n GGML_ASSERT((to_alignment & (to_alignment - 1)) == 0 && \"to_alignment must be power-of-two\");\n\n return ((value + to_alignment - 1) / to_alignment) * to_alignment;\n}\n\n\n// Parses a version string of form \"XX.YY \". On an error returns ggml_cl_version with all zeroes.\nstatic ggml_cl_version parse_cl_version(std::string_view str) {\n size_t major_str_begin = 0;\n size_t major_str_end = str.find(\".\", major_str_begin);\n if (major_str_end == std::string::npos) {\n return {};\n }\n\n size_t minor_str_begin = major_str_end + 1;\n size_t minor_str_end = str.find(\" \", minor_str_begin);\n if (minor_str_end == std::string::npos) {\n return {};\n }\n\n cl_uint version_major;\n if (std::from_chars(str.data() + major_str_begin, str.data() + major_str_end, version_major).ec != std::errc{}) {\n return {};\n }\n\n cl_uint version_minor;\n if (std::from_chars(str.data() + minor_str_begin, str.data() + minor_str_end, version_minor).ec != std::errc{}) {\n return {};\n }\n return { version_major, version_minor };\n}\n\n// Returns OpenCL platform's version. On an error returns ggml_cl_version with all zeroes.\nstatic ggml_cl_version get_opencl_platform_version(cl_platform_id platform) {\n size_t param_size;\n CL_CHECK(clGetPlatformInfo(platform, CL_PLATFORM_VERSION, 0, nullptr, ¶m_size));\n std::unique_ptr param_storage(new char[param_size]);\n CL_CHECK(clGetPlatformInfo(platform, CL_PLATFORM_VERSION, param_size, param_storage.get(), nullptr));\n\n auto param_value = std::string_view(param_storage.get(), param_size);\n const std::string version_prefix = \"OpenCL \"; // Suffix: \"XX.YY \"\n if (param_value.find(version_prefix) != 0) {\n return {};\n }\n param_value.remove_prefix(version_prefix.length());\n return parse_cl_version(param_value);\n}\n\n// Return a version to use in OpenCL C compilation. On an error returns ggml_cl_version with all zeroes.\nstatic ggml_cl_version get_opencl_c_version(ggml_cl_version platform_version, cl_device_id device) {\n size_t param_size;\n\n#if CL_TARGET_OPENCL_VERSION >= 300\n if (platform_version.major >= 3) {\n CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size));\n if (!param_size) {\n return {};\n }\n\n std::unique_ptr versions(new cl_name_version[param_size]);\n CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr));\n unsigned versions_count = param_size / sizeof(cl_name_version);\n\n cl_version version_max = 0;\n for (unsigned i = 0; i < versions_count; i++) {\n version_max = std::max(versions[i].version, version_max);\n }\n\n return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) };\n }\n#else\n GGML_UNUSED(platform_version);\n#endif // CL_TARGET_OPENCL_VERSION >= 300\n\n CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size));\n if (!param_size) {\n return {};\n }\n\n std::unique_ptr param_storage(new char[param_size]);\n CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr));\n auto param_value = std::string_view(param_storage.get(), param_size);\n\n const std::string version_prefix = \"OpenCL C \"; // Suffix: \"XX.YY \"\n if (param_value.find(version_prefix) != 0) {\n return {};\n }\n param_value.remove_prefix(version_prefix.length());\n\n return parse_cl_version(param_value);\n}\n\nstatic ADRENO_GPU_GEN get_adreno_gpu_gen(const char *device_name) {\n if (strstr(device_name, \"730\") ||\n strstr(device_name, \"740\") ||\n strstr(device_name, \"750\")) {\n return ADRENO_GPU_GEN::A7X;\n }\n\n if (strstr(device_name, \"830\") ||\n strstr(device_name, \"840\")) {\n return ADRENO_GPU_GEN::A8X;\n }\n\n if (strstr(device_name, \"X1\")) {\n return ADRENO_GPU_GEN::X1E;\n }\n\n return ADRENO_GPU_GEN::ADRENO_UNKNOWN;\n}\n\nstatic ggml_cl_compiler_version get_adreno_cl_compiler_version(const char *driver_version) {\n std::string driver_ver_str(driver_version);\n ADRENO_CL_COMPILER_TYPE type = ADRENO_CL_COMPILER_TYPE::E031;\n size_t compiler_ver_pos = driver_ver_str.find(\"E031\");\n size_t compiler_ver_len = 13;\n size_t compiler_major_offset = 5;\n size_t compiler_minor_offset = 8;\n size_t compiler_patch_offset = 11;\n\n if (compiler_ver_pos == std::string::npos) {\n compiler_ver_pos = driver_ver_str.find(\"DX\");\n if (compiler_ver_pos == std::string::npos) {\n return {};\n }\n type = ADRENO_CL_COMPILER_TYPE::DX;\n compiler_ver_len = 11;\n compiler_major_offset = 3;\n }\n\n std::string compiler_ver_str = driver_ver_str.substr(compiler_ver_pos, compiler_ver_len);\n int major = std::atoi(compiler_ver_str.substr(compiler_major_offset, 2).c_str());\n int minor = std::atoi(compiler_ver_str.substr(compiler_minor_offset, 2).c_str());\n int patch = std::atoi(compiler_ver_str.substr(compiler_patch_offset, 2).c_str());\n return { type, major, minor, patch };\n}\n\n// cl buffer wrapper\nstruct ggml_cl_buffer {\n cl_mem buffer;\n size_t size;\n\n ggml_cl_buffer()\n : buffer(nullptr), size(0) {}\n\n ~ggml_cl_buffer() {\n if (buffer) {\n CL_CHECK(clReleaseMemObject(buffer));\n }\n }\n\n void allocate(cl_context context, size_t new_size) {\n if (new_size > size) {\n size = new_size;\n if (buffer) {\n CL_CHECK(clReleaseMemObject(buffer));\n }\n cl_int err;\n CL_CHECK((buffer = clCreateBuffer(context, CL_MEM_READ_WRITE, size, NULL, &err), err));\n }\n }\n};\n\n// Profiling\nstruct ProfilingInfo {\n std::string op_name;\n std::string kernel_name;\n\n cl_kernel kernel;\n cl_event evt;\n\n cl_ulong cmd_queued;\n cl_ulong cmd_submit;\n cl_ulong cmd_start;\n cl_ulong cmd_end;\n cl_ulong overhead_start;\n cl_ulong overhead_end;\n // For the times below, see spec for clGetEventProfilingInfo\n // The time kernel spent in cmd queue - SUBMIT - QUEUED\n cl_ulong cmd_queued_duration_ns;\n // The time kernel spent for submission - START - SUBMIT\n cl_ulong cmd_submit_duration_ns;\n // Kernel execution time in nanoseconds - END - START\n cl_ulong cmd_duration_ns;\n // The time for the kernel to complete - COMPLETE - END\n cl_ulong cmd_complete_duration_ns;\n // Total time to finish the kernel - COMPLETE - QUEUED\n cl_ulong cmd_total_duration_ns;\n // Global and local work sizes.\n size_t global_size[3];\n size_t local_size[3];\n // Op output size.\n size_t output_size[4];\n};\n\nstatic void populateProfilingInfo(\n ProfilingInfo& info, cl_event evt, cl_kernel kernel, cl_uint work_dim,\n size_t global_size[3], size_t local_size[3],\n const ggml_tensor * tensor) {\n info.op_name = tensor->name;\n info.kernel = kernel;\n info.evt = evt;\n\n // 0 means not specified, e.g., 2D workgroup, or NULL for driver to choose\n info.local_size[0] = 0;\n info.local_size[1] = 0;\n info.local_size[2] = 0;\n\n info.global_size[0] = 0;\n info.global_size[1] = 0;\n info.global_size[2] = 0;\n\n if (local_size) {\n for (cl_uint i = 0; i < work_dim; ++i) {\n info.local_size[i] = local_size[i];\n }\n }\n\n for (cl_uint i = 0; i < work_dim; ++i) {\n info.global_size[i] = global_size[i];\n }\n\n info.output_size[0] = tensor->ne[0];\n info.output_size[1] = tensor->ne[1];\n info.output_size[2] = tensor->ne[2];\n info.output_size[3] = tensor->ne[3];\n}\n\nstruct ggml_backend_opencl_context;\n\n// backend device context\nstruct ggml_backend_opencl_device_context {\n cl_platform_id platform;\n std::string platform_name;\n\n cl_device_id device;\n std::string device_name;\n cl_device_type device_type;\n std::string device_version;\n\n // Initialized by ggml_cl2_init().\n ggml_backend_opencl_context * backend_ctx = nullptr;\n\n // Initialized by ggml_backend_opencl_device_get_buffer_type()\n ggml_backend_buffer_type buffer_type;\n\n cl_context context = nullptr;\n};\n\n// backend context\nstruct ggml_backend_opencl_context {\n int ref_count;\n\n cl_device_id device;\n std::string device_name;\n\n std::string driver_version;\n\n GPU_FAMILY gpu_family;\n ADRENO_GPU_GEN adreno_gen;\n\n cl_int alignment;\n size_t max_alloc_size;\n size_t max_workgroup_size;\n bool fp16_support;\n bool has_vector_subgroup_broadcast;\n bool disable_fusion;\n ggml_cl_compiler_version adreno_cl_compiler_version;\n\n int adreno_wave_size;\n\n cl_bool non_uniform_workgroups;\n size_t image_max_buffer_size;\n\n cl_context context;\n cl_command_queue queue;\n\n // prealloc buffers for transposing weights and activations\n ggml_cl_buffer prealloc_quant_trans;\n ggml_cl_buffer prealloc_scales_trans;\n ggml_cl_buffer prealloc_act_trans;\n\n // prealloc buffers for src0 and src1\n ggml_cl_buffer prealloc_src0;\n ggml_cl_buffer prealloc_src1;\n\n cl_program program_add;\n cl_program program_add_id;\n cl_program program_clamp;\n cl_program program_cvt;\n cl_program program_diag_mask_inf;\n cl_program program_gelu;\n cl_program program_gemv_noshuffle_general;\n cl_program program_gemv_noshuffle;\n cl_program program_get_rows;\n cl_program program_set_rows;\n cl_program program_glu;\n cl_program program_im2col_f16;\n cl_program program_im2col_f32;\n cl_program program_mul_mat_Ab_Bi_8x4;\n cl_program program_mul_mv_q4_0_f32;\n cl_program program_mul_mv_q4_0_f32_v;\n cl_program program_mul_mv_q4_0_f32_8x_flat;\n cl_program program_mul_mv_q4_0_f32_1d_8x_flat;\n cl_program program_mul_mv_q4_0_f32_1d_16x_flat;\n cl_program program_mul_mv_q6_K;\n cl_program program_mul_mv_q8_0_f32, program_mul_mv_q8_0_f32_flat;\n cl_program program_mul_mv_mxfp4_f32;\n cl_program program_mul_mv_mxfp4_f32_flat;\n cl_program program_mul_mv_f16_f16;\n cl_program program_mul_mv_f16_f32_1row;\n cl_program program_mul_mv_f16_f32_l4;\n cl_program program_mul_mv_f16_f32;\n cl_program program_mul_mv_f32_f32;\n cl_program program_mul;\n cl_program program_mul_mat_f16_f32_tiled;\n cl_program program_mul_mm_f16_f32_kqv;\n cl_program program_mul_mm_f16_f32_kq;\n cl_program program_div;\n cl_program program_sub;\n cl_program program_norm;\n cl_program program_relu;\n cl_program program_rms_norm;\n cl_program program_group_norm;\n cl_program program_rope;\n cl_program program_silu;\n cl_program program_sigmoid;\n cl_program program_softmax_f32;\n cl_program program_softmax_f16;\n cl_program program_softmax_4_f32;\n cl_program program_softmax_4_f16;\n cl_program program_argsort_f32_i32;\n cl_program program_sum_rows_f32;\n cl_program program_pad;\n cl_program program_upscale;\n cl_program program_conv_2d_f16;\n cl_program program_conv_2d_f32;\n cl_program program_conv_2d_f16_f32;\n cl_program program_tsembd;\n cl_program program_gemv_moe_mxfp4_f32, program_gemm_moe_mxfp4_f32;\n cl_program program_mul_mv_id_q4_0_f32_8x_flat;\n cl_program program_mul_mv_id_q8_0_f32, program_mul_mv_id_q8_0_f32_flat;\n cl_program program_mul_mv_id_mxfp4_f32;\n cl_program program_mul_mv_id_mxfp4_f32_flat;\n cl_program program_mul_mm_f32_f32_l4_lm;\n cl_program program_mul_mm_f16_f32_l4_lm;\n cl_program program_mul_mm_q8_0_f32_l4_lm;\n\n cl_kernel kernel_add, kernel_add_row, kernel_add_f16, kernel_add_row_f16;\n cl_kernel kernel_mul, kernel_mul_row, kernel_mul_f16, kernel_mul_row_f16;\n cl_kernel kernel_div, kernel_div_row, kernel_div_f16, kernel_div_row_f16;\n cl_kernel kernel_sub, kernel_sub_row, kernel_sub_f16, kernel_sub_row_f16;\n cl_kernel kernel_add_id;\n cl_kernel kernel_scale_f32, kernel_scale_f32_4;\n cl_kernel kernel_sqr_cont_f32, kernel_sqr_cont_f32_4, kernel_sqr_cont_f16, kernel_sqr_cont_f16_4;\n cl_kernel kernel_sqrt_cont_f32, kernel_sqrt_cont_f32_4, kernel_sqrt_cont_f16, kernel_sqrt_cont_f16_4;\n cl_kernel kernel_mean_f32, kernel_mean_f32_4;\n cl_kernel kernel_silu, kernel_silu_4;\n cl_kernel kernel_gelu, kernel_gelu_4;\n cl_kernel kernel_gelu_erf, kernel_gelu_erf_4;\n cl_kernel kernel_gelu_quick, kernel_gelu_quick_4;\n cl_kernel kernel_relu;\n cl_kernel kernel_sigmoid_f32, kernel_sigmoid_f16;\n cl_kernel kernel_tri;\n cl_kernel kernel_fill;\n cl_kernel kernel_clamp;\n cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_geglu_erf, kernel_geglu_quick,\n kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_geglu_erf_f16, kernel_geglu_quick_f16;\n cl_kernel kernel_norm, kernel_norm_mul_add;\n cl_kernel kernel_rms_norm, kernel_rms_norm_mul;\n cl_kernel kernel_l2_norm_f32;\n cl_kernel kernel_group_norm, kernel_group_norm_mul_add;\n cl_kernel kernel_diag_mask_inf, kernel_diag_mask_inf_8;\n cl_kernel kernel_diag_f32;\n cl_kernel kernel_soft_max, kernel_soft_max_4;\n cl_kernel kernel_soft_max_f16, kernel_soft_max_4_f16;\n std::map, cl_kernel> kernels_flash_attn_f16;\n std::map, cl_kernel> kernels_flash_attn_f16_q1;\n std::map, cl_kernel> kernels_flash_attn_f32;\n std::map, cl_kernel> kernels_flash_attn_f32_q1;\n std::map, cl_kernel> kernels_flash_attn_f32_f16;\n std::map, cl_kernel> kernels_flash_attn_f32_f16_q1;\n std::map, int> kernels_flash_attn_bm;\n std::map, int> kernels_flash_attn_bn;\n cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0;\n cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32;\n cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16;\n cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16;\n cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32;\n cl_kernel kernel_mul_mat_f32_f32;\n cl_kernel kernel_mul_mat_f16_f16;\n cl_kernel kernel_mul_mat_f16_f32_1row;\n cl_kernel kernel_mul_mat_f16_f32;\n cl_kernel kernel_mul_mat_f16_f32_l4;\n cl_kernel kernel_mul_mat_f16_f32_tiled;\n cl_kernel kernel_mul_mm_f16_f32_kqv;\n cl_kernel kernel_mul_mm_f16_f32_kq;\n cl_kernel kernel_mul_mat_q4_0_f32, kernel_mul_mat_q4_0_f32_v;\n cl_kernel kernel_convert_block_q4_0, kernel_restore_block_q4_0;\n cl_kernel kernel_convert_block_q4_1, kernel_restore_block_q4_1;\n cl_kernel kernel_convert_block_mxfp4, kernel_convert_block_mxfp4_trans, kernel_restore_block_mxfp4, kernel_restore_block_mxfp4_trans;\n cl_kernel kernel_convert_block_q8_0, kernel_restore_block_q8_0, kernel_restore_block_q8_0_trans;\n cl_kernel kernel_mul_mat_q4_0_f32_8x_flat;\n cl_kernel kernel_convert_block_q4_0_noshuffle;\n cl_kernel kernel_restore_block_q4_0_noshuffle;\n cl_kernel kernel_convert_block_q4_1_noshuffle;\n cl_kernel kernel_restore_block_q4_1_noshuffle;\n cl_kernel kernel_convert_block_q6_K, kernel_restore_block_q6_K;\n cl_kernel kernel_mul_mat_q4_0_f32_1d_8x_flat, kernel_mul_mat_q4_0_f32_1d_16x_flat;\n cl_kernel kernel_mul_mv_q4_1_f32;\n cl_kernel kernel_mul_mv_q4_1_f32_flat;\n cl_kernel kernel_mul_mv_q4_K_f32;\n cl_kernel kernel_mul_mv_q6_K_f32;\n cl_kernel kernel_mul_mv_q6_K_f32_flat;\n cl_kernel kernel_mul_mv_mxfp4_f32, kernel_mul_mv_mxfp4_f32_flat;\n cl_kernel kernel_mul_mv_q8_0_f32, kernel_mul_mv_q8_0_f32_flat;\n cl_kernel kernel_solve_tri_f32;\n cl_kernel kernel_im2col_f32, kernel_im2col_f16;\n cl_kernel kernel_argsort_f32_i32;\n cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4;\n cl_kernel kernel_cumsum_blk, kernel_cumsum_add;\n cl_kernel kernel_repeat_f32;\n cl_kernel kernel_pad;\n cl_kernel kernel_tanh_f32, kernel_tanh_f32_4, kernel_tanh_f32_nc;\n cl_kernel kernel_tanh_f16, kernel_tanh_f16_4, kernel_tanh_f16_nc;\n cl_kernel kernel_neg_f32, kernel_neg_f32_4, kernel_neg_f32_nc;\n cl_kernel kernel_neg_f16, kernel_neg_f16_4, kernel_neg_f16_nc;\n cl_kernel kernel_exp_f32, kernel_exp_f32_4, kernel_exp_f32_nc;\n cl_kernel kernel_exp_f16, kernel_exp_f16_4, kernel_exp_f16_nc;\n cl_kernel kernel_expm1_f32, kernel_expm1_f32_4, kernel_expm1_f32_nc;\n cl_kernel kernel_expm1_f16, kernel_expm1_f16_4, kernel_expm1_f16_nc;\n cl_kernel kernel_softplus_f32, kernel_softplus_f32_4, kernel_softplus_f32_nc;\n cl_kernel kernel_softplus_f16, kernel_softplus_f16_4, kernel_softplus_f16_nc;\n cl_kernel kernel_upscale;\n cl_kernel kernel_upscale_bilinear;\n cl_kernel kernel_concat_f32;\n cl_kernel kernel_conv_2d_f16;\n cl_kernel kernel_conv_2d_f32;\n cl_kernel kernel_conv_2d_f16_f32;\n cl_kernel kernel_ssm_conv_f32_f32, kernel_ssm_conv_f32_f32_4;\n cl_kernel kernel_timestep_embedding;\n cl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32;\n cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat;\n cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat;\n cl_kernel kernel_mul_mv_id_mxfp4_f32;\n cl_kernel kernel_mul_mv_id_mxfp4_f32_flat;\n cl_kernel kernel_mul_mm_f32_f32_l4_lm;\n cl_kernel kernel_mul_mm_f16_f32_l4_lm;\n cl_kernel kernel_mul_mm_q4_0_f32_l4_lm;\n cl_kernel kernel_mul_mm_q4_1_f32_l4_lm;\n cl_kernel kernel_mul_mm_q8_0_f32_l4_lm;\n cl_kernel kernel_mul_mm_q6_k_f32_l4_lm;\n\n std::vector profiling_info;\n\n void write_profiling_info() {\n FILE * fperf = fopen(\"cl_profiling.csv\", \"w\");\n if (!fperf) {\n GGML_LOG_ERROR(\"Failed to open cl_profiling.csv\\n\");\n return;\n }\n\n // Populate profiling info\n for (ProfilingInfo & info : profiling_info) {\n cl_ulong cmd_queued;\n cl_ulong cmd_submit;\n cl_ulong cmd_start;\n cl_ulong cmd_end;\n cl_ulong cmd_complete;\n\n CL_CHECK(clWaitForEvents(1, &info.evt));\n CL_CHECK(clGetEventProfilingInfo(\n info.evt, CL_PROFILING_COMMAND_QUEUED, sizeof(cl_ulong), &cmd_queued, NULL));\n CL_CHECK(clGetEventProfilingInfo(\n info.evt, CL_PROFILING_COMMAND_SUBMIT, sizeof(cl_ulong), &cmd_submit, NULL));\n CL_CHECK(clGetEventProfilingInfo(\n info.evt, CL_PROFILING_COMMAND_START, sizeof(cl_ulong), &cmd_start, NULL));\n CL_CHECK(clGetEventProfilingInfo(\n info.evt, CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &cmd_end, NULL));\n CL_CHECK(clGetEventProfilingInfo(\n info.evt, CL_PROFILING_COMMAND_COMPLETE, sizeof(cl_ulong), &cmd_complete, NULL));\n CL_CHECK(clReleaseEvent(info.evt));\n\n char kernel_name[512];\n CL_CHECK(clGetKernelInfo(info.kernel, CL_KERNEL_FUNCTION_NAME,\n sizeof(kernel_name), kernel_name, NULL));\n info.kernel_name = kernel_name;\n\n info.cmd_queued = cmd_queued;\n info.cmd_submit = cmd_submit;\n info.cmd_start = cmd_start;\n info.cmd_end = cmd_end;\n\n info.cmd_queued_duration_ns = cmd_submit - cmd_queued;\n info.cmd_submit_duration_ns = cmd_start - cmd_submit;\n info.cmd_duration_ns = cmd_end - cmd_start;\n info.cmd_complete_duration_ns = cmd_complete - cmd_end;\n info.cmd_total_duration_ns = cmd_complete - cmd_queued;\n }\n\n // Dump a csv\n fprintf(fperf, \"op name, kernel name, exec duration (ms), global size, local size, output size\\n\");\n for (const ProfilingInfo & info : profiling_info) {\n fprintf(fperf, \"%s,%s,%f,%zux%zux%zu,%zux%zux%zu,%zux%zux%zux%zu\\n\",\n info.op_name.c_str(), info.kernel_name.c_str(),\n info.cmd_duration_ns/1.e6f,\n info.global_size[0], info.global_size[1], info.global_size[2],\n info.local_size[0], info.local_size[1], info.local_size[2],\n info.output_size[0], info.output_size[1], info.output_size[2], info.output_size[3]);\n }\n fclose(fperf);\n\n // Dump a simple chrome trace\n FILE* ftrace = fopen(\"cl_trace.json\", \"w\");\n if (!ftrace) {\n GGML_LOG_ERROR(\"Failed to open cl_trace.json\\n\");\n return;\n }\n\n fprintf(ftrace, \"[\\n\");\n for (const ProfilingInfo & info : profiling_info) {\n fprintf(ftrace, \"{\\\"name\\\": \\\"%s\\\", \\\"cat\\\": \\\"OpenCL\\\", \\\"ph\\\": \\\"B\\\", \\\"ts\\\": %\" PRIu64 \", \\\"pid\\\": \\\"\\\", \\\"tid\\\": \\\"Host\\\"},\\n\",\n info.kernel_name.c_str(), info.cmd_queued/1000);\n fprintf(ftrace, \"{\\\"name\\\": \\\"%s\\\", \\\"cat\\\": \\\"OpenCL\\\", \\\"ph\\\": \\\"E\\\", \\\"ts\\\": %\" PRIu64 \", \\\"pid\\\": \\\"\\\", \\\"tid\\\": \\\"Host\\\"},\\n\",\n info.kernel_name.c_str(), info.cmd_submit/1000);\n\n fprintf(ftrace, \"{\\\"name\\\": \\\"%s\\\", \\\"cat\\\": \\\"OpenCL\\\", \\\"ph\\\": \\\"B\\\", \\\"ts\\\": %\" PRIu64 \", \\\"pid\\\": \\\"\\\", \\\"tid\\\": \\\"Device\\\"},\\n\",\n info.kernel_name.c_str(), info.cmd_start/1000);\n fprintf(ftrace, \"{\\\"name\\\": \\\"%s\\\", \\\"cat\\\": \\\"OpenCL\\\", \\\"ph\\\": \\\"E\\\", \\\"ts\\\": %\" PRIu64 \", \\\"pid\\\": \\\"\\\", \\\"tid\\\": \\\"Device\\\"},\\n\",\n info.kernel_name.c_str(), info.cmd_end/1000);\n }\n fclose(ftrace);\n }\n\n size_t get_kernel_workgroup_size(cl_kernel kernel) const {\n size_t workgroup_size = 0;\n size_t ret_size = 0;\n CL_CHECK(\n clGetKernelWorkGroupInfo(kernel, device, CL_KERNEL_WORK_GROUP_SIZE,\n sizeof(size_t), &workgroup_size, &ret_size));\n GGML_ASSERT(sizeof(size_t) == ret_size);\n return workgroup_size;\n }\n\n void enqueue_ndrange_kernel(cl_kernel kernel, cl_uint work_dim, size_t *global_work_size, size_t *local_work_size, const ggml_tensor * tensor) {\n#ifdef GGML_OPENCL_PROFILING\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, work_dim, NULL, global_work_size, local_work_size, 0, NULL, &evt));\n\n profiling_info.emplace_back();\n populateProfilingInfo(profiling_info.back(), evt, kernel, work_dim, global_work_size, local_work_size, tensor);\n#else\n GGML_UNUSED(tensor);\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, work_dim, NULL, global_work_size, local_work_size, 0, NULL, NULL));\n#endif\n }\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n // Transpose kernels\n cl_program program_transpose;\n\n cl_kernel kernel_transpose_32;\n cl_kernel kernel_transpose_32_16;\n cl_kernel kernel_transpose_16;\n cl_kernel kernel_transpose_8_buf;\n cl_kernel kernel_transpose_16_buf;\n cl_kernel kernel_transpose_32_buf;\n cl_kernel kernel_transpose_16_4x1;\n\n // Gemm and Gemv related programs, kernels, etc\n cl_program program_CL_gemm;\n cl_program program_CL_gemv_general;\n cl_program program_CL_gemv_4096_1_11008;\n cl_program program_CL_gemv_4096_1_4096;\n cl_program program_CL_gemv_11008_1_4096;\n cl_program program_CL_gemv_32000_1_4096;\n cl_kernel CL_mul_mat_Ab_Bi_8x4;\n cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_general;\n cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_11008;\n cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_4096;\n cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_11008_1_4096;\n cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_32000_1_4096;\n cl_kernel kernel_gemv_noshuffle_q4_1_f32;\n cl_kernel kernel_gemm_noshuffle_q4_1_f32;\n cl_kernel kernel_mul_mm_q8_0_f32_8x4;\n cl_kernel CL_mul_mat_vec_q8_0_f32;\n#endif // GGML_OPENCL_USE_ADRENO_KERNELS\n\n void free() {\n ref_count--;\n if (ref_count == 0) {\n#ifdef GGML_OPENCL_PROFILING\n write_profiling_info();\n profiling_info.clear();\n#endif\n }\n }\n};\n\n// All registered devices with a default device in the front.\nstatic std::vector g_ggml_backend_opencl_devices;\n\ninline std::string read_file(const std::string &path) {\n std::ifstream ifs(path);\n if (!ifs) {\n return \"\";\n }\n std::string text;\n ifs.seekg(0, std::ios::end);\n text.resize(ifs.tellg());\n ifs.seekg(0, std::ios::beg);\n ifs.read(&text[0], text.size());\n return text;\n}\n\nstatic cl_program build_program_from_source(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts) {\n cl_program p;\n char *program_log;\n size_t program_size;\n size_t log_size;\n int err;\n\n program_size = strlen(program_buffer);\n\n p = clCreateProgramWithSource(ctx, 1, (const char**)&program_buffer, &program_size, &err);\n if(err < 0) {\n GGML_LOG_ERROR(\"OpenCL error creating program\");\n exit(1);\n }\n\n err = clBuildProgram(p, 0, NULL, compile_opts.c_str(), NULL, NULL);\n if(err < 0) {\n clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);\n program_log = (char*) malloc(log_size + 1);\n program_log[log_size] = '\\0';\n clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL);\n GGML_LOG_ERROR(\"ggml_opencl: kernel compile error:\\n\\n%s\\n\", program_log);\n free(program_log);\n exit(1);\n }\n\n return p;\n}\n\nstatic void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_version opencl_c_version) {\n cl_int err;\n\n // compiler options for general kernels\n auto opencl_c_std =\n std::string(\"CL\") + std::to_string(opencl_c_version.major) + \".\" + std::to_string(opencl_c_version.minor);\n std::string compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable -cl-unsafe-math-optimizations\"\n \" -cl-finite-math-only -cl-fast-relaxed-math\";\n\n GGML_LOG_INFO(\"ggml_opencl: loading OpenCL kernels\");\n\n // add\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"add.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"add.cl\");\n#endif\n backend_ctx->program_add =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_add = clCreateKernel(backend_ctx->program_add, \"kernel_add\", &err), err));\n CL_CHECK((backend_ctx->kernel_add_row = clCreateKernel(backend_ctx->program_add, \"kernel_add_row\", &err), err));\n CL_CHECK((backend_ctx->kernel_add_f16 = clCreateKernel(backend_ctx->program_add, \"kernel_add_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_add_row_f16 = clCreateKernel(backend_ctx->program_add, \"kernel_add_row_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // add_id\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"add_id.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"add_id.cl\");\n#endif\n backend_ctx->program_add_id =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_add_id = clCreateKernel(backend_ctx->program_add_id, \"kernel_add_id\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // tri\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"tri.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"tri.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_tri = clCreateKernel(prog, \"kernel_tri_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n\n CL_CHECK(clReleaseProgram(prog));\n }\n\n // fill\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"fill.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"fill.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_fill = clCreateKernel(prog, \"kernel_fill_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n\n CL_CHECK(clReleaseProgram(prog));\n }\n\n // clamp\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"clamp.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"clamp.cl\");\n#endif\n backend_ctx->program_clamp =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_clamp = clCreateKernel(backend_ctx->program_clamp, \"kernel_clamp\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // cpy\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"cpy.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"cpy.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_cpy_f16_f16 = clCreateKernel(prog, \"kernel_cpy_f16_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_cpy_f16_f32 = clCreateKernel(prog, \"kernel_cpy_f16_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_cpy_f32_f16 = clCreateKernel(prog, \"kernel_cpy_f32_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_cpy_f32_f32 = clCreateKernel(prog, \"kernel_cpy_f32_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, \"kernel_cpy_i32_i32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // cvt\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"cvt.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"cvt.cl\");\n#endif\n backend_ctx->program_cvt =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_convert_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, \"kernel_convert_block_q4_0_noshuffle\", &err), err));\n CL_CHECK((backend_ctx->kernel_restore_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, \"kernel_restore_block_q4_0_noshuffle\", &err), err));\n CL_CHECK((backend_ctx->kernel_convert_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, \"kernel_convert_block_q4_0\", &err), err));\n CL_CHECK((backend_ctx->kernel_restore_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, \"kernel_restore_block_q4_0\", &err), err));\n CL_CHECK((backend_ctx->kernel_convert_block_q4_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, \"kernel_convert_block_q4_1_noshuffle\", &err), err));\n CL_CHECK((backend_ctx->kernel_restore_block_q4_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, \"kernel_restore_block_q4_1_noshuffle\", &err), err));\n CL_CHECK((backend_ctx->kernel_convert_block_q4_1 = clCreateKernel(backend_ctx->program_cvt, \"kernel_convert_block_q4_1\", &err), err));\n CL_CHECK((backend_ctx->kernel_restore_block_q4_1 = clCreateKernel(backend_ctx->program_cvt, \"kernel_restore_block_q4_1\", &err), err));\n CL_CHECK((backend_ctx->kernel_convert_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, \"kernel_convert_block_mxfp4\", &err), err));\n CL_CHECK((backend_ctx->kernel_convert_block_mxfp4_trans = clCreateKernel(backend_ctx->program_cvt, \"kernel_convert_block_mxfp4_trans\", &err), err));\n CL_CHECK((backend_ctx->kernel_restore_block_mxfp4_trans = clCreateKernel(backend_ctx->program_cvt, \"kernel_restore_block_mxfp4_trans\", &err), err));\n CL_CHECK((backend_ctx->kernel_restore_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, \"kernel_restore_block_mxfp4\", &err), err));\n CL_CHECK((backend_ctx->kernel_convert_block_q8_0 = clCreateKernel(backend_ctx->program_cvt, \"kernel_convert_block_q8_0\", &err), err));\n CL_CHECK((backend_ctx->kernel_restore_block_q8_0 = clCreateKernel(backend_ctx->program_cvt, \"kernel_restore_block_q8_0\", &err), err));\n CL_CHECK((backend_ctx->kernel_restore_block_q8_0_trans = clCreateKernel(backend_ctx->program_cvt, \"kernel_restore_block_q8_0_trans\", &err), err));\n CL_CHECK((backend_ctx->kernel_convert_block_q6_K = clCreateKernel(backend_ctx->program_cvt, \"kernel_convert_block_q6_K\", &err), err));\n CL_CHECK((backend_ctx->kernel_restore_block_q6_K = clCreateKernel(backend_ctx->program_cvt, \"kernel_restore_block_q6_K\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // diag_mask_inf\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"diag_mask_inf.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"diag_mask_inf.cl\");\n#endif\n backend_ctx->program_diag_mask_inf =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_diag_mask_inf_8 = clCreateKernel(backend_ctx->program_diag_mask_inf, \"kernel_diag_mask_inf_8\", &err), err));\n CL_CHECK((backend_ctx->kernel_diag_mask_inf = clCreateKernel(backend_ctx->program_diag_mask_inf, \"kernel_diag_mask_inf\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // diag\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"diag.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"diag.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_diag_f32 = clCreateKernel(prog, \"kernel_diag_f32\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // gelu\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"gelu.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"gelu.cl\");\n#endif\n backend_ctx->program_gelu =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_gelu = clCreateKernel(backend_ctx->program_gelu, \"kernel_gelu\", &err), err));\n CL_CHECK((backend_ctx->kernel_gelu_4 = clCreateKernel(backend_ctx->program_gelu, \"kernel_gelu_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_gelu_erf = clCreateKernel(backend_ctx->program_gelu, \"kernel_gelu_erf\", &err), err));\n CL_CHECK((backend_ctx->kernel_gelu_erf_4 = clCreateKernel(backend_ctx->program_gelu, \"kernel_gelu_erf_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_gelu_quick = clCreateKernel(backend_ctx->program_gelu, \"kernel_gelu_quick\", &err), err));\n CL_CHECK((backend_ctx->kernel_gelu_quick_4 = clCreateKernel(backend_ctx->program_gelu, \"kernel_gelu_quick_4\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // glu\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"glu.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"glu.cl\");\n#endif\n backend_ctx->program_glu =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_geglu = clCreateKernel(backend_ctx->program_glu, \"kernel_geglu\", &err), err));\n CL_CHECK((backend_ctx->kernel_reglu = clCreateKernel(backend_ctx->program_glu, \"kernel_reglu\", &err), err));\n CL_CHECK((backend_ctx->kernel_swiglu = clCreateKernel(backend_ctx->program_glu, \"kernel_swiglu\", &err), err));\n CL_CHECK((backend_ctx->kernel_swiglu_oai = clCreateKernel(backend_ctx->program_glu, \"kernel_swiglu_oai\", &err), err));\n CL_CHECK((backend_ctx->kernel_geglu_erf = clCreateKernel(backend_ctx->program_glu, \"kernel_geglu_erf\", &err), err));\n CL_CHECK((backend_ctx->kernel_geglu_quick = clCreateKernel(backend_ctx->program_glu, \"kernel_geglu_quick\", &err), err));\n CL_CHECK((backend_ctx->kernel_geglu_f16 = clCreateKernel(backend_ctx->program_glu, \"kernel_geglu_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_reglu_f16 = clCreateKernel(backend_ctx->program_glu, \"kernel_reglu_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_swiglu_f16 = clCreateKernel(backend_ctx->program_glu, \"kernel_swiglu_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_geglu_erf_f16 = clCreateKernel(backend_ctx->program_glu, \"kernel_geglu_erf_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_geglu_quick_f16 = clCreateKernel(backend_ctx->program_glu, \"kernel_geglu_quick_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // get_rows\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"get_rows.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"get_rows.cl\");\n#endif\n backend_ctx->program_get_rows =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_get_rows_f32 = clCreateKernel(backend_ctx->program_get_rows, \"kernel_get_rows_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_get_rows_f16 = clCreateKernel(backend_ctx->program_get_rows, \"kernel_get_rows_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_get_rows_q4_0 = clCreateKernel(backend_ctx->program_get_rows, \"kernel_get_rows_q4_0\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // solve_tri_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"solve_tri.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"solve_tri.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_solve_tri_f32 = clCreateKernel(prog, \"kernel_solve_tri_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n CL_CHECK(clReleaseProgram(prog));\n }\n\n // im2col_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"im2col_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"im2col_f32.cl\");\n#endif\n backend_ctx->program_im2col_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_im2col_f32 = clCreateKernel(backend_ctx->program_im2col_f32, \"kernel_im2col_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // im2col_f16\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"im2col_f16.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"im2col_f16.cl\");\n#endif\n backend_ctx->program_im2col_f16 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_im2col_f16 = clCreateKernel(backend_ctx->program_im2col_f16, \"kernel_im2col_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q4_0_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q4_0_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q4_0_f32.cl\");\n#endif\n backend_ctx->program_mul_mv_q4_0_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32, \"kernel_mul_mat_q4_0_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q4_0_f32_v\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q4_0_f32_v.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q4_0_f32_v.cl\");\n#endif\n backend_ctx->program_mul_mv_q4_0_f32_v =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_v = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_v, \"kernel_mul_mat_q4_0_f32_v\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q4_0_f32_8x_flat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q4_0_f32_8x_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q4_0_f32_8x_flat.cl\");\n#endif\n backend_ctx->program_mul_mv_q4_0_f32_8x_flat =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_8x_flat, \"kernel_mul_mat_q4_0_f32_8x_flat\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q4_0_f32_1d_8x_flat\n // This kernel does not compiler on Adreno cl compiler 38.01. Skip it for\n // those compiler versions since it is anyway not used for Adreno.\n if (backend_ctx->gpu_family != ADRENO ||\n backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) ||\n backend_ctx->adreno_cl_compiler_version.type == DX) {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q4_0_f32_1d_8x_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q4_0_f32_1d_8x_flat.cl\");\n#endif\n backend_ctx->program_mul_mv_q4_0_f32_1d_8x_flat =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_1d_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_1d_8x_flat, \"kernel_mul_mat_q4_0_f32_1d_8x_flat\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q4_0_f32_1d_16x_flat\n // This kernel does not compiler on Adreno cl compiler 38.01. Skip it for\n // those compiler versions since it is anyway not used for Adreno.\n if (backend_ctx->gpu_family != ADRENO ||\n backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) ||\n backend_ctx->adreno_cl_compiler_version.type == DX) {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q4_0_f32_1d_16x_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q4_0_f32_1d_16x_flat.cl\");\n#endif\n backend_ctx->program_mul_mv_q4_0_f32_1d_16x_flat =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_1d_16x_flat = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_1d_16x_flat, \"kernel_mul_mat_q4_0_f32_1d_16x_flat\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q4_1_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q4_1_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q4_1_f32.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_q4_1_f32 = clCreateKernel(prog, \"kernel_mul_mv_q4_1_f32\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q4_1_f32_flat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q4_1_f32_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q4_1_f32_flat.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_q4_1_f32_flat = clCreateKernel(prog, \"kernel_mul_mv_q4_1_f32_flat\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q4_k_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q4_k_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q4_k_f32.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_q4_K_f32 = clCreateKernel(prog, \"kernel_mul_mv_q4_K_f32\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q6_k_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q6_k_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q6_k_f32.cl\");\n#endif\n backend_ctx->program_mul_mv_q6_K =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32 = clCreateKernel(backend_ctx->program_mul_mv_q6_K, \"kernel_mul_mv_q6_K_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q6_k_f32_flat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q6_k_f32_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q6_k_f32_flat.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, \"kernel_mul_mv_q6_K_f32_flat\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q8_0_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q8_0_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q8_0_f32.cl\");\n#endif\n backend_ctx->program_mul_mv_q8_0_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32, \"kernel_mul_mv_q8_0_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_q8_0_f32_flat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_q8_0_f32_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_q8_0_f32_flat.cl\");\n#endif\n backend_ctx->program_mul_mv_q8_0_f32_flat =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32_flat, \"kernel_mul_mv_q8_0_f32_flat\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_mxfp4_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_mxfp4_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_mxfp4_f32.cl\");\n#endif\n backend_ctx->program_mul_mv_mxfp4_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32, \"kernel_mul_mv_mxfp4_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_mxfp4_f32_flat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_mxfp4_f32_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_mxfp4_f32_flat.cl\");\n#endif\n backend_ctx->program_mul_mv_mxfp4_f32_flat =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32_flat, \"kernel_mul_mv_mxfp4_f32_flat\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_f16_f16\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_f16_f16.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_f16_f16.cl\");\n#endif\n backend_ctx->program_mul_mv_f16_f16 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_f16_f16 = clCreateKernel(backend_ctx->program_mul_mv_f16_f16, \"kernel_mul_mat_f16_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_f16_f32_1row\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_f16_f32_1row.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_f16_f32_1row.cl\");\n#endif\n backend_ctx->program_mul_mv_f16_f32_1row =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_1row = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_1row, \"kernel_mul_mat_f16_f32_1row\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_f16_f32_l4\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_f16_f32_l4.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_f16_f32_l4.cl\");\n#endif\n backend_ctx->program_mul_mv_f16_f32_l4 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, \"kernel_mul_mat_f16_f32_l4\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_f16_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_f16_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_f16_f32.cl\");\n#endif\n backend_ctx->program_mul_mv_f16_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32, \"kernel_mul_mat_f16_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_f32_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_f32_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_f32_f32.cl\");\n#endif\n backend_ctx->program_mul_mv_f32_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_f32_f32 = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, \"kernel_mul_mat_f32_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mat_f16_f32_tiled\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mat_f16_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mat_f16_f32.cl\");\n#endif\n backend_ctx->program_mul_mat_f16_f32_tiled =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_tiled = clCreateKernel(backend_ctx->program_mul_mat_f16_f32_tiled, \"mul_mat_f16_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mm_f32_f32_l4_lm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mm_f32_f32_l4_lm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mm_f32_f32_l4_lm.cl\");\n#endif\n backend_ctx->program_mul_mm_f32_f32_l4_lm =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mm_f32_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, \"kernel_mul_mm_f32_f32_l4_lm\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mm_f16_f32_l4_lm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mm_f16_f32_l4_lm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mm_f16_f32_l4_lm.cl\");\n#endif\n backend_ctx->program_mul_mm_f16_f32_l4_lm =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_l4_lm, \"kernel_mul_mm_f16_f32_l4_lm\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mm_q4_0_f32_l4_lm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mm_q4_0_f32_l4_lm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mm_q4_0_f32_l4_lm.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm = clCreateKernel(prog, \"kernel_mul_mm_q4_0_f32_l4_lm\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mm_q4_1_f32_l4_lm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mm_q4_1_f32_l4_lm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mm_q4_1_f32_l4_lm.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm = clCreateKernel(prog, \"kernel_mul_mm_q4_1_f32_l4_lm\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mm_q8_0_f32_l4_lm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mm_q8_0_f32_l4_lm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mm_q8_0_f32_l4_lm.cl\");\n#endif\n backend_ctx->program_mul_mm_q8_0_f32_l4_lm =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_q8_0_f32_l4_lm, \"kernel_mul_mm_q8_0_f32_l4_lm\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mm_q6_k_f32_l4_lm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mm_q6_k_f32_l4_lm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mm_q6_k_f32_l4_lm.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm = clCreateKernel(prog, \"kernel_mul_mm_q6_k_f32_l4_lm\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mm_f16_f32_kq_kqv\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mm_f16_f32_kq_kqv.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mm_f16_f32_kq_kqv.cl\");\n#endif\n backend_ctx->program_mul_mm_f16_f32_kqv =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts+\" -DKQV \");\n backend_ctx->program_mul_mm_f16_f32_kq =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kqv = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kqv, \"mul_mm_f16_f32_kqv\", &err), err));\n CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kq = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kq, \"mul_mm_f16_f32_kq\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul.cl\");\n#endif\n backend_ctx->program_mul =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul = clCreateKernel(backend_ctx->program_mul, \"kernel_mul\", &err), err));\n CL_CHECK((backend_ctx->kernel_mul_row = clCreateKernel(backend_ctx->program_mul, \"kernel_mul_row\", &err), err));\n CL_CHECK((backend_ctx->kernel_mul_f16 = clCreateKernel(backend_ctx->program_mul, \"kernel_mul_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_mul_row_f16 = clCreateKernel(backend_ctx->program_mul, \"kernel_mul_row_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // norm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"norm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"norm.cl\");\n#endif\n backend_ctx->program_norm =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_norm = clCreateKernel(backend_ctx->program_norm, \"kernel_norm\", &err), err));\n CL_CHECK((backend_ctx->kernel_norm_mul_add = clCreateKernel(backend_ctx->program_norm, \"kernel_norm_mul_add\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // relu\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"relu.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"relu.cl\");\n#endif\n backend_ctx->program_relu =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_relu = clCreateKernel(backend_ctx->program_relu, \"kernel_relu\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // rms_norm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"rms_norm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"rms_norm.cl\");\n#endif\n backend_ctx->program_rms_norm =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_rms_norm = clCreateKernel(backend_ctx->program_rms_norm, \"kernel_rms_norm\", &err), err));\n CL_CHECK((backend_ctx->kernel_rms_norm_mul = clCreateKernel(backend_ctx->program_rms_norm, \"kernel_rms_norm_mul\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // l2_norm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"l2_norm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"l2_norm.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_l2_norm_f32 = clCreateKernel(prog, \"kernel_l2_norm_f32\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // rope\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"rope.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"rope.cl\");\n#endif\n backend_ctx->program_rope =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_rope_norm_f32 = clCreateKernel(backend_ctx->program_rope, \"kernel_rope_norm_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_rope_norm_f16 = clCreateKernel(backend_ctx->program_rope, \"kernel_rope_norm_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_rope_neox_f32 = clCreateKernel(backend_ctx->program_rope, \"kernel_rope_neox_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_rope_neox_f16 = clCreateKernel(backend_ctx->program_rope, \"kernel_rope_neox_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_rope_multi_f32 = clCreateKernel(backend_ctx->program_rope, \"kernel_rope_multi_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_rope_multi_f16 = clCreateKernel(backend_ctx->program_rope, \"kernel_rope_multi_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_rope_vision_f32 = clCreateKernel(backend_ctx->program_rope, \"kernel_rope_vision_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_rope_vision_f16 = clCreateKernel(backend_ctx->program_rope, \"kernel_rope_vision_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // scale\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"scale.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"scale.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_scale_f32 = clCreateKernel(prog, \"kernel_scale_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_scale_f32_4 = clCreateKernel(prog, \"kernel_scale_f32_4\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // silu\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"silu.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"silu.cl\");\n#endif\n backend_ctx->program_silu =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_silu = clCreateKernel(backend_ctx->program_silu, \"kernel_silu\", &err), err));\n CL_CHECK((backend_ctx->kernel_silu_4 = clCreateKernel(backend_ctx->program_silu, \"kernel_silu_4\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // softmax_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"softmax_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"softmax_f32.cl\");\n#endif\n backend_ctx->program_softmax_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_soft_max = clCreateKernel(backend_ctx->program_softmax_f32, \"kernel_soft_max\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // softmax_f16\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"softmax_f16.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"softmax_f16.cl\");\n#endif\n backend_ctx->program_softmax_f16 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_soft_max_f16 = clCreateKernel(backend_ctx->program_softmax_f16, \"kernel_soft_max_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // softmax_4_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"softmax_4_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"softmax_4_f32.cl\");\n#endif\n backend_ctx->program_softmax_4_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_soft_max_4 = clCreateKernel(backend_ctx->program_softmax_4_f32, \"kernel_soft_max_4\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // softmax_4_f16\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"softmax_4_f16.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"softmax_4_f16.cl\");\n#endif\n backend_ctx->program_softmax_4_f16 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_soft_max_4_f16 = clCreateKernel(backend_ctx->program_softmax_4_f16, \"kernel_soft_max_4_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // flash_attn\n {\n #ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src_f16 {\n #include \"flash_attn_f16.cl.h\"\n };\n const std::string kernel_src_f32 {\n #include \"flash_attn_f32.cl.h\"\n };\n const std::string kernel_src_f32_f16 {\n #include \"flash_attn_f32_f16.cl.h\"\n };\n #else\n const std::string kernel_src_f16 = read_file(\"flash_attn_f16.cl\");\n const std::string kernel_src_f32 = read_file(\"flash_attn_f32.cl\");\n const std::string kernel_src_f32_f16 = read_file(\"flash_attn_f32_f16.cl\");\n #endif\n\n if (!kernel_src_f16.empty() && !kernel_src_f32.empty() && !kernel_src_f32_f16.empty()) {\n const struct { int dk; int dv; int bm; int bn; } fa_dims[] = {\n { 40, 40, 32, 32}, { 64, 64, 64, 64}, { 80, 80, 64, 32}, { 96, 96, 64, 32},\n {112, 112, 32, 32}, {128, 128, 32, 32}, {192, 128, 16, 16},\n {192, 192, 16, 16}, {256, 256, 16, 16},\n };\n\n for (size_t i = 0; i < sizeof(fa_dims)/sizeof(fa_dims[0]); ++i) {\n const int dk = fa_dims[i].dk;\n const int dv = fa_dims[i].dv;\n const int bm = fa_dims[i].bm;\n const int bn = fa_dims[i].bn;\n std::string OPTS = compile_opts +\n \" -D DK=\" + std::to_string(dk) +\n \" -D DV=\" + std::to_string(dv) +\n \" -D BLOCK_M=\" + std::to_string(bm) +\n \" -D BLOCK_N=\" + std::to_string(bn);\n\n cl_program prog_f16 = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f16.c_str(), OPTS);\n cl_kernel k_f16, k_f16_q1;\n CL_CHECK((k_f16 = clCreateKernel(prog_f16, \"flash_attn_f16\", &err), err));\n CL_CHECK((k_f16_q1 = clCreateKernel(prog_f16, \"flash_attn_f16_q1\", &err), err));\n backend_ctx->kernels_flash_attn_f16[{dk, dv}] = k_f16;\n backend_ctx->kernels_flash_attn_f16_q1[{dk, dv}] = k_f16_q1;\n CL_CHECK(clReleaseProgram(prog_f16));\n\n cl_program prog_f32 = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f32.c_str(), OPTS);\n cl_kernel k_f32, k_f32_q1;\n CL_CHECK((k_f32 = clCreateKernel(prog_f32, \"flash_attn_f32\", &err), err));\n CL_CHECK((k_f32_q1 = clCreateKernel(prog_f32, \"flash_attn_f32_q1\", &err), err));\n backend_ctx->kernels_flash_attn_f32[{dk, dv}] = k_f32;\n backend_ctx->kernels_flash_attn_f32_q1[{dk, dv}] = k_f32_q1;\n CL_CHECK(clReleaseProgram(prog_f32));\n\n cl_program prog_f32_f16 = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f32_f16.c_str(), OPTS);\n cl_kernel k_f32_f16, k_f32_f16_q1;\n CL_CHECK((k_f32_f16 = clCreateKernel(prog_f32_f16, \"flash_attn_f32_f16\", &err), err));\n CL_CHECK((k_f32_f16_q1 = clCreateKernel(prog_f32_f16, \"flash_attn_f32_f16_q1\", &err), err));\n backend_ctx->kernels_flash_attn_f32_f16[{dk, dv}] = k_f32_f16;\n backend_ctx->kernels_flash_attn_f32_f16_q1[{dk, dv}] = k_f32_f16_q1;\n CL_CHECK(clReleaseProgram(prog_f32_f16));\n\n backend_ctx->kernels_flash_attn_bm[{dk, dv}] = bm;\n backend_ctx->kernels_flash_attn_bn[{dk, dv}] = bn;\n }\n GGML_LOG_CONT(\".\");\n }\n }\n\n // argsort\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"argsort.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"argsort.cl\");\n#endif\n backend_ctx->program_argsort_f32_i32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_argsort_f32_i32 = clCreateKernel(backend_ctx->program_argsort_f32_i32, \"kernel_argsort_f32_i32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // div\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"div.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"div.cl\");\n#endif\n std::string compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable -cl-finite-math-only \";\n\n backend_ctx->program_div =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_div = clCreateKernel(backend_ctx->program_div, \"kernel_div\", &err), err));\n CL_CHECK((backend_ctx->kernel_div_row = clCreateKernel(backend_ctx->program_div, \"kernel_div_row\", &err), err));\n CL_CHECK((backend_ctx->kernel_div_f16 = clCreateKernel(backend_ctx->program_div, \"kernel_div_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_div_row_f16 = clCreateKernel(backend_ctx->program_div, \"kernel_div_row_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // sqr\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"sqr.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"sqr.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_sqr_cont_f32 = clCreateKernel(prog, \"kernel_sqr_cont_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_sqr_cont_f32_4 = clCreateKernel(prog, \"kernel_sqr_cont_f32_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_sqr_cont_f16 = clCreateKernel(prog, \"kernel_sqr_cont_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_sqr_cont_f16_4 = clCreateKernel(prog, \"kernel_sqr_cont_f16_4\", &err), err));\n\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // sqrt\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"sqrt.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"sqrt.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_sqrt_cont_f32 = clCreateKernel(prog, \"kernel_sqrt_cont_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_sqrt_cont_f32_4 = clCreateKernel(prog, \"kernel_sqrt_cont_f32_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_sqrt_cont_f16 = clCreateKernel(prog, \"kernel_sqrt_cont_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_sqrt_cont_f16_4 = clCreateKernel(prog, \"kernel_sqrt_cont_f16_4\", &err), err));\n\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // mean\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mean.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mean.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mean_f32 = clCreateKernel(prog, \"kernel_mean_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_mean_f32_4 = clCreateKernel(prog, \"kernel_mean_f32_4\", &err), err));\n\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // sub\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"sub.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"sub.cl\");\n#endif\n backend_ctx->program_sub =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_sub = clCreateKernel(backend_ctx->program_sub, \"kernel_sub\", &err), err));\n CL_CHECK((backend_ctx->kernel_sub_row = clCreateKernel(backend_ctx->program_sub, \"kernel_sub_row\", &err), err));\n CL_CHECK((backend_ctx->kernel_sub_f16 = clCreateKernel(backend_ctx->program_sub, \"kernel_sub_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_sub_row_f16 = clCreateKernel(backend_ctx->program_sub, \"kernel_sub_row_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // sum_rows\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"sum_rows.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"sum_rows.cl\");\n#endif\n backend_ctx->program_sum_rows_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_sum_rows_f32 = clCreateKernel(backend_ctx->program_sum_rows_f32, \"kernel_sum_rows_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_sum_rows_f32_4 = clCreateKernel(backend_ctx->program_sum_rows_f32, \"kernel_sum_rows_f32_4\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // cumsum\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"cumsum.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"cumsum.cl\");\n#endif\n cl_program prog;\n prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_cumsum_blk = clCreateKernel(prog, \"kernel_cumsum_blk\", &err), err));\n CL_CHECK((backend_ctx->kernel_cumsum_add = clCreateKernel(prog, \"kernel_cumsum_add\", &err), err));\n GGML_LOG_CONT(\".\");\n CL_CHECK(clReleaseProgram(prog));\n }\n\n // sigmoid\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"sigmoid.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"sigmoid.cl\");\n#endif\n backend_ctx->program_sigmoid =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_sigmoid_f32 = clCreateKernel(backend_ctx->program_sigmoid, \"kernel_sigmoid_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_sigmoid_f16 = clCreateKernel(backend_ctx->program_sigmoid, \"kernel_sigmoid_f16\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // group_norm\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"group_norm.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"group_norm.cl\");\n#endif\n backend_ctx->program_group_norm =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_group_norm = clCreateKernel(backend_ctx->program_group_norm, \"kernel_group_norm\", &err), err));\n CL_CHECK((backend_ctx->kernel_group_norm_mul_add = clCreateKernel(backend_ctx->program_group_norm, \"kernel_group_norm_mul_add\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // repeat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"repeat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"repeat.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_repeat_f32 = clCreateKernel(prog, \"kernel_repeat_f32\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // pad\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"pad.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"pad.cl\");\n#endif\n if (!kernel_src.empty()) {\n backend_ctx->program_pad =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_pad = clCreateKernel(backend_ctx->program_pad, \"kernel_pad\", &err), err));\n GGML_LOG_CONT(\".\");\n } else {\n GGML_LOG_WARN(\"ggml_opencl: pad kernel source not found or empty. Pad operations will not be available.\\n\");\n backend_ctx->program_pad = nullptr;\n backend_ctx->kernel_pad = nullptr;\n }\n }\n\n // tanh\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"tanh.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"tanh.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_tanh_f32 = clCreateKernel(prog, \"kernel_tanh_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_tanh_f32_4 = clCreateKernel(prog, \"kernel_tanh_f32_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_tanh_f32_nc = clCreateKernel(prog, \"kernel_tanh_f32_nc\", &err), err));\n CL_CHECK((backend_ctx->kernel_tanh_f16 = clCreateKernel(prog, \"kernel_tanh_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_tanh_f16_4 = clCreateKernel(prog, \"kernel_tanh_f16_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_tanh_f16_nc = clCreateKernel(prog, \"kernel_tanh_f16_nc\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // neg\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"neg.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"neg.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_neg_f32 = clCreateKernel(prog, \"kernel_neg_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_neg_f32_4 = clCreateKernel(prog, \"kernel_neg_f32_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_neg_f32_nc = clCreateKernel(prog, \"kernel_neg_f32_nc\", &err), err));\n CL_CHECK((backend_ctx->kernel_neg_f16 = clCreateKernel(prog, \"kernel_neg_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_neg_f16_4 = clCreateKernel(prog, \"kernel_neg_f16_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_neg_f16_nc = clCreateKernel(prog, \"kernel_neg_f16_nc\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // exp\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"exp.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"exp.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_exp_f32 = clCreateKernel(prog, \"kernel_exp_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_exp_f32_4 = clCreateKernel(prog, \"kernel_exp_f32_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_exp_f32_nc = clCreateKernel(prog, \"kernel_exp_f32_nc\", &err), err));\n CL_CHECK((backend_ctx->kernel_exp_f16 = clCreateKernel(prog, \"kernel_exp_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_exp_f16_4 = clCreateKernel(prog, \"kernel_exp_f16_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_exp_f16_nc = clCreateKernel(prog, \"kernel_exp_f16_nc\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // expm1\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"expm1.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"expm1.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_expm1_f32 = clCreateKernel(prog, \"kernel_expm1_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_expm1_f32_4 = clCreateKernel(prog, \"kernel_expm1_f32_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_expm1_f32_nc = clCreateKernel(prog, \"kernel_expm1_f32_nc\", &err), err));\n CL_CHECK((backend_ctx->kernel_expm1_f16 = clCreateKernel(prog, \"kernel_expm1_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_expm1_f16_4 = clCreateKernel(prog, \"kernel_expm1_f16_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_expm1_f16_nc = clCreateKernel(prog, \"kernel_expm1_f16_nc\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // softplus\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"softplus.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"softplus.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_softplus_f32 = clCreateKernel(prog, \"kernel_softplus_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_softplus_f32_4 = clCreateKernel(prog, \"kernel_softplus_f32_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_softplus_f32_nc = clCreateKernel(prog, \"kernel_softplus_f32_nc\", &err), err));\n CL_CHECK((backend_ctx->kernel_softplus_f16 = clCreateKernel(prog, \"kernel_softplus_f16\", &err), err));\n CL_CHECK((backend_ctx->kernel_softplus_f16_4 = clCreateKernel(prog, \"kernel_softplus_f16_4\", &err), err));\n CL_CHECK((backend_ctx->kernel_softplus_f16_nc = clCreateKernel(prog, \"kernel_softplus_f16_nc\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // upscale\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"upscale.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"upscale.cl\");\n#endif\n if (!kernel_src.empty()) {\n backend_ctx->program_upscale =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_upscale = clCreateKernel(backend_ctx->program_upscale, \"kernel_upscale\", &err), err));\n if (backend_ctx->program_upscale) {\n cl_int err_bilinear;\n backend_ctx->kernel_upscale_bilinear = clCreateKernel(backend_ctx->program_upscale, \"kernel_upscale_bilinear\", &err_bilinear);\n if (err_bilinear != CL_SUCCESS) {\n GGML_LOG_WARN(\"ggml_opencl: kernel_upscale_bilinear not found in upscale.cl. Bilinear upscale will not be available. Error: %d\\n\", err_bilinear);\n backend_ctx->kernel_upscale_bilinear = nullptr;\n }\n } else {\n backend_ctx->kernel_upscale_bilinear = nullptr;\n }\n GGML_LOG_CONT(\".\");\n } else {\n GGML_LOG_WARN(\"ggml_opencl: upscale kernel source not found or empty. Upscale operations will not be available.\\n\");\n backend_ctx->program_upscale = nullptr;\n backend_ctx->kernel_upscale = nullptr;\n backend_ctx->kernel_upscale_bilinear = nullptr;\n }\n }\n\n // concat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"concat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"concat.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_concat_f32 = clCreateKernel(prog, \"kernel_concat_f32\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // timestep_embedding\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"tsembd.cl.h\"\n };\n#else\n\n const std::string kernel_src = read_file(\"tsembd.cl\");\n#endif\n if (!kernel_src.empty()) {\n backend_ctx->program_tsembd =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_timestep_embedding = clCreateKernel(backend_ctx->program_tsembd, \"kernel_timestep_embedding\", &err), err));\n GGML_LOG_CONT(\".\");\n } else {\n GGML_LOG_WARN(\"ggml_opencl: timestep_embedding kernel source not found or empty. This op will not be available.\\n\");\n backend_ctx->program_tsembd = nullptr;\n backend_ctx->kernel_timestep_embedding = nullptr;\n }\n }\n\n // set_rows\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"set_rows.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"set_rows.cl\");\n#endif\n backend_ctx->program_set_rows =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_set_rows_f32_i64 = clCreateKernel(backend_ctx->program_set_rows, \"kernel_set_rows_f32_i64\", &err), err));\n CL_CHECK((backend_ctx->kernel_set_rows_f32_i32 = clCreateKernel(backend_ctx->program_set_rows, \"kernel_set_rows_f32_i32\", &err), err));\n CL_CHECK((backend_ctx->kernel_set_rows_f16_i64 = clCreateKernel(backend_ctx->program_set_rows, \"kernel_set_rows_f16_i64\", &err), err));\n CL_CHECK((backend_ctx->kernel_set_rows_f16_i32 = clCreateKernel(backend_ctx->program_set_rows, \"kernel_set_rows_f16_i32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // conv2d\n {\n #ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"conv2d.cl.h\"\n };\n const std::string kernel_src_f16_f32 {\n #include \"conv2d_f16_f32.cl.h\"\n };\n #else\n const std::string kernel_src = read_file(\"conv2d.cl\");\n const std::string kernel_src_f16_f32 = read_file(\"conv2d_f16_f32.cl\");\n #endif\n if (!kernel_src.empty()) {\n backend_ctx->program_conv_2d_f16 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), (std::string(compile_opts) + \" -DUSE_FP16=1\").c_str());\n CL_CHECK((backend_ctx->kernel_conv_2d_f16 = clCreateKernel(backend_ctx->program_conv_2d_f16, \"kernel_conv_2d\", &err), err));\n GGML_LOG_CONT(\".\");\n backend_ctx->program_conv_2d_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_conv_2d_f32 = clCreateKernel(backend_ctx->program_conv_2d_f32, \"kernel_conv_2d\", &err), err));\n GGML_LOG_CONT(\".\");\n } else {\n GGML_LOG_WARN(\"ggml_opencl: conv2d kernel source not found or empty. This op will not be available.\\n\");\n backend_ctx->program_conv_2d_f16 = nullptr;\n backend_ctx->kernel_conv_2d_f16 = nullptr;\n backend_ctx->program_conv_2d_f32 = nullptr;\n backend_ctx->kernel_conv_2d_f32 = nullptr;\n }\n if (!kernel_src_f16_f32.empty()) {\n backend_ctx->program_conv_2d_f16_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f16_f32.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_conv_2d_f16_f32 = clCreateKernel(backend_ctx->program_conv_2d_f16_f32, \"kernel_conv_2d\", &err), err));\n GGML_LOG_CONT(\".\");\n } else {\n GGML_LOG_WARN(\"ggml_opencl: conv2d_f16_f32 kernel source not found or empty. This op will not be available.\\n\");\n backend_ctx->program_conv_2d_f16_f32 = nullptr;\n backend_ctx->kernel_conv_2d_f16_f32 = nullptr;\n }\n }\n\n // ssm_conv\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"ssm_conv.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"ssm_conv.cl\");\n#endif\n cl_program prog =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32 = clCreateKernel(prog, \"kernel_ssm_conv_f32_f32\", &err), err));\n CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32_4 = clCreateKernel(prog, \"kernel_ssm_conv_f32_f32_4\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_id_q4_0_f32_8x_flat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_id_q4_0_f32_8x_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_id_q4_0_f32_8x_flat.cl\");\n#endif\n backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_id_q4_0_f32_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat, \"kernel_mul_mv_id_q4_0_f32_8x_flat\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_id_q8_0_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_id_q8_0_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_id_q8_0_f32.cl\");\n#endif\n backend_ctx->program_mul_mv_id_q8_0_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32, \"kernel_mul_mv_id_q8_0_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_id_q8_0_f32_flat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_id_q8_0_f32_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_id_q8_0_f32_flat.cl\");\n#endif\n backend_ctx->program_mul_mv_id_q8_0_f32_flat =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32_flat, \"kernel_mul_mv_id_q8_0_f32_flat\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_id_mxfp4_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_id_mxfp4_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_id_mxfp4_f32.cl\");\n#endif\n backend_ctx->program_mul_mv_id_mxfp4_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32, \"kernel_mul_mv_id_mxfp4_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mv_id_mxfp4_f32_flat\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"mul_mv_id_mxfp4_f32_flat.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"mul_mv_id_mxfp4_f32_flat.cl\");\n#endif\n backend_ctx->program_mul_mv_id_mxfp4_f32_flat =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32_flat, \"kernel_mul_mv_id_mxfp4_f32_flat\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // Adreno kernels\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n // transpose\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"transpose.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"transpose.cl\");\n#endif\n backend_ctx->program_transpose =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n\n CL_CHECK((backend_ctx->kernel_transpose_32_16 = clCreateKernel(backend_ctx->program_transpose, \"kernel_transpose_32_16\", &err), err));\n CL_CHECK((backend_ctx->kernel_transpose_32 = clCreateKernel(backend_ctx->program_transpose, \"kernel_transpose_32\", &err), err));\n CL_CHECK((backend_ctx->kernel_transpose_16 = clCreateKernel(backend_ctx->program_transpose, \"kernel_transpose_16\", &err), err));\n CL_CHECK((backend_ctx->kernel_transpose_8_buf = clCreateKernel(backend_ctx->program_transpose, \"kernel_transpose_8_buf\", &err), err));\n CL_CHECK((backend_ctx->kernel_transpose_16_buf = clCreateKernel(backend_ctx->program_transpose, \"kernel_transpose_16_buf\", &err), err));\n CL_CHECK((backend_ctx->kernel_transpose_32_buf = clCreateKernel(backend_ctx->program_transpose, \"kernel_transpose_32_buf\", &err), err));\n CL_CHECK((backend_ctx->kernel_transpose_16_4x1 = clCreateKernel(backend_ctx->program_transpose, \"kernel_transpose_16_4x1\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // gemv_noshuffle_general\n {\n std::string CL_gemv_compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable \"\n \" -DSIMDGROUP_WIDTH=\" +\n std::to_string(backend_ctx->adreno_wave_size);\n if (backend_ctx->has_vector_subgroup_broadcast) {\n CL_gemv_compile_opts += \" -DVECTOR_SUB_GROUP_BROADCAT \";\n }\n\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src_CL_gemv_general {\n #include \"gemv_noshuffle_general.cl.h\"\n };\n#else\n const std::string kernel_src_CL_gemv_general = read_file(\"gemv_noshuffle_general.cl\");\n#endif\n\n backend_ctx->program_CL_gemv_general = build_program_from_source(\n backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts);\n\n CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_general = clCreateKernel(backend_ctx->program_CL_gemv_general, \"kernel_gemv_noshuffle\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // gemv_noshuffle\n {\n // Gemv 2048, 16384\n std::string CL_gemv_compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable \"\n \" -DLINE_STRIDE_A=2048 \"\n \" -DBLOCK_STRIDE_A=16384 \"\n \" -DSIMDGROUP_WIDTH=\" +\n std::to_string(backend_ctx->adreno_wave_size);\n if (backend_ctx->has_vector_subgroup_broadcast) {\n CL_gemv_compile_opts += \" -DVECTOR_SUB_GROUP_BROADCAT \";\n }\n\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src_CL_gemv {\n #include \"gemv_noshuffle.cl.h\"\n };\n#else\n const std::string kernel_src_CL_gemv = read_file(\"gemv_noshuffle.cl\");\n#endif\n\n backend_ctx->program_CL_gemv_4096_1_4096 = build_program_from_source(\n backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts);\n CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_4096 = clCreateKernel(backend_ctx->program_CL_gemv_4096_1_4096, \"kernel_gemv_noshuffle\", &err), err));\n GGML_LOG_CONT(\".\");\n\n // Gemv 2048, 16384\n CL_gemv_compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable \"\n \" -DLINE_STRIDE_A=2048 \"\n \" -DBLOCK_STRIDE_A=16384 \"\n \" -DSIMDGROUP_WIDTH=\" +\n std::to_string(backend_ctx->adreno_wave_size);\n if (backend_ctx->has_vector_subgroup_broadcast) {\n CL_gemv_compile_opts += \" -DVECTOR_SUB_GROUP_BROADCAT \";\n }\n\n backend_ctx->program_CL_gemv_4096_1_11008 = build_program_from_source(\n backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts);\n CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_11008 = clCreateKernel(backend_ctx->program_CL_gemv_4096_1_11008, \"kernel_gemv_noshuffle\", &err), err));\n GGML_LOG_CONT(\".\");\n\n // Gemv 5504, 44032\n CL_gemv_compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable \"\n \" -DLINE_STRIDE_A=5504 \"\n \" -DBLOCK_STRIDE_A=44032 \"\n \" -DSIMDGROUP_WIDTH=\" +\n std::to_string(backend_ctx->adreno_wave_size);\n if (backend_ctx->has_vector_subgroup_broadcast) {\n CL_gemv_compile_opts += \" -DVECTOR_SUB_GROUP_BROADCAT \";\n }\n\n backend_ctx->program_CL_gemv_11008_1_4096 = build_program_from_source(\n backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts);\n CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_11008_1_4096 = clCreateKernel(backend_ctx->program_CL_gemv_11008_1_4096, \"kernel_gemv_noshuffle\", &err), err));\n GGML_LOG_CONT(\".\");\n\n // Gemv 16000, 128000\n CL_gemv_compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable \"\n \" -DLINE_STRIDE_A=16000 \"\n \" -DBLOCK_STRIDE_A=128000 \"\n \" -DSIMDGROUP_WIDTH=\" +\n std::to_string(backend_ctx->adreno_wave_size);\n\n if (backend_ctx->has_vector_subgroup_broadcast) {\n CL_gemv_compile_opts += \" -DVECTOR_SUB_GROUP_BROADCAT \";\n }\n\n backend_ctx->program_CL_gemv_32000_1_4096 = build_program_from_source(\n backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts);\n CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_32000_1_4096 = clCreateKernel(backend_ctx->program_CL_gemv_32000_1_4096, \"kernel_gemv_noshuffle\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mat_Ab_Bi_8x4\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src_CL_gemm {\n #include \"mul_mat_Ab_Bi_8x4.cl.h\"\n };\n#else\n const std::string kernel_src_CL_gemm = read_file(\"mul_mat_Ab_Bi_8x4.cl\");\n#endif\n backend_ctx->program_CL_gemm = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_CL_gemm.c_str(), compile_opts);\n CL_CHECK((backend_ctx->CL_mul_mat_Ab_Bi_8x4 = clCreateKernel(backend_ctx->program_CL_gemm, \"kernel_mul_mat_Ab_Bi_8x4\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // gemm_noshuffle_q4_1_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"gemm_noshuffle_q4_1_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"gemm_noshuffle_q4_1_f32.cl\");\n#endif\n cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_1_f32 = clCreateKernel(prog, \"kernel_gemm_noshuffle_q4_1_f32\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // gemv_noshuffle_q4_1_f32\n {\n std::string CL_gemv_compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable \";\n if (backend_ctx->has_vector_subgroup_broadcast) {\n CL_gemv_compile_opts += \" -DVECTOR_SUB_GROUP_BROADCAT \";\n }\n\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"gemv_noshuffle_q4_1_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"gemv_noshuffle_q4_1_f32.cl\");\n#endif\n\n cl_program prog = build_program_from_source(\n backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_gemv_compile_opts);\n\n CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32 = clCreateKernel(prog, \"kernel_gemv_noshuffle_q4_1_f32\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n // mul_mm_q8_0_f32_8x4\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src_q8_8x4_gemm {\n #include \"mul_mm_q8_0_f32_8x4.cl.h\"\n };\n#else\n const std::string kernel_src_q8_8x4_gemm = read_file(\"mul_mm_q8_0_f32_8x4.cl\");\n#endif\n backend_ctx->program_CL_gemm = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_q8_8x4_gemm.c_str(), compile_opts);\n CL_CHECK((backend_ctx->kernel_mul_mm_q8_0_f32_8x4 = clCreateKernel(backend_ctx->program_CL_gemm, \"kernel_mul_mm_q8_0_f32_8x4\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // gemv_noshuffle_general_q8_0_f32\n {\n std::string CL_gemv_compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable \"\n \" -DSIMDGROUP_WIDTH=\" +\n std::to_string(backend_ctx->adreno_wave_size);\n if (backend_ctx->has_vector_subgroup_broadcast) {\n CL_gemv_compile_opts += \" -DVECTOR_SUB_GROUP_BROADCAT \";\n }\n\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src_CL_gemv_general {\n #include \"gemv_noshuffle_general_q8_0_f32.cl.h\"\n };\n#else\n const std::string kernel_src_CL_gemv_general = read_file(\"gemv_noshuffle_general_q8_0_f32.cl\");\n#endif\n\n cl_program prog = build_program_from_source(\n backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts);\n\n CL_CHECK((backend_ctx->CL_mul_mat_vec_q8_0_f32 = clCreateKernel(prog, \"kernel_gemv_noshuffle_q8_0_f32\", &err), err));\n CL_CHECK(clReleaseProgram(prog));\n GGML_LOG_CONT(\".\");\n }\n\n std::string CL_moe_compile_opts = std::string(\"-cl-std=\") + opencl_c_std +\n \" -cl-mad-enable \"\n \" -cl-fast-relaxed-math\";\n\n // gemv_moe_mxfp4_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"gemv_moe_mxfp4_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"gemv_moe_mxfp4_f32.cl\");\n#endif\n backend_ctx->program_gemv_moe_mxfp4_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);\n\n CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemv_moe_mxfp4_f32, \"kernel_gemv_moe_mxfp4_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n\n // gemm_moe_mxfp4_f32\n {\n#ifdef GGML_OPENCL_EMBED_KERNELS\n const std::string kernel_src {\n #include \"gemm_moe_mxfp4_f32.cl.h\"\n };\n#else\n const std::string kernel_src = read_file(\"gemm_moe_mxfp4_f32.cl\");\n#endif\n backend_ctx->program_gemm_moe_mxfp4_f32 =\n build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);\n\n CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemm_moe_mxfp4_f32, \"kernel_gemm_moe_mxfp4_f32\", &err), err));\n GGML_LOG_CONT(\".\");\n }\n#endif // GGML_OPENCL_USE_ADRENO_KERNELS\n GGML_LOG_CONT(\"\\n\");\n}\n\n// XXX static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) {\n// XXX static bool initialized = false;\n// XXX static ggml_backend_opencl_context *backend_ctx = nullptr;\n\nstatic ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev);\n\nnamespace /* anonymous */ {\nextern struct ggml_backend_device_i ggml_backend_opencl_device_i;\n}\n\n// Look for available and suitable devices.\nstatic std::vector ggml_opencl_probe_devices(ggml_backend_reg * reg) {\n std::vector found_devices;\n\n#ifdef GGML_OPENCL_PROFILING\n GGML_LOG_INFO(\"ggml_opencl: OpenCL profiling enabled\\n\");\n#endif\n\n struct cl_device;\n struct cl_platform {\n cl_platform_id id;\n unsigned number;\n char name[128];\n char vendor[128];\n struct cl_device * devices;\n unsigned n_devices;\n struct cl_device * default_device;\n };\n\n struct cl_device {\n struct cl_platform * platform;\n cl_device_id id;\n unsigned number;\n cl_device_type type;\n char name[128];\n char version[128];\n };\n\n enum { NPLAT = 16, NDEV = 16 };\n\n struct cl_platform platforms[NPLAT];\n unsigned n_platforms = 0;\n struct cl_device devices[NDEV];\n unsigned n_devices = 0;\n struct cl_device * default_device = NULL;\n unsigned default_platform_number = 0;\n\n cl_platform_id platform_ids[NPLAT];\n if (clGetPlatformIDs(NPLAT, platform_ids, &n_platforms) != CL_SUCCESS) {\n GGML_LOG_ERROR(\"ggml_opencl: platform IDs not available.\\n\");\n return found_devices;\n }\n\n for (unsigned i = 0; i < n_platforms; i++) {\n struct cl_platform * p = &platforms[i];\n p->number = i;\n p->id = platform_ids[i];\n CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL));\n CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL));\n\n cl_device_id device_ids[NDEV];\n cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices);\n if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) {\n p->n_devices = 0;\n } else {\n CL_CHECK(clGetDeviceIDsError);\n }\n p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL;\n p->default_device = NULL;\n\n for (unsigned j = 0; j < p->n_devices; j++) {\n struct cl_device * d = &devices[n_devices];\n d->number = n_devices++;\n d->id = device_ids[j];\n d->platform = p;\n CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_NAME, sizeof(d->name), &d->name, NULL));\n CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_TYPE, sizeof(d->type), &d->type, NULL));\n CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_VERSION, sizeof(d->version), &d->version, NULL));\n\n if (p->default_device == NULL && d->type == CL_DEVICE_TYPE_GPU) {\n p->default_device = d;\n }\n }\n\n if (default_device == NULL && p->default_device != NULL) {\n default_device = p->default_device;\n default_platform_number = i;\n }\n }\n\n if (n_devices == 0) {\n GGML_LOG_ERROR(\"ggml_opencl: could find any OpenCL devices.\\n\");\n return found_devices;\n }\n\n char * user_platform_string = getenv(\"GGML_OPENCL_PLATFORM\");\n char * user_device_string = getenv(\"GGML_OPENCL_DEVICE\");\n int user_platform_number = -1;\n int user_device_number = -1;\n cl_device * candidate_devices = nullptr;\n unsigned n_candidate_devices = 0;\n\n unsigned n;\n if (user_platform_string != NULL && sscanf(user_platform_string, \" %u\", &n) == 1 && n < n_platforms) {\n user_platform_number = (int)n;\n }\n if (user_device_string != NULL && sscanf(user_device_string, \" %u\", &n) == 1 && n < n_devices) {\n user_device_number = (int)n;\n }\n if (user_platform_number != -1 && user_device_number != -1) {\n cl_platform* platform = &platforms[user_platform_number];\n if ((unsigned)user_device_number >= platform->n_devices) {\n GGML_LOG_ERROR(\"ggml_opencl: invalid device number %d\\n\", user_device_number);\n exit(1);\n }\n default_device = &platform->devices[user_device_number];\n candidate_devices = platform->devices;\n n_candidate_devices = platform->n_devices;\n } else {\n // Choose a platform by matching a substring.\n if (user_platform_number == -1 && user_platform_string != NULL && user_platform_string[0] != 0) {\n for (unsigned i = 0; i < n_platforms; i++) {\n struct cl_platform * p = &platforms[i];\n if (strstr(p->name, user_platform_string) != NULL ||\n strstr(p->vendor, user_platform_string) != NULL) {\n user_platform_number = (int)i;\n break;\n }\n }\n if (user_platform_number == -1) {\n GGML_LOG_ERROR(\"ggml_opencl: no platform matching '%s' was found.\\n\", user_platform_string);\n exit(1);\n }\n }\n\n int platform_idx = user_platform_number != -1 ? user_platform_number : default_platform_number;\n struct cl_platform * p = &platforms[platform_idx];\n candidate_devices = p->devices;\n n_candidate_devices = p->n_devices;\n default_device = p->default_device;\n if (n_candidate_devices == 0) {\n GGML_LOG_ERROR(\"ggml_opencl: selected platform '%s' does not have any devices.\\n\", p->name);\n exit(1);\n }\n\n if (user_device_number == -1 && user_device_string != NULL && user_device_string[0] != 0) {\n for (unsigned i = 0; i < n_candidate_devices; i++) {\n struct cl_device * d = &candidate_devices[i];\n if (strstr(d->name, user_device_string) != NULL) {\n user_device_number = d->number;\n break;\n }\n }\n if (user_device_number == -1) {\n GGML_LOG_ERROR(\"ggml_opencl: no device matching '%s' was found.\\n\", user_device_string);\n exit(1);\n }\n }\n if (user_device_number != -1) {\n candidate_devices = &devices[user_device_number];\n n_candidate_devices = 1;\n default_device = &candidate_devices[0];\n }\n\n GGML_ASSERT(n_candidate_devices > 0);\n\n if (default_device == NULL) {\n default_device = &candidate_devices[0];\n }\n }\n\n GGML_ASSERT(n_candidate_devices != 0 && candidate_devices);\n\n // Put the default device in front.\n for (unsigned i = 1; i < n_candidate_devices; i++) {\n if (&candidate_devices[i] == default_device) {\n std::swap(candidate_devices[0], candidate_devices[i]);\n default_device = &candidate_devices[0];\n break;\n }\n }\n\n GGML_LOG_INFO(\"ggml_opencl: selected platform: '%s'\\n\", default_device->platform->name);\n\n std::vector device_ids;\n for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) {\n device_ids.push_back(dev->id);\n }\n\n cl_int err;\n cl_context shared_context;\n cl_context_properties properties[] = { (intptr_t) CL_CONTEXT_PLATFORM, (intptr_t) default_device->platform->id, 0 };\n\n CL_CHECK(\n (shared_context = clCreateContext(properties, device_ids.size(), device_ids.data(), NULL, NULL, &err), err));\n\n for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) {\n GGML_LOG_INFO(\"\\nggml_opencl: device: '%s (%s)'\\n\", dev->name, dev->version);\n\n auto dev_ctx = std::unique_ptr(new ggml_backend_opencl_device_context{\n /*.platform =*/dev->platform->id,\n /*.platform_nane =*/dev->platform->name,\n /*.device =*/dev->id,\n /*.device_name =*/dev->name,\n /*.device_type =*/dev->type,\n /*.device_version =*/dev->version,\n /*.backend_ctx =*/nullptr,\n /*.buffer_type =*/{},\n /*.context =*/shared_context,\n });\n\n found_devices.push_back(ggml_backend_device{\n /* .iface = */ ggml_backend_opencl_device_i,\n /* .reg = */ reg,\n /* .context = */ dev_ctx.get(),\n });\n\n if (!ggml_cl2_init(&found_devices.back())) {\n found_devices.pop_back();\n GGML_LOG_INFO(\"ggml_opencl: drop unsupported device.\\n\");\n continue;\n }\n\n dev_ctx.release();\n }\n\n if (found_devices.size()) {\n auto * dev_ctx = static_cast(found_devices.front().context);\n GGML_LOG_INFO(\"ggml_opencl: default device: '%s (%s)'\\n\", dev_ctx->device_name.c_str(),\n dev_ctx->device_version.c_str());\n\n if (dev_ctx->device_type != CL_DEVICE_TYPE_GPU) {\n GGML_LOG_WARN(\"ggml_opencl: warning, the default device is not a GPU: '%s'.\\n\",\n dev_ctx->device_name.c_str());\n }\n }\n\n return found_devices;\n}\n\n// Initialize device if it is supported (returns nullptr if it is not).\nstatic ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) {\n GGML_ASSERT(dev);\n GGML_ASSERT(dev->context);\n\n ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context;\n GGML_ASSERT(dev_ctx->platform);\n GGML_ASSERT(dev_ctx->device);\n\n if (dev_ctx->backend_ctx) {\n return dev_ctx->backend_ctx;\n }\n\n auto backend_ctx = std::make_unique();\n backend_ctx->device = dev_ctx->device;\n backend_ctx->gpu_family = GPU_FAMILY::UNKNOWN;\n\n // ref_count get increased in ggml_backend_opencl_device_init\n // This function is also used to retrieve backend context, so we don't want\n // to increase ref_count for each call. We only want to increase ref_count\n // when the associated device is initialized\n backend_ctx->ref_count = 0;\n\n if (strstr(dev_ctx->device_name.c_str(), \"Adreno\") ||\n strstr(dev_ctx->device_name.c_str(), \"Qualcomm\") ||\n strstr(dev_ctx->device_version.c_str(), \"Adreno\")) {\n backend_ctx->gpu_family = GPU_FAMILY::ADRENO;\n // Usually device version contains the detailed device name\n backend_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_version.c_str());\n if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN) {\n backend_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_name.c_str());\n }\n\n // Use wave size of 64 for all Adreno GPUs.\n backend_ctx->adreno_wave_size = 64;\n } else if (strstr(dev_ctx->device_name.c_str(), \"Intel\")) {\n backend_ctx->gpu_family = GPU_FAMILY::INTEL;\n } else {\n GGML_LOG_ERROR(\"Unsupported GPU: %s\\n\", dev_ctx->device_name.c_str());\n backend_ctx->gpu_family = GPU_FAMILY::UNKNOWN;\n return nullptr;\n }\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) {\n GGML_LOG_ERROR(\"ggml_opencl: Adreno-specific kernels should not be enabled for non-Adreno GPUs; \"\n \"run on an Adreno GPU or recompile with CMake option `-DGGML_OPENCL_USE_ADRENO_KERNELS=OFF`\\n\");\n return nullptr;\n }\n#endif\n\n // Populate backend device name\n backend_ctx->device_name = dev_ctx->device_name;\n\n // A local ref of cl_device_id for convenience\n cl_device_id device = backend_ctx->device;\n\n ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform);\n\n // Check device OpenCL version, OpenCL 2.0 or above is required\n ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, device);\n if (opencl_c_version.major < 2) {\n GGML_LOG_ERROR(\"ggml_opencl: OpenCL 2.0 or above is required\\n\");\n return nullptr;\n }\n\n // Check driver version\n size_t driver_version_str_size;\n clGetDeviceInfo(device, CL_DRIVER_VERSION, 0, NULL, &driver_version_str_size);\n char *driver_version = (char *)alloca(driver_version_str_size + 1);\n clGetDeviceInfo(device, CL_DRIVER_VERSION, driver_version_str_size, driver_version, NULL);\n driver_version[driver_version_str_size] = '\\0';\n GGML_LOG_INFO(\"ggml_opencl: OpenCL driver: %s\\n\", driver_version);\n backend_ctx->driver_version = driver_version;\n\n backend_ctx->adreno_cl_compiler_version = get_adreno_cl_compiler_version(driver_version);\n backend_ctx->has_vector_subgroup_broadcast =\n (backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) ||\n (backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17);\n GGML_LOG_INFO(\"ggml_opencl: vector subgroup broadcast support: %s\\n\",\n backend_ctx->has_vector_subgroup_broadcast ? \"true\" : \"false\");\n\n size_t ext_str_size;\n clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size);\n char *ext_buffer = (char *)alloca(ext_str_size + 1);\n clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer, NULL);\n ext_buffer[ext_str_size] = '\\0'; // ensure it is null terminated\n // Check if ext_buffer contains cl_khr_fp16\n backend_ctx->fp16_support = strstr(ext_buffer, \"cl_khr_fp16\") != NULL;\n GGML_LOG_INFO(\"ggml_opencl: device FP16 support: %s\\n\", backend_ctx->fp16_support ? \"true\" : \"false\");\n\n // fp16 is required\n if (!backend_ctx->fp16_support) {\n GGML_LOG_ERROR(\"ggml_opencl: device does not support FP16\\n\");\n return nullptr;\n }\n\n // If OpenCL 3.0 is supported, then check for cl_khr_subgroups, which becomes\n // optional in OpenCL 3.0 (cl_khr_subgroup is mandatory in OpenCL 2.x)\n if (opencl_c_version.major == 3 && strstr(ext_buffer, \"cl_khr_subgroups\") == NULL &&\n strstr(ext_buffer, \"cl_intel_subgroups\") == NULL) {\n GGML_LOG_ERROR(\"ggml_opencl: device does not support subgroups (cl_khr_subgroups or cl_intel_subgroups) \"\n \"(note that subgroups is an optional feature in OpenCL 3.0)\\n\");\n return nullptr;\n }\n\n cl_uint base_align_in_bits;\n CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &base_align_in_bits, NULL));\n GGML_ASSERT(base_align_in_bits % 8u == 0);\n backend_ctx->alignment = base_align_in_bits / 8u;\n GGML_LOG_INFO(\"ggml_opencl: mem base addr align: %u\\n\", backend_ctx->alignment);\n\n clGetDeviceInfo(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(size_t), &backend_ctx->max_alloc_size, NULL);\n GGML_LOG_INFO(\"ggml_opencl: max mem alloc size: %zu MB\\n\", backend_ctx->max_alloc_size/1024/1024);\n\n clGetDeviceInfo(device, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, sizeof(size_t), &backend_ctx->image_max_buffer_size, NULL);\n GGML_LOG_INFO(\"ggml_opencl: device max image buffer size (pixels): %lu\\n\", backend_ctx->image_max_buffer_size);\n\n clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), &backend_ctx->max_workgroup_size, NULL);\n GGML_LOG_INFO(\"ggml_opencl: device max workgroup size: %lu\\n\", backend_ctx->max_workgroup_size);\n\n // Check SVM.\n cl_device_svm_capabilities svm_caps;\n CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_SVM_CAPABILITIES, sizeof(cl_device_svm_capabilities), &svm_caps, 0));\n GGML_LOG_INFO(\"ggml_opencl: SVM coarse grain buffer support: %s\\n\",\n svm_caps & CL_DEVICE_SVM_COARSE_GRAIN_BUFFER ? \"true\" : \"false\");\n GGML_LOG_INFO(\"ggml_opencl: SVM fine grain buffer support: %s\\n\",\n svm_caps & CL_DEVICE_SVM_FINE_GRAIN_BUFFER ? \"true\" : \"false\");\n GGML_LOG_INFO(\"ggml_opencl: SVM fine grain system support: %s\\n\",\n svm_caps & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM ? \"true\" : \"false\");\n GGML_LOG_INFO(\"ggml_opencl: SVM atomics support: %s\\n\",\n svm_caps & CL_DEVICE_SVM_ATOMICS ? \"true\" : \"false\");\n\n if (opencl_c_version.major >= 3) {\n // Assume it is not available for 3.0, since it is optional in 3.0.\n // If compiling against 3.0, then we can query.\n backend_ctx->non_uniform_workgroups = false;\n#if CL_TARGET_OPENCL_VERSION >= 300\n CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_NON_UNIFORM_WORK_GROUP_SUPPORT, sizeof(cl_bool),\n &backend_ctx->non_uniform_workgroups, 0));\n#endif\n } else {\n GGML_ASSERT(opencl_c_version.major == 2);\n // Non-uniform workgroup sizes is mandatory feature in v2.x.\n backend_ctx->non_uniform_workgroups = true;\n }\n\n // Print out configurations\n#ifdef GGML_OPENCL_SOA_Q\n GGML_LOG_INFO(\"ggml_opencl: flattening quantized weights representation as struct of arrays (GGML_OPENCL_SOA_Q)\\n\");\n#endif // GGML_OPENCL_SOA_Q\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n GGML_LOG_INFO(\"ggml_opencl: using kernels optimized for Adreno (GGML_OPENCL_USE_ADRENO_KERNELS)\\n\");\n#endif // GGML_OPENCL_USE_ADRENO_KERNELS\n\n cl_int err;\n\n // A local ref of cl_context for convenience\n cl_context context = backend_ctx->context = dev_ctx->context;\n\n //CL_CHECK((queue = clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err),\n // (err != CL_INVALID_QUEUE_PROPERTIES && err != CL_INVALID_VALUE ? err :\n // (queue = clCreateCommandQueue(context, device, 0, &err), err)\n //)));\n cl_command_queue_properties command_queue_props = 0;\n#ifdef GGML_OPENCL_PROFILING\n command_queue_props |= CL_QUEUE_PROFILING_ENABLE;\n#endif\n CL_CHECK((backend_ctx->queue = clCreateCommandQueue(context, device, command_queue_props, &err), err));\n\n // Load kernels\n load_cl_kernels(backend_ctx.get(), opencl_c_version);\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n // Allocate intermediate buffers and images\n size_t required_A_q_d_bytes = 311164928;\n size_t required_A_s_d_bytes = 38895616;\n size_t required_B_d_bytes = 45088768;\n\n // Ensure buffer sizes do not exceed the maximum allocation size\n size_t max_A_q_d_bytes = MIN(required_A_q_d_bytes, backend_ctx->max_alloc_size);\n size_t max_A_s_d_bytes = MIN(required_A_s_d_bytes, backend_ctx->max_alloc_size);\n size_t max_B_d_bytes = MIN(required_B_d_bytes, backend_ctx->max_alloc_size);\n if (required_A_q_d_bytes > backend_ctx->max_alloc_size) {\n GGML_LOG_WARN(\"ggml_opencl: A_q_d buffer size reduced from %zu to %zu due to device limitations.\\n\",\n required_A_q_d_bytes, max_A_q_d_bytes);\n }\n if (required_A_s_d_bytes > backend_ctx->max_alloc_size) {\n GGML_LOG_WARN(\"ggml_opencl: A_s_d buffer size reduced from %zu to %zu due to device limitations.\\n\",\n required_A_s_d_bytes, max_A_s_d_bytes);\n }\n if (required_B_d_bytes > backend_ctx->max_alloc_size) {\n GGML_LOG_WARN(\"ggml_opencl: B_d buffer size reduced from %zu to %zu due to device limitations.\\n\",\n required_B_d_bytes, max_B_d_bytes);\n }\n\n backend_ctx->prealloc_quant_trans.allocate(context, max_A_q_d_bytes);\n backend_ctx->prealloc_scales_trans.allocate(context, max_A_s_d_bytes);\n backend_ctx->prealloc_act_trans.allocate(context, max_B_d_bytes);\n#endif // GGML_OPENCL_USE_ADRENO_KERNELS\n\n backend_ctx->disable_fusion = getenv(\"GGML_OPENCL_DISABLE_FUSION\") != nullptr;\n\n dev_ctx->backend_ctx = backend_ctx.release();\n return dev_ctx->backend_ctx;\n}\n\nstatic void ggml_cl2_free(ggml_backend_t backend) {\n ggml_backend_opencl_context * ctx = (ggml_backend_opencl_context *) backend->context;\n ctx->free();\n\n // The CL context is shared by all backends, release it if all backends have been released\n bool should_release_opencl = true;\n for (auto device : g_ggml_backend_opencl_devices) {\n ggml_backend_opencl_device_context * ctx_dev = (ggml_backend_opencl_device_context *) device.context;\n if (ctx_dev->backend_ctx->ref_count > 0) {\n should_release_opencl = false;\n }\n }\n\n if (should_release_opencl) {\n CL_CHECK(clReleaseContext(ctx->context));\n }\n}\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\nstatic void transpose_2d(\n ggml_backend_opencl_context * backend_ctx,\n cl_kernel kernel,\n cl_mem src, cl_mem dst, size_t size,\n cl_int stride, cl_int rows,\n bool blocking = true\n) {\n static ggml_cl_buffer buf;\n\n cl_event evt;\n cl_int err;\n\n buf.allocate(backend_ctx->context, size);\n\n cl_mem trans;\n cl_buffer_region region;\n\n region.origin = 0;\n region.size = size;\n CL_CHECK((trans = clCreateSubBuffer(\n buf.buffer, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &trans));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &stride));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &rows));\n\n size_t local_size[3] = {64, 1, 1};\n size_t global_size[3] = {(size_t)stride, (size_t)rows, 1};;\n CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL,\n global_size, local_size, 0, NULL, NULL));\n\n if (blocking) {\n CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clReleaseEvent(evt));\n } else {\n CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, NULL));\n }\n\n CL_CHECK(clReleaseMemObject(trans));\n}\n\nstatic void transpose_2d_as_8b(\n ggml_backend_opencl_context * backend_ctx,\n cl_mem src, cl_mem dst, size_t size,\n cl_int stride, cl_int rows,\n bool blocking = true\n) {\n transpose_2d(backend_ctx, backend_ctx->kernel_transpose_8_buf,\n src, dst, size, stride, rows, blocking);\n}\n\nstatic void transpose_2d_as_16b(\n ggml_backend_opencl_context * backend_ctx,\n cl_mem src, cl_mem dst, size_t size,\n cl_int stride, cl_int rows,\n bool blocking = true\n) {\n transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf,\n src, dst, size, stride, rows, blocking);\n}\n\nstatic void transpose_2d_as_32b(\n ggml_backend_opencl_context * backend_ctx,\n cl_mem src, cl_mem dst, size_t size,\n cl_int stride, cl_int rows,\n bool blocking = true\n) {\n transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf,\n src, dst, size, stride, rows, blocking);\n}\n#endif // GGML_OPENCL_USE_ADRENO_KERNELS\n\n//------------------------------------------------------------------------------\n// Tensor extra management\n//------------------------------------------------------------------------------\nstruct ggml_tensor_extra_cl {\n // The buffer object that holds the data.\n cl_mem data_device;\n // The offset into the buffer object. This is primarily for scratch buffer\n // and view operation.\n // NB: this offset no longer includes view offset (view_offs). Whenever this\n // offset is used, view_offs should be considered.\n cl_ulong offset;\n // The actual size of the cl_mem object. This is needed when returning the\n // block to the pool.\n size_t actual_size;\n\n void reset() {\n data_device = nullptr;\n offset = 0;\n actual_size = 0;\n }\n};\n\n// Additional tensor extra structs for quantized tensors.\n// These tensors are loaded from files and should not be allocated in scratch --\n// they should always be allocated from the pool. Hence, they do not have an\n// `offset`, which indicate their locations in the scratch buffer.\nstruct ggml_tensor_extra_cl_q4_0 {\n // Quantized values.\n cl_mem q = nullptr;\n // Quantized values in image1d_buffer_t.\n cl_mem q_img = nullptr;\n // Scales.\n cl_mem d = nullptr;\n // Scales in image1d_buffer_t.\n cl_mem d_img = nullptr;\n // Size of quantized values.\n size_t size_q = 0;\n // Size of scales.\n size_t size_d = 0;\n\n ~ggml_tensor_extra_cl_q4_0() {\n reset();\n }\n\n void reset() {\n // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer.\n // They must be properly released so that the original buffer can be\n // properly released to avoid memory leak.\n if (q != nullptr) {\n CL_CHECK(clReleaseMemObject(q));\n q = nullptr;\n }\n if (d != nullptr) {\n CL_CHECK(clReleaseMemObject(d));\n d = nullptr;\n }\n // Currently, q_img and d_img are only initialized when SMALL_ALLOC is\n // enabled. They point to the images in ggml_backend_opencl_buffer_context.\n // So, there is no need to release them here.\n // TODO: initialize them for non SMALL_PATH path, or remove them.\n q_img = nullptr;\n d_img = nullptr;\n size_q = 0;\n size_d = 0;\n }\n};\n\nstruct ggml_tensor_extra_cl_q4_1 {\n // Quantized values.\n cl_mem q = nullptr;\n // Quantized values in image1d_buffer_t.\n cl_mem q_img = nullptr;\n // Scales.\n cl_mem d = nullptr;\n // Scales in image1d_buffer_t.\n cl_mem d_img = nullptr;\n // Min\n cl_mem m = nullptr;\n // Min in image1d_buffer_t.\n cl_mem m_img = nullptr;\n // Size of quantized values.\n size_t size_q = 0;\n // Size of scales.\n size_t size_d = 0;\n // Size of min values.\n size_t size_m = 0;\n\n ~ggml_tensor_extra_cl_q4_1() {\n reset();\n }\n\n void reset() {\n // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer.\n // They must be properly released so that the original buffer can be\n // properly released to avoid memory leak.\n if (q != nullptr) {\n CL_CHECK(clReleaseMemObject(q));\n q = nullptr;\n }\n if (d != nullptr) {\n CL_CHECK(clReleaseMemObject(d));\n d = nullptr;\n }\n if (m != nullptr) {\n CL_CHECK(clReleaseMemObject(m));\n m = nullptr;\n }\n // Currently, q_img and d_img are only initialized when SMALL_ALLOC is\n // enabled. They point to the images in ggml_backend_opencl_buffer_context.\n // So, there is no need to release them here.\n // TODO: initialize them for non SMALL_PATH path, or remove them.\n q_img = nullptr;\n d_img = nullptr;\n m_img = nullptr;\n size_q = 0;\n size_d = 0;\n size_m = 0;\n }\n};\n\nstruct ggml_tensor_extra_cl_mxfp4 {\n // Quantized values.\n cl_mem q = nullptr;\n // Quantized values in image1d_buffer_t.\n cl_mem q_img = nullptr;\n // Scales in E8M0.\n cl_mem e = nullptr;\n // Scales in image1d_buffer_t.\n cl_mem e_img = nullptr;\n // Size of quantized values.\n size_t size_q = 0;\n // Size of scales.\n size_t size_e = 0;\n\n ~ggml_tensor_extra_cl_mxfp4() {\n reset();\n }\n\n void reset() {\n // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer.\n // They must be properly released so that the original buffer can be\n // properly released to avoid memory leak.\n if (q != nullptr) {\n CL_CHECK(clReleaseMemObject(q));\n q = nullptr;\n }\n if (e != nullptr) {\n CL_CHECK(clReleaseMemObject(e));\n e = nullptr;\n }\n if (q != nullptr) {\n CL_CHECK(clReleaseMemObject(q_img));\n q = nullptr;\n }\n // Currently, q_img and d_img are not used. They can be image1d_buffer_t\n // that wraps around q and d to utilize image access path.\n q_img = nullptr;\n e_img = nullptr;\n size_q = 0;\n size_e = 0;\n }\n};\n\nstruct ggml_tensor_extra_cl_q8_0 {\n cl_mem q = nullptr;\n cl_mem q_img = nullptr;\n\n cl_mem d = nullptr;\n cl_mem d_img = nullptr;\n\n size_t size_q = 0;\n size_t size_d = 0;\n\n ~ggml_tensor_extra_cl_q8_0() {\n reset();\n }\n\n void reset() {\n // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer.\n // They must be properly released so that the original buffer can be\n // properly released to avoid memory leak.\n if (q != nullptr) {\n CL_CHECK(clReleaseMemObject(q));\n q = nullptr;\n }\n if (d != nullptr) {\n CL_CHECK(clReleaseMemObject(d));\n d = nullptr;\n }\n // Currently, q_img and d_img are not used. They can be image1d_buffer_t\n // that wraps around q and d to utilize image access path.\n q_img = nullptr;\n d_img = nullptr;\n size_q = 0;\n size_d = 0;\n }\n};\n\nstruct ggml_tensor_extra_cl_q6_K {\n // Lower 4 bits of quantized weights.\n cl_mem ql = nullptr;\n // Upper 2 bits of quantized weights.\n cl_mem qh = nullptr;\n // Scales for each block.\n cl_mem s = nullptr;\n // Scales for each super block.\n cl_mem d = nullptr;\n\n size_t size_ql = 0;\n size_t size_qh = 0;\n size_t size_s = 0;\n size_t size_d = 0;\n\n ~ggml_tensor_extra_cl_q6_K() {\n reset();\n }\n\n void reset() {\n if (ql != nullptr) {\n CL_CHECK(clReleaseMemObject(ql));\n ql = nullptr;\n }\n if (qh != nullptr) {\n CL_CHECK(clReleaseMemObject(qh));\n qh = nullptr;\n }\n if (s != nullptr) {\n CL_CHECK(clReleaseMemObject(s));\n s = nullptr;\n }\n if (d != nullptr) {\n CL_CHECK(clReleaseMemObject(d));\n d = nullptr;\n }\n\n size_ql = 0;\n size_qh = 0;\n size_s = 0;\n size_d = 0;\n }\n};\n\n//------------------------------------------------------------------------------\n// Backend API\n//------------------------------------------------------------------------------\n\n//\n// backend\n//\nstatic const char * ggml_backend_opencl_name(ggml_backend_t backend) {\n return \"OpenCL\";\n\n UNUSED(backend);\n}\n\nstatic void ggml_backend_opencl_free(ggml_backend_t backend) {\n ggml_cl2_free(backend);\n}\n\nstatic void ggml_backend_opencl_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n GGML_UNUSED(backend);\n GGML_UNUSED(tensor);\n GGML_UNUSED(data);\n GGML_UNUSED(offset);\n GGML_UNUSED(size);\n}\n\nstatic void ggml_backend_opencl_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n GGML_UNUSED(backend);\n GGML_UNUSED(tensor);\n GGML_UNUSED(data);\n GGML_UNUSED(offset);\n GGML_UNUSED(size);\n}\n\nstatic bool ggml_backend_opencl_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) {\n GGML_UNUSED(backend);\n GGML_UNUSED(src);\n GGML_UNUSED(dst);\n return false;\n}\n\nstatic void ggml_backend_opencl_synchronize(ggml_backend_t backend) {\n auto * backend_ctx = static_cast(backend->context);\n\n cl_event evt;\n CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, 0, nullptr, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clReleaseEvent(evt));\n}\n\n// Synchronizes the 'backend_ctx's device with others so that commands\n// enqueued to it won't start until commands in the other devices have\n// completed.\nstatic void sync_with_other_backends(ggml_backend_opencl_context * backend_ctx) {\n if (g_ggml_backend_opencl_devices.size() < 2)\n return; // No other devices to synchronize with.\n\n std::vector events;\n events.reserve(g_ggml_backend_opencl_devices.size());\n\n for (ggml_backend_device & backend_dev : g_ggml_backend_opencl_devices) {\n auto * other_backend_ctx = ggml_cl2_init(&backend_dev);\n if (backend_ctx != other_backend_ctx) {\n cl_event ev;\n CL_CHECK(clEnqueueMarkerWithWaitList(other_backend_ctx->queue, 0, nullptr, &ev));\n CL_CHECK(clFlush(other_backend_ctx->queue));\n events.push_back(ev);\n }\n }\n\n CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, events.size(), events.data(), nullptr));\n for (auto ev : events) {\n CL_CHECK(clReleaseEvent(ev));\n }\n}\n\nstatic void sync_with_other_backends(ggml_backend_t backend) {\n auto * backend_ctx = static_cast(backend->context);\n sync_with_other_backends(backend_ctx);\n}\n\nstatic bool ggml_opencl_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) {\n if (!ggml_can_fuse(cgraph, node_idx, ops)) {\n return false;\n }\n\n if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) {\n const ggml_tensor *rms_norm = cgraph->nodes[node_idx];\n const ggml_tensor *mul = cgraph->nodes[node_idx+1];\n\n GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(rms_norm->type == GGML_TYPE_F32);\n\n // rms_norm only supports f32\n if (mul->src[0]->type != GGML_TYPE_F32 ||\n mul->src[1]->type != GGML_TYPE_F32 ||\n mul->type != GGML_TYPE_F32) {\n return false;\n }\n\n // if rms_norm is the B operand, then we don't handle broadcast\n if (rms_norm == mul->src[1] &&\n !ggml_are_same_shape(mul->src[0], rms_norm)) {\n return false;\n }\n\n // rms_norm assumes contiguous rows\n if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) {\n return false;\n }\n } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) {\n const ggml_tensor *norm = cgraph->nodes[node_idx];\n const ggml_tensor *mul = cgraph->nodes[node_idx+1];\n const ggml_tensor *add = cgraph->nodes[node_idx+2];\n const ggml_tensor *w = mul->src[0] == norm ? mul->src[1] : mul->src[0];\n const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0];\n\n // norm fusion only supports F32\n if (norm->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) {\n return false;\n }\n\n if (norm->src[0]->ne[0] % 4 != 0) {\n return false;\n }\n\n if (!ggml_is_contiguous(norm->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) {\n return false;\n }\n } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_GROUP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) {\n const ggml_tensor *gn = cgraph->nodes[node_idx];\n const ggml_tensor *mul = cgraph->nodes[node_idx+1];\n const ggml_tensor *add = cgraph->nodes[node_idx+2];\n const ggml_tensor *w = mul->src[0] == gn ? mul->src[1] : mul->src[0];\n const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0];\n\n if (gn->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) {\n return false;\n }\n\n if (!ggml_is_contiguous(gn->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) {\n return false;\n }\n }\n\n return true;\n}\n\nstatic void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor);\nstatic void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor);\nstatic void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor);\n\nstatic ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n for (int i = 0; i < cgraph->n_nodes; i++) {\n ggml_tensor * node = cgraph->nodes[i];\n\n // NOTE: this may oversynchronize by synchronizing with\n // backends/devices which don't compute 'cgraph's\n // dependencies.\n sync_with_other_backends(backend);\n\n if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {\n continue;\n }\n\n if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n continue;\n }\n\n if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {\n ggml_opencl_op_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]);\n i += 2;\n continue;\n }\n if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {\n ggml_opencl_op_group_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]);\n i += 2;\n continue;\n }\n if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {\n ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]);\n i++;\n continue;\n }\n\n bool ok = ggml_cl_compute_forward(backend, node);\n if (!ok) {\n GGML_LOG_ERROR(\"%s: error: op not supported %s (%s)\\n\", __func__, node->name, ggml_op_name(node->op));\n }\n GGML_ASSERT(ok);\n }\n\n return GGML_STATUS_SUCCESS;\n}\n\nstatic bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {\n ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *)dev->context;\n ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx;\n\n switch (op->op) {\n case GGML_OP_NONE:\n return true;\n case GGML_OP_GET_ROWS:\n switch (op->src[0]->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n return true;\n case GGML_TYPE_Q4_0:\n#ifdef GGML_OPENCL_SOA_Q\n // We do not support flattened Q4_0 (and possibly other Q's)\n return false;\n#else // GGML_OPENCL_SOA_Q\n return true;\n#endif // GGML_OPENCL_SOA_Q\n default:\n return false;\n }\n case GGML_OP_SET_ROWS:\n {\n // TODO: add support\n // ref: https://github.com/ggml-org/llama.cpp/pull/14274\n#pragma message(\"TODO: implement BF16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, IQ4_NL support (https://github.com/ggml-org/llama.cpp/pull/14661)\")\n if (op->src[0]->type != GGML_TYPE_F32) {\n return false;\n }\n switch (op->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_F32:\n return (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32);\n default:\n return false;\n }\n }\n case GGML_OP_CPY:\n case GGML_OP_DUP:\n case GGML_OP_CONT:\n switch (op->src[0]->type) {\n case GGML_TYPE_F32:\n switch (op->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_F32:\n return true;\n default:\n return false;\n }\n case GGML_TYPE_F16:\n switch (op->type) {\n case GGML_TYPE_F16:\n case GGML_TYPE_F32:\n return true;\n default:\n return false;\n }\n case GGML_TYPE_I32:\n switch (op->type) {\n case GGML_TYPE_I32:\n return true;\n default:\n return false;\n }\n default:\n return false;\n }\n case GGML_OP_SET: {\n return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32) &&\n op->type == op->src[0]->type &&\n op->type == op->src[1]->type;\n }\n case GGML_OP_SCALE:\n return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);\n case GGML_OP_ADD:\n if (op->type == GGML_TYPE_F16) {\n const bool src0_ok = op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32;\n const bool src1_ok = op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32;\n if (src0_ok && src1_ok) {\n return true;\n }\n }\n case GGML_OP_MUL:\n case GGML_OP_DIV:\n case GGML_OP_SUB:\n return (op->src[0]->type == op->src[1]->type) &&\n (op->src[0]->type == op->type) &&\n (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);\n case GGML_OP_ADD_ID:\n return op->src[0]->type == GGML_TYPE_F32;\n case GGML_OP_SQR:\n case GGML_OP_SQRT:\n return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&\n ggml_is_contiguous(op->src[0]);\n case GGML_OP_UNARY:\n switch (ggml_get_unary_op(op)) {\n case GGML_UNARY_OP_GELU:\n case GGML_UNARY_OP_SILU:\n case GGML_UNARY_OP_RELU:\n case GGML_UNARY_OP_GELU_ERF:\n case GGML_UNARY_OP_GELU_QUICK:\n return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;\n case GGML_UNARY_OP_SIGMOID:\n return ggml_is_contiguous(op->src[0]);\n case GGML_UNARY_OP_TANH:\n case GGML_UNARY_OP_NEG:\n case GGML_UNARY_OP_EXP:\n return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;\n case GGML_UNARY_OP_EXPM1:\n return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;\n case GGML_UNARY_OP_SOFTPLUS:\n return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;\n default:\n return false;\n }\n case GGML_OP_GLU:\n switch (ggml_get_glu_op(op)) {\n case GGML_GLU_OP_GEGLU:\n case GGML_GLU_OP_REGLU:\n case GGML_GLU_OP_SWIGLU:\n case GGML_GLU_OP_SWIGLU_OAI:\n case GGML_GLU_OP_GEGLU_ERF:\n case GGML_GLU_OP_GEGLU_QUICK:\n return ggml_is_contiguous_1(op->src[0]) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);\n default:\n return false;\n }\n case GGML_OP_TRI:\n return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op);\n case GGML_OP_FILL:\n return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op);\n case GGML_OP_CLAMP:\n return op->src[0]->type == GGML_TYPE_F32;\n case GGML_OP_SOFT_MAX:\n case GGML_OP_NORM:\n return true;\n case GGML_OP_RMS_NORM:\n return op->ne[0] % 4 == 0 && ggml_is_contiguous_rows(op->src[0]);\n case GGML_OP_L2_NORM:\n return ggml_is_contiguous_rows(op->src[0]);\n case GGML_OP_REPEAT:\n return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; // Assuming F32 for now, can be expanded\n case GGML_OP_PAD:\n // TODO: add circular padding support for opencl, see https://github.com/ggml-org/llama.cpp/pull/16985\n if (ggml_get_op_params_i32(op, 8) != 0) {\n return false;\n }\n return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;\n case GGML_OP_UPSCALE: {\n ggml_scale_mode mode = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & 0xFF);\n const bool antialias = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & GGML_SCALE_FLAG_ANTIALIAS);\n return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 &&\n (mode == GGML_SCALE_MODE_NEAREST || mode == GGML_SCALE_MODE_BILINEAR) && !antialias;\n }\n case GGML_OP_CONV_2D:\n return (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16) ||\n (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) ||\n (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32);\n case GGML_OP_SSM_CONV:\n return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32);\n case GGML_OP_CONCAT:\n return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;\n case GGML_OP_TIMESTEP_EMBEDDING:\n return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;\n case GGML_OP_GROUP_NORM:\n return ggml_is_contiguous(op->src[0]);\n case GGML_OP_MUL_MAT:\n if (op->src[0]->type == GGML_TYPE_F16) {\n return true;\n } else if (op->src[0]->type == GGML_TYPE_F32) {\n return op->src[1]->type == GGML_TYPE_F32;\n } else if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_1 ||\n op->src[0]->type == GGML_TYPE_MXFP4 ||\n op->src[0]->type == GGML_TYPE_Q4_K ||\n op->src[0]->type == GGML_TYPE_Q6_K) {\n return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);\n } else if (op->src[0]->type == GGML_TYPE_Q8_0) {\n return op->src[1]->type == GGML_TYPE_F32;\n }\n return false;\n case GGML_OP_MUL_MAT_ID:\n if (op->src[0]->type == GGML_TYPE_Q4_0 ||\n op->src[0]->type == GGML_TYPE_Q8_0 ||\n op->src[0]->type == GGML_TYPE_MXFP4) {\n if (op->src[1]->type == GGML_TYPE_F32) {\n return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);\n }\n }\n return false;\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n return true;\n case GGML_OP_DIAG:\n return true;\n case GGML_OP_DIAG_MASK_INF:\n return op->ne[3] == 1;\n case GGML_OP_ROPE: {\n const int mode = ((const int32_t *) op->op_params)[2];\n const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;\n const bool is_vision = mode == GGML_ROPE_TYPE_VISION;\n if (is_mrope && !is_vision) {\n if (op->src[0]->type == GGML_TYPE_F32 ||\n op->src[0]->type == GGML_TYPE_F16) {\n return true;\n }\n return false;\n }\n if (is_vision) {\n if (op->src[0]->type == GGML_TYPE_F32 ||\n op->src[0]->type == GGML_TYPE_F16) {\n return true;\n }\n return false;\n }\n return true;\n }\n case GGML_OP_SOLVE_TRI:\n return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);\n case GGML_OP_IM2COL:\n return true;\n case GGML_OP_ARGSORT: {\n cl_kernel kernel = backend_ctx->kernel_argsort_f32_i32;\n int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel);\n\n int cols = 1;\n while (cols < op->ne[0]) {\n cols *= 2;\n }\n\n return cols <= max_workgroup_size && op->src[0]->type == GGML_TYPE_F32;\n }\n case GGML_OP_SUM_ROWS:\n case GGML_OP_CUMSUM:\n return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);\n case GGML_OP_MEAN:\n return op->src[0]->type == GGML_TYPE_F32;\n case GGML_OP_FLASH_ATTN_EXT:\n {\n const ggml_tensor * q = op->src[0];\n const ggml_tensor * k = op->src[1];\n const ggml_tensor * v = op->src[2];\n\n const int dk = q->ne[0];\n const int dv = v->ne[0];\n\n const struct { int dk; int dv; } supported_dims[] = {\n { 40, 40}, { 64, 64}, { 80, 80}, { 96, 96},\n {112, 112}, {128, 128}, {192, 128},\n {192, 192}, {256, 256},\n };\n\n bool dims_supported = false;\n for (size_t i = 0; i < sizeof(supported_dims)/sizeof(supported_dims[0]); ++i) {\n if (supported_dims[i].dk == dk && supported_dims[i].dv == dv) {\n dims_supported = true;\n break;\n }\n }\n if (!dims_supported) {\n return false;\n }\n\n const bool is_f32_f32 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 &&\n v->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;\n const bool is_f16_f16 = q->type == GGML_TYPE_F16 && k->type == GGML_TYPE_F16 &&\n v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16;\n const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 &&\n v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32;\n\n return is_f32_f32 || is_f16_f16 || is_f32_f16;\n }\n default:\n return false;\n }\n}\n\n// Forward declaration - implementation appears later in the file.\nstatic const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type);\n\nstatic ggml_guid_t ggml_backend_opencl_guid() {\n static ggml_guid guid = { 0xde, 0xe0, 0x70, 0xa2, 0x73, 0x4e, 0x4d, 0xbc, 0xb0, 0xc7, 0x4f, 0xd4, 0x6d, 0x4e, 0x90, 0xfe };\n return &guid;\n}\n\nstatic ggml_backend_i ggml_backend_opencl_i = {\n /* .get_name = */ ggml_backend_opencl_name,\n /* .free = */ ggml_backend_opencl_free,\n /* .set_tensor_async = */ NULL, /* ggml_backend_opencl_set_tensor_async */\n /* .get_tensor_async = */ NULL, /* ggml_backend_opencl_get_tensor_async */\n /* .cpy_tensor_async = */ NULL, /* ggml_backend_opencl_cpy_tensor_async */\n /* .synchronize = */ ggml_backend_opencl_synchronize,\n /* .graph_plan_create = */ NULL,\n /* .graph_plan_free = */ NULL,\n /* .graph_plan_update = */ NULL,\n /* .graph_plan_compute = */ NULL,\n /* .graph_compute = */ ggml_backend_opencl_graph_compute,\n /* .event_record = */ NULL,\n /* .event_wait = */ NULL,\n /* .graph_optimize = */ NULL,\n};\n\nggml_backend_t ggml_backend_opencl_init(void) {\n ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_opencl_reg(), 0);\n ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(dev);\n\n ggml_backend_t backend = new ggml_backend {\n /* .guid = */ ggml_backend_opencl_guid(),\n /* .iface = */ ggml_backend_opencl_i,\n /* .device = */ dev,\n /* .context = */ backend_ctx\n };\n\n return backend;\n}\n\nbool ggml_backend_is_opencl(ggml_backend_t backend) {\n return backend && backend->iface.get_name == ggml_backend_opencl_name;\n}\n\n//\n// buffer\n//\nstruct ggml_backend_opencl_buffer_context {\n // A buffer context can hold multiple cl_mem objects. This is for flattening\n // quantized weights and should be used with GGML_OPENCL_SMALL_ALLOC where\n // each tensor is allocated a separate buffer. When flattening is enabled\n // with small allocation, each tensor is backed by two cl_mem objects (for\n // quants and scales) packed into a backend_opencl_buffer.\n ggml_backend_opencl_buffer_context(cl_mem buf)\n : name(\"OpenCL\") {\n buffer.push_back(buf);\n }\n\n ~ggml_backend_opencl_buffer_context() {\n for (cl_mem buf : buffer) {\n CL_CHECK(clReleaseMemObject(buf));\n }\n for (cl_mem im : img) {\n CL_CHECK(clReleaseMemObject(im));\n }\n\n // Delete all extras to trigger their destructors\n for (ggml_tensor_extra_cl * e : temp_tensor_extras) {\n delete e;\n }\n for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) {\n delete e;\n }\n for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0) {\n delete e;\n }\n for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) {\n delete e;\n }\n for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4) {\n delete e;\n }\n for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) {\n delete e;\n }\n for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0) {\n delete e;\n }\n for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) {\n delete e;\n }\n for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K) {\n delete e;\n }\n for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) {\n delete e;\n }\n }\n\n ggml_tensor_extra_cl * ggml_opencl_alloc_temp_tensor_extra() {\n ggml_tensor_extra_cl * extra;\n if (temp_tensor_extras.empty()) {\n extra = new ggml_tensor_extra_cl();\n } else {\n extra = temp_tensor_extras.back();\n temp_tensor_extras.pop_back();\n }\n\n temp_tensor_extras_in_use.push_back(extra);\n\n extra->reset();\n return extra;\n }\n\n ggml_tensor_extra_cl_q4_0 * ggml_opencl_alloc_temp_tensor_extra_q4_0() {\n ggml_tensor_extra_cl_q4_0 * extra;\n if (temp_tensor_extras_q4_0.empty()) {\n extra = new ggml_tensor_extra_cl_q4_0();\n } else {\n extra = temp_tensor_extras_q4_0.back();\n temp_tensor_extras_q4_0.pop_back();\n }\n\n temp_tensor_extras_q4_0_in_use.push_back(extra);\n\n extra->reset();\n return extra;\n }\n\n ggml_tensor_extra_cl_q4_1 * ggml_opencl_alloc_temp_tensor_extra_q4_1() {\n ggml_tensor_extra_cl_q4_1 * extra;\n if (temp_tensor_extras_q4_1.empty()) {\n extra = new ggml_tensor_extra_cl_q4_1();\n } else {\n extra = temp_tensor_extras_q4_1.back();\n temp_tensor_extras_q4_1.pop_back();\n }\n\n temp_tensor_extras_q4_1_in_use.push_back(extra);\n\n extra->reset();\n return extra;\n }\n\n ggml_tensor_extra_cl_mxfp4 * ggml_opencl_alloc_temp_tensor_extra_mxfp4() {\n ggml_tensor_extra_cl_mxfp4 * extra;\n if (temp_tensor_extras_mxfp4.empty()) {\n extra = new ggml_tensor_extra_cl_mxfp4();\n } else {\n extra = temp_tensor_extras_mxfp4.back();\n temp_tensor_extras_mxfp4.pop_back();\n }\n\n temp_tensor_extras_mxfp4_in_use.push_back(extra);\n\n extra->reset();\n return extra;\n }\n\n ggml_tensor_extra_cl_q8_0 * ggml_opencl_alloc_temp_tensor_extra_q8_0() {\n ggml_tensor_extra_cl_q8_0 * extra;\n if (temp_tensor_extras_q8_0.empty()) {\n extra = new ggml_tensor_extra_cl_q8_0();\n } else {\n extra = temp_tensor_extras_q8_0.back();\n temp_tensor_extras_q8_0.pop_back();\n }\n\n temp_tensor_extras_q8_0_in_use.push_back(extra);\n\n extra->reset();\n return extra;\n }\n\n ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() {\n ggml_tensor_extra_cl_q6_K * extra;\n if (temp_tensor_extras_q6_K.empty()) {\n extra = new ggml_tensor_extra_cl_q6_K();\n } else {\n extra = temp_tensor_extras_q6_K.back();\n temp_tensor_extras_q6_K.pop_back();\n }\n\n temp_tensor_extras_q6_K_in_use.push_back(extra);\n\n extra->reset();\n return extra;\n }\n\n void reset() {\n for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) {\n temp_tensor_extras.push_back(e);\n }\n temp_tensor_extras_in_use.clear();\n\n for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) {\n temp_tensor_extras_q4_0.push_back(e);\n }\n temp_tensor_extras_q4_0_in_use.clear();\n\n for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) {\n temp_tensor_extras_q4_1.push_back(e);\n }\n temp_tensor_extras_q4_1_in_use.clear();\n\n for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) {\n temp_tensor_extras_mxfp4.push_back(e);\n }\n temp_tensor_extras_mxfp4_in_use.clear();\n\n for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) {\n temp_tensor_extras_q8_0.push_back(e);\n }\n temp_tensor_extras_q8_0_in_use.clear();\n\n for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) {\n temp_tensor_extras_q6_K.push_back(e);\n }\n temp_tensor_extras_q6_K_in_use.clear();\n }\n\n // Pools for extras. Available extras are in `temp_tensor_extras`. Extras\n // being used are in `temp_tensor_extras_in_use`. At the first run, new\n // extras get created and put in `in_use`. When the buffer is reset via\n // the `reset` callback, all extras in `in_use` get moved to available extras\n // for reuse.\n std::vector temp_tensor_extras;\n std::vector temp_tensor_extras_in_use;\n std::vector temp_tensor_extras_q4_0;\n std::vector temp_tensor_extras_q4_0_in_use;\n std::vector temp_tensor_extras_q4_1;\n std::vector temp_tensor_extras_q4_1_in_use;\n std::vector temp_tensor_extras_mxfp4;\n std::vector temp_tensor_extras_mxfp4_in_use;\n std::vector temp_tensor_extras_q8_0;\n std::vector temp_tensor_extras_q8_0_in_use;\n std::vector temp_tensor_extras_q6_K;\n std::vector temp_tensor_extras_q6_K_in_use;\n\n // The buffer_context is initially created by ggml_backend_buft_alloc_buffer\n // before any tensor is initialized (at the beginning of alloc_tensor_range).\n // Hence, there is always a buffer object in this vector. When each tensor is\n // being initialized, this original buffer object will be released if both\n // flattening and small allocation are enabled, and additional buffer\n // objects will be created in init_tensor to represent flattened quantized\n // weights.\n std::vector buffer;\n // These are image1d_buffer_t objects that wrap around the quants and scales.\n // For Q4_0 quantization, there should be two of them - one for quants and\n // one for scales. They should be populated only when flattening and small\n // allocation are enabled.\n std::vector img;\n std::string name;\n};\n\nstatic void ggml_backend_opencl_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;\n delete ctx;\n}\n\nstatic void * ggml_backend_opencl_buffer_get_base(ggml_backend_buffer_t buffer) {\n ggml_backend_opencl_context * backend_ctx = ggml_cl2_init(buffer->buft->device);\n return (void *) (uintptr_t) backend_ctx->alignment;\n}\n\nstatic enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {\n ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;\n\n ggml_cl2_init(buffer->buft->device);\n\n if (tensor->view_src != nullptr) {\n GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);\n\n ggml_tensor_extra_cl * view_extra = (ggml_tensor_extra_cl *) tensor->view_src->extra;\n GGML_ASSERT(view_extra && \"view_extra is nullptr?\");\n\n // Reuse extra of the parent tensor. The offset of this view tensor\n // becomes `extra->offset + view_offs` and needs to be calculated when\n // it is used. This changes is needed because of the change to\n // ggml_alloc.c in https://github.com/ggml-org/llama.cpp/pull/7640.\n // `buffer` passed in here will always be `tensor->buffer`. It is OK\n // to allocate extras from the same buffer context for ordinary\n // intermediate tensors. But for views into kv cache tensors, doing so\n // would mess up the extras used by kv cache.\n // Before #7640, `buffer` is for intermediate tensors, which is always\n // different from that of kv cache tensors.\n //\n // NB: now extra->offset no longer accounts for view_offs.\n // NB: this should not apply to weight tensors (for end-to-end runs, but\n // may apply for test-backend-ops).\n // FIXME: if any unexpected results are seen, double check the offset -\n // there could be other places that need fix.\n tensor->extra = view_extra;\n } else {\n {\n size_t offset = (char *) tensor->data - (char *) ggml_backend_opencl_buffer_get_base(buffer);\n\n ggml_tensor_extra_cl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra();\n extra->offset = offset;\n extra->data_device = ctx->buffer[0];\n extra->actual_size = ggml_nbytes(tensor);\n\n tensor->extra = extra;\n }\n }\n return GGML_STATUS_SUCCESS;\n}\n\n// The optimized gemm and gemv kernels are used for large matrices without batch.\n// tensor is the quantized weights matrix.\ninline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {\n int64_t threshold_ne0 = 512;\n int64_t threshold_ne1 = 512;\n if (!backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) &&\n backend_ctx->adreno_cl_compiler_version.type != DX) {\n threshold_ne0 = 128;\n threshold_ne1 = 128;\n }\n return tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 &&\n tensor->ne[2] == 1 && tensor->ne[3] == 1;\n}\n\ninline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {\n GGML_UNUSED(backend_ctx);\n int ne01 = tensor->ne[1];\n return ((strstr(tensor->name, \"ffn\") != NULL) || (strstr(tensor->name, \"as\") != NULL)) && (ne01 % 64 == 0);\n}\n\ninline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {\n\n bool adreno_kernel = use_adreno_kernels(backend_ctx, tensor);\n\n size_t elem_num = tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3];\n\n return ((elem_num < 128 * 1024 * 1024) && adreno_kernel); // max element num: 2**27\n}\n\nstatic void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(buffer->buft->device);\n\n cl_context context = backend_ctx->context;\n cl_command_queue queue = backend_ctx->queue;\n\n#ifdef GGML_OPENCL_SOA_Q\n // We separate the quantized bits and scale from block_q4_0 by using an\n // additional kernel, where each thread handles a block. We first read the\n // original weights into a temporary buffer, then create two separate\n // buffers for quantized bits and scales, which are then populated by the\n // conversion kernel.\n if (tensor->type == GGML_TYPE_Q4_0) {\n // Tensors should have been preallocated, therefore they should\n // already have ggml_tensor_extra_cl as extra.\n ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;\n GGML_ASSERT(extra_orig && \"Tesnors in OpenCL backend should have been allocated and initialized\");\n\n // Allocate the new extra and create aliases from the original.\n ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;\n ggml_tensor_extra_cl_q4_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_0();\n\n size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);\n size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2;\n GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && \"Incorrect tensor size\");\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n CL_CHECK(clEnqueueWriteBuffer(\n queue, data_device, CL_TRUE, 0,\n ggml_nbytes(tensor), data, 0, NULL, NULL));\n\n // We consider the specified offset arg as always, although For weights\n // the offset arg should be 0 (we do not assert this).\n //GGML_ASSERT(offset == 0);\n\n // We create subbuffers from the original tensor buffer for scales and\n // quants - i.e., scales and quants are aliases into the buffer object\n // that backs the original tensor. This is a cleaner way to adapt to the\n // new memory management.\n // In the old code, we allocate new buffers for scales and quants\n // respectively, which could still be done but would result in double\n // allocation; properly deallocating the preallocated buffer that backs\n // the tensors is tricky and would leak the backend specific information\n // into the general backend code.\n // Does this create misaligned subbuffers (alignment is 1024) in certain\n // cases ?\n cl_buffer_region region;\n\n // The original tensor memory is divided into scales and quants, i.e.,\n // we first store scales, then quants.\n // Create subbuffer for scales.\n region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);\n region.size = size_d;\n extra->d = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n auto previous_origin = region.origin;\n\n // Create subbuffer for quants.\n region.origin = align_to(previous_origin + size_d, backend_ctx->alignment);\n region.size = size_q;\n extra->q = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n\n //cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0;\n #ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0;\n\n // The optimized kernels need weights in natural order, so unshuffle.\n if (use_adreno_kernels(backend_ctx, tensor)) {\n kernel = backend_ctx->kernel_convert_block_q4_0_noshuffle;\n }\n #else\n cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0;\n #endif // GGML_OPENCL_USE_ADRENO_KERNELS\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d));\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clReleaseMemObject(data_device));\n\n tensor->extra = extra;\n\n // transpose the weights and scales\n #ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n // Only do transpose for large, non batched matrix\n // TODO: use preallocated images instead of sub-buffer then image\n if (use_adreno_kernels(backend_ctx, tensor)) {\n // <----------------------------------------------------------------------------------> //\n // start transpose\n // <----------------------------------------------------------------------------------> //\n int M = tensor->ne[1]; // ne01\n int K = tensor->ne[0]; // ne00\n\n //For matrix-vector multiplication kernel, we assume K is a multiple of 32\n GGML_ASSERT(K % 32 == 0);\n //For transpose kernels, we assume K is a multiple of 4 (satisfied by prior assert), and M is a multiple of 4\n GGML_ASSERT(M % 4 == 0);\n\n // transpose is out of place, so we need to allocate transposed buffers\n // <----------------------------------------------------------------------------------> //\n // use sub_buffer of max buffer size instead\n\n size_t q_size_bytes = K * M / 8 * sizeof(float);\n backend_ctx->prealloc_quant_trans.allocate(context, q_size_bytes);\n\n cl_buffer_region region;\n region.origin = 0;\n region.size = q_size_bytes;\n cl_mem qT_d = clCreateSubBuffer(\n backend_ctx->prealloc_quant_trans.buffer,\n 0,\n CL_BUFFER_CREATE_TYPE_REGION,\n ®ion,\n &err);\n CL_CHECK(err);\n\n bool K_tile_trans = true;\n if ((K / 32) % 4 != 0){\n K_tile_trans =false;\n }\n\n size_t d_size_bytes = M * (K / 32) * 2;\n backend_ctx->prealloc_scales_trans.allocate(context, d_size_bytes);\n\n region.origin = 0;\n region.size = d_size_bytes;\n cl_mem dT_d = clCreateSubBuffer(\n backend_ctx->prealloc_scales_trans.buffer,\n 0,\n CL_BUFFER_CREATE_TYPE_REGION,\n ®ion,\n &err);\n CL_CHECK(err);\n\n // <----------------------------------------------------------------------------------> //\n\n\n // create images from the buffers\n // <----------------------------------------------------------------------------------> //\n cl_mem q_d_image1D;\n cl_mem d_d_image1D;\n cl_mem qT_d_image1D;\n cl_mem dT_d_image1D;\n\n cl_image_format img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT };\n cl_image_desc img_desc_1d;\n\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * K / 4 / 4;\n img_desc_1d.buffer = extra->q;\n q_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err);\n CL_CHECK(err);\n\n img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT };\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * K / 4 / 4;\n img_desc_1d.buffer = qT_d;\n qT_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err);\n CL_CHECK(err);\n\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n if (K_tile_trans) {\n img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT };\n img_desc_1d.image_width = M * K / 32 / 4;\n } else {\n img_fmt_1d = { CL_R, CL_HALF_FLOAT };\n img_desc_1d.image_width = M * K / 32;\n }\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.buffer = extra->d;\n d_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err);\n CL_CHECK(err);\n\n img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT };\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * K / 32 / 4;\n img_desc_1d.buffer = dT_d;\n dT_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err);\n CL_CHECK(err);\n // <----------------------------------------------------------------------------------> //\n\n // set up and call the transpose kernels\n // <----------------------------------------------------------------------------------> //\n // weights\n int height_q = M / 4;\n int width_q = K / 4 / 4;\n kernel = backend_ctx->kernel_transpose_16;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_d_image1D));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qT_d_image1D));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_q));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_q));\n\n size_t local_size_q[3] = {4, 16, 1};\n size_t global_size_q[3] = {static_cast(width_q), static_cast(height_q), 1};\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_size_q, local_size_q, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n\n // scales\n int height_s = M / 4;\n int width_s = K / 32 / 4;\n\n kernel = backend_ctx->kernel_transpose_16;\n if (!K_tile_trans) {\n kernel = backend_ctx->kernel_transpose_16_4x1;\n width_s = K / 32;\n }\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &d_d_image1D));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &dT_d_image1D));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_s));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_s));\n\n size_t local_size_s[3] = {4, 16, 1};\n size_t global_size_s[3] = {static_cast(width_s), static_cast(height_s), 1};\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_size_s, local_size_s, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n // <----------------------------------------------------------------------------------> //\n\n // copy transposed buffer contents to original buffers\n // <----------------------------------------------------------------------------------> //\n // weights\n CL_CHECK(clEnqueueCopyBuffer(queue, qT_d, extra->q, 0, 0, q_size_bytes, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n\n // scales\n CL_CHECK(clEnqueueCopyBuffer(queue, dT_d, extra->d, 0, 0, d_size_bytes, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n // <----------------------------------------------------------------------------------> //\n\n // deallocate transpose buffers\n // <----------------------------------------------------------------------------------> //\n CL_CHECK(clReleaseMemObject(qT_d));\n CL_CHECK(clReleaseMemObject(dT_d));\n\n // deallocate temporary images\n CL_CHECK(clReleaseMemObject(q_d_image1D));\n CL_CHECK(clReleaseMemObject(d_d_image1D));\n CL_CHECK(clReleaseMemObject(qT_d_image1D));\n CL_CHECK(clReleaseMemObject(dT_d_image1D));\n // <----------------------------------------------------------------------------------> //\n // end transpose\n // <----------------------------------------------------------------------------------> //\n }\n #endif // GGML_OPENCL_USE_ADRENO_KERNELS\n\n return;\n\n }\n if (tensor->type == GGML_TYPE_Q4_1) {\n ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;\n GGML_ASSERT(extra_orig && \"Tesnors in OpenCL backend should have been allocated and initialized\");\n\n // Allocate the new extra and create aliases from the original.\n ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;\n ggml_tensor_extra_cl_q4_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_1();\n\n size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);\n size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);\n size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2;\n GGML_ASSERT(size_d + size_m + size_q == ggml_nbytes(tensor) && \"Incorrect tensor size\");\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n CL_CHECK(clEnqueueWriteBuffer(\n queue, data_device, CL_TRUE, 0,\n ggml_nbytes(tensor), data, 0, NULL, NULL));\n\n cl_buffer_region region;\n\n // The original tensor memory is divided into scales and quants, i.e.,\n // we first store scales, mins, then quants.\n // Create subbuffer for scales.\n region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);\n region.size = size_d;\n extra->d = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n auto previous_origin = region.origin;\n\n // Create subbuffer for mins.\n region.origin = align_to(previous_origin + size_d, backend_ctx->alignment);\n region.size = size_m;\n extra->m = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n previous_origin = region.origin;\n\n // Create subbuffer for quants.\n region.origin = align_to(previous_origin + size_m, backend_ctx->alignment);\n region.size = size_q;\n extra->q = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n\n #ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1;\n\n if (use_adreno_kernels(backend_ctx, tensor)) {\n kernel = backend_ctx->kernel_convert_block_q4_1_noshuffle;\n }\n #else\n cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1;\n #endif // GGML_OPENCL_USE_ADRENO_KERNELS\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m));\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clReleaseMemObject(data_device));\n\n tensor->extra = extra;\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n if (use_adreno_kernels(backend_ctx, tensor)) {\n\n int M = tensor->ne[1];\n int K = tensor->ne[0];\n\n GGML_ASSERT(K % 32 == 0);\n\n // Transpose q as ushort\n transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M);\n // Transpose d as ushort\n transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M);\n // Transpose m as ushort\n transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M);\n }\n#endif // GGML_OPENCL_USE_ADRENO_KERNELS\n return;\n }\n if (tensor->type == GGML_TYPE_MXFP4) {\n ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;\n GGML_ASSERT(extra_orig && \"Tesnors in OpenCL backend should have been allocated and initialized\");\n\n // Allocate the new extra and create aliases from the original.\n ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;\n ggml_tensor_extra_cl_mxfp4 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_mxfp4();\n\n size_t size_e = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(char);\n size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2;\n GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && \"Incorrect tensor size\");\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n CL_CHECK(clEnqueueWriteBuffer(\n queue, data_device, CL_TRUE, 0,\n ggml_nbytes(tensor), data, 0, NULL, NULL));\n\n // The original tensor memory is divided into scales and quants, i.e.,\n // we first store scales, then quants.\n cl_buffer_region region;\n\n // Create subbuffer for scales.\n region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);\n region.size = size_e;\n extra->e = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n auto previous_origin = region.origin;\n\n // Create subbuffer for quants.\n region.origin = align_to(previous_origin + size_e, backend_ctx->alignment);\n region.size = size_q;\n extra->q = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n if (use_adreno_moe_kernels(backend_ctx, tensor)) {\n cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4_trans;\n\n int ne00 = tensor->ne[0];\n int ne01 = tensor->ne[1];\n int ne02 = tensor->ne[2];\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01));\n\n size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)};\n size_t local_work_size[3] = {64, 2, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clReleaseMemObject(data_device));\n tensor->extra = extra;\n\n return;\n }\n#endif\n cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e));\n\n size_t global_work_size[3] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[3] = {64, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clReleaseMemObject(data_device));\n\n // Create image for Q\n cl_image_format img_format_q = {CL_RG, CL_UNSIGNED_INT32};\n cl_image_desc img_desc_q = {\n CL_MEM_OBJECT_IMAGE1D_BUFFER,\n static_cast(ggml_nelements(tensor)/32*2),\n 0, 0, 0, 0, 0, 0, 0,\n { extra->q }\n };\n extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err);\n tensor->extra = extra;\n\n return;\n }\n if (tensor->type == GGML_TYPE_Q8_0) {\n ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;\n GGML_ASSERT(extra_orig && \"Tesnors in OpenCL backend should have been allocated and initialized\");\n\n // Allocate the new extra and create aliases from the original.\n ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;\n ggml_tensor_extra_cl_q8_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q8_0();\n\n size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);\n size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)*sizeof(char));\n GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && \"Incorrect tensor size\");\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n CL_CHECK(clEnqueueWriteBuffer(\n queue, data_device, CL_TRUE, 0,\n ggml_nbytes(tensor), data, 0, NULL, NULL));\n\n // The original tensor memory is divided into scales and quants, i.e.,\n // we first store scales, then quants.\n cl_buffer_region region;\n\n // Create subbuffer for scales.\n region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);\n region.size = size_d;\n extra->d = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n auto previous_origin = region.origin;\n\n // Create subbuffer for quants.\n region.origin = align_to(previous_origin + size_d, backend_ctx->alignment);\n region.size = size_q;\n extra->q = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n\n cl_kernel kernel = backend_ctx->kernel_convert_block_q8_0;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d));\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clReleaseMemObject(data_device));\n\n tensor->extra = extra;\n\n // Transpose the weights and scales\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n if (enable_adreno_trans_weight(backend_ctx, tensor)) {\n\n int M = tensor->ne[1]; // ne01\n int K = tensor->ne[0]; // ne00\n\n GGML_ASSERT(K % 32 == 0);\n GGML_ASSERT(M % 4 == 0);\n GGML_ASSERT(tensor->ne[2] == 1);\n GGML_ASSERT(tensor->ne[3] == 1);\n\n // Transpose weights\n size_t q_size_bytes = K * M / 4 * sizeof(float);\n cl_buffer_region region;\n region.origin = 0;\n region.size = q_size_bytes;\n cl_mem qT_d = clCreateSubBuffer(\n backend_ctx->prealloc_quant_trans.buffer,\n 0,\n CL_BUFFER_CREATE_TYPE_REGION,\n ®ion,\n &err);\n CL_CHECK(err);\n\n cl_mem q_d_image1D;\n cl_mem qT_d_image1D;\n\n cl_image_format img_fmt_1d;\n cl_image_desc img_desc_1d;\n\n img_fmt_1d = { CL_RGBA, CL_FLOAT };\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * K / 4 / 4;\n img_desc_1d.buffer = extra->q;\n q_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err);\n CL_CHECK(err);\n\n img_fmt_1d = { CL_RGBA, CL_FLOAT };\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * K / 4 / 4;\n img_desc_1d.buffer = qT_d;\n qT_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err);\n CL_CHECK(err);\n\n int height_q = M / 4;\n int width_q = K / 4 / 4;\n kernel = backend_ctx->kernel_transpose_32;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_d_image1D));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qT_d_image1D));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_q));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_q));\n\n size_t local_size_q[3] = {4, 16, 1};\n size_t global_size_q[3] = {static_cast(width_q), static_cast(height_q), 1};\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_size_q, local_size_q, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n\n // Transpose scales\n size_t d_size_bytes = M * (K / 32) * 2;\n region.origin = 0;\n region.size = d_size_bytes;\n cl_mem dT_d = clCreateSubBuffer(\n backend_ctx->prealloc_scales_trans.buffer,\n 0,\n CL_BUFFER_CREATE_TYPE_REGION,\n ®ion,\n &err);\n CL_CHECK(err);\n\n cl_mem d_d_image1D;\n cl_mem dT_d_image1D;\n\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_fmt_1d = { CL_R, CL_HALF_FLOAT };\n img_desc_1d.image_width = M * K / 32;\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.buffer = extra->d;\n d_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err);\n CL_CHECK(err);\n\n img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT };\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * K / 32 / 4;\n img_desc_1d.buffer = dT_d;\n dT_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err);\n CL_CHECK(err);\n\n int height_s = M / 4;\n int width_s = K / 32;\n\n kernel = backend_ctx->kernel_transpose_16_4x1;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &d_d_image1D));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &dT_d_image1D));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_s));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_s));\n\n size_t local_size_s[3] = {4, 16, 1};\n size_t global_size_s[3] = {static_cast(width_s), static_cast(height_s), 1};\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_size_s, local_size_s, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n\n // copy transposed buffer contents to original buffers\n CL_CHECK(clEnqueueCopyBuffer(queue, qT_d, extra->q, 0, 0, q_size_bytes, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n\n CL_CHECK(clEnqueueCopyBuffer(queue, dT_d, extra->d, 0, 0, d_size_bytes, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n\n CL_CHECK(clReleaseMemObject(qT_d));\n CL_CHECK(clReleaseMemObject(dT_d));\n\n CL_CHECK(clReleaseMemObject(q_d_image1D));\n CL_CHECK(clReleaseMemObject(d_d_image1D));\n CL_CHECK(clReleaseMemObject(qT_d_image1D));\n CL_CHECK(clReleaseMemObject(dT_d_image1D));\n } // end transpose\n#endif // GGML_OPENCL_USE_ADRENO_KERNELS\n\n return;\n }\n if (tensor->type == GGML_TYPE_Q6_K) {\n ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;\n GGML_ASSERT(extra_orig && \"Tesnors in OpenCL backend should have been allocated and initialized\");\n\n // Allocate the new extra and create aliases from the original.\n ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;\n ggml_tensor_extra_cl_q6_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q6_K();\n\n size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2;\n size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4;\n size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16;\n size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);\n GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) &&\n \"Incorrect tensor size\");\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n CL_CHECK(clEnqueueWriteBuffer(\n queue, data_device, CL_TRUE, 0,\n ggml_nbytes(tensor), data, 0, NULL, NULL));\n\n cl_buffer_region region;\n\n // Subbuffer for ql\n region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);\n region.size = size_ql;\n extra->ql = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n auto previous_origin = region.origin;\n\n // Subbuffer for qh\n region.origin = align_to(previous_origin + size_ql, backend_ctx->alignment);\n region.size = size_qh;\n extra->qh = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n previous_origin = region.origin;\n\n // Subbuffer for scales\n region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment);\n region.size = size_s;\n extra->s = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n previous_origin = region.origin;\n\n // Create subbuffer for d.\n region.origin = align_to(previous_origin + size_s, backend_ctx->alignment);\n region.size = size_d;\n extra->d = clCreateSubBuffer(\n extra_orig->data_device, CL_MEM_READ_WRITE,\n CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);\n CL_CHECK(err);\n previous_origin = region.origin;\n\n // Flatten the weights\n cl_kernel kernel = backend_ctx->kernel_convert_block_q6_K;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d));\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clReleaseMemObject(data_device));\n\n extra->size_ql = size_ql;\n extra->size_qh = size_qh;\n extra->size_s = size_s;\n extra->size_d = size_d;\n\n tensor->extra = extra;\n return;\n }\n#endif // GGML_OPENCL_SOA_Q\n\n ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;\n GGML_ASSERT(extra);\n\n CL_CHECK(clEnqueueWriteBuffer(\n queue, extra->data_device, CL_TRUE, extra->offset + offset,\n size, data, 0, NULL, NULL));\n\n GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n GGML_ASSERT(tensor->extra);\n\n ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(buffer->buft->device);\n\n cl_context context = backend_ctx->context;\n cl_command_queue queue = backend_ctx->queue;\n\n // Make sure all previously submitted commands in other devices are finished.\n sync_with_other_backends(backend_ctx);\n\n#ifdef GGML_OPENCL_SOA_Q\n // In end-to-end runs, get_tensor is usually used to get back the logits,\n // where we can simply do clEnqueueReadBuffer since they are f32.\n // However, in test-backend-ops, the GPU graph is copied to the CPU backend,\n // which requires reading back quantized weight tensors.\n // To properly support this, we need to restore block_q4_0 struct arrays\n // from the flattened buffers.\n if (tensor->type == GGML_TYPE_Q4_0) {\n ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *)tensor->extra;\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n if (use_adreno_kernels(backend_ctx, tensor)) {\n cl_int err;\n cl_kernel kernel;\n\n cl_int M = tensor->ne[1]; // ne01\n cl_int K = tensor->ne[0]; // ne00\n\n GGML_ASSERT(K % 32 == 0);\n GGML_ASSERT(M % 4 == 0);\n\n size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2;\n size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t);\n GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && \"Incorrect tensor size\");\n\n cl_mem buf_trans_q;\n cl_mem buf_trans_d;\n\n CL_CHECK((buf_trans_q = clCreateBuffer(context, CL_MEM_READ_WRITE,\n size_q, NULL, &err), err));\n CL_CHECK((buf_trans_d = clCreateBuffer(context, CL_MEM_READ_WRITE,\n size_d, NULL, &err), err));\n\n kernel = backend_ctx->kernel_transpose_16_buf;\n\n // transpose q back\n cl_int stride_k_q = K/4;\n size_t local_size_q[3] = {64, 1, 1};\n size_t global_size_q[3] = {(size_t)M, (size_t)stride_k_q, 1};\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_q));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &M));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &stride_k_q));\n\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_size_q, local_size_q, 0, NULL, NULL));\n\n // transpose scales back\n cl_int stride_k_d = K/32;\n size_t local_size_d[3] = {64, 1, 1};\n size_t global_size_d[3] = {(size_t)M, (size_t)stride_k_d, 1};\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->d));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &M));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &stride_k_d));\n\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_size_d, local_size_d, 0, NULL, NULL));\n\n // unpack\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n\n cl_uchar mask_0F = 0x0F;\n cl_uchar mask_F0 = 0xF0;\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {1, 1, 1};\n\n kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0));\n\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_work_size, local_work_size, 0, NULL, NULL));\n\n // read back to host\n CL_CHECK(clEnqueueReadBuffer(\n queue, data_device, CL_TRUE, offset,\n size, data, 0, NULL, NULL));\n\n CL_CHECK(clReleaseMemObject(data_device));\n CL_CHECK(clReleaseMemObject(buf_trans_q));\n CL_CHECK(clReleaseMemObject(buf_trans_d));\n\n return;\n }\n#endif\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n\n cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device));\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {1, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clEnqueueReadBuffer(\n queue, data_device, CL_TRUE, offset,\n size, data, 0, NULL, NULL));\n CL_CHECK(clReleaseMemObject(data_device));\n return;\n }\n if (tensor->type == GGML_TYPE_Q4_1) {\n ggml_tensor_extra_cl_q4_1 * extra = (ggml_tensor_extra_cl_q4_1 *)tensor->extra;\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n if (use_adreno_kernels(backend_ctx, tensor)) {\n static ggml_cl_buffer buf_trans_q;\n static ggml_cl_buffer buf_trans_m;\n static ggml_cl_buffer buf_trans_d;\n static ggml_cl_buffer buf_unpacked;\n\n cl_int M = tensor->ne[1];\n cl_int K = tensor->ne[0];\n\n GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0);\n\n size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2;\n size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t);\n size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t);\n GGML_ASSERT(size_d + size_q + size_m == ggml_nbytes(tensor) && \"Incorrect tensor size\");\n\n buf_trans_q.allocate(backend_ctx->context, size_q);\n buf_trans_m.allocate(backend_ctx->context, size_m);\n buf_trans_d.allocate(backend_ctx->context, size_d);\n buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));\n\n // transpose q, d, m back\n transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4);\n transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32);\n transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32);\n\n cl_uchar mask_0F = 0x0F;\n cl_uchar mask_F0 = 0xF0;\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {1, 1, 1};\n\n cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_noshuffle;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_m.buffer));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0));\n\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL));\n CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));\n return;\n }\n#endif\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n\n cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device));\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {1, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clEnqueueReadBuffer(\n queue, data_device, CL_TRUE, offset,\n size, data, 0, NULL, NULL));\n CL_CHECK(clReleaseMemObject(data_device));\n return;\n }\n if (tensor->type == GGML_TYPE_MXFP4) {\n ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra;\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n if (use_adreno_moe_kernels(backend_ctx, tensor)) {\n cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4_trans;\n\n int ne00 = tensor->ne[0];\n int ne01 = tensor->ne[1];\n int ne02 = tensor->ne[2];\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01));\n\n size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)};\n size_t local_work_size[3] = {64, 2, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clEnqueueReadBuffer(\n queue, data_device, CL_TRUE, offset,\n size, data, 0, NULL, NULL));\n CL_CHECK(clReleaseMemObject(data_device));\n return;\n }\n#endif\n cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device));\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {1, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clEnqueueReadBuffer(\n queue, data_device, CL_TRUE, offset,\n size, data, 0, NULL, NULL));\n CL_CHECK(clReleaseMemObject(data_device));\n return;\n }\n if (tensor->type == GGML_TYPE_Q8_0) {\n ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)tensor->extra;\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n if (enable_adreno_trans_weight(backend_ctx, tensor)) {\n cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans;\n\n int ne00 = tensor->ne[0];\n int ne01 = tensor->ne[1];\n GGML_ASSERT(tensor->ne[2] == 1);\n GGML_ASSERT(tensor->ne[3] == 1);\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01));\n\n size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), 1, 1};\n size_t local_work_size[3] = {64, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n\n CL_CHECK(clEnqueueReadBuffer(\n queue, data_device, CL_TRUE, offset,\n size, data, 0, NULL, NULL));\n CL_CHECK(clReleaseMemObject(data_device));\n return;\n }\n#endif\n cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device));\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {1, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clEnqueueReadBuffer(\n queue, data_device, CL_TRUE, offset,\n size, data, 0, NULL, NULL));\n CL_CHECK(clReleaseMemObject(data_device));\n return;\n }\n if (tensor->type == GGML_TYPE_Q6_K) {\n ggml_tensor_extra_cl_q6_K * extra = (ggml_tensor_extra_cl_q6_K *)tensor->extra;\n\n cl_int err;\n cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,\n ggml_nbytes(tensor), NULL, &err);\n CL_CHECK(err);\n\n cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device));\n\n size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};\n size_t local_work_size[] = {1, 1, 1};\n\n cl_event evt;\n CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,\n global_work_size, local_work_size, 0, NULL, &evt));\n CL_CHECK(clWaitForEvents(1, &evt));\n CL_CHECK(clEnqueueReadBuffer(\n queue, data_device, CL_TRUE, offset,\n size, data, 0, NULL, NULL));\n CL_CHECK(clReleaseMemObject(data_device));\n return;\n }\n#endif // GGML_OPENCL_SOA_Q\n\n ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;\n\n CL_CHECK(clEnqueueReadBuffer(\n queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset,\n size, data, 0, NULL, NULL));\n\n GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_opencl_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n ggml_backend_dev_t dev = buffer->buft->device;\n ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(dev);\n cl_command_queue queue = backend_ctx->queue;\n\n ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;\n for (cl_mem buf : ctx->buffer) {\n CL_CHECK(clEnqueueFillBuffer(queue, buf, &value, sizeof(value), 0, buffer->size, 0, NULL, NULL));\n }\n CL_CHECK(clFinish(queue));\n}\n\nstatic void ggml_backend_opencl_buffer_reset(ggml_backend_buffer_t buffer) {\n ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;\n ctx->reset();\n}\n\nstatic ggml_backend_buffer_i ggml_backend_opencl_buffer_interface = {\n /* .free_buffer = */ ggml_backend_opencl_buffer_free_buffer,\n /* .get_base = */ ggml_backend_opencl_buffer_get_base,\n /* .init_tensor = */ ggml_backend_opencl_buffer_init_tensor,\n /* .memset_tensor = */ NULL,\n /* .set_tensor = */ ggml_backend_opencl_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_opencl_buffer_get_tensor,\n /* .cpy_tensor = */ NULL,\n /* .clear = */ ggml_backend_opencl_buffer_clear,\n /* .reset = */ ggml_backend_opencl_buffer_reset,\n};\n\n//\n// buffer type\n//\n\nstatic const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type) {\n return \"OpenCL\";\n\n GGML_UNUSED(buffer_type);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buffer_type, size_t size) {\n ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(buffer_type->device);\n\n // clCreateBuffer returns -61 for size 0\n size = std::max(size, (size_t)1);\n\n cl_int err;\n cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err);\n if (err != CL_SUCCESS) {\n GGML_LOG_INFO(\"%s: failed to allocate %.2f MiB\\n\", __func__, size / 1024.0 / 1024.0);\n return nullptr;\n }\n\n ggml_backend_opencl_buffer_context * ctx = new ggml_backend_opencl_buffer_context(mem);\n\n return ggml_backend_buffer_init(buffer_type, ggml_backend_opencl_buffer_interface, ctx, size);\n}\n\nstatic size_t ggml_backend_opencl_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) {\n ggml_backend_opencl_context * backend_ctx = ggml_cl2_init(buffer_type->device);\n return backend_ctx->alignment;\n}\n\nstatic size_t ggml_backend_opencl_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) {\n static size_t max_size = -1;\n if (max_size == (size_t)-1) {\n ggml_backend_opencl_context * backend_ctx = ggml_cl2_init(buffer_type->device);\n max_size = backend_ctx->max_alloc_size;\n }\n return max_size;\n}\n\nstatic bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {\n return ggml_backend_is_opencl(backend);\n\n UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = {\n /* .get_name = */ ggml_backend_opencl_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment,\n /* .get_max_size = */ ggml_backend_opencl_buffer_type_get_max_size,\n /* .get_alloc_size = */ NULL,\n /* .is_host = */ NULL,\n};\n\n//\n// backend device\n//\n\nstatic const char * ggml_backend_opencl_device_get_name(ggml_backend_dev_t dev) {\n return \"GPUOpenCL\";\n\n GGML_UNUSED(dev);\n}\n\nstatic const char * ggml_backend_opencl_device_get_description(ggml_backend_dev_t dev) {\n ggml_backend_opencl_device_context *dev_ctx = (ggml_backend_opencl_device_context *) dev->context;\n return dev_ctx->device_name.c_str();\n}\n\nstatic void ggml_backend_opencl_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n // no memory to report\n *free = 0;\n *total = 0;\n\n GGML_UNUSED(dev);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_opencl_device_get_type(ggml_backend_dev_t dev) {\n return GGML_BACKEND_DEVICE_TYPE_GPU;\n\n GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_opencl_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {\n props->name = ggml_backend_opencl_device_get_name(dev);\n props->description = ggml_backend_opencl_device_get_description(dev);\n props->type = ggml_backend_opencl_device_get_type(dev);\n ggml_backend_opencl_device_get_memory(dev, &props->memory_free, &props->memory_total);\n props->caps = ggml_backend_dev_caps {\n /* .async = */ false,\n /* .host_buffer = */ false,\n /* .buffer_from_host_ptr = */ false,\n /* .events = */ false,\n };\n}\n\nstatic ggml_backend_t ggml_backend_opencl_device_init(ggml_backend_dev_t dev, const char * params) {\n ggml_backend_opencl_context * backend_ctx = ggml_cl2_init(dev);\n // Getting a new reference to the backend, increase ref_count\n backend_ctx->ref_count++;\n\n ggml_backend_t backend = new ggml_backend {\n /* .guid = */ ggml_backend_opencl_guid(),\n /* .interface = */ ggml_backend_opencl_i,\n /* .device = */ dev,\n /* .context = */ backend_ctx,\n };\n\n return backend;\n\n GGML_UNUSED(params);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_opencl_device_get_buffer_type(ggml_backend_dev_t dev) {\n auto * dev_ctx = static_cast(dev->context);\n\n dev_ctx->buffer_type = ggml_backend_buffer_type{\n /* .iface = */ ggml_backend_opencl_buffer_type_interface,\n /* .device = */ dev,\n /* .context = */ nullptr,\n };\n\n return &dev_ctx->buffer_type;\n}\n\nstatic ggml_backend_buffer_t ggml_backend_opencl_device_buffer_from_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {\n GGML_UNUSED(dev);\n GGML_UNUSED(ptr);\n GGML_UNUSED(size);\n GGML_UNUSED(max_tensor_size);\n return nullptr;\n}\n\nstatic bool ggml_backend_opencl_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {\n return ggml_opencl_supports_op(dev, op);\n}\n\nstatic bool ggml_backend_opencl_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n // Check 'dev' and 'buffer_type' are not objects belonging to this backend.\n if (dev->iface.get_name != ggml_backend_opencl_device_get_name ||\n buft->iface.get_name != ggml_backend_opencl_buffer_type_get_name) {\n return false;\n }\n\n // Check cl_context is the same. clEnqueue* commands may not use\n // buffers from another cl_context.\n ggml_backend_opencl_context * backend_ctx0 = ggml_cl2_init(dev);\n ggml_backend_opencl_context * backend_ctx1 = ggml_cl2_init(buft->device);\n return backend_ctx0->context == backend_ctx1->context;\n}\n\nnamespace /* anonymous */ {\nstruct ggml_backend_device_i ggml_backend_opencl_device_i = {\n /* .get_name = */ ggml_backend_opencl_device_get_name,\n /* .get_description = */ ggml_backend_opencl_device_get_description,\n /* .get_memory = */ ggml_backend_opencl_device_get_memory,\n /* .get_type = */ ggml_backend_opencl_device_get_type,\n /* .get_props = */ ggml_backend_opencl_device_get_props,\n /* .init_backend = */ ggml_backend_opencl_device_init,\n /* .get_buffer_type = */ ggml_backend_opencl_device_get_buffer_type,\n /* .get_host_buffer_type = */ NULL,\n /* .buffer_from_host_ptr = */ ggml_backend_opencl_device_buffer_from_ptr,\n /* .supports_op = */ ggml_backend_opencl_device_supports_op,\n /* .supports_buft = */ ggml_backend_opencl_device_supports_buft,\n /* .offload_op = */ NULL,\n /* .event_new = */ NULL,\n /* .event_free = */ NULL,\n /* .event_synchronize = */ NULL,\n};\n}\n\n// Backend registry\n\nstatic const char * ggml_backend_opencl_reg_get_name(ggml_backend_reg_t reg) {\n return \"OpenCL\";\n\n GGML_UNUSED(reg);\n}\n\nstatic size_t ggml_backend_opencl_reg_device_count(ggml_backend_reg_t reg) {\n return g_ggml_backend_opencl_devices.size();\n\n GGML_UNUSED(reg);\n}\n\nstatic ggml_backend_dev_t ggml_backend_opencl_reg_device_get(ggml_backend_reg_t reg, size_t index) {\n GGML_ASSERT(index < ggml_backend_opencl_reg_device_count(reg));\n\n return &g_ggml_backend_opencl_devices[index];\n\n GGML_UNUSED(reg);\n GGML_UNUSED(index);\n}\n\nstatic struct ggml_backend_reg_i ggml_backend_opencl_reg_i = {\n /* .get_name = */ ggml_backend_opencl_reg_get_name,\n /* .device_count = */ ggml_backend_opencl_reg_device_count,\n /* .device_get = */ ggml_backend_opencl_reg_device_get,\n /* .get_proc_address = */ NULL,\n};\n\nggml_backend_reg_t ggml_backend_opencl_reg(void) {\n static std::mutex mutex;\n static ggml_backend_reg reg;\n static bool initialized = false;\n std::lock_guard lock(mutex);\n\n if (initialized) {\n return ®\n }\n initialized = true;\n\n g_ggml_backend_opencl_devices = ggml_opencl_probe_devices(®);\n\n reg = ggml_backend_reg{\n /* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_opencl_reg_i,\n /* .context = */ NULL,\n };\n\n return ®\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_opencl_reg)\n\n//------------------------------------------------------------------------------\n// Debugging utils\n//------------------------------------------------------------------------------\n#if 0\n#define QK4_0 32\ntypedef struct {\n ggml_fp16_t d; // delta\n uint8_t qs[QK4_0 / 2]; // nibbles / quants\n} block_q4_0;\nstatic_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2,\n \"wrong q4_0 block size/padding\");\n\n#include \n#ifdef __cplusplus\n#include \"half.hpp\"\n#endif\n\nstatic void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tensor) {\n void * buf = malloc(ggml_nbytes(tensor));\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n cl_command_queue queue = backend_ctx->queue;\n#ifdef GGML_OPENCL_SOA_Q\n void * buf_q;\n void * buf_d;\n#endif\n\n // Make sure everything is done.\n CL_CHECK(clFinish(queue));\n\n#ifdef GGML_OPENCL_SOA_Q\n if (tensor->type == GGML_TYPE_Q4_0) {\n ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *) tensor->extra;\n GGML_ASSERT(extra);\n\n size_t size_q = ggml_nelements(tensor)/QK4_0 * QK4_0/2;\n size_t size_d = ggml_nelements(tensor)/QK4_0 * sizeof(ggml_fp16_t);\n GGML_ASSERT(size_q + size_d == ggml_nbytes(tensor));\n buf_q = malloc(size_q);\n buf_d = malloc(size_d);\n\n CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL));\n CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, size_d, buf_d, 0, NULL, NULL));\n CL_CHECK(clFinish(queue));\n } else if (tensor->type == GGML_TYPE_MXFP4) {\n ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *) tensor->extra;\n GGML_ASSERT(extra);\n\n size_t size_q = ggml_nelements(tensor)/QK_MXFP4 * QK_MXFP4/2;\n size_t size_e = ggml_nelements(tensor)/QK_MXFP4 * sizeof(char);\n GGML_ASSERT(size_q + size_e == ggml_nbytes(tensor));\n buf_q = malloc(size_q);\n buf_d = malloc(size_e);\n\n CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL));\n CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, size_e, buf_d, 0, NULL, NULL));\n CL_CHECK(clFinish(queue));\n } else {\n // Read out the tensor from GPU memory.\n ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;\n GGML_ASSERT(extra);\n\n CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE,\n extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL));\n CL_CHECK(clFinish(queue));\n }\n#else\n // Read out the tensor from GPU memory.\n ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;\n GGML_ASSERT(extra);\n\n CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE,\n extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL));\n CL_CHECK(clFinish(queue));\n#endif // GGML_OPENCL_SOA_Q\n\n // Open file and dump.\n char fname[512];\n snprintf(fname, sizeof(fname), \"./tensor-dumps/%s.txt\", tensor->name);\n FILE * f = fopen(fname, \"w\");\n if (!f) {\n printf(\"Failed to open %s\\n\", fname);\n return;\n }\n\n if (tensor->type == GGML_TYPE_F32) {\n float * data = (float *) buf;\n for (int i = 0; i < ggml_nelements(tensor); ++i) {\n if (isnan(data[i])) {\n printf(\"NaN found: %s\\n\", tensor->name);\n break;\n }\n fprintf(f, \"%f\\n\", data[i]);\n }\n } else if (tensor->type == GGML_TYPE_I32) {\n int * data = (int *) buf;\n for (int i = 0; i < ggml_nelements(tensor); ++i) {\n if (isnan(data[i])) {\n printf(\"NaN found: %s\\n\", tensor->name);\n break;\n }\n fprintf(f, \"%d\\n\", data[i]);\n }\n } else if (tensor->type == GGML_TYPE_F16) {\n#ifdef __cplusplus\n half_float::half * data = (half_float::half *) buf;\n for (int i = 0; i < ggml_nelements(tensor); ++i) {\n if (std::isnan(data[i])) {\n printf(\"NaN found: %s\\n\", tensor->name);\n break;\n }\n fprintf(f, \"%f\\n\", float(data[i]));\n }\n#endif\n } else if (tensor->type == GGML_TYPE_Q4_0) {\n#ifdef GGML_OPENCL_SOA_Q\n ggml_fp16_t * data_d = (ggml_fp16_t *)buf_d;\n unsigned char * data_q = (unsigned char *)buf_q;\n\n for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) {\n fprintf(f, \"%04x, \", data_d[i]);\n for (int k = 0; k < QK4_0/2; ++k) {\n fprintf(f, \"%02x, \", data_q[k]);\n }\n fprintf(f, \"\\n\");\n data_q += QK4_0/2;\n }\n free(buf_d);\n free(buf_q);\n#else\n block_q4_0 * data = (block_q4_0 *) buf;\n for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) {\n fprintf(f, \"%04x, \", data[i].d);\n for (int k = 0; k < QK4_0/2; ++k) {\n fprintf(f, \"%02x, \", data[i].qs[k]);\n }\n fprintf(f, \"\\n\");\n }\n#endif // GGML_OPENCL_SOA_Q\n }\n free(buf);\n fflush(f);\n fclose(f);\n}\n#else\n#define dump_tensor(tensor)\n#endif\n\n//------------------------------------------------------------------------------\n// Ops\n//------------------------------------------------------------------------------\n\nstatic bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) {\n const int64_t ne10 = src1->ne[0];\n\n const int64_t ne0 = dst->ne[0];\n const int64_t ne1 = dst->ne[1];\n\n // TODO: find the optimal values for these\n return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) &&\n src1->type == GGML_TYPE_F32 &&\n dst->type == GGML_TYPE_F32 &&\n (ne0 >= 32 && ne1 >= 32 && ne10 >= 32);\n}\n\n// Copy a noncontiguous tensor to contiguous tensor. ne[] remains the same but\n// nb[] is recalculated such that tensor is contiguous.\nstatic void ggml_cl_copy_to_contiguous(ggml_backend_t backend, const ggml_tensor * src, cl_mem dst,\n cl_ulong &nb0, cl_ulong &nb1, cl_ulong &nb2, cl_ulong &nb3) {\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n const int tensor_type_size = ggml_type_size(src->type);\n\n const int ne00 = src->ne[0];\n const int ne01 = src->ne[1];\n const int ne02 = src->ne[2];\n const int ne03 = src->ne[3];\n\n const cl_ulong nb00 = src->nb[0];\n const cl_ulong nb01 = src->nb[1];\n const cl_ulong nb02 = src->nb[2];\n const cl_ulong nb03 = src->nb[3];\n\n const int ne0 = src->ne[0];\n const int ne1 = src->ne[1];\n const int ne2 = src->ne[2];\n const int ne3 = src->ne[3];\n\n nb0 = tensor_type_size;\n nb1 = tensor_type_size*ne00;\n nb2 = tensor_type_size*ne00*ne01;\n nb3 = tensor_type_size*ne00*ne01*ne02;\n\n ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *)src->extra;\n\n cl_ulong offset0 = extra->offset + src->view_offs;\n cl_ulong offsetd = 0;\n\n cl_kernel kernel;\n\n switch (src->type) {\n case GGML_TYPE_F32:\n kernel = backend_ctx->kernel_cpy_f32_f32;\n break;\n case GGML_TYPE_F16:\n kernel = backend_ctx->kernel_cpy_f16_f16;\n break;\n default:\n GGML_ASSERT(false && \"not implemented\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &dst));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne2));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne3));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb3));\n\n const int nth = MIN(64, ne00);\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src);\n}\n\nstatic void ggml_cl_nop(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n UNUSED(backend);\n UNUSED(src0);\n UNUSED(src1);\n UNUSED(dst);\n}\n\nstatic void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n GGML_TENSOR_LOCALS(int, ne0, src0, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb);\n GGML_TENSOR_LOCALS(int, ne1, src1, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb);\n GGML_TENSOR_LOCALS(int, ne, dst, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n switch (src0->type) {\n case GGML_TYPE_F32:\n kernel = backend_ctx->kernel_get_rows_f32;\n break;\n case GGML_TYPE_F16:\n kernel = backend_ctx->kernel_get_rows_f16;\n break;\n case GGML_TYPE_Q4_0:\n kernel = backend_ctx->kernel_get_rows_q4_0;\n break;\n default:\n GGML_ASSERT(false && \"not implemented\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3));\n\n int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel);\n int nth = 1;\n while (nth < ne00 && 2*nth <= max_workgroup_size) {\n nth *= 2;\n }\n\n size_t global_work_size[] = {(size_t)ne10*nth, (size_t)ne11, (size_t)ne12};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32);\n\n // ne0 = ne00\n // ne2 = ne02\n // ne3 = ne03\n\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const int ne11 = src1->ne[1];\n const int ne12 = src1->ne[2];\n\n const cl_ulong nb10 = src1->nb[0];\n const cl_ulong nb11 = src1->nb[1];\n const cl_ulong nb12 = src1->nb[2];\n\n const int ne0 = dst->ne[0];\n\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n const int nblk0 = ne0/ggml_blck_size(dst->type);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n switch (dst->type) {\n case GGML_TYPE_F32:\n if (src1->type == GGML_TYPE_I64) {\n kernel = backend_ctx->kernel_set_rows_f32_i64;\n } else {\n kernel = backend_ctx->kernel_set_rows_f32_i32;\n }\n break;\n case GGML_TYPE_F16:\n if (src1->type == GGML_TYPE_I64) {\n kernel = backend_ctx->kernel_set_rows_f16_i64;\n } else {\n kernel = backend_ctx->kernel_set_rows_f16_i32;\n }\n break;\n default:\n GGML_ABORT(\"not implemented\");\n }\n\n fastdiv_vals ne11_ = init_fastdiv_values(ne11);\n fastdiv_vals ne12_ = init_fastdiv_values(ne12);\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(fastdiv_vals), &ne11_));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(fastdiv_vals), &ne12_));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &nblk0));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb3));\n\n int nth0 = 64;\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 32;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n }\n\n int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel);\n while (nth0 < nblk0 && nth0 < max_workgroup_size) {\n nth0 *= 2;\n }\n\n int rows_per_workgroup = 1;\n if (nth0 > nblk0) {\n rows_per_workgroup = nth0 / nblk0;\n nth0 = nblk0;\n }\n\n size_t global_work_size[] = {\n (size_t)(ne01 + rows_per_workgroup - 1)/rows_per_workgroup*nth0,\n (size_t)ne02*rows_per_workgroup,\n (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth0, (size_t)rows_per_workgroup, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const int ne10 = src1->ne[0];\n const int ne11 = src1->ne[1];\n const int ne12 = src1->ne[2];\n const int ne13 = src1->ne[3];\n\n const cl_ulong nb10 = src1->nb[0];\n const cl_ulong nb11 = src1->nb[1];\n const cl_ulong nb12 = src1->nb[2];\n const cl_ulong nb13 = src1->nb[3];\n\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n const int ne2 = dst->ne[2];\n const int ne3 = dst->ne[3];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0;\n\n if (bcast_row) {\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ne11 == 1);\n }\n\n if (dst->type == GGML_TYPE_F32) {\n GGML_ASSERT(src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32);\n if (bcast_row) {\n kernel = backend_ctx->kernel_add_row;\n const int ne = ne00 / 4;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne));\n } else {\n kernel = backend_ctx->kernel_add;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2));\n CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3));\n CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3));\n }\n } else if (dst->type == GGML_TYPE_F16) {\n GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);\n const int type_src0 = (src0->type == GGML_TYPE_F32);\n const int type_src1 = (src1->type == GGML_TYPE_F32);\n if (bcast_row) {\n kernel = backend_ctx->kernel_add_row_f16;\n const int ne = ne00 / 4;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &type_src0));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &type_src1));\n } else {\n kernel = backend_ctx->kernel_add_f16;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2));\n CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3));\n CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3));\n CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &type_src0));\n CL_CHECK(clSetKernelArg(kernel, 31, sizeof(int), &type_src1));\n }\n } else {\n GGML_ASSERT(false && \"unsupported data types for add\");\n }\n\n if (bcast_row) {\n int n = ggml_nelements(dst)/4;\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size_ptr, dst);\n } else {\n unsigned int nth = MIN(64, ne0);\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_add_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n const ggml_tensor * src2 = dst->src[2];\n GGML_ASSERT(src2);\n GGML_ASSERT(src2->extra);\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(src2->type == GGML_TYPE_I32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous_rows(src0));\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n\n const cl_ulong nb11 = src1->nb[1];\n\n const cl_ulong nb21 = src2->nb[1];\n\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offset2 = extra2->offset + src2->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel = backend_ctx->kernel_add_id;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb21));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1));\n\n int nth = MIN(ne00, (int) backend_ctx->get_kernel_workgroup_size(kernel));\n size_t global_work_size[] = { (size_t)ne01*nth, (size_t)ne02, 1 };\n size_t local_work_size[] = { (size_t)nth, 1, 1 };\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_mul(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n GGML_ASSERT(src0->type == src1->type);\n GGML_ASSERT(src0->type == dst->type);\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const int ne10 = src1->ne[0];\n const int ne11 = src1->ne[1];\n const int ne12 = src1->ne[2];\n const int ne13 = src1->ne[3]; UNUSED(ne13);\n\n const cl_ulong nb10 = src1->nb[0];\n const cl_ulong nb11 = src1->nb[1];\n const cl_ulong nb12 = src1->nb[2];\n const cl_ulong nb13 = src1->nb[3]; UNUSED(nb13);\n\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n const int ne2 = dst->ne[2];\n const int ne3 = dst->ne[3];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n bool bcast_row = false;\n cl_kernel kernel;\n\n if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n // src1 is a row\n GGML_ASSERT(ne11 == 1);\n\n bcast_row = true;\n int ne = ne00 / 4;\n\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_mul_row;\n } else {\n kernel = backend_ctx->kernel_mul_row_f16;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne));\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_mul;\n } else {\n kernel = backend_ctx->kernel_mul_f16;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2));\n CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3));\n CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3));\n }\n\n if (bcast_row) {\n int n = ggml_nelements(dst)/4;\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr; // Let driver choose the work-group sizes.\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n } else {\n unsigned int nth = MIN(64, ne0);\n size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_div(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n GGML_ASSERT(src0->type == src1->type);\n GGML_ASSERT(src0->type == dst->type);\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const int ne10 = src1->ne[0];\n const int ne11 = src1->ne[1];\n const int ne12 = src1->ne[2];\n const int ne13 = src1->ne[3];\n\n const cl_ulong nb10 = src1->nb[0];\n const cl_ulong nb11 = src1->nb[1];\n const cl_ulong nb12 = src1->nb[2];\n const cl_ulong nb13 = src1->nb[3];\n\n const int ne0 = dst->ne[0];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n bool bcast_row = false;\n cl_kernel kernel;\n\n if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n // src1 is a row\n GGML_ASSERT(ne11 == 1);\n\n bcast_row = true;\n int ne = ne00 / 4;\n\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_div_row;\n } else {\n kernel = backend_ctx->kernel_div_row_f16;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne));\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_div;\n } else {\n kernel = backend_ctx->kernel_div_f16;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3));\n }\n\n if (bcast_row) {\n int n = ggml_nelements(dst)/4;\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n } else {\n unsigned int nth = MIN(64, ne0);\n size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_sub(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n GGML_ASSERT(src0->type == src1->type);\n GGML_ASSERT(src0->type == dst->type);\n GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const int ne10 = src1->ne[0];\n const int ne11 = src1->ne[1];\n const int ne12 = src1->ne[2];\n const int ne13 = src1->ne[3];\n\n const cl_ulong nb10 = src1->nb[0];\n const cl_ulong nb11 = src1->nb[1];\n const cl_ulong nb12 = src1->nb[2];\n const cl_ulong nb13 = src1->nb[3];\n\n const int ne0 = dst->ne[0];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n bool bcast_row = false;\n cl_kernel kernel;\n\n if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n // src1 is a row\n GGML_ASSERT(ne11 == 1);\n\n bcast_row = true;\n int ne = ne00 / 4;\n\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_sub_row;\n } else {\n kernel = backend_ctx->kernel_sub_row_f16;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne));\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_sub;\n } else {\n kernel = backend_ctx->kernel_sub_f16;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3));\n }\n\n if (bcast_row) {\n int n = ggml_nelements(dst)/4;\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n } else {\n unsigned int nth = MIN(64, ne0);\n size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_sqr(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n // Currently assumes src0 is contiguous\n int n = ggml_nelements(dst);\n if (n % 4 == 0) {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_sqr_cont_f32_4;\n } else {\n kernel = backend_ctx->kernel_sqr_cont_f16_4;\n }\n n /= 4;\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_sqr_cont_f32;\n } else {\n kernel = backend_ctx->kernel_sqr_cont_f16;\n }\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_sqrt(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n // Currently assumes src0 is contiguous\n int n = ggml_nelements(dst);\n if (n % 4 == 0) {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_sqrt_cont_f32_4;\n } else {\n kernel = backend_ctx->kernel_sqrt_cont_f16_4;\n }\n n /= 4;\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_sqrt_cont_f32;\n } else {\n kernel = backend_ctx->kernel_sqrt_cont_f16;\n }\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_UNUSED(src1);\n\n GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n cl_kernel kernel;\n\n const bool is_c4 = ne00 % 4 == 0;\n if (is_c4) {\n kernel = backend_ctx->kernel_mean_f32_4;\n } else {\n kernel = backend_ctx->kernel_mean_f32;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb3));\n\n size_t global_work_size[] = {64 * (size_t)ne01, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n int ne01 = src0->ne[1];\n cl_ulong nb00 = src0->nb[0];\n cl_ulong nb01 = src0->nb[1];\n cl_ulong nb02 = src0->nb[2];\n\n int ne10 = src1->ne[0];\n cl_ulong nb11 = src1->nb[1];\n\n int ne1 = dst->ne[1];\n int ne2 = dst->ne[2];\n cl_ulong nb0 = dst->nb[0];\n cl_ulong nb1 = dst->nb[1];\n cl_ulong nb2 = dst->nb[2];\n\n cl_kernel kernel = backend_ctx->kernel_ssm_conv_f32_f32;\n\n if (ne10 % 4 == 0) {\n kernel = backend_ctx->kernel_ssm_conv_f32_f32_4;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2));\n\n size_t global_work_size[] = {(size_t)ne01, (size_t)ne1, (size_t)ne2};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (ne01 % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_gelu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n int n = ggml_nelements(dst);\n\n if (n % 4 == 0) {\n kernel = backend_ctx->kernel_gelu_4;\n n /= 4;\n } else {\n kernel = backend_ctx->kernel_gelu;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_gelu_erf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n int n = ggml_nelements(dst);\n\n if (n % 4 == 0) {\n kernel = backend_ctx->kernel_gelu_erf_4;\n n /= 4;\n } else {\n kernel = backend_ctx->kernel_gelu_erf;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_gelu_quick(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n int n = ggml_nelements(dst);\n\n if (n % 4 == 0) {\n kernel = backend_ctx->kernel_gelu_quick_4;\n n /= 4;\n } else {\n kernel = backend_ctx->kernel_gelu_quick;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_silu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n int n = ggml_nelements(dst);\n\n if (n % 4 == 0) {\n kernel = backend_ctx->kernel_silu_4;\n n /= 4;\n } else {\n kernel = backend_ctx->kernel_silu;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr; // Let driver choose the work-group sizes.\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_relu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel = backend_ctx->kernel_relu;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n const int64_t n = ggml_nelements(dst);\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr; // Let driver choose the work-group sizes.\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_sigmoid(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_sigmoid_f32;\n } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {\n kernel = backend_ctx->kernel_sigmoid_f16;\n } else {\n GGML_ASSERT(false && \"Unsupported data types for sigmoid (input and output must be both f32 or f16)\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n const int64_t n = ggml_nelements(dst);\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr; // Let driver choose the work-group sizes.\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_tri(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int tri_type = ggml_get_op_params_i32(dst, 0);\n const int64_t n = ggml_nelements(dst);\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n\n cl_kernel kernel = backend_ctx->kernel_tri;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &tri_type));\n\n size_t local_work_size[1] = { 256 };\n size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] };\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_fill(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src0);\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n float v = 0.0f;\n memcpy(&v, ((int32_t *) dst->op_params), sizeof(float));\n\n const int64_t n = ggml_nelements(dst);\n\n cl_kernel kernel = backend_ctx->kernel_fill;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(float), &v));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &n));\n\n size_t local_work_size[1] = { 256 };\n size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] };\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n float min;\n float max;\n memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float));\n memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float));\n\n cl_kernel kernel = backend_ctx->kernel_clamp;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &min));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &max));\n\n const int64_t n = ggml_nelements(dst);\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr; // Let driver choose the work-group sizes.\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n\n const int ne00 = src0 ? src0->ne[0] : 0;\n const int ne01 = src0 ? src0->ne[1] : 0;\n const int ne02 = src0 ? src0->ne[2] : 0;\n const int ne03 = src0 ? src0->ne[3] : 0;\n\n const cl_ulong nb01 = src0 ? src0->nb[1] : 0;\n const cl_ulong nb02 = src0 ? src0->nb[2] : 0;\n const cl_ulong nb03 = src0 ? src0->nb[3] : 0;\n\n const int nth = MIN(64, ne00);\n\n cl_kernel kernel = backend_ctx->kernel_norm;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth, NULL));\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n //ggml_backend_opencl_device_context * dev_ctx =\n // (ggml_backend_opencl_device_context *)backend->device->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n\n const int ne00 = src0 ? src0->ne[0] : 0;\n const int ne01 = src0 ? src0->ne[1] : 0;\n const int ne02 = src0 ? src0->ne[2] : 0;\n const int ne03 = src0 ? src0->ne[3] : 0;\n\n const cl_ulong nb01 = src0 ? src0->nb[1] : 0;\n const cl_ulong nb02 = src0 ? src0->nb[2] : 0;\n const cl_ulong nb03 = src0 ? src0->nb[3] : 0;\n\n GGML_ASSERT(ne00 % 4 == 0);\n\n const int nth = MIN(64, ne00);\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n cl_kernel kernel = backend_ctx->kernel_rms_norm;\n\n // Note, this kernel declares local memory in kernel args and the size\n // depends on subgroup size.\n // Note, this requires OpenCL 2.1 and above\n // For now we use fixed subgroup size to simplify support for OpenCL 2.0.\n size_t sgs;\n //CL_CHECK(clGetKernelSubGroupInfo(kernel, dev_ctx->device,\n // CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE,\n // sizeof(local_work_size), local_work_size,\n // sizeof(size_t), &sgs, NULL));\n if (backend_ctx->gpu_family == ADRENO) {\n sgs = 64;\n } else if (backend_ctx->gpu_family == INTEL) {\n sgs = 32;\n } else {\n GGML_ASSERT(false && \"Unsupported GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps));\n // This is local memory - the size depends on subgroup size.\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL));\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor) {\n GGML_ASSERT(mul_tensor);\n GGML_ASSERT(rms_norm_tensor);\n\n // src0 is the src of rms_norm, src1 is the other src of mul (one being rms_norm)\n const ggml_tensor * src0 = rms_norm_tensor->src[0];\n const ggml_tensor * src1;\n if (mul_tensor->src[0] == rms_norm_tensor) {\n src1 = mul_tensor->src[1];\n } else if (mul_tensor->src[1] == rms_norm_tensor) {\n src1 = mul_tensor->src[0];\n } else {\n GGML_ASSERT(false && \"Invalid args for rms_norm and mul\");\n }\n const ggml_tensor * dst = mul_tensor;\n\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n float eps;\n memcpy(&eps, rms_norm_tensor->op_params, sizeof(float));\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const int ne10 = src1->ne[0];\n const int ne11 = src1->ne[1];\n const int ne12 = src1->ne[2];\n const int ne13 = src1->ne[3];\n\n const cl_ulong nb11 = src1->nb[1];\n const cl_ulong nb12 = src1->nb[2];\n const cl_ulong nb13 = src1->nb[3];\n\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n GGML_ASSERT(ne00 % 4 == 0);\n\n size_t sgs;\n if (backend_ctx->gpu_family == ADRENO) {\n sgs = 64;\n } else if (backend_ctx->gpu_family == INTEL) {\n sgs = 32;\n } else {\n GGML_ASSERT(false && \"Unsupported GPU\");\n }\n\n cl_kernel kernel = backend_ctx->kernel_rms_norm_mul;\n\n int nth = sgs;\n int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel);\n while (nth < ne00 && nth < max_workgroup_size) {\n nth *= 2;\n }\n nth = MIN(nth, max_workgroup_size);\n nth = MIN(nth, ne00);\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &eps));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float)*sgs, NULL));\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) {\n GGML_ASSERT(norm_tensor && mul_tensor && add_tensor);\n\n const ggml_tensor * src0 = norm_tensor->src[0];\n const ggml_tensor * src1 = mul_tensor->src[0] == norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0];\n const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0];\n const ggml_tensor * dst = add_tensor;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offset2 = extra2->offset + src2->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n float eps;\n memcpy(&eps, norm_tensor->op_params, sizeof(float));\n\n const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3];\n const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3];\n const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3];\n const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3];\n const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3];\n const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3];\n const cl_ulong nbd1 = dst->nb[1], nbd2 = dst->nb[2], nbd3 = dst->nb[3];\n\n size_t sgs;\n if (backend_ctx->gpu_family == ADRENO) sgs = 64;\n else if (backend_ctx->gpu_family == INTEL) sgs = 32;\n else GGML_ASSERT(false && \"Unsupported GPU\");\n\n cl_kernel kernel = backend_ctx->kernel_norm_mul_add;\n\n int nth = sgs;\n int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel);\n while (nth < ne00/4 && nth < max_workgroup_size) nth *= 2;\n nth = MIN(nth, max_workgroup_size);\n nth = MIN(nth, ne00/4);\n\n size_t gws[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t lws[] = {(size_t)nth, 1, 1};\n size_t num_subgroups = (nth + sgs - 1) / sgs;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22));\n CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23));\n CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21));\n CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22));\n CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23));\n CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nbd1));\n CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nbd2));\n CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nbd3));\n CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps));\n CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_float2) * num_subgroups, NULL));\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);\n}\n\nstatic void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) {\n GGML_ASSERT(gn_tensor && mul_tensor && add_tensor);\n\n const ggml_tensor * src0 = gn_tensor->src[0];\n const ggml_tensor * src1 = mul_tensor->src[0] == gn_tensor ? mul_tensor->src[1] : mul_tensor->src[0];\n const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0];\n const ggml_tensor * dst = add_tensor;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offset2 = extra2->offset + src2->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n int groups;\n float eps;\n memcpy(&groups, gn_tensor->op_params, sizeof(int));\n memcpy(&eps, (char *)gn_tensor->op_params + sizeof(int), sizeof(float));\n\n cl_kernel kernel = backend_ctx->kernel_group_norm_mul_add;\n int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel);\n int ne = ggml_nelements(src0);\n int group_size = ne / groups;\n\n size_t lws[] = { (size_t)MIN(max_workgroup_size, group_size) };\n size_t gws[] = { (size_t)groups * lws[0] };\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &group_size));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(float), &eps));\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 1, gws, lws, dst);\n}\n\nstatic void ggml_cl_group_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n int32_t n_groups = ((const int32_t *) dst->op_params)[0];\n int32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + n_groups - 1) / n_groups);\n float eps = ((const float *) dst->op_params)[1];\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne = ne00*ne01*ne02;\n\n cl_kernel kernel = backend_ctx->kernel_group_norm;\n\n size_t sgs = 64;\n if (backend_ctx->gpu_family == ADRENO) {\n sgs = 64;\n } else if (backend_ctx->gpu_family == INTEL) {\n sgs = 32;\n } else {\n GGML_ASSERT(false && \"Unsupported GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &group_size));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(float), &eps));\n\n size_t global_work_size[] = {(size_t)n_groups*sgs, 1, 1};\n size_t local_work_size[] = {(size_t)sgs, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_l2_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n float eps;\n memcpy(&eps, dst->op_params, sizeof(float));\n\n GGML_TENSOR_LOCALS(int, ne0, src0, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb);\n\n size_t sgs;\n if (backend_ctx->gpu_family == ADRENO) {\n sgs = 64;\n } else if (backend_ctx->gpu_family == INTEL) {\n sgs = 32;\n } else {\n GGML_ASSERT(false && \"Unsupported GPU\");\n }\n\n cl_kernel kernel = backend_ctx->kernel_l2_norm_f32;\n\n int nth = sgs;\n while (nth < ne00 && nth < (int)backend_ctx->get_kernel_workgroup_size(kernel)) {\n nth *= 2;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL));\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n cl_kernel kernel;\n\n if (ggml_is_contiguous(src0)) {\n // Handle contiguous input\n int n = ggml_nelements(dst);\n if (n % 4 == 0) {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_tanh_f32_4;\n } else {\n kernel = backend_ctx->kernel_tanh_f16_4;\n }\n n /= 4;\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_tanh_f32;\n } else {\n kernel = backend_ctx->kernel_tanh_f16;\n }\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n } else {\n // Handle non-contiguous input\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_tanh_f32_nc;\n } else {\n kernel = backend_ctx->kernel_tanh_f16_nc;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3));\n\n int nth = 64;\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_neg(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n GGML_TENSOR_LOCALS(int, ne0, src0, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb);\n GGML_TENSOR_LOCALS(int, ne, dst, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb);\n\n cl_kernel kernel;\n\n if (ggml_is_contiguous(src0)) {\n // Handle contiguous input\n int n = ggml_nelements(dst);\n if (n % 4 == 0) {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_neg_f32_4;\n } else {\n kernel = backend_ctx->kernel_neg_f16_4;\n }\n n /= 4;\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_neg_f32;\n } else {\n kernel = backend_ctx->kernel_neg_f16;\n }\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n));\n\n size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n } else {\n // Handle non-contiguous input\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_neg_f32_nc;\n } else {\n kernel = backend_ctx->kernel_neg_f16_nc;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3));\n\n int nth = 64;\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_exp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n GGML_TENSOR_LOCALS(int, ne0, src0, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb);\n GGML_TENSOR_LOCALS(int, ne, dst, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb);\n\n cl_kernel kernel;\n\n if (ggml_is_contiguous(src0)) {\n // Handle contiguous input\n int n = ggml_nelements(dst);\n if (n % 4 == 0) {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_exp_f32_4;\n } else {\n kernel = backend_ctx->kernel_exp_f16_4;\n }\n n /= 4;\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_exp_f32;\n } else {\n kernel = backend_ctx->kernel_exp_f16;\n }\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n));\n\n size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n } else {\n // Handle non-contiguous input\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_exp_f32_nc;\n } else {\n kernel = backend_ctx->kernel_exp_f16_nc;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3));\n\n int nth = 64;\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_expm1(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n cl_kernel kernel;\n\n if (ggml_is_contiguous(src0)) {\n // Handle contiguous input\n int n = ggml_nelements(dst);\n if (n % 4 == 0) {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_expm1_f32_4;\n } else {\n kernel = backend_ctx->kernel_expm1_f16_4;\n }\n n /= 4;\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_expm1_f32;\n } else {\n kernel = backend_ctx->kernel_expm1_f16;\n }\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n } else {\n // Handle non-contiguous input\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_expm1_f32_nc;\n } else {\n kernel = backend_ctx->kernel_expm1_f16_nc;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3));\n\n int nth = 64;\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n cl_kernel kernel;\n\n if (ggml_is_contiguous(src0)) {\n // Handle contiguous input\n int n = ggml_nelements(dst);\n if (n % 4 == 0) {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_softplus_f32_4;\n } else {\n kernel = backend_ctx->kernel_softplus_f16_4;\n }\n n /= 4;\n } else {\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_softplus_f32;\n } else {\n kernel = backend_ctx->kernel_softplus_f16;\n }\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n } else {\n // Handle non-contiguous input\n if (src0->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_softplus_f32_nc;\n } else {\n kernel = backend_ctx->kernel_softplus_f16_nc;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3));\n\n int nth = 64;\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_repeat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1_shape_def, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_ASSERT(dst->type == src0->type);\n\n UNUSED(src1_shape_def);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n const int ne2 = dst->ne[2];\n const int ne3 = dst->ne[3];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n cl_kernel kernel = backend_ctx->kernel_repeat_f32;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3));\n\n int nth = 64;\n\n size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_pad(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n if (backend_ctx->kernel_pad == nullptr) {\n GGML_LOG_WARN(\"%s: pad kernel not available, skipping OpenCL execution.\\n\", __func__);\n return;\n }\n\n ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong off_src0 = extra_src0->offset + src0->view_offs;\n cl_ulong off_dst = extra_dst->offset + dst->view_offs;\n\n const int s_ne0 = src0->ne[0];\n const int s_ne1 = src0->ne[1];\n const int s_ne2 = src0->ne[2];\n const int s_ne3 = src0->ne[3];\n\n const int s_nb0 = src0->nb[0];\n const int s_nb1 = src0->nb[1];\n const int s_nb2 = src0->nb[2];\n const int s_nb3 = src0->nb[3];\n\n const int d_ne0 = dst->ne[0];\n const int d_ne1 = dst->ne[1];\n const int d_ne2 = dst->ne[2];\n const int d_ne3 = dst->ne[3];\n\n const int d_nb0 = dst->nb[0];\n const int d_nb1 = dst->nb[1];\n const int d_nb2 = dst->nb[2];\n const int d_nb3 = dst->nb[3];\n\n const int lp0 = ((const int*)(dst->op_params))[0];\n const int rp0 = ((const int*)(dst->op_params))[1];\n const int lp1 = ((const int*)(dst->op_params))[2];\n const int rp1 = ((const int*)(dst->op_params))[3];\n const int lp2 = ((const int*)(dst->op_params))[4];\n const int rp2 = ((const int*)(dst->op_params))[5];\n const int lp3 = ((const int*)(dst->op_params))[6];\n const int rp3 = ((const int*)(dst->op_params))[7];\n\n cl_kernel kernel = backend_ctx->kernel_pad;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &s_ne0));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &s_ne1));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &s_ne2));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &s_ne3));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &s_nb0));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &s_nb1));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &s_nb2));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &s_nb3));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &d_ne0));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &d_ne1));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &d_ne2));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &d_ne3));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &d_nb0));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &d_nb1));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &d_nb2));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &d_nb3));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &lp0));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &rp0));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &lp1));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &rp1));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &lp2));\n CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &rp2));\n CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &lp3));\n CL_CHECK(clSetKernelArg(kernel, 27, sizeof(int), &rp3));\n\n size_t lws0 = 64;\n size_t gws0 = (( (size_t)d_ne0 + lws0 - 1 ) / lws0) * lws0;\n\n size_t global_work_size[] = { gws0, (size_t)d_ne1, (size_t)d_ne2*d_ne3 };\n size_t local_work_size[] = { lws0, 1, 1 };\n\n size_t * local_work_size_ptr = local_work_size;\n if (d_ne0 % lws0 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_upscale(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n const int mode_flags = (ggml_scale_mode) ggml_get_op_params_i32(dst, 0);\n const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF);\n cl_kernel kernel = nullptr;\n\n if (mode == GGML_SCALE_MODE_NEAREST) {\n kernel = backend_ctx->kernel_upscale;\n if (kernel == nullptr) {\n GGML_LOG_WARN(\"%s: nearest upscale kernel not available, skipping OpenCL execution.\\n\", __func__);\n return;\n }\n } else if (mode == GGML_SCALE_MODE_BILINEAR) {\n kernel = backend_ctx->kernel_upscale_bilinear;\n if (kernel == nullptr) {\n GGML_LOG_WARN(\"%s: bilinear upscale kernel not available, skipping OpenCL execution.\\n\", __func__);\n return;\n }\n } else {\n GGML_LOG_WARN(\"%s: unsupported upscale mode %d, skipping OpenCL execution.\\n\", __func__, mode);\n return;\n }\n\n ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong off_src0 = extra_src0->offset + src0->view_offs;\n cl_ulong off_dst = extra_dst->offset + dst->view_offs;\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n const int ne2 = dst->ne[2];\n const int ne3 = dst->ne[3];\n\n float sf0 = (float)ne0 / ne00;\n float sf1 = (float)ne1 / ne01;\n float sf2 = (float)ne2 / ne02;\n float sf3 = (float)ne3 / ne03;\n\n float pixel_offset = 0.5f;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb03));\n\n if (mode == GGML_SCALE_MODE_NEAREST) {\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne2));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne3));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &sf0));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &sf1));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf2));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf3));\n } else if (mode == GGML_SCALE_MODE_BILINEAR) {\n if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) {\n sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0;\n sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1;\n pixel_offset = 0.0f;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne2));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne3));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf0));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf1));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(float), &sf2));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(float), &sf3));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float), &pixel_offset));\n }\n\n\n size_t dst_total_elements = (size_t)ne0 * ne1 * ne2 * ne3;\n if (dst_total_elements == 0) {\n return;\n }\n size_t global_work_size[] = { dst_total_elements, 1, 1 };\n size_t local_work_size_pref = 256;\n size_t local_work_size[] = { MIN(local_work_size_pref, dst_total_elements), 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (dst_total_elements % local_work_size[0] != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const cl_ulong nb10 = src1->nb[0];\n const cl_ulong nb11 = src1->nb[1];\n const cl_ulong nb12 = src1->nb[2];\n const cl_ulong nb13 = src1->nb[3];\n\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n const int ne2 = dst->ne[2];\n const int ne3 = dst->ne[3];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n const cl_int dim = ((const int32_t *) dst->op_params)[0];\n GGML_ASSERT(dim >= 0 && dim <= 3);\n\n int nth = MIN(64, ne0);\n\n cl_kernel kernel = backend_ctx->kernel_concat_f32;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_int), &dim));\n\n size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_timestep_embedding(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n if (backend_ctx->kernel_timestep_embedding == nullptr) {\n GGML_LOG_WARN(\"%s: timestep_embedding kernel not available, skipping OpenCL execution.\\n\", __func__);\n return;\n }\n\n ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong off_src0 = extra_src0->offset + src0->view_offs;\n cl_ulong off_dst = extra_dst->offset + dst->view_offs;\n\n const int logical_dim = dst->op_params[0];\n const int max_period = dst->op_params[1];\n const int dst_nb1_bytes = dst->nb[1];\n\n cl_kernel kernel = backend_ctx->kernel_timestep_embedding;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &dst_nb1_bytes));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &logical_dim));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &max_period));\n\n size_t gws0 = (size_t)(((logical_dim + 1) / 2) + 1);\n\n size_t gws1 = (size_t)src0->ne[0];\n\n size_t global_work_size[] = {gws0, gws1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, NULL, dst);\n}\n\nstatic void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) {\n const ggml_tensor * v = dst->src[2];\n const ggml_tensor * mask = dst->src[3];\n const ggml_tensor * sinks = dst->src[4];\n GGML_ASSERT(q->extra);\n GGML_ASSERT(k->extra);\n GGML_ASSERT(v->extra);\n GGML_ASSERT(dst->extra);\n if (mask) {\n GGML_ASSERT(mask->extra);\n }\n if (sinks) {\n GGML_ASSERT(sinks->extra);\n }\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n const int n_q = q->ne[1];\n const int n_kv = k->ne[1];\n const int d_head_q = q->ne[0];\n const int d_head_v = v->ne[0];\n const int n_head = q->ne[2];\n const int n_head_kv = k->ne[2];\n const int n_batch = q->ne[3];\n\n cl_kernel kernel = NULL;\n\n const bool is_f16 = q->type == GGML_TYPE_F16;\n const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16;\n const std::pair dk_dv = {d_head_q, d_head_v};\n\n if (n_q == 1) {\n if (is_mixed) {\n kernel = backend_ctx->kernels_flash_attn_f32_f16_q1.at(dk_dv);\n } else if (is_f16) {\n kernel = backend_ctx->kernels_flash_attn_f16_q1.at(dk_dv);\n } else {\n kernel = backend_ctx->kernels_flash_attn_f32_q1.at(dk_dv);\n }\n } else {\n if (is_mixed) {\n kernel = backend_ctx->kernels_flash_attn_f32_f16.at(dk_dv);\n } else if (is_f16) {\n kernel = backend_ctx->kernels_flash_attn_f16.at(dk_dv);\n } else {\n kernel = backend_ctx->kernels_flash_attn_f32.at(dk_dv);\n }\n }\n GGML_ASSERT(kernel != NULL);\n\n ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra;\n ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *)k->extra;\n ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra;\n ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra;\n ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL;\n ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL;\n\n cl_ulong offset_q = extra_q->offset + q->view_offs;\n cl_ulong offset_k = extra_k->offset + k->view_offs;\n cl_ulong offset_v = extra_v->offset + v->view_offs;\n cl_ulong offset_o = extra_o->offset + dst->view_offs;\n cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL;\n cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0;\n cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL;\n cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0;\n\n const cl_ulong q_nb1 = q->nb[1], q_nb2 = q->nb[2], q_nb3 = q->nb[3];\n const cl_ulong k_nb1 = k->nb[1], k_nb2 = k->nb[2], k_nb3 = k->nb[3];\n const cl_ulong v_nb1 = v->nb[1], v_nb2 = v->nb[2], v_nb3 = v->nb[3];\n const cl_ulong o_nb1 = dst->nb[1], o_nb2 = dst->nb[2], o_nb3 = dst->nb[3];\n const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0;\n const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0;\n const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0;\n const int mask_ne2 = mask ? mask->ne[2] : 0;\n const int mask_ne3 = mask ? mask->ne[3] : 0;\n\n float scale, max_bias, logit_softcap;\n const float * params = (const float *)dst->op_params;\n scale = params[0];\n max_bias = params[1];\n logit_softcap = params[2];\n\n const int is_causal = (mask == NULL && n_q > 1 && n_q == n_kv);\n\n const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0;\n const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f;\n const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f);\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_k->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_k));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra_v->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_v));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_o->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_o));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(float), &scale));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_q));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_kv));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &is_causal));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_head));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &q_nb1)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &q_nb2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &q_nb3));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &k_nb1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &k_nb2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &k_nb3));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &v_nb1)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &v_nb2)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &v_nb3));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &o_nb1)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &o_nb2)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &o_nb3));\n CL_CHECK(clSetKernelArg(kernel, 25, sizeof(float), &max_bias));\n CL_CHECK(clSetKernelArg(kernel, 26, sizeof(float), &m0));\n CL_CHECK(clSetKernelArg(kernel, 27, sizeof(float), &m1));\n CL_CHECK(clSetKernelArg(kernel, 28, sizeof(int), &n_head_log2_val));\n CL_CHECK(clSetKernelArg(kernel, 29, sizeof(float), &logit_softcap));\n CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &n_head_kv));\n CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_mem), &mask_buffer));\n CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &offset_mask));\n CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &mask_nb1));\n CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &mask_nb2));\n CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &mask_nb3));\n CL_CHECK(clSetKernelArg(kernel, 36, sizeof(int), &mask_ne2));\n CL_CHECK(clSetKernelArg(kernel, 37, sizeof(int), &mask_ne3));\n CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_mem), &sinks_buffer));\n CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_ulong), &offset_sinks));\n\n if (n_q == 1) {\n const size_t wg_size = 64;\n size_t local_work_size[] = { wg_size, 1 };\n size_t global_work_size[] = { wg_size, (size_t)(n_head * n_batch) };\n backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst);\n } else {\n const int block_m = backend_ctx->kernels_flash_attn_bm.at(dk_dv);\n const size_t wg_size = block_m;\n size_t local_work_size[] = { wg_size, 1 };\n size_t global_work_size[] = { (size_t)((n_q + block_m - 1) / block_m) * wg_size, (size_t)(n_head * n_batch) };\n backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_mul_mat_f16_f32_tiled(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int M = src0->ne[1];\n const int N = src1->ne[1];\n const int K = src0->ne[0];\n\n cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_tiled;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(int), &M));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &N));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &K));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd));\n\n // Tiling parameters. These need to be tuned for optimal performance.\n // They must match the #defines in the kernel mul_mat_f16_f32.cl.\n //\n // OPWM / OPWN: Output tile size per Work-Group. A work-group computes a tile of size OPWM x OPWN.\n // TPWM / TPWN: Threads per Work-group. This is the work-group size.\n // OPTM / OPTN: Output elements per Thread. Each thread computes OPTM x OPTN elements.\n //\n // The following relationships must hold:\n // OPWM = TPWM * OPTM\n // OPWN = TPWN * OPTN\n //\n const int OPWM = 64;\n const int OPWN = 64;\n const int TPWM = 16;\n const int TPWN = 8;\n\n size_t local_work_size[2] = { TPWM, TPWN };\n size_t global_work_size[2] = {\n (size_t) ((M + OPWM - 1) / OPWM) * TPWM,\n (size_t) ((N + OPWN - 1) / OPWN) * TPWN,\n };\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_conv_2d(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_TENSOR_BINARY_OP_LOCALS;\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const cl_uint Cout = ne03; const cl_uint Cin = ne02; const cl_uint N = ne13;\n const cl_uint KW = ne00; const cl_uint KH = ne01; const cl_uint W = ne10; const cl_uint H = ne11; const cl_uint OW = ne0; const cl_uint OH = ne1;\n\n const cl_uint s0 = dst->op_params[0]; const cl_uint s1 = dst->op_params[1];\n const cl_uint p0 = dst->op_params[2]; const cl_uint p1 = dst->op_params[3];\n const cl_uint d0 = dst->op_params[4]; const cl_uint d1 = dst->op_params[5];\n\n const cl_uint cl_nb01 = nb01/ggml_type_size(src0->type); const cl_uint cl_nb02 = nb02/ggml_type_size(src0->type); const cl_uint cl_nb03 = nb03/ggml_type_size(src0->type);\n const cl_uint cl_nb11 = nb11/ggml_type_size(src1->type); const cl_uint cl_nb12 = nb12/ggml_type_size(src1->type); const cl_uint cl_nb13 = nb13/ggml_type_size(src1->type);\n const cl_uint cl_nb1 = nb1/ggml_type_size(dst->type); const cl_uint cl_nb2 = nb2/ggml_type_size(dst->type); const cl_uint cl_nb3 = nb3/ggml_type_size(dst->type);\n\n const int64_t NPQ = (int64_t)N * OW * OH;\n\n const uint32_t BS_K = 64;\n const uint32_t BS_NPQ = 64;\n const uint32_t BS_CRS = 16;\n const uint32_t VEC_SIZE = 4;\n\n const uint32_t TS_K = 4;\n const uint32_t TS_NPQ = 8;\n\n const uint32_t WG_K = BS_K / TS_K;\n const uint32_t WG_NPQ = BS_NPQ / TS_NPQ;\n\n auto splitWork = [](uint32_t work_size, uint32_t block_size) { return (block_size + work_size - 1) / block_size; };\n const uint32_t NB_K = splitWork(Cout, BS_K);\n const uint32_t NB_NPQ = splitWork(NPQ, BS_NPQ);\n\n cl_kernel kernel;\n size_t shmem_size;\n\n if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {\n kernel = backend_ctx->kernel_conv_2d_f16;\n shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_half4));\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_conv_2d_f32;\n shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_float) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4));\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_conv_2d_f16_f32;\n shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4));\n } else {\n GGML_ASSERT(false && \"Unsupported data type combination for conv2d\");\n }\n\n cl_uint idx = 0;\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, idx++, shmem_size, NULL));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cout)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cin)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &N));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KW)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KH)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &W)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OW)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OH));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p1));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d1));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb01)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb02)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb03));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb11)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb12)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb13));\n CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb2)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb3));\n\n size_t global_work_size[] = { (size_t)NB_K * WG_K, (size_t)NB_NPQ * WG_NPQ, 1 };\n size_t local_work_size[] = { (size_t)WG_K, (size_t)WG_NPQ, 1 };\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n\n const int ne10 = src1->ne[0];\n const int ne11 = src1->ne[1];\n const int ne12 = src1->ne[2];\n\n const cl_ulong nb10 = src1->nb[0];\n\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n\n GGML_ASSERT(ne00 == ne10);\n\n cl_kernel kernel;\n cl_context context = backend_ctx->context;\n\n cl_int status;\n cl_image_format img_fmt_1d;\n cl_image_desc img_desc_1d;\n cl_buffer_region region;\n cl_mem A_image1d;\n cl_mem A_sub_buffer;\n cl_mem B_sub_buffer;\n cl_mem D_image1d;\n cl_mem D_sub_buffer;\n\n int M = ne01;\n int N = ne1;\n int K = ne00;\n\n if (nb01 > nb02) {\n // KQ\n kernel = backend_ctx->kernel_mul_mm_f16_f32_kq;\n } else {\n // KQV\n kernel = backend_ctx->kernel_mul_mm_f16_f32_kqv;\n }\n // create sub-buffer for A\n // <--------------------------------------------> //\n extra0 = src0->view_src ? (ggml_tensor_extra_cl *)src0->view_src->extra : (ggml_tensor_extra_cl *)src0->extra;\n\n region.origin = (extra0->offset);\n if (nb01 > nb02) {\n // KQ\n region.size = nb01 * ne01;\n } else {\n // KQV\n region.size = nb02 * ne02;\n }\n\n A_sub_buffer = clCreateSubBuffer((extra0->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);\n CL_CHECK(status);\n\n // <--------------------------------------------> //\n\n // create sub-buffer for B\n // <--------------------------------------------> //\n region.origin = (extra1->offset);\n region.size = nb10 * ne10 * ne11 * ne12;\n B_sub_buffer = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);\n CL_CHECK(status);\n // <--------------------------------------------> //\n\n img_fmt_1d = {CL_RGBA, CL_FLOAT};\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n if (nb01 > nb02) {\n img_desc_1d.image_width = (nb01 * ne01 / 4)/4;\n }\n else {\n img_desc_1d.image_width = (nb02 * ne02 / 4)/4;\n }\n img_desc_1d.buffer = A_sub_buffer;\n A_image1d = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status);\n CL_CHECK(status);\n\n // create sub-buffer for output C\n // <--------------------------------------------> //\n region.origin = (extrad->offset);\n region.size = ne0 * ne1 * dst->ne[2] * dst->nb[0]; // size of C in bytes\n D_sub_buffer = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);\n CL_CHECK(status);\n // <--------------------------------------------> //\n\n // create image for C output\n // <--------------------------------------------> //\n img_fmt_1d = {CL_R, CL_FLOAT};\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4;\n img_desc_1d.buffer = D_sub_buffer;\n D_image1d = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status);\n CL_CHECK(status);\n // <--------------------------------------------> //\n\n int offset_src0 = 0;\n int offset_src1 = 0;\n\n // set kernel args\n // <--------------------------------------------> //\n cl_uint k_arg = 0;\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src0));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_sub_buffer));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src1));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &D_image1d));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &extrad->offset));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &nb01));\n\n size_t global_work_size[3] = {64, static_cast(((M+63)/64)), static_cast(((N+31)/32)*ne12)};\n size_t local_work_size[3] = {64, 1, 2};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n\n // deallocate sub buffers and images\n // <--------------------------------------------> //\n CL_CHECK(clReleaseMemObject(A_image1d));\n CL_CHECK(clReleaseMemObject(D_image1d));\n CL_CHECK(clReleaseMemObject(A_sub_buffer));\n CL_CHECK(clReleaseMemObject(B_sub_buffer));\n CL_CHECK(clReleaseMemObject(D_sub_buffer));\n}\n\nstatic void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra;\n\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n\n const int ne1 = dst->ne[1];\n\n GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);\n\n cl_context context = backend_ctx->context;\n cl_kernel kernel;\n\n cl_int err;\n cl_image_format img_fmt;\n cl_image_desc img_desc;\n cl_buffer_region region;\n\n int M = ne01;\n int N = ne1;\n int K = ne00;\n\n if (ne1 == 1) {\n cl_mem q_img = nullptr;\n cl_mem b_sub_buf = nullptr;\n cl_mem b_img = nullptr;\n\n // image for q\n img_fmt = { CL_R, CL_UNSIGNED_INT32};\n memset(&img_desc, 0, sizeof(img_desc));\n img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc.image_width = M * K / 2 / 4;\n img_desc.buffer = extra0_q4_1->q;\n CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));\n\n // subbuffer for activations\n region.origin = offset1;\n region.size = K * N * sizeof(float);\n CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));\n\n // image for activations\n img_fmt = {CL_RGBA, CL_FLOAT};\n memset(&img_desc, 0, sizeof(img_desc));\n img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc.image_width = K * N / 4;\n img_desc.buffer = b_sub_buf;\n CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));\n\n kernel = backend_ctx->kernel_gemv_noshuffle_q4_1_f32;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01));\n\n size_t local_work_size[3] = {64, 4, 1};\n size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n\n CL_CHECK(clReleaseMemObject(q_img));\n CL_CHECK(clReleaseMemObject(b_sub_buf));\n CL_CHECK(clReleaseMemObject(b_img));\n } else {\n cl_mem b_sub_buf = nullptr;\n cl_mem b_sub_buf_trans = nullptr;\n cl_mem b_img = nullptr;\n cl_mem b_img_trans = nullptr;\n\n // subbuffer for activations\n region.origin = offset1;\n region.size = K * N * sizeof(float);\n CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));\n\n // image for activations\n img_fmt = {CL_RGBA, CL_FLOAT};\n memset(&img_desc, 0, sizeof(img_desc));\n img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc.image_width = K * N / 4;\n img_desc.buffer = b_sub_buf;\n CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));\n\n // pad N to multiple of 8\n int extra_elements = N % 8;\n int padding = 0;\n if (extra_elements > 0){\n padding = 8 - extra_elements;\n }\n\n // subbuffer for transposed activations\n region.origin = 0;\n region.size = K * (N + padding) * sizeof(float)/2;\n backend_ctx->prealloc_act_trans.allocate(context, region.size);\n CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));\n\n // image for transposed activations\n img_fmt = {CL_RGBA, CL_HALF_FLOAT};\n memset(&img_desc, 0, sizeof(img_desc));\n img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc.image_width = K * (N + padding) / 4;\n img_desc.buffer = b_sub_buf_trans;\n CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err));\n\n // transpose activations\n int height_B = N/4;\n if (height_B == 0) {\n height_B = 1;\n }\n int width_B = K/4;\n int padded_height_B = (N + padding)/4;\n\n kernel = backend_ctx->kernel_transpose_32_16;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B));\n\n size_t local_work_size_t[2] = { 1, 16 };\n size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B };\n backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst);\n\n // gemm\n kernel = backend_ctx->kernel_gemm_noshuffle_q4_1_f32;\n int padded_N = N + padding;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1));\n\n size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1};\n size_t local_work_size[3] = {1, 128, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n\n CL_CHECK(clReleaseMemObject(b_sub_buf));\n CL_CHECK(clReleaseMemObject(b_sub_buf_trans));\n CL_CHECK(clReleaseMemObject(b_img));\n CL_CHECK(clReleaseMemObject(b_img_trans));\n }\n#else\n GGML_UNUSED(backend);\n GGML_UNUSED(src0);\n GGML_UNUSED(src1);\n GGML_UNUSED(dst);\n#endif\n}\n\nstatic void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n const enum ggml_type src0t = src0->type;\n const enum ggml_type src1t = src1->type;\n\n GGML_ASSERT(src0t == GGML_TYPE_Q8_0);\n GGML_ASSERT(src1t == GGML_TYPE_F32);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra;\n\n GGML_ASSERT(src1->view_offs == 0);\n GGML_ASSERT(dst->view_offs == 0);\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n\n const int ne10 = src1->ne[0];\n const int ne12 = src1->ne[2];\n\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n\n GGML_ASSERT(ne00 == ne10);\n GGML_ASSERT((ne00 % 32) == 0);\n GGML_ASSERT(ne0 == ne01);\n\n cl_context context = backend_ctx->context;\n cl_kernel kernel;\n\n // init CL objects\n cl_int status;\n cl_image_format img_fmt_1d;\n cl_image_desc img_desc_1d;\n cl_buffer_region region;\n cl_mem A_image1d;\n cl_mem B_image1d;\n cl_mem B_sub_buffer;\n cl_mem S_image1d;\n\n cl_mem D_image1d;\n cl_mem D_sub_buffer;\n\n int M = ne01;\n int N = ne1;\n int K = ne00;\n\n // create an image for A\n img_fmt_1d = { CL_R, CL_FLOAT};\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * K / 4; // Divide by 4 for char -> float\n img_desc_1d.buffer = extra0_q8_0->q;\n A_image1d = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status);\n CL_CHECK(status);\n\n // create an image for Scale\n img_fmt_1d = { CL_R, CL_HALF_FLOAT};\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * K / 32; // Block size is 32\n img_desc_1d.buffer = extra0_q8_0->d;\n S_image1d = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status);\n CL_CHECK(status);\n\n // create a sub_buffer for B\n region.origin = (extra1->offset); // + src1->view_offs);\n region.size = K * N * sizeof(float);\n B_sub_buffer = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);\n CL_CHECK(status);\n\n // create an image for B from sub_buffer: RGBA (OCL)\n img_fmt_1d = {CL_RGBA, CL_FLOAT};\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = K * N / 4;\n img_desc_1d.buffer = B_sub_buffer;\n B_image1d = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status);\n CL_CHECK(status);\n\n // Create subbuffer and image1d_buffer for dst\n region.origin = (extrad->offset); // + dst->view_offs;\n region.size = M * N * sizeof(float);\n D_sub_buffer = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);\n CL_CHECK(status);\n\n img_fmt_1d = {CL_R, CL_FLOAT};\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * N;\n img_desc_1d.buffer = D_sub_buffer;\n D_image1d = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status);\n CL_CHECK(status);\n\n size_t local_work_size[3] = {1, 1, 1};\n size_t global_work_size[3] = {1, 1, 1};\n\n if (N == 1) {\n kernel = backend_ctx->CL_mul_mat_vec_q8_0_f32;\n\n int r2 = 1;\n int r3 = 1;\n cl_uint k_arg = 0;\n\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q8_0->d));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_image1d));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &extra1->offset));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &extrad->offset));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3));\n\n size_t wavesize = backend_ctx->adreno_wave_size;\n local_work_size[0] = wavesize;\n local_work_size[1] = 4; // reduce factor\n local_work_size[2] = 1;\n\n global_work_size[0] = ((M + wavesize - 1) / wavesize) * wavesize;\n global_work_size[1] = 4; // reduce factor\n global_work_size[2] = 1;\n } else {\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n cl_mem B_image1d_trans = nullptr;\n // for B transpose\n cl_mem B_d = nullptr;\n int padding;\n\n //how many extra elements beyond multiple of 8\n int extra_elements = N % 8;\n\n //how much padding to add\n padding = 0;\n if (extra_elements > 0){\n padding = 8 - extra_elements;\n }\n\n // Specify the starting offset (in bytes)\n region.origin = 0;\n // Specify the size of the sub-buffer (divide by 2 for FP16)\n region.size = K * (N + padding) * sizeof(float)/2;\n backend_ctx->prealloc_act_trans.allocate(context, region.size);\n B_d = clCreateSubBuffer(\n backend_ctx->prealloc_act_trans.buffer,\n 0,\n CL_BUFFER_CREATE_TYPE_REGION,\n ®ion,\n &status);\n CL_CHECK(status);\n\n cl_image_format image_format_B_d_output = { CL_RGBA, CL_HALF_FLOAT }; //(CL_HALF_FLOAT for FP16)\n cl_image_desc image_desc_B_d_output = {\n CL_MEM_OBJECT_IMAGE1D_BUFFER,\n static_cast(K * (N + padding)/4),\n 0, 0, 0, 0, 0, 0, 0, { B_d }\n };\n B_image1d_trans = clCreateImage(\n context,\n 0,\n &image_format_B_d_output,\n &image_desc_B_d_output,\n NULL,\n &status);\n CL_CHECK(status);\n\n int height_B = N/4;\n if (height_B == 0) {\n height_B = 1;\n }\n int width_B = K/4;\n int padded_height_B = (N + padding)/4;\n\n kernel = backend_ctx->kernel_transpose_32_16;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &B_image1d));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &B_image1d_trans));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B));\n\n size_t local_size_t[2] = { 1, 16 };\n size_t global_size_t[2] = {\n static_cast(width_B),\n static_cast(padded_height_B)\n };\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst);\n\n kernel = backend_ctx->kernel_mul_mm_q8_0_f32_8x4;\n\n int N_with_padding = N + padding;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &B_image1d_trans));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &N_with_padding));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd));\n\n global_work_size[0] = (size_t)(N + 7) / 8;\n global_work_size[1] = (size_t)(M + 3) / 4;\n global_work_size[2] = 1;\n\n local_work_size[0] = 2;\n local_work_size[1] = 128;\n local_work_size[2] = 1;\n }\n\n // enqueue kernel with profiling\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n\n // deallocate sub buffers and images\n CL_CHECK(clReleaseMemObject(A_image1d));\n CL_CHECK(clReleaseMemObject(B_sub_buffer));\n CL_CHECK(clReleaseMemObject(B_image1d));\n CL_CHECK(clReleaseMemObject(S_image1d));\n CL_CHECK(clReleaseMemObject(D_sub_buffer));\n CL_CHECK(clReleaseMemObject(D_image1d));\n#else\n GGML_UNUSED(backend);\n GGML_UNUSED(src0);\n GGML_UNUSED(src1);\n GGML_UNUSED(dst);\n#endif\n}\n\nstatic void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT;\n const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT;\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n#ifdef GGML_OPENCL_SOA_Q\n ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra;\n ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra;\n ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)src0->extra;\n ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra;\n ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra;\n#endif\n\n const int ne00 = src0 ? src0->ne[0] : 0;\n const int ne01 = src0 ? src0->ne[1] : 0;\n const int ne02 = src0 ? src0->ne[2] : 0;\n const int ne03 = src0 ? src0->ne[3] : 0;\n\n const cl_ulong nb00 = src0 ? src0->nb[0] : 0;\n const cl_ulong nb01 = src0 ? src0->nb[1] : 0;\n const cl_ulong nb02 = src0 ? src0->nb[2] : 0;\n const cl_ulong nb03 = src0 ? src0->nb[3] : 0;\n\n const int ne10 = src1 ? src1->ne[0] : 0;\n const int ne11 = src1 ? src1->ne[1] : 0;\n const int ne12 = src1 ? src1->ne[2] : 0;\n const int ne13 = src1 ? src1->ne[3] : 0;\n\n const cl_ulong nb10 = src1 ? src1->nb[0] : 0;\n const cl_ulong nb11 = src1 ? src1->nb[1] : 0;\n const cl_ulong nb12 = src1 ? src1->nb[2] : 0;\n const cl_ulong nb13 = src1 ? src1->nb[3] : 0;\n\n const int ne0 = dst ? dst->ne[0] : 0;\n const int ne1 = dst ? dst->ne[1] : 0;\n\n int r2 = ne12/ne02;\n int r3 = ne13/ne03;\n\n GGML_ASSERT(ne00 == ne10);\n\n int nth0 = 32;\n int nth1 = 1;\n int nrows = 1;\n // The number of values produced by each subgroup\n int ndst = 4;\n\n cl_kernel kernel;\n\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n cl_context context = backend_ctx->context;\n\n if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32){\n if (ne01 >= 64 && ne1 >= 32 && ne00 >= 16 && (ne12 % ne02) == 0 &&\n // dst is wrapped with image1d_buffer, the size limit applies, also src0\n (ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4 <= backend_ctx->image_max_buffer_size)) {\n // For KQ\n if (ggml_is_permuted(src0) && ggml_is_permuted(src1) &&\n ((nb01 * ne01 / 4)/4 <= backend_ctx->image_max_buffer_size) &&\n nb00 <= nb02 &&\n nb02 <= nb01 &&\n nb01 <= nb03 &&\n nb10 <= nb12 &&\n nb12 <= nb11 &&\n nb11 <= nb13) {\n ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst);\n return;\n }\n // For KQV\n if (!ggml_is_contiguous(src0) && ggml_is_contiguous(src1) &&\n ((nb02 * ne02 / 4)/4 <= backend_ctx->image_max_buffer_size)) {\n ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst);\n return;\n }\n }\n }\n\n if (ne01 && ne1 && use_adreno_kernels(backend_ctx, src0)) {\n\n // init CL objects\n // <--------------------------------------------> //\n cl_int status;\n cl_image_format img_fmt_1d;\n cl_image_desc img_desc_1d;\n cl_buffer_region region;\n cl_mem A_image1d = nullptr;\n cl_mem B_image1d = nullptr;\n cl_mem B_sub_buffer = nullptr;\n cl_mem C_d = nullptr;\n // for B transpose\n cl_mem B_d = nullptr;\n cl_mem B_d_input_image = nullptr;\n // <--------------------------------------------> //\n\n // define matrix dimensions\n // <--------------------------------------------> //\n int M = ne01;\n int N = ne1;\n int K = ne00;\n int padding;\n // <--------------------------------------------> //\n\n // NOTE: Kernels using image1d_buffer_t (e.g., src0_q) would normally require\n // a limit check, but q4_0 / q4_1 tensors are very unlikely to exceed that\n // limit, so the check is omitted.\n\n // q4_1 x fp32\n if (src0t == GGML_TYPE_Q4_1 && src1t == GGML_TYPE_F32) {\n ggml_cl_mul_mat_q4_1_f32_adreno(backend, src0, src1, dst);\n return;\n }\n\n // q8_0 x fp32\n if (src0t == GGML_TYPE_Q8_0 && src1t == GGML_TYPE_F32 &&\n enable_adreno_trans_weight(backend_ctx, src0)) {\n ggml_cl_mul_mat_q8_0_f32_adreno(backend, src0, src1, dst);\n return;\n }\n\n // q4_0 x fp32\n if(src0t == GGML_TYPE_Q4_0 && src1t == GGML_TYPE_F32) {\n // TODO: remove duplicate definitions of image description + format -- move to top\n\n // create an image for A\n // <--------------------------------------------> //\n if (N == 1) {\n img_fmt_1d = { CL_R, CL_UNSIGNED_INT32};\n } else {\n img_fmt_1d = { CL_R, CL_FLOAT};\n }\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.image_width = M * K / 2 / 4; // Divide by 4 for char -> float\n img_desc_1d.buffer = extra0_q4_0->q;\n A_image1d = clCreateImage(\n context,\n CL_MEM_READ_ONLY,\n &img_fmt_1d,\n &img_desc_1d,\n NULL,\n &status);\n CL_CHECK(status);\n // <--------------------------------------------> //\n\n\n // create a sub_buffer for B\n // <--------------------------------------------> //\n region.origin = (extra1->offset);\n region.size = K * N * sizeof(float);\n B_sub_buffer = clCreateSubBuffer(\n extra1->data_device,\n 0,\n CL_BUFFER_CREATE_TYPE_REGION,\n ®ion,\n &status);\n CL_CHECK(status);\n // <--------------------------------------------> //\n\n // transpose activation for Skyler's gemm\n if (N != 1) {\n //how many extra elements beyond multiple of 8\n int extra_elements = N % 8;\n\n //how much padding to add\n padding = 0;\n if (extra_elements > 0){\n padding = 8 - extra_elements;\n }\n\n // Specify the starting offset (in bytes)\n region.origin = 0;\n // Specify the size of the sub-buffer (divide by 2 for FP16)\n region.size = K * (N + padding) * sizeof(float)/2;\n backend_ctx->prealloc_act_trans.allocate(context, region.size);\n\n B_d = clCreateSubBuffer(\n backend_ctx->prealloc_act_trans.buffer,\n 0,\n CL_BUFFER_CREATE_TYPE_REGION,\n ®ion,\n &status);\n CL_CHECK(status);\n\n cl_image_format image_format_B_d_input = { CL_RGBA, CL_FLOAT };\n cl_image_desc image_desc_B_d_input = {\n CL_MEM_OBJECT_IMAGE1D_BUFFER,\n static_cast(K * N / 4),\n 0, 0, 0, 0, 0, 0, 0, { B_sub_buffer }\n };\n B_d_input_image = clCreateImage(\n context,\n 0,\n &image_format_B_d_input,\n &image_desc_B_d_input,\n NULL,\n &status);\n CL_CHECK(status);\n\n cl_image_format image_format_B_d_output = { CL_RGBA, CL_HALF_FLOAT }; //(CL_HALF_FLOAT for FP16)\n cl_image_desc image_desc_B_d_output = {\n CL_MEM_OBJECT_IMAGE1D_BUFFER,\n static_cast(K * (N + padding)/4),\n 0, 0, 0, 0, 0, 0, 0, { B_d }\n };\n B_image1d = clCreateImage(\n context,\n 0,\n &image_format_B_d_output,\n &image_desc_B_d_output,\n NULL,\n &status);\n CL_CHECK(status);\n\n int height_B = N/4;\n if (height_B == 0) {\n height_B = 1;\n }\n int width_B = K/4;\n int padded_height_B = (N + padding)/4;\n\n kernel = backend_ctx->kernel_transpose_32_16;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &B_d_input_image));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &B_image1d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B));\n\n size_t local_size_t[2] = { 1, 16 };\n //WGS tuning\n if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) {\n local_size_t[0]=4;\n local_size_t[1]=8;\n } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) {\n local_size_t[0]=2;\n local_size_t[1]=8;\n } else if(ne0 == 4096 && ne1 == 128 && ne10 == 11008) {\n local_size_t[0]=1;\n local_size_t[1]=8;\n } else if(ne0 == 32000 && ne1 == 128 && ne10 == 4096) {\n local_size_t[0]=2;\n local_size_t[1]=8;\n }\n\n size_t global_size_t[2] = {\n static_cast(width_B),\n static_cast(padded_height_B)\n };\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst);\n } else {\n // no need to transpose B in other cases\n // create an image for B from sub_buffer\n // <--------------------------------------------> //\n img_fmt_1d = {CL_RGBA, CL_FLOAT};\n\n memset(&img_desc_1d, 0, sizeof(img_desc_1d));\n img_desc_1d.image_width = K * N / 4;\n img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;\n img_desc_1d.buffer = B_sub_buffer;\n B_image1d = clCreateImage(\n context,\n CL_MEM_READ_ONLY,\n &img_fmt_1d,\n &img_desc_1d,\n NULL,\n &status);\n CL_CHECK(status);\n // <--------------------------------------------> //\n }\n\n // choose gemm or gemv kernel\n // <--------------------------------------------> //\n if (N == 1) {\n kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_general;\n if (M == 4096 && K == 4096) {\n kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_4096;\n } else if (M == 4096 && K == 11008) {\n kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_11008;\n } else if (M == 11008 && K == 4096) {\n kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_11008_1_4096;\n } else if (M == 32000 && K == 4096) {\n kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_32000_1_4096;\n }\n } else {\n kernel = backend_ctx->CL_mul_mat_Ab_Bi_8x4;\n }\n // <--------------------------------------------> //\n\n // set kernel args\n // <--------------------------------------------> //\n cl_uint k_arg = 0;\n\n if (N == 1) {\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_image1d));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &extra1->offset));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &extrad->offset));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3));\n } else {\n region.origin = extrad->offset; // Specify the starting offset (in bytes)\n region.size = M * N * sizeof(float); // Specify the size of the sub-buffer\n C_d = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);\n CL_CHECK(status);\n\n int padded_N = ne1 + padding;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); //A_q_dextra0_q4_0->q\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); //A_s_d\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &B_image1d)); //B_d\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &C_d)); //C_d\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); //M\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &padded_N)); //N with padding\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); //K\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1)); //N without padding\n }\n // <--------------------------------------------> //\n\n // choose workgroup size\n // <--------------------------------------------> //\n size_t global_work_size[3] = {\n 64, static_cast((M+63)/64), static_cast((N+31)/32)};\n size_t local_work_size[3] = {64, 2, 4};\n\n global_work_size[0] = (size_t)(ceil((float)ne1/8));\n global_work_size[1] = (size_t)(ne01/4);\n global_work_size[2] = (size_t)(1);\n\n local_work_size[0] = (size_t)(1); //4x32 for FP32\n local_work_size[1] = (size_t)(128);\n local_work_size[2] = (size_t)(1);\n\n //WGS tuning\n if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) {\n local_work_size[0] = 1;\n local_work_size[1] = 128;\n } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) {\n local_work_size[0] = 2;\n local_work_size[1] = 64;\n } else if (ne0 == 4096 && ne1 == 128 && ne10 == 11008) {\n local_work_size[0] = 2;\n local_work_size[1] = 64;\n } else if (ne0 == 32000 && ne1 == 128 && ne10 == 4096) {\n local_work_size[0] = 2;\n local_work_size[1] = 64;\n }\n\n if (N == 1) {\n size_t wavesize = backend_ctx->adreno_wave_size;\n local_work_size[0] = wavesize; // localsize\n local_work_size[1] = 4; // reduce factor\n local_work_size[2] = 1;\n\n global_work_size[0] = (((M / 2) + wavesize - 1) / wavesize) * wavesize;\n global_work_size[1] = 4; // reduce factor\n global_work_size[2] = 1;\n }\n // <--------------------------------------------> //\n\n // enqueue kernel with profiling\n // <--------------------------------------------> //\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n // <--------------------------------------------> //\n\n // deallocate sub buffers and images\n // <--------------------------------------------> //\n CL_CHECK(clReleaseMemObject(A_image1d));\n CL_CHECK(clReleaseMemObject(B_sub_buffer));\n CL_CHECK(clReleaseMemObject(B_image1d));\n\n if (N != 1) {\n CL_CHECK(clReleaseMemObject(B_d));\n CL_CHECK(clReleaseMemObject(B_d_input_image));\n CL_CHECK(clReleaseMemObject(C_d));\n }\n // <--------------------------------------------> //\n\n return;\n }\n } // if (ne01 && ne1)\n#endif // GGML_OPENCL_USE_ADRENO_KERNELS\n\n // GEMM using local memory\n // Current BK = 16, so ne00 % 16 == 0\n if (src1t == GGML_TYPE_F32 &&\n ne00 % 16 == 0 &&\n ne11 > 1) {\n switch(src0t) {\n case GGML_TYPE_F32: {\n kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm;\n nth0 = 128; // calculated as (BM*BN)/(TM*TN)\n\n int batch_stride_a = ne00*ne01;\n int batch_stride_b = ne10*ne11;\n int batch_stride_d = ne0*ne1;\n\n cl_mem mem_src0 = extra0->data_device;\n cl_mem mem_src1 = extra1->data_device;\n\n cl_ulong nb00_cont = nb00;\n cl_ulong nb01_cont = nb01;\n cl_ulong nb02_cont = nb02;\n cl_ulong nb03_cont = nb03;\n\n cl_ulong nb10_cont = nb10;\n cl_ulong nb11_cont = nb11;\n cl_ulong nb12_cont = nb12;\n cl_ulong nb13_cont = nb13;\n\n cl_ulong offset0_cont = offset0;\n cl_ulong offset1_cont = offset1;\n\n if (!ggml_is_contiguous(src0)) {\n backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0));\n ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer,\n nb00_cont, nb01_cont, nb02_cont, nb03_cont);\n mem_src0 = backend_ctx->prealloc_src0.buffer;\n offset0_cont = 0;\n }\n\n if (!ggml_is_contiguous(src1)) {\n backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1));\n ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer,\n nb10_cont, nb11_cont, nb12_cont, nb13_cont);\n mem_src1 = backend_ctx->prealloc_src1.buffer;\n offset1_cont = 0;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));\n\n // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed.\n size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n return;\n }\n case GGML_TYPE_F16: {\n kernel = backend_ctx->kernel_mul_mm_f16_f32_l4_lm;\n nth0 = 128; // calculated as (BM*BN)/(TM*TN)\n\n int batch_stride_a = ne00*ne01;\n int batch_stride_b = ne10*ne11;\n int batch_stride_d = ne0*ne1;\n\n cl_mem mem_src0 = extra0->data_device;\n cl_mem mem_src1 = extra1->data_device;\n\n cl_ulong nb00_cont = nb00;\n cl_ulong nb01_cont = nb01;\n cl_ulong nb02_cont = nb02;\n cl_ulong nb03_cont = nb03;\n\n cl_ulong nb10_cont = nb10;\n cl_ulong nb11_cont = nb11;\n cl_ulong nb12_cont = nb12;\n cl_ulong nb13_cont = nb13;\n\n cl_ulong offset0_cont = offset0;\n cl_ulong offset1_cont = offset1;\n\n if (!ggml_is_contiguous(src0)) {\n backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0));\n ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer,\n nb00_cont, nb01_cont, nb02_cont, nb03_cont);\n mem_src0 = backend_ctx->prealloc_src0.buffer;\n offset0_cont = 0;\n }\n\n if (!ggml_is_contiguous(src1)) {\n backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1));\n ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer,\n nb10_cont, nb11_cont, nb12_cont, nb13_cont);\n mem_src1 = backend_ctx->prealloc_src1.buffer;\n offset1_cont = 0;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));\n\n // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed.\n size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n return;\n }\n case GGML_TYPE_Q4_0: {\n if (ne11 < 32) {\n break;\n }\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {\n break;\n }\n\n kernel = backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm;\n nth0 = 128; // calculated as (BM*BN)/(TM*TN)\n\n int batch_stride_a = ne00*ne01;\n int batch_stride_b = ne10*ne11;\n int batch_stride_d = ne0*ne1;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));\n\n // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed.\n size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n return;\n }\n case GGML_TYPE_Q4_1: {\n if (ne11 < 32) {\n break;\n }\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {\n break;\n }\n\n kernel = backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm;\n nth0 = 128; // calculated as (BM*BN)/(TM*TN)\n\n int batch_stride_a = ne00*ne01;\n int batch_stride_b = ne10*ne11;\n int batch_stride_d = ne0*ne1;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3));\n\n // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed.\n size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n return;\n }\n case GGML_TYPE_Q8_0: {\n if (ne11 < 32) {\n break;\n }\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {\n break;\n }\n\n kernel = backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm;\n nth0 = 128; // calculated as (BM*BN)/(TM*TN)\n\n int batch_stride_a = ne00*ne01;\n int batch_stride_b = ne10*ne11;\n int batch_stride_d = ne0*ne1;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));\n\n // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed.\n size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n return;\n }\n case GGML_TYPE_Q6_K: {\n if (ne11 < 32) {\n break;\n }\n if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {\n break;\n }\n\n kernel = backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm;\n nth0 = 128; // calculated as (BM*BN)/(TM*TN)\n\n int batch_stride_a = ne00*ne01;\n int batch_stride_b = ne10*ne11;\n int batch_stride_d = ne0*ne1;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3));\n\n // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed.\n size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n return;\n }\n default:\n break;\n }\n }\n\n if (src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32 &&\n src0->ne[1] > 32 && // M > 32\n src1->ne[1] > 32 && // N > 32\n src0->ne[0] > 32 && // K > 32\n src0->ne[2] == 1 && src0->ne[3] == 1 &&\n src1->ne[2] == 1 && src1->ne[3] == 1 &&\n ggml_is_contiguous(src0) && ggml_is_contiguous(src1) &&\n backend_ctx->kernel_mul_mat_f16_f32_tiled != NULL) {\n ggml_cl_mul_mat_f16_f32_tiled(backend, src0, src1, dst);\n return;\n }\n\n if (!ggml_is_transposed(src0) &&\n !ggml_is_transposed(src1) &&\n src1t == GGML_TYPE_F32 &&\n ne00%32 == 0 &&\n ne11 > 2) {\n#ifdef GGML_OPENCL_SOA_Q\n // Set up kernel.\n switch(src0t) {\n case GGML_TYPE_Q4_0:\n // This should have been satisfied.\n GGML_ASSERT(ne11 == ne1);\n GGML_ASSERT(ne01 == ne0);\n\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 1;\n\n kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_16x_flat;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 1;\n\n kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_8x_flat;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));\n break;\n default:\n break;\n }\n\n // Launch kernel.\n if (src0t == GGML_TYPE_Q4_0) {\n size_t global_work_size[] = {(size_t)(ne01 + 7)/8*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};\n\n if (backend_ctx->gpu_family == INTEL) {\n // Set global size for Intel. It uses 16x output values.\n global_work_size[0] = (size_t)(ne01 + 15)/16*nth0;\n global_work_size[1] = (size_t)ne11*nth1;\n global_work_size[2] = (size_t)ne12*ne13;\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n return;\n }\n#else // GGML_OPENCL_SOA_Q\n // TODO: add block_q4_0 variant.\n#endif // GGML_OPENCL_SOA_Q\n }\n\n // use custom matrix x vector kernel\n switch (src0t) {\n case GGML_TYPE_F32:\n //GGML_ASSERT(ne02 == ne12);\n GGML_ASSERT(src1t == GGML_TYPE_F32);\n kernel = backend_ctx->kernel_mul_mat_f32_f32;\n nrows = 4;\n\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 32;\n nth1 = 1;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 1;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3));\n break;\n case GGML_TYPE_F16:\n //GGML_ASSERT(ne02 == ne12);\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 32;\n nth1 = 1;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 1;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n if (src1t == GGML_TYPE_F32) {\n if (ne11 * ne12 < 4) {\n kernel = backend_ctx->kernel_mul_mat_f16_f32_1row;\n } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {\n kernel = backend_ctx->kernel_mul_mat_f16_f32_l4;\n nrows = ne11;\n } else {\n kernel = backend_ctx->kernel_mul_mat_f16_f32;\n nrows = 4;\n }\n } else {\n kernel = backend_ctx->kernel_mul_mat_f16_f16;\n nrows = 4;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3));\n break;\n case GGML_TYPE_Q4_0:\n // This should have been satisfied.\n GGML_ASSERT(ne11 == ne1);\n GGML_ASSERT(ne01 == ne0);\n\n#ifdef GGML_OPENCL_SOA_Q\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 1;\n\n kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat;\n ndst = 8;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 1;\n\n kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat;\n ndst =8;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));\n#else // GGML_OPENCL_SOA_Q\n if (backend_ctx->gpu_family == INTEL) {\n // Use 1D local size. Each workgroup is a SIMD group. Each SIMD\n // group produces N_DST (4 for Q4_0 kernel) values in the result.\n // The number of workgroups on dim 0 (the leading dimension) is\n // the nearest multiple of 4 that covers ne0 (equals ne01).\n nth0 = 16;\n nth1 = 1;\n\n kernel = backend_ctx->kernel_mul_mat_q4_0_f32;\n ndst = 4;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 1;\n\n kernel = backend_ctx->kernel_mul_mat_q4_0_f32_v;\n ndst = 4;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));\n#endif // GGML_OPENCL_SOA_Q\n break;\n case GGML_TYPE_Q4_1: {\n#ifdef GGML_OPENCL_SOA_Q\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 1;\n ndst = 4;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 1;\n ndst = 4;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n kernel = backend_ctx->kernel_mul_mv_q4_1_f32_flat;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3));\n#else\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 1;\n ndst = 4;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 1;\n ndst = 4;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n kernel = backend_ctx->kernel_mul_mv_q4_1_f32;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));\n#endif // GGML_OPENCL_SOA_Q\n break;\n }\n case GGML_TYPE_Q8_0: {\n#ifdef GGML_OPENCL_SOA_Q\n kernel = backend_ctx->kernel_mul_mv_q8_0_f32_flat;\n\n // nth0 - subgroup size\n // nth1 - number of subgroups per workgroup\n // ndst - number of output values per workgroup = output per subgroup * number of subgroups\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 2;\n ndst = nth1*4;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 2;\n ndst = nth1*4;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));\n#else\n kernel = backend_ctx->kernel_mul_mv_q8_0_f32;\n\n // nth0 - subgroup size\n // nth1 - number of subgroups per workgroup\n // ndst - number of output values per workgroup = output per subgroup * number of subgroups\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 2;\n ndst = nth1*4;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 2;\n ndst = nth1*4;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));\n#endif // GGML_OPENCL_SOA_Q\n break;\n }\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K: {\n kernel = backend_ctx->kernel_mul_mv_q4_K_f32;\n\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 1;\n ndst = 4;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 1;\n ndst = 4;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));\n break;\n }\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_Q6_K:\n#ifdef GGML_OPENCL_SOA_Q\n kernel = backend_ctx->kernel_mul_mv_q6_K_f32_flat;\n\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 2;\n ndst = 4;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 2;\n ndst = 4;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3));\n#else\n kernel = backend_ctx->kernel_mul_mv_q6_K_f32;\n\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 2;\n ndst = 1;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 2;\n ndst = 1;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));\n#endif // GGML_OPENCL_SOA_Q\n break;\n case GGML_TYPE_MXFP4: {\n#ifdef GGML_OPENCL_SOA_Q\n kernel = backend_ctx->kernel_mul_mv_mxfp4_f32_flat;\n\n cl_mem q;\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 2;\n ndst = nth1*2;\n\n q = extra0_mxfp4->q;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 2;\n ndst = nth1*2;\n\n q = extra0_mxfp4->q_img;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_mxfp4->e));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3));\n#else\n kernel = backend_ctx->kernel_mul_mv_mxfp4_f32;\n\n if (backend_ctx->gpu_family == INTEL) {\n nth0 = 16;\n nth1 = 2;\n ndst = nth1*2;\n } else if (backend_ctx->gpu_family == ADRENO) {\n nth0 = 64;\n nth1 = 2;\n ndst = nth1*2;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float)*nth0,nullptr));\n#endif\n break;\n }\n default:\n GGML_ASSERT(false && \"not implemented\");\n }\n\n if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_MXFP4 ||\n src0t == GGML_TYPE_Q4_1 ||\n src0t == GGML_TYPE_Q8_0 ||\n src0t == GGML_TYPE_Q2_K) {\n // Each SIMD group produces N_DST values in the result. Assuming each\n // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will\n // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size\n // (number of workgroups) will be a nearest multiple of\n // N_DST*N_SIMDGROUP to cover the size of the dimension. Below, 4 is\n // N_DST*N_SIMDGROUP (see the kernel for Q4_0 matmul).\n size_t global_work_size[] = {(size_t)(ne01 + ndst-1)/ndst*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n } else if (src0t == GGML_TYPE_Q4_K) {\n size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n } else if (src0t == GGML_TYPE_Q3_K) {\n GGML_ASSERT(false && \"not implemented\");\n } else if (src0t == GGML_TYPE_Q5_K) {\n GGML_ASSERT(false && \"not implemented\");\n } else if (src0t == GGML_TYPE_Q6_K) {\n size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n } else {\n int64_t ny = (ne11 + nrows - 1)/nrows;\n\n size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13};\n size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n }\n}\n\nstatic void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n const ggml_tensor * src2 = dst->src[2];\n GGML_ASSERT(src2);\n GGML_ASSERT(src2->extra);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offset2 = extra2->offset + src2->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n GGML_UNUSED(offset0);\n\n#ifdef GGML_OPENCL_SOA_Q\n ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra;\n ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)src0->extra;\n ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra;\n#endif\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const int ne10 = src1->ne[0];\n const int ne11 = src1->ne[1];\n const int ne12 = src1->ne[2];\n const int ne13 = src1->ne[3];\n\n const cl_ulong nb11 = src1->nb[1];\n const cl_ulong nb12 = src1->nb[2];\n const cl_ulong nb13 = src1->nb[3];\n\n const int ne20 = src2->ne[0];\n const int ne21 = src2->ne[1];\n\n const cl_ulong nb21 = src2->nb[1];\n const cl_ulong nb20 = src2->nb[0];\n\n UNUSED(nb20);\n\n const int ne0 = dst->ne[0];\n const int ne1 = dst->ne[1];\n\n const int r2 = ne12/ne02;\n const int r3 = ne13/ne03;\n const int dst_rows = ne20*ne21; // ne20 = n_used_experts, ne21 = n_rows\n\n GGML_ASSERT(ne00 == ne10);\n\n int sgs = 32; // subgroup size\n int nsg = 1; // number of subgroups\n int nrows = 1; // number of row in src1\n int ndst = 4; // number of values produced by each subgroup\n\n cl_kernel kernel;\n\n // subgroup mat vec\n switch (src0->type) {\n case GGML_TYPE_Q4_0: {\n kernel = backend_ctx->kernel_mul_mv_id_q4_0_f32_8x_flat;\n\n if (backend_ctx->gpu_family == INTEL) {\n sgs = 16;\n nsg = 1;\n ndst = 8;\n } else if (backend_ctx->gpu_family == ADRENO) {\n sgs = 64;\n nsg = 1;\n ndst = 8;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne20));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne21));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb21));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &r3));\n\n break;\n }\n case GGML_TYPE_Q8_0: {\n#ifdef GGML_OPENCL_SOA_Q\n kernel = backend_ctx->kernel_mul_mv_id_q8_0_f32_flat;\n\n if (backend_ctx->gpu_family == INTEL) {\n sgs = 16;\n nsg = 2;\n ndst = 4;\n } else if (backend_ctx->gpu_family == ADRENO) {\n sgs = 64;\n nsg = 2;\n ndst = 4;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne20));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne21));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb21));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne1));\n#else\n kernel = backend_ctx->kernel_mul_mv_id_q8_0_f32;\n\n if (backend_ctx->gpu_family == INTEL) {\n sgs = 16;\n nsg = 2;\n ndst = 4;\n } else if (backend_ctx->gpu_family == ADRENO) {\n sgs = 64;\n nsg = 2;\n ndst = 4;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne20));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne21));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb21));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne1));\n#endif // GGML_OPENCL_SOA_Q\n break;\n }\n case GGML_TYPE_MXFP4: {\n#ifdef GGML_OPENCL_USE_ADRENO_KERNELS\n if (use_adreno_moe_kernels(backend_ctx, src0)) {\n cl_int status;\n\n size_t local_size[3] = {64, 2, 1};\n size_t global_size[3] = {64, 2, 1};\n\n cl_mem src1_sub_buffer, buf_src1_image, buf_src2;\n\n int tile_size = 320;\n if (ne12 == 1) { // for gemv\n kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32;\n\n // create a sub_buffer for src2\n cl_buffer_region region;\n region.origin = offset2;\n region.size = ne20 * ne21 * sizeof(int);\n buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);\n CL_CHECK(status);\n\n // set thread grid\n global_size[0] = static_cast(ne01);\n global_size[1] = 4;\n global_size[2] = static_cast(ne20);\n local_size[1] = 4;\n } else { // for gemm\n kernel = backend_ctx->kernel_gemm_moe_mxfp4_f32;\n\n // preprocess router table\n int num_tiles_per_expert = (ne01 + tile_size - 1) / tile_size;\n void * host_src2_reorder = malloc(ne20 * ne21 * 4 * num_tiles_per_expert * sizeof(short));\n void * host_src2 = malloc(ne21 * nb21);\n CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, extra2->data_device, CL_TRUE, offset2, ne21 * nb21, host_src2, 0, NULL, NULL));\n int total_experts = nb21 / nb20;\n int out_idx = 0;\n for (int i_expert = 0; i_expert < ne02; i_expert++) {\n for (int i_tile = 0; i_tile < num_tiles_per_expert; i_tile++) {\n for (int j = 0; j < ne21; j++) {\n for (int i = 0; i < ne20; i++) {\n int expert = ((int *)host_src2)[j * total_experts + i];\n if (i_expert == expert) {\n ((short *)host_src2_reorder)[out_idx] = static_cast(expert);\n ((short *)host_src2_reorder)[out_idx + 1] = static_cast(j * ne11 + (i % ne11));\n ((short *)host_src2_reorder)[out_idx + 2] = static_cast(j * ne20 + i);\n ((short *)host_src2_reorder)[out_idx + 3] = static_cast(i_tile);\n out_idx += 4;\n }\n }\n }\n }\n }\n buf_src2 = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, ne20 * ne21 * 4 * num_tiles_per_expert * sizeof(short), host_src2_reorder, &status);\n CL_CHECK(status);\n\n // set thread grid\n global_size[0] = static_cast(tile_size);\n global_size[2] = static_cast(ne20 * ne21 * num_tiles_per_expert);\n }\n\n // create a sub_buffer for src1\n cl_buffer_region region;\n region.origin = offset1;\n region.size = ne10 * ne11 * ne12 * sizeof(float);\n src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);\n CL_CHECK(status);\n\n // create image for src1\n cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};\n cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}};\n buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);\n CL_CHECK(status);\n\n // Set kernel args\n int arg_idx = 0;\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q));\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->e));\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image));\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2));\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01));\n if (ne12 == 1) {\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11));\n } else {\n CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &tile_size));\n }\n\n // launch kernel\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst);\n\n // deallocate sub buffers and images\n CL_CHECK(clReleaseMemObject(src1_sub_buffer));\n CL_CHECK(clReleaseMemObject(buf_src1_image));\n CL_CHECK(clReleaseMemObject(buf_src2));\n return;\n } // else fallback to generic kernel\n#endif // GGML_OPENCL_USE_ADRENO_KERNELS\n\n#ifdef GGML_OPENCL_SOA_Q\n kernel = backend_ctx->kernel_mul_mv_id_mxfp4_f32_flat;\n\n cl_mem q;\n if (backend_ctx->gpu_family == INTEL) {\n sgs = 16;\n nsg = 2;\n ndst = 2;\n\n q = extra0_mxfp4->q;\n } else if (backend_ctx->gpu_family == ADRENO) {\n sgs = 64;\n nsg = 1;\n ndst = 4;\n\n q = extra0_mxfp4->q_img;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_mxfp4->e));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne20));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne21));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb21));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3));\n#else // GGML_OPENCL_SOA_Q\n kernel = backend_ctx->kernel_mul_mv_id_mxfp4_f32;\n\n if (backend_ctx->gpu_family == INTEL) {\n sgs = 16;\n nsg = 2;\n ndst = 2;\n } else if (backend_ctx->gpu_family == ADRENO) {\n sgs = 64;\n nsg = 2;\n ndst = 2;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne20));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne21));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb21));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float)*sgs,nullptr));\n#endif // GGML_OPENCL_SOA_Q\n break;\n }\n default:\n GGML_ASSERT(false && \"not implemented\");;\n }\n\n int _ne1 = 1;\n int ne123 = dst_rows;\n\n size_t global_work_size[] = {(size_t)(ne01+ndst*nsg-1)/(ndst*nsg)*sgs, (size_t)(_ne1+nrows-1)/nrows*nsg, (size_t)ne123};\n size_t local_work_size[] = {(size_t)sgs, (size_t)nsg, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_scale(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_UNUSED(src1);\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n float scale;\n float bias;\n memcpy(&scale, ((int32_t *) dst->op_params) + 0, sizeof(float));\n memcpy(&bias, ((int32_t *) dst->op_params) + 1, sizeof(float));\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n int n = ggml_nelements(dst);\n\n if (n % 4 == 0) {\n kernel = backend_ctx->kernel_scale_f32_4;\n n /= 4;\n } else {\n kernel = backend_ctx->kernel_scale_f32;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &scale));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &bias));\n\n size_t global_work_size[] = {(size_t)n, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr; // Let driver choose the work-group sizes.\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n}\n\nstatic void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n\n // GGML_OP_CPY happens between src0 and src1.\n // GGML_OP_DUP and GGML_OP_CONT happen between src0 and dst.\n UNUSED(dst);\n\n GGML_TENSOR_LOCALS(int, ne0, src0, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb);\n GGML_TENSOR_LOCALS(int, ne1, src1, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb);\n\n const enum ggml_type src0t = src0->type;\n const enum ggml_type src1t = src1->type;\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n\n cl_kernel kernel;\n\n switch (src0t) {\n case GGML_TYPE_F32:\n switch (src1t) {\n case GGML_TYPE_F16:\n kernel = backend_ctx->kernel_cpy_f32_f16;\n break;\n case GGML_TYPE_F32:\n kernel = backend_ctx->kernel_cpy_f32_f32;\n break;\n default:\n GGML_ASSERT(false && \"not implemented\");\n }\n break;\n case GGML_TYPE_F16:\n switch (src1t) {\n case GGML_TYPE_F16:\n kernel = backend_ctx->kernel_cpy_f16_f16;\n break;\n case GGML_TYPE_F32:\n kernel = backend_ctx->kernel_cpy_f16_f32;\n break;\n default:\n GGML_ASSERT(false && \"not implemented\");\n }\n break;\n case GGML_TYPE_I32:\n switch (src1t) {\n case GGML_TYPE_I32:\n kernel = backend_ctx->kernel_cpy_i32_i32;\n break;\n default:\n GGML_ASSERT(false && \"not implemented\");\n }\n break;\n default:\n GGML_ASSERT(false && \"not implemented\");\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13));\n\n const int nth = MIN(64, ne00);\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src1);\n}\n\nstatic void ggml_cl_dup(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n ggml_cl_cpy(backend, src0, dst, nullptr);\n UNUSED(src1);\n}\n\nstatic void ggml_cl_set(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n GGML_ASSERT((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32) &&\n src1->type == src0->type && dst->type == src0->type);\n\n GGML_TENSOR_LOCALS(int, ne0, src0, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb);\n GGML_TENSOR_LOCALS(int, ne1, src1, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb);\n GGML_TENSOR_LOCALS(int, ne, dst, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const cl_ulong pnb1 = ((const int32_t *)dst->op_params)[0];\n const cl_ulong pnb2 = ((const int32_t *)dst->op_params)[1];\n const cl_ulong pnb3 = ((const int32_t *)dst->op_params)[2];\n const cl_ulong offs = ((const int32_t *)dst->op_params)[3];\n const bool inplace = (bool)((const int32_t *)dst->op_params)[4];\n\n cl_kernel kernel = nullptr;\n\n // for inplace case, dst is a view of src0 and is updated on top of it\n // so for non-inplace case, copy src0 to dst first\n if (!inplace) {\n ggml_cl_cpy(backend, src0, dst, nullptr);\n }\n\n // then copy src1 to dst with specified offset\n if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_cpy_f32_f32;\n } else if (src1->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_I32) {\n kernel = backend_ctx->kernel_cpy_i32_i32;\n } else {\n GGML_ASSERT(false && \"not implemented\");\n }\n\n offsetd += offs;\n cl_ulong nb = ggml_element_size(dst);\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb10));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &pnb1));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &pnb2));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &pnb3));\n\n int max_local_size = backend_ctx->get_kernel_workgroup_size(kernel);\n\n const int nth = MIN(max_local_size, ne00);\n\n size_t global_work_size[] = {(size_t)ne11*nth, (size_t)ne12, (size_t)ne13};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_diag_mask_inf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n int n_past = ((int32_t *)(dst->op_params))[0];\n\n const int ne00 = src0 ? src0->ne[0] : 0;\n const int ne01 = src0 ? src0->ne[1] : 0;\n const int ne02 = src0 ? src0->ne[2] : 0;\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel;\n\n if (ne00%8 == 0) {\n kernel = backend_ctx->kernel_diag_mask_inf_8;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_past));\n\n size_t global_work_size[] = {(size_t)ne00*ne01*ne02/8, 1, 1};\n size_t local_work_size[] = {64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n } else {\n kernel = backend_ctx->kernel_diag_mask_inf;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_past));\n\n size_t global_work_size[] = {(size_t)ne00, (size_t)ne01, (size_t)ne02};\n size_t local_work_size[] = {64, 1, 1};\n\n size_t * local_work_size_ptr = local_work_size;\n if (ne00 % 64 != 0 && !backend_ctx->non_uniform_workgroups) {\n local_work_size_ptr = nullptr; // Let driver choose the work-group sizes.\n }\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);\n }\n}\n\nstatic void ggml_cl_diag(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n UNUSED(src1);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n GGML_TENSOR_LOCALS(int, ne0, src0, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb);\n GGML_TENSOR_LOCALS(int, ne, dst, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb);\n\n cl_kernel kernel = backend_ctx->kernel_diag_f32;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb3));\n\n int nth = 64;\n\n size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_soft_max(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n // Softmax can now fuse KQ mask and KQ scale, which used to be two additional\n // ops before softmax. It now also fuses alibi if `max_bias > 0`. For llama,\n // alibi is not used; however, for some other models, it is used.\n // KQ_mask\n if (src1) {\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n }\n\n const ggml_tensor * src2 = dst->src[2];\n if (src2) {\n GGML_ASSERT(src2->extra);\n }\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n ggml_tensor_extra_cl * extra1 = src1 ? (ggml_tensor_extra_cl *)src1->extra : nullptr;\n ggml_tensor_extra_cl * extra2 = src2 ? (ggml_tensor_extra_cl *)src2->extra : nullptr;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_ulong offset1 = extra1 ? extra1->offset + src1->view_offs : offset0;\n cl_ulong offset2 = extra2 ? extra2->offset + src2->view_offs : offset0;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_long nb01 = src0->nb[1];\n const cl_long nb02 = src0->nb[2];\n const cl_long nb03 = src0->nb[3];\n\n const int ne12 = src1 ? src1->ne[2] : 0;\n const int ne13 = src1 ? src1->ne[3] : 0;\n\n const cl_long nb11 = src1 ? src1->nb[1] : 0;\n const cl_long nb12 = src1 ? src1->nb[2] : 0;\n const cl_long nb13 = src1 ? src1->nb[3] : 0;\n\n const cl_long nb1 = dst->nb[1];\n const cl_long nb2 = dst->nb[2];\n const cl_long nb3 = dst->nb[3];\n\n float scale, max_bias;\n memcpy(&scale, dst->op_params + 0, sizeof(float));\n memcpy(&max_bias, dst->op_params + 1, sizeof(float));\n\n const int n_head = src0->ne[2];\n const int n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));\n\n const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);\n\n // Local size must be wave size. Each workgroup is a wave, working on a row,\n // where a row corresponds to leading dimension.\n int nth = MIN(32, ne00);\n\n if (backend_ctx->gpu_family == INTEL) {\n // This is the same as the initial value.\n nth = MIN(32, ne00);\n }\n else if (backend_ctx->gpu_family == ADRENO) {\n nth = 64;\n } else {\n GGML_ASSERT(false && \"TODO: Unknown GPU\");\n }\n\n cl_kernel kernel;\n\n if (ne00%4 == 0) {\n if (use_f16) {\n kernel = backend_ctx->kernel_soft_max_4_f16;\n } else {\n kernel = backend_ctx->kernel_soft_max_4;\n }\n } else {\n if (use_f16) {\n kernel = backend_ctx->kernel_soft_max_f16;\n } else {\n kernel = backend_ctx->kernel_soft_max;\n }\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), extra1 ? &extra1->data_device : &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), extra2 ? &extra2->data_device : &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb3));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(float), &scale));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(float), &max_bias));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(float), &m0));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &m1));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &n_head_log2));\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n ggml_tensor * src2 = dst->src[2];\n ggml_tensor_extra_cl * extra2 = src2 ? (ggml_tensor_extra_cl *)src2->extra : nullptr;\n\n cl_ulong offset2 = extra2 ? extra2->offset + src2->view_offs : offset0;\n\n const int ne00 = src0 ? src0->ne[0] : 0;\n const int ne01 = src0 ? src0->ne[1] : 0;\n const int ne02 = src0 ? src0->ne[2] : 0;\n const int ne03 = src0 ? src0->ne[3] : 0;\n\n const cl_ulong nb00 = src0 ? src0->nb[0] : 0;\n const cl_ulong nb01 = src0 ? src0->nb[1] : 0;\n const cl_ulong nb02 = src0 ? src0->nb[2] : 0;\n const cl_ulong nb03 = src0 ? src0->nb[3] : 0;\n\n const int ne10 = src1 ? src1->ne[0] : 0;\n const int ne11 = src1 ? src1->ne[1] : 0; UNUSED(ne11);\n const int ne12 = src1 ? src1->ne[2] : 0; UNUSED(ne12);\n const int ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13);\n\n const int ne0 = dst ? dst->ne[0] : 0;\n const int ne1 = dst ? dst->ne[1] : 0;\n const int ne2 = dst ? dst->ne[2] : 0;\n const int ne3 = dst ? dst->ne[3] : 0;\n\n const cl_ulong nb0 = dst ? dst->nb[0] : 0;\n const cl_ulong nb1 = dst ? dst->nb[1] : 0;\n const cl_ulong nb2 = dst ? dst->nb[2] : 0;\n const cl_ulong nb3 = dst ? dst->nb[3] : 0;\n\n GGML_ASSERT(ne10 % ne02 == 0);\n GGML_ASSERT(ne10 >= ne02);\n\n int nth = MIN(64, ne00);\n\n const int n_past = ((int *) dst->op_params)[0];\n const int n_dims = ((int *) dst->op_params)[1];\n const int mode = ((int *) dst->op_params)[2];\n const int n_ctx_orig = ((int32_t *) dst->op_params)[4];\n\n float freq_base;\n float freq_scale;\n float ext_factor;\n float attn_factor;\n float beta_fast;\n float beta_slow;\n int32_t sections[4];\n\n memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));\n memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));\n memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));\n memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));\n memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));\n memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));\n memcpy(§ions, (int32_t *) dst->op_params + 11, sizeof(int32_t)*4);\n\n const bool is_neox = mode & 2;\n const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;\n const bool is_vision = mode == GGML_ROPE_TYPE_VISION;\n const int is_imrope = mode == GGML_ROPE_TYPE_IMROPE;\n\n if (is_mrope) {\n GGML_ASSERT(sections[0] > 0 || sections[1] > 0 || sections[2] > 0);\n }\n\n if (is_vision) {\n GGML_ASSERT(n_dims == ne00/2);\n }\n\n cl_kernel kernel;\n\n if (is_neox) {\n switch (src0->type) {\n case GGML_TYPE_F32:\n kernel = backend_ctx->kernel_rope_neox_f32;\n break;\n case GGML_TYPE_F16:\n kernel = backend_ctx->kernel_rope_neox_f16;\n break;\n default:\n GGML_ASSERT(false);\n };\n } else if (is_mrope && !is_vision) {\n switch (src0->type) {\n case GGML_TYPE_F32:\n kernel = backend_ctx->kernel_rope_multi_f32;\n break;\n case GGML_TYPE_F16:\n kernel = backend_ctx->kernel_rope_multi_f16;\n break;\n default:\n GGML_ASSERT(false);\n };\n } else if (is_vision) {\n switch (src0->type) {\n case GGML_TYPE_F32:\n kernel = backend_ctx->kernel_rope_vision_f32;\n break;\n case GGML_TYPE_F16:\n kernel = backend_ctx->kernel_rope_vision_f16;\n break;\n default:\n GGML_ASSERT(false);\n }\n } else {\n switch (src0->type) {\n case GGML_TYPE_F32:\n kernel = backend_ctx->kernel_rope_norm_f32;\n break;\n case GGML_TYPE_F16:\n kernel = backend_ctx->kernel_rope_norm_f16;\n break;\n default:\n GGML_ASSERT(false);\n };\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), extra2 ? &extra2->data_device : &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne1));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne2));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne3));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb0));\n CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &nb3));\n CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &n_past));\n CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &n_dims));\n CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &n_ctx_orig));\n CL_CHECK(clSetKernelArg(kernel, 27, sizeof(float), &freq_base));\n CL_CHECK(clSetKernelArg(kernel, 28, sizeof(float), &freq_scale));\n CL_CHECK(clSetKernelArg(kernel, 29, sizeof(float), &ext_factor));\n CL_CHECK(clSetKernelArg(kernel, 30, sizeof(float), &attn_factor));\n CL_CHECK(clSetKernelArg(kernel, 31, sizeof(float), &beta_fast));\n CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &beta_slow));\n // both mrope and vision kernels have sections\n if (is_mrope || is_vision) {\n CL_CHECK(clSetKernelArg(kernel, 33, sizeof(int32_t)*4, §ions));\n }\n // only mrope has is_imrope\n if (is_mrope && !is_vision) {\n CL_CHECK(clSetKernelArg(kernel, 34, sizeof(int), &is_imrope));\n }\n\n size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_solve_tri(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_kernel kernel = backend_ctx->kernel_solve_tri_f32;\n GGML_ASSERT(kernel != nullptr);\n\n const int n = src0->ne[0];\n const int k = src1->ne[0];\n\n const cl_ulong nb00 = src0->nb[0];\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const cl_ulong nb10 = src1->nb[0];\n const cl_ulong nb11 = src1->nb[1];\n const cl_ulong nb12 = src1->nb[2];\n const cl_ulong nb13 = src1->nb[3];\n\n const cl_ulong nb0 = dst->nb[0];\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &k));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong),&nb02));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong),&nb03));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong),&nb10));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong),&nb11));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong),&nb12));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong),&nb13));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong),&nb0));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong),&nb1));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong),&nb2));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong),&nb3));\n\n size_t global_work_size[3]= { (size_t)k, (size_t)dst->ne[2], (size_t)dst->ne[3]};\n size_t local_work_size[] = {16, 4, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_im2col(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n // src0 - filter, src1 - input\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset1 = extra1->offset + src1->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int32_t s0 = ((const int32_t*)(dst->op_params))[0];\n const int32_t s1 = ((const int32_t*)(dst->op_params))[1];\n const int32_t p0 = ((const int32_t*)(dst->op_params))[2];\n const int32_t p1 = ((const int32_t*)(dst->op_params))[3];\n const int32_t d0 = ((const int32_t*)(dst->op_params))[4];\n const int32_t d1 = ((const int32_t*)(dst->op_params))[5];\n\n const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;\n\n const cl_long IC = src1->ne[is_2D ? 2 : 1];\n const cl_long IH = is_2D ? src1->ne[1] : 1;\n const cl_long IW = src1->ne[0];\n\n const cl_long KH = is_2D ? src0->ne[1] : 1;\n const cl_long KW = src0->ne[0];\n\n const cl_long OH = is_2D ? dst->ne[2] : 1;\n const cl_long OW = dst->ne[1];\n\n // nb is byte offset, src is type float32\n const cl_ulong delta_offset = src1->nb[is_2D ? 2 : 1]/4;\n const cl_long batch = src1->ne[is_2D ? 3 : 2];\n const cl_ulong batch_offset = src1->nb[is_2D ? 3 : 2]/4;\n\n const cl_long pelements = OW*KW*KH;\n const cl_long CHW = IC*KH*KW;\n\n cl_kernel kernel;\n\n if(dst->type == GGML_TYPE_F16) {\n kernel = backend_ctx->kernel_im2col_f16;\n } else {\n kernel = backend_ctx->kernel_im2col_f32;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra1->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &batch_offset));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &delta_offset));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_long), &IW));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_long), &IH));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_long), &IC));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_long), &OW));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_long), &OH));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_long), &KW));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_long), &KH));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_long), &pelements));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_long), &CHW));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &s0));\n CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &s1));\n CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &p0));\n CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &p1));\n CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &d0));\n CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &d1));\n\n const int num_blocks = (pelements + 256 - 1) / 256;\n size_t global_work_size[] = {(size_t)num_blocks*256, (size_t)OH, (size_t)batch*IC};\n size_t local_work_size[] = {256, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_UNUSED(src1);\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_I32);\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int ne00 = src0->ne[0];\n const int nrows = ggml_nrows(src0);\n\n int ne00_padded = 1;\n while (ne00_padded < ne00) {\n ne00_padded *= 2;\n }\n\n int order = (enum ggml_sort_order) dst->op_params[0];\n\n cl_kernel kernel = backend_ctx->kernel_argsort_f32_i32;\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00_padded));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &order));\n CL_CHECK(clSetKernelArg(kernel, 7, ne00_padded*sizeof(int), NULL));\n\n size_t global_work_size[] = {(size_t)ne00_padded, (size_t)nrows, (size_t)1};\n size_t local_work_size[] = {(size_t)ne00_padded, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_sum_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_UNUSED(src1);\n\n GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n const int ne00 = src0->ne[0];\n const int ne01 = src0->ne[1];\n const int ne02 = src0->ne[2];\n const int ne03 = src0->ne[3];\n\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb02 = src0->nb[2];\n const cl_ulong nb03 = src0->nb[3];\n\n const cl_ulong nb1 = dst->nb[1];\n const cl_ulong nb2 = dst->nb[2];\n const cl_ulong nb3 = dst->nb[3];\n\n cl_kernel kernel;\n\n const bool is_c4 = ne00 % 4 == 0;\n if (is_c4) {\n kernel = backend_ctx->kernel_sum_rows_f32_4;\n } else {\n kernel = backend_ctx->kernel_sum_rows_f32;\n }\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb2));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb3));\n\n size_t global_work_size[] = {64 * (size_t)ne01, (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = {(size_t)64, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\nstatic void ggml_cl_cumsum(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n GGML_UNUSED(src1);\n\n GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n GGML_TENSOR_LOCALS(int, ne0, src0, ne);\n GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb);\n\n cl_kernel kernel = backend_ctx->kernel_cumsum_blk;\n\n int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel);\n int nth = 1;\n while (nth < ne00 && 2*nth <= max_workgroup_size) {\n nth *= 2;\n }\n\n GGML_ASSERT(ne00 <= nth*nth);\n\n const int net0 = CEIL_DIV(ne00, nth);\n const int net1 = ne01;\n const int net2 = ne02;\n const int net3 = ne03;\n\n const cl_ulong nbt0 = sizeof(float);\n const cl_ulong nbt1 = net0*nbt0;\n const cl_ulong nbt2 = net1*nbt1;\n const cl_ulong nbt3 = net2*nbt2;\n\n static ggml_cl_buffer tmp_buffer;\n tmp_buffer.allocate(backend_ctx->context, net0*ne01*ne02*ne03*sizeof(float));\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &tmp_buffer.buffer));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &net0));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &net1));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &net2));\n\n size_t global_work_size[] = { (size_t)(nth*net0*ne01), (size_t)ne02, (size_t)ne03};\n size_t local_work_size[] = { (size_t)nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n\n if(ne00 > nth) {\n // if a single workgroup cannot handle an entire row, each workgroup\n // computes a partial sum and stores to dst, tmp_buffer contains the sum\n // of the each workgroup; cumsum this buffer and add to the partial sums in dst\n cl_ulong offsett = 0;\n kernel = backend_ctx->kernel_cumsum_blk;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &tmp_buffer.buffer));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsett));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &tmp_buffer.buffer));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tmp_buffer.buffer));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsett));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &net0));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nbt0));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nbt1));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nbt2));\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nbt3));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &net0));\n CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &net1));\n CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &net2));\n\n size_t global_work_size_1[] = { (size_t)net1*nth, (size_t)net2, (size_t)net3};\n size_t local_work_size_1[] = { (size_t)nth, 1, 1};\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size_1, local_work_size_1, dst);\n\n kernel = backend_ctx->kernel_cumsum_add;\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &tmp_buffer.buffer));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne02));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne03));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &nbt0));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &nbt1));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &nbt2));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &nbt3));\n\n size_t global_work_size_2[] = { (size_t)(nth*net0*ne01), (size_t)ne02, (size_t)ne03};\n size_t local_work_size_2[] = { (size_t)nth, 1, 1};\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size_2, local_work_size_2, dst);\n }\n}\n\nstatic void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->extra);\n GGML_ASSERT(dst);\n GGML_ASSERT(dst->extra);\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n\n if (src1) {\n GGML_ASSERT(src1);\n GGML_ASSERT(src1->extra);\n GGML_ASSERT(ggml_are_same_shape(src0, src1));\n }\n\n ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;\n\n cl_kernel kernel;\n switch (ggml_get_glu_op(dst)) {\n case GGML_GLU_OP_GEGLU:\n if (dst->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_geglu;\n } else {\n kernel = backend_ctx->kernel_geglu_f16;\n }\n break;\n case GGML_GLU_OP_REGLU:\n if (dst->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_reglu;\n } else {\n kernel = backend_ctx->kernel_reglu_f16;\n }\n break;\n case GGML_GLU_OP_SWIGLU:\n if (dst->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_swiglu;\n } else {\n kernel = backend_ctx->kernel_swiglu_f16;\n }\n break;\n case GGML_GLU_OP_SWIGLU_OAI:\n kernel = backend_ctx->kernel_swiglu_oai;\n break;\n case GGML_GLU_OP_GEGLU_ERF:\n if (dst->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_geglu_erf;\n } else {\n kernel = backend_ctx->kernel_geglu_erf_f16;\n }\n break;\n case GGML_GLU_OP_GEGLU_QUICK:\n if (dst->type == GGML_TYPE_F32) {\n kernel = backend_ctx->kernel_geglu_quick;\n } else {\n kernel = backend_ctx->kernel_geglu_quick_f16;\n }\n break;\n default:\n GGML_ABORT(\"Unsupported glu op\");\n }\n\n ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;\n ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;\n\n ggml_tensor_extra_cl * extra1 = src1 ? (ggml_tensor_extra_cl *)src1->extra : nullptr;\n\n cl_ulong offset0 = extra0->offset + src0->view_offs;\n cl_ulong offsetd = extrad->offset + dst->view_offs;\n\n cl_ulong offset1 = extra1 ? extra1->offset + src1->view_offs : offset0;\n\n const int ne0 = dst->ne[0];\n\n const cl_ulong nb01 = src0->nb[1];\n const cl_ulong nb11 = src1 ? src1->nb[1] : nb01;\n\n const cl_ulong nb1 = dst->nb[1];\n\n const int swp = ggml_get_op_params_i32(dst, 1);\n const float alpha = ggml_get_op_params_f32(dst, 2);\n const float limit = ggml_get_op_params_f32(dst, 3);\n\n const int ne00_off = src1 ? 0 : (swp ? ne0 : 0);\n const int ne10_off = src1 ? 0 : (swp ? 0 : ne0);\n\n CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));\n CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), src1 ? &extra1->data_device : &extra0->data_device));\n CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));\n CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));\n CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));\n CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));\n CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb11));\n CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne0));\n CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb1));\n CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00_off));\n CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10_off));\n\n if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI) {\n CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &limit));\n CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &alpha));\n }\n\n const size_t nrows = ggml_nrows(src0);\n size_t nth = 512;\n size_t global_work_size[] = {nrows*nth, 1, 1};\n size_t local_work_size[] = {nth, 1, 1};\n\n backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);\n}\n\n//------------------------------------------------------------------------------\n// Op offloading\n//------------------------------------------------------------------------------\n\ntypedef void (*ggml_cl_func_t)(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);\n\nbool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor) {\n ggml_cl_func_t func = nullptr;\n\n ggml_tensor * src0 = tensor->src[0];\n ggml_tensor * src1 = tensor->src[1];\n\n const bool any_on_device = tensor->extra\n || (src0 != nullptr && src0->extra)\n || (src1 != nullptr && src1->extra);\n\n switch (tensor->op) {\n case GGML_OP_GET_ROWS:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_get_rows;\n break;\n case GGML_OP_SET_ROWS:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_set_rows;\n break;\n case GGML_OP_CPY:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_cpy;\n break;\n case GGML_OP_SET:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_set;\n break;\n case GGML_OP_DUP:\n case GGML_OP_CONT:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_dup;\n break;\n case GGML_OP_ADD:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_add;\n break;\n case GGML_OP_ADD_ID:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_add_id;\n break;\n case GGML_OP_MUL:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_mul;\n break;\n case GGML_OP_DIV:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_div;\n break;\n case GGML_OP_SUB:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_sub;\n break;\n case GGML_OP_SQR:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_sqr;\n break;\n case GGML_OP_SQRT:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_sqrt;\n break;\n case GGML_OP_MEAN:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_mean;\n break;\n case GGML_OP_UNARY:\n switch (ggml_get_unary_op(tensor)) {\n case GGML_UNARY_OP_GELU:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_gelu;\n break;\n case GGML_UNARY_OP_GELU_ERF:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_gelu_erf;\n break;\n case GGML_UNARY_OP_GELU_QUICK:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_gelu_quick;\n break;\n case GGML_UNARY_OP_SILU:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_silu;\n break;\n case GGML_UNARY_OP_RELU:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_relu;\n break;\n case GGML_UNARY_OP_SIGMOID:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_sigmoid;\n break;\n case GGML_UNARY_OP_TANH:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_tanh;\n break;\n case GGML_UNARY_OP_NEG:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_neg;\n break;\n case GGML_UNARY_OP_EXP:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_exp;\n break;\n case GGML_UNARY_OP_EXPM1:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_expm1;\n break;\n case GGML_UNARY_OP_SOFTPLUS:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_softplus;\n break;\n default:\n return false;\n } break;\n case GGML_OP_GLU:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_glu;\n break;\n case GGML_OP_TRI:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_tri;\n break;\n case GGML_OP_FILL:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_fill;\n break;\n case GGML_OP_CLAMP:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_clamp;\n break;\n case GGML_OP_NORM:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_norm;\n break;\n case GGML_OP_RMS_NORM:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_rms_norm;\n break;\n case GGML_OP_L2_NORM:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_l2_norm;\n break;\n case GGML_OP_GROUP_NORM:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_group_norm;\n break;\n case GGML_OP_REPEAT:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_repeat;\n break;\n case GGML_OP_PAD:\n if (!any_on_device) {\n return false;\n }\n ggml_cl_pad(backend, tensor->src[0], tensor);\n return true;\n case GGML_OP_UPSCALE:\n if (!any_on_device) {\n return false;\n }\n ggml_cl_upscale(backend, tensor->src[0], tensor);\n return true;\n case GGML_OP_CONV_2D:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_conv_2d;\n break;\n case GGML_OP_SSM_CONV:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_ssm_conv;\n break;\n case GGML_OP_CONCAT:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_concat;\n break;\n case GGML_OP_TIMESTEP_EMBEDDING:\n if (!any_on_device) {\n return false;\n }\n ggml_cl_timestep_embedding(backend, tensor->src[0], tensor);\n return true;\n case GGML_OP_MUL_MAT:\n if (!any_on_device && !ggml_cl_can_mul_mat(tensor->src[0], tensor->src[1], tensor)) {\n return false;\n }\n func = ggml_cl_mul_mat;\n break;\n case GGML_OP_MUL_MAT_ID:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_mul_mat_id;\n break;\n case GGML_OP_SCALE:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_scale;\n break;\n case GGML_OP_RESHAPE:\n case GGML_OP_VIEW:\n case GGML_OP_PERMUTE:\n case GGML_OP_TRANSPOSE:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_nop;\n break;\n case GGML_OP_DIAG:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_diag;\n break;\n case GGML_OP_DIAG_MASK_INF:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_diag_mask_inf;\n break;\n case GGML_OP_SOFT_MAX:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_soft_max;\n break;\n case GGML_OP_ROPE:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_rope;\n break;\n case GGML_OP_SOLVE_TRI:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_solve_tri;\n break;\n case GGML_OP_IM2COL:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_im2col;\n break;\n case GGML_OP_ARGSORT:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_argsort;\n break;\n case GGML_OP_SUM_ROWS:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_sum_rows;\n break;\n case GGML_OP_CUMSUM:\n if (!any_on_device) {\n return false;\n }\n func = ggml_cl_cumsum;\n break;\n case GGML_OP_FLASH_ATTN_EXT:\n if (!any_on_device) {\n return false;\n }\n ggml_cl_flash_attn(backend, tensor->src[0], tensor->src[1], tensor);\n return true;\n default:\n return false;\n }\n\n func(backend, tensor->src[0], tensor->src[1], tensor);\n return true;\n}\n" }, { "path": "src/ggml-opencl/kernels/add.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// add\n//------------------------------------------------------------------------------\n\n// general-purpose kernel for addition of two tensors\n// pros: works for non-contiguous tensors, supports broadcast across dims 1, 2 and 3\n// cons: not very efficient\nkernel void kernel_add(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03 % ne13;\n int i12 = i02 % ne12;\n int i11 = i01 % ne11;\n\n global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;\n global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;\n global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const int i10 = i0 % ne10;\n *((global float *)(dst_ptr + i0*nb0)) = *((global float *)(src0_ptr + i0*nb00)) + *((global float *)(src1_ptr + i10*nb10));\n }\n}\n\n// assumption: src1 is a row\n// broadcast src1 into src0\nkernel void kernel_add_row(\n global float4 * src0,\n ulong offset0,\n global float4 * src1,\n ulong offset1,\n global float4 * dst,\n ulong offsetd,\n int ne\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float4*)((global char*)dst + offsetd);\n\n // This performs better than using %.\n uint gid = get_global_id(0);\n uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne\n dst[gid] = src0[gid] + src1[idx1];\n}\n\nkernel void kernel_add_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int type_src0,\n int type_src1\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03 % ne13;\n int i12 = i02 % ne12;\n int i11 = i01 % ne11;\n\n global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;\n global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;\n global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const int i10 = i0 % ne10;\n\n half v0, v1;\n if (type_src0 == 1) {\n v0 = convert_half(*((global float *)(src0_ptr + i0*nb00)));\n } else {\n v0 = *((global half *)(src0_ptr + i0*nb00));\n }\n\n if (type_src1 == 1) {\n v1 = convert_half(*((global float *)(src1_ptr + i10*nb10)));\n } else {\n v1 = *((global half *)(src1_ptr + i10*nb10));\n }\n\n *((global half *)(dst_ptr + i0*nb0)) = v0 + v1;\n }\n}\n\nkernel void kernel_add_row_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global half4 * dst,\n ulong offsetd,\n int ne,\n int type_src0,\n int type_src1\n) {\n dst = (global half4*)((global char*)dst + offsetd);\n\n // This performs better than using %.\n uint gid = get_global_id(0);\n uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne\n\n half4 v0, v1;\n if (type_src0 == 1) {\n global float4* src0_f32 = (global float4*)((global char*)src0 + offset0);\n v0 = convert_half4(src0_f32[gid]);\n } else {\n global half4* src0_f16 = (global half4*)((global char*)src0 + offset0);\n v0 = src0_f16[gid];\n }\n\n if (type_src1 == 1) {\n global float4* src1_f32 = (global float4*)((global char*)src1 + offset1);\n v1 = convert_half4(src1_f32[idx1]);\n } else {\n global half4* src1_f16 = (global half4*)((global char*)src1 + offset1);\n v1 = src1_f16[idx1];\n }\n\n dst[gid] = v0 + v1;\n}\n" }, { "path": "src/ggml-opencl/kernels/add_id.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// add_id\n//------------------------------------------------------------------------------\nkernel void kernel_add_id(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * src2,\n ulong offset2,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb02,\n ulong nb11,\n ulong nb21,\n int ne0,\n int ne1\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n src2 = (global char*)((global char*)src2 + offset2);\n dst = (global char*)((global char*)dst + offsetd);\n\n int i1 = get_group_id(0);\n int i2 = get_group_id(1);\n\n const int i11 = *((global const int *) (src2 + i1*sizeof(int) + i2*nb21));\n\n const size_t nb1 = ne0 * sizeof(float);\n const size_t nb2 = ne1 * nb1;\n\n global float * dst_row = (global float *)((global char *)dst + i1*nb1 + i2*nb2);\n global float * src0_row = (global float *)((global char *)src0 + i1*nb01 + i2*nb02);\n global float * src1_row = (global float *)((global char *)src1 + i11*nb11);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n dst_row[i0] = src0_row[i0] + src1_row[i0];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/argsort.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define SWAP(x, y, T) { T tmp = (x); (x) = (y); (y) = tmp; }\n\nenum ggml_sort_order {\n GGML_SORT_ORDER_ASC,\n GGML_SORT_ORDER_DESC,\n};\n\nkernel void kernel_argsort_f32_i32(\n global float * src0,\n ulong offset0,\n global int * dst,\n ulong offsetd,\n const int ne00,\n const int ne00_pad,\n const int order,\n local int * dst_row\n) {\n // bitonic sort\n int col = get_local_id(0);\n int row = get_group_id(1);\n\n if (col >= ne00_pad) {\n return;\n }\n\n src0 = (global char *)((global char *)src0 + offset0);\n dst = (global float *)((global char *)dst + offsetd);\n\n global float * x_row = src0 + row * ne00;\n\n // initialize indices\n dst_row[col] = col;\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n for (int k = 2; k <= ne00_pad; k *= 2) {\n for (int j = k / 2; j > 0; j /= 2) {\n int ixj = col ^ j;\n if (ixj > col) {\n if ((col & k) == 0) {\n if (dst_row[col] >= ne00 ||\n (dst_row[ixj] < ne00 && (order == GGML_SORT_ORDER_ASC ?\n x_row[dst_row[col]] > x_row[dst_row[ixj]] :\n x_row[dst_row[col]] < x_row[dst_row[ixj]]))\n ) {\n SWAP(dst_row[col], dst_row[ixj], int);\n }\n } else {\n if (dst_row[ixj] >= ne00 ||\n (dst_row[col] < ne00 && (order == GGML_SORT_ORDER_ASC ?\n x_row[dst_row[col]] < x_row[dst_row[ixj]] :\n x_row[dst_row[col]] > x_row[dst_row[ixj]]))\n ) {\n SWAP(dst_row[col], dst_row[ixj], int);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n }\n\n // copy the result to dst without the padding\n if (col < ne00) {\n dst[row * ne00 + col] = dst_row[col];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/clamp.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// clamp\n//------------------------------------------------------------------------------\nkernel void kernel_clamp(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n float min,\n float max\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = src0[get_global_id(0)] < min ?\n min :\n (src0[get_global_id(0)] > max ? max : src0[get_global_id(0)]);\n}\n" }, { "path": "src/ggml-opencl/kernels/concat.cl", "content": "kernel void kernel_concat_f32(\n global const char * src0,\n ulong offset0,\n global const char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int dim\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n int o[4] = {0, 0, 0, 0};\n o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));\n\n global const float * x;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);\n } else {\n x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);\n }\n\n global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = *x;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/conv2d.cl", "content": "#ifdef USE_FP16\n#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#define T_FLOAT half\n#define T_FLOAT4 half4\n#define VSTORE_T_FLOAT4(data, offset, p) vstore_half4_rte(data, offset, p)\n#else\n#define T_FLOAT float\n#define T_FLOAT4 float4\n#define VSTORE_T_FLOAT4(data, offset, p) vstore4(data, offset, p)\n#endif\n\n#if defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#else\n#define REQD_SUBGROUP_SIZE_128\n#endif\n\n#define T_ACCUM float4\n#define VEC_SIZE 4\n\n#define BS_K 64\n#define BS_NPQ 64\n#define BS_CRS 16\n\n#define TS_K 4\n#define TS_NPQ 8\n\n#define WG_K (BS_K / TS_K)\n#define WG_NPQ (BS_NPQ / TS_NPQ)\n\n#define BS_NPQ_VEC (BS_NPQ / VEC_SIZE)\n#define TS_NPQ_VEC (TS_NPQ / VEC_SIZE)\n\nstatic inline uint splitWork(uint work_size, uint block_size){\n return (work_size + block_size - 1) / block_size;\n}\n\nREQD_SUBGROUP_SIZE_128\nkernel void kernel_conv_2d(\n global void* p_knl,\n ulong off_knl,\n global void* p_src,\n ulong off_src,\n global void* p_dst,\n ulong off_dst,\n local void* shared,\n uint Cout, uint Cin, uint N,\n uint KW, uint KH, uint W, uint H, uint OW, uint OH,\n uint s0, uint s1, uint p0, uint p1, uint d0, uint d1,\n uint nb01, uint nb02, uint nb03,\n uint nb11, uint nb12, uint nb13,\n uint nb1, uint nb2, uint nb3\n) {\n global T_FLOAT* knl_data = (global T_FLOAT*) ((global char*)p_knl + off_knl);\n global T_FLOAT* src_data = (global T_FLOAT*) ((global char*)p_src + off_src);\n global T_FLOAT* dst_data = (global T_FLOAT*) ((global char*)p_dst + off_dst);\n\n const uint K = Cout;\n const uint CRS = Cin*KH*KW;\n const uint NPQ = N*OH*OW;\n\n const uint lid_k = get_local_id(0);\n const uint lid_npq = get_local_id(1);\n const uint tid = lid_npq * WG_K + lid_k;\n\n const uint B_idx_K = get_group_id(0);\n const uint B_idx_NPQ = get_group_id(1);\n\n const uint offset_k = B_idx_K * BS_K;\n const uint offset_npq = B_idx_NPQ * BS_NPQ;\n\n local T_FLOAT* Ash = (local T_FLOAT*)shared;\n local T_FLOAT4* Bsh = (local T_FLOAT4*) &Ash[BS_K * BS_CRS];\n\n T_ACCUM regC[TS_K][TS_NPQ_VEC];\n for (int i = 0; i < TS_K; ++i) {\n for (int j = 0; j < TS_NPQ_VEC; ++j) {\n regC[i][j] = (T_ACCUM)(0.0f);\n }\n }\n\n const uint NB_CRS = splitWork(CRS, BS_CRS);\n\n for (uint B_idx_CRS = 0; B_idx_CRS < NB_CRS; ++B_idx_CRS) {\n const uint offset_crs = B_idx_CRS * BS_CRS;\n\n for (int i = tid; i < BS_K * BS_CRS; i += (WG_K * WG_NPQ)) {\n const uint k_l = i / BS_CRS;\n const uint crs_l = i % BS_CRS;\n const uint k_g = offset_k + k_l;\n const uint crs_g = offset_crs + crs_l;\n\n if (k_g < K && crs_g < CRS) {\n const uint Cin_idx = crs_g / (KW*KH);\n const uint KH_idx = (crs_g - Cin_idx*KW*KH) / KW;\n const uint KW_idx = crs_g - Cin_idx*KW*KH - KH_idx*KW;\n const uint knl_idx = KW_idx + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03;\n Ash[k_l * BS_CRS + crs_l] = knl_data[knl_idx];\n } else {\n Ash[k_l * BS_CRS + crs_l] = (T_FLOAT)0.0f;\n }\n }\n\n for (int i = tid; i < BS_CRS * BS_NPQ_VEC; i += (WG_K * WG_NPQ)) {\n const uint crs_l = i / BS_NPQ_VEC;\n const uint npq_l_vec = i % BS_NPQ_VEC;\n const uint crs_g = offset_crs + crs_l;\n\n T_FLOAT4 val = (T_FLOAT4)(0.0f);\n if (crs_g < CRS) {\n const uint Cin_idx = crs_g / (KW * KH);\n const uint KH_idx = (crs_g - Cin_idx * KW * KH) / KW;\n const uint KW_idx = crs_g - Cin_idx * KW * KH - KH_idx * KW;\n for (int v = 0; v < VEC_SIZE; ++v) {\n const uint npq_g = offset_npq + npq_l_vec * VEC_SIZE + v;\n if (npq_g < NPQ) {\n const uint N_idx = npq_g / (OH * OW);\n const uint pq_idx = npq_g % (OH * OW);\n const uint OH_idx = pq_idx / OW;\n const uint OW_idx = pq_idx % OW;\n const int H_idx = (int)(OH_idx * s1 + KH_idx * d1 - p1);\n const int W_idx = (int)(OW_idx * s0 + KW_idx * d0 - p0);\n\n if (H_idx >= 0 && H_idx < H && W_idx >= 0 && W_idx < W) {\n const uint src_idx = W_idx + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13;\n ((T_FLOAT*)&val)[v] = src_data[src_idx];\n }\n }\n }\n }\n Bsh[crs_l * BS_NPQ_VEC + npq_l_vec] = val;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n #pragma unroll\n for (uint crs_l = 0; crs_l < BS_CRS; ++crs_l) {\n T_FLOAT regA[TS_K];\n for (uint k_l_reg = 0; k_l_reg < TS_K; ++k_l_reg) {\n regA[k_l_reg] = Ash[(lid_k * TS_K + k_l_reg) * BS_CRS + crs_l];\n }\n\n for (uint npq_l_vec_reg = 0; npq_l_vec_reg < TS_NPQ_VEC; ++npq_l_vec_reg) {\n T_FLOAT4 regB = Bsh[crs_l * BS_NPQ_VEC + lid_npq * TS_NPQ_VEC + npq_l_vec_reg];\n for (uint k_l_reg = 0; k_l_reg < TS_K; ++k_l_reg) {\n regC[k_l_reg][npq_l_vec_reg] = mad(convert_float(regA[k_l_reg]), convert_float4(regB), regC[k_l_reg][npq_l_vec_reg]);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n for (uint k_l_reg = 0; k_l_reg < TS_K; ++k_l_reg) {\n const uint k_g = offset_k + lid_k * TS_K + k_l_reg;\n if (k_g >= K) continue;\n\n for (uint npq_l_vec_reg = 0; npq_l_vec_reg < TS_NPQ_VEC; ++npq_l_vec_reg) {\n const uint npq_g_base = offset_npq + (lid_npq * TS_NPQ_VEC + npq_l_vec_reg) * VEC_SIZE;\n\n const uint N_idx = npq_g_base / (OH * OW);\n const uint pq_idx = npq_g_base % (OH * OW);\n const uint OH_idx = pq_idx / OW;\n const uint OW_idx = pq_idx % OW;\n\n if (nb1 == OW && OW_idx + VEC_SIZE <= OW && npq_g_base + VEC_SIZE <= NPQ) {\n const uint dst_idx = OW_idx + OH_idx*nb1 + k_g*nb2 + N_idx*nb3;\n VSTORE_T_FLOAT4(regC[k_l_reg][npq_l_vec_reg], 0, &dst_data[dst_idx]);\n } else {\n T_ACCUM res = regC[k_l_reg][npq_l_vec_reg];\n for (int v = 0; v < VEC_SIZE; ++v) {\n const uint npq_g = npq_g_base + v;\n if (npq_g < NPQ) {\n const uint N_idx_s = npq_g / (OH*OW);\n const uint pq_idx_s = npq_g % (OH*OW);\n const uint OH_idx_s = pq_idx_s / OW;\n const uint OW_idx_s = pq_idx_s % OW;\n const uint dst_idx_s = OW_idx_s + OH_idx_s*nb1 + k_g*nb2 + N_idx_s*nb3;\n dst_data[dst_idx_s] = (T_FLOAT)(((float*)&res)[v]);\n }\n }\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/conv2d_f16_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#if defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#else\n#define REQD_SUBGROUP_SIZE_128\n#endif\n\n#define T_ACCUM float4\n#define VEC_SIZE 4\n\n#define BS_K 64\n#define BS_NPQ 64\n#define BS_CRS 16\n\n#define TS_K 4\n#define TS_NPQ 8\n\n#define WG_K (BS_K / TS_K)\n#define WG_NPQ (BS_NPQ / TS_NPQ)\n\n#define BS_NPQ_VEC (BS_NPQ / VEC_SIZE)\n#define TS_NPQ_VEC (TS_NPQ / VEC_SIZE)\n\nstatic inline uint splitWork(uint work_size, uint block_size){\n return (work_size + block_size - 1) / block_size;\n}\n\nREQD_SUBGROUP_SIZE_128\nkernel void kernel_conv_2d(\n global void* p_knl,\n ulong off_knl,\n global void* p_src,\n ulong off_src,\n global void* p_dst,\n ulong off_dst,\n local void* shared,\n uint Cout, uint Cin, uint N,\n uint KW, uint KH, uint W, uint H, uint OW, uint OH,\n uint s0, uint s1, uint p0, uint p1, uint d0, uint d1,\n uint nb01, uint nb02, uint nb03,\n uint nb11, uint nb12, uint nb13,\n uint nb1, uint nb2, uint nb3\n) {\n global half* knl_data = (global half*) ((global char*)p_knl + off_knl);\n global float* src_data = (global float*) ((global char*)p_src + off_src);\n global float* dst_data = (global float*) ((global char*)p_dst + off_dst);\n\n const uint K = Cout;\n const uint CRS = Cin*KH*KW;\n const uint NPQ = N*OH*OW;\n\n const uint lid_k = get_local_id(0);\n const uint lid_npq = get_local_id(1);\n const uint tid = lid_npq * WG_K + lid_k;\n\n const uint B_idx_K = get_group_id(0);\n const uint B_idx_NPQ = get_group_id(1);\n\n const uint offset_k = B_idx_K * BS_K;\n const uint offset_npq = B_idx_NPQ * BS_NPQ;\n\n local half* Ash = (local half*)shared;\n local float4* Bsh = (local float4*) &Ash[BS_K * BS_CRS];\n\n T_ACCUM regC[TS_K][TS_NPQ_VEC];\n for (int i = 0; i < TS_K; ++i) {\n for (int j = 0; j < TS_NPQ_VEC; ++j) {\n regC[i][j] = (T_ACCUM)(0.0f);\n }\n }\n\n const uint NB_CRS = splitWork(CRS, BS_CRS);\n\n for (uint B_idx_CRS = 0; B_idx_CRS < NB_CRS; ++B_idx_CRS) {\n const uint offset_crs = B_idx_CRS * BS_CRS;\n\n for (int i = tid; i < BS_K * BS_CRS; i += (WG_K * WG_NPQ)) {\n const uint k_l = i / BS_CRS;\n const uint crs_l = i % BS_CRS;\n const uint k_g = offset_k + k_l;\n const uint crs_g = offset_crs + crs_l;\n\n if (k_g < K && crs_g < CRS) {\n const uint Cin_idx = crs_g / (KW*KH);\n const uint KH_idx = (crs_g - Cin_idx*KW*KH) / KW;\n const uint KW_idx = crs_g - Cin_idx*KW*KH - KH_idx*KW;\n const uint knl_idx = KW_idx + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03;\n Ash[k_l * BS_CRS + crs_l] = knl_data[knl_idx];\n } else {\n Ash[k_l * BS_CRS + crs_l] = (half)0.0f;\n }\n }\n\n for (int i = tid; i < BS_CRS * BS_NPQ_VEC; i += (WG_K * WG_NPQ)) {\n const uint crs_l = i / BS_NPQ_VEC;\n const uint npq_l_vec = i % BS_NPQ_VEC;\n const uint crs_g = offset_crs + crs_l;\n\n float4 val = (float4)(0.0f);\n if (crs_g < CRS) {\n const uint Cin_idx = crs_g / (KW * KH);\n const uint KH_idx = (crs_g - Cin_idx * KW * KH) / KW;\n const uint KW_idx = crs_g - Cin_idx * KW * KH - KH_idx * KW;\n for (int v = 0; v < VEC_SIZE; ++v) {\n const uint npq_g = offset_npq + npq_l_vec * VEC_SIZE + v;\n if (npq_g < NPQ) {\n const uint N_idx = npq_g / (OH * OW);\n const uint pq_idx = npq_g % (OH * OW);\n const uint OH_idx = pq_idx / OW;\n const uint OW_idx = pq_idx % OW;\n const int H_idx = (int)(OH_idx * s1 + KH_idx * d1 - p1);\n const int W_idx = (int)(OW_idx * s0 + KW_idx * d0 - p0);\n\n if (H_idx >= 0 && H_idx < H && W_idx >= 0 && W_idx < W) {\n const uint src_idx = W_idx + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13;\n ((float*)&val)[v] = src_data[src_idx];\n }\n }\n }\n }\n Bsh[crs_l * BS_NPQ_VEC + npq_l_vec] = val;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n #pragma unroll\n for (uint crs_l = 0; crs_l < BS_CRS; ++crs_l) {\n half regA[TS_K];\n for (uint k_l_reg = 0; k_l_reg < TS_K; ++k_l_reg) {\n regA[k_l_reg] = Ash[(lid_k * TS_K + k_l_reg) * BS_CRS + crs_l];\n }\n\n for (uint npq_l_vec_reg = 0; npq_l_vec_reg < TS_NPQ_VEC; ++npq_l_vec_reg) {\n float4 regB = Bsh[crs_l * BS_NPQ_VEC + lid_npq * TS_NPQ_VEC + npq_l_vec_reg];\n for (uint k_l_reg = 0; k_l_reg < TS_K; ++k_l_reg) {\n regC[k_l_reg][npq_l_vec_reg] = mad(convert_float(regA[k_l_reg]), regB, regC[k_l_reg][npq_l_vec_reg]);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n for (uint k_l_reg = 0; k_l_reg < TS_K; ++k_l_reg) {\n const uint k_g = offset_k + lid_k * TS_K + k_l_reg;\n if (k_g >= K) continue;\n\n for (uint npq_l_vec_reg = 0; npq_l_vec_reg < TS_NPQ_VEC; ++npq_l_vec_reg) {\n const uint npq_g_base = offset_npq + (lid_npq * TS_NPQ_VEC + npq_l_vec_reg) * VEC_SIZE;\n\n const uint N_idx = npq_g_base / (OH * OW);\n const uint pq_idx = npq_g_base % (OH * OW);\n const uint OH_idx = pq_idx / OW;\n const uint OW_idx = pq_idx % OW;\n\n if (nb1 == OW && OW_idx + VEC_SIZE <= OW && npq_g_base + VEC_SIZE <= NPQ) {\n const uint dst_idx = OW_idx + OH_idx*nb1 + k_g*nb2 + N_idx*nb3;\n vstore4(regC[k_l_reg][npq_l_vec_reg], 0, &dst_data[dst_idx]);\n } else {\n T_ACCUM res = regC[k_l_reg][npq_l_vec_reg];\n for (int v = 0; v < VEC_SIZE; ++v) {\n const uint npq_g = npq_g_base + v;\n if (npq_g < NPQ) {\n const uint N_idx_s = npq_g / (OH*OW);\n const uint pq_idx_s = npq_g % (OH*OW);\n const uint OH_idx_s = pq_idx_s / OW;\n const uint OW_idx_s = pq_idx_s % OW;\n const uint dst_idx_s = OW_idx_s + OH_idx_s*nb1 + k_g*nb2 + N_idx_s*nb3;\n dst_data[dst_idx_s] = ((float*)&res)[v];\n }\n }\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/cpy.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// cpy\n//------------------------------------------------------------------------------\n\nkernel void kernel_cpy_f16_f16(\n global half * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;\n\n int i3 = n / (ne2*ne1*ne0);\n int i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);\n int i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;\n int i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);\n\n global half * dst_data = (global half *) ((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n global const half * src = (global half *)((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);\n dst_data[i00] = src[0];\n }\n}\n\nkernel void kernel_cpy_f16_f32(\n global half * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;\n\n int i3 = n / (ne2*ne1*ne0);\n int i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);\n int i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;\n int i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);\n\n global float * dst_data = (global float *) ((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n global half * src = (global half *)((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);\n dst_data[i00] = src[0];\n }\n}\n\nkernel void kernel_cpy_f32_f16(\n global float * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;\n\n int i3 = n / (ne2*ne1*ne0);\n int i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);\n int i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;\n int i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);\n\n global half * dst_data = (global half *) ((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n global const float * src = (global float *)((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);\n\n dst_data[i00] = src[0];\n }\n}\n\nkernel void kernel_cpy_f32_f32(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;\n\n int i3 = n / (ne2*ne1*ne0);\n int i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);\n int i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;\n int i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);\n\n global float * dst_data = (global float *) ((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n global const float * src = (global float *)((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);\n\n dst_data[i00] = src[0];\n }\n}\n\nkernel void kernel_cpy_i32_i32(\n global int * src0,\n ulong offset0,\n global int * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = (global int*)((global char*)src0 + offset0);\n dst = (global int*)((global char*)dst + offsetd);\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;\n\n int i3 = n / (ne2*ne1*ne0);\n int i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);\n int i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;\n int i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);\n\n global int * dst_data = (global int *) ((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n global const int * src = (global int *)((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);\n\n dst_data[i00] = src[0];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/cumsum.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n// max workgroup size is usually 1024, this covers various subgroups sizes\n#define MAX_SUBGROUPS 128\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_32\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_cumsum_blk(\n global char * src0,\n ulong offset0,\n global char * tmp,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n uint net0,\n uint net1,\n uint net2\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n const int nth = get_local_size(0);\n const int tid = get_local_id(0);\n\n const uint sg_size = get_sub_group_size();\n const uint sg_id = get_sub_group_id();\n const uint sg_lid = get_sub_group_local_id();\n\n const int ib = i1 / ne01;\n const int i00 = ib * nth;\n const int i01 = i1 % ne01;\n const int i02 = i2;\n const int i03 = i3;\n\n global const float * src0_row = (global const float *)(src0 + i03*nb03 + i02*nb02 + i01*nb01);\n global float * tmp_row = (global float *)tmp + net0 * i01 + net0 * net1 * i02 + net0 * net1 * net2 * i03;\n global float * dst_row = (global float *)dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;\n\n __local float partial[MAX_SUBGROUPS];\n\n float v = 0.0f;\n if (i00 + tid < ne00) {\n v = src0_row[i00 + tid];\n }\n\n float s = sub_group_scan_inclusive_add(v);\n if (sg_lid == sg_size - 1) {\n partial[sg_id] = s;\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n\n // NB: subgroup size should be larger than number of subgroups\n // assuming max workgroup size of 1024, subgroup size should be >= 32\n if (sg_id == 0) {\n float x = 0.0f;\n if (sg_lid < get_num_sub_groups()) {\n x = partial[sg_lid];\n }\n float ex = sub_group_scan_exclusive_add(x);\n if (sg_lid < get_num_sub_groups()) {\n partial[sg_lid] = ex;\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n\n s += partial[sg_id];\n\n if (i00 + tid < ne00) {\n dst_row[i00 + tid] = s;\n }\n if (ne00 > nth && tid == nth - 1) {\n tmp_row[ib] = s;\n }\n}\n\nkernel void kernel_cumsum_add(\n global char * tmp,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n uint nbt0,\n uint nbt1,\n uint nbt2,\n uint nbt3\n) {\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n const int nth = get_local_size(0);\n const int tid = get_local_id(0);\n\n const int ib = i1 / ne01;\n if (ib == 0) {\n return;\n }\n const int i00 = ib * nth;\n const int i01 = i1 % ne01;\n const int i02 = i2;\n const int i03 = i3;\n\n global float * tmp_row = (global float *)(tmp + nbt1 * i01 + nbt2 * i02 + nbt3 * i03);\n global float * dst_row = (global float *)dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;\n\n if (i00 + tid < ne00) {\n dst_row[i00 + tid] += tmp_row[ib - 1];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/cvt.cl", "content": "//------------------------------------------------------------------------------\n// This file is contains kernels for data conversion.\n// These kernels are used when loading the model, so its performance is less\n// important.\n//------------------------------------------------------------------------------\n#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_0 32\n#define QR4_0 2\n#define QK4_1 32\n#define QR4_1 2\n#define QK5_0 32\n#define QR5_0 2\n#define QK5_1 32\n#define QR5_1 2\n#define QK8_0 32\n#define QR8_0 1\n#define QK_K 256\n#define K_QUANTS_PER_ITERATION 2\n\ntypedef char int8_t;\ntypedef uchar uint8_t;\ntypedef short int16_t;\ntypedef ushort uint16_t;\ntypedef int int32_t;\ntypedef uint uint32_t;\n\n//------------------------------------------------------------------------------\n// block_q4_0\n//------------------------------------------------------------------------------\nstruct block_q4_0\n{\n half d;\n uint8_t qs[QK4_0 / 2];\n};\n\n//------------------------------------------------------------------------------\n// block_q4_1\n//------------------------------------------------------------------------------\nstruct block_q4_1 {\n half d; // delta\n half m; // min\n uchar qs[QK4_1 / 2]; // nibbles / quants\n};\n\n//------------------------------------------------------------------------------\n// block_q6_K\n//------------------------------------------------------------------------------\nstruct block_q6_K {\n uint8_t ql[QK_K/2]; // quants, lower 4 bits\n uint8_t qh[QK_K/4]; // quants, upper 2 bits\n int8_t scales[QK_K/16]; // scales, quantized with 8 bits\n half d; // super-block scale\n};\n\n//------------------------------------------------------------------------------\n// kernel_convert_block_q4_0\n// Convert the block_q4_0 format to 2 separate arrays (AOS -> SOA).\n// This kernel does not deshuffle the bits.\n//------------------------------------------------------------------------------\nkernel void kernel_convert_block_q4_0(\n global struct block_q4_0 * src0,\n global uchar * dst_q,\n global half * dst_d\n) {\n global struct block_q4_0 * b = (global struct block_q4_0 *) src0 + get_global_id(0);\n global uchar * q = (global uchar *) dst_q + QK4_0/2*get_global_id(0);\n global half * d = (global half *) dst_d + get_global_id(0);\n\n *d = b->d;\n\n for (int i = 0; i < QK4_0/2; ++i) {\n q[i] = b->qs[i];\n }\n}\n\nkernel void kernel_restore_block_q4_0(\n global uchar * src_q,\n global half * src_d,\n global struct block_q4_0 * dst\n) {\n global struct block_q4_0 * b = (global struct block_q4_0 *) dst + get_global_id(0);\n global uchar * q = (global uchar *) src_q + QK4_0/2*get_global_id(0);\n global half * d = (global half *) src_d + get_global_id(0);\n\n b->d = *d;\n for (int i = 0; i < QK4_0/2; ++i) {\n b->qs[i] = q[i];\n }\n}\n\n//------------------------------------------------------------------------------\n// kernel_convert_block_q4_0_noshuffle\n// Flatten q4_0 weights and unshuffle the bits\n//------------------------------------------------------------------------------\n\nkernel void kernel_convert_block_q4_0_noshuffle(\n global struct block_q4_0 * src0,\n global uchar * dst_q,\n global half * dst_d\n) {\n global struct block_q4_0 * b = (global struct block_q4_0 *) src0 + get_global_id(0);\n global uchar * q = (global uchar *) dst_q + QK4_0/2*get_global_id(0);\n global half * d = (global half *) dst_d + get_global_id(0);\n\n *d = b->d;\n for (int i = 0; i < QK4_0/4; ++i) {\n uchar x0 = b->qs[2*i + 0];\n uchar x1 = b->qs[2*i + 1];\n\n q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);\n q[i + QK4_0/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);\n\n#ifdef ADRENO_GPU\n // Workaround for adreno - must have the following printf statement for\n // the kernel to work properly. Otherwise it produces incorrect result.\n // convert_uchar above also seems necessary.\n // Compare against a large number so that it does not print anything.\n // get_sub_group_local_id() also works.\n if (get_global_id(0) == 65536*4096) {\n printf(\"%04x - %02x\\n\", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));\n }\n#endif\n }\n}\n\nkernel void kernel_restore_block_q4_0_noshuffle(\n global uchar * src_q,\n global half * src_d,\n global struct block_q4_0 * dst,\n uchar mask_0F,\n uchar mask_F0\n) {\n global struct block_q4_0 * b = (global struct block_q4_0 *) dst + get_global_id(0);\n global uchar * q = (global uchar *) src_q + QK4_0/2*get_global_id(0);\n global half * d = (global half *) src_d + get_global_id(0);\n\n b->d = *d;\n for (int i = 0; i < QK4_0/4; ++i) {\n uchar x0 = q[i + 0 ] ;\n uchar x1 = q[i + QK4_0/4];\n\n b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));\n b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));\n }\n}\n\n//------------------------------------------------------------------------------\n// kernel_convert_block_q4_1\n// Convert the block_q4_1 format to 2 separate arrays (AOS -> SOA).\n// This kernel does not deshuffle the bits.\n//------------------------------------------------------------------------------\nkernel void kernel_convert_block_q4_1(\n global struct block_q4_1 * src0,\n global uchar * dst_q,\n global half * dst_d,\n global half * dst_m\n) {\n global struct block_q4_1 * b = (global struct block_q4_1 *) src0 + get_global_id(0);\n global uchar * q = (global uchar *) dst_q + QK4_1/2*get_global_id(0);\n global half * d = (global half *) dst_d + get_global_id(0);\n global half * m = (global half *) dst_m + get_global_id(0);\n\n *d = b->d;\n *m = b->m;\n\n for (int i = 0; i < QK4_1/2; ++i) {\n q[i] = b->qs[i];\n }\n}\n\nkernel void kernel_restore_block_q4_1(\n global uchar * src_q,\n global half * src_d,\n global half * src_m,\n global struct block_q4_1 * dst\n) {\n global struct block_q4_1 * b = (global struct block_q4_1 *) dst + get_global_id(0);\n global uchar * q = (global uchar *) src_q + QK4_1/2*get_global_id(0);\n global half * d = (global half *) src_d + get_global_id(0);\n global half * m = (global half *) src_m + get_global_id(0);\n\n b->d = *d;\n b->m = *m;\n for (int i = 0; i < QK4_1/2; ++i) {\n b->qs[i] = q[i];\n }\n}\n\nkernel void kernel_convert_block_q4_1_noshuffle(\n global struct block_q4_1 * src0,\n global uchar * dst_q,\n global half * dst_d,\n global half * dst_m\n) {\n global struct block_q4_1 * b = (global struct block_q4_1 *) src0 + get_global_id(0);\n global uchar * q = (global uchar *) dst_q + QK4_1/2*get_global_id(0);\n global half * d = (global half *) dst_d + get_global_id(0);\n global half * m = (global half *) dst_m + get_global_id(0);\n\n *d = b->d;\n *m = b->m;\n for (int i = 0; i < QK4_1/4; ++i) {\n uchar x0 = b->qs[2*i + 0];\n uchar x1 = b->qs[2*i + 1];\n\n q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);\n q[i + QK4_1/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);\n\n#ifdef ADRENO_GPU\n if (get_global_id(0) == 65536*4096) {\n printf(\"%04x - %02x\\n\", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));\n }\n#endif\n }\n}\n\nkernel void kernel_restore_block_q4_1_noshuffle(\n global uchar * src_q,\n global half * src_d,\n global half * src_m,\n global struct block_q4_1 * dst,\n uchar mask_0F,\n uchar mask_F0\n) {\n global struct block_q4_1 * b = (global struct block_q4_1 *) dst + get_global_id(0);\n global uchar * q = (global uchar *) src_q + QK4_1/2*get_global_id(0);\n global half * d = (global half *) src_d + get_global_id(0);\n global half * m = (global half *) src_m + get_global_id(0);\n\n b->d = *d;\n b->m = *m;\n for (int i = 0; i < QK4_1/4; ++i) {\n uchar x0 = q[i + 0 ] ;\n uchar x1 = q[i + QK4_1/4];\n\n b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));\n b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));\n }\n}\n\n//------------------------------------------------------------------------------\n// block_mxfp4\n//------------------------------------------------------------------------------\n#define QK_MXFP4 32\nstruct block_mxfp4 {\n uchar e; // E8M0\n uchar qs[QK_MXFP4 / 2];\n};\n\n//------------------------------------------------------------------------------\n// kernel_convert_block_mxfp4\n// Convert the block_mxfp4 format to 2 separate arrays (AOS -> SOA).\n// This kernel does not deshuffle the bits.\n//------------------------------------------------------------------------------\nkernel void kernel_convert_block_mxfp4(\n global struct block_mxfp4 * src0,\n global uchar * dst_q,\n global uchar * dst_e\n) {\n global struct block_mxfp4 * b = (global struct block_mxfp4 *) src0 + get_global_id(0);\n global uchar * q = (global uchar *) dst_q + QK_MXFP4 / 2 * get_global_id(0);\n global uchar * e = (global uchar *) dst_e + get_global_id(0);\n\n *e = b->e;\n\n for (int i = 0; i < QK_MXFP4 / 2; ++i) {\n q[i] = b->qs[i];\n }\n}\n\nkernel void kernel_convert_block_mxfp4_trans(\n global struct block_mxfp4 * src0,\n __global uint4 * dst_q,\n __global uchar * dst_e,\n uint ne00,\n uint ne01\n) {\n int i00 = get_global_id(1);\n uint i01 = get_global_id(0);\n uint i02 = get_global_id(2);\n\n uint ne00_blk = ne00 / QK_MXFP4;\n uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;\n uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;\n\n global struct block_mxfp4 * b = src0 + src_blk_offset;\n\n dst_q[dst_blk_offset] = ((global uint4 *)(&(b->qs[0])))[0];\n dst_e[dst_blk_offset] = b->e;\n}\n\nkernel void kernel_restore_block_mxfp4(\n global uchar * src_q,\n global half * src_e,\n global struct block_mxfp4 * dst\n) {\n global struct block_mxfp4 * b = (global struct block_mxfp4 *) dst + get_global_id(0);\n global uchar * q = (global uchar *) src_q + QK_MXFP4 / 2 * get_global_id(0);\n global uchar * e = (global uchar *) src_e + get_global_id(0);\n\n b->e = *e;\n for (int i = 0; i < QK_MXFP4 / 2; ++i) {\n b->qs[i] = q[i];\n }\n}\n\nkernel void kernel_restore_block_mxfp4_trans(\n __global uint4 * src_q,\n __global uchar * src_e,\n global struct block_mxfp4 * dst,\n uint ne00,\n uint ne01\n) {\n int i00 = get_global_id(1);\n uint i01 = get_global_id(0);\n uint i02 = get_global_id(2);\n\n uint ne00_blk = ne00 / QK_MXFP4;\n uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;\n uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;\n\n global struct block_mxfp4 * b = dst + dst_blk_offset;\n\n ((global uint4 *)(&(b->qs[0])))[0] = src_q[src_blk_offset];\n b->e = src_e[src_blk_offset];\n}\n\n//------------------------------------------------------------------------------\n// block_q8_0\n//------------------------------------------------------------------------------\ntypedef struct {\n half d; // delta\n char qs[QK8_0]; // quants\n} block_q8_0;\n\nkernel void kernel_convert_block_q8_0(\n global block_q8_0 * src0,\n global uchar * dst_q,\n global half * dst_d\n) {\n global block_q8_0 * b = (global block_q8_0 *) src0 + get_global_id(0);\n global uchar * q = (global uchar *) dst_q + QK8_0*get_global_id(0);\n global half * d = (global half *) dst_d + get_global_id(0);\n\n *d = b->d;\n\n for (int i = 0; i < QK8_0; ++i) {\n q[i] = b->qs[i];\n }\n}\n\nkernel void kernel_restore_block_q8_0(\n global uchar * src_q,\n global half * src_d,\n global block_q8_0 * dst\n) {\n global block_q8_0 * b = (global block_q8_0 *) dst + get_global_id(0);\n global uchar * q = (global uchar *) src_q + QK8_0*get_global_id(0);\n global half * d = (global half *) src_d + get_global_id(0);\n\n b->d = *d;\n for (int i = 0; i < QK8_0; ++i) {\n b->qs[i] = q[i];\n }\n}\n\nkernel void kernel_restore_block_q8_0_trans(\n global uchar * src_q,\n global half * src_d,\n global block_q8_0 * dst,\n uint ne00,\n uint ne01\n){\n uint num_blk_per_row = ne00 / QK8_0;\n\n global block_q8_0 * b = (global block_q8_0 *) dst + get_global_id(0) * num_blk_per_row;\n global uchar * q = (global uchar *) src_q + get_global_id(0) * 4; // 4 8-bit packed\n global half * d = (global half *) src_d + get_global_id(0);\n\n for (uint blk = 0; blk < num_blk_per_row; blk++) {\n b->d = *d;\n\n for (uint i = 0; i < QK8_0; i+=4) {\n b->qs[i] = q[0];\n b->qs[i+1] = q[1];\n b->qs[i+2] = q[2];\n b->qs[i+3] = q[3];\n\n q += 4 * ne01; // M stride\n }\n\n d += ne01;\n\n b++;\n }\n}\n\n//------------------------------------------------------------------------------\n// kernel_convert_block_q6_K\n// Convert the block_q6_K format to 3 separate arrays (AOS -> SOA).\n// This kernel does not deshuffle the bits.\n// Each thread processes a super block.\n//------------------------------------------------------------------------------\nkernel void kernel_convert_block_q6_K(\n global struct block_q6_K * src0,\n global uchar * dst_ql,\n global uchar * dst_qh,\n global char * dst_s,\n global half * dst_d\n) {\n global struct block_q6_K * b = (global struct block_q6_K *) src0 + get_global_id(0);\n global uchar * ql = (global uchar *) dst_ql + QK_K/2*get_global_id(0);\n global uchar * qh = (global uchar *) dst_qh + QK_K/4*get_global_id(0);\n global char * s = (global char *) dst_s + QK_K/16*get_global_id(0);\n global half * d = (global half *) dst_d + get_global_id(0);\n\n *d = b->d;\n\n for (int i = 0; i < QK_K/2; ++i) {\n ql[i] = b->ql[i];\n }\n for (int i = 0; i < QK_K/4; ++i) {\n qh[i] = b->qh[i];\n }\n for (int i = 0; i < QK_K/16; ++i) {\n s[i] = b->scales[i];\n }\n}\n\n// Restore block_q6_K from flattened arrays.\n// Each thread processes a super block.\nkernel void kernel_restore_block_q6_K(\n global uchar * dst_ql,\n global uchar * dst_qh,\n global char * dst_s,\n global half * dst_d,\n global struct block_q6_K * dst\n) {\n global struct block_q6_K * b = (global struct block_q6_K *) dst + get_global_id(0);\n global uchar * ql = (global uchar *) dst_ql + QK_K/2*get_global_id(0);\n global uchar * qh = (global uchar *) dst_qh + QK_K/4*get_global_id(0);\n global char * s = (global char *) dst_s + QK_K/16*get_global_id(0);\n global half * d = (global half *) dst_d + get_global_id(0);\n\n b->d = *d;\n\n for (int i = 0; i < QK_K/2; ++i) {\n b->ql[i] = ql[i];\n }\n for (int i = 0; i < QK_K/4; ++i) {\n b->qh[i] = qh[i];\n }\n for (int i = 0; i < QK_K/16; ++i) {\n b->scales[i] = s[i];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/diag.cl", "content": "kernel void kernel_diag_f32(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n ulong nb0,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n int i3 = get_group_id(2);\n int i2 = get_group_id(1);\n int i1 = get_group_id(0);\n\n global const float * src0_ptr = (global const float *)(src0 + i2*nb02 + i3*nb03);\n global float * dst_ptr = (global float *)(dst + i1*nb01 + i2*nb2 + i3*nb3);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n dst_ptr[i0] = i0 == i1 ? src0_ptr[i0] : 0.0f;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/diag_mask_inf.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// diag_mask_inf kernels\n//------------------------------------------------------------------------------\nkernel void kernel_diag_mask_inf(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int n_past\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i02 = get_global_id(2);\n int i01 = get_global_id(1);\n int i00 = get_global_id(0);\n\n if (i00 > n_past + i01) {\n dst[i02*ne01*ne00 + i01*ne00 + i00] = -INFINITY;\n } else {\n dst[i02*ne01*ne00 + i01*ne00 + i00] = src0[i02*ne01*ne00 + i01*ne00 + i00];\n }\n}\n\nkernel void kernel_diag_mask_inf_8(\n global float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int n_past\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n int i = 2*get_global_id(0);\n\n dst[i+0] = src0[i+0];\n dst[i+1] = src0[i+1];\n int i4 = 4*i;\n int i02 = i4/(ne00*ne01); i4 -= i02*ne00*ne01;\n int i01 = i4/(ne00); i4 -= i01*ne00;\n int i00 = i4;\n for (int k = 3; k >= 0; --k) {\n if (i00 + 4 + k <= n_past + i01) {\n break;\n }\n (&dst[i+1])[k] = -INFINITY;\n if (i00 + k > n_past + i01) {\n (&dst[i])[k] = -INFINITY;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/div.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// div\n//------------------------------------------------------------------------------\nkernel void kernel_div(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03 % ne13;\n int i12 = i02 % ne12;\n int i11 = i01 % ne11;\n\n global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;\n global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;\n global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const int i10 = i0 % ne10;\n *((global float *)(dst_ptr + i0*nb0)) = *((global float *)(src0_ptr + i0*nb00)) / *((global float *)(src1_ptr + i10*nb10));\n }\n}\n\n// assumption: src1 is a row\n// broadcast src1 into src0\nkernel void kernel_div_row(\n global float4 * src0,\n ulong offset0,\n global float4 * src1,\n ulong offset1,\n global float4 * dst,\n ulong offsetd,\n int ne\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float4*)((global char*)dst + offsetd);\n\n // This performs better than using %.\n uint gid = get_global_id(0);\n uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne\n dst[gid] = src0[gid] / src1[idx1];\n}\n\nkernel void kernel_div_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03 % ne13;\n int i12 = i02 % ne12;\n int i11 = i01 % ne11;\n\n global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;\n global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;\n global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const int i10 = i0 % ne10;\n *((global half *)(dst_ptr + i0*nb0)) = *((global half *)(src0_ptr + i0*nb00)) / *((global half *)(src1_ptr + i10*nb10));\n }\n}\n\nkernel void kernel_div_row_f16(\n global half4 * src0,\n ulong offset0,\n global half4 * src1,\n ulong offset1,\n global half4 * dst,\n ulong offsetd,\n int ne\n) {\n src0 = (global half4*)((global char*)src0 + offset0);\n src1 = (global half4*)((global char*)src1 + offset1);\n dst = (global half4*)((global char*)dst + offsetd);\n\n // This performs better than using %.\n uint gid = get_global_id(0);\n uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne\n dst[gid] = src0[gid] / src1[idx1];\n}\n" }, { "path": "src/ggml-opencl/kernels/embed_kernel.py", "content": "#\n\nimport sys\nimport logging\nlogger = logging.getLogger(\"opencl-embed-kernel\")\n\n\ndef main():\n logging.basicConfig(level=logging.INFO)\n\n if len(sys.argv) != 3:\n logger.info(\"Usage: python embed_kernel.py \")\n sys.exit(1)\n\n ifile = open(sys.argv[1], \"r\")\n ofile = open(sys.argv[2], \"w\")\n\n for i in ifile:\n ofile.write('R\"({})\"\\n'.format(i))\n\n ifile.close()\n ofile.close()\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "src/ggml-opencl/kernels/exp.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\nkernel void kernel_exp_f32(\n global const float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int n\n) {\n if (get_global_id(0) >= n) {\n return;\n }\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = exp(src0[get_global_id(0)]);\n}\n\nkernel void kernel_exp_f32_4(\n global const float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd,\n int n\n) {\n if (get_global_id(0) >= n) {\n return;\n }\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = exp(src0[get_global_id(0)]);\n}\n\nkernel void kernel_exp_f16(\n global const half * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd,\n int n\n) {\n if (get_global_id(0) >= n) {\n return;\n }\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = exp(src0[get_global_id(0)]);\n}\n\nkernel void kernel_exp_f16_4(\n global const half4 * src0,\n ulong offset0,\n global half4 * dst,\n ulong offsetd,\n int n\n) {\n if (get_global_id(0) >= n) {\n return;\n }\n src0 = (global half4*)((global char*)src0 + offset0);\n dst = (global half4*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = exp(src0[get_global_id(0)]);\n}\n\nkernel void kernel_exp_f32_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = exp(*x);\n }\n}\n\nkernel void kernel_exp_f16_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = exp(*x);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/expm1.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// expm1\n//------------------------------------------------------------------------------\n\nkernel void kernel_expm1_f32(\n global const float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = exp(src0[get_global_id(0)]) - 1.0f;\n}\n\nkernel void kernel_expm1_f32_4(\n global const float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = exp(src0[get_global_id(0)]) - 1.0f;\n}\n\nkernel void kernel_expm1_f16(\n global const half * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd\n) {\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = exp(src0[get_global_id(0)]) - 1.0h;\n}\n\nkernel void kernel_expm1_f16_4(\n global const half4 * src0,\n ulong offset0,\n global half4 * dst,\n ulong offsetd\n) {\n src0 = (global half4*)((global char*)src0 + offset0);\n dst = (global half4*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = exp(src0[get_global_id(0)]) - 1.0h;\n}\n\nkernel void kernel_expm1_f32_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = exp(*x) - 1.0f;\n }\n}\n\nkernel void kernel_expm1_f16_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = exp(*x) - 1.0f;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/fill.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// fill\n//------------------------------------------------------------------------------\n__kernel void kernel_fill_f32(\n __global float *dst,\n ulong offsetd,\n float v,\n int n\n\n) {\n dst = (global float*)((global char*)dst + offsetd);\n if(get_global_id(0) < n){\n dst[get_global_id(0)] = v;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/flash_attn_f16.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define ACC_TYPE float\n#define ACC_TYPE4 float4\n#define DATA_TYPE half\n#define DATA_TYPE4 half4\n#define CONVERT_ACC4(x) convert_float4(x)\n#define CONVERT_DATA4(x) convert_half4(x)\n\n#define DK_VEC (DK/4)\n#define DV_VEC (DV/4)\n#define WG_SIZE (BLOCK_M)\n#define Q1_WG_SIZE 64\n\ninline float get_alibi_slope(\n const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1\n) {\n if (max_bias <= 0.0f) {\n return 1.0f;\n }\n const float base = h < n_head_log2 ? m0 : m1;\n const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n return pow(base, exph);\n}\n__kernel void flash_attn_f16(\n const global void * q_void, ulong q_offset,\n const global void * k_void, ulong k_offset,\n const global void * v_void, ulong v_offset,\n global void * o_void, ulong o_offset,\n const float scale,\n const int n_q,\n const int n_kv,\n const int is_causal,\n const int n_head,\n const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,\n const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,\n const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,\n const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,\n const float max_bias,\n const float m0,\n const float m1,\n const int n_head_log2,\n const float logit_softcap,\n const int n_head_kv,\n const global void* mask_void,\n const ulong mask_offset,\n const ulong mask_nb1,\n const ulong mask_nb2,\n const ulong mask_nb3,\n const int mask_ne2,\n const int mask_ne3,\n const global void* sinks_void,\n const ulong sinks_offset\n) {\n const int tid = get_local_id(0);\n const int block_q_idx = get_group_id(0);\n const int head_batch_idx = get_global_id(1);\n\n const int my_query_row = block_q_idx * BLOCK_M + tid;\n\n const int batch_idx = head_batch_idx / n_head;\n const int head_idx = head_batch_idx % n_head;\n\n const int gqa_ratio = n_head / n_head_kv;\n const int head_kv_idx = head_idx / gqa_ratio;\n\n const global char* q_base = (const global char*)q_void + q_offset;\n const global char* k_base = (const global char*)k_void + k_offset;\n const global char* v_base = (const global char*)v_void + v_offset;\n global char* o_base = (global char*)o_void + o_offset;\n\n const global char* mask_base = NULL;\n if (mask_void != NULL) {\n const int mask_head_idx = head_idx % mask_ne2;\n const int mask_batch_idx = batch_idx % mask_ne3;\n mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;\n }\n\n ACC_TYPE4 q_priv[DK_VEC];\n if (my_query_row < n_q) {\n const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1;\n const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset);\n #pragma unroll\n for (int i = 0; i < DK_VEC; ++i) {\n q_priv[i] = CONVERT_ACC4(q_ptr[i]);\n }\n }\n\n ACC_TYPE4 o_acc[DV_VEC];\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_acc[i] = (ACC_TYPE4)(0.0f);\n }\n ACC_TYPE m_i = -INFINITY;\n ACC_TYPE l_i = 0.0f;\n\n float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);\n\n __local DATA_TYPE4 l_k[BLOCK_N][DK_VEC];\n __local DATA_TYPE4 l_v[BLOCK_N][DV_VEC];\n\n for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) {\n for (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) {\n const int row = i / DK_VEC;\n const int col = i % DK_VEC;\n const int k_row_idx = k_start + row;\n if (k_row_idx < n_kv) {\n const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1;\n l_k[row][col] = ((__global DATA_TYPE4*)(k_base + k_row_offset))[col];\n }\n }\n for (int i = tid; i < BLOCK_N * DV_VEC; i += WG_SIZE) {\n const int row = i / DV_VEC;\n const int col = i % DV_VEC;\n const int v_row_idx = k_start + row;\n if (v_row_idx < n_kv) {\n const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1;\n l_v[row][col] = ((__global DATA_TYPE4*)(v_base + v_row_offset))[col];\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if (my_query_row >= n_q) {\n continue;\n }\n\n for (int j = 0; j < BLOCK_N; j += 2) {\n const int k_row0 = k_start + j;\n const int k_row1 = k_start + j + 1;\n\n ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f);\n ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f);\n #pragma unroll\n for (int k = 0; k < DK_VEC; k++) {\n dot_acc0 = mad(q_priv[k], CONVERT_ACC4(l_k[j][k]), dot_acc0);\n dot_acc1 = mad(q_priv[k], CONVERT_ACC4(l_k[j+1][k]), dot_acc1);\n }\n ACC_TYPE score0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale;\n ACC_TYPE score1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale;\n\n if (is_causal) {\n if (k_row0 > (n_kv - n_q + my_query_row)) score0 = -INFINITY;\n if (k_row1 > (n_kv - n_q + my_query_row)) score1 = -INFINITY;\n }\n\n if (k_row0 >= n_kv) score0 = -INFINITY;\n if (k_row1 >= n_kv) score1 = -INFINITY;\n\n if (mask_base != NULL) {\n const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base + my_query_row * mask_nb1);\n if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0];\n if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1];\n }\n\n if (logit_softcap > 0.0f) {\n score0 = logit_softcap * tanh(score0 / logit_softcap);\n score1 = logit_softcap * tanh(score1 / logit_softcap);\n }\n\n const ACC_TYPE m_new = max(m_i, max(score0, score1));\n const ACC_TYPE p0 = exp(score0 - m_new);\n const ACC_TYPE p1 = exp(score1 - m_new);\n const ACC_TYPE scale_prev = exp(m_i - m_new);\n\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_acc[i] = o_acc[i] * scale_prev + p0 * CONVERT_ACC4(l_v[j][i]) + p1 * CONVERT_ACC4(l_v[j+1][i]);\n }\n l_i = l_i * scale_prev + p0 + p1;\n m_i = m_new;\n }\n }\n\n if (my_query_row < n_q) {\n if (sinks_void != NULL) {\n const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);\n const ACC_TYPE m_sink = sinks_ptr[head_idx];\n const ACC_TYPE m_final = max(m_i, m_sink);\n\n const ACC_TYPE scale_o = exp(m_i - m_final);\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_acc[i] *= scale_o;\n }\n\n l_i = l_i * exp(m_i - m_final) + exp(m_sink - m_final);\n }\n\n const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1;\n global DATA_TYPE4 *o_row = (global DATA_TYPE4 *)(o_base + o_row_offset);\n if (l_i > 0.0f) {\n const ACC_TYPE l_inv = 1.0f / l_i;\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_row[i] = CONVERT_DATA4(o_acc[i] * l_inv);\n }\n } else {\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_row[i] = (DATA_TYPE4)(0.0f);\n }\n }\n }\n}\n\n__kernel void flash_attn_f16_q1(\n const global void * q_void, ulong q_offset,\n const global void * k_void, ulong k_offset,\n const global void * v_void, ulong v_offset,\n global void * o_void, ulong o_offset,\n const float scale,\n const int n_q,\n const int n_kv,\n const int is_causal,\n const int n_head,\n const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,\n const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,\n const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,\n const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,\n const float max_bias,\n const float m0,\n const float m1,\n const int n_head_log2,\n const float logit_softcap,\n const int n_head_kv,\n const global void* mask_void,\n const ulong mask_offset,\n const ulong mask_nb1,\n const ulong mask_nb2,\n const ulong mask_nb3,\n const int mask_ne2,\n const int mask_ne3,\n const global void* sinks_void,\n const ulong sinks_offset\n) {\n const int tid = get_local_id(0);\n const int head_batch_idx = get_global_id(1);\n\n const int batch_idx = head_batch_idx / n_head;\n const int head_idx = head_batch_idx % n_head;\n\n const int gqa_ratio = n_head / n_head_kv;\n const int head_kv_idx = head_idx / gqa_ratio;\n\n const global char* q_base = (const global char*)q_void + q_offset;\n const global char* k_base = (const global char*)k_void + k_offset;\n const global char* v_base = (const global char*)v_void + v_offset;\n global char* o_base = (global char*)o_void + o_offset;\n\n const global char* mask_base = NULL;\n if (mask_void != NULL) {\n const int mask_head_idx = head_idx % mask_ne2;\n const int mask_batch_idx = batch_idx % mask_ne3;\n mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;\n }\n\n ACC_TYPE4 q_priv[DK_VEC];\n const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;\n const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset);\n #pragma unroll\n for (int i = 0; i < DK_VEC; ++i) {\n q_priv[i] = CONVERT_ACC4(q_ptr[i]);\n }\n\n float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);\n\n const global ACC_TYPE* sinks_ptr = NULL;\n if (sinks_void != NULL) {\n sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);\n }\n\n ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : -INFINITY;\n for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {\n const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;\n const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset);\n ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);\n #pragma unroll\n for (int k = 0; k < DK_VEC; k++) {\n dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc);\n }\n ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;\n if (mask_base != NULL) {\n const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base);\n score += slope * (ACC_TYPE)mask_ptr[k_idx];\n }\n if (logit_softcap > 0.0f) {\n score = logit_softcap * tanh(score / logit_softcap);\n }\n m_i = max(m_i, score);\n }\n\n __local ACC_TYPE local_m[Q1_WG_SIZE];\n local_m[tid] = m_i;\n barrier(CLK_LOCAL_MEM_FENCE);\n #pragma unroll\n for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {\n if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]);\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n const ACC_TYPE m_final = local_m[0];\n\n ACC_TYPE4 o_acc[DV_VEC];\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);\n ACC_TYPE l_i = 0.0f;\n\n for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {\n const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;\n const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;\n const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset);\n const global DATA_TYPE4* v_ptr = (const global DATA_TYPE4*)(v_base + v_row_offset);\n ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);\n #pragma unroll\n for (int k = 0; k < DK_VEC; k++) {\n dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc);\n }\n ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;\n if (mask_base != NULL) {\n const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base);\n score += slope * (ACC_TYPE)mask_ptr[k_idx];\n }\n if (logit_softcap > 0.0f) {\n score = logit_softcap * tanh(score / logit_softcap);\n }\n const ACC_TYPE p = exp(score - m_final);\n l_i += p;\n #pragma unroll\n for (int i = 0; i < DV_VEC; i++) {\n o_acc[i] = mad(p, CONVERT_ACC4(v_ptr[i]), o_acc[i]);\n }\n }\n\n __local ACC_TYPE local_l[Q1_WG_SIZE];\n __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE];\n local_l[tid] = l_i;\n barrier(CLK_LOCAL_MEM_FENCE);\n #pragma unroll\n for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {\n if (tid < s) local_l[tid] += local_l[tid + s];\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;\n global DATA_TYPE4 *o_row = (global DATA_TYPE4 *)(o_base + o_row_offset);\n ACC_TYPE l_final = local_l[0];\n\n if (sinks_ptr != NULL) {\n l_final += exp(sinks_ptr[head_idx] - m_final);\n }\n\n if (l_final > 0.0f) {\n const ACC_TYPE l_inv = 1.0f / l_final;\n for (int i = 0; i < DV_VEC; i++) {\n local_o_comp[tid] = o_acc[i];\n barrier(CLK_LOCAL_MEM_FENCE);\n #pragma unroll\n for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {\n if (tid < s) local_o_comp[tid] += local_o_comp[tid + s];\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n if (tid == 0) {\n o_row[i] = CONVERT_DATA4(local_o_comp[0] * l_inv);\n }\n }\n } else if (tid == 0) {\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) o_row[i] = (DATA_TYPE4)(0.0f);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/flash_attn_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define ACC_TYPE float\n#define ACC_TYPE4 float4\n#define DATA_TYPE float\n#define DATA_TYPE4 float4\n#define MASK_DATA_TYPE half\n#define CONVERT_ACC4(x) (x)\n#define CONVERT_DATA4(x) (x)\n\n#define DK_VEC (DK/4)\n#define DV_VEC (DV/4)\n#define WG_SIZE (BLOCK_M)\n#define Q1_WG_SIZE 64\n\ninline float get_alibi_slope(\n const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1\n) {\n if (max_bias <= 0.0f) {\n return 1.0f;\n }\n const float base = h < n_head_log2 ? m0 : m1;\n const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n return pow(base, exph);\n}\n__kernel void flash_attn_f32(\n const global void * q_void, ulong q_offset,\n const global void * k_void, ulong k_offset,\n const global void * v_void, ulong v_offset,\n global void * o_void, ulong o_offset,\n const float scale,\n const int n_q,\n const int n_kv,\n const int is_causal,\n const int n_head,\n const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,\n const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,\n const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,\n const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,\n const float max_bias,\n const float m0,\n const float m1,\n const int n_head_log2,\n const float logit_softcap,\n const int n_head_kv,\n const global void* mask_void,\n const ulong mask_offset,\n const ulong mask_nb1,\n const ulong mask_nb2,\n const ulong mask_nb3,\n const int mask_ne2,\n const int mask_ne3,\n const global void* sinks_void,\n const ulong sinks_offset\n) {\n const int tid = get_local_id(0);\n const int block_q_idx = get_group_id(0);\n const int head_batch_idx = get_global_id(1);\n\n const int my_query_row = block_q_idx * BLOCK_M + tid;\n\n const int batch_idx = head_batch_idx / n_head;\n const int head_idx = head_batch_idx % n_head;\n\n const int gqa_ratio = n_head / n_head_kv;\n const int head_kv_idx = head_idx / gqa_ratio;\n\n const global char* q_base = (const global char*)q_void + q_offset;\n const global char* k_base = (const global char*)k_void + k_offset;\n const global char* v_base = (const global char*)v_void + v_offset;\n global char* o_base = (global char*)o_void + o_offset;\n\n const global char* mask_base = NULL;\n if (mask_void != NULL) {\n const int mask_head_idx = head_idx % mask_ne2;\n const int mask_batch_idx = batch_idx % mask_ne3;\n mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;\n }\n\n ACC_TYPE4 q_priv[DK_VEC];\n if (my_query_row < n_q) {\n const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1;\n const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset);\n #pragma unroll\n for (int i = 0; i < DK_VEC; ++i) {\n q_priv[i] = CONVERT_ACC4(q_ptr[i]);\n }\n }\n\n ACC_TYPE4 o_acc[DV_VEC];\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_acc[i] = (ACC_TYPE4)(0.0f);\n }\n ACC_TYPE m_i = -INFINITY;\n ACC_TYPE l_i = 0.0f;\n\n float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);\n\n __local DATA_TYPE4 l_k[BLOCK_N][DK_VEC];\n __local DATA_TYPE4 l_v[BLOCK_N][DV_VEC];\n\n for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) {\n for (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) {\n const int row = i / DK_VEC;\n const int col = i % DK_VEC;\n const int k_row_idx = k_start + row;\n if (k_row_idx < n_kv) {\n const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1;\n l_k[row][col] = ((__global DATA_TYPE4*)(k_base + k_row_offset))[col];\n }\n }\n for (int i = tid; i < BLOCK_N * DV_VEC; i += WG_SIZE) {\n const int row = i / DV_VEC;\n const int col = i % DV_VEC;\n const int v_row_idx = k_start + row;\n if (v_row_idx < n_kv) {\n const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1;\n l_v[row][col] = ((__global DATA_TYPE4*)(v_base + v_row_offset))[col];\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if (my_query_row >= n_q) {\n continue;\n }\n\n for (int j = 0; j < BLOCK_N; j += 2) {\n const int k_row0 = k_start + j;\n const int k_row1 = k_start + j + 1;\n\n ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f);\n ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f);\n #pragma unroll\n for (int k = 0; k < DK_VEC; k++) {\n dot_acc0 = mad(q_priv[k], CONVERT_ACC4(l_k[j][k]), dot_acc0);\n dot_acc1 = mad(q_priv[k], CONVERT_ACC4(l_k[j+1][k]), dot_acc1);\n }\n ACC_TYPE score0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale;\n ACC_TYPE score1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale;\n\n if (is_causal) {\n if (k_row0 > (n_kv - n_q + my_query_row)) score0 = -INFINITY;\n if (k_row1 > (n_kv - n_q + my_query_row)) score1 = -INFINITY;\n }\n\n if (k_row0 >= n_kv) score0 = -INFINITY;\n if (k_row1 >= n_kv) score1 = -INFINITY;\n\n if (mask_base != NULL) {\n const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1);\n if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0];\n if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1];\n }\n\n if (logit_softcap > 0.0f) {\n score0 = logit_softcap * tanh(score0 / logit_softcap);\n score1 = logit_softcap * tanh(score1 / logit_softcap);\n }\n\n const ACC_TYPE m_new = max(m_i, max(score0, score1));\n const ACC_TYPE p0 = exp(score0 - m_new);\n const ACC_TYPE p1 = exp(score1 - m_new);\n const ACC_TYPE scale_prev = exp(m_i - m_new);\n\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_acc[i] = o_acc[i] * scale_prev + p0 * CONVERT_ACC4(l_v[j][i]) + p1 * CONVERT_ACC4(l_v[j+1][i]);\n }\n l_i = l_i * scale_prev + p0 + p1;\n m_i = m_new;\n }\n }\n\n if (my_query_row < n_q) {\n if (sinks_void != NULL) {\n const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);\n const ACC_TYPE m_sink = sinks_ptr[head_idx];\n const ACC_TYPE m_final = max(m_i, m_sink);\n\n const ACC_TYPE scale_o = exp(m_i - m_final);\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_acc[i] *= scale_o;\n }\n\n l_i = l_i * exp(m_i - m_final) + exp(m_sink - m_final);\n }\n\n const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1;\n global DATA_TYPE4 *o_row = (global DATA_TYPE4 *)(o_base + o_row_offset);\n if (l_i > 0.0f) {\n const ACC_TYPE l_inv = 1.0f / l_i;\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_row[i] = CONVERT_DATA4(o_acc[i] * l_inv);\n }\n } else {\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_row[i] = (DATA_TYPE4)(0.0f);\n }\n }\n }\n}\n\n__kernel void flash_attn_f32_q1(\n const global void * q_void, ulong q_offset,\n const global void * k_void, ulong k_offset,\n const global void * v_void, ulong v_offset,\n global void * o_void, ulong o_offset,\n const float scale,\n const int n_q,\n const int n_kv,\n const int is_causal,\n const int n_head,\n const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,\n const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,\n const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,\n const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,\n const float max_bias,\n const float m0,\n const float m1,\n const int n_head_log2,\n const float logit_softcap,\n const int n_head_kv,\n const global void* mask_void,\n const ulong mask_offset,\n const ulong mask_nb1,\n const ulong mask_nb2,\n const ulong mask_nb3,\n const int mask_ne2,\n const int mask_ne3,\n const global void* sinks_void,\n const ulong sinks_offset\n) {\n const int tid = get_local_id(0);\n const int head_batch_idx = get_global_id(1);\n\n const int batch_idx = head_batch_idx / n_head;\n const int head_idx = head_batch_idx % n_head;\n\n const int gqa_ratio = n_head / n_head_kv;\n const int head_kv_idx = head_idx / gqa_ratio;\n\n const global char* q_base = (const global char*)q_void + q_offset;\n const global char* k_base = (const global char*)k_void + k_offset;\n const global char* v_base = (const global char*)v_void + v_offset;\n global char* o_base = (global char*)o_void + o_offset;\n\n const global char* mask_base = NULL;\n if (mask_void != NULL) {\n const int mask_head_idx = head_idx % mask_ne2;\n const int mask_batch_idx = batch_idx % mask_ne3;\n mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;\n }\n\n ACC_TYPE4 q_priv[DK_VEC];\n const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;\n const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset);\n #pragma unroll\n for (int i = 0; i < DK_VEC; ++i) {\n q_priv[i] = CONVERT_ACC4(q_ptr[i]);\n }\n\n float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);\n\n const global ACC_TYPE* sinks_ptr = NULL;\n if (sinks_void != NULL) {\n sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);\n }\n\n ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : -INFINITY;\n for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {\n const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;\n const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset);\n ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);\n #pragma unroll\n for (int k = 0; k < DK_VEC; k++) {\n dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc);\n }\n ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;\n if (mask_base != NULL) {\n const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);\n score += slope * (ACC_TYPE)mask_ptr[k_idx];\n }\n if (logit_softcap > 0.0f) {\n score = logit_softcap * tanh(score / logit_softcap);\n }\n m_i = max(m_i, score);\n }\n\n __local ACC_TYPE local_m[Q1_WG_SIZE];\n local_m[tid] = m_i;\n barrier(CLK_LOCAL_MEM_FENCE);\n #pragma unroll\n for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {\n if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]);\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n const ACC_TYPE m_final = local_m[0];\n\n ACC_TYPE4 o_acc[DV_VEC];\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);\n ACC_TYPE l_i = 0.0f;\n\n for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {\n const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;\n const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;\n const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset);\n const global DATA_TYPE4* v_ptr = (const global DATA_TYPE4*)(v_base + v_row_offset);\n ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);\n #pragma unroll\n for (int k = 0; k < DK_VEC; k++) {\n dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc);\n }\n ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;\n if (mask_base != NULL) {\n const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);\n score += slope * (ACC_TYPE)mask_ptr[k_idx];\n }\n if (logit_softcap > 0.0f) {\n score = logit_softcap * tanh(score / logit_softcap);\n }\n const ACC_TYPE p = exp(score - m_final);\n l_i += p;\n #pragma unroll\n for (int i = 0; i < DV_VEC; i++) {\n o_acc[i] = mad(p, CONVERT_ACC4(v_ptr[i]), o_acc[i]);\n }\n }\n\n __local ACC_TYPE local_l[Q1_WG_SIZE];\n __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE];\n local_l[tid] = l_i;\n barrier(CLK_LOCAL_MEM_FENCE);\n #pragma unroll\n for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {\n if (tid < s) local_l[tid] += local_l[tid + s];\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;\n global DATA_TYPE4 *o_row = (global DATA_TYPE4 *)(o_base + o_row_offset);\n ACC_TYPE l_final = local_l[0];\n\n if (sinks_ptr != NULL) {\n l_final += exp(sinks_ptr[head_idx] - m_final);\n }\n\n if (l_final > 0.0f) {\n const ACC_TYPE l_inv = 1.0f / l_final;\n for (int i = 0; i < DV_VEC; i++) {\n local_o_comp[tid] = o_acc[i];\n barrier(CLK_LOCAL_MEM_FENCE);\n #pragma unroll\n for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {\n if (tid < s) local_o_comp[tid] += local_o_comp[tid + s];\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n if (tid == 0) {\n o_row[i] = CONVERT_DATA4(local_o_comp[0] * l_inv);\n }\n }\n } else if (tid == 0) {\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) o_row[i] = (DATA_TYPE4)(0.0f);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/flash_attn_f32_f16.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define ACC_TYPE float\n#define ACC_TYPE4 float4\n#define Q_DATA_TYPE4 float4\n#define KV_DATA_TYPE4 half4\n#define O_DATA_TYPE4 float4\n#define MASK_DATA_TYPE half\n#define CONVERT_Q_ACC4(x) (x)\n#define CONVERT_KV_ACC4(x) convert_float4(x)\n#define CONVERT_O_DATA4(x) (x)\n\n#define DK_VEC (DK/4)\n#define DV_VEC (DV/4)\n#define WG_SIZE (BLOCK_M)\n#define Q1_WG_SIZE 64\n\ninline float get_alibi_slope(\n const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1\n) {\n if (max_bias <= 0.0f) {\n return 1.0f;\n }\n const float base = h < n_head_log2 ? m0 : m1;\n const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n return pow(base, exph);\n}\n__kernel void flash_attn_f32_f16(\n const global void * q_void, ulong q_offset,\n const global void * k_void, ulong k_offset,\n const global void * v_void, ulong v_offset,\n global void * o_void, ulong o_offset,\n const float scale,\n const int n_q,\n const int n_kv,\n const int is_causal,\n const int n_head,\n const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,\n const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,\n const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,\n const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,\n const float max_bias,\n const float m0,\n const float m1,\n const int n_head_log2,\n const float logit_softcap,\n const int n_head_kv,\n const global void* mask_void,\n const ulong mask_offset,\n const ulong mask_nb1,\n const ulong mask_nb2,\n const ulong mask_nb3,\n const int mask_ne2,\n const int mask_ne3,\n const global void* sinks_void,\n const ulong sinks_offset\n) {\n const int tid = get_local_id(0);\n const int block_q_idx = get_group_id(0);\n const int head_batch_idx = get_global_id(1);\n\n const int my_query_row = block_q_idx * BLOCK_M + tid;\n\n const int batch_idx = head_batch_idx / n_head;\n const int head_idx = head_batch_idx % n_head;\n\n const int gqa_ratio = n_head / n_head_kv;\n const int head_kv_idx = head_idx / gqa_ratio;\n\n const global char* q_base = (const global char*)q_void + q_offset;\n const global char* k_base = (const global char*)k_void + k_offset;\n const global char* v_base = (const global char*)v_void + v_offset;\n global char* o_base = (global char*)o_void + o_offset;\n\n const global char* mask_base = NULL;\n if (mask_void != NULL) {\n const int mask_head_idx = head_idx % mask_ne2;\n const int mask_batch_idx = batch_idx % mask_ne3;\n mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;\n }\n\n ACC_TYPE4 q_priv[DK_VEC];\n if (my_query_row < n_q) {\n const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1;\n const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset);\n #pragma unroll\n for (int i = 0; i < DK_VEC; ++i) {\n q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]);\n }\n }\n\n ACC_TYPE4 o_acc[DV_VEC];\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_acc[i] = (ACC_TYPE4)(0.0f);\n }\n ACC_TYPE m_i = -INFINITY;\n ACC_TYPE l_i = 0.0f;\n\n float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);\n\n __local KV_DATA_TYPE4 l_k[BLOCK_N][DK_VEC];\n __local KV_DATA_TYPE4 l_v[BLOCK_N][DV_VEC];\n\n for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) {\n for (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) {\n const int row = i / DK_VEC;\n const int col = i % DK_VEC;\n const int k_row_idx = k_start + row;\n if (k_row_idx < n_kv) {\n const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1;\n l_k[row][col] = ((__global KV_DATA_TYPE4*)(k_base + k_row_offset))[col];\n }\n }\n for (int i = tid; i < BLOCK_N * DV_VEC; i += WG_SIZE) {\n const int row = i / DV_VEC;\n const int col = i % DV_VEC;\n const int v_row_idx = k_start + row;\n if (v_row_idx < n_kv) {\n const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1;\n l_v[row][col] = ((__global KV_DATA_TYPE4*)(v_base + v_row_offset))[col];\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if (my_query_row >= n_q) {\n continue;\n }\n\n for (int j = 0; j < BLOCK_N; j += 2) {\n const int k_row0 = k_start + j;\n const int k_row1 = k_start + j + 1;\n\n ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f);\n ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f);\n #pragma unroll\n for (int k = 0; k < DK_VEC; k++) {\n dot_acc0 = mad(q_priv[k], CONVERT_KV_ACC4(l_k[j][k]), dot_acc0);\n dot_acc1 = mad(q_priv[k], CONVERT_KV_ACC4(l_k[j+1][k]), dot_acc1);\n }\n ACC_TYPE score0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale;\n ACC_TYPE score1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale;\n\n if (is_causal) {\n if (k_row0 > (n_kv - n_q + my_query_row)) score0 = -INFINITY;\n if (k_row1 > (n_kv - n_q + my_query_row)) score1 = -INFINITY;\n }\n\n if (k_row0 >= n_kv) score0 = -INFINITY;\n if (k_row1 >= n_kv) score1 = -INFINITY;\n\n if (mask_base != NULL) {\n const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1);\n if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0];\n if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1];\n }\n\n if (logit_softcap > 0.0f) {\n score0 = logit_softcap * tanh(score0 / logit_softcap);\n score1 = logit_softcap * tanh(score1 / logit_softcap);\n }\n\n const ACC_TYPE m_new = max(m_i, max(score0, score1));\n const ACC_TYPE p0 = exp(score0 - m_new);\n const ACC_TYPE p1 = exp(score1 - m_new);\n const ACC_TYPE scale_prev = exp(m_i - m_new);\n\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_acc[i] = o_acc[i] * scale_prev + p0 * CONVERT_KV_ACC4(l_v[j][i]) + p1 * CONVERT_KV_ACC4(l_v[j+1][i]);\n }\n l_i = l_i * scale_prev + p0 + p1;\n m_i = m_new;\n }\n }\n\n if (my_query_row < n_q) {\n if (sinks_void != NULL) {\n const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);\n const ACC_TYPE m_sink = sinks_ptr[head_idx];\n const ACC_TYPE m_final = max(m_i, m_sink);\n\n const ACC_TYPE scale_o = exp(m_i - m_final);\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_acc[i] *= scale_o;\n }\n\n l_i = l_i * exp(m_i - m_final) + exp(m_sink - m_final);\n }\n\n const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1;\n global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset);\n if (l_i > 0.0f) {\n const ACC_TYPE l_inv = 1.0f / l_i;\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_row[i] = CONVERT_O_DATA4(o_acc[i] * l_inv);\n }\n } else {\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) {\n o_row[i] = (O_DATA_TYPE4)(0.0f);\n }\n }\n }\n}\n\n__kernel void flash_attn_f32_f16_q1(\n const global void * q_void, ulong q_offset,\n const global void * k_void, ulong k_offset,\n const global void * v_void, ulong v_offset,\n global void * o_void, ulong o_offset,\n const float scale,\n const int n_q,\n const int n_kv,\n const int is_causal,\n const int n_head,\n const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,\n const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,\n const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,\n const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,\n const float max_bias,\n const float m0,\n const float m1,\n const int n_head_log2,\n const float logit_softcap,\n const int n_head_kv,\n const global void* mask_void,\n const ulong mask_offset,\n const ulong mask_nb1,\n const ulong mask_nb2,\n const ulong mask_nb3,\n const int mask_ne2,\n const int mask_ne3,\n const global void* sinks_void,\n const ulong sinks_offset\n) {\n const int tid = get_local_id(0);\n const int head_batch_idx = get_global_id(1);\n\n const int batch_idx = head_batch_idx / n_head;\n const int head_idx = head_batch_idx % n_head;\n\n const int gqa_ratio = n_head / n_head_kv;\n const int head_kv_idx = head_idx / gqa_ratio;\n\n const global char* q_base = (const global char*)q_void + q_offset;\n const global char* k_base = (const global char*)k_void + k_offset;\n const global char* v_base = (const global char*)v_void + v_offset;\n global char* o_base = (global char*)o_void + o_offset;\n\n const global char* mask_base = NULL;\n if (mask_void != NULL) {\n const int mask_head_idx = head_idx % mask_ne2;\n const int mask_batch_idx = batch_idx % mask_ne3;\n mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;\n }\n\n ACC_TYPE4 q_priv[DK_VEC];\n const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;\n const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset);\n #pragma unroll\n for (int i = 0; i < DK_VEC; ++i) {\n q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]);\n }\n\n float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);\n\n const global ACC_TYPE* sinks_ptr = NULL;\n if (sinks_void != NULL) {\n sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);\n }\n\n ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : -INFINITY;\n for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {\n const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;\n const global KV_DATA_TYPE4* k_ptr = (const global KV_DATA_TYPE4*)(k_base + k_row_offset);\n ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);\n #pragma unroll\n for (int k = 0; k < DK_VEC; k++) {\n dot_acc = mad(q_priv[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);\n }\n ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;\n if (mask_base != NULL) {\n const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);\n score += slope * (ACC_TYPE)mask_ptr[k_idx];\n }\n if (logit_softcap > 0.0f) {\n score = logit_softcap * tanh(score / logit_softcap);\n }\n m_i = max(m_i, score);\n }\n\n __local ACC_TYPE local_m[Q1_WG_SIZE];\n local_m[tid] = m_i;\n barrier(CLK_LOCAL_MEM_FENCE);\n #pragma unroll\n for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {\n if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]);\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n const ACC_TYPE m_final = local_m[0];\n\n ACC_TYPE4 o_acc[DV_VEC];\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);\n ACC_TYPE l_i = 0.0f;\n\n for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {\n const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;\n const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;\n const global KV_DATA_TYPE4* k_ptr = (const global KV_DATA_TYPE4*)(k_base + k_row_offset);\n const global KV_DATA_TYPE4* v_ptr = (const global KV_DATA_TYPE4*)(v_base + v_row_offset);\n ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);\n #pragma unroll\n for (int k = 0; k < DK_VEC; k++) {\n dot_acc = mad(q_priv[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);\n }\n ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;\n if (mask_base != NULL) {\n const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);\n score += slope * (ACC_TYPE)mask_ptr[k_idx];\n }\n if (logit_softcap > 0.0f) {\n score = logit_softcap * tanh(score / logit_softcap);\n }\n const ACC_TYPE p = exp(score - m_final);\n l_i += p;\n #pragma unroll\n for (int i = 0; i < DV_VEC; i++) {\n o_acc[i] = mad(p, CONVERT_KV_ACC4(v_ptr[i]), o_acc[i]);\n }\n }\n\n __local ACC_TYPE local_l[Q1_WG_SIZE];\n __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE];\n local_l[tid] = l_i;\n barrier(CLK_LOCAL_MEM_FENCE);\n #pragma unroll\n for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {\n if (tid < s) local_l[tid] += local_l[tid + s];\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;\n global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset);\n ACC_TYPE l_final = local_l[0];\n\n if (sinks_ptr != NULL) {\n l_final += exp(sinks_ptr[head_idx] - m_final);\n }\n\n if (l_final > 0.0f) {\n const ACC_TYPE l_inv = 1.0f / l_final;\n for (int i = 0; i < DV_VEC; i++) {\n local_o_comp[tid] = o_acc[i];\n barrier(CLK_LOCAL_MEM_FENCE);\n #pragma unroll\n for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {\n if (tid < s) local_o_comp[tid] += local_o_comp[tid + s];\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n if (tid == 0) {\n o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv);\n }\n }\n } else if (tid == 0) {\n #pragma unroll\n for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/gelu.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// gelu\n//------------------------------------------------------------------------------\n#define GELU_COEF_A 0.044715f\n#define GELU_QUICK_COEF -1.702f\n#define SQRT_2_OVER_PI 0.79788456080286535587989211986876f\n#define SQRT_2_INV 0.70710678118654752440084436210484f\n\nkernel void kernel_gelu(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n float x = src0[get_global_id(0)];\n\n dst[get_global_id(0)] = 0.5f*x*(1.0f + tanh(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));\n}\n\nkernel void kernel_gelu_4(\n global float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n float4 x = src0[get_global_id(0)];\n\n dst[get_global_id(0)] = 0.5f*x*(1.0f + tanh(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));\n}\n\nkernel void kernel_gelu_erf(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n float x = src0[get_global_id(0)];\n dst[get_global_id(0)] = 0.5f*x*(1.0f + erf(x*SQRT_2_INV));\n}\n\nkernel void kernel_gelu_erf_4(\n global float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n float4 x = src0[get_global_id(0)];\n dst[get_global_id(0)] = 0.5f*x*(1.0f + erf(x*SQRT_2_INV));\n}\n\nkernel void kernel_gelu_quick(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n float x = src0[get_global_id(0)];\n dst[get_global_id(0)] = x*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x)));\n}\n\nkernel void kernel_gelu_quick_4(\n global float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n float4 x = src0[get_global_id(0)];\n dst[get_global_id(0)] = x*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x)));\n}\n" }, { "path": "src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n\n#define QK_MXFP4 32\n#define N_SIMDGROUP 2\n#define SIMDGROUP_WIDTH 64\n\nstatic inline half8 mxfp4_to_fp16_packed8(ushort2 fp4x8) { //, ushort 0x0E00, ushort 0x8000) {\n ushort2 fp16_packed_a_0, fp16_packed_b_0, bias_a, bias_b, sign_a, sign_b;\n fp16_packed_a_0.lo = (fp4x8.s0 << 9) & 0x0E00;\n fp16_packed_a_0.hi = (fp4x8.s0 << 5) & 0x0E00;\n fp16_packed_b_0.lo = (fp4x8.s0 << 1) & 0x0E00;\n fp16_packed_b_0.hi = (fp4x8.s0 >> 3) & 0x0E00;\n\n bias_a.lo = (fp16_packed_a_0.lo != 0) ? 0x3800 : 0x0;\n bias_a.hi = (fp16_packed_a_0.hi != 0) ? 0x3800 : 0x0;\n bias_b.lo = (fp16_packed_b_0.lo != 0) ? 0x3800 : 0x0;\n bias_b.hi = (fp16_packed_b_0.hi != 0) ? 0x3800 : 0x0;\n\n fp16_packed_a_0.lo = (fp16_packed_a_0.lo != 0x0200) ? fp16_packed_a_0.lo : 0x0;\n fp16_packed_a_0.hi = (fp16_packed_a_0.hi != 0x0200) ? fp16_packed_a_0.hi : 0x0;\n fp16_packed_b_0.lo = (fp16_packed_b_0.lo != 0x0200) ? fp16_packed_b_0.lo : 0x0;\n fp16_packed_b_0.hi = (fp16_packed_b_0.hi != 0x0200) ? fp16_packed_b_0.hi : 0x0;\n\n sign_a.lo = (fp4x8.s0 << 12) & 0x8000;\n sign_a.hi = (fp4x8.s0 << 8) & 0x8000;\n sign_b.lo = (fp4x8.s0 << 4) & 0x8000;\n sign_b.hi = fp4x8.s0 & 0x8000;\n\n fp16_packed_a_0 = sign_a + bias_a + fp16_packed_a_0;\n fp16_packed_b_0 = sign_b + bias_b + fp16_packed_b_0;\n\n ushort2 fp16_packed_a_1, fp16_packed_b_1;\n fp16_packed_a_1.lo = (fp4x8.s1 << 9) & 0x0E00;\n fp16_packed_a_1.hi = (fp4x8.s1 << 5) & 0x0E00;\n fp16_packed_b_1.lo = (fp4x8.s1 << 1) & 0x0E00;\n fp16_packed_b_1.hi = (fp4x8.s1 >> 3) & 0x0E00;\n\n bias_a.lo = (fp16_packed_a_1.lo != 0) ? 0x3800 : 0x0;\n bias_a.hi = (fp16_packed_a_1.hi != 0) ? 0x3800 : 0x0;\n bias_b.lo = (fp16_packed_b_1.lo != 0) ? 0x3800 : 0x0;\n bias_b.hi = (fp16_packed_b_1.hi != 0) ? 0x3800 : 0x0;\n\n fp16_packed_a_1.lo = (fp16_packed_a_1.lo != 0x0200) ? fp16_packed_a_1.lo : 0x0;\n fp16_packed_a_1.hi = (fp16_packed_a_1.hi != 0x0200) ? fp16_packed_a_1.hi : 0x0;\n fp16_packed_b_1.lo = (fp16_packed_b_1.lo != 0x0200) ? fp16_packed_b_1.lo : 0x0;\n fp16_packed_b_1.hi = (fp16_packed_b_1.hi != 0x0200) ? fp16_packed_b_1.hi : 0x0;\n\n sign_a.lo = (fp4x8.s1 << 12) & 0x8000;\n sign_a.hi = (fp4x8.s1 << 8) & 0x8000;\n sign_b.lo = (fp4x8.s1 << 4) & 0x8000;\n sign_b.hi = fp4x8.s1 & 0x8000;\n\n fp16_packed_a_1 = sign_a + bias_a + fp16_packed_a_1;\n fp16_packed_b_1 = sign_b + bias_b + fp16_packed_b_1;\n\n return as_half8((ushort8)(fp16_packed_a_0, fp16_packed_b_0, fp16_packed_a_1, fp16_packed_b_1));\n}\n\nstatic inline float e8m0_to_fp32(uchar x) {\n int bits;\n bits = (x == 0) ? 0x00400000 : ((uint) x << 23);\n return as_float(bits);\n}\n\n\n__attribute__((qcom_reqd_sub_group_size(\"half\")))\n__kernel void kernel_gemm_moe_mxfp4_f32(\n __global uint4 * src0_q,\n __global uchar * src0_e,\n __read_only image1d_buffer_t src1,\n __global ushort4 * src2,\n __global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int tile_size\n) {\n uint i01 = get_global_id(0);\n uint i20 = get_global_id(2);\n uint sgid = get_local_id(1);\n uint slid = get_sub_group_local_id();\n\n ushort4 router = src2[i20];\n ushort expert_id = router.x;\n ushort i11 = router.y;\n ushort i1 = router.z;\n ushort tile_id = router.w;\n\n if (tile_id * tile_size + i01 >= ne01) { // handle edge case when ne01 is not multiple of tile_size\n return;\n }\n\n uint expert_offset = expert_id * ne00 * ne01 / 32;\n uint tile_offset = expert_offset + tile_id * tile_size + i01;\n\n __private float sum = 0.0f; // each thread calculate partial sum of one output\n\n // loop along ne00 in block granularity, skip 4 blocks every iter\n for (uint ib00 = sgid; ib00 < (ne00 / QK_MXFP4); ib00 += N_SIMDGROUP) {\n // load one block of q\n uint4 regQ = src0_q[tile_offset + ib00 * ne01];\n // convert 8 fp4 to fp16\n half8 fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s0));\n\n uint offset = i11 * ne00 / 4 + ib00 * 8;\n float4 shared_y4;\n shared_y4 = read_imagef(src1, (offset + 0));\n float4 acc = shared_y4 * (float4)(fp16x8.s0, fp16x8.s2, fp16x8.s4, fp16x8.s6);\n\n shared_y4 = read_imagef(src1, (offset + 4));\n acc += shared_y4 * (float4)(fp16x8.s1, fp16x8.s3, fp16x8.s5, fp16x8.s7);\n\n\n fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s1));\n\n shared_y4 = read_imagef(src1, (offset + 1));\n acc += shared_y4 * (float4)(fp16x8.s0, fp16x8.s2, fp16x8.s4, fp16x8.s6);\n\n shared_y4 = read_imagef(src1, (offset + 5));\n acc += shared_y4 * (float4)(fp16x8.s1, fp16x8.s3, fp16x8.s5, fp16x8.s7);\n\n\n fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s2));\n\n shared_y4 = read_imagef(src1, (offset + 2));\n acc += shared_y4 * (float4)(fp16x8.s0, fp16x8.s2, fp16x8.s4, fp16x8.s6);\n\n shared_y4 = read_imagef(src1, (offset + 6));\n acc += shared_y4 * (float4)(fp16x8.s1, fp16x8.s3, fp16x8.s5, fp16x8.s7);\n\n\n fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s3));\n\n shared_y4 = read_imagef(src1, (offset + 3));\n acc += shared_y4 * (float4)(fp16x8.s0, fp16x8.s2, fp16x8.s4, fp16x8.s6);\n\n shared_y4 = read_imagef(src1, (offset + 7));\n acc += shared_y4 * (float4)(fp16x8.s1, fp16x8.s3, fp16x8.s5, fp16x8.s7);\n\n uchar regE = src0_e[tile_offset + ib00 * ne01];\n sum += e8m0_to_fp32(regE) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3));\n }\n\n // reduction in local memory, assumes #subgroups=4\n __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)];\n if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum;\n // if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum;\n // if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum;\n barrier(CLK_LOCAL_MEM_FENCE);\n if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];\n // if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];\n // if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];\n\n // 1 outputs per thread in subgroup 0\n if (sgid == 0) {\n dst = dst + (offsetd >> 2);\n dst[i01 + tile_id * tile_size + i1 * ne01] = sum;\n }\n\n}\n" }, { "path": "src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n\n#ifdef cl_qcom_reqd_sub_group_size\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_128\n#endif\n\nkernel void kernel_gemm_noshuffle_q4_1_f32(\n global const ushort * src0_q,\n global const half * src0_d,\n global const half * src0_m,\n read_only image1d_buffer_t src1,\n global float * dst,\n ulong offsetd,\n int m,\n int n,\n int k,\n int n_no_padding\n) {\n dst = (global float *)((global char *)dst + offsetd);\n\n int m_4 = m >> 2;\n int n_4 = n >> 2;\n\n int gy = get_global_id(0);\n int gx = get_global_id(1);\n int gx_2 = gx << 2;\n\n half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;\n half8 B;\n half4 dequantized_weights;\n\n global const ushort* weight_ptr = src0_q + gx_2;\n global const half* scale_ptr = src0_d + gx_2;\n global const half* min_ptr = src0_m + gx_2;\n\n for(int i = 0; i < k; i += 4) {\n B.s0123 = read_imageh(src1, gy*2 + (i)*(n_4));\n B.s4567 = read_imageh(src1, gy*2 + (i)*(n_4)+1);\n\n ushort4 bits4 = vload4(0, weight_ptr + (i/4)*(m));\n\n half4 scale = vload4(0, scale_ptr + (i/32)*(m));\n half4 minv = vload4(0, min_ptr + (i/32)*(m));\n\n // j=0\n dequantized_weights.s0 = (bits4.s0 & (0x000F)) * scale.s0 + minv.s0;\n dequantized_weights.s1 = (bits4.s1 & (0x000F)) * scale.s1 + minv.s1;\n dequantized_weights.s2 = (bits4.s2 & (0x000F)) * scale.s2 + minv.s2;\n dequantized_weights.s3 = (bits4.s3 & (0x000F)) * scale.s3 + minv.s3;\n c0 += B * dequantized_weights.s0;\n c1 += B * dequantized_weights.s1;\n c2 += B * dequantized_weights.s2;\n c3 += B * dequantized_weights.s3;\n\n // j=1\n B.s0123 = read_imageh(src1, gy*2 + (i+1)*(n_4));\n B.s4567 = read_imageh(src1, gy*2 + (i+1)*(n_4)+1);\n dequantized_weights.s0 = ((bits4.s0 & (0x00F0)) >> 4) * scale.s0 + minv.s0;\n dequantized_weights.s1 = ((bits4.s1 & (0x00F0)) >> 4) * scale.s1 + minv.s1;\n dequantized_weights.s2 = ((bits4.s2 & (0x00F0)) >> 4) * scale.s2 + minv.s2;\n dequantized_weights.s3 = ((bits4.s3 & (0x00F0)) >> 4) * scale.s3 + minv.s3;\n c0 += B * dequantized_weights.s0;\n c1 += B * dequantized_weights.s1;\n c2 += B * dequantized_weights.s2;\n c3 += B * dequantized_weights.s3;\n\n // j=2\n B.s0123 = read_imageh(src1, gy*2 + (i+2)*(n_4));\n B.s4567 = read_imageh(src1, gy*2 + (i+2)*(n_4)+1);\n dequantized_weights.s0 = ((bits4.s0 & (0x0F00)) >> 8) * scale.s0 + minv.s0;\n dequantized_weights.s1 = ((bits4.s1 & (0x0F00)) >> 8) * scale.s1 + minv.s1;\n dequantized_weights.s2 = ((bits4.s2 & (0x0F00)) >> 8) * scale.s2 + minv.s2;\n dequantized_weights.s3 = ((bits4.s3 & (0x0F00)) >> 8) * scale.s3 + minv.s3;\n c0 += B * dequantized_weights.s0;\n c1 += B * dequantized_weights.s1;\n c2 += B * dequantized_weights.s2;\n c3 += B * dequantized_weights.s3;\n\n // j=3\n B.s0123 = read_imageh(src1, gy*2 + (i+3)*(n_4));\n B.s4567 = read_imageh(src1, gy*2 + (i+3)*(n_4)+1);\n dequantized_weights.s0 = ((bits4.s0 & (0xF000)) >> 12) * scale.s0 + minv.s0;\n dequantized_weights.s1 = ((bits4.s1 & (0xF000)) >> 12) * scale.s1 + minv.s1;\n dequantized_weights.s2 = ((bits4.s2 & (0xF000)) >> 12) * scale.s2 + minv.s2;\n dequantized_weights.s3 = ((bits4.s3 & (0xF000)) >> 12) * scale.s3 + minv.s3;\n c0 += B * dequantized_weights.s0;\n c1 += B * dequantized_weights.s1;\n c2 += B * dequantized_weights.s2;\n c3 += B * dequantized_weights.s3;\n }\n\n int idx = (gy<<3)*m + (gx<<2);\n\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n\n#define QK_MXFP4 32\n#define N_SIMDGROUP 4\n#define SIMDGROUP_WIDTH 64\n\nstatic inline half8 mxfp4_to_fp16_packed8(ushort2 fp4x8) { //, ushort 0x0E00, ushort 0x8000) {\n ushort2 fp16_packed_a_0, fp16_packed_b_0, bias_a, bias_b, sign_a, sign_b;\n fp16_packed_a_0.lo = (fp4x8.s0 << 9) & 0x0E00;\n fp16_packed_a_0.hi = (fp4x8.s0 << 5) & 0x0E00;\n fp16_packed_b_0.lo = (fp4x8.s0 << 1) & 0x0E00;\n fp16_packed_b_0.hi = (fp4x8.s0 >> 3) & 0x0E00;\n\n bias_a.lo = (fp16_packed_a_0.lo != 0) ? 0x3800 : 0x0;\n bias_a.hi = (fp16_packed_a_0.hi != 0) ? 0x3800 : 0x0;\n bias_b.lo = (fp16_packed_b_0.lo != 0) ? 0x3800 : 0x0;\n bias_b.hi = (fp16_packed_b_0.hi != 0) ? 0x3800 : 0x0;\n\n fp16_packed_a_0.lo = (fp16_packed_a_0.lo != 0x0200) ? fp16_packed_a_0.lo : 0x0;\n fp16_packed_a_0.hi = (fp16_packed_a_0.hi != 0x0200) ? fp16_packed_a_0.hi : 0x0;\n fp16_packed_b_0.lo = (fp16_packed_b_0.lo != 0x0200) ? fp16_packed_b_0.lo : 0x0;\n fp16_packed_b_0.hi = (fp16_packed_b_0.hi != 0x0200) ? fp16_packed_b_0.hi : 0x0;\n\n sign_a.lo = (fp4x8.s0 << 12) & 0x8000;\n sign_a.hi = (fp4x8.s0 << 8) & 0x8000;\n sign_b.lo = (fp4x8.s0 << 4) & 0x8000;\n sign_b.hi = fp4x8.s0 & 0x8000;\n\n fp16_packed_a_0 = sign_a + bias_a + fp16_packed_a_0;\n fp16_packed_b_0 = sign_b + bias_b + fp16_packed_b_0;\n\n ushort2 fp16_packed_a_1, fp16_packed_b_1;\n fp16_packed_a_1.lo = (fp4x8.s1 << 9) & 0x0E00;\n fp16_packed_a_1.hi = (fp4x8.s1 << 5) & 0x0E00;\n fp16_packed_b_1.lo = (fp4x8.s1 << 1) & 0x0E00;\n fp16_packed_b_1.hi = (fp4x8.s1 >> 3) & 0x0E00;\n\n bias_a.lo = (fp16_packed_a_1.lo != 0) ? 0x3800 : 0x0;\n bias_a.hi = (fp16_packed_a_1.hi != 0) ? 0x3800 : 0x0;\n bias_b.lo = (fp16_packed_b_1.lo != 0) ? 0x3800 : 0x0;\n bias_b.hi = (fp16_packed_b_1.hi != 0) ? 0x3800 : 0x0;\n\n fp16_packed_a_1.lo = (fp16_packed_a_1.lo != 0x0200) ? fp16_packed_a_1.lo : 0x0;\n fp16_packed_a_1.hi = (fp16_packed_a_1.hi != 0x0200) ? fp16_packed_a_1.hi : 0x0;\n fp16_packed_b_1.lo = (fp16_packed_b_1.lo != 0x0200) ? fp16_packed_b_1.lo : 0x0;\n fp16_packed_b_1.hi = (fp16_packed_b_1.hi != 0x0200) ? fp16_packed_b_1.hi : 0x0;\n\n sign_a.lo = (fp4x8.s1 << 12) & 0x8000;\n sign_a.hi = (fp4x8.s1 << 8) & 0x8000;\n sign_b.lo = (fp4x8.s1 << 4) & 0x8000;\n sign_b.hi = fp4x8.s1 & 0x8000;\n\n fp16_packed_a_1 = sign_a + bias_a + fp16_packed_a_1;\n fp16_packed_b_1 = sign_b + bias_b + fp16_packed_b_1;\n\n return as_half8((ushort8)(fp16_packed_a_0, fp16_packed_b_0, fp16_packed_a_1, fp16_packed_b_1));\n}\n\nstatic inline float e8m0_to_fp32(uchar x) {\n int bits;\n bits = (x == 0) ? 0x00400000 : ((uint) x << 23);\n return as_float(bits);\n}\n\n\n__attribute__((qcom_reqd_sub_group_size(\"half\")))\n__kernel void kernel_gemv_moe_mxfp4_f32(\n __global uint4 * src0_q,\n __global uchar * src0_e,\n __read_only image1d_buffer_t src1,\n __global uint * src2,\n __global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne11\n) {\n uint i01 = get_global_id(0);\n uint i20 = get_global_id(2);\n uint sgid = get_local_id(1);\n uint slid = get_sub_group_local_id();\n\n uint i11 = i20 % ne11;\n\n uint expert_id = src2[i20];\n uint expert_offset = expert_id * ne00 * ne01 / 32;\n\n __private float sum = 0.0f; // each thread calculate partial sum of one output\n\n // loop along ne00 in block granularity, skip 4 blocks every iter\n for (uint ib00 = sgid; ib00 < (ne00 / QK_MXFP4); ib00 += N_SIMDGROUP) {\n\n // load one block of q\n uint4 regQ = src0_q[expert_offset + ib00 * ne01 + i01];\n\n uint offset = i11 * ne00 / 4 + ib00 * 8;\n\n half8 fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s0));\n\n float4 shared_y4;\n shared_y4 = read_imagef(src1, (offset + 0));\n float4 acc = shared_y4 * (float4)(fp16x8.s0, fp16x8.s2, fp16x8.s4, fp16x8.s6);\n\n shared_y4 = read_imagef(src1, (offset + 4));\n acc += shared_y4 * (float4)(fp16x8.s1, fp16x8.s3, fp16x8.s5, fp16x8.s7);\n\n\n fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s1));\n\n shared_y4 = read_imagef(src1, (offset + 1));\n acc += shared_y4 * (float4)(fp16x8.s0, fp16x8.s2, fp16x8.s4, fp16x8.s6);\n\n shared_y4 = read_imagef(src1, (offset + 5));\n acc += shared_y4 * (float4)(fp16x8.s1, fp16x8.s3, fp16x8.s5, fp16x8.s7);\n\n\n fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s2));\n\n shared_y4 = read_imagef(src1, (offset + 2));\n acc += shared_y4 * (float4)(fp16x8.s0, fp16x8.s2, fp16x8.s4, fp16x8.s6);\n\n shared_y4 = read_imagef(src1, (offset + 6));\n acc += shared_y4 * (float4)(fp16x8.s1, fp16x8.s3, fp16x8.s5, fp16x8.s7);\n\n\n fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s3));\n\n shared_y4 = read_imagef(src1, (offset + 3));\n acc += shared_y4 * (float4)(fp16x8.s0, fp16x8.s2, fp16x8.s4, fp16x8.s6);\n\n shared_y4 = read_imagef(src1, (offset + 7));\n acc += shared_y4 * (float4)(fp16x8.s1, fp16x8.s3, fp16x8.s5, fp16x8.s7);\n\n uchar regE = src0_e[ib00 * ne01 + i01 + expert_offset];\n sum += e8m0_to_fp32(regE) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3));\n }\n\n // reduction in local memory, assumes #subgroups=4\n __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)];\n if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum;\n if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum;\n if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum;\n barrier(CLK_LOCAL_MEM_FENCE);\n if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];\n if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];\n if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];\n\n // 1 outputs per thread in subgroup 0\n if (sgid == 0) {\n dst = dst + (offsetd >> 2);\n dst[i01 + i20 * ne01] = sum;\n }\n\n}\n" }, { "path": "src/ggml-opencl/kernels/gemv_noshuffle.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n\n#ifdef cl_qcom_reqd_sub_group_size\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#endif\n\n// assume\n#define QK4_0 32\n#define N_SIMDGROUP 4\n\n#define dequantizeBlockAccum_ns_sgbroadcast_1_hi(total_sums, bits4, scale, y) \\\n float shared_y; \\\n shared_y = sub_group_broadcast(y.s0, 0); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 0); \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 0); \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 0); \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 0); \\\n total_sums.s0 += ((bits4.s2 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 0); \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 0); \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 0); \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s0, 1); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 1); \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 1); \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 1); \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 1); \\\n total_sums.s0 += ((bits4.s6 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 1); \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 1); \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 1); \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n\n\n#define dequantizeBlockAccum_ns_sgbroadcast_1_lo(total_sums, bits4, scale, y) \\\n shared_y = sub_group_broadcast(y.s0, 2); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 2); \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 2); \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 2); \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 2); \\\n total_sums.s0 += ((bits4.s2 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 2); \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 2); \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 2); \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s0, 3); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 3); \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 3); \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 3); \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 3); \\\n total_sums.s0 += ((bits4.s6 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 3); \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 3); \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 3); \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n\n\n#define dequantizeBlockAccum_ns_sgbroadcast_8_hi(total_sums, bits4, scale, y) \\\n float8 shared_y; \\\n shared_y = sub_group_broadcast(y, 0); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) - 8) * scale.s0 * shared_y.s0; \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s1; \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s2; \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s3; \\\n total_sums.s0 += ((bits4.s2 & 0x000F) - 8) * scale.s0 * shared_y.s4; \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s5; \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s6; \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s7; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) - 8) * scale.s1 * shared_y.s0; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s1; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s2; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s3; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) - 8) * scale.s1 * shared_y.s4; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s5; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \\\n shared_y = sub_group_broadcast(y, 1); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) - 8) * scale.s0 * shared_y.s0; \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s1; \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s2; \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s3; \\\n total_sums.s0 += ((bits4.s6 & 0x000F) - 8) * scale.s0 * shared_y.s4; \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s5; \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s6; \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s7; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) - 8) * scale.s1 * shared_y.s0; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s1; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s2; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s3; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) - 8) * scale.s1 * shared_y.s4; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s5; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \\\n\n\n#define dequantizeBlockAccum_ns_sgbroadcast_8_lo(total_sums, bits4, scale, y) \\\n shared_y = sub_group_broadcast(y, 2); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) - 8) * scale.s0 * shared_y.s0; \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s1; \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s2; \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s3; \\\n total_sums.s0 += ((bits4.s2 & 0x000F) - 8) * scale.s0 * shared_y.s4; \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s5; \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s6; \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s7; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) - 8) * scale.s1 * shared_y.s0; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s1; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s2; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s3; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) - 8) * scale.s1 * shared_y.s4; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s5; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \\\n shared_y = sub_group_broadcast(y, 3); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) - 8) * scale.s0 * shared_y.s0; \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s1; \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s2; \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s3; \\\n total_sums.s0 += ((bits4.s6 & 0x000F) - 8) * scale.s0 * shared_y.s4; \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s5; \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s6; \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s7; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) - 8) * scale.s1 * shared_y.s0; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s1; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s2; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s3; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) - 8) * scale.s1 * shared_y.s4; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s5; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \\\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\n__kernel void kernel_gemv_noshuffle(\n __read_only image1d_buffer_t src0_q, // quantized A\n global half2 * src0_d, // A scales\n __read_only image1d_buffer_t src1, // B\n ulong offset1, // offset to B (0)\n global float * dst, // C\n ulong offsetd, // offset to C (0)\n uint K, // K\n int ne01, // M\n int ne02, // 1\n int ne10, // K\n int ne12, // 1\n int ne0, // M\n int ne1, // N\n int r2, // 1\n int r3)\n{\n uint groupId = get_local_id(1);\n uint gid = get_global_id(0);\n ushort slid = get_sub_group_local_id();\n\n __private uint4 regA;\n __private half2 regS;\n __private float8 regB;\n\n __private float2 totalSum = (float2)(0.0f);\n\n // loop along K in block granularity, skip 4 blocks every iter\n for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) {\n regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows\n // first 4 fibers in each wave load 8 B values to its private scope\n if (slid < 4) {\n regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));\n regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));\n }\n\n // load half weights for two blocks in consecutive rows\n regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;\n regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;\n regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;\n regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;\n#ifdef VECTOR_SUB_GROUP_BROADCAT\n dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB);\n#else\n dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB);\n#endif // VECTOR_SUB_GROUP_BROADCAT\n\n regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;\n regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;\n regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;\n regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;\n#ifdef VECTOR_SUB_GROUP_BROADCAT\n dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB);\n#else\n dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regB);\n#endif // VECTOR_SUB_GROUP_BROADCAT\n }\n\n // reduction in local memory, assumes #wave=4\n __local float2 reduceLM[SIMDGROUP_WIDTH * 3];\n if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum;\n if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum;\n if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum;\n barrier(CLK_LOCAL_MEM_FENCE);\n if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];\n if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];\n if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];\n\n // 2 outputs per fiber in wave 0\n if (groupId == 0) {\n dst = (global float*)((global char*)dst + offsetd);\n vstore2(totalSum, 0, &(dst[gid * 2]));\n }\n\n}\n" }, { "path": "src/ggml-opencl/kernels/gemv_noshuffle_general.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n\n#ifdef cl_qcom_reqd_sub_group_size\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#endif\n\n// assume\n#define QK4_0 32\n#define N_SIMDGROUP 4\n\n#define dequantizeBlockAccum_ns_sgbroadcast_1_hi(total_sums, bits4, scale, y) \\\n float shared_y; \\\n shared_y = sub_group_broadcast(y.s0, 0); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 0); \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 0); \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 0); \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 0); \\\n total_sums.s0 += ((bits4.s2 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 0); \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 0); \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 0); \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s0, 1); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 1); \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 1); \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 1); \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 1); \\\n total_sums.s0 += ((bits4.s6 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 1); \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 1); \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 1); \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n\n\n#define dequantizeBlockAccum_ns_sgbroadcast_1_lo(total_sums, bits4, scale, y) \\\n shared_y = sub_group_broadcast(y.s0, 2); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 2); \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 2); \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 2); \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 2); \\\n total_sums.s0 += ((bits4.s2 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 2); \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 2); \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 2); \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s0, 3); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 3); \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 3); \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 3); \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 3); \\\n total_sums.s0 += ((bits4.s6 & 0x000F) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 3); \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 3); \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 3); \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) - 8) * scale.s0 * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y; \\\n\n\n#define dequantizeBlockAccum_ns_sgbroadcast_8_hi(total_sums, bits4, scale, y) \\\n float8 shared_y; \\\n shared_y = sub_group_broadcast(y, 0); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) - 8) * scale.s0 * shared_y.s0; \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s1; \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s2; \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s3; \\\n total_sums.s0 += ((bits4.s2 & 0x000F) - 8) * scale.s0 * shared_y.s4; \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s5; \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s6; \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s7; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) - 8) * scale.s1 * shared_y.s0; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s1; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s2; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s3; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) - 8) * scale.s1 * shared_y.s4; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s5; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \\\n shared_y = sub_group_broadcast(y, 1); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) - 8) * scale.s0 * shared_y.s0; \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s1; \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s2; \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s3; \\\n total_sums.s0 += ((bits4.s6 & 0x000F) - 8) * scale.s0 * shared_y.s4; \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s5; \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s6; \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s7; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) - 8) * scale.s1 * shared_y.s0; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s1; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s2; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s3; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) - 8) * scale.s1 * shared_y.s4; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s5; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \\\n\n\n#define dequantizeBlockAccum_ns_sgbroadcast_8_lo(total_sums, bits4, scale, y) \\\n shared_y = sub_group_broadcast(y, 2); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) - 8) * scale.s0 * shared_y.s0; \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s1; \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s2; \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s3; \\\n total_sums.s0 += ((bits4.s2 & 0x000F) - 8) * scale.s0 * shared_y.s4; \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s5; \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s6; \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s7; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) - 8) * scale.s1 * shared_y.s0; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s1; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s2; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s3; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) - 8) * scale.s1 * shared_y.s4; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s5; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \\\n shared_y = sub_group_broadcast(y, 3); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) - 8) * scale.s0 * shared_y.s0; \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s1; \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s2; \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s3; \\\n total_sums.s0 += ((bits4.s6 & 0x000F) - 8) * scale.s0 * shared_y.s4; \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale.s0 * shared_y.s5; \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale.s0 * shared_y.s6; \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) - 8) * scale.s0 * shared_y.s7; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) - 8) * scale.s1 * shared_y.s0; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s1; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s2; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s3; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) - 8) * scale.s1 * shared_y.s4; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale.s1 * shared_y.s5; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale.s1 * shared_y.s6; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) - 8) * scale.s1 * shared_y.s7; \\\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\n__kernel void kernel_gemv_noshuffle(\n __read_only image1d_buffer_t src0_q, // quantized A\n global half2 * src0_d, // A scales\n __read_only image1d_buffer_t src1, // B\n ulong offset1, // offset to B (0)\n global float * dst, // C\n ulong offsetd, // offset to C (0)\n int ne00, // K\n int ne01, // M\n int ne02, // 1\n int ne10, // K\n int ne12, // 1\n int ne0, // M\n int ne1, // N\n int r2, // 1\n int r3)\n{\n uint groupId = get_local_id(1);\n uint gid = get_global_id(0);\n ushort slid = get_sub_group_local_id();\n\n uint K = ne00;\n uint M = ne01;\n\n uint LINE_STRIDE_A = M / 2;\n uint BLOCK_STRIDE_A = N_SIMDGROUP * M;\n\n __private uint4 regA;\n __private half2 regS;\n __private float8 regB;\n\n __private float2 totalSum = (float2)(0.0f);\n\n // loop along K in block granularity, skip 4 blocks every iter\n for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) {\n regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows\n // first 4 fibers in each wave load 8 B values to its private scope\n if (slid < 4) {\n regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));\n regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));\n }\n\n // load half weights for two blocks in consecutive rows\n regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;\n regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;\n regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;\n regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;\n#ifdef VECTOR_SUB_GROUP_BROADCAT\n dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB);\n#else\n dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB);\n#endif // VECTOR_SUB_GROUP_BROADCAT\n\n regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;\n regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;\n regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;\n regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;\n#ifdef VECTOR_SUB_GROUP_BROADCAT\n dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB);\n#else\n dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regB);\n#endif // VECTOR_SUB_GROUP_BROADCAT\n }\n\n // reduction in local memory, assumes #wave=4\n __local float2 reduceLM[SIMDGROUP_WIDTH * 3];\n if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum;\n if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum;\n if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum;\n barrier(CLK_LOCAL_MEM_FENCE);\n if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];\n if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];\n if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];\n\n // 2 outputs per fiber in wave 0\n if (groupId == 0) {\n dst = (global float*)((global char*)dst + offsetd);\n vstore2(totalSum, 0, &(dst[gid * 2]));\n }\n\n}\n" }, { "path": "src/ggml-opencl/kernels/gemv_noshuffle_general_q8_0_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n\n#ifdef cl_qcom_reqd_sub_group_size\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#endif\n\n#define QK8_0 32\n#define N_SIMDGROUP 4\n\n#define dequantizeBlockAccum_ns_sgbroadcast_1(total_sums, bits8, scale, y) \\\n float shared_y; \\\n char elem; \\\n \\\n shared_y = sub_group_broadcast(y.s0, 0); \\\n elem = (char)(bits8.s0 & 0x000000FF); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 0); \\\n elem = (char)((bits8.s0 & 0x0000FF00) >> 8); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 0); \\\n elem = (char)((bits8.s0 & 0x00FF0000) >> 16); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 0); \\\n elem = (char)((bits8.s0 & 0xFF000000) >> 24); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n \\\n shared_y = sub_group_broadcast(y.s4, 0); \\\n elem = (char)(bits8.s1 & 0x000000FF); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 0); \\\n elem = (char)((bits8.s1 & 0x0000FF00) >> 8); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 0); \\\n elem = (char)((bits8.s1 & 0x00FF0000) >> 16); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 0); \\\n elem = (char)((bits8.s1 & 0xFF000000) >> 24); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n \\\n shared_y = sub_group_broadcast(y.s0, 1); \\\n elem = (char)(bits8.s2 & 0x000000FF); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 1); \\\n elem = (char)((bits8.s2 & 0x0000FF00) >> 8); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 1); \\\n elem = (char)((bits8.s2 & 0x00FF0000) >> 16); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 1); \\\n elem = (char)((bits8.s2 & 0xFF000000) >> 24); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n \\\n shared_y = sub_group_broadcast(y.s4, 1); \\\n elem = (char)(bits8.s3 & 0x000000FF); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 1); \\\n elem = (char)((bits8.s3 & 0x0000FF00) >> 8); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 1); \\\n elem = (char)((bits8.s3 & 0x00FF0000) >> 16); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 1); \\\n elem = (char)((bits8.s3 & 0xFF000000) >> 24); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n \\\n shared_y = sub_group_broadcast(y.s0, 2); \\\n elem = (char)(bits8.s4 & 0x000000FF); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 2); \\\n elem = (char)((bits8.s4 & 0x0000FF00) >> 8); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 2); \\\n elem = (char)((bits8.s4 & 0x00FF0000) >> 16); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 2); \\\n elem = (char)((bits8.s4 & 0xFF000000) >> 24); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n \\\n shared_y = sub_group_broadcast(y.s4, 2); \\\n elem = (char)(bits8.s5 & 0x000000FF); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 2); \\\n elem = (char)((bits8.s5 & 0x0000FF00) >> 8); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 2); \\\n elem = (char)((bits8.s5 & 0x00FF0000) >> 16); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 2); \\\n elem = (char)((bits8.s5 & 0xFF000000) >> 24); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n \\\n shared_y = sub_group_broadcast(y.s0, 3); \\\n elem = (char)(bits8.s6 & 0x000000FF); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 3); \\\n elem = (char)((bits8.s6 & 0x0000FF00) >> 8); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 3); \\\n elem = (char)((bits8.s6 & 0x00FF0000) >> 16); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 3); \\\n elem = (char)((bits8.s6 & 0xFF000000) >> 24); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n \\\n shared_y = sub_group_broadcast(y.s4, 3); \\\n elem = (char)(bits8.s7 & 0x000000FF); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 3); \\\n elem = (char)((bits8.s7 & 0x0000FF00) >> 8); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 3); \\\n elem = (char)((bits8.s7 & 0x00FF0000) >> 16); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 3); \\\n elem = (char)((bits8.s7 & 0xFF000000) >> 24); \\\n total_sums += convert_int(elem) * scale * shared_y; \\\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\n__kernel void kernel_gemv_noshuffle_q8_0_f32(\n __read_only image1d_buffer_t src0_q, // quantized A\n global half * src0_d, // A scales\n __read_only image1d_buffer_t src1, // B\n ulong offset1, // offset to B (0)\n global float * dst, // C\n ulong offsetd, // offset to C\n int ne00, // K\n int ne01, // M\n int ne02, // 1\n int ne10, // K\n int ne12, // 1\n int ne0, // M\n int ne1, // N\n int r2, // 1\n int r3)\n{\n uint groupId = get_local_id(1);\n uint gid = get_global_id(0);\n ushort slid = get_sub_group_local_id();\n\n uint K = ne00;\n uint M = ne01;\n\n uint LINE_STRIDE_A = M;\n uint BLOCK_STRIDE_A = 8 * M; // 32 / 4 = 8\n\n __private uint8 regA;\n __private half regS;\n __private float8 regB;\n\n __private float totalSum = (float)(0.0f);\n\n // loop along K in block granularity, skip 4 blocks every iter\n #pragma unroll 1 /* tell compiler not to unroll */\n for (uint k = groupId; k < (K / QK8_0); k += N_SIMDGROUP) {\n regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of one rows\n // first 4 fibers in each wave load 8 B values to its private scope\n if (slid < 4) {\n regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));\n regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));\n }\n\n // load weights for one block in consecutive rows\n regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;\n regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;\n regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;\n regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;\n regA.s4 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;\n regA.s5 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;\n regA.s6 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;\n regA.s7 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;\n\n dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, regS, regB);\n }\n\n // reduction in local memory, assumes #wave=4\n __local float reduceLM[SIMDGROUP_WIDTH * 3];\n if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum;\n if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum;\n if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum;\n barrier(CLK_LOCAL_MEM_FENCE);\n if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];\n if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];\n if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];\n\n // 1 outputs per fiber in wave 0\n if (groupId == 0) {\n dst = (global float*)((global char*)dst + offsetd);\n dst[gid] = totalSum;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n\n#ifdef cl_qcom_reqd_sub_group_size\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#endif\n\n#define QK4_0 32\n#define NSUBGROUPS 4\n#define SUBGROUP_SIZE 64\n\n#define dequantizeBlockAccum_ns_sgbroadcast_1_hi(total_sums, bits4, scale, minv, y) \\\n float shared_y; \\\n shared_y = sub_group_broadcast(y.s0, 0); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 0); \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 0); \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 0); \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 0); \\\n total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 0); \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 0); \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 0); \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s0, 1); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 1); \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 1); \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 1); \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 1); \\\n total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 1); \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 1); \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 1); \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y; \\\n\n\n#define dequantizeBlockAccum_ns_sgbroadcast_1_lo(total_sums, bits4, scale, minv, y) \\\n shared_y = sub_group_broadcast(y.s0, 2); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 2); \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 2); \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 2); \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 2); \\\n total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 2); \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 2); \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 2); \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s0, 3); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s1, 3); \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s2, 3); \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s3, 3); \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s4, 3); \\\n total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s5, 3); \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s6, 3); \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y; \\\n shared_y = sub_group_broadcast(y.s7, 3); \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y; \\\n\n\n#define dequantizeBlockAccum_ns_sgbroadcast_8_hi(total_sums, bits4, scale, minv, y) \\\n float8 shared_y; \\\n shared_y = sub_group_broadcast(y, 0); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 + minv.s0) * shared_y.s0; \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y.s1; \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y.s2; \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y.s3; \\\n total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 + minv.s0) * shared_y.s4; \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y.s5; \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y.s6; \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y.s7; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 + minv.s1) * shared_y.s0; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y.s1; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y.s2; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y.s3; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 + minv.s1) * shared_y.s4; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y.s5; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y.s6; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y.s7; \\\n shared_y = sub_group_broadcast(y, 1); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 + minv.s0) * shared_y.s0; \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y.s1; \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y.s2; \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y.s3; \\\n total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 + minv.s0) * shared_y.s4; \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y.s5; \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y.s6; \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y.s7; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 + minv.s1) * shared_y.s0; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y.s1; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y.s2; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y.s3; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 + minv.s1) * shared_y.s4; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y.s5; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y.s6; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y.s7; \\\n\n\n#define dequantizeBlockAccum_ns_sgbroadcast_8_lo(total_sums, bits4, scale, minv, y) \\\n shared_y = sub_group_broadcast(y, 2); \\\n total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 + minv.s0) * shared_y.s0; \\\n total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y.s1; \\\n total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y.s2; \\\n total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y.s3; \\\n total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 + minv.s0) * shared_y.s4; \\\n total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y.s5; \\\n total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y.s6; \\\n total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y.s7; \\\n total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 + minv.s1) * shared_y.s0; \\\n total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y.s1; \\\n total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y.s2; \\\n total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y.s3; \\\n total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 + minv.s1) * shared_y.s4; \\\n total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y.s5; \\\n total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y.s6; \\\n total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y.s7; \\\n shared_y = sub_group_broadcast(y, 3); \\\n total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 + minv.s0) * shared_y.s0; \\\n total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y.s1; \\\n total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y.s2; \\\n total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y.s3; \\\n total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 + minv.s0) * shared_y.s4; \\\n total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 + minv.s0) * shared_y.s5; \\\n total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 + minv.s0) * shared_y.s6; \\\n total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 + minv.s0) * shared_y.s7; \\\n total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 + minv.s1) * shared_y.s0; \\\n total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y.s1; \\\n total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y.s2; \\\n total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y.s3; \\\n total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 + minv.s1) * shared_y.s4; \\\n total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 + minv.s1) * shared_y.s5; \\\n total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 + minv.s1) * shared_y.s6; \\\n total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 + minv.s1) * shared_y.s7; \\\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_gemv_noshuffle_q4_1_f32(\n read_only image1d_buffer_t src0_q,\n global half2 * src0_d,\n global half2 * src0_m,\n read_only image1d_buffer_t src1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01)\n{\n uint groupId = get_local_id(1);\n uint gid = get_global_id(0);\n ushort slid = get_sub_group_local_id();\n\n uint K = ne00;\n uint M = ne01;\n\n uint LINE_STRIDE_A = M / 2;\n uint BLOCK_STRIDE_A = NSUBGROUPS * M;\n\n private uint4 regA;\n private half2 regS;\n private half2 regM;\n private float8 regB;\n\n private float2 totalSum = (float2)(0.0f);\n\n // loop along K in block granularity, skip 4 blocks every iter\n for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) {\n regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows\n regM = src0_m[gid + k * LINE_STRIDE_A]; // each fiber loads min of two rows\n // first 4 fibers in each wave load 8 B values to its private scope\n if (slid < 4) {\n regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));\n regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));\n }\n\n // load half weights for two blocks in consecutive rows\n regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;\n regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;\n regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;\n regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;\n#ifdef VECTOR_SUB_GROUP_BROADCAT\n dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB);\n#else\n dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB);\n#endif // VECTOR_SUB_GROUP_BROADCAT\n\n regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;\n regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;\n regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;\n regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;\n#ifdef VECTOR_SUB_GROUP_BROADCAT\n dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB);\n#else\n dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regM, regB);\n#endif // VECTOR_SUB_GROUP_BROADCAT\n }\n\n // reduction in local memory, assumes #wave=4\n local float2 reduceLM[SUBGROUP_SIZE * 3];\n if (groupId == 1) {\n reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum;\n }\n if (groupId == 2) {\n reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum;\n }\n if (groupId == 3) {\n reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if (groupId == 0) {\n totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid];\n }\n if (groupId == 0) {\n totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid];\n }\n if (groupId == 0) {\n totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid];\n }\n\n // 2 outputs per fiber in wave 0\n if (groupId == 0) {\n dst = (global float*)((global char*)dst + offsetd);\n vstore2(totalSum, 0, &(dst[gid * 2]));\n }\n\n}\n" }, { "path": "src/ggml-opencl/kernels/get_rows.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\ntypedef char int8_t;\ntypedef uchar uint8_t;\ntypedef short int16_t;\ntypedef ushort uint16_t;\ntypedef int int32_t;\ntypedef uint uint32_t;\n\n#define QK4_0 32\n\n//------------------------------------------------------------------------------\n// block_q4_0\n//------------------------------------------------------------------------------\nstruct block_q4_0\n{\n half d;\n uint8_t qs[QK4_0 / 2];\n};\n\n\n//------------------------------------------------------------------------------\n// dequantize_q4_0_f32, dequantize_q4_0_f16\n//------------------------------------------------------------------------------\nvoid dequantize_q4_0_f32(global struct block_q4_0 * xb, short il, float16 * reg) {\n global ushort * qs = ((global ushort *)xb + 1);\n float d1 = il ? (xb->d / 16.h) : xb->d;\n float d2 = d1 / 256.f;\n float md = -8.h * xb->d;\n ushort mask0 = il ? 0x00F0 : 0x000F;\n ushort mask1 = mask0 << 8;\n\n reg->s0 = d1 * (qs[0] & mask0) + md;\n reg->s1 = d2 * (qs[0] & mask1) + md;\n\n reg->s2 = d1 * (qs[1] & mask0) + md;\n reg->s3 = d2 * (qs[1] & mask1) + md;\n\n reg->s4 = d1 * (qs[2] & mask0) + md;\n reg->s5 = d2 * (qs[2] & mask1) + md;\n\n reg->s6 = d1 * (qs[3] & mask0) + md;\n reg->s7 = d2 * (qs[3] & mask1) + md;\n\n reg->s8 = d1 * (qs[4] & mask0) + md;\n reg->s9 = d2 * (qs[4] & mask1) + md;\n\n reg->sa = d1 * (qs[5] & mask0) + md;\n reg->sb = d2 * (qs[5] & mask1) + md;\n\n reg->sc = d1 * (qs[6] & mask0) + md;\n reg->sd = d2 * (qs[6] & mask1) + md;\n\n reg->se = d1 * (qs[7] & mask0) + md;\n reg->sf = d2 * (qs[7] & mask1) + md;\n}\n\n\n//------------------------------------------------------------------------------\n// get_rows\n//------------------------------------------------------------------------------\nkernel void kernel_get_rows_f32(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i10 = get_group_id(0);\n int i11 = get_group_id(1);\n int i12 = get_group_id(2);\n\n int r = ((global int *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0];\n\n int i02 = i11;\n int i03 = i12;\n\n for (int ind = get_local_id(0); ind < ne00; ind += get_local_size(0)) {\n if (ind >= ne00) {\n return;\n }\n ((global float *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1))[ind] =\n ((global float *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03))[ind];\n }\n}\n\nkernel void kernel_get_rows_f16(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i10 = get_group_id(0);\n int i11 = get_group_id(1);\n int i12 = get_group_id(2);\n\n int r = ((global int32_t *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0];\n\n int i02 = i11;\n int i03 = i12;\n\n for (int ind = get_local_id(0); ind < ne00; ind += get_local_size(0)) {\n if (ind >= ne00) {\n return;\n }\n ((global float *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1))[ind] =\n ((global half *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03))[ind];\n }\n}\n\nkernel void kernel_get_rows_q4_0(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n const int NL = 2;\n\n int i10 = get_group_id(0);\n int i11 = get_group_id(1);\n int i12 = get_group_id(2);\n\n int r = ((global int32_t *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0];\n\n int i02 = i11;\n int i03 = i12;\n\n for (int ind = get_local_id(0); ind < ne00/16; ind += get_local_size(0)) {\n float16 temp;\n if (ind >= ne00) {\n return;\n }\n dequantize_q4_0_f32(\n ((global struct block_q4_0 *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03)) + ind/NL, ind%NL, &temp);\n *(((global float16 *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1)) + ind) = temp;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/glu.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define GELU_COEF_A 0.044715f\n#define GELU_QUICK_COEF -1.702f\n#define SQRT_2_OVER_PI 0.79788456080286535587989211986876f\n#define SQRT_2_INV 0.70710678118654752440084436210484f\n\n//------------------------------------------------------------------------------\n// geglu\n//------------------------------------------------------------------------------\nkernel void kernel_geglu(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global float * src0_row = (global float *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global float * src1_row = (global float *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global float * dst_row = (global float *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n const float gelu = 0.5f*x0*(1.0f + tanh(SQRT_2_OVER_PI*x0*(1.0f + GELU_COEF_A*x0*x0)));\n\n dst_row[i0] = gelu*x1;\n }\n}\n\nkernel void kernel_geglu_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global half * src0_row = (global half *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global half * src1_row = (global half *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global half * dst_row = (global half *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const half x0 = src0_row[i0];\n const half x1 = src1_row[i0];\n\n const half gelu = 0.5f*x0*(1.0f + tanh(SQRT_2_OVER_PI*x0*(1.0f + GELU_COEF_A*x0*x0)));\n\n dst_row[i0] = gelu*x1;\n }\n}\n\n//------------------------------------------------------------------------------\n// reglu\n//------------------------------------------------------------------------------\nkernel void kernel_reglu(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global float * src0_row = (global float *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global float * src1_row = (global float *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global float * dst_row = (global float *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n dst_row[i0] = x0*x1*(x0 > 0.0f);\n }\n}\n\nkernel void kernel_reglu_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global half * src0_row = (global half *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global half * src1_row = (global half *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global half * dst_row = (global half *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const half x0 = src0_row[i0];\n const half x1 = src1_row[i0];\n\n dst_row[i0] = x0*x1*(x0 > 0.0f);\n }\n}\n\n//------------------------------------------------------------------------------\n// swiglu\n//------------------------------------------------------------------------------\nkernel void kernel_swiglu(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global float * src0_row = (global float *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global float * src1_row = (global float *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global float * dst_row = (global float *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n const float silu = x0 / (1.0f + exp(-x0));\n\n dst_row[i0] = silu*x1;\n }\n}\n\nkernel void kernel_swiglu_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global half * src0_row = (global half *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global half * src1_row = (global half *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global half * dst_row = (global half *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const half x0 = src0_row[i0];\n const half x1 = src1_row[i0];\n\n const half silu = x0 / (1.0f + exp(-x0));\n\n dst_row[i0] = silu*x1;\n }\n}\n\n//------------------------------------------------------------------------------\n// swiglu_oai\n//------------------------------------------------------------------------------\nkernel void kernel_swiglu_oai(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off,\n float limit,\n float alpha\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global float * src0_row = (global float *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global float * src1_row = (global float *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global float * dst_row = (global float *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n float x0 = src0_row[i0];\n float x1 = src1_row[i0];\n\n x0 = min(x0, limit);\n x1 = max(min(x1, limit), -limit);\n\n float out_glu = x0 / (1.0f + exp(-x0 * alpha));\n out_glu = out_glu * (1.0f + x1);\n\n dst_row[i0] = out_glu;\n }\n}\n\n//------------------------------------------------------------------------------\n// geglu_erf\n//------------------------------------------------------------------------------\nkernel void kernel_geglu_erf(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global float * src0_row = (global float *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global float * src1_row = (global float *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global float * dst_row = (global float *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n const float gelu_erf = 0.5f*x0*(1.0f + erf(x0*SQRT_2_INV));\n\n dst_row[i0] = gelu_erf*x1;\n }\n}\n\nkernel void kernel_geglu_erf_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global half * src0_row = (global half *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global half * src1_row = (global half *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global half * dst_row = (global half *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const half x0 = src0_row[i0];\n const half x1 = src1_row[i0];\n\n const half gelu_erf = 0.5f*x0*(1.0f + erf(x0*SQRT_2_INV));\n\n dst_row[i0] = gelu_erf*x1;\n }\n}\n\n//------------------------------------------------------------------------------\n// geglu_quick\n//------------------------------------------------------------------------------\nkernel void kernel_geglu_quick(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global float * src0_row = (global float *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global float * src1_row = (global float *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global float * dst_row = (global float *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const float x0 = src0_row[i0];\n const float x1 = src1_row[i0];\n\n const float gelu_quick = x0*(1.0f/(1.0f + exp(GELU_QUICK_COEF*x0)));\n\n dst_row[i0] = gelu_quick*x1;\n }\n}\n\nkernel void kernel_geglu_quick_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb01,\n ulong nb11,\n int ne0,\n ulong nb1,\n int ne00_off,\n int ne10_off\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n global half * src0_row = (global half *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off;\n global half * src1_row = (global half *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off;\n global half * dst_row = (global half *) ((global char *) dst + get_group_id(0)*nb1);\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const half x0 = src0_row[i0];\n const half x1 = src1_row[i0];\n\n const half gelu_quick = x0*(1.0f/(1.0f + exp(GELU_QUICK_COEF*x0)));\n\n dst_row[i0] = gelu_quick*x1;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/group_norm.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n// Workgroup must be a subgroup\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_32\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_group_norm(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int ne,\n int group_size,\n float eps\n) {\n src0 = (global float *)((global char *)src0 + offset0);\n dst = (global float *)((global char *)dst + offsetd);\n\n int start = get_group_id(0) * group_size;\n int end = start + group_size;\n\n start += get_local_id(0);\n\n if (end >= ne) {\n end = ne;\n }\n\n float tmp = 0.0f;\n\n for (int j = start; j < end; j += get_local_size(0)) {\n tmp += src0[j];\n }\n\n tmp = sub_group_reduce_add(tmp);\n\n const float mean = tmp / group_size;\n tmp = 0.0f;\n\n for (int j = start; j < end; j += get_local_size(0)) {\n float xi = src0[j] - mean;\n dst[j] = xi;\n tmp += xi * xi;\n }\n\n tmp = sub_group_reduce_add(tmp);\n\n const float variance = tmp / group_size;\n const float scale = 1.0f/sqrt(variance + eps);\n for (int j = start; j < end; j += get_local_size(0)) {\n dst[j] *= scale;\n }\n}\n\n//------------------------------------------------------------------------------\n// group_norm_mul_add\n//------------------------------------------------------------------------------\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_32\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_group_norm_mul_add(\n global float * src0, ulong offset0,\n global float * src1, ulong offset1,\n global float * src2, ulong offset2,\n global float * dst, ulong offsetd,\n int ne,\n int group_size,\n float eps\n) {\n src0 = (global float *)((global char *)src0 + offset0);\n src1 = (global float *)((global char *)src1 + offset1);\n src2 = (global float *)((global char *)src2 + offset2);\n dst = (global float *)((global char *)dst + offsetd);\n\n int start = get_group_id(0) * group_size;\n int end = start + group_size;\n if (end > ne) {\n end = ne;\n }\n\n float sum = 0.0f;\n float sum_sq = 0.0f;\n\n for (int j = start + get_local_id(0); j < end; j += get_local_size(0)) {\n float val = src0[j];\n sum += val;\n sum_sq += val*val;\n }\n\n sum = sub_group_reduce_add(sum);\n sum_sq = sub_group_reduce_add(sum_sq);\n\n const float mean = sum / group_size;\n const float var = sum_sq / group_size - mean * mean;\n const float scale = rsqrt(var + eps);\n\n for (int j = start + get_local_id(0); j < end; j += get_local_size(0)) {\n dst[j] = ((src0[j] - mean) * scale) * src1[j] + src2[j];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/im2col_f16.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\nkernel void kernel_im2col_f16(\n global float * src1,\n ulong offset1,\n global half * dst,\n ulong offsetd,\n ulong batch_offset,\n ulong delta_offset,\n long IW,\n long IH,\n long IC,\n long OW,\n long OH,\n long KW,\n long KH,\n long pelements,\n long CHW,\n int s0,\n int s1,\n int p0,\n int p1,\n int d0,\n int d1\n) {\n long i = get_global_id(0);\n if (i >= pelements) {\n return;\n }\n\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global half*)((global char*)dst + offsetd);\n\n long ksize = OW * KH;\n long kx = i / ksize;\n long kd = kx * ksize;\n long ky = (i - kd) / OW;\n long ix = i % OW;\n\n long oh = get_group_id(1);\n long batch = get_group_id(2) / IC;\n long ic = get_group_id(2) % IC;\n\n long iiw = ix * s0 + kx * d0 - p0;\n long iih = oh * s1 + ky * d1 - p1;\n\n long offset_dst =\n ((batch * OH + oh) * OW + ix) * CHW +\n (ic * (KW * KH) + ky * KW + kx);\n\n if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {\n dst[offset_dst] = 0.0f;\n } else {\n long offset_src = ic * delta_offset + batch * batch_offset;\n dst[offset_dst] = src1[offset_src + iih * IW + iiw];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/im2col_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\nkernel void kernel_im2col_f32(\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n ulong batch_offset,\n ulong delta_offset,\n long IW,\n long IH,\n long IC,\n long OW,\n long OH,\n long KW,\n long KH,\n long pelements,\n long CHW,\n int s0,\n int s1,\n int p0,\n int p1,\n int d0,\n int d1\n) {\n long i = get_global_id(0);\n if (i >= pelements) {\n return;\n }\n\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n long ksize = OW * KH;\n long kx = i / ksize;\n long kd = kx * ksize;\n long ky = (i - kd) / OW;\n long ix = i % OW;\n\n long oh = get_group_id(1);\n long batch = get_group_id(2) / IC;\n long ic = get_group_id(2) % IC;\n\n long iiw = ix * s0 + kx * d0 - p0;\n long iih = oh * s1 + ky * d1 - p1;\n\n long offset_dst =\n ((batch * OH + oh) * OW + ix) * CHW +\n (ic * (KW * KH) + ky * KW + kx);\n\n if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {\n dst[offset_dst] = 0.0f;\n } else {\n long offset_src = ic * delta_offset + batch * batch_offset;\n dst[offset_dst] = src1[offset_src + iih * IW + iiw];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/l2_norm.cl", "content": "#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_32\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_l2_norm_f32(\n global void * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n float eps,\n local float * sum\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n global float * x = (global float *) ((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01);\n global float * y = (global float *) (dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);\n\n float sumf = 0;\n\n // parallel sum\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n sumf += x[i00] * x[i00];\n }\n sumf = sub_group_reduce_add(sumf);\n\n if (get_sub_group_local_id() == 0) {\n sum[get_sub_group_id()] = sumf;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n // broadcast\n for (uint i = get_local_size(0) / get_max_sub_group_size() / 2; i > 0; i /= 2) {\n if (get_local_id(0) < i) {\n sum[get_local_id(0)] += sum[get_local_id(0) + i];\n }\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n const float scale = 1.0f/max(sqrt(sum[0]), eps);\n\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n y[i00] = x[i00] * scale;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mean.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n\n// Most devices have max workgroup size of 1024, so this is enough for subgroup\n// sizes of 16, 32, 64 and 128. Increase this value for smaller subgroups sizes\n#define MAX_SUBGROUPS 64\nkernel void kernel_mean_f32(\n global char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n const int lid = get_local_id(0);\n const int lsize = get_local_size(0);\n\n const uint sg_size = get_sub_group_size();\n const uint sg_id = get_sub_group_id();\n const uint sg_lid = get_sub_group_local_id();\n\n __local float lmem[MAX_SUBGROUPS];\n\n if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {\n return;\n }\n\n if(sg_id == 0){\n lmem[sg_lid] = 0.0f;\n }\n\n global float * src_row = (global float *) (src0 + i1*nb01 + i2*nb02 + i3*nb03);\n global float * dst_row = (global float *) (dst + i1*nb1 + i2*nb2 + i3*nb3);\n\n float sumf = 0.0f;\n\n for (int i0 = lid; i0 < ne00; i0 += lsize) {\n sumf += src_row[i0];\n }\n\n sumf = sub_group_reduce_add(sumf);\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if(sg_lid == 0){\n lmem[sg_id] = sumf;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n sumf = lmem[sg_lid];\n sumf = sub_group_reduce_add(sumf);\n\n if (lid == 0) {\n dst_row[0] = sumf / ne00;\n }\n}\n\nkernel void kernel_mean_f32_4(\n global char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n const int lid = get_local_id(0);\n const int lsize = get_local_size(0);\n\n const uint sg_size = get_sub_group_size();\n const uint sg_id = get_sub_group_id();\n const uint sg_lid = get_sub_group_local_id();\n\n __local float lmem[MAX_SUBGROUPS];\n\n if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {\n return;\n }\n\n if(sg_id == 0){\n lmem[sg_lid] = 0.0f;\n }\n\n global float4 * src_row = (global float4 *) (src0 + i1*nb01 + i2*nb02 + i3*nb03);\n global float * dst_row = (global float *) (dst + i1*nb1 + i2*nb2 + i3*nb3);\n\n float4 sum_vec = (float4)0.0f;\n\n for (int i0 = lid; i0 < ne00 / 4; i0 += lsize) {\n sum_vec += src_row[i0];\n }\n\n float sumf = dot(sum_vec, (float4)(1.0f));\n sumf = sub_group_reduce_add(sumf);\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if(sg_lid == 0){\n lmem[sg_id] = sumf;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n sumf = lmem[sg_lid];\n sumf = sub_group_reduce_add(sumf);\n\n if (lid == 0) {\n dst_row[0] = sumf / ne00;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// mul\n//------------------------------------------------------------------------------\nkernel void kernel_mul(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03 % ne13;\n int i12 = i02 % ne12;\n int i11 = i01 % ne11;\n\n global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;\n global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;\n global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const int i10 = i0 % ne10;\n *((global float *)(dst_ptr + i0*nb0)) = *((global float *)(src0_ptr + i0*nb00)) * *((global float *)(src1_ptr + i10*nb10));\n }\n}\n\n// assumption: src1 is a row\n// broadcast src1 into src0\nkernel void kernel_mul_row(\n global float4 * src0,\n ulong offset0,\n global float4 * src1,\n ulong offset1,\n global float4 * dst,\n ulong offsetd,\n int ne\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float4*)((global char*)dst + offsetd);\n\n // This performs better than using %.\n uint gid = get_global_id(0);\n uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne\n dst[gid] = src0[gid] * src1[idx1];\n}\n\nkernel void kernel_mul_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03 % ne13;\n int i12 = i02 % ne12;\n int i11 = i01 % ne11;\n\n global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;\n global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;\n global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const int i10 = i0 % ne10;\n *((global half *)(dst_ptr + i0*nb0)) = *((global half *)(src0_ptr + i0*nb00)) * *((global half *)(src1_ptr + i10*nb10));\n }\n}\n\nkernel void kernel_mul_row_f16(\n global half4 * src0,\n ulong offset0,\n global half4 * src1,\n ulong offset1,\n global half4 * dst,\n ulong offsetd,\n int ne\n) {\n src0 = (global half4*)((global char*)src0 + offset0);\n src1 = (global half4*)((global char*)src1 + offset1);\n dst = (global half4*)((global char*)dst + offsetd);\n\n // This performs better than using %.\n uint gid = get_global_id(0);\n uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne\n dst[gid] = src0[gid] * src1[idx1];\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl", "content": "// src0_q, src0_d, src1 are transposed as a preprocessing step\n// 4-bit weights are transposed in groups of 4 (unsigned short int)\n// consider weights originally \"next to each other\", now \"on top of each other\"\n// each fiber computes a 8x4 tile of output elements\n// using unshuffled weights\n\n#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n\n#ifdef cl_qcom_reqd_sub_group_size\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_128\n#endif\n\nkernel void kernel_mul_mat_Ab_Bi_8x4(\n global const ushort * src0_q, // quantized A\n global const half * src0_d, // A scales\n __read_only image1d_buffer_t src1, // B (1d image)\n global float * dst, // C\n int m, // M\n int n, // N with padding\n int k, // K\n int n_no_padding // N without padding\n) {\n\n int m_4 = m >> 2;\n int n_4 = n >> 2;\n\n int gy = get_global_id(0);\n int gx = get_global_id(1);\n int gx_2 = gx << 2;\n\n half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; // 8x4 output elements\n half8 B; // registers for activations\n half4 dequantized_weights; // registers for dequantized weights\n __global const ushort* weight_ptr = src0_q + gx_2; // pointer for weights\n __global const half* scale_ptr = src0_d + gx_2; // pointer for scales\n\n for(int i=0; i> 4) - 8) * scale.s0; // dequantize a row of the 16 weights\n dequantized_weights.s1 = (((bits4.s1 & (0x00F0)) >> 4) - 8) * scale.s1;\n dequantized_weights.s2 = (((bits4.s2 & (0x00F0)) >> 4) - 8) * scale.s2;\n dequantized_weights.s3 = (((bits4.s3 & (0x00F0)) >> 4) - 8) * scale.s3;\n c0 += B * dequantized_weights.s0; //vector-scalar multiplication to accumulate\n c1 += B * dequantized_weights.s1;\n c2 += B * dequantized_weights.s2;\n c3 += B * dequantized_weights.s3;\n\n // j=2\n B.s0123 = read_imageh(src1, gy*2 + (i+2)*(n_4));\n B.s4567 = read_imageh(src1, gy*2 + (i+2)*(n_4)+1);\n dequantized_weights.s0 = (((bits4.s0 & (0x0F00)) >> 8) - 8) * scale.s0; // dequantize a row of the 16 weights\n dequantized_weights.s1 = (((bits4.s1 & (0x0F00)) >> 8) - 8) * scale.s1;\n dequantized_weights.s2 = (((bits4.s2 & (0x0F00)) >> 8) - 8) * scale.s2;\n dequantized_weights.s3 = (((bits4.s3 & (0x0F00)) >> 8) - 8) * scale.s3;\n c0 += B * dequantized_weights.s0; // vector-scalar multiplication to accumulate\n c1 += B * dequantized_weights.s1;\n c2 += B * dequantized_weights.s2;\n c3 += B * dequantized_weights.s3;\n\n // j=3\n B.s0123 = read_imageh(src1, gy*2 + (i+3)*(n_4));\n B.s4567 = read_imageh(src1, gy*2 + (i+3)*(n_4)+1);\n dequantized_weights.s0 = (((bits4.s0 & (0xF000)) >> 12) - 8) * scale.s0; // dequantize a row of the 16 weights\n dequantized_weights.s1 = (((bits4.s1 & (0xF000)) >> 12) - 8) * scale.s1;\n dequantized_weights.s2 = (((bits4.s2 & (0xF000)) >> 12) - 8) * scale.s2;\n dequantized_weights.s3 = (((bits4.s3 & (0xF000)) >> 12) - 8) * scale.s3;\n c0 += B * dequantized_weights.s0; // vector-scalar multiplication to accumulate\n c1 += B * dequantized_weights.s1;\n c2 += B * dequantized_weights.s2;\n c3 += B * dequantized_weights.s3;\n }\n\n int idx = (gy<<3)*m + (gx<<2); // vectorized store 16 elements\n\n // conditional check if store is to a valid location. Required when N is not a multiple of 8\n // if statements allow registers to be reused for each store\n // provides a performance boost due to reduced register footprint, which increases number of concurrent waves\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mat_f16_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#if defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#else\n#define REQD_SUBGROUP_SIZE_128\n#endif\n\n#define OPWM 64\n#define OPWN 64\n#define CPWK 8\n#define OPTM 4\n#define OPTN 8\n\n#define WG_M (OPWM / OPTM)\n#define WG_N (OPWN / OPTN)\n#define VEC_K (CPWK / 4)\n\nREQD_SUBGROUP_SIZE_128\n__kernel void mul_mat_f16_f32(\n const int M, const int N, const int K,\n __global const void* A_void, ulong A_offset,\n __global const void* B_void, ulong B_offset,\n __global void* C_void, ulong C_offset) {\n\n __global const half* A = (__global const half* )((__global const char*)A_void + A_offset);\n __global const float* B = (__global const float*)((__global const char*)B_void + B_offset);\n __global float* C = (__global float*)((__global char*)C_void + C_offset);\n\n const int lidm = get_local_id(0);\n const int lidn = get_local_id(1);\n const int lid = lidn * WG_M + lidm;\n\n const int offsetM = get_group_id(0) * OPWM;\n const int offsetN = get_group_id(1) * OPWN;\n\n __local half4 Alocal[OPWM][VEC_K];\n __local float4 Blocal[OPWN][VEC_K];\n\n float sum[OPTM][OPTN];\n\n for (int wm = 0; wm < OPTM; wm++) {\n for (int wn = 0; wn < OPTN; wn++) {\n sum[wm][wn] = 0.0f;\n }\n }\n\n const int numTiles = (K + CPWK - 1) / CPWK;\n\n const int load_row_a = lid % OPWM;\n const int load_vec_k_a = lid / OPWM;\n const int global_row_a = offsetM + load_row_a;\n\n const int load_row_b = lid % OPWN;\n const int load_vec_k_b = lid / OPWN;\n const int global_row_b = offsetN + load_row_b;\n\n for (int t = 0; t < numTiles; t++) {\n const int k_start = t * CPWK;\n const int k_vec_start_a = k_start + load_vec_k_a * 4;\n const int k_vec_start_b = k_start + load_vec_k_b * 4;\n\n if (global_row_a < M && k_vec_start_a < K) {\n if (k_vec_start_a + 3 < K) {\n Alocal[load_row_a][load_vec_k_a] = vload4(0, A + global_row_a * K + k_vec_start_a);\n } else {\n half4 tempA = (half4)(0.0h);\n if (k_vec_start_a < K) tempA.s0 = A[global_row_a * K + k_vec_start_a];\n if (k_vec_start_a + 1 < K) tempA.s1 = A[global_row_a * K + k_vec_start_a + 1];\n if (k_vec_start_a + 2 < K) tempA.s2 = A[global_row_a * K + k_vec_start_a + 2];\n Alocal[load_row_a][load_vec_k_a] = tempA;\n }\n } else {\n Alocal[load_row_a][load_vec_k_a] = (half4)(0.0h);\n }\n\n if (global_row_b < N && k_vec_start_b < K) {\n if (k_vec_start_b + 3 < K) {\n Blocal[load_row_b][load_vec_k_b] = vload4(0, B + global_row_b * K + k_vec_start_b);\n } else {\n float4 tempB = (float4)(0.0f);\n if (k_vec_start_b < K) tempB.s0 = B[global_row_b * K + k_vec_start_b];\n if (k_vec_start_b + 1 < K) tempB.s1 = B[global_row_b * K + k_vec_start_b + 1];\n if (k_vec_start_b + 2 < K) tempB.s2 = B[global_row_b * K + k_vec_start_b + 2];\n Blocal[load_row_b][load_vec_k_b] = tempB;\n }\n } else {\n Blocal[load_row_b][load_vec_k_b] = (float4)(0.0f);\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n #pragma unroll\n for (int k_vec = 0; k_vec < VEC_K; k_vec++) {\n float4 a_fvecs[OPTM];\n int current_row_a = lidm;\n for (int wm = 0; wm < OPTM; wm++) {\n a_fvecs[wm] = convert_float4(Alocal[current_row_a][k_vec]);\n current_row_a += WG_M;\n }\n\n float4 b_fvecs[OPTN];\n int current_row_b = lidn;\n for (int wn = 0; wn < OPTN; wn++) {\n b_fvecs[wn] = Blocal[current_row_b][k_vec];\n current_row_b += WG_N;\n }\n\n for (int wm = 0; wm < OPTM; wm++) {\n for (int wn = 0; wn < OPTN; wn++) {\n sum[wm][wn] += dot(a_fvecs[wm], b_fvecs[wn]);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n for (int wm = 0; wm < OPTM; wm++) {\n int globalRow = offsetM + lidm + wm * WG_M;\n if (globalRow < M) {\n for (int wn = 0; wn < OPTN; wn++) {\n int globalCol = offsetN + lidn + wn * WG_N;\n if (globalCol < N) {\n C[globalCol * M + globalRow] = sum[wm][wn];\n }\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n\n#define LM_FIRST_256B 0\n#define LM_SECOND_256B 64\n#define LM_THIRD_256B 128\n#define LM_FOURTH_256B 192\n\n\ninline float16 mm_load_a(\n image1d_buffer_t matrix_A,\n uint subMatrixAStartInElements,\n int nb01,\n int line_stride_matrix_A_in_bytes\n) {\n __private float8 regA;\n size_t sub_block_id_m = get_local_id(0);\n\n#ifdef KQV\n uint a_texCoord = subMatrixAStartInElements/2 + (sub_block_id_m * nb01/4);\n#else // KQ\n uint a_texCoord = subMatrixAStartInElements/2 + (sub_block_id_m * line_stride_matrix_A_in_bytes/4);\n#endif\n\n regA.s0123 = read_imagef(matrix_A, a_texCoord/4);\n regA.s4567 = read_imagef(matrix_A, (a_texCoord+4)/4);\n\n return convert_float16(as_half16(regA));\n}\n\ninline float4 alu_32(\n float16 regA,\n __local float4* matrix_B_vec\n) {\n\n __private float4 rC = 0;\n int i = get_sub_group_id() * 64;\n\n rC += regA.s0 * matrix_B_vec[i];\n rC += regA.s1 * matrix_B_vec[i + 16];\n rC += regA.s4 * matrix_B_vec[i + 1];\n rC += regA.s5 * matrix_B_vec[i + 17];\n rC += regA.s8 * matrix_B_vec[i + 2];\n rC += regA.s9 * matrix_B_vec[i + 18];\n rC += regA.sc * matrix_B_vec[i + 3];\n rC += regA.sd * matrix_B_vec[i + 19];\n\n i += 32;\n\n rC += regA.s2 * matrix_B_vec[i];\n rC += regA.s3 * matrix_B_vec[i + 16];\n rC += regA.s6 * matrix_B_vec[i + 1];\n rC += regA.s7 * matrix_B_vec[i + 17];\n rC += regA.sa * matrix_B_vec[i + 2];\n rC += regA.sb * matrix_B_vec[i + 18];\n rC += regA.se * matrix_B_vec[i + 3];\n rC += regA.sf * matrix_B_vec[i + 19];\n\n return rC;\n}\n\ninline float16 alu_16(\n float16 regA,\n __local float* matrix_B_local\n) {\n float16 out;\n __local float4* matrix_B_vec = (__local float4*)matrix_B_local;\n\n out.s0123 = alu_32(regA, matrix_B_vec);\n out.s4567 = alu_32(regA, matrix_B_vec + 4);\n out.s89ab = alu_32(regA, matrix_B_vec + 8);\n out.scdef = alu_32(regA, matrix_B_vec + 12);\n\n return out;\n}\n\ninline void mm_mad(\n __local float* matrix_B_local,\n float16 regA,\n float8 regB,\n uint b_localOffsetInWords,\n float16* regC0_ptr,\n float16* regC1_ptr\n) {\n int offset = b_localOffsetInWords + get_sub_group_id() * 256;\n\n matrix_B_local[offset + LM_FIRST_256B] = regB.s0;\n matrix_B_local[offset + LM_SECOND_256B] = regB.s1;\n matrix_B_local[offset + LM_THIRD_256B] = regB.s2;\n matrix_B_local[offset + LM_FOURTH_256B] = regB.s3;\n\n float16 add0 = alu_16(regA, matrix_B_local);\n *regC0_ptr += add0;\n\n matrix_B_local[offset + LM_FIRST_256B] = regB.s4;\n matrix_B_local[offset + LM_SECOND_256B] = regB.s5;\n matrix_B_local[offset + LM_THIRD_256B] = regB.s6;\n matrix_B_local[offset + LM_FOURTH_256B] = regB.s7;\n\n float16 add1 = alu_16(regA, matrix_B_local);\n *regC1_ptr += add1;\n}\n\ninline void mm_store_c_N(\n __write_only image1d_buffer_t matrix_C,\n float16 regC0,\n float16 regC1,\n uint subMatrixCStartInElements,\n int line_stride_matrix_C_in_bytes,\n int mask\n) {\n size_t sub_block_id_m = get_local_id(0);\n\n uint strideInWords = line_stride_matrix_C_in_bytes/4;\n uint c_coordInWords_0 = (subMatrixCStartInElements + sub_block_id_m);\n\n uint c_coordInWords_1 = c_coordInWords_0 + 1 * strideInWords;\n uint c_coordInWords_2 = c_coordInWords_0 + 2 * strideInWords;\n uint c_coordInWords_3 = c_coordInWords_0 + 3 * strideInWords;\n uint c_coordInWords_4 = c_coordInWords_0 + 4 * strideInWords;\n uint c_coordInWords_5 = c_coordInWords_0 + 5 * strideInWords;\n uint c_coordInWords_6 = c_coordInWords_0 + 6 * strideInWords;\n uint c_coordInWords_7 = c_coordInWords_0 + 7 * strideInWords;\n uint c_coordInWords_8 = c_coordInWords_0 + 8 * strideInWords;\n uint c_coordInWords_9 = c_coordInWords_0 + 9 * strideInWords;\n uint c_coordInWords_10 = c_coordInWords_0 + 10 * strideInWords;\n uint c_coordInWords_11 = c_coordInWords_0 + 11 * strideInWords;\n uint c_coordInWords_12 = c_coordInWords_0 + 12 * strideInWords;\n uint c_coordInWords_13 = c_coordInWords_0 + 13 * strideInWords;\n uint c_coordInWords_14 = c_coordInWords_0 + 14 * strideInWords;\n uint c_coordInWords_15 = c_coordInWords_0 + 15 * strideInWords;\n uint c_coordInWords_16 = c_coordInWords_0 + 16 * strideInWords;\n uint c_coordInWords_17 = c_coordInWords_0 + 17 * strideInWords;\n uint c_coordInWords_18 = c_coordInWords_0 + 18 * strideInWords;\n uint c_coordInWords_19 = c_coordInWords_0 + 19 * strideInWords;\n uint c_coordInWords_20 = c_coordInWords_0 + 20 * strideInWords;\n uint c_coordInWords_21 = c_coordInWords_0 + 21 * strideInWords;\n uint c_coordInWords_22 = c_coordInWords_0 + 22 * strideInWords;\n uint c_coordInWords_23 = c_coordInWords_0 + 23 * strideInWords;\n uint c_coordInWords_24 = c_coordInWords_0 + 24 * strideInWords;\n uint c_coordInWords_25 = c_coordInWords_0 + 25 * strideInWords;\n uint c_coordInWords_26 = c_coordInWords_0 + 26 * strideInWords;\n uint c_coordInWords_27 = c_coordInWords_0 + 27 * strideInWords;\n uint c_coordInWords_28 = c_coordInWords_0 + 28 * strideInWords;\n uint c_coordInWords_29 = c_coordInWords_0 + 29 * strideInWords;\n uint c_coordInWords_30 = c_coordInWords_0 + 30 * strideInWords;\n uint c_coordInWords_31 = c_coordInWords_0 + 31 * strideInWords;\n\n if (mask > 0) { write_imagef(matrix_C, c_coordInWords_0, regC0.s0); }\n if (mask > 1) { write_imagef(matrix_C, c_coordInWords_1, regC0.s1); }\n if (mask > 2) { write_imagef(matrix_C, c_coordInWords_2, regC0.s2); }\n if (mask > 3) { write_imagef(matrix_C, c_coordInWords_3, regC0.s3); }\n if (mask > 4) { write_imagef(matrix_C, c_coordInWords_4, regC0.s4); }\n if (mask > 5) { write_imagef(matrix_C, c_coordInWords_5, regC0.s5); }\n if (mask > 6) { write_imagef(matrix_C, c_coordInWords_6, regC0.s6); }\n if (mask > 7) { write_imagef(matrix_C, c_coordInWords_7, regC0.s7); }\n if (mask > 8) { write_imagef(matrix_C, c_coordInWords_8, regC0.s8); }\n if (mask > 9) { write_imagef(matrix_C, c_coordInWords_9, regC0.s9); }\n if (mask > 10) { write_imagef(matrix_C, c_coordInWords_10, regC0.sa); }\n if (mask > 11) { write_imagef(matrix_C, c_coordInWords_11, regC0.sb); }\n if (mask > 12) { write_imagef(matrix_C, c_coordInWords_12, regC0.sc); }\n if (mask > 13) { write_imagef(matrix_C, c_coordInWords_13, regC0.sd); }\n if (mask > 14) { write_imagef(matrix_C, c_coordInWords_14, regC0.se); }\n if (mask > 15) { write_imagef(matrix_C, c_coordInWords_15, regC0.sf); }\n if (mask > 16) { write_imagef(matrix_C, c_coordInWords_16, regC1.s0); }\n if (mask > 17) { write_imagef(matrix_C, c_coordInWords_17, regC1.s1); }\n if (mask > 18) { write_imagef(matrix_C, c_coordInWords_18, regC1.s2); }\n if (mask > 19) { write_imagef(matrix_C, c_coordInWords_19, regC1.s3); }\n if (mask > 20) { write_imagef(matrix_C, c_coordInWords_20, regC1.s4); }\n if (mask > 21) { write_imagef(matrix_C, c_coordInWords_21, regC1.s5); }\n if (mask > 22) { write_imagef(matrix_C, c_coordInWords_22, regC1.s6); }\n if (mask > 23) { write_imagef(matrix_C, c_coordInWords_23, regC1.s7); }\n if (mask > 24) { write_imagef(matrix_C, c_coordInWords_24, regC1.s8); }\n if (mask > 25) { write_imagef(matrix_C, c_coordInWords_25, regC1.s9); }\n if (mask > 26) { write_imagef(matrix_C, c_coordInWords_26, regC1.sa); }\n if (mask > 27) { write_imagef(matrix_C, c_coordInWords_27, regC1.sb); }\n if (mask > 28) { write_imagef(matrix_C, c_coordInWords_28, regC1.sc); }\n if (mask > 29) { write_imagef(matrix_C, c_coordInWords_29, regC1.sd); }\n if (mask > 30) { write_imagef(matrix_C, c_coordInWords_30, regC1.se); }\n if (mask > 31) { write_imagef(matrix_C, c_coordInWords_31, regC1.sf); }\n}\n\n#define TILESIZE_K 16\n#define TILESIZE_M 64\n#define TILESIZE_N 32\n#ifdef KQV\n__kernel void mul_mm_f16_f32_kqv(\n#else\n__kernel void mul_mm_f16_f32_kq(\n#endif\n __read_only image1d_buffer_t matrix_A,\n int offset0,\n __global float* matrix_B,\n int offset1,\n __write_only image1d_buffer_t matrix_C,\n int offsetd,\n int M, int K, int N,\n int D_A,\n int D_B,\n int nb01\n) {\n\n uint block_id_m = get_global_id(1);\n uint block_id_n = get_global_id(2) % ((N+TILESIZE_N-1)/TILESIZE_N);\n uint block_id_d = get_global_id(2) / ((N+TILESIZE_N-1)/TILESIZE_N);\n\n __private float16 regA;\n __private float8 regB;\n __private float16 regC0;\n __private float16 regC1;\n\n const uint col = block_id_m * TILESIZE_M;\n const uint row = block_id_n * TILESIZE_N;\n const uint depth_A = block_id_d / (D_B/D_A);\n const uint depth_B = block_id_d;\n\n#ifdef KQV\n int line_stride_matrix_A_in_bytes = nb01 * M;\n int line_stride_matrix_B_in_bytes = K * N * 4;\n#else\n int line_stride_matrix_A_in_bytes = K * D_A * 2;\n int line_stride_matrix_B_in_bytes = K * D_B * 4;\n#endif\n\n int line_stride_matrix_C_in_bytes = M * 4;\n\n const uint strideAinElements = line_stride_matrix_A_in_bytes / 2;\n const uint strideBinElements = line_stride_matrix_B_in_bytes / 4;\n\n size_t sub_block_id_m = get_local_id(0);\n\n uint b_localOffsetInWords = (sub_block_id_m/16)*16\n + ((((sub_block_id_m)>>0)&1)<<2)\n + ((((sub_block_id_m)>>1)&1)<<3)\n + ((((sub_block_id_m)>>2)&1)<<0)\n + ((((sub_block_id_m)>>3)&1)<<1);\n\n uint2 b_globalOffsetInWords_xy = {((sub_block_id_m%4)*4), (sub_block_id_m>>2)};\n uint b_globalOffsetInWords00, b_globalOffsetInWords16;\n#ifdef KQV\n b_globalOffsetInWords00 = b_globalOffsetInWords_xy.x + b_globalOffsetInWords_xy.y*K;\n b_globalOffsetInWords16 = b_globalOffsetInWords00 + (16 * K);\n uint subMatrixAStartInElements = depth_A * strideAinElements + col * nb01 / 2;\n uint subMatrixBStartInElements = depth_B * strideBinElements + row * K;\n#else\n b_globalOffsetInWords00 = b_globalOffsetInWords_xy.x + b_globalOffsetInWords_xy.y*line_stride_matrix_B_in_bytes/4;\n b_globalOffsetInWords16 = b_globalOffsetInWords00 + (16 * line_stride_matrix_B_in_bytes/4);\n uint subMatrixAStartInElements = col * strideAinElements + depth_A * K;\n uint subMatrixBStartInElements = row * strideBinElements + depth_B * K;\n#endif\n\n __local float matrix_B_local[1024];\n\n for (uint step=0; step < K; step+=TILESIZE_K) {\n size_t sub_block_id_m = get_local_id(0);\n regA = mm_load_a(matrix_A, subMatrixAStartInElements, nb01, line_stride_matrix_A_in_bytes);\n\n uint b_coordInWords00 = subMatrixBStartInElements + b_globalOffsetInWords00;\n uint b_coordInWords16 = subMatrixBStartInElements + b_globalOffsetInWords16;\n\n regB.s0123 = vload4(b_coordInWords00/4, matrix_B);\n regB.s4567 = vload4(b_coordInWords16/4, matrix_B);\n\n mm_mad(matrix_B_local, regA, regB, b_localOffsetInWords, ®C0, ®C1);\n\n subMatrixAStartInElements += TILESIZE_K;\n subMatrixBStartInElements += TILESIZE_K;\n }\n\n uint subMatrixCStartInElements = depth_B * N * M + row * M + col;\n mm_store_c_N(matrix_C, regC0, regC1, subMatrixCStartInElements, line_stride_matrix_C_in_bytes, (N-block_id_n*32));\n}\n\n" }, { "path": "src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define LOAD_VEC_A 4\n#define LOAD_VEC_B 4\n\n#define BM 64\n#define BN 64\n#define BK 16\n#define TM 4\n#define TN 8\n\nkernel void kernel_mul_mm_f16_f32_l4_lm(\n global half4 * src0,\n ulong offset0,\n global float4 * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n\n int ne00,\n int ne01,\n int ne02,\n int ne11,\n int ne12,\n\n int stride_a,\n int stride_b,\n int stride_d,\n\n int batch_stride_a,\n int batch_stride_b,\n int batch_stride_d,\n\n int r2,\n int r3\n) {\n src0 = (global half4*)((global char*)src0 + offset0);\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n local half buf_a[BM * BK];\n local float buf_b[BN * BK];\n\n const int batch_idx = get_global_id(2);\n\n const int i13 = batch_idx / ne12;\n const int i12 = batch_idx % ne12;\n\n const int i03 = i13 / r3;\n const int i02 = i12 / r2;\n\n const int batch_idx_a = i03 * ne02 + i02;\n\n const int ir = get_group_id(0);\n const int ic = get_group_id(1);\n\n const int tid = get_local_id(0);\n const int th_r = tid % (BM / TM);\n const int th_c = tid / (BM / TM);\n\n const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);\n const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);\n const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);\n const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);\n\n const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;\n const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;\n\n int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A;\n int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;\n\n float sums[TM * TN];\n half cache_a[TM];\n float cache_b[TN];\n\n for (int i = 0; i < TM * TN; i++) {\n sums[i] = 0.0f;\n }\n\n for (int block = 0; block < ne00; block += BK) {\n for (int l = 0; l < BM; l += loadstride_a) {\n if (ir*BM + loadc_a + l < ne01) {\n const int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a;\n buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = src0[idx].s0;\n buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = src0[idx].s1;\n buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = src0[idx].s2;\n buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = src0[idx].s3;\n } else {\n buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0h;\n buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0h;\n buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0h;\n buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0h;\n }\n }\n\n for (int l = 0; l < BN; l += loadstride_b) {\n if (ic*BN + loadc_b + l < ne11) {\n const int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;\n } else {\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0h;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0h;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0h;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0h;\n }\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n pos_a += BK / LOAD_VEC_A;\n pos_b += BK / LOAD_VEC_B;\n\n for (int i = 0; i < BK; i++) {\n for (int j = 0; j < TM; j++) {\n cache_a[j] = buf_a[(i) * BM + th_r * TM + j];\n }\n for (int j = 0; j < TN; j++) {\n cache_b[j] = buf_b[(i) * BN + th_c * TN + j];\n }\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n const int sums_idx = cc*TM + cr;\n sums[sums_idx] = mad(convert_float(cache_a[cr]), cache_b[cc], sums[sums_idx]);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n const int dr = ir * BM + th_r * TM;\n const int dc = ic * BN + th_c * TN;\n\n const int offsets = batch_idx * batch_stride_d;\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n if (dr + cr < ne01 && dc + cc < ne11) {\n dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define LOAD_VEC_A 4\n#define LOAD_VEC_B 4\n\n#define BM 64\n#define BN 64\n#define BK 16\n#define TM 4\n#define TN 8\n\nkernel void kernel_mul_mm_f32_f32_l4_lm(\n global float4 * src0,\n ulong offset0,\n global float4 * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n\n int ne00,\n int ne01,\n int ne02,\n int ne11,\n int ne12,\n\n int stride_a,\n int stride_b,\n int stride_d,\n\n int batch_stride_a,\n int batch_stride_b,\n int batch_stride_d,\n\n int r2,\n int r3\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n local float buf_a[BM * BK];\n local float buf_b[BN * BK];\n\n const int batch_idx = get_global_id(2);\n\n const int i13 = batch_idx / ne12;\n const int i12 = batch_idx % ne12;\n\n const int i03 = i13 / r3;\n const int i02 = i12 / r2;\n\n const int batch_idx_a = i03 * ne02 + i02;\n\n const int ir = get_group_id(0);\n const int ic = get_group_id(1);\n\n const int tid = get_local_id(0);\n const int th_r = tid % (BM / TM);\n const int th_c = tid / (BM / TM);\n\n const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);\n const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);\n const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);\n const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);\n\n const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;\n const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;\n\n int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A;\n int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;\n\n float sums[TM * TN];\n float cache_a[TM];\n float cache_b[TN];\n\n for (int i = 0; i < TM * TN; i++) {\n sums[i] = 0.0f;\n }\n\n for (int block = 0; block < ne00; block += BK) {\n for (int l = 0; l < BM; l += loadstride_a) {\n if (ir*BM + loadc_a + l < ne01) {\n const int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a;\n buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = src0[idx].s0;\n buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = src0[idx].s1;\n buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = src0[idx].s2;\n buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = src0[idx].s3;\n } else {\n buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;\n }\n }\n\n for (int l = 0; l < BN; l += loadstride_b) {\n if (ic*BN + loadc_b + l < ne11) {\n const int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;\n } else {\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;\n }\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n pos_a += BK / LOAD_VEC_A;\n pos_b += BK / LOAD_VEC_B;\n\n for (int i = 0; i < BK; i++) {\n for (int j = 0; j < TM; j++) {\n cache_a[j] = buf_a[(i) * BM + th_r * TM + j];\n }\n\n for (int j = 0; j < TN; j++) {\n cache_b[j] = buf_b[(i) * BN + th_c * TN + j];\n }\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n const int sums_idx = cc*TM + cr;\n sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n const int dr = ir * BM + th_r * TM;\n const int dc = ic * BN + th_c * TN;\n\n const int offsets = batch_idx * batch_stride_d;\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n if (dr + cr < ne01 && dc + cc < ne11) {\n dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define LOAD_VEC_A 8\n#define LOAD_VEC_B 4\n\n#define BM 64\n#define BN 64\n#define BK 32\n#define TM 4\n#define TN 8\n\nkernel void kernel_mul_mm_q4_0_f32_l4_lm(\n global uchar4 * src0_q,\n global half * src0_d,\n global float4 * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n\n int ne00,\n int ne01,\n int ne02,\n int ne11,\n int ne12,\n\n int stride_a,\n int stride_b,\n int stride_d,\n\n int batch_stride_a,\n int batch_stride_b,\n int batch_stride_d,\n\n int r2,\n int r3\n) {\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float *)((global char*)dst + offsetd);\n\n local float buf_a[BM * BK];\n local float buf_b[BN * BK];\n\n const int batch_idx = get_global_id(2);\n\n const int i13 = batch_idx / ne12;\n const int i12 = batch_idx % ne12;\n\n const int i03 = i13 / r3;\n const int i02 = i12 / r2;\n\n const int batch_idx_a = i03 * ne02 + i02;\n\n const int ir = get_group_id(0);\n const int ic = get_group_id(1);\n\n const int tid = get_local_id(0);\n const int th_r = tid % (BM / TM);\n const int th_c = tid / (BM / TM);\n\n const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);\n const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);\n const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);\n const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);\n\n const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;\n const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;\n\n int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A;\n int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;\n\n float sums[TM * TN];\n float cache_a[TM];\n float cache_b[TN];\n\n for (int i = 0; i < TM * TN; i++) {\n sums[i] = 0.0f;\n }\n\n for (int block = 0; block < ne00; block += BK) {\n for (int l = 0; l < BM; l += loadstride_a) {\n if (ir*BM + loadc_a + l < ne01) {\n int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a;\n int ib = idx / 4;\n int iqs = idx % 4;\n\n float d = (float)src0_d[ib];\n global uchar4 * qs = src0_q + ib*4 + iqs;\n uchar4 q = *qs;\n float4 v1 = (convert_float4((uchar4)((q.s0 )&0x0F, (q.s1 )&0x0F, (q.s2 )&0x0F, (q.s3 )&0x0F)) - 8.0f)*d;\n float4 v2 = (convert_float4((uchar4)((q.s0>>4)&0x0F, (q.s1>>4)&0x0F, (q.s2>>4)&0x0F, (q.s3>>4)&0x0F)) - 8.0f)*d;\n\n buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = v1.s0;\n buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = v1.s1;\n buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = v1.s2;\n buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = v1.s3;\n buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = v2.s0;\n buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = v2.s1;\n buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = v2.s2;\n buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = v2.s3;\n } else {\n buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = 0.0f;\n }\n }\n\n for (int l = 0; l < BN; l += loadstride_b) {\n if (ic*BN + loadc_b + l < ne11) {\n int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;\n } else {\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;\n }\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n pos_a += BK / LOAD_VEC_A;\n pos_b += BK / LOAD_VEC_B;\n\n for (int i = 0; i < BK; i++) {\n for (int j = 0; j < TM; j++) {\n cache_a[j] = buf_a[(i) * BM + th_r * TM + j];\n }\n\n for (int j = 0; j < TN; j++) {\n cache_b[j] = buf_b[(i) * BN + th_c * TN + j];\n }\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n const int sums_idx = cc*TM + cr;\n sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n const int dr = ir * BM + th_r * TM;\n const int dc = ic * BN + th_c * TN;\n\n const int offsets = batch_idx * batch_stride_d;\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n if (dr + cr < ne01 && dc + cc < ne11) {\n dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define LOAD_VEC_A 8\n#define LOAD_VEC_B 4\n\n#define BM 64\n#define BN 64\n#define BK 32\n#define TM 4\n#define TN 8\n\nkernel void kernel_mul_mm_q4_1_f32_l4_lm(\n global uchar4 * src0_q,\n global half * src0_d,\n global half * src0_m,\n global float4 * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n\n int ne00,\n int ne01,\n int ne02,\n int ne11,\n int ne12,\n\n int stride_a,\n int stride_b,\n int stride_d,\n\n int batch_stride_a,\n int batch_stride_b,\n int batch_stride_d,\n\n int r2,\n int r3\n) {\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float *)((global char*)dst + offsetd);\n\n local float buf_a[BM * BK];\n local float buf_b[BN * BK];\n\n const int batch_idx = get_global_id(2);\n\n const int i13 = batch_idx / ne12;\n const int i12 = batch_idx % ne12;\n\n const int i03 = i13 / r3;\n const int i02 = i12 / r2;\n\n const int batch_idx_a = i03 * ne02 + i02;\n\n const int ir = get_group_id(0);\n const int ic = get_group_id(1);\n\n const int tid = get_local_id(0);\n const int th_r = tid % (BM / TM);\n const int th_c = tid / (BM / TM);\n\n const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);\n const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);\n const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);\n const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);\n\n const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;\n const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;\n\n int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A;\n int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;\n\n float sums[TM * TN];\n float cache_a[TM];\n float cache_b[TN];\n\n for (int i = 0; i < TM * TN; i++) {\n sums[i] = 0.0f;\n }\n\n for (int block = 0; block < ne00; block += BK) {\n for (int l = 0; l < BM; l += loadstride_a) {\n if (ir*BM + loadc_a + l < ne01) {\n int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a;\n int ib = idx / 4;\n int iqs = idx % 4;\n\n float d = (float)src0_d[ib];\n float m = (float)src0_m[ib];\n global uchar4 * qs = src0_q + ib*4 + iqs;\n uchar4 q = *qs;\n float4 v1 = (convert_float4((uchar4)((q.s0 )&0x0F, (q.s1 )&0x0F, (q.s2 )&0x0F, (q.s3 )&0x0F)))*d + m;\n float4 v2 = (convert_float4((uchar4)((q.s0>>4)&0x0F, (q.s1>>4)&0x0F, (q.s2>>4)&0x0F, (q.s3>>4)&0x0F)))*d + m;\n\n buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = v1.s0;\n buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = v1.s1;\n buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = v1.s2;\n buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = v1.s3;\n buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = v2.s0;\n buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = v2.s1;\n buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = v2.s2;\n buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = v2.s3;\n } else {\n buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = 0.0f;\n }\n }\n\n for (int l = 0; l < BN; l += loadstride_b) {\n if (ic*BN + loadc_b + l < ne11) {\n int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;\n } else {\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;\n }\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n pos_a += BK / LOAD_VEC_A;\n pos_b += BK / LOAD_VEC_B;\n\n for (int i = 0; i < BK; i++) {\n for (int j = 0; j < TM; j++) {\n cache_a[j] = buf_a[(i) * BM + th_r * TM + j];\n }\n\n for (int j = 0; j < TN; j++) {\n cache_b[j] = buf_b[(i) * BN + th_c * TN + j];\n }\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n const int sums_idx = cc*TM + cr;\n sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n const int dr = ir * BM + th_r * TM;\n const int dc = ic * BN + th_c * TN;\n\n const int offsets = batch_idx * batch_stride_d;\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n if (dr + cr < ne01 && dc + cc < ne11) {\n dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define LOAD_VEC_A 2\n#define LOAD_VEC_B 4\n\n#define BM 64\n#define BN 64\n#define BK 32\n#define TM 4\n#define TN 8\n\nkernel void kernel_mul_mm_q6_k_f32_l4_lm(\n global uchar * src0_ql,\n global uchar * src0_qh,\n global char * src0_s,\n global half * src0_d,\n global float4 * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n\n int ne00,\n int ne01,\n int ne02,\n int ne11,\n int ne12,\n\n int stride_a,\n int stride_b,\n int stride_d,\n\n int batch_stride_a,\n int batch_stride_b,\n int batch_stride_d,\n\n int r2,\n int r3\n) {\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float *)((global char*)dst + offsetd);\n\n local float buf_a[BM * BK];\n local float buf_b[BN * BK];\n\n const int batch_idx = get_global_id(2);\n\n const int i13 = batch_idx / ne12;\n const int i12 = batch_idx % ne12;\n\n const int i03 = i13 / r3;\n const int i02 = i12 / r2;\n\n const int batch_idx_a = i03 * ne02 + i02;\n\n const int ir = get_group_id(0);\n const int ic = get_group_id(1);\n\n const int tid = get_local_id(0);\n const int th_r = tid % (BM / TM);\n const int th_c = tid / (BM / TM);\n\n const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);\n const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);\n const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);\n const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);\n\n const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;\n const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;\n\n int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A;\n int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;\n\n float sums[TM * TN];\n float cache_a[TM];\n float cache_b[TN];\n\n for (int i = 0; i < TM * TN; i++) {\n sums[i] = 0.0f;\n }\n\n for (int block = 0; block < ne00; block += BK) {\n for (int l = 0; l < BM; l += loadstride_a) {\n if (ir*BM + loadc_a + l < ne01) {\n int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a;\n\n int ib = idx / 128; // 2 values per idx\n int iqs = idx % 128; // 0..127\n\n int n = iqs / 64; // 0,1\n int b = (iqs % 64) / 32; // 0,1\n int is_b = (iqs % 16) / 8; // 0,1\n int qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6\n int is = 8 * n + qhshift + is_b; // 0..15\n int qsi = n * 64 + (iqs % 32) * 2; // 0,2,4..126\n int qhi = n * 32 + (iqs % 16) * 2; // 0,2,4..62\n\n float dscale = (float)src0_d[ib] * (float)src0_s[ib*16 + is];\n\n buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = dscale * convert_float(convert_char(((src0_ql[128*ib + qsi + 0] >> (b * 4)) & 0xF) | (((src0_qh[64*ib + qhi + 0] >> qhshift) & 3) << 4)) - 32);\n buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = dscale * convert_float(convert_char(((src0_ql[128*ib + qsi + 1] >> (b * 4)) & 0xF) | (((src0_qh[64*ib + qhi + 1] >> qhshift) & 3) << 4)) - 32);\n } else {\n buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;\n }\n }\n\n for (int l = 0; l < BN; l += loadstride_b) {\n if (ic*BN + loadc_b + l < ne11) {\n int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;\n } else {\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;\n }\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n pos_a += BK / LOAD_VEC_A;\n pos_b += BK / LOAD_VEC_B;\n\n for (int i = 0; i < BK; i++) {\n for (int j = 0; j < TM; j++) {\n cache_a[j] = buf_a[(i) * BM + th_r * TM + j];\n }\n\n for (int j = 0; j < TN; j++) {\n cache_b[j] = buf_b[(i) * BN + th_c * TN + j];\n }\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n const int sums_idx = cc*TM + cr;\n sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n const int dr = ir * BM + th_r * TM;\n const int dc = ic * BN + th_c * TN;\n\n const int offsets = batch_idx * batch_stride_d;\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n if (dr + cr < ne01 && dc + cc < ne11) {\n dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mm_q8_0_f32_8x4.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n\n#ifdef cl_qcom_reqd_sub_group_size\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_128\n#endif\n\nkernel void kernel_mul_mm_q8_0_f32_8x4(\n global const uint * src0_q,\n global const half * src0_d,\n __read_only image1d_buffer_t src1,\n global float * dst,\n int k,\n int m,\n int n,\n int n_no_padding,\n ulong offsetd\n) {\n\n int m_4 = m >> 2;\n int n_4 = n >> 2;\n\n int gy = get_global_id(0);\n int gx = get_global_id(1);\n int gx_2 = gx << 2;\n dst = (global float *)((global char*)dst + offsetd);\n\n\n half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;\n half8 B;\n half4 deq;\n\n __global const uint* wptr = src0_q + gx_2;\n __global const half* sptr = src0_d + gx_2;\n\n for (int i = 0; i < k; i += 4) {\n uint4 pack4 = vload4(0, wptr + (i / 4) * m);\n half4 scale = vload4(0, sptr + (i / 32) * m);\n\n char4 p0 = as_char4(pack4.s0);\n char4 p1 = as_char4(pack4.s1);\n char4 p2 = as_char4(pack4.s2);\n char4 p3 = as_char4(pack4.s3);\n\n // ------------------- j = 0 (k = i+0) -------------------\n B.s0123 = read_imageh(src1, gy * 2 + (i + 0) * n_4);\n B.s4567 = read_imageh(src1, gy * 2 + (i + 0) * n_4 + 1);\n\n half4 wj0 = convert_half4((char4)(p0.s0, p1.s0, p2.s0, p3.s0)) * scale;\n\n c0 += B * wj0.s0;\n c1 += B * wj0.s1;\n c2 += B * wj0.s2;\n c3 += B * wj0.s3;\n\n // ------------------- j = 1 (k = i+1) -------------------\n B.s0123 = read_imageh(src1, gy * 2 + (i + 1) * n_4);\n B.s4567 = read_imageh(src1, gy * 2 + (i + 1) * n_4 + 1);\n\n half4 wj1 = convert_half4((char4)(p0.s1, p1.s1, p2.s1, p3.s1)) * scale;\n\n c0 += B * wj1.s0;\n c1 += B * wj1.s1;\n c2 += B * wj1.s2;\n c3 += B * wj1.s3;\n\n // ------------------- j = 2 (k = i+2) -------------------\n B.s0123 = read_imageh(src1, gy * 2 + (i + 2) * n_4);\n B.s4567 = read_imageh(src1, gy * 2 + (i + 2) * n_4 + 1);\n\n half4 wj2 = convert_half4((char4)(p0.s2, p1.s2, p2.s2, p3.s2)) * scale;\n\n c0 += B * wj2.s0;\n c1 += B * wj2.s1;\n c2 += B * wj2.s2;\n c3 += B * wj2.s3;\n\n // ------------------- j = 3 (k = i+3) -------------------\n B.s0123 = read_imageh(src1, gy * 2 + (i + 3) * n_4);\n B.s4567 = read_imageh(src1, gy * 2 + (i + 3) * n_4 + 1);\n\n half4 wj3 = convert_half4((char4)(p0.s3, p1.s3, p2.s3, p3.s3)) * scale;\n\n c0 += B * wj3.s0;\n c1 += B * wj3.s1;\n c2 += B * wj3.s2;\n c3 += B * wj3.s3;\n }\n\n int idx = (gy << 3) * m + (gx << 2);\n\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);\n idx += m;\n }\n if(idx+3 < m*n_no_padding){\n vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#define LOAD_VEC_A 4\n#define LOAD_VEC_B 4\n\n#define BM 64\n#define BN 64\n#define BK 32\n#define TM 4\n#define TN 8\n\nkernel void kernel_mul_mm_q8_0_f32_l4_lm(\n global char4 * src0_q,\n global half * src0_d,\n global float4 * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n\n int ne00,\n int ne01,\n int ne02,\n int ne11,\n int ne12,\n\n int stride_a,\n int stride_b,\n int stride_d,\n\n int batch_stride_a,\n int batch_stride_b,\n int batch_stride_d,\n\n int r2,\n int r3\n) {\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float *)((global char*)dst + offsetd);\n\n local float buf_a[BM * BK];\n local float buf_b[BN * BK];\n\n const int batch_idx = get_global_id(2);\n\n const int i13 = batch_idx / ne12;\n const int i12 = batch_idx % ne12;\n\n const int i03 = i13 / r3;\n const int i02 = i12 / r2;\n\n const int batch_idx_a = i03 * ne02 + i02;\n\n const int ir = get_group_id(0);\n const int ic = get_group_id(1);\n\n const int tid = get_local_id(0);\n const int th_r = tid % (BM / TM);\n const int th_c = tid / (BM / TM);\n\n const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);\n const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);\n const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);\n const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);\n\n const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;\n const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;\n\n int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A;\n int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;\n\n float sums[TM * TN];\n float cache_a[TM];\n float cache_b[TN];\n\n for (int i = 0; i < TM * TN; i++) {\n sums[i] = 0.0f;\n }\n\n for (int block = 0; block < ne00; block += BK) {\n for (int l = 0; l < BM; l += loadstride_a) {\n if (ir*BM + loadc_a + l < ne01) {\n int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a;\n int ib = idx / 8;\n int iqs = idx % 8;\n\n float d = (float)src0_d[ib];\n global char4 * qs = src0_q + ib*8 + iqs;\n char4 q = *qs;\n float4 v = convert_float4(q)*d;\n\n buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v.s0;\n buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v.s1;\n buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v.s2;\n buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v.s3;\n } else {\n buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;\n buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;\n }\n }\n\n for (int l = 0; l < BN; l += loadstride_b) {\n if (ic*BN + loadc_b + l < ne11) {\n int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;\n } else {\n buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;\n buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;\n }\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n pos_a += BK / LOAD_VEC_A;\n pos_b += BK / LOAD_VEC_B;\n\n for (int i = 0; i < BK; i++) {\n for (int j = 0; j < TM; j++) {\n cache_a[j] = buf_a[(i) * BM + th_r * TM + j];\n }\n\n for (int j = 0; j < TN; j++) {\n cache_b[j] = buf_b[(i) * BN + th_c * TN + j];\n }\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n const int sums_idx = cc*TM + cr;\n sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);\n }\n }\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n\n const int dr = ir * BM + th_r * TM;\n const int dc = ic * BN + th_c * TN;\n\n const int offsets = batch_idx * batch_stride_d;\n\n for (int cc = 0; cc < TN; cc++) {\n for (int cr = 0; cr < TM; cr++) {\n if (dr + cr < ne01 && dc + cc < ne11) {\n dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_f16_f16.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define N_F16_F16 4\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_f16_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3)\n{\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n int r0 = get_group_id(0);\n int rb = get_group_id(1)*N_F16_F16;\n int im = get_group_id(2);\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n\n global half * x = (global half *) (src0 + offset_src0);\n\n if (ne00 < 128) {\n for (int row = 0; row < N_F16_F16; ++row) {\n int r1 = rb + row;\n if (r1 >= ne11) {\n break;\n }\n\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global half * y = (global half *) (src1 + offset_src1);\n\n float sumf = 0;\n for (int i = get_sub_group_local_id(); i < ne00; i += get_max_sub_group_size()) {\n sumf += (half) x[i] * (half) y[i];\n }\n\n float all_sum = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;\n }\n }\n } else {\n global half4 * x4 = (global half4 *)x;\n for (int row = 0; row < N_F16_F16; ++row) {\n int r1 = rb + row;\n if (r1 >= ne11) {\n break;\n }\n\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global half * y = (global half *) (src1 + offset_src1);\n global half4 * y4 = (global half4 *) y;\n\n float sumf = 0;\n for (int i = get_sub_group_local_id(); i < ne00/4; i += get_max_sub_group_size()) {\n sumf += (half) x4[i].s0 * y4[i].s0;\n sumf += (half) x4[i].s1 * y4[i].s1;\n sumf += (half) x4[i].s2 * y4[i].s2;\n sumf += (half) x4[i].s3 * y4[i].s3;\n }\n\n float all_sum = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n for (int i = 4*(ne00/4); i < ne00; ++i) {\n all_sum += (half) x[i] * y[i];\n }\n dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_f16_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define N_F16_F32 4\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_f16_f32(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n int r0 = get_group_id(0);\n int rb = get_group_id(1)*N_F16_F32;\n int im = get_group_id(2);\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n\n global half * x = (global half *) (src0 + offset_src0);\n\n if (ne00 < 128) {\n for (int row = 0; row < N_F16_F32; ++row) {\n int r1 = rb + row;\n if (r1 >= ne11) {\n break;\n }\n\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global float * y = (global float *) (src1 + offset_src1);\n\n float sumf = 0;\n for (int i = get_sub_group_local_id(); i < ne00; i += get_max_sub_group_size()) {\n sumf += convert_float(x[i]) * y[i];\n }\n\n float all_sum = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;\n }\n }\n } else {\n global half4 * x4 = (global half4 *)x;\n for (int row = 0; row < N_F16_F32; ++row) {\n int r1 = rb + row;\n if (r1 >= ne11) {\n break;\n }\n\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global float * y = (global float *) (src1 + offset_src1);\n global float4 * y4 = (global float4 *) y;\n\n float sumf = 0;\n for (int i = get_sub_group_local_id(); i < ne00/4; i += get_max_sub_group_size()) {\n sumf += convert_float(x4[i].s0) * y4[i].s0;\n sumf += convert_float(x4[i].s1) * y4[i].s1;\n sumf += convert_float(x4[i].s2) * y4[i].s2;\n sumf += convert_float(x4[i].s3) * y4[i].s3;\n }\n\n float all_sum = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n for (int i = 4*(ne00/4); i < ne00; ++i) {\n all_sum += (float) x[i] * y[i];\n }\n dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_f16_f32_1row(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global half * x = (global half *) (src0 + offset_src0);\n global float * y = (global float *) (src1 + offset_src1);\n\n float sumf = 0;\n if (ne00 < 128) {\n for (int i = get_sub_group_local_id(); i < ne00; i += get_max_sub_group_size()) {\n sumf += (float) x[i] * (float) y[i];\n }\n float all_sum = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;\n }\n } else {\n global half4 * x4 = (global half4 *) x;\n global float4 * y4 = (global float4 *) y;\n for (int i = get_sub_group_local_id(); i < ne00/4; i += get_max_sub_group_size()) {\n sumf += (float) x4[i].s0 * y4[i].s0;\n sumf += (float) x4[i].s1 * y4[i].s1;\n sumf += (float) x4[i].s2 * y4[i].s2;\n sumf += (float) x4[i].s3 * y4[i].s3;\n }\n float all_sum = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n for (int i = 4*(ne00/4); i < ne00; ++i) {\n all_sum += (float) x[i] * y[i];\n }\n dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;\n }\n }\n\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n// Assumes row size (ne00) is a multiple of 4\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_f16_f32_l4(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n int nrows = ne11;\n int r0 = get_group_id(0);\n int im = get_group_id(2);\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n\n global half4 * x4 = (global half4 *) (src0 + offset_src0);\n\n for (int r1 = 0; r1 < nrows; ++r1) {\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global float4 * y4 = (global float4 *) (src1 + offset_src1);\n\n float sumf = 0;\n for (int i = get_sub_group_local_id(); i < ne00/4; i += get_max_sub_group_size()) {\n sumf += convert_float(x4[i].s0) * y4[i].s0;\n sumf += convert_float(x4[i].s1) * y4[i].s1;\n sumf += convert_float(x4[i].s2) * y4[i].s2;\n sumf += convert_float(x4[i].s3) * y4[i].s3;\n }\n\n float all_sum = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_f32_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define N_F32_F32 4\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_f32_f32(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n int r0 = get_group_id(0);\n int rb = get_group_id(1)*N_F32_F32;\n int im = get_group_id(2);\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n\n global float * x = (global float *) (src0 + offset_src0);\n\n if (ne00 < 128) {\n for (int row = 0; row < N_F32_F32; ++row) {\n int r1 = rb + row;\n if (r1 >= ne11) {\n break;\n }\n\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global float * y = (global float *) (src1 + offset_src1);\n\n float sumf = 0;\n for (int i = get_sub_group_local_id(); i < ne00; i += get_max_sub_group_size()) {\n sumf += (float) x[i] * (float) y[i];\n }\n\n float all_sum = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;\n }\n }\n } else {\n global float4 * x4 = (global float4 *)x;\n for (int row = 0; row < N_F32_F32; ++row) {\n int r1 = rb + row;\n if (r1 >= ne11) {\n break;\n }\n\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global float * y = (global float *) (src1 + offset_src1);\n global float4 * y4 = (global float4 *) y;\n\n float sumf = 0;\n for (int i = get_sub_group_local_id(); i < ne00/4; i += get_max_sub_group_size()) {\n sumf += (float) x4[i].s0 * y4[i].s0;\n sumf += (float) x4[i].s1 * y4[i].s1;\n sumf += (float) x4[i].s2 * y4[i].s2;\n sumf += (float) x4[i].s3 * y4[i].s3;\n }\n\n float all_sum = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n for (int i = 4*(ne00/4); i < ne00; ++i) {\n all_sum += (float) x[i] * y[i];\n }\n dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK_MXFP4 32\ntypedef struct {\n uchar e; // E8M0\n uchar qs[QK_MXFP4/2];\n} block_mxfp4;\n\nconstant static float kvalues_mxfp4_f[16] = {\n 0, .5f, 1.f, 1.5f, 2.f, 3.f, 4.f, 6.f, -0, -.5f, -1.f, -1.5f, -2.f, -3.f, -4.f, -6.f\n};\n\nstatic inline float e8m0_to_fp32(uchar x) {\n int bits;\n\n if (x == 0) {\n bits = 0x00400000;\n } else {\n bits = (uint) x << 23;\n }\n\n return as_float(bits);\n}\n\n#ifdef INTEL_GPU\n#define N_R0_MXFP4 2 // number of rows each subgroup works on\n#define N_SG_MXFP4 2 // number of subgroups in a work group\n#define N_SIMDWIDTH 16 // subgroup size\n#elif defined (ADRENO_GPU)\n#define N_R0_MXFP4 2\n#define N_SG_MXFP4 2\n#define N_SIMDWIDTH 64\n#endif\n\ninline void mul_mv_mxfp4_f32(\n global char * src0,\n global char * src1,\n global char * dst,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3,\n local char * shmem\n) {\n local float * shmem_f32 = (local float *) shmem;\n int nb = ne00/QK_MXFP4;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = 0;\n\n int first_row = (r0 * N_SG_MXFP4 + get_sub_group_id()) * N_R0_MXFP4;\n\n uint i12 = im%ne12;\n uint i13 = im/ne12;\n\n ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global block_mxfp4 * x = (global block_mxfp4 *) (src0 + offset_src0);\n global float * y = (global float *) (src1 + offset_src1);\n\n const short ix = get_sub_group_local_id()/2; // 0...15\n const short it = get_sub_group_local_id()%2; // 0 or 1\n\n shmem_f32[get_sub_group_local_id()] = kvalues_mxfp4_f[get_sub_group_local_id()%16];\n barrier(CLK_LOCAL_MEM_FENCE);\n\n float4 yl[4];\n float sumf[N_R0_MXFP4] = {0.f};\n\n global float * yb = y + ix * QK_MXFP4 + it * 8;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n global float4 * y4 = (global float4 *)yb;\n yl[0] = y4[0];\n yl[1] = y4[4];\n yl[2] = y4[1];\n yl[3] = y4[5];\n\n for (short row = 0; row < N_R0_MXFP4; row++) {\n global block_mxfp4 * xb = x + row*nb + ib;\n global uchar * q2 = (global uchar *)(xb->qs + 8*it);\n\n float4 acc1 = yl[0]*(float4)(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]);\n float4 acc2 = yl[1]*(float4)(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]);\n float4 acc3 = yl[2]*(float4)(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]);\n float4 acc4 = yl[3]*(float4)(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]);\n\n acc1 = (acc1 + acc3) + (acc2 + acc4);\n\n sumf[row] += e8m0_to_fp32(xb->e) * ((acc1.s0 + acc1.s1) + (acc1.s2 + acc1.s3));\n }\n\n yb += (N_SIMDWIDTH/2) * QK_MXFP4;\n }\n\n global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;\n\n for (int row = 0; row < N_R0_MXFP4 && first_row + row < ne0; ++row) {\n float sum_all = sub_group_reduce_add(sumf[row]);\n if (get_sub_group_local_id() == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_id_mxfp4_f32(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * src2,\n ulong offset2,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne11,\n int ne12,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne20,\n int ne21,\n ulong nb21,\n int ne0,\n int ne1,\n int r2,\n int r3,\n local char * shmem\n) {\n src0 = (global char *)((global char *)src0 + offset0);\n src1 = (global char *)((global char *)src1 + offset1);\n src2 = (global char *)((global char *)src2 + offset2);\n dst = (global char *)((global char *)dst + offsetd);\n\n const int iid1 = get_group_id(2)/ne20;\n const int idx = get_group_id(2)%ne20;\n\n int i02 = ((global int *) (src2 + iid1*nb21))[idx];\n\n int i11 = idx % ne11;\n int i12 = iid1;\n\n int i1 = idx;\n int i2 = i12;\n\n global char * src0_cur = src0 + i02*nb02;\n global char * src1_cur = src1 + i11*nb11 + i12*nb12;\n\n global char * dst_cur = dst + (i1*ne0 + i2*ne1*ne0)*sizeof(float);\n\n mul_mv_mxfp4_f32(src0_cur, src1_cur, dst_cur,\n ne00, nb01, nb02, nb03, ne12, nb11, nb12, nb13, ne0, ne1, r2, r3, shmem);\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK_MXFP4 32\n\nstatic inline half4 mxfp4_to_fp16_packed(ushort fp4x4) {\n ushort2 fp16_packed_a, fp16_packed_b, bias_a, bias_b, sign_a, sign_b;\n fp16_packed_a.lo = (fp4x4 << 9) & 0x0E00;\n fp16_packed_a.hi = (fp4x4 << 5) & 0x0E00;\n fp16_packed_b.lo = (fp4x4 << 1) & 0x0E00;\n fp16_packed_b.hi = (fp4x4 >> 3) & 0x0E00;\n\n bias_a.lo = (fp16_packed_a.lo == 0) ? 0x0 : 0x3800;\n bias_a.hi = (fp16_packed_a.hi == 0) ? 0x0 : 0x3800;\n bias_b.lo = (fp16_packed_b.lo == 0) ? 0x0 : 0x3800;\n bias_b.hi = (fp16_packed_b.hi == 0) ? 0x0 : 0x3800;\n\n fp16_packed_a.lo = (fp16_packed_a.lo == 0x0200) ? 0x0 : fp16_packed_a.lo;\n fp16_packed_a.hi = (fp16_packed_a.hi == 0x0200) ? 0x0 : fp16_packed_a.hi;\n fp16_packed_b.lo = (fp16_packed_b.lo == 0x0200) ? 0x0 : fp16_packed_b.lo;\n fp16_packed_b.hi = (fp16_packed_b.hi == 0x0200) ? 0x0 : fp16_packed_b.hi;\n\n sign_a.lo = (fp4x4 << 12) & 0x8000;\n sign_a.hi = (fp4x4 << 8) & 0x8000;\n sign_b.lo = (fp4x4 << 4) & 0x8000;\n sign_b.hi = fp4x4 & 0x8000;\n\n fp16_packed_a = sign_a + bias_a + fp16_packed_a;\n fp16_packed_b = sign_b + bias_b + fp16_packed_b;\n\n return as_half4((ushort4)(fp16_packed_a, fp16_packed_b));\n}\n\nstatic inline float e8m0_to_fp32(uchar x) {\n int bits;\n bits = (x == 0) ? 0x00400000 : ((uint) x << 23);\n return as_float(bits);\n}\n\n#ifdef INTEL_GPU\n#define N_R0_MXFP4 2 // number of rows each subgroup works on\n#define N_SG_MXFP4 2 // number of subgroups in a work group\n#define N_SIMDWIDTH 16 // subgroup size\n#elif defined (ADRENO_GPU)\n#define N_R0_MXFP4 4\n#define N_SG_MXFP4 1\n#define N_SIMDWIDTH 64\n#define SRC0Q_IMG\n#endif\n\nkernel void kernel_mul_mv_id_mxfp4_f32_flat(\n#ifdef SRC0Q_IMG\n __read_only image1d_buffer_t src0_q,\n#else\n global uchar * src0_q,\n#endif\n global uchar * src0_e,\n global uchar * src1,\n ulong offset1,\n global uchar * src2,\n ulong offset2,\n global uchar * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne11,\n int ne12,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne20,\n int ne21,\n ulong nb21,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n dst = dst + offsetd;\n\n const int iid1 = get_group_id(2) / ne20;\n const int idx = get_group_id(2) % ne20;\n\n uint i02 = ((global uint *) (src2 + offset2 + iid1 * nb21))[idx];\n\n int i11 = idx % ne11;\n\n int nb = ne00 / QK_MXFP4;\n\n uint src0_off = i02*nb02;\n src0_off /= 17; // 17 = sizeof(block_mxfp4)\n\n src0_e = src0_e + src0_off;\n\n dst = dst + (idx * ne0 + iid1 * ne1 * ne0) * sizeof(float);\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n\n int first_row = (r0 * N_SG_MXFP4 + get_sub_group_id()) * N_R0_MXFP4;\n\n uint offset_src0 = first_row*nb01;\n offset_src0 /= 17; // 17 = sizeof(block_mxfp4)\n#ifdef SRC0Q_IMG\n ulong offset_q = src0_off + offset_src0;\n#else\n src0_q = src0_q + src0_off*16;\n global uchar16 * x_q = (global uchar16 *)(src0_q) + offset_src0;\n#endif\n global uchar * x_e = src0_e + offset_src0;\n\n const short ix = get_sub_group_local_id() >> 1;\n const short it = get_sub_group_local_id() & 1;\n\n float sumf[N_R0_MXFP4] = {0.f};\n\n src1 = src1 + offset1 + i11 * nb11 + iid1 * nb12;\n global float * y = (global float *) (src1 + r1 * nb11);\n global float * yb = y + ix * QK_MXFP4 + it * 8;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH / 2) {\n global float4 * y4 = (global float4 *)yb;\n\n #pragma unroll\n for (short row = 0; row < N_R0_MXFP4; row++) {\n uchar xb_e = x_e[row * nb + ib];\n#ifdef SRC0Q_IMG\n ushort4 xb_q = as_ushort4(read_imageui(src0_q, (offset_q + row * nb + ib) * 2 + it).xy);\n#else\n ushort4 xb_q = vload4(0, (global ushort *)((global uchar *)(x_q + row * nb + ib) + 8 * it));\n#endif\n\n half4 fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s0);\n half4 fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s1);\n float4 acc1 = y4[0] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);\n acc1 += y4[4] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);\n\n fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s2);\n fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s3);\n acc1 += y4[1] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);\n acc1 += y4[5] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);\n\n sumf[row] += e8m0_to_fp32(xb_e) * ((acc1.s0 + acc1.s1) + (acc1.s2 + acc1.s3));\n }\n\n yb += (N_SIMDWIDTH / 2) * QK_MXFP4;\n }\n\n global float * dst_f32 = (global float *)dst + (ulong)r1 * ne0;\n\n for (int row = 0; row < N_R0_MXFP4 && first_row + row < ne0; ++row) {\n float sum_all = sub_group_reduce_add(sumf[row]);\n if (get_sub_group_local_id() == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_0 32\n\ntypedef char int8_t;\ntypedef uchar uint8_t;\ntypedef short int16_t;\ntypedef ushort uint16_t;\ntypedef int int32_t;\ntypedef uint uint32_t;\n\n//------------------------------------------------------------------------------\n// block_q4_0\n//------------------------------------------------------------------------------\nstruct block_q4_0\n{\n half d;\n uint8_t qs[QK4_0 / 2];\n};\n\n// This function requires the original shuffled weights.\n// As a reminder, the original weights are shuffled so that (q[0], q[16]) are\n// packed together in a byte, so are (q[1], q[17]) and so on.\ninline float block_q_4_0_dot_y_flat(\n global uchar * x,\n global half * dh,\n float sumy,\n float16 yl,\n int il\n) {\n float d = *dh;\n global ushort * qs = ((global ushort *)x + il/2);\n float acc = 0.f;\n\n acc += yl.s0 * (qs[0] & 0x000F);\n acc += yl.s1 * (qs[0] & 0x0F00);\n acc += yl.s8 * (qs[0] & 0x00F0);\n acc += yl.s9 * (qs[0] & 0xF000);\n\n acc += yl.s2 * (qs[1] & 0x000F);\n acc += yl.s3 * (qs[1] & 0x0F00);\n acc += yl.sa * (qs[1] & 0x00F0);\n acc += yl.sb * (qs[1] & 0xF000);\n\n acc += yl.s4 * (qs[2] & 0x000F);\n acc += yl.s5 * (qs[2] & 0x0F00);\n acc += yl.sc * (qs[2] & 0x00F0);\n acc += yl.sd * (qs[2] & 0xF000);\n\n acc += yl.s6 * (qs[3] & 0x000F);\n acc += yl.s7 * (qs[3] & 0x0F00);\n acc += yl.se * (qs[3] & 0x00F0);\n acc += yl.sf * (qs[3] & 0xF000);\n\n return d * (sumy * -8.f + acc);\n}\n\n//\n// This variant outputs 8 values.\n//\n#undef N_DST\n#undef N_SIMDGROUP\n#undef N_SIMDWIDTH\n\n#ifdef INTEL_GPU\n#define N_DST 8 // each SIMD group works on 8 rows\n#define N_SIMDGROUP 1 // number of SIMD groups in a thread group\n#define N_SIMDWIDTH 16 // subgroup size\n#elif defined (ADRENO_GPU)\n#define N_DST 8\n#define N_SIMDGROUP 1\n#define N_SIMDWIDTH 64\n#endif\n\ninline void mul_vec_q_n_f32_8x_flat(\n global char * src0_q,\n global half * src0_d,\n global float * src1,\n global float * dst,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n const ulong nb = ne00/QK4_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = 0;\n\n int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n // The number of scales is the same as the number of blocks.\n ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.\n ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;\n\n global uchar * x = (global uchar *) src0_q + offset0_q;\n global half * d = (global half *) src0_d + offset0_d;\n global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n float16 yl;\n float8 sumf = 0.f;\n\n int ix = get_sub_group_local_id()/2;\n int il = 8*(get_sub_group_local_id()%2);\n\n global float * yb = y + ix*QK4_0 + il;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n float sumy = 0.f;\n\n sumy += yb[0];\n sumy += yb[1];\n sumy += yb[2];\n sumy += yb[3];\n sumy += yb[4];\n sumy += yb[5];\n sumy += yb[6];\n sumy += yb[7];\n\n sumy += yb[16];\n sumy += yb[17];\n sumy += yb[18];\n sumy += yb[19];\n sumy += yb[20];\n sumy += yb[21];\n sumy += yb[22];\n sumy += yb[23];\n\n yl.s0 = yb[0];\n yl.s1 = yb[1]/256.f;\n\n yl.s2 = yb[2];\n yl.s3 = yb[3]/256.f;\n\n yl.s4 = yb[4];\n yl.s5 = yb[5]/256.f;\n\n yl.s6 = yb[6];\n yl.s7 = yb[7]/256.f;\n\n yl.s8 = yb[16]/16.f;\n yl.s9 = yb[17]/4096.f;\n\n yl.sa = yb[18]/16.f;\n yl.sb = yb[19]/4096.f;\n\n yl.sc = yb[20]/16.f;\n yl.sd = yb[21]/4096.f;\n\n yl.se = yb[22]/16.f;\n yl.sf = yb[23]/4096.f;\n\n sumf.s0 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 0*nb*QK4_0/2, d + ib + 0*nb, sumy, yl, il);\n sumf.s1 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 1*nb*QK4_0/2, d + ib + 1*nb, sumy, yl, il);\n sumf.s2 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 2*nb*QK4_0/2, d + ib + 2*nb, sumy, yl, il);\n sumf.s3 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 3*nb*QK4_0/2, d + ib + 3*nb, sumy, yl, il);\n\n sumf.s4 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 4*nb*QK4_0/2, d + ib + 4*nb, sumy, yl, il);\n sumf.s5 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 5*nb*QK4_0/2, d + ib + 5*nb, sumy, yl, il);\n sumf.s6 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 6*nb*QK4_0/2, d + ib + 6*nb, sumy, yl, il);\n sumf.s7 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 7*nb*QK4_0/2, d + ib + 7*nb, sumy, yl, il);\n\n yb += QK4_0 * (N_SIMDWIDTH/2);\n }\n\n float8 tot = (float8)(\n sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),\n sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),\n sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7)\n );\n\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;\n }\n\n if (first_row + 4 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;\n }\n if (first_row + 5 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;\n }\n if (first_row + 6 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;\n }\n if (first_row + 7 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;\n }\n }\n}\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_id_q4_0_f32_8x_flat(\n global char * src0_q,\n global half * src0_d,\n global float * src1,\n ulong offset1,\n global char * src2,\n ulong offset2,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n ulong nb00,\n ulong nb02,\n int ne10,\n int ne11,\n int ne12,\n ulong nb11,\n ulong nb12,\n int ne20,\n int ne21,\n ulong nb21,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src1 = (global float *)((global char *)src1 + offset1);\n src2 = (global char *)((global char *)src2 + offset2);\n dst = (global float *)((global char *)dst + offsetd);\n\n const int iid1 = get_group_id(2)/ne20;\n const int idx = get_group_id(2)%ne20;\n\n const int i02 = ((global int *)(src2 + iid1*nb21))[idx];\n\n const int i11 = idx%ne11;\n const int i12 = iid1;\n\n const int i1 = idx;\n const int i2 = i12;\n\n global char * src0_q_cur = src0_q + (i02*nb02/nb00)*(QK4_0/2);\n global half * src0_d_cur = src0_d + (i02*nb02/nb00);\n global float * src1_cur = (global float *)((global char *) src1 + i11*nb11 + i12*nb12);\n global float * dst_cur = dst + i1*ne0 + i2*ne1*ne0;\n\n mul_vec_q_n_f32_8x_flat(src0_q_cur, src0_d_cur, src1_cur, dst_cur, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK8_0 32\ntypedef struct {\n half d; // delta\n char qs[QK8_0]; // quants\n} block_q8_0;\n\n#define NB_Q8_0 8\n\n#ifdef INTEL_GPU\n#define N_R0_Q8_0 4 // number of rows each subgroup works on\n#define N_SG_Q8_0 2 // number of subgroups in a work group\n#define N_SIMDWIDTH 16 // subgroup size\n#elif defined (ADRENO_GPU)\n#define N_R0_Q8_0 4\n#define N_SG_Q8_0 2\n#define N_SIMDWIDTH 64\n#endif\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_id_q8_0_f32(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * src2,\n ulong offset2,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n ulong nb01,\n ulong nb02,\n int ne11,\n int ne12,\n ulong nb11,\n ulong nb12,\n int ne20,\n int ne21,\n ulong nb21,\n int ne0,\n int ne1\n) {\n src0 = (global char *)((global char *)src0 + offset0);\n src1 = (global char *)((global char *)src1 + offset1);\n src2 = (global char *)((global char *)src2 + offset2);\n dst = (global char *)((global char *)dst + offsetd);\n\n int iid1 = get_group_id(2)/ne20;\n int idx = get_group_id(2)%ne20;\n\n int i02 = ((global int *) (src2 + iid1*nb21))[idx];\n\n int i11_ = idx % ne11;\n int i12_ = iid1;\n\n int i1 = idx;\n int i2 = i12_;\n\n global char * src0_cur = src0 + i02*nb02;\n global char * src1_cur = src1 + i11_*nb11 + i12_*nb12;\n\n global char * dst_cur = dst + (i1*ne0 + i2*ne1*ne0)*sizeof(float);\n\n int nb = ne00/QK8_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n\n int first_row = (r0*N_SG_Q8_0 + get_sub_group_id()) * N_R0_Q8_0;\n\n ulong offset_src1 = r1*nb11;\n global float * y = (global float *) (src1_cur + offset_src1);\n\n // pointers to src0 rows\n global block_q8_0 * ax[N_R0_Q8_0];\n for (int row = 0; row < N_R0_Q8_0; ++row) {\n ulong offset_src0 = (first_row + row)*nb01;\n ax[row] = (global block_q8_0 *) ((global char *) src0_cur + offset_src0);\n }\n\n float yl[NB_Q8_0];\n float sumf[N_R0_Q8_0] = { 0.f };\n\n const short ix = get_sub_group_local_id()/4;\n const short il = get_sub_group_local_id()%4;\n\n global float * yb = y + ix*QK8_0 + il*NB_Q8_0;\n\n // each thread handles NB_Q8_0 quants at a time\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) {\n for (short i = 0; i < NB_Q8_0; ++i) {\n yl[i] = yb[i];\n }\n\n for (short row = 0; row < N_R0_Q8_0; row++) {\n global char * qs = ax[row][ib].qs + il*NB_Q8_0;\n float sumq = 0.f;\n for (short iq = 0; iq < NB_Q8_0; ++iq) {\n sumq += qs[iq] * yl[iq];\n }\n sumf[row] += sumq*ax[row][ib].d;\n }\n\n yb += N_SIMDWIDTH*NB_Q8_0;\n }\n\n global float * dst_f32 = (global float *) dst_cur + (ulong)r1*ne0;\n\n for (int row = 0; row < N_R0_Q8_0; ++row) {\n float tot = sub_group_reduce_add(sumf[row]);\n\n if (get_sub_group_local_id() == 0 && first_row + row < ne01) {\n dst_f32[first_row + row] = tot;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK8_0 32\ntypedef struct {\n half d; // delta\n char qs[QK8_0]; // quants\n} block_q8_0;\n\n#define NB_Q8_0 8\n\n#ifdef INTEL_GPU\n#define N_R0_Q8_0 4 // number of rows each subgroup works on\n#define N_SG_Q8_0 2 // number of subgroups in a work group\n#define N_SIMDWIDTH 16 // subgroup size\n#elif defined (ADRENO_GPU)\n#define N_R0_Q8_0 4\n#define N_SG_Q8_0 2\n#define N_SIMDWIDTH 64\n#endif\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_id_q8_0_f32_flat(\n global char * src0_q,\n global half * src0_d,\n global char * src1,\n ulong offset1,\n global char * src2,\n ulong offset2,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n ulong nb01,\n ulong nb02,\n int ne11,\n int ne12,\n ulong nb11,\n ulong nb12,\n int ne20,\n int ne21,\n ulong nb21,\n int ne0,\n int ne1\n) {\n src1 = (global char *)((global char *)src1 + offset1);\n src2 = (global char *)((global char *)src2 + offset2);\n dst = (global char *)((global char *)dst + offsetd);\n\n int iid1 = (int)get_group_id(2)/ne20;\n int idx = (int)get_group_id(2)%ne20;\n\n int i02 = ((global int *) (src2 + iid1*nb21))[idx];\n\n int i11_ = idx % ne11;\n int i12_ = iid1;\n\n int i1 = idx;\n int i2 = i12_;\n\n // 34 == sizeof(block_q8_0)\n uint src0_off = i02*nb02;\n src0_off /= 34;\n\n global char * src0_q_cur = src0_q + src0_off*sizeof(char)*QK8_0;\n global half * src0_d_cur = src0_d + src0_off;\n global char * src1_cur = src1 + i11_*nb11 + i12_*nb12;\n\n global char * dst_cur = dst + (i1*ne0 + i2*ne1*ne0)*sizeof(float);\n\n int nb = ne00/QK8_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n\n int first_row = (r0*N_SG_Q8_0 + get_sub_group_id()) * N_R0_Q8_0;\n\n ulong offset_src1 = r1*nb11;\n global float * y = (global float *) (src1_cur + offset_src1);\n\n // pointers to src0 rows\n uint offset_src0_base = first_row*nb01;\n\n global char * ax0, * ax1, * ax2, * ax3;\n global half * ad0, * ad1, * ad2, * ad3;\n uint offset_src0;\n\n offset_src0 = offset_src0_base + 0*nb01;\n offset_src0 = offset_src0/34;\n ax0 = (global char *) ((global char *) src0_q_cur + offset_src0*sizeof(char)*QK8_0);\n ad0 = (global half *) ((global char *) src0_d_cur + offset_src0*sizeof(half));\n\n offset_src0 = offset_src0_base + 1*nb01;\n offset_src0 = offset_src0/34;\n ax1 = (global char *) ((global char *) src0_q_cur + offset_src0*sizeof(char)*QK8_0);\n ad1 = (global half *) ((global char *) src0_d_cur + offset_src0*sizeof(half));\n\n offset_src0 = offset_src0_base + 2*nb01;\n offset_src0 = offset_src0/34;\n ax2 = (global char *) ((global char *) src0_q_cur + offset_src0*sizeof(char)*QK8_0);\n ad2 = (global half *) ((global char *) src0_d_cur + offset_src0*sizeof(half));\n\n offset_src0 = offset_src0_base + 3*nb01;\n offset_src0 = offset_src0/34;\n ax3 = (global char *) ((global char *) src0_q_cur + offset_src0*sizeof(char)*QK8_0);\n ad3 = (global half *) ((global char *) src0_d_cur + offset_src0*sizeof(half));\n\n const short ix = get_sub_group_local_id()/4;\n const short il = get_sub_group_local_id()%4;\n\n global float * yb = y + ix*QK8_0 + il*NB_Q8_0;\n\n float8 yl;\n float8 qv;\n float4 sumf = 0.f;\n float sumq = 0.f;\n global char * qs;\n\n // each thread handles NB_Q8_0 quants at a time\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) {\n yl = vload8(0, yb);\n\n qs = ax0 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;\n qv = convert_float8(vload8(0, qs));\n sumq = 0;\n sumq += qv.s0*yl.s0;\n sumq += qv.s1*yl.s1;\n sumq += qv.s2*yl.s2;\n sumq += qv.s3*yl.s3;\n sumq += qv.s4*yl.s4;\n sumq += qv.s5*yl.s5;\n sumq += qv.s6*yl.s6;\n sumq += qv.s7*yl.s7;\n sumf.s0 += sumq*ad0[ib];\n\n qs = ax1 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;\n qv = convert_float8(vload8(0, qs));\n sumq = 0;\n sumq += qv.s0*yl.s0;\n sumq += qv.s1*yl.s1;\n sumq += qv.s2*yl.s2;\n sumq += qv.s3*yl.s3;\n sumq += qv.s4*yl.s4;\n sumq += qv.s5*yl.s5;\n sumq += qv.s6*yl.s6;\n sumq += qv.s7*yl.s7;\n sumf.s1 += sumq*ad1[ib];\n\n qs = ax2 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;\n qv = convert_float8(vload8(0, qs));\n sumq = 0;\n sumq += qv.s0*yl.s0;\n sumq += qv.s1*yl.s1;\n sumq += qv.s2*yl.s2;\n sumq += qv.s3*yl.s3;\n sumq += qv.s4*yl.s4;\n sumq += qv.s5*yl.s5;\n sumq += qv.s6*yl.s6;\n sumq += qv.s7*yl.s7;\n sumf.s2 += sumq*ad2[ib];\n\n qs = ax3 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;\n qv = convert_float8(vload8(0, qs));\n sumq = 0;\n sumq += qv.s0*yl.s0;\n sumq += qv.s1*yl.s1;\n sumq += qv.s2*yl.s2;\n sumq += qv.s3*yl.s3;\n sumq += qv.s4*yl.s4;\n sumq += qv.s5*yl.s5;\n sumq += qv.s6*yl.s6;\n sumq += qv.s7*yl.s7;\n sumf.s3 += sumq*ad3[ib];\n\n yb += N_SIMDWIDTH*NB_Q8_0;\n }\n\n global float * dst_f32 = (global float *) dst_cur + (ulong)r1*ne0;\n\n float4 tot = (float4)(\n sub_group_reduce_add(sumf.s0),\n sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2),\n sub_group_reduce_add(sumf.s3)\n );\n\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst_f32[first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst_f32[first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst_f32[first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst_f32[first_row + 3] = tot.s3;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK_MXFP4 32\ntypedef struct {\n uchar e; // E8M0\n uchar qs[QK_MXFP4/2];\n} block_mxfp4;\n\nconstant static float kvalues_mxfp4_f[16] = {\n 0, .5f, 1.f, 1.5f, 2.f, 3.f, 4.f, 6.f, -0, -.5f, -1.f, -1.5f, -2.f, -3.f, -4.f, -6.f\n};\n\nstatic inline float e8m0_to_fp32(uchar x) {\n int bits;\n\n if (x == 0) {\n bits = 0x00400000;\n } else {\n bits = (uint) x << 23;\n }\n\n return as_float(bits);\n}\n\n#ifdef INTEL_GPU\n#define N_R0_MXFP4 2 // number of rows each subgroup works on\n#define N_SG_MXFP4 2 // number of subgroups in a work group\n#define N_SIMDWIDTH 16 // subgroup size\n#elif defined (ADRENO_GPU)\n#define N_R0_MXFP4 2\n#define N_SG_MXFP4 2\n#define N_SIMDWIDTH 64\n#endif\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_mxfp4_f32(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3,\n local char * shmem\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n local float * shmem_f32 = (local float *) shmem;\n int nb = ne00/QK_MXFP4;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n int first_row = (r0 * N_SG_MXFP4 + get_sub_group_id()) * N_R0_MXFP4;\n\n uint i12 = im%ne12;\n uint i13 = im/ne12;\n\n ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global block_mxfp4 * x = (global block_mxfp4 *) (src0 + offset_src0);\n global float * y = (global float *) (src1 + offset_src1);\n\n const short ix = get_sub_group_local_id()/2; // 0...15\n const short it = get_sub_group_local_id()%2; // 0 or 1\n\n shmem_f32[get_sub_group_local_id()] = kvalues_mxfp4_f[get_sub_group_local_id()%16];\n barrier(CLK_LOCAL_MEM_FENCE);\n\n float4 yl[4];\n float sumf[N_R0_MXFP4] = {0.f};\n\n global float * yb = y + ix * QK_MXFP4 + it * 8;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n global float4 * y4 = (global float4 *)yb;\n yl[0] = y4[0];\n yl[1] = y4[4];\n yl[2] = y4[1];\n yl[3] = y4[5];\n\n for (short row = 0; row < N_R0_MXFP4; row++) {\n global block_mxfp4 * xb = x + row*nb + ib;\n global uchar * q2 = (global uchar *)(xb->qs + 8*it);\n\n float4 acc1 = yl[0]*(float4)(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]);\n float4 acc2 = yl[1]*(float4)(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]);\n float4 acc3 = yl[2]*(float4)(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]);\n float4 acc4 = yl[3]*(float4)(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]);\n\n acc1 = (acc1 + acc3) + (acc2 + acc4);\n\n sumf[row] += e8m0_to_fp32(xb->e) * ((acc1.s0 + acc1.s1) + (acc1.s2 + acc1.s3));\n }\n\n yb += (N_SIMDWIDTH/2) * QK_MXFP4;\n }\n\n global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;\n\n for (int row = 0; row < N_R0_MXFP4 && first_row + row < ne0; ++row) {\n float sum_all = sub_group_reduce_add(sumf[row]);\n if (get_sub_group_local_id() == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK_MXFP4 32\n\nstatic inline half4 mxfp4_to_fp16_packed(ushort fp4x4) {\n ushort2 fp16_packed_a, fp16_packed_b, bias_a, bias_b, sign_a, sign_b;\n fp16_packed_a.lo = (fp4x4 << 9) & 0x0E00;\n fp16_packed_a.hi = (fp4x4 << 5) & 0x0E00;\n fp16_packed_b.lo = (fp4x4 << 1) & 0x0E00;\n fp16_packed_b.hi = (fp4x4 >> 3) & 0x0E00;\n\n bias_a.lo = (fp16_packed_a.lo == 0) ? 0x0 : 0x3800;\n bias_a.hi = (fp16_packed_a.hi == 0) ? 0x0 : 0x3800;\n bias_b.lo = (fp16_packed_b.lo == 0) ? 0x0 : 0x3800;\n bias_b.hi = (fp16_packed_b.hi == 0) ? 0x0 : 0x3800;\n\n fp16_packed_a.lo = (fp16_packed_a.lo == 0x0200) ? 0x0 : fp16_packed_a.lo;\n fp16_packed_a.hi = (fp16_packed_a.hi == 0x0200) ? 0x0 : fp16_packed_a.hi;\n fp16_packed_b.lo = (fp16_packed_b.lo == 0x0200) ? 0x0 : fp16_packed_b.lo;\n fp16_packed_b.hi = (fp16_packed_b.hi == 0x0200) ? 0x0 : fp16_packed_b.hi;\n\n sign_a.lo = (fp4x4 << 12) & 0x8000;\n sign_a.hi = (fp4x4 << 8) & 0x8000;\n sign_b.lo = (fp4x4 << 4) & 0x8000;\n sign_b.hi = fp4x4 & 0x8000;\n\n fp16_packed_a = sign_a + bias_a + fp16_packed_a;\n fp16_packed_b = sign_b + bias_b + fp16_packed_b;\n\n return as_half4((ushort4)(fp16_packed_a, fp16_packed_b));\n}\n\nstatic inline float e8m0_to_fp32(uchar x) {\n int bits;\n bits = (x == 0) ? 0x00400000 : ((uint) x << 23);\n return as_float(bits);\n}\n\n#ifdef INTEL_GPU\n#define N_R0_MXFP4 2 // number of rows each subgroup works on\n#define N_SG_MXFP4 2 // number of subgroups in a work group\n#define N_SIMDWIDTH 16 // subgroup size\n#elif defined (ADRENO_GPU)\n#define N_R0_MXFP4 2\n#define N_SG_MXFP4 2\n#define N_SIMDWIDTH 64\n#define SRC0Q_IMG\n#endif\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_mxfp4_f32_flat(\n#ifdef SRC0Q_IMG\n __read_only image1d_buffer_t src0_q,\n#else\n global uchar * src0_q,\n#endif\n global uchar * src0_e,\n global uchar * src1,\n ulong offset1,\n global uchar * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int nb = ne00 / QK_MXFP4;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n int first_row = (r0 * N_SG_MXFP4 + get_sub_group_id()) * N_R0_MXFP4;\n\n uint i12 = im % ne12;\n uint i13 = im / ne12;\n\n uint offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n // 17 = sizeof(block_mxfp4)\n offset_src0 /= 17;\n#ifdef SRC0Q_IMG\n ulong offset_q = offset_src0;\n#else\n global uchar16 * x_q = (global uchar16 *)(src0_q) + offset_src0;\n#endif\n global uchar * x_e = src0_e + offset_src0;\n\n ulong offset_src1 = r1 * nb11 + i12 * nb12 + i13 * nb13;\n global float * y = (global float *)(src1 + offset_src1);\n\n const short ix = get_sub_group_local_id() >> 1; // 0...15\n const short it = get_sub_group_local_id() & 1; // 0 or 1\n\n float sumf[N_R0_MXFP4] = {0.f};\n\n global float * yb = y + ix * QK_MXFP4 + it * 8;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n global float4 * y4 = (global float4 *)yb;\n\n #pragma unroll\n for (short row = 0; row < N_R0_MXFP4; row++) {\n uchar xb_e = x_e[row * nb + ib];\n#ifdef SRC0Q_IMG\n ushort4 xb_q = as_ushort4(read_imageui(src0_q, (offset_q + row * nb + ib) * 2 + it).xy);\n#else\n ushort4 xb_q = vload4(0, (global ushort *)((global uchar *)(x_q + row * nb + ib) + 8 * it));\n#endif\n\n half4 fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s0);\n half4 fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s1);\n float4 acc1 = y4[0] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);\n acc1 += y4[4] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);\n\n fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s2);\n fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s3);\n acc1 += y4[1] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);\n acc1 += y4[5] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);\n\n sumf[row] += e8m0_to_fp32(xb_e) * ((acc1.s0 + acc1.s1) + (acc1.s2 + acc1.s3));\n }\n\n yb += (N_SIMDWIDTH/2) * QK_MXFP4;\n }\n\n global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;\n\n for (int row = 0; row < N_R0_MXFP4 && first_row + row < ne0; ++row) {\n float sum_all = sub_group_reduce_add(sumf[row]);\n if (get_sub_group_local_id() == 0) {\n dst_f32[first_row + row] = sum_all;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_0 32\n#define QR4_0 2\n#define QK4_1 32\n#define QR4_1 2\n#define QK5_0 32\n#define QR5_0 2\n#define QK5_1 32\n#define QR5_1 2\n#define QK8_0 32\n#define QR8_0 1\n#define QK_K 256\n#define K_QUANTS_PER_ITERATION 2\n\ntypedef char int8_t;\ntypedef uchar uint8_t;\ntypedef short int16_t;\ntypedef ushort uint16_t;\ntypedef int int32_t;\ntypedef uint uint32_t;\n\n//------------------------------------------------------------------------------\n// block_q4_0\n//------------------------------------------------------------------------------\nstruct block_q4_0\n{\n half d;\n uint8_t qs[QK4_0 / 2];\n};\n\n//------------------------------------------------------------------------------\n// mul_vec_q_n_f32\n//------------------------------------------------------------------------------\n// function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i])\n// il indicates where the q4 quants begin (0 or QK4_0/4)\n// we assume that the yl's have been multiplied with the appropriate scale factor\n// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)\ninline float block_q_4_0_dot_y(\n global struct block_q4_0 * qb_curr,\n float sumy,\n private float * yl,\n int il\n) {\n float d = qb_curr->d;\n float2 acc = 0.f;\n global ushort * qs = ((global ushort *)qb_curr + 1 + il/2);\n for (int i = 0; i < 8; i+=2) {\n acc.s0 += yl[i + 0] * (qs[i / 2] & 0x000F)\n + yl[i + 1] * (qs[i / 2] & 0x0F00);\n acc.s1 += yl[i + 8] * (qs[i / 2] & 0x00F0)\n + yl[i + 9] * (qs[i / 2] & 0xF000);\n }\n return d * (sumy * -8.f + acc.s0 + acc.s1);\n}\n\n#ifdef INTEL_GPU\n#define N_DST 4 // each SIMD group works on 4 rows\n#define N_SIMDGROUP 1 // number of SIMD groups in a thread group\n#define N_SIMDWIDTH 16 // assuming SIMD group size is 16\n#elif defined (ADRENO_GPU)\n#define N_DST 4\n#define N_SIMDGROUP 1\n#define N_SIMDWIDTH 64\n#endif\n\ninline void mul_vec_q_n_f32(\n global void * src0,\n global float * src1,\n global float * dst,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n\n const ulong nb = ne00/QK4_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n // (r0 * N_SIMDGROUP + get_sub_group_id()) is essenatially the linear global\n // id of a SIMD group in the grid.\n int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n\n global struct block_q4_0 * x = (global struct block_q4_0 *) src0 + offset0;\n global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n float yl[16]; // src1 vector cache\n float sumf[N_DST]={0.f};\n\n int ix = get_sub_group_local_id()/2;\n int il = 8*(get_sub_group_local_id()%2);\n\n global float * yb = y + ix * QK4_0 + il;\n\n // each thread in a SIMD group deals with half a block.\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n float sumy = 0;\n for (int i = 0; i < 8; i += 2) {\n sumy += yb[i] + yb[i+1];\n yl[i+0] = yb[i+ 0];\n yl[i+1] = yb[i+ 1]/256.f;\n sumy += yb[i+16] + yb[i+17];\n yl[i+8] = yb[i+16]/16.f;\n yl[i+9] = yb[i+17]/4096.f;\n }\n\n for (int row = 0; row < N_DST; row++) {\n sumf[row] += block_q_4_0_dot_y(x+ib+row*nb, sumy, yl, il);\n }\n\n // One thread in a SIMD group (i.e., subgroup) handles a half block,\n // hence then entire SIMD group handles SIMDWIDTH/2 blocks.\n // y points to the activation matrix (of type float). Therefore for\n // one thread, the # of blocks y should advance is SIMDWIDTH/2 (because\n // SIMDWIDTH/2 blocks are processed by a SIMD group) - in terms of\n // floats, it is QK4_0 * (SIMDWIDTH/2), where QK4_0 is the block size.\n yb += QK4_0 * (N_SIMDWIDTH/2);\n }\n\n // The above does not work for Adreno - it produces incorrect results for\n // row = 1, 2, 3 and only row = 0 gives the correct result.\n // If N_DST is changed, the below array must be initialized accordingly.\n // This also seems to perform better on Intel.\n float tot[N_DST] = {\n sub_group_reduce_add(sumf[0]), sub_group_reduce_add(sumf[1]),\n sub_group_reduce_add(sumf[2]), sub_group_reduce_add(sumf[3])};\n for (int row = 0; row < N_DST; ++row) {\n if (get_sub_group_local_id() == 0 && first_row + row < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot[row];\n }\n }\n}\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_q4_0_f32(\n global void * src0,\n ulong offset0,\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_0 32\n#define QR4_0 2\n#define QK4_1 32\n#define QR4_1 2\n#define QK5_0 32\n#define QR5_0 2\n#define QK5_1 32\n#define QR5_1 2\n#define QK8_0 32\n#define QR8_0 1\n#define QK_K 256\n#define K_QUANTS_PER_ITERATION 2\n\ntypedef char int8_t;\ntypedef uchar uint8_t;\ntypedef short int16_t;\ntypedef ushort uint16_t;\ntypedef int int32_t;\ntypedef uint uint32_t;\n\n//------------------------------------------------------------------------------\n// block_q4_0\n//------------------------------------------------------------------------------\nstruct block_q4_0\n{\n half d;\n uint8_t qs[QK4_0 / 2];\n};\n\ninline float mm_block_q_4_0_dot_y_flat(\n global uchar * x,\n global half * dh,\n float sumy,\n float16 yl,\n int il\n) {\n float d = *dh;\n global ushort * qs = ((global ushort *)x + il/2);\n float acc = 0.f;\n\n acc += yl.s0 * (qs[0] & 0x000F);\n acc += yl.s1 * (qs[0] & 0x0F00);\n acc += yl.s8 * (qs[0] & 0x00F0);\n acc += yl.s9 * (qs[0] & 0xF000);\n\n acc += yl.s2 * (qs[1] & 0x000F);\n acc += yl.s3 * (qs[1] & 0x0F00);\n acc += yl.sa * (qs[1] & 0x00F0);\n acc += yl.sb * (qs[1] & 0xF000);\n\n acc += yl.s4 * (qs[2] & 0x000F);\n acc += yl.s5 * (qs[2] & 0x0F00);\n acc += yl.sc * (qs[2] & 0x00F0);\n acc += yl.sd * (qs[2] & 0xF000);\n\n acc += yl.s6 * (qs[3] & 0x000F);\n acc += yl.s7 * (qs[3] & 0x0F00);\n acc += yl.se * (qs[3] & 0x00F0);\n acc += yl.sf * (qs[3] & 0xF000);\n\n return d * (sumy * -8.f + acc);\n}\n\n#ifdef INTEL_GPU\n#define N_DST 16 // each SIMD group works on 8 rows (in weights matrix)\n#define N_SIMDGROUP 1 // number of SIMD groups in a thread group\n#define N_SIMDWIDTH 16 // assuming SIMD group size is 16\n#elif defined (ADRENO_GPU)\n#define N_DST 16\n#define N_SIMDGROUP 1\n#define N_SIMDWIDTH 64\n#endif\n//\n// This variant performs 1d blocking with 16x output.\n// Eeach simdgroup outputs 16 values on `n0` dim (row in the output matrix).\n//\ninline void mul_mat_q_n_f32_1d_16x_flat(\n global uchar * src0_q,\n global half * src0_d,\n global float * src1,\n global float * dst,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n const int nb = ne00/QK4_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n // (r0 * N_SIMDGROUP + get_sub_group_id()) is the linear global id of\n // a SIMD group in the grid. Each SIMD group produces N_DST values in the\n // result, hence uses nb blocks, i.e., the offset becomes first_row*nb.\n // Currently with llama2 7B, im is always 0.\n // TODO: how to handle im/gqa*(nb*ne0)?\n int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n // The number of scales is the same as the number of blocks.\n ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.\n ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;\n\n global uchar * x = (global uchar *) src0_q + offset0_q;\n global half * d = (global half *) src0_d + offset0_d;\n global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n float16 yl;\n float16 sumf = (float16)(0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f,\n 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f);\n\n int ix = get_sub_group_local_id()/2;\n int il = 8*(get_sub_group_local_id()%2);\n\n global float * yb = y + ix*QK4_0 + il;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n float sumy = 0.f;\n\n sumy += yb[0];\n sumy += yb[1];\n sumy += yb[2];\n sumy += yb[3];\n sumy += yb[4];\n sumy += yb[5];\n sumy += yb[6];\n sumy += yb[7];\n\n sumy += yb[16];\n sumy += yb[17];\n sumy += yb[18];\n sumy += yb[19];\n sumy += yb[20];\n sumy += yb[21];\n sumy += yb[22];\n sumy += yb[23];\n\n yl.s0 = yb[0];\n yl.s1 = yb[1]/256.f;\n\n yl.s2 = yb[2];\n yl.s3 = yb[3]/256.f;\n\n yl.s4 = yb[4];\n yl.s5 = yb[5]/256.f;\n\n yl.s6 = yb[6];\n yl.s7 = yb[7]/256.f;\n\n yl.s8 = yb[16]/16.f;\n yl.s9 = yb[17]/4096.f;\n\n yl.sa = yb[18]/16.f;\n yl.sb = yb[19]/4096.f;\n\n yl.sc = yb[20]/16.f;\n yl.sd = yb[21]/4096.f;\n\n yl.se = yb[22]/16.f;\n yl.sf = yb[23]/4096.f;\n\n sumf.s0 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 0*nb*QK4_0/2, d + ib + 0*nb, sumy, yl, il);\n sumf.s1 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 1*nb*QK4_0/2, d + ib + 1*nb, sumy, yl, il);\n sumf.s2 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 2*nb*QK4_0/2, d + ib + 2*nb, sumy, yl, il);\n sumf.s3 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 3*nb*QK4_0/2, d + ib + 3*nb, sumy, yl, il);\n\n sumf.s4 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 4*nb*QK4_0/2, d + ib + 4*nb, sumy, yl, il);\n sumf.s5 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 5*nb*QK4_0/2, d + ib + 5*nb, sumy, yl, il);\n sumf.s6 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 6*nb*QK4_0/2, d + ib + 6*nb, sumy, yl, il);\n sumf.s7 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 7*nb*QK4_0/2, d + ib + 7*nb, sumy, yl, il);\n\n sumf.s8 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 8*nb*QK4_0/2, d + ib + 8*nb, sumy, yl, il);\n sumf.s9 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 9*nb*QK4_0/2, d + ib + 9*nb, sumy, yl, il);\n sumf.sa += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 10*nb*QK4_0/2, d + ib + 10*nb, sumy, yl, il);\n sumf.sb += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 11*nb*QK4_0/2, d + ib + 11*nb, sumy, yl, il);\n\n sumf.sc += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 12*nb*QK4_0/2, d + ib + 12*nb, sumy, yl, il);\n sumf.sd += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 13*nb*QK4_0/2, d + ib + 13*nb, sumy, yl, il);\n sumf.se += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 14*nb*QK4_0/2, d + ib + 14*nb, sumy, yl, il);\n sumf.sf += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 15*nb*QK4_0/2, d + ib + 15*nb, sumy, yl, il);\n\n yb += QK4_0 * (N_SIMDWIDTH/2);\n }\n\n float16 tot = (float16)(\n sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),\n sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),\n sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7),\n\n sub_group_reduce_add(sumf.s8), sub_group_reduce_add(sumf.s9),\n sub_group_reduce_add(sumf.sa), sub_group_reduce_add(sumf.sb),\n sub_group_reduce_add(sumf.sc), sub_group_reduce_add(sumf.sd),\n sub_group_reduce_add(sumf.se), sub_group_reduce_add(sumf.sf)\n );\n\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;\n }\n\n if (first_row + 4 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;\n }\n if (first_row + 5 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;\n }\n if (first_row + 6 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;\n }\n if (first_row + 7 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;\n }\n\n if (first_row + 8 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 8] = tot.s8;\n }\n if (first_row + 9 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 9] = tot.s9;\n }\n if (first_row + 10 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 10] = tot.sa;\n }\n if (first_row + 11 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 11] = tot.sb;\n }\n\n if (first_row + 12 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 12] = tot.sc;\n }\n if (first_row + 13 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 13] = tot.sd;\n }\n if (first_row + 14 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 14] = tot.se;\n }\n if (first_row + 15 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 15] = tot.sf;\n }\n }\n}\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_q4_0_f32_1d_16x_flat(\n global uchar * src0_q,\n global half * src0_d,\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n mul_mat_q_n_f32_1d_16x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_0 32\n#define QR4_0 2\n#define QK4_1 32\n#define QR4_1 2\n#define QK5_0 32\n#define QR5_0 2\n#define QK5_1 32\n#define QR5_1 2\n#define QK8_0 32\n#define QR8_0 1\n#define QK_K 256\n#define K_QUANTS_PER_ITERATION 2\n\ntypedef char int8_t;\ntypedef uchar uint8_t;\ntypedef short int16_t;\ntypedef ushort uint16_t;\ntypedef int int32_t;\ntypedef uint uint32_t;\n\n//------------------------------------------------------------------------------\n// block_q4_0\n//------------------------------------------------------------------------------\nstruct block_q4_0\n{\n half d;\n uint8_t qs[QK4_0 / 2];\n};\n\ninline float mm_block_q_4_0_dot_y_flat(\n global uchar * x,\n global half * dh,\n float sumy,\n float16 yl,\n int il\n) {\n float d = *dh;\n global ushort * qs = ((global ushort *)x + il/2);\n float acc = 0.f;\n\n acc += yl.s0 * (qs[0] & 0x000F);\n acc += yl.s1 * (qs[0] & 0x0F00);\n acc += yl.s8 * (qs[0] & 0x00F0);\n acc += yl.s9 * (qs[0] & 0xF000);\n\n acc += yl.s2 * (qs[1] & 0x000F);\n acc += yl.s3 * (qs[1] & 0x0F00);\n acc += yl.sa * (qs[1] & 0x00F0);\n acc += yl.sb * (qs[1] & 0xF000);\n\n acc += yl.s4 * (qs[2] & 0x000F);\n acc += yl.s5 * (qs[2] & 0x0F00);\n acc += yl.sc * (qs[2] & 0x00F0);\n acc += yl.sd * (qs[2] & 0xF000);\n\n acc += yl.s6 * (qs[3] & 0x000F);\n acc += yl.s7 * (qs[3] & 0x0F00);\n acc += yl.se * (qs[3] & 0x00F0);\n acc += yl.sf * (qs[3] & 0xF000);\n\n return d * (sumy * -8.f + acc);\n}\n\n#ifdef INTEL_GPU\n#define N_DST 8 // each SIMD group works on 8 rows (in weights matrix)\n#define N_SIMDGROUP 1 // number of SIMD groups in a thread group\n#define N_SIMDWIDTH 16 // assuming SIMD group size is 16\n#elif defined (ADRENO_GPU)\n#define N_DST 8\n#define N_SIMDGROUP 1\n#define N_SIMDWIDTH 64\n#endif\n//\n// This variant performs 1d blocking with 8x output.\n// Eeach simdgroup outputs 8 values on `n0` dim (row in the output matrix).\n//\ninline void mul_mat_q_n_f32_1d_8x_flat(\n global uchar * src0_q,\n global half * src0_d,\n global float * src1,\n global float * dst,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n const int nb = ne00/QK4_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n // (r0 * N_SIMDGROUP + get_sub_group_id()) is the linear global id of\n // a SIMD group in the grid. Each SIMD group produces N_DST values in the\n // result, hence uses nb blocks, i.e., the offset becomes first_row*nb.\n // Currently with llama2 7B, im is always 0.\n // TODO: how to handle im/gqa*(nb*ne0)?\n int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n // The number of scales is the same as the number of blocks.\n ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.\n ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;\n\n global uchar * x = (global uchar *) src0_q + offset0_q;\n global half * d = (global half *) src0_d + offset0_d;\n global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n float16 yl;\n float8 sumf = (float8)(0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f);\n\n int ix = get_sub_group_local_id()/2;\n int il = 8*(get_sub_group_local_id()%2);\n\n global float * yb = y + ix*QK4_0 + il;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n float sumy = 0.f;\n\n sumy += yb[0];\n sumy += yb[1];\n sumy += yb[2];\n sumy += yb[3];\n sumy += yb[4];\n sumy += yb[5];\n sumy += yb[6];\n sumy += yb[7];\n\n sumy += yb[16];\n sumy += yb[17];\n sumy += yb[18];\n sumy += yb[19];\n sumy += yb[20];\n sumy += yb[21];\n sumy += yb[22];\n sumy += yb[23];\n\n yl.s0 = yb[0];\n yl.s1 = yb[1]/256.f;\n\n yl.s2 = yb[2];\n yl.s3 = yb[3]/256.f;\n\n yl.s4 = yb[4];\n yl.s5 = yb[5]/256.f;\n\n yl.s6 = yb[6];\n yl.s7 = yb[7]/256.f;\n\n yl.s8 = yb[16]/16.f;\n yl.s9 = yb[17]/4096.f;\n\n yl.sa = yb[18]/16.f;\n yl.sb = yb[19]/4096.f;\n\n yl.sc = yb[20]/16.f;\n yl.sd = yb[21]/4096.f;\n\n yl.se = yb[22]/16.f;\n yl.sf = yb[23]/4096.f;\n\n sumf.s0 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 0*nb*QK4_0/2, d + ib + 0*nb, sumy, yl, il);\n sumf.s1 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 1*nb*QK4_0/2, d + ib + 1*nb, sumy, yl, il);\n sumf.s2 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 2*nb*QK4_0/2, d + ib + 2*nb, sumy, yl, il);\n sumf.s3 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 3*nb*QK4_0/2, d + ib + 3*nb, sumy, yl, il);\n\n sumf.s4 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 4*nb*QK4_0/2, d + ib + 4*nb, sumy, yl, il);\n sumf.s5 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 5*nb*QK4_0/2, d + ib + 5*nb, sumy, yl, il);\n sumf.s6 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 6*nb*QK4_0/2, d + ib + 6*nb, sumy, yl, il);\n sumf.s7 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 7*nb*QK4_0/2, d + ib + 7*nb, sumy, yl, il);\n\n yb += QK4_0 * (N_SIMDWIDTH/2);\n }\n\n float8 tot = (float8)(\n sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),\n sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),\n sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7)\n );\n\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;\n }\n\n if (first_row + 4 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;\n }\n if (first_row + 5 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;\n }\n if (first_row + 6 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;\n }\n if (first_row + 7 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;\n }\n }\n}\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_q4_0_f32_1d_8x_flat(\n global uchar * src0_q,\n global half * src0_d,\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n mul_mat_q_n_f32_1d_8x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_0 32\n#define QR4_0 2\n#define QK4_1 32\n#define QR4_1 2\n#define QK5_0 32\n#define QR5_0 2\n#define QK5_1 32\n#define QR5_1 2\n#define QK8_0 32\n#define QR8_0 1\n#define QK_K 256\n#define K_QUANTS_PER_ITERATION 2\n\ntypedef char int8_t;\ntypedef uchar uint8_t;\ntypedef short int16_t;\ntypedef ushort uint16_t;\ntypedef int int32_t;\ntypedef uint uint32_t;\n\n//------------------------------------------------------------------------------\n// block_q4_0\n//------------------------------------------------------------------------------\nstruct block_q4_0\n{\n half d;\n uint8_t qs[QK4_0 / 2];\n};\n\n// This function requires the original shuffled weights.\n// As a reminder, the original weights are shuffled so that (q[0], q[16]) are\n// packed together in a byte, so are (q[1], q[17]) and so on.\ninline float block_q_4_0_dot_y_flat(\n global uchar * x,\n global half * dh,\n float sumy,\n float16 yl,\n int il\n) {\n float d = *dh;\n global ushort * qs = ((global ushort *)x + il/2);\n float acc = 0.f;\n\n acc += yl.s0 * (qs[0] & 0x000F);\n acc += yl.s1 * (qs[0] & 0x0F00);\n acc += yl.s8 * (qs[0] & 0x00F0);\n acc += yl.s9 * (qs[0] & 0xF000);\n\n acc += yl.s2 * (qs[1] & 0x000F);\n acc += yl.s3 * (qs[1] & 0x0F00);\n acc += yl.sa * (qs[1] & 0x00F0);\n acc += yl.sb * (qs[1] & 0xF000);\n\n acc += yl.s4 * (qs[2] & 0x000F);\n acc += yl.s5 * (qs[2] & 0x0F00);\n acc += yl.sc * (qs[2] & 0x00F0);\n acc += yl.sd * (qs[2] & 0xF000);\n\n acc += yl.s6 * (qs[3] & 0x000F);\n acc += yl.s7 * (qs[3] & 0x0F00);\n acc += yl.se * (qs[3] & 0x00F0);\n acc += yl.sf * (qs[3] & 0xF000);\n\n return d * (sumy * -8.f + acc);\n}\n\n//\n// This variant outputs 8 values.\n//\n#undef N_DST\n#undef N_SIMDGROUP\n#undef N_SIMDWIDTH\n\n#ifdef INTEL_GPU\n#define N_DST 8 // each SIMD group works on 8 rows\n#define N_SIMDGROUP 1 // number of SIMD groups in a thread group\n#define N_SIMDWIDTH 16 // assuming SIMD group size is 32\n#elif defined (ADRENO_GPU)\n#define N_DST 8\n#define N_SIMDGROUP 1\n#define N_SIMDWIDTH 64\n#endif\n\ninline void mul_vec_q_n_f32_8x_flat(\n global uchar * src0_q,\n global half * src0_d,\n global float * src1,\n global float * dst,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n const ulong nb = ne00/QK4_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n // (r0 * N_SIMDGROUP + get_sub_group_id()) is the linear global id of\n // a SIMD group in the grid. Each SIMD group produces N_DST values in the\n // result, hence uses nb blocks, i.e., the offset becomes first_row*nb.\n // Currently with llama2 7B, im is always 0.\n // TODO: how to handle im/gqa*(nb*ne0)?\n int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n // The number of scales is the same as the number of blocks.\n ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.\n ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;\n\n global uchar * x = (global uchar *) src0_q + offset0_q;\n global half * d = (global half *) src0_d + offset0_d;\n global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n float16 yl;\n float8 sumf = 0.f;\n\n int ix = get_sub_group_local_id()/2;\n int il = 8*(get_sub_group_local_id()%2);\n\n global float * yb = y + ix*QK4_0 + il;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n float sumy = 0.f;\n\n sumy += yb[0];\n sumy += yb[1];\n sumy += yb[2];\n sumy += yb[3];\n sumy += yb[4];\n sumy += yb[5];\n sumy += yb[6];\n sumy += yb[7];\n\n sumy += yb[16];\n sumy += yb[17];\n sumy += yb[18];\n sumy += yb[19];\n sumy += yb[20];\n sumy += yb[21];\n sumy += yb[22];\n sumy += yb[23];\n\n yl.s0 = yb[0];\n yl.s1 = yb[1]/256.f;\n\n yl.s2 = yb[2];\n yl.s3 = yb[3]/256.f;\n\n yl.s4 = yb[4];\n yl.s5 = yb[5]/256.f;\n\n yl.s6 = yb[6];\n yl.s7 = yb[7]/256.f;\n\n yl.s8 = yb[16]/16.f;\n yl.s9 = yb[17]/4096.f;\n\n yl.sa = yb[18]/16.f;\n yl.sb = yb[19]/4096.f;\n\n yl.sc = yb[20]/16.f;\n yl.sd = yb[21]/4096.f;\n\n yl.se = yb[22]/16.f;\n yl.sf = yb[23]/4096.f;\n\n sumf.s0 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 0*nb*QK4_0/2, d + ib + 0*nb, sumy, yl, il);\n sumf.s1 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 1*nb*QK4_0/2, d + ib + 1*nb, sumy, yl, il);\n sumf.s2 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 2*nb*QK4_0/2, d + ib + 2*nb, sumy, yl, il);\n sumf.s3 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 3*nb*QK4_0/2, d + ib + 3*nb, sumy, yl, il);\n\n sumf.s4 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 4*nb*QK4_0/2, d + ib + 4*nb, sumy, yl, il);\n sumf.s5 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 5*nb*QK4_0/2, d + ib + 5*nb, sumy, yl, il);\n sumf.s6 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 6*nb*QK4_0/2, d + ib + 6*nb, sumy, yl, il);\n sumf.s7 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 7*nb*QK4_0/2, d + ib + 7*nb, sumy, yl, il);\n\n yb += QK4_0 * (N_SIMDWIDTH/2);\n }\n\n float8 tot = (float8)(\n sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),\n sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),\n sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7)\n );\n\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;\n }\n\n if (first_row + 4 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;\n }\n if (first_row + 5 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;\n }\n if (first_row + 6 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;\n }\n if (first_row + 7 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;\n }\n }\n}\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_q4_0_f32_8x_flat(\n global uchar * src0_q,\n global half * src0_d,\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n mul_vec_q_n_f32_8x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_0 32\n#define QR4_0 2\n#define QK4_1 32\n#define QR4_1 2\n#define QK5_0 32\n#define QR5_0 2\n#define QK5_1 32\n#define QR5_1 2\n#define QK8_0 32\n#define QR8_0 1\n#define QK_K 256\n#define K_QUANTS_PER_ITERATION 2\n\ntypedef char int8_t;\ntypedef uchar uint8_t;\ntypedef short int16_t;\ntypedef ushort uint16_t;\ntypedef int int32_t;\ntypedef uint uint32_t;\n\n//------------------------------------------------------------------------------\n// block_q4_0\n//------------------------------------------------------------------------------\nstruct block_q4_0\n{\n half d;\n uint8_t qs[QK4_0 / 2];\n};\n\n//\n// This variant unrolls the loops and uses vector types instead of pointers.\n// It improves performance on Adreno but not so much on Intel.\n//\ninline float block_q_4_0_dot_y_v(\n global struct block_q4_0 * qb_curr,\n float sumy,\n float16 yl,\n int il\n) {\n float d = qb_curr->d;\n float acc = 0.f;\n global ushort * qs = ((global ushort *)qb_curr + 1 + il/2);\n\n acc += yl.s0 * (qs[0] & 0x000F);\n acc += yl.s1 * (qs[0] & 0x0F00);\n acc += yl.s8 * (qs[0] & 0x00F0);\n acc += yl.s9 * (qs[0] & 0xF000);\n\n acc += yl.s2 * (qs[1] & 0x000F);\n acc += yl.s3 * (qs[1] & 0x0F00);\n acc += yl.sa * (qs[1] & 0x00F0);\n acc += yl.sb * (qs[1] & 0xF000);\n\n acc += yl.s4 * (qs[2] & 0x000F);\n acc += yl.s5 * (qs[2] & 0x0F00);\n acc += yl.sc * (qs[2] & 0x00F0);\n acc += yl.sd * (qs[2] & 0xF000);\n\n acc += yl.s6 * (qs[3] & 0x000F);\n acc += yl.s7 * (qs[3] & 0x0F00);\n acc += yl.se * (qs[3] & 0x00F0);\n acc += yl.sf * (qs[3] & 0xF000);\n\n return d * (sumy * -8.f + acc);\n}\n\n#undef N_DST\n#undef N_SIMDGROUP\n#undef N_SIMDWIDTH\n\n#ifdef INTEL_GPU\n#define N_DST 4 // each SIMD group works on 4 rows\n#define N_SIMDGROUP 1 // number of SIMD groups in a thread group\n#define N_SIMDWIDTH 16 // assuming SIMD group size is 16\n#elif defined (ADRENO_GPU)\n#define N_DST 4\n#define N_SIMDGROUP 1\n#define N_SIMDWIDTH 64\n#endif\n\ninline void mul_vec_q_n_f32_v(\n global void * src0,\n global float * src1,\n global float * dst,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n const ulong nb = ne00/QK4_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n // (r0 * N_SIMDGROUP + get_sub_group_id()) is essenatially the linear global\n // id of a SIMD group in the grid.\n int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n\n global struct block_q4_0 * x = (global struct block_q4_0 *) src0 + offset0;\n global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n float16 yl; // src1 vector cache\n float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);\n\n int ix = get_sub_group_local_id()/2;\n int il = 8*(get_sub_group_local_id()%2);\n\n global float * yb = y + ix * QK4_0 + il;\n\n // each thread in a SIMD group deals with half a block.\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n float sumy = 0;\n\n sumy += yb[0];\n sumy += yb[1];\n sumy += yb[2];\n sumy += yb[3];\n sumy += yb[4];\n sumy += yb[5];\n sumy += yb[6];\n sumy += yb[7];\n\n sumy += yb[16];\n sumy += yb[17];\n sumy += yb[18];\n sumy += yb[19];\n sumy += yb[20];\n sumy += yb[21];\n sumy += yb[22];\n sumy += yb[23];\n\n\n yl.s0 = yb[0];\n yl.s1 = yb[1]/256.f;\n\n yl.s2 = yb[2];\n yl.s3 = yb[3]/256.f;\n\n yl.s4 = yb[4];\n yl.s5 = yb[5]/256.f;\n\n yl.s6 = yb[6];\n yl.s7 = yb[7]/256.f;\n\n yl.s8 = yb[16]/16.f;\n yl.s9 = yb[17]/4096.f;\n\n yl.sa = yb[18]/16.f;\n yl.sb = yb[19]/4096.f;\n\n yl.sc = yb[20]/16.f;\n yl.sd = yb[21]/4096.f;\n\n yl.se = yb[22]/16.f;\n yl.sf = yb[23]/4096.f;\n\n sumf.s0 += block_q_4_0_dot_y_v(x+ib+0*nb, sumy, yl, il);\n sumf.s1 += block_q_4_0_dot_y_v(x+ib+1*nb, sumy, yl, il);\n sumf.s2 += block_q_4_0_dot_y_v(x+ib+2*nb, sumy, yl, il);\n sumf.s3 += block_q_4_0_dot_y_v(x+ib+3*nb, sumy, yl, il);\n\n // One thread in a SIMD group (i.e., subgroup) handles a half block,\n // hence then entire SIMD group handles SIMDWIDTH/2 blocks.\n // y points to the activation matrix (of type float). Therefore for\n // one thread, the # of blocks y should advance is SIMDWIDTH/2 (because\n // SIMDWIDTH/2 blocks are processed by a SIMD group) - in terms of\n // floats, it is QK4_0 * (SIMDWIDTH/2), where QK4_0 is the block size.\n yb += QK4_0 * (N_SIMDWIDTH/2);\n }\n\n // The above does not work for Adreno - it produces incorrect results for\n // row = 1, 2, 3 and only row = 0 gives the correct result.\n // If N_DST is changed, the below array must be initialized accordingly.\n // This also seems to perform better on Intel.\n float4 tot = (float4)(\n sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)\n );\n\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;\n }\n }\n}\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mat_q4_0_f32_v(\n global void * src0,\n ulong offset0,\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n mul_vec_q_n_f32_v(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_1 32\n\nstruct block_q4_1 {\n half d; // delta\n half m; // min\n uchar qs[QK4_1 / 2]; // nibbles / quants\n};\n\ninline float block_q4_1_dot_y(\n global const struct block_q4_1 * qb_curr,\n float sumy,\n float16 yl,\n int il\n) {\n float d = qb_curr->d;\n float m = qb_curr->m;\n\n float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f);\n\n global const ushort * qs = ((global const ushort *) qb_curr + 2 + il/2);\n\n acc.s0 += yl.s0 * (qs[0] & 0x000F);\n acc.s0 += yl.s1 * (qs[0] & 0x0F00);\n acc.s0 += yl.s8 * (qs[0] & 0x00F0);\n acc.s3 += yl.s9 * (qs[0] & 0xF000);\n\n acc.s0 += yl.s2 * (qs[1] & 0x000F);\n acc.s1 += yl.s3 * (qs[1] & 0x0F00);\n acc.s2 += yl.sa * (qs[1] & 0x00F0);\n acc.s3 += yl.sb * (qs[1] & 0xF000);\n\n acc.s0 += yl.s4 * (qs[2] & 0x000F);\n acc.s1 += yl.s5 * (qs[2] & 0x0F00);\n acc.s2 += yl.sc * (qs[2] & 0x00F0);\n acc.s3 += yl.sd * (qs[2] & 0xF000);\n\n acc.s0 += yl.s6 * (qs[3] & 0x000F);\n acc.s1 += yl.s7 * (qs[3] & 0x0F00);\n acc.s2 += yl.se * (qs[3] & 0x00F0);\n acc.s3 += yl.sf * (qs[3] & 0xF000);\n\n return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3) + sumy * m;\n}\n\n#undef N_DST\n#undef N_SIMDGROUP\n#undef N_SIMDWIDTH\n\n#ifdef INTEL_GPU\n#define N_DST 4 // each subgroup works on 4 rows\n#define N_SIMDGROUP 1 // number of subgroups in a thread group\n#define N_SIMDWIDTH 16 // assuming subgroup size is 16\n#elif defined (ADRENO_GPU)\n#define N_DST 4\n#define N_SIMDGROUP 1\n#define N_SIMDWIDTH 64\n#endif\n\ninline void mul_vec_q_n_f32(\n global void * src0,\n global float * src1,\n global float * dst,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n const ulong nb = ne00/QK4_1;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n\n global struct block_q4_1 * x = (global struct block_q4_1 *) src0 + offset0;\n global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n float16 yl;\n float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);\n\n int ix = get_sub_group_local_id()/2;\n int il = 8*(get_sub_group_local_id()%2);\n\n global float * yb = y + ix * QK4_1 + il;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n float sumy = 0;\n\n sumy += yb[0];\n sumy += yb[1];\n sumy += yb[2];\n sumy += yb[3];\n sumy += yb[4];\n sumy += yb[5];\n sumy += yb[6];\n sumy += yb[7];\n\n sumy += yb[16];\n sumy += yb[17];\n sumy += yb[18];\n sumy += yb[19];\n sumy += yb[20];\n sumy += yb[21];\n sumy += yb[22];\n sumy += yb[23];\n\n\n yl.s0 = yb[0];\n yl.s1 = yb[1]/256.f;\n\n yl.s2 = yb[2];\n yl.s3 = yb[3]/256.f;\n\n yl.s4 = yb[4];\n yl.s5 = yb[5]/256.f;\n\n yl.s6 = yb[6];\n yl.s7 = yb[7]/256.f;\n\n yl.s8 = yb[16]/16.f;\n yl.s9 = yb[17]/4096.f;\n\n yl.sa = yb[18]/16.f;\n yl.sb = yb[19]/4096.f;\n\n yl.sc = yb[20]/16.f;\n yl.sd = yb[21]/4096.f;\n\n yl.se = yb[22]/16.f;\n yl.sf = yb[23]/4096.f;\n\n sumf.s0 += block_q4_1_dot_y(x+ib+0*nb, sumy, yl, il);\n sumf.s1 += block_q4_1_dot_y(x+ib+1*nb, sumy, yl, il);\n sumf.s2 += block_q4_1_dot_y(x+ib+2*nb, sumy, yl, il);\n sumf.s3 += block_q4_1_dot_y(x+ib+3*nb, sumy, yl, il);\n\n yb += QK4_1 * (N_SIMDWIDTH/2);\n }\n\n float4 tot = (float4)(\n sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)\n );\n\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;\n }\n }\n}\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_q4_1_f32(\n global void * src0,\n ulong offset0,\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_1 32\n\nstruct block_q4_1 {\n half d; // delta\n half m; // min\n uchar qs[QK4_1 / 2]; // nibbles / quants\n};\n\ninline float block_q4_1_dot_y_flat(\n global const uchar * x,\n global const half * dh,\n global const half * mh,\n float sumy,\n float16 yl,\n int il\n) {\n float d = *dh;\n float m = *mh;\n global const ushort * qs = ((global const ushort *) x + il/2);\n\n float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f);\n\n acc.s0 += yl.s0 * (qs[0] & 0x000F);\n acc.s0 += yl.s1 * (qs[0] & 0x0F00);\n acc.s0 += yl.s8 * (qs[0] & 0x00F0);\n acc.s3 += yl.s9 * (qs[0] & 0xF000);\n\n acc.s0 += yl.s2 * (qs[1] & 0x000F);\n acc.s1 += yl.s3 * (qs[1] & 0x0F00);\n acc.s2 += yl.sa * (qs[1] & 0x00F0);\n acc.s3 += yl.sb * (qs[1] & 0xF000);\n\n acc.s0 += yl.s4 * (qs[2] & 0x000F);\n acc.s1 += yl.s5 * (qs[2] & 0x0F00);\n acc.s2 += yl.sc * (qs[2] & 0x00F0);\n acc.s3 += yl.sd * (qs[2] & 0xF000);\n\n acc.s0 += yl.s6 * (qs[3] & 0x000F);\n acc.s1 += yl.s7 * (qs[3] & 0x0F00);\n acc.s2 += yl.se * (qs[3] & 0x00F0);\n acc.s3 += yl.sf * (qs[3] & 0xF000);\n\n return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3) + sumy * m;\n}\n\n#undef N_DST\n#undef N_SIMDGROUP\n#undef N_SIMDWIDTH\n\n#ifdef INTEL_GPU\n#define N_DST 4 // each subgroup works on 4 rows\n#define N_SIMDGROUP 1 // number of subgroups in a thread group\n#define N_SIMDWIDTH 16 // assuming subgroup size is 16\n#elif defined (ADRENO_GPU)\n#define N_DST 4\n#define N_SIMDGROUP 1\n#define N_SIMDWIDTH 64\n#endif\n\ninline void mul_vec_q_n_f32_flat(\n global void * src0_q,\n global void * src0_d,\n global void * src0_m,\n global float * src1,\n global float * dst,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n const ulong nb = ne00/QK4_1;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n\n // The number of scales/mins is the same as the number of blocks.\n ulong offset0_dm = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02));\n // Each block contains QK4_1/2 uchars, hence offset for qs is as follows.\n ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_1/2;\n\n global uchar * x = (global uchar *) src0_q + offset0_q;\n global half * d = (global half *) src0_d + offset0_dm;\n global half * m = (global half *) src0_m + offset0_dm;\n global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n float16 yl;\n float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);\n\n int ix = get_sub_group_local_id()/2;\n int il = 8*(get_sub_group_local_id()%2);\n\n global float * yb = y + ix * QK4_1 + il;\n\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {\n float sumy = 0;\n\n sumy += yb[0];\n sumy += yb[1];\n sumy += yb[2];\n sumy += yb[3];\n sumy += yb[4];\n sumy += yb[5];\n sumy += yb[6];\n sumy += yb[7];\n\n sumy += yb[16];\n sumy += yb[17];\n sumy += yb[18];\n sumy += yb[19];\n sumy += yb[20];\n sumy += yb[21];\n sumy += yb[22];\n sumy += yb[23];\n\n\n yl.s0 = yb[0];\n yl.s1 = yb[1]/256.f;\n\n yl.s2 = yb[2];\n yl.s3 = yb[3]/256.f;\n\n yl.s4 = yb[4];\n yl.s5 = yb[5]/256.f;\n\n yl.s6 = yb[6];\n yl.s7 = yb[7]/256.f;\n\n yl.s8 = yb[16]/16.f;\n yl.s9 = yb[17]/4096.f;\n\n yl.sa = yb[18]/16.f;\n yl.sb = yb[19]/4096.f;\n\n yl.sc = yb[20]/16.f;\n yl.sd = yb[21]/4096.f;\n\n yl.se = yb[22]/16.f;\n yl.sf = yb[23]/4096.f;\n\n sumf.s0 += block_q4_1_dot_y_flat(x + ib*QK4_1/2 + 0*nb*QK4_1/2, d + ib + 0*nb, m + ib + 0*nb, sumy, yl, il);\n sumf.s1 += block_q4_1_dot_y_flat(x + ib*QK4_1/2 + 1*nb*QK4_1/2, d + ib + 1*nb, m + ib + 1*nb, sumy, yl, il);\n sumf.s2 += block_q4_1_dot_y_flat(x + ib*QK4_1/2 + 2*nb*QK4_1/2, d + ib + 2*nb, m + ib + 2*nb, sumy, yl, il);\n sumf.s3 += block_q4_1_dot_y_flat(x + ib*QK4_1/2 + 3*nb*QK4_1/2, d + ib + 3*nb, m + ib + 3*nb, sumy, yl, il);\n\n yb += QK4_1 * (N_SIMDWIDTH/2);\n }\n\n float4 tot = (float4)(\n sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)\n );\n\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;\n }\n }\n}\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_q4_1_f32_flat(\n global void * src0_q,\n global void * src0_d,\n global void * src0_m,\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n mul_vec_q_n_f32_flat(src0_q, src0_d, src0_m, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl", "content": "#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n//------------------------------------------------------------------------------\n// block_q4_K\n//------------------------------------------------------------------------------\n#define QK_K 256\n#define K_SCALE_SIZE 12\n\n// 8 blocks of 32 elements each\n// weight is represented as x = a * q + b\ntypedef struct {\n half d; // super-block scale for quantized scales\n half dmin; // super-block scale for quantized mins\n\n uchar scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits\n uchar qs[QK_K/2]; // 4-bit quants\n} block_q4_K;\n\n#undef N_DST\n#undef N_SIMDGROUP\n#undef N_SIMDWIDTH\n\n#ifdef INTEL_GPU\n#define N_DST 4 // number of rows each SIMD group works on\n#define N_SIMDGROUP 1 // number of SIMD groups in a thread group\n#define N_SIMDWIDTH 16 // SIMD group size\n#elif defined (ADRENO_GPU)\n#define N_DST 4\n#define N_SIMDGROUP 1\n#define N_SIMDWIDTH 64\n#endif\n\n#undef BLOCK_STRIDE\n// number of (super) blocks each subgroup processes\n// each thread in a subgroup processes a block (32 weights)\n#define BLOCK_STRIDE (N_SIMDWIDTH/8)\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_q4_K_f32(\n global char * src0,\n int offset0,\n global char * src1,\n int offset1,\n global char * dst,\n int offsetd,\n int ne00,\n int ne01,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n ushort kmask1 = 0x3f3f;\n ushort kmask2 = 0x0f0f;\n ushort kmask3 = 0xc0c0;\n\n int ix = get_sub_group_local_id()/8; // super block index\n int it = get_sub_group_local_id()%8; // block index (inside super block)\n int iq = it/4; // 0 or 1 - first or second half of the super block\n int ir = it%4; // 0...3 - block index in the half super block\n\n int nb = ne00/QK_K;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n int offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n int offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;\n\n global block_q4_K * x = (global block_q4_K *) (src0 + offset_src0);\n global float * y = (global float *) (src1 + offset_src1);\n\n float yl[16];\n float yh[16];\n float sumf[N_DST] = {0.f};\n float all_sum;\n\n global float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;\n\n ushort sc16[4];\n uchar * sc8 = (uchar *)sc16;\n\n for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {\n float4 sumy = {0.f, 0.f, 0.f, 0.f};\n for (int i = 0; i < 8; ++i) {\n yl[i+0] = y4[i+0];\n sumy.s0 += yl[i+0];\n\n yl[i+8] = y4[i+32];\n sumy.s1 += yl[i+8];\n\n yh[i+0] = y4[i+128];\n sumy.s2 += yh[i+0];\n\n yh[i+8] = y4[i+160];\n sumy.s3 += yh[i+8];\n }\n\n global ushort * sc = (global ushort *)x[ib].scales + iq;\n global ushort * q1 = (global ushort *)x[ib].qs + 16 * iq + 4 * ir;\n global half * dh = &x[ib].d;\n\n for (int row = 0; row < N_DST; row++) {\n sc16[0] = sc[0] & kmask1;\n sc16[1] = sc[2] & kmask1;\n sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);\n sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);\n\n global ushort * q2 = q1 + 32;\n\n float4 acc1 = {0.f, 0.f, 0.f, 0.f};\n float4 acc2 = {0.f, 0.f, 0.f, 0.f};\n for (int i = 0; i < 8; i += 2) {\n acc1.s0 += yl[i+0] * (q1[i/2] & 0x000F);\n acc1.s1 += yl[i+1] * (q1[i/2] & 0x0F00);\n acc1.s2 += yl[i+8] * (q1[i/2] & 0x00F0);\n acc1.s3 += yl[i+9] * (q1[i/2] & 0xF000);\n acc2.s0 += yh[i+0] * (q2[i/2] & 0x000F);\n acc2.s1 += yh[i+1] * (q2[i/2] & 0x0F00);\n acc2.s2 += yh[i+8] * (q2[i/2] & 0x00F0);\n acc2.s3 += yh[i+9] * (q2[i/2] & 0xF000);\n }\n\n float dall = dh[0];\n float dmin = dh[1];\n sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc1.s1) * sc8[0] +\n (acc1.s2 + 1.f/256.f * acc1.s3) * sc8[1] * 1.f/16.f +\n (acc2.s0 + 1.f/256.f * acc2.s1) * sc8[4] +\n (acc2.s2 + 1.f/256.f * acc2.s3) * sc8[5] * 1.f/16.f) -\n dmin * (sumy.s0 * sc8[2] + sumy.s1 * sc8[3] + sumy.s2 * sc8[6] + sumy.s3 * sc8[7]);\n\n q1 += nb01/2;\n sc += nb01/2;\n dh += nb01/2;\n }\n\n y4 += BLOCK_STRIDE * QK_K;\n }\n\n global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;\n\n for (int row = 0; row < N_DST; ++row) {\n all_sum = sub_group_reduce_add(sumf[row]);\n if (first_row + row < ne01) {\n if (get_sub_group_local_id() == 0) {\n dst_f32[first_row + row] = all_sum;\n }\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK4_0 32\n#define QR4_0 2\n#define QK4_1 32\n#define QR4_1 2\n#define QK5_0 32\n#define QR5_0 2\n#define QK5_1 32\n#define QR5_1 2\n#define QK8_0 32\n#define QR8_0 1\n#define QK_K 256\n#define K_QUANTS_PER_ITERATION 2\n\ntypedef char int8_t;\ntypedef uchar uint8_t;\ntypedef short int16_t;\ntypedef ushort uint16_t;\ntypedef int int32_t;\ntypedef uint uint32_t;\n\n//------------------------------------------------------------------------------\n// block_q6_K\n//------------------------------------------------------------------------------\n// 6-bit quantization\n// weight is represented as x = a * q\n// 16 blocks of 16 elements each\n// Effectively 6.5625 bits per weight\ntypedef struct {\n uint8_t ql[QK_K/2]; // quants, lower 4 bits\n uint8_t qh[QK_K/4]; // quants, upper 2 bits\n int8_t scales[QK_K/16]; // scales, quantized with 8 bits\n half d; // super-block scale\n} block_q6_K;\n\n//------------------------------------------------------------------------------\n// kernel_mul_mv_q6_K_f32\n//------------------------------------------------------------------------------\n\n#undef N_DST\n#undef N_SIMDGROUP\n#undef N_SIMDWIDTH\n\n#ifdef INTEL_GPU\n#define N_DST 1 // number of rows each SIMD group works on\n#define N_SIMDGROUP 2 // number of SIMD groups in a thread group\n#define N_SIMDWIDTH 16 // SIMD group size\n#elif defined (ADRENO_GPU)\n#define N_DST 1\n#define N_SIMDGROUP 2\n#define N_SIMDWIDTH 64\n#endif\n\n#define BLOCK_STRIDE (N_SIMDWIDTH/16) // number of blocks each subgroup processes\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_q6_K_f32(\n global void * src0,\n ulong offset0,\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n uchar kmask1 = 0x03;\n uchar kmask2 = 0x0C;\n uchar kmask3 = 0x30;\n uchar kmask4 = 0xC0;\n\n int nb = ne00/QK_K;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n int row = N_SIMDGROUP * r0 + get_sub_group_id();\n\n if (row >= ne01) {\n return;\n }\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n ulong offset_src0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n\n global block_q6_K * x = (global block_q6_K *) src0 + row*nb + offset_src0;\n global float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n float sumf = 0;\n\n // For Q6_K quantization, 16 values forms a subblock, 16 subblock forms a\n // block. Values in a subblock shares a scale that is quantized with 8 bits;\n // the entire block shares a single floating point scale.\n // For work distribution, each thread processes a subblock (16 weights), hence\n // 16 threads process a (super) block -- a subgroup thus handles SIMDWIDTH/16\n // (super) blocks -- this is the block stride.\n // The 16 threads that process a (super) block are split into 2 portions, each has\n // 8 threads; each portion works on 8 subblocks.\n // For subgroup of 16 threads, the entire subgroup works on a single (super) block\n // before moving to the next (super) block. Thread0 - thread7 work on the\n // first 8 subblocks; thread8 - thread15 works on the last 8 subblocks.\n // Thread0 - thread3 work on subblocks 0, 2, 4, 6; thread4 - thread7 work on\n // subblocks 1, 3, 5, 7. Each thread does not work on an entire subblock, but\n // works on a total of 16 weight values.\n int tid = get_sub_group_local_id()/BLOCK_STRIDE; // first block_stride groups have tid=0\n int ix = get_sub_group_local_id()%BLOCK_STRIDE; // first block is 0..block_stride-1\n int ip = tid/8; // first or second half of (super) block (0 or 1)\n int il = tid%8; // each half has 8 parts, one per scale\n int n = 4; // 4 scales at a time (and 4 sums)\n int l0 = n*il; // offset into half-block, 0..28\n int is = 8*ip + l0/16; // 0, 1, 8, 9\n\n int y_offset = 128*ip + l0;\n int q_offset_l = 64*ip + l0;\n int q_offset_h = 32*ip + l0;\n\n for (int i = ix; i < nb; i += BLOCK_STRIDE) {\n\n global uint8_t * q1 = x[i].ql + q_offset_l;\n global uint8_t * q2 = q1 + QK_K/8;\n global uint8_t * qh = x[i].qh + q_offset_h;\n global int8_t * sc = x[i].scales + is;\n\n global float * y = yy + i * QK_K + y_offset;\n\n float dall = x[i].d;\n\n float4 sums = {0.f, 0.f, 0.f, 0.f};\n\n sums.s0 += y[0+ 0] * ((float)((q1[0] & 0xF) | ((qh[0] & kmask1) << 4)) - 32.f);\n sums.s1 += y[0+32] * ((float)((q2[0] & 0xF) | ((qh[0] & kmask2) << 2)) - 32.f);\n sums.s2 += y[0+64] * ((float)((q1[0] >> 4) | ((qh[0] & kmask3) << 0)) - 32.f);\n sums.s3 += y[0+96] * ((float)((q2[0] >> 4) | ((qh[0] & kmask4) >> 2)) - 32.f);\n\n sums.s0 += y[1+ 0] * ((float)((q1[1] & 0xF) | ((qh[1] & kmask1) << 4)) - 32.f);\n sums.s1 += y[1+32] * ((float)((q2[1] & 0xF) | ((qh[1] & kmask2) << 2)) - 32.f);\n sums.s2 += y[1+64] * ((float)((q1[1] >> 4) | ((qh[1] & kmask3) << 0)) - 32.f);\n sums.s3 += y[1+96] * ((float)((q2[1] >> 4) | ((qh[1] & kmask4) >> 2)) - 32.f);\n\n sums.s0 += y[2+ 0] * ((float)((q1[2] & 0xF) | ((qh[2] & kmask1) << 4)) - 32.f);\n sums.s1 += y[2+32] * ((float)((q2[2] & 0xF) | ((qh[2] & kmask2) << 2)) - 32.f);\n sums.s2 += y[2+64] * ((float)((q1[2] >> 4) | ((qh[2] & kmask3) << 0)) - 32.f);\n sums.s3 += y[2+96] * ((float)((q2[2] >> 4) | ((qh[2] & kmask4) >> 2)) - 32.f);\n\n sums.s0 += y[3+ 0] * ((float)((q1[3] & 0xF) | ((qh[3] & kmask1) << 4)) - 32.f);\n sums.s1 += y[3+32] * ((float)((q2[3] & 0xF) | ((qh[3] & kmask2) << 2)) - 32.f);\n sums.s2 += y[3+64] * ((float)((q1[3] >> 4) | ((qh[3] & kmask3) << 0)) - 32.f);\n sums.s3 += y[3+96] * ((float)((q2[3] >> 4) | ((qh[3] & kmask4) >> 2)) - 32.f);\n\n sumf += dall * (sums.s0 * sc[0] + sums.s1 * sc[2] + sums.s2 * sc[4] + sums.s3 * sc[6]);\n }\n\n float tot = sub_group_reduce_add(sumf);\n if (get_sub_group_local_id() == 0) {\n dst[r1*ne0 + im*ne0*ne1 + row] = tot;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n//------------------------------------------------------------------------------\n// kernel_mul_mv_q6_K_f32_flat\n//------------------------------------------------------------------------------\n#define Q6_K_MASK1 0x03\n#define Q6_K_MASK2 0x0C\n#define Q6_K_MASK3 0x30\n#define Q6_K_MASK4 0xC0\n\n#define QK_K 256\n\ninline float block_q_6_K_dot_y_flat(\n global uchar * blk_ql,\n global uchar * blk_qh,\n global char * blk_scales,\n global half * blk_d,\n global float * yy,\n int ib,\n int ip,\n int is,\n int l0\n) {\n int y_offset = 128*ip + l0;\n int q_offset_l = 64*ip + l0;\n int q_offset_h = 32*ip + l0;\n\n global uchar * q1 = blk_ql + ib*128 + q_offset_l;\n global uchar * q2 = q1 + QK_K/8;\n global uchar * qh = blk_qh + ib*64 + q_offset_h;\n global char * sc = blk_scales + ib*16 + is;\n\n global float * y = yy + ib * QK_K + y_offset;\n\n float dall = blk_d[ib];\n\n float sumf = 0;\n float4 sums = {0.f, 0.f, 0.f, 0.f};\n\n sums.s0 += y[0+ 0] * ((float)((q1[0] & 0xF) | ((qh[0] & Q6_K_MASK1) << 4)) - 32.f);\n sums.s1 += y[0+32] * ((float)((q2[0] & 0xF) | ((qh[0] & Q6_K_MASK2) << 2)) - 32.f);\n sums.s2 += y[0+64] * ((float)((q1[0] >> 4) | ((qh[0] & Q6_K_MASK3) << 0)) - 32.f);\n sums.s3 += y[0+96] * ((float)((q2[0] >> 4) | ((qh[0] & Q6_K_MASK4) >> 2)) - 32.f);\n\n sums.s0 += y[1+ 0] * ((float)((q1[1] & 0xF) | ((qh[1] & Q6_K_MASK1) << 4)) - 32.f);\n sums.s1 += y[1+32] * ((float)((q2[1] & 0xF) | ((qh[1] & Q6_K_MASK2) << 2)) - 32.f);\n sums.s2 += y[1+64] * ((float)((q1[1] >> 4) | ((qh[1] & Q6_K_MASK3) << 0)) - 32.f);\n sums.s3 += y[1+96] * ((float)((q2[1] >> 4) | ((qh[1] & Q6_K_MASK4) >> 2)) - 32.f);\n\n sums.s0 += y[2+ 0] * ((float)((q1[2] & 0xF) | ((qh[2] & Q6_K_MASK1) << 4)) - 32.f);\n sums.s1 += y[2+32] * ((float)((q2[2] & 0xF) | ((qh[2] & Q6_K_MASK2) << 2)) - 32.f);\n sums.s2 += y[2+64] * ((float)((q1[2] >> 4) | ((qh[2] & Q6_K_MASK3) << 0)) - 32.f);\n sums.s3 += y[2+96] * ((float)((q2[2] >> 4) | ((qh[2] & Q6_K_MASK4) >> 2)) - 32.f);\n\n sums.s0 += y[3+ 0] * ((float)((q1[3] & 0xF) | ((qh[3] & Q6_K_MASK1) << 4)) - 32.f);\n sums.s1 += y[3+32] * ((float)((q2[3] & 0xF) | ((qh[3] & Q6_K_MASK2) << 2)) - 32.f);\n sums.s2 += y[3+64] * ((float)((q1[3] >> 4) | ((qh[3] & Q6_K_MASK3) << 0)) - 32.f);\n sums.s3 += y[3+96] * ((float)((q2[3] >> 4) | ((qh[3] & Q6_K_MASK4) >> 2)) - 32.f);\n\n sumf += dall * (sums.s0 * sc[0] + sums.s1 * sc[2] + sums.s2 * sc[4] + sums.s3 * sc[6]);\n\n return sumf;\n}\n\n#undef N_DST\n#undef N_SIMDGROUP\n#undef N_SIMDWIDTH\n\n#ifdef INTEL_GPU\n#define N_DST 4\n#define N_SIMDGROUP 2\n#define N_SIMDWIDTH 16\n#elif defined (ADRENO_GPU)\n#define N_DST 4\n#define N_SIMDGROUP 2\n#define N_SIMDWIDTH 64\n#endif\n\n#define BLOCK_STRIDE (N_SIMDWIDTH/16) // number of blocks each subgroup processes\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_q6_K_f32_flat(\n global uchar * src0_ql,\n global uchar * src0_qh,\n global char * src0_s,\n global half * src0_d,\n global float * src1,\n ulong offset1,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne10,\n int ne12,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src1 = (global float*)((global char*)src1 + offset1);\n dst = (global float*)((global char*)dst + offsetd);\n\n int nb = ne00/QK_K;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n int i12 = im%ne12;\n int i13 = im/ne12;\n\n int first_row = (N_SIMDGROUP * r0 + get_sub_group_id()) * N_DST;\n\n ulong offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);\n ulong offset_src0_ql = offset_src0 * 128;\n ulong offset_src0_qh = offset_src0 * 64;\n ulong offset_src0_s = offset_src0 * 16;\n ulong offset_src0_d = offset_src0;\n\n global uchar * blk_ql = (global uchar *) src0_ql + offset_src0_ql;\n global uchar * blk_qh = (global uchar *) src0_qh + offset_src0_qh;\n global char * blk_scales = (global char *) src0_s + offset_src0_s;\n global half * blk_d = (global half *) src0_d + offset_src0_d;\n global float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1;\n\n int tid = get_sub_group_local_id()/BLOCK_STRIDE; // first block_stride groups have tid=0\n int ix = get_sub_group_local_id()%BLOCK_STRIDE; // first block is 0..block_stride-1\n int ip = tid/8; // first or second half of (super) block (0 or 1)\n int il = tid%8; // each half has 8 parts, one per scale\n int n = 4; // 4 scales at a time (and 4 sums)\n int l0 = n*il; // offset into half-block, 0..28\n int is = 8*ip + l0/16; // 0, 1, 8, 9\n\n float4 sumf = 0;\n\n for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {\n if (first_row + 0 < ne01) {\n sumf.s0 += block_q_6_K_dot_y_flat(blk_ql + 0*nb*128, blk_qh + 0*nb*64, blk_scales + 0*nb*16, blk_d + 0*nb, yy, ib, ip, is, l0);\n }\n if (first_row + 1 < ne01) {\n sumf.s1 += block_q_6_K_dot_y_flat(blk_ql + 1*nb*128, blk_qh + 1*nb*64, blk_scales + 1*nb*16, blk_d + 1*nb, yy, ib, ip, is, l0);\n }\n if (first_row + 2 < ne01) {\n sumf.s2 += block_q_6_K_dot_y_flat(blk_ql + 2*nb*128, blk_qh + 2*nb*64, blk_scales + 2*nb*16, blk_d + 2*nb, yy, ib, ip, is, l0);\n }\n if (first_row + 3 < ne01) {\n sumf.s3 += block_q_6_K_dot_y_flat(blk_ql + 3*nb*128, blk_qh + 3*nb*64, blk_scales + 3*nb*16, blk_d + 3*nb, yy, ib, ip, is, l0);\n }\n }\n\n float4 tot = (float4)(\n sub_group_reduce_add(sumf.s0),\n sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2),\n sub_group_reduce_add(sumf.s3)\n );\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK8_0 32\ntypedef struct {\n half d; // delta\n char qs[QK8_0]; // quants\n} block_q8_0;\n\n#define NB_Q8_0 8\n\n#ifdef INTEL_GPU\n#define N_R0_Q8_0 4 // number of rows each subgroup works on\n#define N_SG_Q8_0 2 // number of subgroups in a work group\n#define N_SIMDWIDTH 16 // subgroup size\n#elif defined (ADRENO_GPU)\n#define N_R0_Q8_0 4\n#define N_SG_Q8_0 2\n#define N_SIMDWIDTH 64\n#endif\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_q8_0_f32(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src0 = (global char*)((global char*)src0 + offset0);\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n int nb = ne00/QK8_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n int first_row = (r0*N_SG_Q8_0 + get_sub_group_id()) * N_R0_Q8_0;\n\n uint i12 = im%ne12;\n uint i13 = im/ne12;\n\n ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13;\n global float * y = (global float *) (src1 + offset_src1);\n\n // pointers to src0 rows\n global block_q8_0 * ax[N_R0_Q8_0];\n for (int row = 0; row < N_R0_Q8_0; ++row) {\n ulong offset_src0 = (first_row + row)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n ax[row] = (global block_q8_0 *) ((global char *) src0 + offset_src0);\n }\n\n float yl[NB_Q8_0];\n float sumf[N_R0_Q8_0] = { 0.f };\n\n const short ix = get_sub_group_local_id()/4;\n const short il = get_sub_group_local_id()%4;\n\n global float * yb = y + ix*QK8_0 + il*NB_Q8_0;\n\n // each thread handles NB_Q8_0 quants at a time\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) {\n for (short i = 0; i < NB_Q8_0; ++i) {\n yl[i] = yb[i];\n }\n\n for (short row = 0; row < N_R0_Q8_0; row++) {\n global char * qs = ax[row][ib].qs + il*NB_Q8_0;\n float sumq = 0.f;\n for (short iq = 0; iq < NB_Q8_0; ++iq) {\n sumq += qs[iq] * yl[iq];\n }\n sumf[row] += sumq*ax[row][ib].d;\n }\n\n yb += N_SIMDWIDTH*NB_Q8_0;\n }\n\n global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;\n\n for (int row = 0; row < N_R0_Q8_0; ++row) {\n float tot = sub_group_reduce_add(sumf[row]);\n\n if (get_sub_group_local_id() == 0 && first_row + row < ne01) {\n dst_f32[first_row + row] = tot;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#define QK8_0 32\ntypedef struct {\n half d; // delta\n char qs[QK8_0]; // quants\n} block_q8_0;\n\n#define NB_Q8_0 8\n\n#ifdef INTEL_GPU\n#define N_R0_Q8_0 4 // number of rows each subgroup works on\n#define N_SG_Q8_0 2 // number of subgroups in a work group\n#define N_SIMDWIDTH 16 // subgroup size\n#elif defined (ADRENO_GPU)\n#define N_R0_Q8_0 4\n#define N_SG_Q8_0 2\n#define N_SIMDWIDTH 64\n#endif\n\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_16\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_mul_mv_q8_0_f32_flat(\n global char * src0_q,\n global half * src0_d,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n int ne1,\n int r2,\n int r3\n) {\n src1 = (global char*)((global char*)src1 + offset1);\n dst = (global char*)((global char*)dst + offsetd);\n\n int nb = ne00/QK8_0;\n\n int r0 = get_group_id(0);\n int r1 = get_group_id(1);\n int im = get_group_id(2);\n\n int first_row = (r0*N_SG_Q8_0 + get_sub_group_id()) * N_R0_Q8_0;\n\n uint i12 = im%ne12;\n uint i13 = im/ne12;\n\n ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13;\n global float * y = (global float *) (src1 + offset_src1);\n\n // pointers to src0 rows\n uint offset_src0_base = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;\n\n global char * ax0, * ax1, * ax2, * ax3;\n global half * ad0, * ad1, * ad2, * ad3;\n uint offset_src0;\n\n offset_src0 = offset_src0_base + 0*nb01;\n offset_src0 = offset_src0/34;\n ax0 = (global char *) ((global char *) src0_q + offset_src0*sizeof(char)*QK8_0);\n ad0 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));\n\n offset_src0 = offset_src0_base + 1*nb01;\n offset_src0 = offset_src0/34;\n ax1 = (global char *) ((global char *) src0_q + offset_src0*sizeof(char)*QK8_0);\n ad1 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));\n\n offset_src0 = offset_src0_base + 2*nb01;\n offset_src0 = offset_src0/34;\n ax2 = (global char *) ((global char *) src0_q + offset_src0*sizeof(char)*QK8_0);\n ad2 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));\n\n offset_src0 = offset_src0_base + 3*nb01;\n offset_src0 = offset_src0/34;\n ax3 = (global char *) ((global char *) src0_q + offset_src0*sizeof(char)*QK8_0);\n ad3 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));\n\n const short ix = get_sub_group_local_id()/4;\n const short il = get_sub_group_local_id()%4;\n\n global float * yb = y + ix*QK8_0 + il*NB_Q8_0;\n\n float8 yl;\n float8 qv;\n float4 sumf = 0.f;\n float sumq = 0.f;\n global char * qs;\n\n // each thread handles NB_Q8_0 quants at a time\n for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) {\n yl = vload8(0, yb);\n\n qs = ax0 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;\n qv = convert_float8(vload8(0, qs));\n sumq = 0;\n sumq += qv.s0*yl.s0;\n sumq += qv.s1*yl.s1;\n sumq += qv.s2*yl.s2;\n sumq += qv.s3*yl.s3;\n sumq += qv.s4*yl.s4;\n sumq += qv.s5*yl.s5;\n sumq += qv.s6*yl.s6;\n sumq += qv.s7*yl.s7;\n sumf.s0 += sumq*ad0[ib];\n\n qs = ax1 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;\n qv = convert_float8(vload8(0, qs));\n sumq = 0;\n sumq += qv.s0*yl.s0;\n sumq += qv.s1*yl.s1;\n sumq += qv.s2*yl.s2;\n sumq += qv.s3*yl.s3;\n sumq += qv.s4*yl.s4;\n sumq += qv.s5*yl.s5;\n sumq += qv.s6*yl.s6;\n sumq += qv.s7*yl.s7;\n sumf.s1 += sumq*ad1[ib];\n\n qs = ax2 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;\n qv = convert_float8(vload8(0, qs));\n sumq = 0;\n sumq += qv.s0*yl.s0;\n sumq += qv.s1*yl.s1;\n sumq += qv.s2*yl.s2;\n sumq += qv.s3*yl.s3;\n sumq += qv.s4*yl.s4;\n sumq += qv.s5*yl.s5;\n sumq += qv.s6*yl.s6;\n sumq += qv.s7*yl.s7;\n sumf.s2 += sumq*ad2[ib];\n\n qs = ax3 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;\n qv = convert_float8(vload8(0, qs));\n sumq = 0;\n sumq += qv.s0*yl.s0;\n sumq += qv.s1*yl.s1;\n sumq += qv.s2*yl.s2;\n sumq += qv.s3*yl.s3;\n sumq += qv.s4*yl.s4;\n sumq += qv.s5*yl.s5;\n sumq += qv.s6*yl.s6;\n sumq += qv.s7*yl.s7;\n sumf.s3 += sumq*ad3[ib];\n\n yb += N_SIMDWIDTH*NB_Q8_0;\n }\n\n global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;\n\n float4 tot = (float4)(\n sub_group_reduce_add(sumf.s0),\n sub_group_reduce_add(sumf.s1),\n sub_group_reduce_add(sumf.s2),\n sub_group_reduce_add(sumf.s3)\n );\n\n if (get_sub_group_local_id() == 0) {\n if (first_row + 0 < ne01) {\n dst_f32[first_row + 0] = tot.s0;\n }\n if (first_row + 1 < ne01) {\n dst_f32[first_row + 1] = tot.s1;\n }\n if (first_row + 2 < ne01) {\n dst_f32[first_row + 2] = tot.s2;\n }\n if (first_row + 3 < ne01) {\n dst_f32[first_row + 3] = tot.s3;\n }\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/neg.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\nkernel void kernel_neg_f32(\n global const float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int n\n) {\n if (get_global_id(0) >= n) {\n return;\n }\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = -src0[get_global_id(0)];\n}\n\nkernel void kernel_neg_f32_4(\n global const float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd,\n int n\n) {\n if (get_global_id(0) >= n) {\n return;\n }\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = -src0[get_global_id(0)];\n}\n\nkernel void kernel_neg_f16(\n global const half * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd,\n int n\n) {\n if (get_global_id(0) >= n) {\n return;\n }\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = -src0[get_global_id(0)];\n}\n\nkernel void kernel_neg_f16_4(\n global const half4 * src0,\n ulong offset0,\n global half4 * dst,\n ulong offsetd,\n int n\n) {\n if (get_global_id(0) >= n) {\n return;\n }\n src0 = (global half4*)((global char*)src0 + offset0);\n dst = (global half4*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = -src0[get_global_id(0)];\n}\n\nkernel void kernel_neg_f32_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = -*x;\n }\n}\n\nkernel void kernel_neg_f16_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = -*x;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/norm.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n//------------------------------------------------------------------------------\n// norm\n//------------------------------------------------------------------------------\nkernel void kernel_norm(\n global void * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n float eps,\n local float * sum\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n dst = (global void*)((global char*)dst + offsetd);\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n global float * x = (global float *) ((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01);\n\n // MEAN\n // parallel sum\n sum[get_local_id(0)] = 0.0f;\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n sum[get_local_id(0)] += x[i00];\n }\n // reduce\n barrier(CLK_LOCAL_MEM_FENCE);\n for (uint i = get_local_size(0)/2; i > 0; i /= 2) {\n if (get_local_id(0) < i) {\n sum[get_local_id(0)] += sum[get_local_id(0) + i];\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n float mean = sum[0] / ne00;\n\n // recenter and VARIANCE\n barrier(CLK_LOCAL_MEM_FENCE);\n global float * y = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;\n sum[get_local_id(0)] = 0.0f;\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n y[i00] = x[i00] - mean;\n sum[get_local_id(0)] += y[i00] * y[i00];\n }\n\n // reduce\n barrier(CLK_LOCAL_MEM_FENCE);\n for (uint i = get_local_size(0)/2; i > 0; i /= 2) {\n if (get_local_id(0) < i) {\n sum[get_local_id(0)] += sum[get_local_id(0) + i];\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n }\n float variance = sum[0] / ne00;\n\n float scale = 1.0f/sqrt(variance + eps);\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n y[i00] = y[i00] * scale;\n }\n}\n\n//------------------------------------------------------------------------------\n// norm_mul_add\n//------------------------------------------------------------------------------\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_32\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_norm_mul_add(\n global char * src0_ptr, ulong src0_offset,\n global char * src1_ptr, ulong src1_offset,\n global char * src2_ptr, ulong src2_offset,\n global char * dst_ptr, ulong dst_offset,\n int ne00, int ne01, int ne02, int ne03,\n ulong nb01, ulong nb02, ulong nb03,\n int ne10, int ne11, int ne12, int ne13,\n ulong nb11, ulong nb12, ulong nb13,\n int ne20, int ne21, int ne22, int ne23,\n ulong nb21, ulong nb22, ulong nb23,\n ulong nbd1, ulong nbd2, ulong nbd3,\n float eps,\n local float2 * sums\n) {\n const int i03 = get_group_id(2);\n const int i02 = get_group_id(1);\n const int i01 = get_group_id(0);\n\n global float4 * x = (global float4 *)(src0_ptr + src0_offset + i01*nb01 + i02*nb02 + i03*nb03);\n global float4 * w = (global float4 *)(src1_ptr + src1_offset + (i01%ne11)*nb11 + (i02%ne12)*nb12 + (i03%ne13)*nb13);\n global float4 * b = (global float4 *)(src2_ptr + src2_offset + (i01%ne21)*nb21 + (i02%ne22)*nb22 + (i03%ne23)*nb23);\n global float4 * y = (global float4 *)(dst_ptr + dst_offset + i01*nbd1 + i02*nbd2 + i03*nbd3);\n\n float p_sum = 0.0f;\n float p_sum_sq = 0.0f;\n\n const int n_chunks = ne00 / 4;\n for (int i00 = get_local_id(0); i00 < n_chunks; i00 += get_local_size(0)) {\n float4 val = x[i00];\n p_sum += val.x + val.y + val.z + val.w;\n p_sum_sq += dot(val, val);\n }\n\n p_sum = sub_group_reduce_add(p_sum);\n p_sum_sq = sub_group_reduce_add(p_sum_sq);\n\n if (get_sub_group_local_id() == 0) {\n sums[get_sub_group_id()] = (float2)(p_sum, p_sum_sq);\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if (get_local_id(0) == 0) {\n float sum = 0.0f;\n float sum_sq = 0.0f;\n for (uint i = 0; i < get_num_sub_groups(); ++i) {\n float2 s = sums[i];\n sum += s.x;\n sum_sq += s.y;\n }\n\n const float inv_ne00 = 1.0f / (float)ne00;\n const float mean = sum * inv_ne00;\n const float variance = mad(-mean, mean, sum_sq * inv_ne00);\n\n sums[0] = (float2)(mean, rsqrt(variance + eps));\n }\n barrier(CLK_LOCAL_MEM_FENCE);\n\n const float2 mean_scale = sums[0];\n const float mean = mean_scale.x;\n const float scale = mean_scale.y;\n const float neg_mean_scale = -mean * scale;\n\n for (int i00 = get_local_id(0); i00 < n_chunks; i00 += get_local_size(0)) {\n const int w_idx = ne10 > 1 ? i00 : 0;\n const int b_idx = ne20 > 1 ? i00 : 0;\n const float4 norm_x = mad(x[i00], (float4)scale, (float4)neg_mean_scale);\n y[i00] = mad(norm_x, w[w_idx], b[b_idx]);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/pad.cl", "content": "kernel void kernel_pad(\n global void * src0,\n ulong offset0,\n global void * dst,\n ulong offsetd,\n int ne00, int ne01, int ne02, int ne03,\n ulong nb00, ulong nb01, ulong nb02, ulong nb03,\n int ne0, int ne1, int ne2, int ne3,\n ulong nb0, ulong nb1, ulong nb2, ulong nb3,\n int lp0, int rp0,\n int lp1, int rp1,\n int lp2, int rp2,\n int lp3, int rp3\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i0 = get_global_id(0);\n int i1 = get_group_id(1);\n int i2 = get_group_id(2) % ne2;\n int i3 = get_group_id(2) / ne2;\n\n if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {\n return;\n }\n\n uint src0_idx = (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01 + (i0 - lp0)*nb00;\n uint dst_idx = i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0;\n\n global float * src0_ptr = (global float *)((global char *)src0 + src0_idx);\n global float * dst_ptr = (global float *)((global char *)dst + dst_idx);\n\n bool in_src_bounds = (i0 >= lp0 && i0 < ne0 - rp0) &&\n (i1 >= lp1 && i1 < ne1 - rp1) &&\n (i2 >= lp2 && i2 < ne2 - rp2) &&\n (i3 >= lp3 && i3 < ne3 - rp3);\n\n *dst_ptr = in_src_bounds ? *src0_ptr : 0.0f;\n}\n" }, { "path": "src/ggml-opencl/kernels/relu.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// relu\n//------------------------------------------------------------------------------\nkernel void kernel_relu(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = fmax(0.0f, src0[get_global_id(0)]);\n}\n" }, { "path": "src/ggml-opencl/kernels/repeat.cl", "content": "kernel void kernel_repeat_f32(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n const int i03 = i3%ne03;\n const int i02 = i2%ne02;\n const int i01 = i1%ne01;\n\n global const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;\n global char * dst_ptr = dst + i3*nb3 + i2*nb2 + i1*nb1;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const int i00 = i0%ne00;\n *((global float *)(dst_ptr + i0*nb0)) = *((global float *)(src0_ptr + i00*nb00));\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/rms_norm.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n//------------------------------------------------------------------------------\n// rms_norm\n//------------------------------------------------------------------------------\n// This kernel depends on subgroup size.\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_32\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_rms_norm(\n global void * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n float eps,\n local float * sum // Note, the size depends on number of subgroups\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n global float4 * x = (global float4 *) ((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01);\n global float * x_scalar = (global float *) x;\n float4 sumf = 0;\n float all_sum = 0;\n\n // parallel sum\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n sumf += x[i00] * x[i00];\n }\n all_sum = sumf.s0 + sumf.s1 + sumf.s2 + sumf.s3;\n all_sum = sub_group_reduce_add(all_sum);\n if (get_sub_group_local_id() == 0) {\n sum[get_sub_group_id()] = all_sum;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n // broadcast\n for (uint i = get_local_size(0) / get_max_sub_group_size() / 2; i > 0; i /= 2) {\n if (get_local_id(0) < i) {\n sum[get_local_id(0)] += sum[get_local_id(0) + i];\n }\n }\n if (get_local_id(0) == 0) {\n for (int i = 4 * (ne00 / 4); i < ne00; i++) {\n sum[0] += x_scalar[i];\n }\n sum[0] /= ne00;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n const float mean = sum[0];\n const float scale = 1.0f/sqrt(mean + eps);\n\n global float4 * y = (global float4 *) (dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);\n global float * y_scalar = (global float *) y;\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n y[i00] = x[i00] * scale;\n }\n if (get_local_id(0) == 0) {\n for (int i00 = 4 * (ne00 / 4); i00 < ne00; i00++) {\n y_scalar[i00] = x_scalar[i00] * scale;\n }\n }\n}\n\n//------------------------------------------------------------------------------\n// rms_norm_mul\n//------------------------------------------------------------------------------\n#ifdef INTEL_GPU\nREQD_SUBGROUP_SIZE_32\n#elif defined (ADRENO_GPU)\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_rms_norm_mul(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n float eps,\n local float * sum\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n // The size of sum is sizeof(float)*subgroup_size.\n // Each subgroup writes its partial sum to this array.\n // So the number of subgroups per workgroup for this kernel cannot exceed the subgroup size.\n // This is generally true -\n // for subgroup size 64, workgroup size should be less than 4096 (the max is usually 1024).\n if (get_sub_group_id() == 0) {\n sum[get_sub_group_local_id()] = 0.0f;\n }\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);\n global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11);\n\n float sumf = 0;\n\n // parallel sum\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n sumf += dot(x[i00], x[i00]);\n }\n sumf = sub_group_reduce_add(sumf);\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if (get_sub_group_local_id() == 0) {\n sum[get_sub_group_id()] = sumf;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n //for (uint i = get_local_size(0) / get_max_sub_group_size() / 2; i > 0; i /= 2) {\n // if (get_local_id(0) < i) {\n // sum[get_local_id(0)] += sum[get_local_id(0) + i];\n // }\n //}\n //if (get_local_id(0) == 0) {\n // sum[0] /= ne00;\n //}\n\n //barrier(CLK_LOCAL_MEM_FENCE);\n\n sumf = sum[get_sub_group_local_id()];\n sumf = sub_group_reduce_add(sumf);\n\n float mean = sumf / ne00;\n float scale = 1.0f/sqrt(mean + eps);\n\n global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1);\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n y[i00] = (x[i00] * scale) * f[i00%(ne10/4)];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/rope.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// kernel_rope\n//------------------------------------------------------------------------------\nfloat rope_yarn_ramp(float low, float high, int i0) {\n const float y = (i0 / 2 - low) / max(0.001f, high - low);\n return 1.0f - min(1.0f, max(0.0f, y));\n}\n\n// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn\n// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.\nfloat2 rope_yarn(\n float theta_extrap, float freq_scale, float2 corr_dims, int i0, float ext_factor, float mscale\n) {\n // Get n-d rotational scaling corrected for extrapolation\n float theta_interp = freq_scale * theta_extrap;\n float theta = theta_interp;\n if (ext_factor != 0.0f) {\n float ramp_mix = rope_yarn_ramp(corr_dims.s0, corr_dims.s1, i0) * ext_factor;\n theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;\n\n // Get n-d magnitude scaling corrected for interpolation\n mscale *= 1.0f + 0.1f * log(1.0f / freq_scale);\n }\n return (float2)(cos(theta) * mscale, sin(theta) * mscale);\n}\n\n// Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get\n// `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`\nfloat rope_yarn_corr_factor(int n_dims, int n_ctx_orig, float n_rot, float base) {\n return n_dims * log(n_ctx_orig / (n_rot * 2 * M_PI_F)) / (2 * log(base));\n}\n\nfloat2 rope_yarn_corr_dims(\n int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow\n) {\n // start and end correction dims\n return (float2)(\n max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_fast, freq_base))),\n min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_slow, freq_base)))\n );\n}\n\nkernel void kernel_rope_norm_f32(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * src2,\n ulong offset2,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int n_past,\n int n_dims,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n src2 = (global float*)((global char*)src2 + offset2);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i3 = get_group_id(2);\n int i2 = get_group_id(1);\n int i1 = get_group_id(0);\n\n float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);\n\n global int * pos = src1;\n\n float theta_base = (float) pos[i2];\n float inv_ndims = -1.f/n_dims;\n\n for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {\n if (i0 < n_dims) {\n int ic = i0/2;\n\n float theta = theta_base * pow(freq_base, inv_ndims*i0);\n\n float freq_factor = src2 != src0 ? src2[ic] : 1.0f;\n\n float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);\n\n global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n float x0 = src[0];\n float x1 = src[1];\n\n dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;\n dst_data[1] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;\n } else {\n global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\nkernel void kernel_rope_norm_f16(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * src2,\n ulong offset2,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int n_past,\n int n_dims,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n src2 = (global float*)((global char*)src2 + offset2);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i3 = get_group_id(2);\n int i2 = get_group_id(1);\n int i1 = get_group_id(0);\n\n float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);\n\n global int * pos = src1;\n\n float theta_base = (float) pos[i2];\n float inv_ndims = -1.f/n_dims;\n\n for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {\n if (i0 < n_dims) {\n int ic = i0/2;\n\n float theta = theta_base * pow(freq_base, inv_ndims*i0);\n\n float freq_factor = src2 != src0 ? src2[ic] : 1.0f;\n\n float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);\n\n global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n float x0 = src[0];\n float x1 = src[1];\n\n dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;\n dst_data[1] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;\n } else {\n global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\nkernel void kernel_rope_neox_f32(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * src2,\n ulong offset2,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int n_past,\n int n_dims,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n src2 = (global float*)((global char*)src2 + offset2);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i3 = get_group_id(2);\n int i2 = get_group_id(1);\n int i1 = get_group_id(0);\n\n float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);\n\n global int * pos = src1;\n\n float theta_base = (float) pos[i2];\n float inv_ndims = -1.f/n_dims;\n\n for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {\n if (i0 < n_dims) {\n int ic = i0/2;\n\n const float theta = theta_base * pow(freq_base, inv_ndims*i0);\n\n const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;\n\n float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);\n\n global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);\n global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);\n\n const float x0 = src[0];\n const float x1 = src[n_dims/2];\n\n dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;\n dst_data[n_dims/2] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;\n } else {\n global float * const src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\nkernel void kernel_rope_neox_f16(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * src2,\n ulong offset2,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int n_past,\n int n_dims,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n src2 = (global float*)((global char*)src2 + offset2);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i3 = get_group_id(2);\n int i2 = get_group_id(1);\n int i1 = get_group_id(0);\n\n float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);\n\n global int * pos = src1;\n\n float theta_base = (float) pos[i2];\n float inv_ndims = -1.f/n_dims;\n\n for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {\n if (i0 < n_dims) {\n int ic = i0/2;\n\n const float theta = theta_base * pow(freq_base, inv_ndims*i0);\n\n const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;\n\n float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);\n\n global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);\n global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);\n\n const float x0 = src[0];\n const float x1 = src[n_dims/2];\n\n dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;\n dst_data[n_dims/2] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;\n } else {\n global half * const src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\nkernel void kernel_rope_multi_f32(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * src2,\n ulong offset2,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int n_past,\n int n_dims,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow,\n int4 sections,\n int is_imrope\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n src2 = (global float*)((global char*)src2 + offset2);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i3 = get_group_id(2);\n int i2 = get_group_id(1);\n int i1 = get_group_id(0);\n\n float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);\n\n global int * pos = src1;\n\n const int sect_dims = sections.s0 + sections.s1 + sections.s2 + sections.s3;\n const int sec_w = sections.s1 + sections.s0;\n\n float inv_ndims = -1.f/n_dims;\n\n for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {\n if (i0 < n_dims) {\n int ic = i0/2;\n\n const int sector = (i0 / 2) % sect_dims;\n float theta_base = 0.0f;\n\n if (is_imrope) {\n if (sector % 3 == 1 && sector < 3 * sections.s1) { // h\n theta_base = (float) pos[i2 + ne02 * 1];\n } else if (sector % 3 == 2 && sector < 3 * sections.s2) { // w\n theta_base = (float) pos[i2 + ne02 * 2];\n } else if (sector % 3 == 0 && sector < 3 * sections.s0) { // t\n theta_base = (float) pos[i2 + ne02 * 0];\n } else { // e\n theta_base = (float) pos[i2 + ne02 * 3];\n }\n } else {\n if (sector < sections.s0) {\n theta_base = pos[i2];\n }\n else if (sector >= sections.s0 && sector < sec_w) {\n theta_base = pos[i2 + ne2 * 1];\n }\n else if (sector >= sec_w && sector < sec_w + sections.s2) {\n theta_base = pos[i2 + ne2 * 2];\n }\n else if (sector >= sec_w + sections.s2) {\n theta_base = pos[i2 + ne2 * 3];\n }\n }\n\n const float theta = theta_base * pow(freq_base, inv_ndims*i0);\n\n const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;\n\n float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);\n\n global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);\n global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);\n\n const float x0 = src[0];\n const float x1 = src[n_dims/2];\n\n dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;\n dst_data[n_dims/2] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;\n } else {\n global float * const src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\nkernel void kernel_rope_multi_f16(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * src2,\n ulong offset2,\n global half * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int n_past,\n int n_dims,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow,\n int4 sections,\n int is_imrope\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n src2 = (global float*)((global char*)src2 + offset2);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i3 = get_group_id(2);\n int i2 = get_group_id(1);\n int i1 = get_group_id(0);\n\n float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);\n\n global int * pos = src1;\n\n const int sect_dims = sections.s0 + sections.s1 + sections.s2 + sections.s3;\n const int sec_w = sections.s1 + sections.s0;\n\n float inv_ndims = -1.f/n_dims;\n\n for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {\n if (i0 < n_dims) {\n int ic = i0/2;\n\n const int sector = (i0 / 2) % sect_dims;\n float theta_base = 0.0f;\n\n if (is_imrope) {\n if (sector % 3 == 1 && sector < 3 * sections.s1) { // h\n theta_base = (float) pos[i2 + ne02 * 1];\n } else if (sector % 3 == 2 && sector < 3 * sections.s2) { // w\n theta_base = (float) pos[i2 + ne02 * 2];\n } else if (sector % 3 == 0 && sector < 3 * sections.s0) { // t\n theta_base = (float) pos[i2 + ne02 * 0];\n } else { // e\n theta_base = (float) pos[i2 + ne02 * 3];\n }\n } else {\n if (sector < sections.s0) {\n theta_base = pos[i2];\n }\n else if (sector >= sections.s0 && sector < sec_w) {\n theta_base = pos[i2 + ne2 * 1];\n }\n else if (sector >= sec_w && sector < sec_w + sections.s2) {\n theta_base = pos[i2 + ne2 * 2];\n }\n else if (sector >= sec_w + sections.s2) {\n theta_base = pos[i2 + ne2 * 3];\n }\n }\n\n const float theta = theta_base * pow(freq_base, inv_ndims*i0);\n\n const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;\n\n float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);\n\n global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);\n global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);\n\n const float x0 = src[0];\n const float x1 = src[n_dims/2];\n\n dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;\n dst_data[n_dims/2] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;\n } else {\n global half * const src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n dst_data[0] = src[0];\n dst_data[1] = src[1];\n }\n }\n}\n\nkernel void kernel_rope_vision_f32(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * src2,\n ulong offset2,\n global float * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int n_past,\n int n_dims,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow,\n int4 sections\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n src2 = (global float*)((global char*)src2 + offset2);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i3 = get_group_id(2);\n int i2 = get_group_id(1);\n int i1 = get_group_id(0);\n\n float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);\n\n global int * pos = src1;\n\n const int sect_dims = sections.s0 + sections.s1;\n const int sec_w = sections.s1 + sections.s0;\n\n float inv_ndims = -1.f/n_dims;\n\n for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {\n int ic = i0/2;\n\n const int sector = (i0/2) % sect_dims;\n float theta_base = 0.0f;\n\n if (sector < sections.s0) {\n const int p = sector;\n theta_base = pos[i2] * pow(freq_base, inv_ndims*2.0f*p);\n } else if (sector >= sections.s0 && sector < sec_w) {\n const int p = sector - sections.s0;\n theta_base = pos[i2 + ne2] * pow(freq_base, inv_ndims*2.0f*p);\n }\n\n const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;\n\n float2 cos_sin_theta = rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);\n\n global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);\n global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);\n\n const float x0 = src[0];\n const float x1 = src[n_dims];\n\n dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;\n dst_data[n_dims] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;\n }\n}\n\nkernel void kernel_rope_vision_f16(\n global void * src0,\n ulong offset0,\n global int * src1,\n ulong offset1,\n global float * src2,\n ulong offset2,\n global half * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne0,\n int ne1,\n int ne2,\n int ne3,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n int n_past,\n int n_dims,\n int n_ctx_orig,\n float freq_base,\n float freq_scale,\n float ext_factor,\n float attn_factor,\n float beta_fast,\n float beta_slow,\n int4 sections\n) {\n src0 = (global void*)((global char*)src0 + offset0);\n src1 = (global int*)((global char*)src1 + offset1);\n src2 = (global float*)((global char*)src2 + offset2);\n dst = (global float*)((global char*)dst + offsetd);\n\n int i3 = get_group_id(2);\n int i2 = get_group_id(1);\n int i1 = get_group_id(0);\n\n float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);\n\n global int * pos = src1;\n\n const int sect_dims = sections.s0 + sections.s1;\n const int sec_w = sections.s1 + sections.s0;\n\n float inv_ndims = -1.f/n_dims;\n\n for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {\n int ic = i0/2;\n\n const int sector = (i0/2) % sect_dims;\n float theta_base = 0.0f;\n\n if (sector < sections.s0) {\n const int p = sector;\n theta_base = pos[i2] * pow(freq_base, inv_ndims*2.0f*p);\n } else if (sector >= sections.s0 && sector < sec_w) {\n const int p = sector - sections.s0;\n theta_base = pos[i2 + ne2] * pow(freq_base, inv_ndims*2.0f*p);\n }\n\n const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;\n\n float2 cos_sin_theta = rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);\n\n global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);\n global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);\n\n const float x0 = src[0];\n const float x1 = src[n_dims];\n\n dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;\n dst_data[n_dims] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/scale.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\nkernel void kernel_scale_f32(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n float scale,\n float bias\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n dst[get_global_id(0)] = src0[get_global_id(0)] * scale + bias;\n}\n\nkernel void kernel_scale_f32_4(\n global float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd,\n float scale,\n float bias\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n dst[get_global_id(0)] = src0[get_global_id(0)] * scale + bias;\n}\n" }, { "path": "src/ggml-opencl/kernels/set_rows.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n// v = { mp, L, d }\ninline uint fastdiv(uint n, uint4 v) {\n uint msbs;\n msbs = mul_hi(n, v.s0);\n return (msbs + n) >> v.s1;\n}\ninline uint fastmod(uint n, uint4 v) {\n uint q = fastdiv(n, v);\n return n - q * v.s2;\n}\n\nkernel void kernel_set_rows_f32_i64(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne01,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n uint4 ne11,\n uint4 ne12,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n int nblk0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);\n\n if (i01 >= ne01) {\n return;\n }\n\n //int i12 = i03%ne12;\n //int i11 = i02%ne11;\n int i12 = fastmod(i03, ne12);\n int i11 = fastmod(i02, ne11);\n\n int i10 = i01;\n long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];\n\n global float * dst_row = (global float *) (dst + i1*nb1 + i02*nb2 + i03*nb3);\n global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);\n\n for (int ind = get_local_id(0); ind < nblk0; ind += get_local_size(0)) {\n dst_row[ind] = (float)src_row[ind];\n }\n}\n\nkernel void kernel_set_rows_f16_i64(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne01,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n uint4 ne11,\n uint4 ne12,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n int nblk0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);\n\n if (i01 >= ne01) {\n return;\n }\n\n //int i12 = i03%ne12;\n //int i11 = i02%ne11;\n int i12 = fastmod(i03, ne12);\n int i11 = fastmod(i02, ne11);\n\n int i10 = i01;\n long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];\n\n global half * dst_row = (global half *) (dst + i1*nb1 + i02*nb2 + i03*nb3);\n global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);\n\n for (int ind = get_local_id(0); ind < nblk0; ind += get_local_size(0)) {\n dst_row[ind] = src_row[ind];\n }\n}\n\nkernel void kernel_set_rows_f32_i32(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne01,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n uint4 ne11,\n uint4 ne12,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n int nblk0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);\n\n if (i01 >= ne01) {\n return;\n }\n\n //int i12 = i03%ne12;\n //int i11 = i02%ne11;\n int i12 = fastmod(i03, ne12);\n int i11 = fastmod(i02, ne11);\n\n int i10 = i01;\n int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];\n\n global float * dst_row = (global float *) (dst + i1*nb1 + i02*nb2 + i03*nb3);\n global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);\n\n for (int ind = get_local_id(0); ind < nblk0; ind += get_local_size(0)) {\n dst_row[ind] = (float)src_row[ind];\n }\n}\n\nkernel void kernel_set_rows_f16_i32(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n int ne01,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n uint4 ne11,\n uint4 ne12,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n int nblk0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);\n\n if (i01 >= ne01) {\n return;\n }\n\n //int i12 = i03%ne12;\n //int i11 = i02%ne11;\n int i12 = fastmod(i03, ne12);\n int i11 = fastmod(i02, ne11);\n\n int i10 = i01;\n int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];\n\n global half * dst_row = (global half *) (dst + i1*nb1 + i02*nb2 + i03*nb3);\n global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);\n\n for (int ind = get_local_id(0); ind < nblk0; ind += get_local_size(0)) {\n dst_row[ind] = src_row[ind];\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/sigmoid.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// sigmoid\n//------------------------------------------------------------------------------\n\nkernel void kernel_sigmoid_f32(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = 1.0f / (1.0f + exp(-src0[get_global_id(0)]));\n}\n\nkernel void kernel_sigmoid_f16(\n global half * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd\n) {\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = 1.0f / (1.0f + exp(-src0[get_global_id(0)]));\n}\n" }, { "path": "src/ggml-opencl/kernels/silu.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// silu\n//------------------------------------------------------------------------------\nkernel void kernel_silu(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n float x = src0[get_global_id(0)];\n dst[get_global_id(0)] = x / (1.0f + exp(-x));\n}\n\nkernel void kernel_silu_4(\n global float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n float4 x = src0[get_global_id(0)];\n dst[get_global_id(0)] = x / (1.0f + exp(-x));\n}\n" }, { "path": "src/ggml-opencl/kernels/softmax_4_f16.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_soft_max_4_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * src2,\n ulong offset2,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n int ne13,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n float scale,\n float max_bias,\n float m0,\n float m1,\n int n_head_log2\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n src2 = src2 + offset2;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03%ne13;\n int i12 = i02%ne12;\n int i11 = i01;\n\n global float4 * psrc4 = (global float4 *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);\n global half4 * pmask = src1 != src0 ? (global half4 *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0;\n global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0;\n global float4 * pdst4 = (global float4 *)(dst + i01*nb1 + i02*nb2 + i03*nb3);\n\n float slope = 1.0f;\n\n // ALiBi\n if (max_bias > 0.0f) {\n int h = i02;\n\n float base = h < n_head_log2 ? m0 : m1;\n int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n slope = pow(base, exp);\n }\n\n // parallel max\n float4 lmax4 = psrc2 ? psrc2[i02] : -INFINITY;\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n lmax4 = fmax(lmax4, psrc4[i00]*scale + slope*(pmask ? convert_float4(pmask[i00]) : 0.0f));\n }\n float lmax = fmax(fmax(lmax4.s0, lmax4.s1), fmax(lmax4.s2, lmax4.s3));\n\n const float max = sub_group_reduce_max(lmax);\n\n // parallel sum\n float4 lsum4 = 0.0f;\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n const float4 exp_psrc4 = exp((psrc4[i00]*scale + slope*(pmask ? convert_float4(pmask[i00]) : 0.0f)) - max);\n lsum4 += exp_psrc4;\n pdst4[i00] = exp_psrc4;\n }\n float lsum = lsum4.s0 + lsum4.s1 + lsum4.s2 + lsum4.s3;\n\n float sum = sub_group_reduce_add(lsum);\n\n if (psrc2) {\n sum += exp(psrc2[i02] - max);\n }\n\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n pdst4[i00] /= sum;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/softmax_4_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_soft_max_4(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * src2,\n ulong offset2,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n int ne13,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n float scale,\n float max_bias,\n float m0,\n float m1,\n int n_head_log2\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n src2 = src2 + offset2;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03%ne13;\n int i12 = i02%ne12;\n int i11 = i01;\n\n global float4 * psrc4 = (global float4 *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);\n global float4 * pmask = src1 != src0 ? (global float4 *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0;\n global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0;\n global float4 * pdst4 = (global float4 *)(dst + i01*nb1 + i02*nb2 + i03*nb3);\n\n float slope = 1.0f;\n\n // ALiBi\n if (max_bias > 0.0f) {\n int h = i02;\n\n float base = h < n_head_log2 ? m0 : m1;\n int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n slope = pow(base, exp);\n }\n\n // parallel max\n float4 lmax4 = psrc2 ? psrc2[i02] : -INFINITY;\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f));\n }\n float lmax = fmax(fmax(lmax4.s0, lmax4.s1), fmax(lmax4.s2, lmax4.s3));\n\n const float max = sub_group_reduce_max(lmax);\n\n // parallel sum\n float4 lsum4 = 0.0f;\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)) - max);\n lsum4 += exp_psrc4;\n pdst4[i00] = exp_psrc4;\n }\n float lsum = lsum4.s0 + lsum4.s1 + lsum4.s2 + lsum4.s3;\n\n float sum = sub_group_reduce_add(lsum);\n\n if (psrc2) {\n sum += exp(psrc2[i02] - max);\n }\n\n for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {\n pdst4[i00] /= sum;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/softmax_f16.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_soft_max_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * src2,\n ulong offset2,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n int ne13,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n float scale,\n float max_bias,\n float m0,\n float m1,\n int n_head_log2\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n src2 = src2 + offset2;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03%ne13;\n int i12 = i02%ne12;\n int i11 = i01;\n\n global float * psrc0 = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);\n global half * pmask = src1 != src0 ? (global half *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0;\n global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0;\n global float * pdst = (global float *)(dst + i01*nb1 + i02*nb2 + i03*nb3);\n\n float slope = 1.0f;\n\n // ALiBi\n if (max_bias > 0.0f) {\n int h = i02;\n\n float base = h < n_head_log2 ? m0 : m1;\n int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n slope = pow(base, exp);\n }\n\n // parallel max\n float lmax = psrc2 ? psrc2[i02] : -INFINITY;\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n lmax = fmax(lmax, psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f));\n }\n float max = sub_group_reduce_max(lmax);\n\n // parallel sum\n float lsum = 0.0f;\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)) - max);\n lsum += exp_psrc0;\n // Remember the result of exp here. exp is expensive, so we really do not\n // wish to compute it twice.\n pdst[i00] = exp_psrc0;\n }\n\n float sum = sub_group_reduce_add(lsum);\n\n if (psrc2) {\n sum += exp(psrc2[i02] - max);\n }\n\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n pdst[i00] /= sum;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/softmax_f32.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n#ifdef cl_intel_subgroups\n#pragma OPENCL EXTENSION cl_intel_subgroups : enable\n#else\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n#endif\n\n#ifdef cl_intel_required_subgroup_size\n#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable\n#define INTEL_GPU 1\n#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))\n#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))\n#elif defined(cl_qcom_reqd_sub_group_size)\n#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable\n#define ADRENO_GPU 1\n#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size(\"half\")))\n#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size(\"full\")))\n#endif\n\n#ifdef ADRENO_GPU\nREQD_SUBGROUP_SIZE_64\n#endif\nkernel void kernel_soft_max(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * src2,\n ulong offset2,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne12,\n int ne13,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n ulong nb1,\n ulong nb2,\n ulong nb3,\n float scale,\n float max_bias,\n float m0,\n float m1,\n int n_head_log2\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n src2 = src2 + offset2;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03%ne13;\n int i12 = i02%ne12;\n int i11 = i01;\n\n global float * psrc0 = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);\n global float * pmask = src1 != src0 ? (global float *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0;\n global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0;\n global float * pdst = (global float *)(dst + i01*nb1 + i02*nb2 + i03*nb3);\n\n float slope = 1.0f;\n\n // ALiBi\n if (max_bias > 0.0f) {\n int h = i02;\n\n float base = h < n_head_log2 ? m0 : m1;\n int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n slope = pow(base, exp);\n }\n\n // parallel max\n float lmax = psrc2 ? psrc2[i02] : -INFINITY;\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n lmax = fmax(lmax, psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f));\n }\n float max = sub_group_reduce_max(lmax);\n\n // parallel sum\n float lsum = 0.0f;\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)) - max);\n lsum += exp_psrc0;\n // Remember the result of exp here. exp is expensive, so we really do not\n // wish to compute it twice.\n pdst[i00] = exp_psrc0;\n }\n\n float sum = sub_group_reduce_add(lsum);\n\n if (psrc2) {\n sum += exp(psrc2[i02] - max);\n }\n\n for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {\n pdst[i00] /= sum;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/softplus.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// softplus\n//------------------------------------------------------------------------------\n\nkernel void kernel_softplus_f32(\n global const float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = (src0[get_global_id(0)] > 20.0f) ? src0[get_global_id(0)] : log(1.0f + exp(src0[get_global_id(0)]));\n}\n\nkernel void kernel_softplus_f32_4(\n global const float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = (src0[get_global_id(0)] > 20.0f) ? src0[get_global_id(0)] : log(1.0f + exp(src0[get_global_id(0)]));\n}\n\nkernel void kernel_softplus_f16(\n global const half * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd\n) {\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n const float x = convert_float(src0[get_global_id(0)]);\n dst[get_global_id(0)] = convert_half_rte((x > 20.0f) ? x : log(1.0f + exp(x)));\n}\n\nkernel void kernel_softplus_f16_4(\n global const half4 * src0,\n ulong offset0,\n global half4 * dst,\n ulong offsetd\n) {\n src0 = (global half4*)((global char*)src0 + offset0);\n dst = (global half4*)((global char*)dst + offsetd);\n\n const float4 x = convert_float4(src0[get_global_id(0)]);\n dst[get_global_id(0)] = convert_half4_rte((x > 20.0f) ? x : log(1.0f + exp(x)));\n}\n\nkernel void kernel_softplus_f32_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = (*x > 20.0f) ? *x : log(1.0f + exp(*x));\n }\n}\n\nkernel void kernel_softplus_f16_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const half * hx = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global half * hy = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n const float x = convert_float(*hx);\n *hy = convert_half_rte((x > 20.0f) ? x : log(1.0f + exp(x)));\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/solve_tri.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// solve_tri\n//------------------------------------------------------------------------------\nkernel void kernel_solve_tri_f32(\n global uchar * src0,\n ulong offset0,\n global uchar * src1,\n ulong offset1,\n global uchar * dst,\n ulong offsetd,\n int n,\n int k,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n int col = get_global_id(0);\n int i2 = get_global_id(1);\n int i3 = get_global_id(2);\n\n global const uchar * Lb = src0 + offset0 + i2 * nb02 + i3 * nb03;\n global const uchar * Bb = src1 + offset1 + i2 * nb12 + i3 * nb13;\n global uchar * Xb = dst + offsetd + i2 * nb2 + i3 * nb3;\n\n for(int row = 0; row < n; ++row){\n global const float *pB = (global const float *)(Bb + row * nb11 + col * nb10);\n\n float sum = 0.0f;\n for(int j = 0; j < row; ++j){\n global const float *pL = (global const float *)(Lb + row * nb01 + j * nb00);\n global const float *pX = (global const float *)(Xb + j * nb1 + col * nb0);\n sum += (*pL) * (*pX);\n }\n\n global const float * pDiag = (global const float *)(Lb + row * nb01 + row *nb00);\n global float * pOut = (global float *)(Xb + row * nb1 + col *nb0);\n\n *pOut = ((* pB) - sum) / (*pDiag);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/sqr.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\nkernel void kernel_sqr_cont_f32(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n uint gid = get_global_id(0);\n dst[gid] = src0[gid] * src0[gid];\n}\n\nkernel void kernel_sqr_cont_f32_4(\n global float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n uint gid = get_global_id(0);\n dst[gid] = src0[gid] * src0[gid];\n}\n\nkernel void kernel_sqr_cont_f16(\n global half * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd\n) {\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n uint gid = get_global_id(0);\n dst[gid] = src0[gid] * src0[gid];\n}\n\nkernel void kernel_sqr_cont_f16_4(\n global half4 * src0,\n ulong offset0,\n global half4 * dst,\n ulong offsetd\n) {\n src0 = (global half4*)((global char*)src0 + offset0);\n dst = (global half4*)((global char*)dst + offsetd);\n\n uint gid = get_global_id(0);\n dst[gid] = src0[gid] * src0[gid];\n}\n" }, { "path": "src/ggml-opencl/kernels/sqrt.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\nkernel void kernel_sqrt_cont_f32(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n uint gid = get_global_id(0);\n dst[gid] = sqrt(src0[gid]);\n}\n\nkernel void kernel_sqrt_cont_f32_4(\n global float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n uint gid = get_global_id(0);\n dst[gid] = sqrt(src0[gid]);\n}\n\nkernel void kernel_sqrt_cont_f16(\n global half * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd\n) {\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n uint gid = get_global_id(0);\n dst[gid] = convert_half(sqrt(convert_float(src0[gid])));\n}\n\nkernel void kernel_sqrt_cont_f16_4(\n global half4 * src0,\n ulong offset0,\n global half4 * dst,\n ulong offsetd\n) {\n src0 = (global half4*)((global char*)src0 + offset0);\n dst = (global half4*)((global char*)dst + offsetd);\n\n uint gid = get_global_id(0);\n dst[gid] = convert_half4(sqrt(convert_float4(src0[gid])));\n}\n" }, { "path": "src/ggml-opencl/kernels/ssm_conv.cl", "content": "kernel void kernel_ssm_conv_f32_f32(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n int ne10,\n ulong nb11,\n ulong nb0,\n ulong nb1,\n ulong nb2\n){\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int ir = get_global_id(0);\n int i2 = get_global_id(1);\n int i3 = get_global_id(2);\n\n int nc = ne10;\n\n global float * s = (global float *) (src0 + ir*nb01 + i2*nb00 + i3*nb02);\n global float * c = (global float *) (src1 + ir*nb11);\n global float * d = (global float *) (dst + ir*nb0 + i2*nb1 + i3*nb2);\n\n float sumf = 0.0f;\n\n for (int i0 = 0; i0 < nc; ++i0) {\n sumf += s[i0] * c[i0];\n }\n\n d[0] = sumf;\n}\n\nkernel void kernel_ssm_conv_f32_f32_4(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n int ne10,\n ulong nb11,\n ulong nb0,\n ulong nb1,\n ulong nb2\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int ir = get_global_id(0);\n int i2 = get_global_id(1);\n int i3 = get_global_id(2);\n\n int nc = ne10;\n\n global float4 * s = (global float4 *) (src0 + ir*nb01 + i2*nb00 + i3*nb02);\n global float4 * c = (global float4 *) (src1 + ir*nb11);\n global float * d = (global float *) (dst + ir*nb0 + i2*nb1 + i3*nb2);\n\n float sumf = 0.0f;\n\n for (int i0 = 0; i0 < nc/4; ++i0) {\n sumf += dot(s[i0], c[i0]);\n }\n\n d[0] = sumf;\n}\n" }, { "path": "src/ggml-opencl/kernels/sub.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// div\n//------------------------------------------------------------------------------\nkernel void kernel_sub(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03 % ne13;\n int i12 = i02 % ne12;\n int i11 = i01 % ne11;\n\n global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;\n global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;\n global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const int i10 = i0 % ne10;\n *((global float *)(dst_ptr + i0*nb0)) = *((global float *)(src0_ptr + i0*nb00)) - *((global float *)(src1_ptr + i10*nb10));\n }\n}\n\n// assumption: src1 is a row\n// broadcast src1 into src0\nkernel void kernel_sub_row(\n global float4 * src0,\n ulong offset0,\n global float4 * src1,\n ulong offset1,\n global float4 * dst,\n ulong offsetd,\n int ne\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n src1 = (global float4*)((global char*)src1 + offset1);\n dst = (global float4*)((global char*)dst + offsetd);\n\n // This performs better than using %.\n uint gid = get_global_id(0);\n uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne\n dst[gid] = src0[gid] - src1[idx1];\n}\n\nkernel void kernel_sub_f16(\n global char * src0,\n ulong offset0,\n global char * src1,\n ulong offset1,\n global char * dst,\n ulong offsetd,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n ulong nb10,\n ulong nb11,\n ulong nb12,\n ulong nb13,\n int ne0,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n src1 = src1 + offset1;\n dst = dst + offsetd;\n\n int i03 = get_group_id(2);\n int i02 = get_group_id(1);\n int i01 = get_group_id(0);\n\n int i13 = i03 % ne13;\n int i12 = i02 % ne12;\n int i11 = i01 % ne11;\n\n global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;\n global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;\n global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;\n\n for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {\n const int i10 = i0 % ne10;\n *((global half *)(dst_ptr + i0*nb0)) = *((global half *)(src0_ptr + i0*nb00)) - *((global half *)(src1_ptr + i10*nb10));\n }\n}\n\nkernel void kernel_sub_row_f16(\n global half4 * src0,\n ulong offset0,\n global half4 * src1,\n ulong offset1,\n global half4 * dst,\n ulong offsetd,\n int ne\n) {\n src0 = (global half4*)((global char*)src0 + offset0);\n src1 = (global half4*)((global char*)src1 + offset1);\n dst = (global half4*)((global char*)dst + offsetd);\n\n // This performs better than using %.\n uint gid = get_global_id(0);\n uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne\n dst[gid] = src0[gid] - src1[idx1];\n}\n" }, { "path": "src/ggml-opencl/kernels/sum_rows.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n\n// Most devices have max workgroup size of 1024, so this is enough for subgroup\n// sizes of 16, 32, 64 and 128. Increase this value for smaller subgroups sizes\n#define MAX_SUBGROUPS 64\nkernel void kernel_sum_rows_f32(\n global char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n const int lid = get_local_id(0);\n const int lsize = get_local_size(0);\n\n const uint sg_size = get_sub_group_size();\n const uint sg_id = get_sub_group_id();\n const uint sg_lid = get_sub_group_local_id();\n\n __local float lmem[MAX_SUBGROUPS];\n\n if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {\n return;\n }\n\n if(sg_id == 0){\n lmem[sg_lid] = 0.0f;\n }\n\n global float * src_row = (global float *) (src0 + i1*nb01 + i2*nb02 + i3*nb03);\n global float * dst_row = (global float *) (dst + i1*nb1 + i2*nb2 + i3*nb3);\n\n float sumf = 0.0f;\n\n for (int i0 = lid; i0 < ne00; i0 += lsize) {\n sumf += src_row[i0];\n }\n\n sumf = sub_group_reduce_add(sumf);\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if(sg_lid == 0){\n lmem[sg_id] = sumf;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n sumf = lmem[sg_lid];\n sumf = sub_group_reduce_add(sumf);\n\n if (lid == 0) {\n dst_row[0] = sumf;\n }\n}\n\nkernel void kernel_sum_rows_f32_4(\n global char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n int ne01,\n int ne02,\n int ne03,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n const int lid = get_local_id(0);\n const int lsize = get_local_size(0);\n\n const uint sg_size = get_sub_group_size();\n const uint sg_id = get_sub_group_id();\n const uint sg_lid = get_sub_group_local_id();\n\n __local float lmem[MAX_SUBGROUPS];\n\n if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {\n return;\n }\n\n if(sg_id == 0){\n lmem[sg_lid] = 0.0f;\n }\n\n global float4 * src_row = (global float4 *) (src0 + i1*nb01 + i2*nb02 + i3*nb03);\n global float * dst_row = (global float *) (dst + i1*nb1 + i2*nb2 + i3*nb3);\n\n float4 sum_vec = (float4)0.0f;\n\n for (int i0 = lid; i0 < ne00 / 4; i0 += lsize) {\n sum_vec += src_row[i0];\n }\n\n float sumf = dot(sum_vec, (float4)(1.0f));\n sumf = sub_group_reduce_add(sumf);\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n if(sg_lid == 0){\n lmem[sg_id] = sumf;\n }\n\n barrier(CLK_LOCAL_MEM_FENCE);\n\n sumf = lmem[sg_lid];\n sumf = sub_group_reduce_add(sumf);\n\n if (lid == 0) {\n dst_row[0] = sumf;\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/tanh.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\nkernel void kernel_tanh_f32(\n global const float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = tanh(src0[get_global_id(0)]);\n}\n\nkernel void kernel_tanh_f32_4(\n global const float4 * src0,\n ulong offset0,\n global float4 * dst,\n ulong offsetd\n) {\n src0 = (global float4*)((global char*)src0 + offset0);\n dst = (global float4*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = tanh(src0[get_global_id(0)]);\n}\n\nkernel void kernel_tanh_f16(\n global const half * src0,\n ulong offset0,\n global half * dst,\n ulong offsetd\n) {\n src0 = (global half*)((global char*)src0 + offset0);\n dst = (global half*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = tanh(src0[get_global_id(0)]);\n}\n\nkernel void kernel_tanh_f16_4(\n global const half4 * src0,\n ulong offset0,\n global half4 * dst,\n ulong offsetd\n) {\n src0 = (global half4*)((global char*)src0 + offset0);\n dst = (global half4*)((global char*)dst + offsetd);\n\n dst[get_global_id(0)] = tanh(src0[get_global_id(0)]);\n}\n\nkernel void kernel_tanh_f32_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = tanh(*x);\n }\n}\n\nkernel void kernel_tanh_f16_nc(\n global const char * src0,\n ulong offset0,\n global char * dst,\n ulong offsetd,\n int ne00,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n ulong nb0,\n ulong nb1,\n ulong nb2,\n ulong nb3\n) {\n src0 = src0 + offset0;\n dst = dst + offsetd;\n\n const int i3 = get_group_id(2);\n const int i2 = get_group_id(1);\n const int i1 = get_group_id(0);\n\n for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\n global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n *y = tanh(*x);\n }\n}\n" }, { "path": "src/ggml-opencl/kernels/transpose.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n// 16-bit transpose, loading/storing a 4x4 tile of elements\nkernel void kernel_transpose_16(\n __read_only image1d_buffer_t input,\n __write_only image1d_buffer_t output,\n const uint rows,\n const uint cols\n) {\n\n const int i = get_global_id(0);\n const int j = get_global_id(1);\n const int i_2 = i<<2;\n const int j_2 = j<<2;\n\n half4 temp0 = read_imageh(input, (j_2+0)*cols+i);\n half4 temp1 = read_imageh(input, (j_2+1)*cols+i);\n half4 temp2 = read_imageh(input, (j_2+2)*cols+i);\n half4 temp3 = read_imageh(input, (j_2+3)*cols+i);\n\n write_imageh(output, (i_2+0)*rows+j, (half4)(temp0.s0, temp1.s0, temp2.s0, temp3.s0));\n write_imageh(output, (i_2+1)*rows+j, (half4)(temp0.s1, temp1.s1, temp2.s1, temp3.s1));\n write_imageh(output, (i_2+2)*rows+j, (half4)(temp0.s2, temp1.s2, temp2.s2, temp3.s2));\n write_imageh(output, (i_2+3)*rows+j, (half4)(temp0.s3, temp1.s3, temp2.s3, temp3.s3));\n}\n\n// Padded kernel for irregular shape\nkernel void kernel_transpose_16_4x1(\n __read_only image1d_buffer_t input,\n __write_only image1d_buffer_t output,\n const uint rows,\n const uint cols\n) {\n\n const int i = get_global_id(0);\n const int j = get_global_id(1);\n const int j_2 = j << 2;\n\n half temp0 = read_imageh(input, (j_2 + 0) * cols + i).x;\n half temp1 = read_imageh(input, (j_2 + 1) * cols + i).x;\n half temp2 = read_imageh(input, (j_2 + 2) * cols + i).x;\n half temp3 = read_imageh(input, (j_2 + 3) * cols + i).x;\n\n write_imageh(output, i * rows + j, (half4)(temp0, temp1, temp2, temp3));\n}\n\n// Transpose treating each element as 8-bit using buffer\nkernel void kernel_transpose_8_buf(\n global const uchar * input,\n global uchar * output,\n const int ldi,\n const int ldo\n) {\n const int x = get_global_id(0);\n const int y = get_global_id(1);\n\n output[x*ldo + y] = input[y*ldi + x];\n}\n\n// Transpose treating each element as 16-bit using buffer\nkernel void kernel_transpose_16_buf(\n global const ushort * input,\n global ushort * output,\n const int ldi,\n const int ldo\n) {\n const int x = get_global_id(0);\n const int y = get_global_id(1);\n\n output[x*ldo + y] = input[y*ldi + x];\n}\n\n// Transpose treating each element as 32-bit using buffer\nkernel void kernel_transpose_32_buf(\n global const uint * input,\n global uint * output,\n const int ldi,\n const int ldo\n) {\n const int x = get_global_id(0);\n const int y = get_global_id(1);\n\n output[x*ldo + y] = input[y*ldi + x];\n}\n\n// 32-bit transpose, loading/storing a 4x4 tile of elements\nkernel void kernel_transpose_32(\n __read_only image1d_buffer_t input,\n __write_only image1d_buffer_t output,\n const uint rows,\n const uint cols\n) {\n\n const int i = get_global_id(0);\n const int j = get_global_id(1);\n const int i_2 = i<<2;\n const int j_2 = j<<2;\n\n float4 temp0 = read_imagef(input, (j_2+0)*cols+i);\n float4 temp1 = read_imagef(input, (j_2+1)*cols+i);\n float4 temp2 = read_imagef(input, (j_2+2)*cols+i);\n float4 temp3 = read_imagef(input, (j_2+3)*cols+i);\n\n write_imagef(output, (i_2+0)*rows+j, (float4)(temp0.s0, temp1.s0, temp2.s0, temp3.s0));\n write_imagef(output, (i_2+1)*rows+j, (float4)(temp0.s1, temp1.s1, temp2.s1, temp3.s1));\n write_imagef(output, (i_2+2)*rows+j, (float4)(temp0.s2, temp1.s2, temp2.s2, temp3.s2));\n write_imagef(output, (i_2+3)*rows+j, (float4)(temp0.s3, temp1.s3, temp2.s3, temp3.s3));\n\n}\n\n// 32-bit transpose, loading/storing a 4x4 tile of elements\n// Only used for activations\n// converts to FP16\n// also adds zero padding for non multiple of 8 prompt lengths\nkernel void kernel_transpose_32_16(__read_only image1d_buffer_t input, __write_only image1d_buffer_t output, const uint rows, const uint cols, const uint padded_rows) {\n\n const int i = get_global_id(0);\n const int j = get_global_id(1);\n const int i_2 = i<<2;\n const int j_2 = j<<2;\n half4 temp0 = {0,0,0,0}; // initialize outputs to 0\n half4 temp1 = {0,0,0,0};\n half4 temp2 = {0,0,0,0};\n half4 temp3 = {0,0,0,0};\n\n if((j_2+0)*cols+i*4+3 < rows*cols*16){ // only load from a valid location. Otherwise keep register data as 0\n temp0 = read_imageh(input, (j_2+0)*cols+i);\n }\n if((j_2+1)*cols+i*4+3 < rows*cols*16){\n temp1 = read_imageh(input, (j_2+1)*cols+i);\n }\n if((j_2+2)*cols+i*4+3 < rows*cols*16){\n temp2 = read_imageh(input, (j_2+2)*cols+i);\n }\n if((j_2+3)*cols+i*4+3 < rows*cols*16){\n temp3 = read_imageh(input, (j_2+3)*cols+i);\n }\n\n write_imageh(output, (i_2+0)*padded_rows+j, (half4)(temp0.s0, temp1.s0, temp2.s0, temp3.s0)); // no conditionals for output, includes zero padding\n write_imageh(output, (i_2+1)*padded_rows+j, (half4)(temp0.s1, temp1.s1, temp2.s1, temp3.s1));\n write_imageh(output, (i_2+2)*padded_rows+j, (half4)(temp0.s2, temp1.s2, temp2.s2, temp3.s2));\n write_imageh(output, (i_2+3)*padded_rows+j, (half4)(temp0.s3, temp1.s3, temp2.s3, temp3.s3));\n}\n" }, { "path": "src/ggml-opencl/kernels/tri.cl", "content": "#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\n//------------------------------------------------------------------------------\n// tri\n//------------------------------------------------------------------------------\n__kernel void kernel_tri_f32(\n global float * src0,\n ulong offset0,\n global float * dst,\n ulong offsetd,\n int n,\n int ne0,\n int ne1,\n int tri_type\n) {\n src0 = (global float*)((global char*)src0 + offset0);\n dst = (global float*)((global char*)dst + offsetd);\n\n int idx = get_global_id(0);\n if (idx >= n) return;\n\n int i0 = idx % ne0;\n int i1 = (idx / ne0) % ne1;\n\n int keep = 0;\n if (tri_type == 0) keep = (i0 >= i1);\n else if (tri_type == 1) keep = (i0 > i1);\n else if (tri_type == 2) keep = (i0 <= i1);\n else keep = (i0 < i1);\n\n dst[idx] = keep ? src0[idx] : 0.0f;\n}\n" }, { "path": "src/ggml-opencl/kernels/tsembd.cl", "content": "kernel void kernel_timestep_embedding(\n global const void * p_timesteps,\n ulong off_timesteps,\n global void * p_dst,\n ulong off_dst,\n int dst_nb1_bytes,\n int logical_dim,\n int max_period\n) {\n int local_i;\n int local_j;\n int local_half_dim;\n float local_timestep_val;\n float local_freq;\n float local_arg;\n global float * local_embed_data_ptr;\n global const float * local_timesteps_input_ptr;\n global float * local_dst_output_base_ptr;\n\n local_timesteps_input_ptr = (global const float *)((global char *)p_timesteps + off_timesteps);\n local_dst_output_base_ptr = (global float *)((global char *)p_dst + off_dst);\n\n local_i = get_global_id(1);\n local_j = get_global_id(0);\n\n local_half_dim = logical_dim / 2;\n local_embed_data_ptr = (global float *)((global char *)local_dst_output_base_ptr + local_i * dst_nb1_bytes);\n\n if (logical_dim % 2 != 0 && local_j == local_half_dim) {\n local_embed_data_ptr[2 * local_half_dim] = 0.0f;\n }\n\n if (local_j >= local_half_dim) {\n return;\n }\n\n local_timestep_val = local_timesteps_input_ptr[local_i];\n\n if (local_half_dim == 0) {\n local_freq = 1.0f;\n } else {\n local_freq = exp(-log((float)max_period) * (float)local_j / (float)local_half_dim);\n }\n\n local_arg = local_timestep_val * local_freq;\n local_embed_data_ptr[local_j] = cos(local_arg);\n local_embed_data_ptr[local_j + local_half_dim] = sin(local_arg);\n}\n" }, { "path": "src/ggml-opencl/kernels/upscale.cl", "content": "kernel void kernel_upscale(\n global const void * p_src0,\n ulong off_src0,\n global void * p_dst,\n ulong off_dst,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne10,\n int ne11,\n int ne12,\n int ne13,\n float sf0,\n float sf1,\n float sf2,\n float sf3\n) {\n global const char * src_base = (global const char *)p_src0 + off_src0;\n global float * dst_base = (global float *)((global char *)p_dst + off_dst);\n\n int index = get_global_id(0);\n int dst_total_elements = ne10 * ne11 * ne12 * ne13;\n\n if (index >= dst_total_elements) {\n return;\n }\n\n int i10 = index % ne10;\n int i11 = (index / ne10) % ne11;\n int i12 = (index / (ne10 * ne11)) % ne12;\n int i13 = index / (ne10 * ne11 * ne12);\n\n int i00 = (int)(i10 / sf0);\n int i01 = (int)(i11 / sf1);\n int i02 = (int)(i12 / sf2);\n int i03 = (int)(i13 / sf3);\n\n ulong offset_src_element = (ulong)i03 * nb03 + (ulong)i02 * nb02 + (ulong)i01 * nb01 + (ulong)i00 * nb00;\n global const float * src_element_ptr = (global const float *)(src_base + offset_src_element);\n\n dst_base[index] = *src_element_ptr;\n}\n\nkernel void kernel_upscale_bilinear(\n global const void * p_src0,\n ulong off_src0,\n global void * p_dst,\n ulong off_dst,\n ulong nb00,\n ulong nb01,\n ulong nb02,\n ulong nb03,\n int ne00_src,\n int ne01_src,\n int ne10_dst,\n int ne11_dst,\n int ne12_dst,\n int ne13_dst,\n float sf0,\n float sf1,\n float sf2,\n float sf3,\n float pixel_offset\n) {\n global const char * src_base = (global const char *)p_src0 + off_src0;\n global float * dst_base = (global float *)((global char *)p_dst + off_dst);\n\n int index = get_global_id(0);\n int dst_total_elements = ne10_dst * ne11_dst * ne12_dst * ne13_dst;\n\n if (index >= dst_total_elements) {\n return;\n }\n\n int i10_dst = index % ne10_dst;\n int i11_dst = (index / ne10_dst) % ne11_dst;\n int i12_dst = (index / (ne10_dst * ne11_dst)) % ne12_dst;\n int i13_dst = index / (ne10_dst * ne11_dst * ne12_dst);\n\n int i02_src = (int)(i12_dst / sf2);\n int i03_src = (int)(i13_dst / sf3);\n\n float y_src_f = ((float)i11_dst + pixel_offset) / sf1 - pixel_offset;\n long y0_src = (long)floor(y_src_f);\n long y1_src = y0_src + 1;\n\n y0_src = max(0L, min(y0_src, (long)ne01_src - 1));\n y1_src = max(0L, min(y1_src, (long)ne01_src - 1));\n\n float dy = y_src_f - (float)y0_src;\n dy = max(0.0f, min(dy, 1.0f));\n\n float x_src_f = ((float)i10_dst + pixel_offset) / sf0 - pixel_offset;\n long x0_src = (long)floor(x_src_f);\n long x1_src = x0_src + 1;\n\n x0_src = max(0L, min(x0_src, (long)ne00_src - 1));\n x1_src = max(0L, min(x1_src, (long)ne00_src - 1));\n\n float dx = x_src_f - (float)x0_src;\n dx = max(0.0f, min(dx, 1.0f));\n\n global const float * p_a = (global const float *)(src_base + (ulong)x0_src * nb00 + (ulong)y0_src * nb01 + (ulong)i02_src * nb02 + (ulong)i03_src * nb03);\n global const float * p_b = (global const float *)(src_base + (ulong)x1_src * nb00 + (ulong)y0_src * nb01 + (ulong)i02_src * nb02 + (ulong)i03_src * nb03);\n global const float * p_c = (global const float *)(src_base + (ulong)x0_src * nb00 + (ulong)y1_src * nb01 + (ulong)i02_src * nb02 + (ulong)i03_src * nb03);\n global const float * p_d = (global const float *)(src_base + (ulong)x1_src * nb00 + (ulong)y1_src * nb01 + (ulong)i02_src * nb02 + (ulong)i03_src * nb03);\n\n const float val_a = *p_a;\n const float val_b = *p_b;\n const float val_c = *p_c;\n const float val_d = *p_d;\n\n float result = val_a * (1.0f - dx) * (1.0f - dy) +\n val_b * dx * (1.0f - dy) +\n val_c * (1.0f - dx) * dy +\n val_d * dx * dy;\n\n dst_base[index] = result;\n}\n" }, { "path": "src/ggml-openvino/.clang-format", "content": "---\n# Override root .clang-format\nAlignConsecutiveAssignments: false\nAlignConsecutiveDeclarations: false\nCpp11BracedListStyle: true\nSpacesInContainerLiterals: false\nBreakBeforeBraces: Attach\nAccessModifierOffset: -4\nIndentCaseBlocks: false\nIndentCaseLabels: false\n\nLanguage: Cpp\nAlignAfterOpenBracket: Align\nAlignArrayOfStructures: Left\nAlignConsecutiveBitFields: AcrossComments\nAlignConsecutiveMacros: AcrossComments\n# AlignConsecutiveShortCaseStatements: AcrossComments\nAlignEscapedNewlines: Left # LeftWithLastLine\nAlignOperands: Align\nAlignTrailingComments:\n Kind: Always\n OverEmptyLines: 1\nAllowAllArgumentsOnNextLine: true\nAllowAllParametersOfDeclarationOnNextLine: false\n# AllowBreakBeforeNoexceptSpecifier: OnlyWithParen\nAllowShortBlocksOnASingleLine: Never\nAllowShortCaseLabelsOnASingleLine: false\nAllowShortFunctionsOnASingleLine: Inline\nAllowShortIfStatementsOnASingleLine: Never\nAllowShortLambdasOnASingleLine: Inline\nAllowShortLoopsOnASingleLine: false\nAlwaysBreakBeforeMultilineStrings: true\n# Treat CUDA keywords/attributes as \"attribute macros\" and avoid breaking lines inside them\nAttributeMacros:\n - __host__\n - __device__\n - __global__\n - __forceinline__\n - __launch_bounds__\nBinPackArguments: true\nBinPackParameters: false # OnePerLine\nBitFieldColonSpacing: Both\n# BreakAdjacentStringLiterals: true\nBreakAfterAttributes: Never\nBreakBeforeBinaryOperators: None\nBreakBeforeInlineASMColon: OnlyMultiline\nBreakBeforeTernaryOperators: false\n# BreakBinaryOperations: Never\nBreakConstructorInitializers: AfterColon\n# BreakFunctionDefinitionParameters: false\nBreakInheritanceList: AfterComma\nBreakStringLiterals: true\n# BreakTemplateDeclarations: Yes\nColumnLimit: 120\nCommentPragmas: '^ IWYU pragma:'\nCompactNamespaces: false\nConstructorInitializerIndentWidth: 4\nContinuationIndentWidth: 4\nDerivePointerAlignment: false\nDisableFormat: false\nEmptyLineBeforeAccessModifier: Leave\nEmptyLineAfterAccessModifier: Never\nExperimentalAutoDetectBinPacking: false\nFixNamespaceComments: true\nIncludeBlocks: Regroup\nIncludeCategories:\n - Regex: '\".*\"'\n Priority: 1\n SortPriority: 0\n - Regex: '^<.*\\.h>'\n Priority: 2\n SortPriority: 0\n - Regex: '^<.*'\n Priority: 3\n SortPriority: 0\n - Regex: '.*'\n Priority: 4\n SortPriority: 0\nIncludeIsMainRegex: '([-_](test|unittest))?$'\nIncludeIsMainSourceRegex: ''\nIndentAccessModifiers: false\nIndentExternBlock: NoIndent\nIndentGotoLabels: false\nIndentPPDirectives: AfterHash\nIndentWidth: 4\nIndentWrappedFunctionNames: false\nInsertBraces: true # NOTE: may lead to incorrect formatting\nInsertNewlineAtEOF: true\nJavaScriptQuotes: Leave\nJavaScriptWrapImports: true\nKeepEmptyLinesAtTheStartOfBlocks: false\nLambdaBodyIndentation: Signature\nLineEnding: LF\nMacroBlockBegin: ''\nMacroBlockEnd: ''\nMaxEmptyLinesToKeep: 1\nNamespaceIndentation: None\nObjCBinPackProtocolList: Auto\nObjCBlockIndentWidth: 4\nObjCSpaceAfterProperty: true\nObjCSpaceBeforeProtocolList: true\nPPIndentWidth: -1\nPackConstructorInitializers: CurrentLine\nPenaltyBreakAssignment: 2\nPenaltyBreakBeforeFirstCallParameter: 1\nPenaltyBreakComment: 300\nPenaltyBreakFirstLessLess: 120\nPenaltyBreakString: 1000\nPenaltyBreakTemplateDeclaration: 10\nPenaltyExcessCharacter: 1000000\nPenaltyReturnTypeOnItsOwnLine: 200\nPointerAlignment: Middle\nQualifierAlignment: Left\n#QualifierOrder: ['static', 'inline', 'friend', 'constexpr', 'const', 'volatile', 'type', 'restrict']\nRawStringFormats:\n - Language: Cpp\n Delimiters:\n - cc\n - CC\n - cpp\n - Cpp\n - CPP\n - 'c++'\n - 'C++'\n CanonicalDelimiter: ''\nReferenceAlignment: Middle\nReflowComments: false # IndentOnly\nSeparateDefinitionBlocks: Always\nSortIncludes: CaseInsensitive\nSortUsingDeclarations: LexicographicNumeric\nSpaceAfterCStyleCast: true\nSpaceAfterLogicalNot: false\nSpaceAfterTemplateKeyword: true\nSpaceBeforeAssignmentOperators: true\nSpaceBeforeCpp11BracedList: false\nSpaceBeforeCtorInitializerColon: true\nSpaceBeforeInheritanceColon: true\nSpaceBeforeParens: ControlStatements\nSpaceBeforeRangeBasedForLoopColon: true\nSpaceInEmptyBlock: false\nSpaceInEmptyParentheses: false\nSpacesBeforeTrailingComments: 2\nSpacesInAngles: Never\nSpacesInLineCommentPrefix:\n Minimum: 1\n Maximum: -1\nSpacesInParentheses: false\nSpacesInSquareBrackets: false\nSpaceBeforeSquareBrackets: false\nStandard: c++17\nTabWidth: 4\nUseTab: Never\nWhitespaceSensitiveMacros: ['STRINGIZE']\n...\n" }, { "path": "src/ggml-openvino/CMakeLists.txt", "content": "find_package(OpenVINO REQUIRED)\nfind_package(OpenCL REQUIRED)\n\ninclude(\"${OpenVINO_DIR}/../3rdparty/tbb/lib/cmake/TBB/TBBConfig.cmake\")\n\nfile(GLOB_RECURSE GGML_HEADERS_OPENVINO \"*.h\" \"*.hpp\")\nfile(GLOB_RECURSE GGML_SOURCES_OPENVINO \"*.cpp\")\n\nggml_add_backend_library(ggml-openvino\n ${GGML_SOURCES_OPENVINO}\n ${GGML_HEADERS_OPENVINO}\n)\n\ntarget_link_libraries(ggml-openvino PRIVATE openvino::runtime TBB::tbb OpenCL::OpenCL)\n\nif (GGML_OPENVINO)\n if (CMAKE_SYSTEM_PROCESSOR STREQUAL \"aarch64\")\n elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL \"x86_64\" OR CMAKE_SYSTEM_PROCESSOR STREQUAL \"amd64\" OR CMAKE_SYSTEM_PROCESSOR STREQUAL \"AMD64\")\n else()\n message(FATAL_ERROR \"OpenVINO: OpenVINO toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}\")\n endif()\nendif()\n" }, { "path": "src/ggml-openvino/ggml-decoder.cpp", "content": "#include \"ggml-decoder.h\"\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-openvino-extra.h\"\n#include \"ggml-openvino.h\"\n#include \"ggml-quants.h\"\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nGgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph,\n ModelParams & model_params,\n ComputeParams & compute_params,\n std::map> & model_weights,\n bool is_static,\n bool is_stateful,\n bool is_prefill,\n int prefill_chunk_size) :\n m_is_static(is_static),\n m_is_stateful(is_stateful),\n m_is_prefill(is_prefill),\n m_naive(false),\n m_prefill_chunk_size(prefill_chunk_size),\n m_cgraph(cgraph),\n m_model_weights(model_weights),\n m_model_params(model_params),\n m_compute_params(compute_params) {\n if (auto * env = getenv(\"GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS\"); env && std::string(env) != \"0\") {\n#ifdef _WIN32\n _putenv_s(\"GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS\", \"\");\n#else\n unsetenv(\"GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS\");\n#endif\n print_tensor_address_map(cgraph);\n }\n\n validate_cgraph();\n\n set_input_output();\n compute_model_inputs();\n compute_model_outputs();\n\n for (int node_n = 0; node_n < cgraph->n_nodes; node_n++) {\n m_node_info_list[node_n].node_op_case = compute_op_case(m_node_info_list[node_n].node);\n m_node_info_list[node_n].node_op_type = compute_op_type(m_node_info_list[node_n].node);\n }\n\n add_extra_inputs();\n}\n\nvoid GgmlOvDecoder::update_io(ggml_cgraph * cgraph) {\n m_cgraph = cgraph;\n m_model_inputs.clear();\n m_model_outputs.clear();\n m_node_info_list.clear();\n set_input_output();\n compute_model_inputs();\n compute_model_outputs();\n}\n\nGgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, std::map> & model_weights) {\n m_cgraph = cgraph;\n m_model_weights = model_weights;\n m_naive = true;\n set_input_output();\n compute_model_inputs();\n compute_model_outputs();\n for (int node_n = 0; node_n < cgraph->n_nodes; node_n++) {\n m_node_info_list[node_n].node_op_case = compute_op_case(m_node_info_list[node_n].node);\n m_node_info_list[node_n].node_op_type = compute_op_type(m_node_info_list[node_n].node);\n }\n}\n\nvoid GgmlOvDecoder::set_input_output() {\n for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) {\n auto node = m_cgraph->nodes[node_n];\n\n NodeInfo current_node_info;\n auto node_name = std::string(node->name);\n auto node_output_name = node_name;\n auto * node_output = node;\n if (node->op == GGML_OP_SET_ROWS) {\n // SET_ROWS updates the tensor in place. For later ov op that uses the\n // the view_src of SET_ROWS, we need to make sure they get the updated tensor\n // by putting the view_src name in the tensor_map in\n // /src/frontends/ggml/src/translate_session.cpp\n node_output_name = std::string(node->view_src->name);\n node_output = node->view_src;\n }\n\n current_node_info.node = node;\n current_node_info.node_name = node_name;\n current_node_info.node_output = node_output;\n current_node_info.node_output_name = node_output_name;\n current_node_info.node_op_case = 0;\n current_node_info.data_addr = node->data;\n\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n auto * src = node->src[i];\n if (src == nullptr) {\n continue;\n }\n auto src_name = std::string(src->name);\n if (src->flags & GGML_TENSOR_FLAG_INPUT) {\n src_name = get_graph_input_ov_name(src, node);\n }\n current_node_info.node_inputs[src_name] = src;\n current_node_info.node_inputs_names.push_back(src_name);\n }\n\n m_node_info_list.push_back(current_node_info);\n }\n}\n\nint GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {\n int op_case = 0;\n switch (node->op) {\n case GGML_OP_RESHAPE: {\n auto * src = node->src[0];\n if (src->op == GGML_OP_RESHAPE && src->src[0]->ne[0] == node->ne[0] && src->src[0]->ne[1] == node->ne[1]) {\n op_case = 4;\n } else if (node->ne[0] * node->ne[1] == src->ne[0]) {\n op_case = 1;\n } else if (src->ne[0] * src->ne[1] == node->ne[0]) {\n op_case = 2;\n if (src->ne[2] * src->ne[3] == node->ne[1]) {\n op_case = 5;\n }\n } else if (src->ne[0] * src->ne[1] == node->ne[1]) {\n op_case = 3;\n } else if (src->ne[1] * src->ne[2] == node->ne[1]) {\n op_case = 6;\n }\n break;\n }\n case GGML_OP_CONT: {\n if (node->src[0]->op == GGML_OP_PERMUTE) {\n op_case = 1;\n } else if (node->src[0]->op == GGML_OP_TRANSPOSE) {\n op_case = 2;\n } else if (node->src[0]->op == GGML_OP_VIEW) {\n op_case = 3;\n }\n break;\n }\n case GGML_OP_PERMUTE: {\n if (node->src[0]->op != GGML_OP_VIEW) {\n op_case = 1;\n } else if (node->src[0]->src[0]->op == GGML_OP_NONE) {\n // kv cache tensor\n std::string src_name(node->view_src->name);\n int layer = extract_layer_from_name(src_name);\n if (!is_swa_layer(layer)) {\n op_case = 2;\n } else {\n op_case = 3;\n }\n } else {\n // rope'ed query tensor\n op_case = 4;\n }\n break;\n }\n case GGML_OP_MUL_MAT: {\n if (node->src[0]->op == GGML_OP_CONT && node->src[0]->src[0]->op == GGML_OP_TRANSPOSE) {\n op_case = 2;\n } else if (node->src[0]->op == GGML_OP_VIEW && node->src[1]->op == GGML_OP_VIEW) {\n op_case = 3;\n }\n break;\n }\n case GGML_OP_GET_ROWS: {\n if (node->src[1]->op == GGML_OP_VIEW) {\n op_case = 2;\n }\n break;\n }\n case GGML_OP_ROPE: {\n if (node->src[0]->op == GGML_OP_VIEW) {\n op_case = 2;\n }\n break;\n }\n case GGML_OP_VIEW: {\n if (node->src[0]->op == GGML_OP_VIEW) {\n auto * src = node->src[0];\n if (ggml_nelements(node) != ggml_nelements(src)) {\n throw std::runtime_error(\"Unsupported VIEW case\");\n }\n op_case = 2;\n }\n {\n auto * src = node->src[0];\n if ((ggml_nelements(node) != ggml_nelements(src)) && m_naive) {\n // Compare each dimension of node and src, if only one dimension differs then op_case=3\n int diff_count = 0;\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n if (node->ne[i] != src->ne[i]) {\n diff_count++;\n }\n }\n if (diff_count == 1) {\n op_case = 3;\n }\n }\n }\n break;\n }\n default:\n break;\n }\n return op_case;\n}\n\nint extract_layer_from_name(const std::string & name) {\n size_t pos1 = name.find(\"_l\");\n assert(pos1 != std::string::npos);\n pos1 += 2;\n size_t pos2 = name.find(' ', pos1);\n if (pos2 == std::string::npos) {\n pos2 = name.length();\n }\n std::string layer_str = name.substr(pos1, pos2 - pos1);\n int layer = std::stoi(layer_str);\n return layer;\n}\n\nstd::pair GgmlOvDecoder::compute_llm_params(ggml_cgraph * cgraph, bool is_static) {\n ModelParams model_params;\n ComputeParams compute_params;\n for (int i = 0; i < cgraph->n_nodes; i++) {\n auto * node = cgraph->nodes[i];\n std::string name = std::string(node->name);\n if (node->op == GGML_OP_FLASH_ATTN_EXT) {\n model_params.n_heads = node->src[0]->ne[2];\n model_params.n_heads_kv = node->src[1]->ne[2];\n model_params.head_size = node->src[0]->ne[0];\n compute_params.input_len = node->src[0]->ne[1];\n\n auto * cache_k_perm = node->src[1];\n if (cache_k_perm->op == GGML_OP_CPY) {\n cache_k_perm = cache_k_perm->src[0];\n }\n assert(cache_k_perm->op == GGML_OP_PERMUTE);\n auto * cache_k_view = cache_k_perm->src[0];\n assert(cache_k_view->op == GGML_OP_VIEW);\n\n auto * cache_k = cache_k_view->src[0];\n int layer = extract_layer_from_name(cache_k->name);\n auto * mask = node->src[3];\n std::string mask_name(mask->name);\n\n model_params.kv_buffer_ctx_id = ggml_backend_openvino_buffer_get_ctx_id(cache_k->buffer);\n if (mask_name.find(\"swa\") != std::string::npos) {\n model_params.swa_layers.push_back(layer);\n model_params.ctx_per_seq_swa = cache_k->ne[1];\n } else {\n model_params.ctx_per_seq = cache_k->ne[1];\n model_params.n_seq = cache_k->ne[2];\n }\n\n compute_params.n_seq_active = mask->ne[3];\n auto seq_size = cache_k->ne[0] * cache_k->ne[1] * ggml_type_size(cache_k->type);\n size_t offset;\n memcpy(&offset, cache_k_view->op_params, sizeof(size_t));\n compute_params.seq_active_start = offset / seq_size;\n compute_params.token_len_per_seq = node->ne[2];\n\n if (mask_name.find(\"swa\") != std::string::npos) {\n compute_params.attention_size_swa = mask->ne[0];\n } else {\n compute_params.attention_size = mask->ne[0];\n }\n if (is_static) {\n compute_params.attention_size = model_params.ctx_per_seq;\n compute_params.attention_size_swa = model_params.ctx_per_seq_swa;\n compute_params.token_len_per_seq = 1;\n }\n break;\n }\n if (node->op == GGML_OP_ROPE) {\n memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 15);\n }\n }\n auto * output_tensor = cgraph->nodes[cgraph->n_nodes - 1];\n compute_params.output_len = output_tensor->ne[1];\n // for NPU, output_len is always 1 except for llama-perplexity\n if (is_static && compute_params.output_len == 0) {\n compute_params.output_len = 1;\n }\n model_params.ctx = model_params.ctx_per_seq * model_params.n_seq;\n model_params.ctx_swa = model_params.ctx_per_seq_swa * model_params.n_seq;\n return {model_params, compute_params};\n}\n\nvoid GgmlOvDecoder::validate_cgraph() const {\n if (m_model_params.n_seq > 1 && m_is_static == true) {\n throw std::runtime_error(\"n_seq > 1 is not supported on NPU. Try setting -np 1.\");\n }\n}\n\nov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, const ggml_tensor * input) const {\n if (m_naive) {\n return input!= nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)};\n }\n auto name = std::string(input->name);\n ov::PartialShape input_shape;\n\n if (is_inp_tok(input, op) || is_inp_pos(input, op)) {\n // tokens or positions\n int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1;\n input_shape = ov::PartialShape{1, 1, 1, len};\n\n } else if (is_output_idx(input, op)) {\n // output index\n input_shape = ov::PartialShape{1, 1, 1, m_is_static ? m_compute_params.output_len : -1};\n\n } else if (is_inp_mask(input, op)) {\n // mask\n if (m_is_static) {\n input_shape = ov::PartialShape{1, 1, m_is_prefill ? m_prefill_chunk_size : 1, m_model_params.ctx};\n } else if (m_is_stateful) {\n input_shape = ov::PartialShape{1, 1, -1, -1};\n } else {\n input_shape = ov::PartialShape{-1, 1, -1, -1};\n }\n\n } else if (is_kvcache(input, op)) {\n // kvcache\n input_shape = ov::PartialShape{get_shape(input)};\n if (!m_is_static) {\n // do not fix ctx size to make llama-bench work across test params\n input_shape[2] = -1;\n }\n if (is_stateful()) {\n // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size]\n // to stateful layout [1, seq, n_heads_kv, head_size].\n assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 &&\n input_shape[2].is_dynamic() &&\n input_shape[3] == (m_model_params.n_heads_kv * m_model_params.head_size));\n input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv,\n m_model_params.head_size};\n }\n\n } else if (is_kv_idx(input, op)) {\n // kv update index\n int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1;\n input_shape = ov::PartialShape{1, 1, 1, len};\n\n } else {\n input_shape = ov::PartialShape{get_shape(input)};\n }\n return input_shape;\n}\n\nvoid GgmlOvDecoder::add_extra_inputs() {\n // Extra inputs:\n // 1. `attention_size`, used in FLASH_ATTN where the shape of the matmul's are 256 aligned,\n // see llama_kv_cache_unified::get_n_kv and llama_kv_cache_unified::get_padding.\n // 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch\n\n auto create_1d_input = [this](const std::string & name, int64_t value) {\n if (m_is_static) {\n auto constant =\n std::make_shared(ov::element::i64, ov::Shape{1}, std::vector{value});\n constant->set_friendly_name(name);\n m_model_extra_inputs[name] = constant;\n } else {\n auto param_node = std::make_shared(ov::element::i64, ov::Shape{1});\n param_node->set_friendly_name(name);\n param_node->output(0).get_tensor().set_names({name});\n m_model_extra_inputs[name] = param_node;\n\n auto tensor = std::make_shared(ov::element::i64, ov::Shape{1});\n *tensor->data() = value;\n m_model_extra_input_values[name] = tensor;\n }\n };\n\n create_1d_input(\"attention_size\", m_compute_params.attention_size);\n if (m_compute_params.attention_size_swa != -1) {\n create_1d_input(\"attention_size_swa\", m_compute_params.attention_size_swa);\n }\n create_1d_input(\"n_seq_active\", m_compute_params.n_seq_active);\n create_1d_input(\"seq_active_start\", m_compute_params.seq_active_start);\n create_1d_input(\"seq_active_end\", m_compute_params.seq_active_start + m_compute_params.n_seq_active);\n create_1d_input(\"token_len_per_seq\", m_compute_params.token_len_per_seq);\n // create_1d_input(\"token_len\", m_token_len_per_seq * m_n_seq_active);\n}\n\nbool GgmlOvDecoder::node_is_used_as_src(const int node_idx) {\n ggml_tensor * node = m_cgraph->nodes[node_idx];\n for (int i = node_idx; i < m_cgraph->n_nodes; i++) {\n ggml_tensor * other_node = m_cgraph->nodes[i];\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n if (other_node->src[j] == node) {\n return true;\n }\n }\n }\n return false;\n}\n\nvoid GgmlOvDecoder::compute_model_inputs() {\n m_model_inputs.clear();\n m_inputs.clear();\n for (int i = 0; i < m_cgraph->n_nodes; i++) {\n ggml_tensor * node = m_cgraph->nodes[i];\n // the node op is NONE means this node maybe as input of later nodes, we should add it to model inputs for this node.\n if (node->op == GGML_OP_NONE && node_is_used_as_src(i)) {\n std::string node_name(node->name);\n if (m_model_weights.find(node_name) == m_model_weights.end()) {\n m_inputs[node_name] = node;\n auto param_node =\n std::make_shared(get_ov_type(node), get_graph_input_shape(node, nullptr));\n param_node->set_friendly_name(node_name);\n param_node->output(0).get_tensor().set_names({node_name});\n m_model_inputs[node_name] = param_node;\n }\n continue;\n }\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n auto * src = node->src[i];\n if (src == nullptr) {\n continue;\n }\n std::string src_name = std::string(src->name);\n if (src->flags & GGML_TENSOR_FLAG_INPUT) {\n src_name = get_graph_input_ov_name(src, node);\n }\n if (m_model_weights.find(src_name) != m_model_weights.end()) {\n continue;\n }\n\n bool is_intermediate_node = false;\n for (const auto & node_info : m_node_info_list) {\n if (node_info.node == src) {\n is_intermediate_node = true;\n break;\n }\n }\n if (is_intermediate_node) {\n continue;\n }\n if (m_model_inputs.find(src_name) != m_model_inputs.end()) {\n continue;\n }\n\n m_inputs[src_name] = src;\n\n ggml_backend_buffer * buffer = src->buffer;\n // GGML_BACKEND_BUFFER_USAGE_ANY are kv caches\n if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) {\n if (auto it = std::find(m_model_params.kv_names.begin(), m_model_params.kv_names.end(), src_name);\n it == m_model_params.kv_names.end()) {\n m_model_params.kv_names.push_back(src_name);\n }\n }\n ov::PartialShape param_shape = get_graph_input_shape(node, src);\n auto param_node = std::make_shared(get_ov_type(src), param_shape);\n param_node->set_friendly_name(src_name);\n param_node->output(0).get_tensor().set_names({src_name});\n m_model_inputs[src_name] = param_node;\n }\n }\n}\n\nvoid GgmlOvDecoder::compute_model_outputs() {\n m_model_outputs.clear();\n m_model_output_names.clear();\n for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) {\n auto * cur_node = m_cgraph->nodes[node_n];\n // if the node op is NONE means this node is not used at all, we can skip it directly without adding to model outputs.\n if (cur_node->op == GGML_OP_NONE) {\n continue;\n }\n auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)];\n if (cur_node_use_count == 0) {\n // The output of SET_ROWS is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src.\n if (cur_node != nullptr && cur_node->op == GGML_OP_SET_ROWS) {\n cur_node = cur_node->view_src;\n }\n } else {\n int input_use_count = 0;\n for (int i = 0; i < m_cgraph->n_nodes; i++) {\n ggml_tensor * node = m_cgraph->nodes[i];\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n if (node->src[j] != NULL && node->src[j] == cur_node) {\n input_use_count++;\n }\n }\n }\n if (input_use_count == cur_node_use_count) {\n cur_node = nullptr;\n }\n }\n if (cur_node != nullptr) {\n std::string node_output_name(cur_node->name);\n m_model_outputs[node_output_name] = cur_node;\n m_model_output_names.push_back(node_output_name);\n }\n }\n}\n\nconst ggml_tensor * GgmlOvDecoder::get_tensor_used_op(const ggml_tensor * tensor) const {\n if (tensor == nullptr) {\n return nullptr;\n }\n for (int i = 0; i < m_cgraph->n_nodes; i++) {\n const auto * node = m_cgraph->nodes[i];\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n if (node->src[j] == tensor) {\n return node;\n }\n }\n }\n return nullptr;\n}\n\nconst ggml_tensor * GgmlOvDecoder::get_tensor_from_name(const std::string & name) const {\n for (int i = 0; i < m_cgraph->n_nodes; i++) {\n const auto * node = m_cgraph->nodes[i];\n for (int j = 0; j < GGML_MAX_SRC; j++) {\n const auto * src = node->src[j];\n if (src == nullptr) {\n break;\n }\n if (std::string(src->name) == name) {\n return src;\n }\n }\n }\n return nullptr;\n}\n\nstd::map GgmlOvDecoder::get_kv_param_res_names() const {\n std::map kv_param_res_names;\n for (const auto & name : m_model_params.kv_names) {\n kv_param_res_names[name] = name;\n }\n return kv_param_res_names;\n}\n\nstd::map> GgmlOvDecoder::create_weight_nodes(ggml_cgraph * cgraph, bool naive) {\n static std::mutex weights_mutex;\n std::lock_guard lock(weights_mutex);\n\n std::map> model_weights;\n auto * nodes = cgraph->nodes;\n auto n_nodes = cgraph->n_nodes;\n for (int node_i = 0; node_i < n_nodes; node_i++) {\n auto * node = nodes[node_i];\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n auto * src = node->src[i];\n if (src == nullptr) {\n continue;\n }\n\n std::string src_name(src->name);\n if (is_rope_freqs_weight(src, node)) {\n src_name = \"rope_freqs.weight\";\n }\n if (!src->view_src) {\n ggml_backend_buffer * buffer = src->buffer;\n if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) {\n if (model_weights.find(src_name) == model_weights.end()) {\n auto weight_node = create_weight_node(src, naive);\n weight_node->set_friendly_name(src_name);\n model_weights[src_name] = weight_node;\n }\n }\n }\n }\n }\n return model_weights;\n}\n\nstd::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor, bool naive) {\n const bool is_ov_buffer = ggml_backend_buffer_is_openvino(tensor->buffer);\n\n // Check if we have a pre-built constant from the OpenVINO backend buffer\n // This is set during ggml_backend_openvino_buffer_set_tensor\n if (tensor->extra) {\n OPENVINO_ASSERT(is_ov_buffer, \"Unsupported weight tensor: \" + std::string(tensor->name) +\n \" Possibly this is a cpu backend repacked quantized weights\");\n // Cast to our extra base type and check the type\n auto * extra_base = static_cast(tensor->extra);\n\n if (extra_base->type == ggml_openvino_extra_base::Type::WEIGHT) {\n // F16/F32/BF16 weight with shared-memory constant\n auto * weight_extra = static_cast(tensor->extra);\n if (weight_extra->weight_node) {\n // GGML_LOG_DEBUG(\"%s: using pre-built weight node for %s\\n\", __func__, tensor->name);\n return weight_extra->weight_node;\n }\n } else if (extra_base->type == ggml_openvino_extra_base::Type::QUANTIZED_WEIGHT) {\n // Quantized weight with pre-extracted data\n auto * quant_extra = static_cast(tensor->extra);\n if (quant_extra->weight_node) {\n // GGML_LOG_DEBUG(\"%s: using pre-extracted quantized weight node for %s\\n\", __func__, tensor->name);\n return quant_extra->weight_node;\n }\n }\n }\n\n // There are three cases where we need to create a new weight node:\n // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor\n // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used\n // 3. test-backend-ops. buffers in test-backend-ops does not set USAGE_WEIGHT so backend_buffer_set_tensor will not create weight node\n\n // GGML_LOG_DEBUG(\"%s: creating new weight node for %s\\n\", __func__, tensor->name);\n static const std::set weight_types = {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16,\n GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1,\n GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K};\n if (weight_types.find(tensor->type) == weight_types.end()) {\n throw std::runtime_error(\"Unexpected weight tensor type: \" + std::string(tensor->name) + \" with type \" +\n ggml_type_name(tensor->type));\n }\n\n OvWeight ov_weight;\n if (ggml_is_quantized(tensor->type)) {\n auto use_bias = naive;\n if (is_ov_buffer) {\n // For quantized weights, copy raw data to a temp buffer first because\n // process_weight_tensor reads from data and writes extracted results\n // (weights/scales/zp) to output_base_ptr — they would overlap if both\n // point to tensor->data.\n size_t raw_size = ggml_nbytes(tensor);\n std::vector tmp(raw_size);\n memcpy(tmp.data(), tensor->data, raw_size);\n ov_weight = process_weight_tensor(tensor, tmp.data(), tensor->data, use_bias);\n } else {\n ov_weight = process_weight_tensor(tensor, tensor->data, nullptr, use_bias);\n }\n } else {\n // For non-quantized weights (F16/F32/BF16), data is already in tensor->data.\n // process_weight_tensor will create an ov::Tensor wrapping tensor->data directly.\n ov_weight = process_weight_tensor(tensor, tensor->data, tensor->data);\n }\n\n ov_weight.weight_node->set_friendly_name(tensor->name);\n if (!is_ov_buffer) {\n return ov_weight.weight_node;\n }\n\n ggml_openvino_extra_base * extra;\n if (ov_weight.is_quantized()) {\n extra = new ggml_openvino_quantized_weight_extra(std::move(ov_weight.weights), std::move(ov_weight.scales),\n std::move(ov_weight.zp), ov_weight.weight_node);\n } else {\n extra = new ggml_openvino_weight_extra(std::move(ov_weight.weights), ov_weight.weight_node);\n }\n ggml_openvino_buffer_register_extra(tensor, extra);\n\n return ov_weight.weight_node;\n}\n\nvoid GgmlOvDecoder::dump_cgraph(const ggml_cgraph * cgraph, std::string & filename) {\n std::ofstream file(filename);\n if (!file.is_open()) {\n std::cerr << \"Failed to open file\" << std::endl;\n return;\n }\n\n file << \"=== GRAPH ===\\n\";\n\n // clang-format off\n file << \"n_nodes = \" << cgraph->n_nodes << \"\\n\";\n file << \" \" << std::setw(3) << \"nodes\"\n << std::setw(15) << \"shape\"\n << std::setw(20) << \"op\"\n << std::setw(20) << \"name\"\n << std::setw(3) << \" \"\n << std::setw(62) << \"stride\"\n << std::setw(20) << \"buffer_type\"\n << \"\\n\";\n for (int i = 0; i < cgraph->n_nodes; i++) {\n ggml_tensor * node = cgraph->nodes[i];\n\n // Get buffer type name\n const char * buf_name = \"none\";\n ggml_backend_buffer_t buf = node->view_src ? node->view_src->buffer : node->buffer;\n if (buf) {\n buf_name = ggml_backend_buffer_name(buf);\n }\n\n file << \" - \" << std::setw(3) << i << \": [ \"\n << std::setw(5) << node->ne[0] << \", \"\n << std::setw(5) << node->ne[1] << \", \"\n << std::setw(5) << node->ne[2] << \", \"\n << std::setw(5) << node->ne[3] << \"] \"\n << std::left << std::setw(20) << ggml_op_name(node->op) << std::right << \" \"\n << std::left << std::setw(45) << node->name << std::right\n << std::setw(2) << \"[ \"\n << std::setw(0) << node->nb[0] << \", \"\n << std::setw(5) << node->nb[1] << \", \"\n << std::setw(5) << node->nb[2] << \", \"\n << std::setw(5) << node->nb[3] << \"] \"\n << std::right << std::setw(15) << buf_name << std::right\n << \"\\n\";\n\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (auto* src = node->src[i]) {\n // Get buffer type name for source\n const char * src_buf_name = \"none\";\n ggml_backend_buffer_t src_buf = src->view_src ? src->view_src->buffer : src->buffer;\n if (src_buf) {\n src_buf_name = ggml_backend_buffer_name(src_buf);\n }\n\n file << std::setw(10) << \" [ \"\n << std::setw(5) << src->ne[0] << \", \"\n << std::setw(5) << src->ne[1] << \", \"\n << std::setw(5) << src->ne[2] << \", \"\n << std::setw(5) << src->ne[3] << \"] \"\n << std::setw(12)\n << i << \": \" << std::left << std::setw(12) << ggml_op_name(src->op) << std::right;\n file << std::left << std::setw(30) << src->name << std::right\n << std::setw(16) << \"[ \"\n << std::setw(0) << src->nb[0] << \", \"\n << std::setw(5) << src->nb[1] << \", \"\n << std::setw(5) << src->nb[2] << \", \"\n << std::setw(5) << src->nb[3] << \"] \"\n << std::right << std::setw(15) << src_buf_name << std::right\n << \"\\n\";\n }\n }\n }\n\n file << \"n_leafs = \" << cgraph->n_leafs << \"\\n\";\n for (int i = 0; i < cgraph->n_leafs; i++) {\n ggml_tensor * node = cgraph->leafs[i];\n\n // Get buffer type name for leaf\n const char * leaf_buf_name = \"none\";\n ggml_backend_buffer_t leaf_buf = node->view_src ? node->view_src->buffer : node->buffer;\n if (leaf_buf) {\n leaf_buf_name = ggml_backend_buffer_name(leaf_buf);\n }\n\n file << \" - \" << std::setw(3) << i << \": [ \"\n << std::setw(5) << node->ne[0] << \", \"\n << std::setw(5) << node->ne[1] << \"] \"\n << std::setw(8) << ggml_op_name(node->op) << \" \"\n << std::setw(16) << ggml_get_name(node)\n << std::setw(20) << leaf_buf_name << \"\\n\";\n }\n // clang-format on\n file << \"========================================\\n\";\n\n file.close();\n}\n\nvoid print_tensor_address_map(const ggml_cgraph * cgraph) {\n std::map> address_map;\n for (int node_n = 0; node_n < cgraph->n_nodes; node_n++) {\n auto * node = cgraph->nodes[node_n];\n if (node->data) {\n auto it = address_map.find(node->data);\n if (it == address_map.end()) {\n address_map[node->data] = std::vector();\n }\n address_map[node->data].push_back(node->name);\n }\n }\n for (const auto & pair : address_map) {\n std::cout << \"Address: \" << pair.first << std::endl;\n for (const auto & name : pair.second) {\n std::cout << name << \" ; \";\n }\n std::cout << std::endl << std::endl;\n }\n}\n\nov::Shape GgmlOvDecoder::get_shape(const ggml_tensor * tensor) {\n std::vector shape;\n for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {\n shape.push_back(static_cast(tensor->ne[i]));\n }\n return shape;\n}\n\nstd::vector GgmlOvDecoder::get_stride(const ggml_tensor * tensor) {\n std::vector stride;\n for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {\n stride.push_back(static_cast(tensor->nb[i]));\n }\n return stride;\n}\n\nov::element::Type GgmlOvDecoder::get_ov_type(const ggml_tensor * tensor) {\n switch (tensor->type) {\n case GGML_TYPE_F64:\n return ov::element::f64;\n case GGML_TYPE_F32:\n return ov::element::f32;\n case GGML_TYPE_F16:\n return ov::element::f16;\n case GGML_TYPE_BF16:\n return ov::element::bf16;\n case GGML_TYPE_I8:\n return ov::element::i8;\n case GGML_TYPE_I16:\n return ov::element::i16;\n case GGML_TYPE_I32:\n return ov::element::i32;\n case GGML_TYPE_I64:\n return ov::element::i64;\n default:\n return ov::element::dynamic;\n }\n}\n\nov::PartialShape GgmlOvDecoder::get_input_shape(int node_idx, const std::string & name) const {\n return ov::PartialShape(get_shape(m_node_info_list[node_idx].node_inputs.at(name)));\n}\n\nstd::vector GgmlOvDecoder::get_input_stride(int node_idx, const std::string & name) const {\n return get_stride(m_node_info_list[node_idx].node_inputs.at(name));\n}\n\nov::element::Type GgmlOvDecoder::get_input_type(int node_idx, const std::string & name) const {\n return get_ov_type(m_node_info_list[node_idx].node_inputs.at(name));\n}\n\nsize_t GgmlOvDecoder::get_input_size() const {\n return m_model_inputs.size();\n}\n\nsize_t GgmlOvDecoder::get_input_size(int node_idx) const {\n return m_node_info_list[node_idx].node_inputs_names.size();\n}\n\nstd::vector GgmlOvDecoder::get_input_names(int node_idx) const {\n return m_node_info_list[node_idx].node_inputs_names;\n}\n\nov::PartialShape GgmlOvDecoder::get_output_shape(int node_idx) const {\n auto * ggml_tensor = m_node_info_list[node_idx].node_output;\n return ov::PartialShape(get_shape(ggml_tensor));\n}\n\nov::element::Type GgmlOvDecoder::get_output_type(const int node_idx) const {\n return get_ov_type(m_node_info_list[node_idx].node);\n}\n\nstd::vector GgmlOvDecoder::get_output_names(int node_idx) const {\n return {m_node_info_list[node_idx].node_output_name};\n}\n\nconst std::string & GgmlOvDecoder::get_op_name() const {\n static const std::string unknown_name = \"UNKNOWN_OP_NAME\";\n return unknown_name;\n}\n\nconst std::string & GgmlOvDecoder::get_op_name(int node_idx) const {\n return m_node_info_list[node_idx].node_name;\n}\n\nint32_t * GgmlOvDecoder::get_input_op_params(int node_idx, const std::string & name) const {\n return m_node_info_list[node_idx].node_inputs.at(name)->op_params;\n}\n\nint32_t * GgmlOvDecoder::get_output_op_params(int node_idx) const {\n return m_node_info_list[node_idx].node->op_params;\n}\n\nvoid GgmlOvDecoder::visit_subgraph(std::function, int node_idx)> node_visitor) const {\n for (int node_idx = 0; node_idx < m_cgraph->n_nodes; node_idx++) {\n if (m_cgraph->nodes[node_idx]->op == GGML_OP_NONE) {\n continue;\n }\n node_visitor(std::make_shared(*this), node_idx);\n }\n}\n\nstd::string GgmlOvDecoder::compute_op_type(const ggml_tensor * node) {\n static const std::map ops = {\n {GGML_OP_NONE, \"GGML_OP_NONE\" },\n {GGML_OP_ACC, \"GGML_OP_ACC\" },\n {GGML_OP_ADD, \"GGML_OP_ADD\" },\n {GGML_OP_ADD1, \"GGML_OP_ADD1\" },\n {GGML_OP_CONT, \"GGML_OP_CONT\" },\n {GGML_OP_DIV, \"GGML_OP_DIV\" },\n {GGML_OP_DUP, \"GGML_OP_DUP\" },\n {GGML_OP_GET_ROWS, \"GGML_OP_GET_ROWS\" },\n {GGML_OP_MUL, \"GGML_OP_MUL\" },\n {GGML_OP_MUL_MAT, \"GGML_OP_MUL_MAT\" },\n {GGML_OP_PERMUTE, \"GGML_OP_PERMUTE\" },\n {GGML_OP_RESHAPE, \"GGML_OP_RESHAPE\" },\n {GGML_OP_RMS_NORM, \"GGML_OP_RMS_NORM\" },\n {GGML_OP_ROPE, \"GGML_OP_ROPE\" },\n {GGML_OP_SCALE, \"GGML_OP_SCALE\" },\n {GGML_OP_SOFT_MAX, \"GGML_OP_SOFT_MAX\" },\n {GGML_OP_SUB, \"GGML_OP_SUB\" },\n {GGML_OP_TRANSPOSE, \"GGML_OP_TRANSPOSE\" },\n {GGML_OP_VIEW, \"GGML_OP_VIEW\" },\n {GGML_OP_SET_ROWS, \"GGML_OP_SET_ROWS\" },\n {GGML_OP_CPY, \"GGML_OP_CPY\" },\n {GGML_OP_FLASH_ATTN_EXT, \"GGML_OP_FLASH_ATTN_EXT\"},\n };\n static const std::map unary_ops = {\n {GGML_UNARY_OP_ABS, \"GGML_UNARY_OP_ABS\" },\n {GGML_UNARY_OP_SGN, \"GGML_UNARY_OP_SGN\" },\n {GGML_UNARY_OP_NEG, \"GGML_UNARY_OP_NEG\" },\n {GGML_UNARY_OP_STEP, \"GGML_UNARY_OP_STEP\" },\n {GGML_UNARY_OP_TANH, \"GGML_UNARY_OP_TANH\" },\n {GGML_UNARY_OP_ELU, \"GGML_UNARY_OP_ELU\" },\n {GGML_UNARY_OP_RELU, \"GGML_UNARY_OP_RELU\" },\n {GGML_UNARY_OP_SIGMOID, \"GGML_UNARY_OP_SIGMOID\" },\n {GGML_UNARY_OP_GELU, \"GGML_UNARY_OP_GELU\" },\n {GGML_UNARY_OP_GELU_QUICK, \"GGML_UNARY_OP_GELU_QUICK\" },\n {GGML_UNARY_OP_SILU, \"GGML_UNARY_OP_SILU\" },\n {GGML_UNARY_OP_HARDSWISH, \"GGML_UNARY_OP_HARDSWISH\" },\n {GGML_UNARY_OP_HARDSIGMOID, \"GGML_UNARY_OP_HARDSIGMOID\"},\n {GGML_UNARY_OP_EXP, \"GGML_UNARY_OP_EXP\" },\n {GGML_UNARY_OP_COUNT, \"GGML_UNARY_OP_COUNT\" }\n };\n static const std::map glu_ops = {\n {GGML_GLU_OP_SWIGLU, \"GGML_GLU_OP_SWIGLU\"},\n {GGML_GLU_OP_GEGLU, \"GGML_GLU_OP_GEGLU\" },\n {GGML_GLU_OP_REGLU, \"GGML_GLU_OP_REGLU\" }\n };\n\n switch (node->op) {\n case GGML_OP_UNARY:\n return unary_ops.at(ggml_get_unary_op(node));\n case GGML_OP_GLU:\n return glu_ops.at(ggml_get_glu_op(node));\n default:\n return ops.at(node->op);\n }\n static const std::string unknown_op = \"UNKNOWN_GGML_OP\";\n return unknown_op;\n}\n\nconst std::string & GgmlOvDecoder::get_op_type(int node_idx) const {\n return m_node_info_list[node_idx].node_op_type;\n}\n\nconst std::string & GgmlOvDecoder::get_op_type() const {\n static const std::string unknown_op = \"UNKNOWN_GGML_OP\";\n return unknown_op;\n}\n" }, { "path": "src/ggml-openvino/ggml-decoder.h", "content": "#pragma once\n\n#include \"ggml-quants.h\"\n#include \"ggml.h\"\n#include \"openvino/decoder.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nstruct ModelParams {\n int ctx = -1;\n int ctx_swa = -1;\n int ctx_per_seq = -1;\n int ctx_per_seq_swa = -1;\n int n_seq = 1;\n int n_heads = -1;\n int n_heads_kv = -1;\n int head_size = -1;\n int32_t rope_params[15];\n std::vector swa_layers;\n\n std::vector kv_names;\n size_t kv_buffer_ctx_id = 0;\n\n bool same_rope_params(const ModelParams & other) const {\n return memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0;\n }\n\n bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other); }\n\n bool can_reuse_statically(const ModelParams & other) const { return same_rope_params(other) && ctx == other.ctx; }\n\n bool kv_buffer_changed(const ModelParams & other) const { return kv_buffer_ctx_id != other.kv_buffer_ctx_id; }\n};\n\nstruct ComputeParams {\n int n_seq_active = 1;\n int seq_active_start = 0;\n int attention_size = -1;\n int attention_size_swa = -1;\n int input_len = -1;\n int token_len_per_seq = -1;\n int past_kv_len = -1;\n int output_len = 1;\n};\n\nclass GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {\npublic:\n struct NodeInfo {\n ggml_tensor * node;\n std::string node_name;\n std::string node_op_type;\n std::map node_inputs;\n std::vector node_inputs_names;\n ggml_tensor * node_output;\n std::string node_output_name;\n int node_op_case = 0;\n void * data_addr;\n };\n // Graph decoder\n GgmlOvDecoder(ggml_cgraph * cgraph,\n ModelParams & model_params,\n ComputeParams & compute_params,\n std::map> & model_weights,\n bool is_static,\n bool is_stateful = false,\n bool is_prefill = false,\n int prefill_chunk_size = 256);\n\n // Naive graph decoder\n GgmlOvDecoder(ggml_cgraph * cgraph, std::map> & model_weights);\n\n virtual ov::Any get_attribute(const std::string & name) const override {\n return nullptr;\n GGML_UNUSED(name);\n }\n\n virtual ov::PartialShape get_input_shape(int node_idx, const std::string & name) const override;\n\n virtual std::vector get_input_stride(int node_idx, const std::string & name) const override;\n\n virtual ov::element::Type get_input_type(int node_idx, const std::string & name) const override;\n\n virtual size_t get_input_size() const override;\n\n virtual size_t get_input_size(int node_idx) const override;\n\n virtual void get_input_node(size_t input_port_idx,\n std::string & producer_name,\n std::string & producer_output_port_name,\n size_t & producer_output_port_index) const override {\n GGML_UNUSED(input_port_idx);\n GGML_UNUSED(producer_name);\n GGML_UNUSED(producer_output_port_name);\n GGML_UNUSED(producer_output_port_index);\n }\n\n virtual std::vector get_input_names(int node_idx) const override;\n\n virtual ov::PartialShape get_output_shape(int node_idx) const override;\n\n virtual ov::element::Type get_output_type(int node_idx) const override;\n\n virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const override;\n\n virtual int32_t * get_output_op_params(int node_idx) const override;\n\n virtual std::vector get_output_names(int node_idx) const override;\n\n virtual const std::string & get_op_type() const override;\n\n virtual const std::string & get_op_type(int node_idx) const override;\n\n virtual const std::string & get_op_name() const override;\n\n virtual const std::string & get_op_name(int node_idx) const override;\n\n virtual void visit_subgraph(std::function, int node_idx)> node_visitor) const override;\n\n ggml_tensor * get_input_ggml_tensor(const std::string & name) const { return m_inputs.at(name); }\n\n virtual int get_op_case(int node_idx) const override { return m_node_info_list[node_idx].node_op_case; }\n\n virtual const std::map> & get_model_inputs() const override {\n return m_model_inputs;\n }\n\n virtual const std::map> & get_model_extra_inputs() const override {\n return m_model_extra_inputs;\n }\n\n virtual const std::map> & get_model_extra_input_values() const {\n return m_model_extra_input_values;\n }\n\n virtual const std::map> & get_model_weights() const override {\n return m_model_weights;\n }\n\n virtual std::vector get_model_output_names() const override {\n return m_model_output_names;\n }\n\n const std::map & get_model_outputs() const { return m_model_outputs; }\n\n virtual int get_ctx_size() const { return m_model_params.ctx; }\n\n virtual int get_ctx_swa_size() const { return m_model_params.ctx_swa; }\n\n virtual int get_ctx_per_seq() const { return m_model_params.ctx_per_seq; }\n\n virtual int get_ctx_per_seq_swa() const { return m_model_params.ctx_per_seq_swa; }\n\n virtual int get_n_seq() const { return m_model_params.n_seq; }\n\n virtual int is_swa_layer(int layer) const override {\n return std::find(m_model_params.swa_layers.begin(), m_model_params.swa_layers.end(), layer) !=\n m_model_params.swa_layers.end();\n }\n\n int get_past_kv_len() const { return m_compute_params.past_kv_len; }\n\n int get_input_len() const { return m_compute_params.input_len; }\n\n virtual int32_t * get_rope_params() const override { return const_cast(m_model_params.rope_params); }\n\n virtual std::map get_kv_param_res_names() const override;\n\n virtual bool is_static() const override { return m_is_static; }\n\n virtual bool is_stateful() const override { return m_is_stateful; }\n\n ov::PartialShape get_graph_input_shape(const ggml_tensor * op, const ggml_tensor * input) const;\n\n static void dump_cgraph(const ggml_cgraph * cgraph, std::string & filename);\n\n static std::shared_ptr create_weight_node(ggml_tensor * tensor, bool naive = false);\n\n static std::map> create_weight_nodes(ggml_cgraph * cgraph,\n bool naive = false);\n\n const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const;\n\n const ggml_tensor * get_tensor_from_name(const std::string & name) const;\n\n void clear_model_weights() { m_model_weights.clear(); }\n\n static std::pair compute_llm_params(ggml_cgraph * cgraph, bool is_static);\n\n ModelParams get_model_params() const { return m_model_params; }\n\n ComputeParams get_compute_params() const { return m_compute_params; }\n\n void set_model_params(const ModelParams & model_params) { m_model_params = model_params; }\n\n void set_compute_params(const ComputeParams & compute_params) { m_compute_params = compute_params; }\n\n bool m_is_static = false;\n bool m_is_stateful = false;\n bool m_is_prefill = false;\n bool m_naive = false;\n int m_prefill_chunk_size = 0;\n\n static ov::Shape get_shape(const ggml_tensor * tensor);\n static std::vector get_stride(const ggml_tensor * tensor);\n static ov::element::Type get_ov_type(const ggml_tensor * tensor);\n static std::string compute_op_type(const ggml_tensor * node);\n void add_extra_inputs();\n\n void update_io(ggml_cgraph * cgraph);\n\n inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) {\n return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE;\n }\n\n inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) {\n return op->op == GGML_OP_ROPE && tensor == op->src[1];\n }\n\n inline static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) {\n return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM;\n }\n\n inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) {\n return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]);\n }\n\n inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) {\n return op->op == GGML_OP_ROPE && tensor == op->src[2];\n }\n\n inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {\n return op->op == GGML_OP_SET_ROWS && op->src[2] == tensor;\n }\n\n inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) {\n return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor;\n }\n\n inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {\n return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE;\n }\n\n static std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) {\n if (is_inp_tok(tensor, op)) {\n return \"inp_tokens\";\n }\n if (is_inp_pos(tensor, op)) {\n return \"inp_pos\";\n }\n if (is_inp_emb(tensor, op)) {\n return \"embd\";\n }\n if (is_output_idx(tensor, op)) {\n return \"inp_out_ids\";\n }\n if (is_inp_mask(tensor, op)) {\n return std::string(tensor->name).find(\"swa\") == std::string::npos ? \"self_kq_mask\" : \"self_kq_mask_swa\";\n }\n return tensor->name;\n }\n\nprivate:\n void set_input_output();\n int compute_op_case(const ggml_tensor * node) const;\n bool node_is_used_as_src(const int node_idx);\n void compute_model_inputs();\n void compute_model_outputs();\n\n void validate_cgraph() const;\n\n ggml_cgraph * m_cgraph = nullptr;\n std::map m_inputs;\n\n std::map> m_model_inputs;\n std::map> m_model_extra_inputs;\n std::map> m_model_extra_input_values;\n std::map> m_model_weights;\n std::map m_model_outputs;\n std::vector m_model_output_names;\n std::vector m_node_info_list;\n\n ModelParams m_model_params;\n ComputeParams m_compute_params;\n};\n\nvoid print_tensor_address_map(const ggml_cgraph * cgraph);\n\nint extract_layer_from_name(const std::string & name);\n" }, { "path": "src/ggml-openvino/ggml-openvino-extra.cpp", "content": "#include \"ggml-openvino-extra.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml.h\"\n\n#include \n#include \n#include \n#include \n\nov::Core & ov_singleton_core() {\n static ov::Core core;\n return core;\n}\n\n// =====================================================\n// Device Configuration Implementations\n// =====================================================\n\nvoid ggml_openvino_device_config::init() {\n if (initialized) {\n return;\n }\n device_name = getenv(\"GGML_OPENVINO_DEVICE\") ? getenv(\"GGML_OPENVINO_DEVICE\") : \"CPU\";\n auto available_devices = ov_singleton_core().get_available_devices();\n if (std::find(available_devices.begin(), available_devices.end(), device_name) == available_devices.end()) {\n GGML_LOG_WARN(\"GGML OpenVINO Backend: device %s is not available, fallback to CPU\\n\", device_name.c_str());\n device_name = \"CPU\";\n }\n is_npu = (device_name == \"NPU\");\n\n auto * cache_dir = getenv(\"GGML_OPENVINO_CACHE_DIR\");\n if (device_name == \"NPU\") {\n compile_config = {\n {\"NPU_COMPILER_DYNAMIC_QUANTIZATION\", \"YES\" },\n {\"NPU_USE_NPUW\", \"YES\" },\n {\"NPUW_DEVICES\", \"NPU\" },\n {\"NPUW_FOLD\", \"YES\" },\n {\"NPUW_WEIGHTS_BANK\", \"shared\"},\n {\"NPUW_FUNCALL_FOR_ALL\", \"YES\" },\n {\"NPUW_FUNCALL_ASYNC\", \"YES\" },\n {\"NPUW_DQ\", \"YES\" },\n {\"NPUW_DQ_FULL\", \"NO\" },\n };\n if (cache_dir) {\n compile_config[\"NPUW_CACHE_DIR\"] = cache_dir;\n }\n } else if (cache_dir) {\n ov_singleton_core().set_property(ov::cache_dir(cache_dir));\n }\n\n // Initialize remote context with queue sharing for GPU\n if (device_name == \"GPU\") {\n // Create OpenCL context and queue\n cl_int err;\n cl_platform_id platform;\n err = clGetPlatformIDs(1, &platform, nullptr);\n if (err != CL_SUCCESS) {\n GGML_LOG_ERROR(\"Failed to get OpenCL platform: %d\\n\", err);\n return;\n }\n\n cl_device_id cl_device;\n err = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 1, &cl_device, nullptr);\n if (err != CL_SUCCESS) {\n GGML_LOG_ERROR(\"Failed to get OpenCL device: %d\\n\", err);\n return;\n }\n\n cl_context cl_ctx = clCreateContext(nullptr, 1, &cl_device, nullptr, nullptr, &err);\n if (err != CL_SUCCESS) {\n GGML_LOG_ERROR(\"Failed to create OpenCL context: %d\\n\", err);\n return;\n }\n\n cl_queue = clCreateCommandQueueWithProperties(cl_ctx, cl_device, nullptr, &err);\n if (err != CL_SUCCESS) {\n GGML_LOG_ERROR(\"Failed to create OpenCL command queue: %d\\n\", err);\n clReleaseContext(cl_ctx);\n return;\n }\n\n // Create OpenVINO remote context with queue sharing\n remote_context = ov::intel_gpu::ocl::ClContext(ov_singleton_core(), cl_queue);\n\n // Release the context (queue keeps a reference)\n clReleaseContext(cl_ctx);\n } else if (device_name == \"NPU\") {\n // remote tensor is not used for NPU yet\n // remote_context = ov_singleton_core().get_default_context(device_name);\n }\n\n initialized = true;\n}\n\nggml_openvino_device_config::~ggml_openvino_device_config() {\n if (cl_queue != nullptr) {\n clReleaseCommandQueue(cl_queue);\n cl_queue = nullptr;\n }\n}\n\n// Get the global device config singleton\nggml_openvino_device_config & ggml_openvino_get_device_config() {\n static ggml_openvino_device_config config;\n return config;\n}\n\n// Initialize device config (call during backend init)\nvoid ggml_openvino_init_device_config() {\n ggml_openvino_get_device_config().init();\n}\n\n// Get the device name\nconst std::string & ggml_openvino_get_device_name() {\n return ggml_openvino_get_device_config().device_name;\n}\n\n// Check if running on NPU\nbool ggml_openvino_is_npu() {\n return ggml_openvino_get_device_config().is_npu;\n}\n\n// Get the remote context for the current device (returns empty optional for CPU)\nstd::optional ggml_openvino_get_remote_context() {\n return ggml_openvino_get_device_config().remote_context;\n}\n\n// Get the compile config for the current device\nconst ov::AnyMap & ggml_openvino_get_compile_config() {\n return ggml_openvino_get_device_config().compile_config;\n}\n\n// Get the OpenCL command queue for GPU operations\ncl_command_queue ggml_openvino_get_cl_queue() {\n return ggml_openvino_get_device_config().cl_queue;\n}\n\n// Get the clEnqueueMemFillINTEL function pointer (lazy load)\nclEnqueueMemFillINTEL_fn ggml_openvino_get_clEnqueueMemFillINTEL() {\n static clEnqueueMemFillINTEL_fn fn = nullptr;\n static bool loaded = false;\n if (!loaded) {\n loaded = true;\n cl_platform_id platform;\n if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) {\n fn = (clEnqueueMemFillINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, \"clEnqueueMemFillINTEL\");\n }\n }\n return fn;\n}\n\n// Get the clEnqueueMemcpyINTEL function pointer (lazy load)\nclEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL() {\n static clEnqueueMemcpyINTEL_fn fn = nullptr;\n static bool loaded = false;\n if (!loaded) {\n loaded = true;\n cl_platform_id platform;\n if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) {\n fn = (clEnqueueMemcpyINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, \"clEnqueueMemcpyINTEL\");\n }\n }\n return fn;\n}\n\n// Get requantization type for a tensor type (returns nullopt if no requant needed)\nstd::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant) {\n if (no_requant) {\n return std::nullopt;\n }\n if (strncmp(tensor->name, \"token_embd.weight\", 17) == 0) {\n return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 : ExtraQuantType::Q8_0_C);\n }\n if (strncmp(tensor->name, \"output.weight\", 13) == 0) {\n return ExtraQuantType::Q8_0_C;\n }\n if (ggml_openvino_is_npu()) {\n return ExtraQuantType::Q4_0_128;\n }\n switch (tensor->type) {\n case GGML_TYPE_Q6_K:\n case GGML_TYPE_Q5_K:\n return ExtraQuantType::Q8_0_C;\n default:\n return std::nullopt;\n }\n}\n\n// =====================================================\n// Extracted Layout Calculation\n// =====================================================\n\nggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_tensor * tensor, bool use_bias) {\n ggml_openvino_extracted_layout layout = {};\n layout.is_symmetric = false;\n\n if (!ggml_is_quantized(tensor->type)) {\n return layout;\n }\n\n // Only handle 2D weight tensors\n if (tensor->ne[2] != 1 || tensor->ne[3] != 1) {\n return layout;\n }\n\n int64_t n_elements = ggml_nelements(tensor);\n const size_t alignment = 64; // Good for SIMD\n\n // Check if requantization is needed (NPU-specific)\n auto requant_type = ggml_openvino_get_requant_type(tensor, use_bias);\n if (requant_type.has_value()) {\n layout.is_requant = true;\n layout.requant_type = requant_type;\n\n // Special case: requant to F16 - just store F16 weights, no scales/zp\n if (requant_type.value() == ExtraQuantType::F16) {\n layout.weights_size = n_elements * sizeof(uint16_t); // F16 = 2 bytes\n layout.total_size = layout.weights_size;\n layout.weights_offset = 0;\n // No scales/zp for F16\n return layout;\n }\n\n // Requant to different quantized format (e.g., Q4_0_128)\n switch (requant_type.value()) {\n case ExtraQuantType::Q4_0_128:\n layout.is_u4 = true;\n layout.weights_per_block = 128;\n layout.is_symmetric = true;\n break;\n case ExtraQuantType::Q4_0_C:\n layout.is_u4 = true;\n layout.weights_per_block = tensor->ne[0];\n layout.is_symmetric = true;\n break;\n case ExtraQuantType::Q8_0_32:\n layout.is_u4 = false;\n layout.weights_per_block = 32;\n layout.is_symmetric = true;\n break;\n case ExtraQuantType::Q8_0_C:\n layout.is_u4 = false;\n layout.weights_per_block = tensor->ne[0];\n layout.is_symmetric = true;\n break;\n case ExtraQuantType::Q8_1_C:\n layout.is_u4 = false;\n layout.weights_per_block = tensor->ne[0];\n break;\n default:\n layout.weights_per_block = -1;\n GGML_ABORT(\"Code of re-quantizing to channel-wise is not updated\");\n break;\n }\n\n if (layout.is_requant) {\n // Calculate sizes for requantized format\n layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements;\n int64_t n_blocks = n_elements / layout.weights_per_block;\n layout.scales_size = n_blocks * sizeof(uint16_t);\n // For symmetric quantization, we only need one zp value (not one per block)\n // Zero points are stored in U4 or U8 format matching the weight type\n size_t n_zp_elements = layout.is_symmetric ? 1 : n_blocks;\n layout.zp_size = layout.is_u4 ? ((n_zp_elements + 1) / 2) : n_zp_elements;\n\n layout.weights_offset = 0;\n layout.scales_offset = ((layout.weights_size + alignment - 1) / alignment) * alignment;\n layout.zp_offset = layout.scales_offset + ((layout.scales_size + alignment - 1) / alignment) * alignment;\n layout.total_size = layout.zp_offset + layout.zp_size;\n layout.total_size = std::max(layout.total_size, ggml_nbytes(tensor));\n return layout;\n }\n }\n\n // Normal extraction (no requant) - determine format based on tensor type\n layout.is_u4 = false;\n layout.weights_per_block = 32;\n layout.is_symmetric = false;\n\n switch (tensor->type) {\n case GGML_TYPE_Q4_0:\n layout.is_u4 = true;\n layout.is_symmetric = true;\n break;\n\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q4_K:\n layout.is_u4 = true;\n break;\n\n case GGML_TYPE_Q8_0:\n layout.is_symmetric = true;\n break;\n\n case GGML_TYPE_Q6_K:\n layout.weights_per_block = 16;\n layout.is_symmetric = true;\n break;\n\n case GGML_TYPE_Q5_K:\n break;\n\n default:\n // Unsupported quantization type\n return layout;\n }\n\n // Calculate sizes\n // Weights: U4 = n_elements/2 bytes, U8 = n_elements bytes\n layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements;\n\n // Scales: F16 per block\n int64_t n_blocks = n_elements / layout.weights_per_block;\n layout.scales_size = n_blocks * sizeof(uint16_t); // F16 = 2 bytes\n // Zero points: U4 or U8 matching weight type\n // For symmetric quantization, we only need one zp value (not one per block)\n size_t n_zp_elements = layout.is_symmetric ? 1 : n_blocks;\n layout.zp_size = layout.is_u4 ? ((n_zp_elements + 1) / 2) : n_zp_elements;\n\n // Layout in buffer: [weights | scales | zp] with alignment\n layout.weights_offset = 0;\n layout.scales_offset = ((layout.weights_size + alignment - 1) / alignment) * alignment;\n layout.zp_offset = layout.scales_offset + ((layout.scales_size + alignment - 1) / alignment) * alignment;\n layout.total_size = layout.zp_offset + layout.zp_size;\n layout.total_size = std::max(layout.total_size, ggml_nbytes(tensor));\n\n return layout;\n}\n\nggml_openvino_tensor_extra * ggml_openvino_create_tensor_extra(const ggml_tensor * tensor, bool is_remote) {\n ov::Shape shape;\n for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {\n shape.push_back(static_cast(tensor->ne[i]));\n }\n\n ov::element::Type element_type;\n switch (tensor->type) {\n case GGML_TYPE_F32:\n element_type = ov::element::f32;\n break;\n case GGML_TYPE_F16:\n element_type = ov::element::f16;\n break;\n case GGML_TYPE_BF16:\n element_type = ov::element::bf16;\n break;\n case GGML_TYPE_I32:\n element_type = ov::element::i32;\n break;\n case GGML_TYPE_I64:\n element_type = ov::element::i64;\n break;\n default:\n // GGML_LOG_WARN(\"%s: unsupported tensor type for ov::Tensor: %s\\n\", __func__, ggml_type_name(tensor->type));\n return nullptr;\n }\n\n const auto & device_name = ggml_openvino_get_device_name();\n auto remote_context = ggml_openvino_get_remote_context();\n\n std::shared_ptr ov_tensor;\n if (is_remote) {\n GGML_ASSERT(device_name == \"GPU\");\n auto gpu_context = remote_context->as();\n auto usm_tensor = gpu_context.create_tensor(element_type, shape, tensor->data);\n ov_tensor = std::make_shared(std::move(usm_tensor));\n } else {\n ov_tensor = std::make_shared(element_type, shape, tensor->data);\n }\n\n return new ggml_openvino_tensor_extra(ov_tensor);\n}\n" }, { "path": "src/ggml-openvino/ggml-openvino-extra.h", "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"openvino/runtime/core.hpp\"\n\n#define CL_TARGET_OPENCL_VERSION 300\n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n// ExtraQuantType enum - defines requantization target formats\nenum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q8_0_C, Q8_0_32 };\n\nov::Core & ov_singleton_core();\n\n// Get the remote context for the current device (returns empty optional for CPU)\nstd::optional ggml_openvino_get_remote_context();\n\n// Get the compile config for the current device\nconst ov::AnyMap & ggml_openvino_get_compile_config();\n\n// Get the OpenCL command queue for GPU operations (returns nullptr for CPU/NPU)\ncl_command_queue ggml_openvino_get_cl_queue();\n\n// Intel USM extension function type\ntypedef cl_int(CL_API_CALL * clEnqueueMemFillINTEL_fn)(cl_command_queue queue,\n void * dst_ptr,\n const void * pattern,\n size_t pattern_size,\n size_t size,\n cl_uint num_events_in_wait_list,\n const cl_event * event_wait_list,\n cl_event * event);\n\ntypedef cl_int(CL_API_CALL * clEnqueueMemcpyINTEL_fn)(cl_command_queue queue,\n cl_bool blocking,\n void * dst_ptr,\n const void * src_ptr,\n size_t size,\n cl_uint num_events_in_wait_list,\n const cl_event * event_wait_list,\n cl_event * event);\n\n// Get the clEnqueueMemFillINTEL function pointer (returns nullptr if not available)\nclEnqueueMemFillINTEL_fn ggml_openvino_get_clEnqueueMemFillINTEL();\n\n// Get the clEnqueueMemcpyINTEL function pointer (returns nullptr if not available)\nclEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL();\n\n// =====================================================\n// Global Device Configuration (singleton)\n// =====================================================\n// Initialized once during backend init from GGML_OPENVINO_DEVICE env var\n\nstruct ggml_openvino_device_config {\n std::string device_name = \"CPU\";\n bool is_npu = false;\n bool initialized = false;\n std::optional remote_context;\n ov::AnyMap compile_config;\n cl_command_queue cl_queue = nullptr;\n\n void init();\n ~ggml_openvino_device_config();\n};\n\n// Get the global device config singleton\nggml_openvino_device_config & ggml_openvino_get_device_config();\n\n// Initialize device config (call during backend init)\nvoid ggml_openvino_init_device_config();\n\n// Get the device name\nconst std::string & ggml_openvino_get_device_name();\n\n// Check if running on NPU\nbool ggml_openvino_is_npu();\n\n// Get requantization type for a tensor type (returns nullopt if no requant needed)\nstd::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false);\n\n// =====================================================\n// OpenVINO Tensor Extra Types\n// =====================================================\n// These types are stored in tensor->extra by the OpenVINO backend buffer.\n// They allow:\n// 1. Pre-built ov::Constant nodes for weights (avoiding memcpy during graph construction)\n// 2. ov::Tensor wrappers for KV cache / compute tensors (for direct use with infer_request)\n\n// Base class for OpenVINO tensor extra data\nstruct ggml_openvino_extra_base {\n enum class Type { WEIGHT, QUANTIZED_WEIGHT, TENSOR };\n Type type;\n virtual ~ggml_openvino_extra_base() = default;\nprotected:\n explicit ggml_openvino_extra_base(Type t) : type(t) {}\n};\n\n// Extra data for F16/F32/BF16 weight tensors - stores the pre-built weight node\nstruct ggml_openvino_weight_extra : public ggml_openvino_extra_base {\n ov::Tensor weights; // The underlying weight data tensor\n std::shared_ptr weight_node; // Pre-built OpenVINO weight node\n\n ggml_openvino_weight_extra(ov::Tensor w, std::shared_ptr n) :\n ggml_openvino_extra_base(Type::WEIGHT),\n weights(std::move(w)),\n weight_node(std::move(n)) {}\n};\n\n// Extra data for quantized weight tensors - stores extracted weights/scales/zp and weight node\nstruct ggml_openvino_quantized_weight_extra : public ggml_openvino_extra_base {\n ov::Tensor weights; // U4 or U8 extracted weights\n ov::Tensor scales; // F16 scales\n ov::Tensor zp; // U4 or U8 zero points (same type as weights)\n std::shared_ptr weight_node; // Pre-built OpenVINO weight subgraph\n\n ggml_openvino_quantized_weight_extra(ov::Tensor w, ov::Tensor s, ov::Tensor z, std::shared_ptr n) :\n ggml_openvino_extra_base(Type::QUANTIZED_WEIGHT),\n weights(std::move(w)),\n scales(std::move(s)),\n zp(std::move(z)),\n weight_node(std::move(n)) {}\n};\n\n// Extra data for KV cache / compute tensors - stores ov::Tensor for infer_request\nstruct ggml_openvino_tensor_extra : public ggml_openvino_extra_base {\n std::shared_ptr tensor; // For direct use with infer_request\n\n explicit ggml_openvino_tensor_extra(std::shared_ptr t)\n : ggml_openvino_extra_base(Type::TENSOR), tensor(std::move(t)) {}\n};\n\n// =====================================================\n// Extracted Size Calculation for Quantized Tensors\n// =====================================================\n// For quantized tensors, we need extra space to store extracted weights, scales, and zero points.\n// Returns the total size needed in the buffer for extracted data.\n\nstruct ggml_openvino_extracted_layout {\n size_t total_size = 0; // Total bytes needed\n size_t weights_offset = 0; // Offset to weights in buffer\n size_t weights_size = 0; // Size of weights in bytes\n size_t scales_offset = 0; // Offset to scales in buffer\n size_t scales_size = 0; // Size of scales in bytes\n size_t zp_offset = 0; // Offset to zero points in buffer\n size_t zp_size = 0; // Size of zero points in bytes (U4 or U8)\n bool is_u4; // true for U4 weights, false for U8\n int64_t weights_per_block; // weights per scale/zp block\n bool is_symmetric; // true for symmetric quantization\n\n // Requantization info\n bool is_requant = false; // true if this tensor needs requantization\n std::optional requant_type; // target requant type if is_requant\n};\n\n// Calculate the buffer layout for extracted quantized data\nggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_tensor * tensor, bool use_bias = false);\n\nggml_openvino_tensor_extra * ggml_openvino_create_tensor_extra(const ggml_tensor * tensor, bool is_remote);\n\n// Register an extra with the tensor's OpenVINO buffer context for proper lifetime management.\n// This sets tensor->extra and tracks the extra in the buffer context for cleanup.\nvoid ggml_openvino_buffer_register_extra(ggml_tensor * tensor, ggml_openvino_extra_base * extra);\n\n// =====================================================\n// OpenVINO Backend Context and Interface\n// =====================================================\nstruct ggml_backend_openvino_context {\n int device = 0;\n std::string name = \"OpenVINO\";\n std::string description = \"OpenVINO Backend Context\";\n\n std::shared_ptr runtime_context = nullptr;\n\n ggml_backend_openvino_context() = default;\n};\n" }, { "path": "src/ggml-openvino/ggml-openvino.cpp", "content": "#include \"ggml-openvino.h\"\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-openvino-extra.h\"\n#include \"ggml-openvino/utils.h\"\n#include \"ggml-quants.h\"\n#include \"ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_WIN32)\n# define WIN32_LEAN_AND_MEAN\n# ifndef NOMINMAX\n# define NOMINMAX\n# endif\n# include \n#else\n# include \n#endif\n\n// =====================================================\n// OpenVINO Buffer Implementation using ov::Tensor\n// =====================================================\n//\n// Design: This implementation uses a hybrid approach:\n// 1. For weight tensors: Store a pre-built ov::op::v0::Constant in tensor->extra\n// - This avoids the memcpy during graph construction\n// - For quantized weights, the constant is already converted to OpenVINO format\n// 2. For KV cache / compute tensors: Store an ov::Tensor in tensor->extra\n// - This can be directly passed to infer_request\n// - Future: can be changed to ov::RemoteTensor for GPU/NPU\n//\n// This design is similar to:\n// - CUDA split buffer: tensor->extra stores device pointers\n// - CPU repack buffer: tensor->extra stores tensor_traits with repacked data\n// =====================================================\n\n// Buffer context that manages per-tensor allocations (no contiguous buffer for weights)\nstruct ggml_backend_openvino_buffer_context {\n int device;\n std::string name;\n size_t id;\n\n // For non-weight buffers (KV cache, compute), we still use contiguous allocation\n void * data;\n size_t size;\n bool is_remote;\n\n // Wrapping of the buffer\n std::shared_ptr ov_buffer;\n\n // Track all extras for cleanup\n std::map tensor_extras;\n\n // Used for re-allocation on device for kvcache\n void * data_prev;\n\n ggml_backend_openvino_buffer_context(int device, size_t size, bool is_remote = false) :\n device(device),\n name(std::string(GGML_OPENVINO_NAME) + std::to_string(device)),\n id([]() {\n static std::atomic next_id{1};\n return next_id.fetch_add(1);\n }()),\n data(nullptr),\n size(size),\n is_remote(is_remote) {\n if (size == 0) {\n return;\n }\n\n const auto & device_name = ggml_openvino_get_device_name();\n\n if (is_remote) {\n GGML_ASSERT(device_name == \"GPU\");\n auto remote_context = ggml_openvino_get_remote_context();\n auto gpu_context = remote_context->as();\n ov::intel_gpu::ocl::USMTensor usm_tensor =\n gpu_context.create_usm_device_tensor(ov::element::u8, ov::Shape{size});\n data = usm_tensor.get();\n ov_buffer = std::make_shared(std::move(usm_tensor));\n } else {\n data = ggml_aligned_malloc(size);\n ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data);\n }\n\n if (data == nullptr) {\n GGML_LOG_ERROR(\"%s: failed to allocate %zu bytes\\n\", __func__, size);\n return;\n }\n\n if (reinterpret_cast(data) % TENSOR_ALIGNMENT != 0) {\n GGML_LOG_ERROR(\"%s: %s buffer is not aligned to %d bytes\\n\", __func__, device_name.c_str(),\n TENSOR_ALIGNMENT);\n GGML_ABORT(\"fatal error\");\n }\n }\n\n ~ggml_backend_openvino_buffer_context() {\n // Clean up all tensor extras\n // GGML_LOG_DEBUG(\"Deleting OpenVINO buffer context #%zu for device %d, size %zu MB\\n\", id, device,\n // size / 1024 / 1024);\n for (auto & pair : tensor_extras) {\n delete pair.second;\n }\n tensor_extras.clear();\n if (!is_remote && data != nullptr) {\n ggml_aligned_free(data, size);\n }\n }\n};\n\n// Buffer type context (per-device)\nstruct ggml_backend_openvino_buffer_type_context {\n int device;\n std::string name;\n};\n\n// Buffer interface functions\nstatic void ggml_backend_openvino_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n delete ctx;\n}\n\nstatic void * ggml_backend_openvino_buffer_get_base(ggml_backend_buffer_t buffer) {\n ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n return ctx->data;\n}\n\nstatic enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {\n // GGML_LOG_DEBUG(\"%s: buffer usage=%d, tensor name=%s\\n\", __func__, buffer->usage, tensor->name);\n ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n\n // Put kvcache on device memory for GPU (NPU memory is too small even for kvcache)\n if (strncmp(tensor->name, \"cache_\", 6) == 0 && !ctx->is_remote && ggml_openvino_get_device_name() == \"GPU\" &&\n !getenv(\"GGML_OPENVINO_STATEFUL_EXECUTION\")) {\n GGML_ASSERT(ctx->tensor_extras.empty());\n auto device = ctx->device;\n auto size = ctx->size;\n auto * data_prev = ctx->data;\n delete ctx;\n ctx = new ggml_backend_openvino_buffer_context(device, size, true);\n buffer->context = ctx;\n tensor->data = (char *) ctx->data + ((char *) tensor->data - (char *) data_prev);\n }\n\n // Views share the extra from view_src\n if (tensor->view_src != nullptr) {\n GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);\n if (tensor->view_src->extra != nullptr) {\n tensor->extra = tensor->view_src->extra;\n }\n return GGML_STATUS_SUCCESS;\n }\n\n ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n\n if (tensor->data != nullptr && !ggml_is_quantized(tensor->type)) {\n ggml_openvino_tensor_extra * extra = ggml_openvino_create_tensor_extra(tensor, ctx->is_remote);\n if (extra != nullptr) {\n auto it = ctx->tensor_extras.find(tensor);\n if (it != ctx->tensor_extras.end()) {\n delete it->second;\n }\n ctx->tensor_extras[tensor] = extra;\n tensor->extra = extra;\n }\n }\n\n return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_openvino_buffer_memset_tensor(ggml_backend_buffer_t buffer,\n ggml_tensor * tensor,\n uint8_t value,\n size_t offset,\n size_t size) {\n // GGML_LOG_DEBUG(\"%s: buffer usage=%d, tensor name=%s\\n\", __func__, buffer->usage, tensor->name);\n GGML_ASSERT(tensor != nullptr && tensor->data != nullptr);\n ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n\n if (ctx->is_remote) {\n // For remote (device) buffers, use OpenCL USM memfill\n cl_command_queue queue = ggml_openvino_get_cl_queue();\n auto mem_fill_fn = ggml_openvino_get_clEnqueueMemFillINTEL();\n if (queue != nullptr && mem_fill_fn != nullptr) {\n uint8_t pattern = value;\n cl_int err = mem_fill_fn(queue, (char *) tensor->data + offset, &pattern, sizeof(pattern), size, 0, nullptr,\n nullptr);\n if (err != CL_SUCCESS) {\n GGML_LOG_ERROR(\"%s: clEnqueueMemFillINTEL failed with error %d\\n\", __func__, err);\n }\n clFinish(queue);\n } else {\n GGML_LOG_ERROR(\"%s: no OpenCL queue or clEnqueueMemFillINTEL not available for GPU buffer\\n\", __func__);\n }\n } else {\n memset((char *) tensor->data + offset, value, size);\n }\n}\n\nstatic void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer,\n ggml_tensor * tensor,\n const void * data,\n size_t offset,\n size_t size) {\n // GGML_LOG_DEBUG(\"%s: buffer usage=%d, tensor name=%s\\n\", __func__, buffer->usage, tensor->name);\n GGML_ASSERT(tensor != nullptr && tensor->data != nullptr);\n ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n\n // Check if this is a weight buffer (usage is set BEFORE set_tensor is called, except in test-backend-ops)\n bool is_weight_buffer = (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS);\n // Full tensor set: offset=0, full size, not a view\n bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr);\n // 2D tensor (typical weight shape)\n bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1);\n\n if (is_weight_buffer && is_full_tensor_set && is_2d) {\n try {\n auto result = process_weight_tensor(tensor, data, tensor->data);\n result.weight_node->set_friendly_name(tensor->name);\n\n // const auto & layout = result.layout;\n ggml_openvino_extra_base * extra;\n\n // Quantized path with extracted weight/scale/zp tensors\n if (result.is_quantized()) {\n extra = new ggml_openvino_quantized_weight_extra(std::move(result.weights), std::move(result.scales),\n std::move(result.zp), result.weight_node);\n\n // if (layout.is_requant) {\n // GGML_LOG_DEBUG(\"%s: requantized %s to %s (u%d, block_size=%ld)\\n\", __func__, tensor->name,\n // extra_quant_type_name(layout.requant_type.value()), layout.is_u4 ? 4 : 8,\n // layout.weights_per_block);\n // } else {\n // int64_t n_blocks = ggml_nelements(tensor) / layout.weights_per_block;\n // GGML_LOG_DEBUG(\"%s: extracted quantized weight node for %s (u%d, %zu weights, %ld blocks)\\n\",\n // __func__, tensor->name, layout.is_u4 ? 4 : 8, layout.weights_size, n_blocks);\n // }\n } else {\n // F16/F32/BF16 weight or F16-requant\n extra = new ggml_openvino_weight_extra(std::move(result.weights), result.weight_node);\n\n // if (layout.total_size > 0) {\n // GGML_LOG_DEBUG(\"%s: requantized %s to F16\\n\", __func__, tensor->name);\n // } else {\n // GGML_LOG_DEBUG(\"%s: created shared-memory weight node for %s\\n\", __func__, tensor->name);\n // }\n }\n\n ctx->tensor_extras[tensor] = extra;\n tensor->extra = extra;\n\n } catch (const std::exception & e) {\n GGML_LOG_ERROR(\"%s: failed to process weight tensor for %s: %s\\n\", __func__, tensor->name, e.what());\n memcpy((char *) tensor->data + offset, data, size);\n }\n } else {\n // Non-weight tensor (KV cache, activations, etc.) - copy data. test-backend-ops also goes here\n if (ctx->is_remote) {\n cl_command_queue queue = ggml_openvino_get_cl_queue();\n auto mem_cpy_fn = ggml_openvino_get_clEnqueueMemcpyINTEL();\n if (queue != nullptr && mem_cpy_fn != nullptr) {\n cl_int err =\n mem_cpy_fn(queue, CL_TRUE, (char *) tensor->data + offset, data, size, 0, nullptr, nullptr);\n if (err != CL_SUCCESS) {\n GGML_LOG_ERROR(\"%s: clEnqueueMemcpyINTEL failed with error %d\\n\", __func__, err);\n }\n } else {\n GGML_LOG_ERROR(\"%s: no OpenCL queue or clEnqueueMemcpyINTEL not available for GPU buffer\\n\", __func__);\n }\n } else {\n memcpy((char *) tensor->data + offset, data, size);\n }\n\n ggml_openvino_tensor_extra * extra = ggml_openvino_create_tensor_extra(tensor, ctx->is_remote);\n if (extra == nullptr) {\n // GGML_LOG_ERROR(\"%s: failed to create tensor extra for %s\\n\", __func__, tensor->name);\n return;\n }\n\n auto it = ctx->tensor_extras.find(tensor);\n if (it != ctx->tensor_extras.end()) {\n delete it->second;\n }\n ctx->tensor_extras[tensor] = extra;\n tensor->extra = extra;\n }\n}\n\nstatic void ggml_backend_openvino_buffer_get_tensor(ggml_backend_buffer_t buffer,\n const ggml_tensor * tensor,\n void * data,\n size_t offset,\n size_t size) {\n // GGML_LOG_DEBUG(\"%s: buffer usage=%d, tensor name=%s\\n\", __func__, buffer->usage, tensor->name);\n GGML_ASSERT(tensor != nullptr && tensor->data != nullptr);\n ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n\n if (ctx->is_remote) {\n // For remote (device) buffers, use OpenCL USM memcpy (device-to-host)\n cl_command_queue queue = ggml_openvino_get_cl_queue();\n auto mem_cpy_fn = ggml_openvino_get_clEnqueueMemcpyINTEL();\n if (queue != nullptr && mem_cpy_fn != nullptr) {\n cl_int err =\n mem_cpy_fn(queue, CL_TRUE, data, (const char *) tensor->data + offset, size, 0, nullptr, nullptr);\n if (err != CL_SUCCESS) {\n GGML_LOG_ERROR(\"%s: clEnqueueMemcpyINTEL failed with error %d\\n\", __func__, err);\n }\n } else {\n GGML_LOG_ERROR(\"%s: no OpenCL queue or clEnqueueMemcpyINTEL not available for GPU buffer\\n\", __func__);\n }\n } else {\n memcpy(data, (const char *) tensor->data + offset, size);\n }\n}\n\nstatic bool ggml_backend_openvino_buffer_cpy_tensor(ggml_backend_buffer_t buffer,\n const ggml_tensor * src,\n ggml_tensor * dst) {\n // GGML_LOG_DEBUG(\"%s: src tensor name=%s, dst tensor name=%s\\n\", __func__, src->name, dst->name);\n GGML_ASSERT(src != nullptr && dst != nullptr);\n ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n\n if (ctx->is_remote) {\n // For remote (device) buffers, use OpenCL USM memcpy\n cl_command_queue queue = ggml_openvino_get_cl_queue();\n auto mem_cpy_fn = ggml_openvino_get_clEnqueueMemcpyINTEL();\n if (queue == nullptr || mem_cpy_fn == nullptr) {\n GGML_LOG_ERROR(\"%s: no OpenCL queue or clEnqueueMemcpyINTEL not available for GPU buffer\\n\", __func__);\n return false;\n }\n // Can copy from host to device\n if (ggml_backend_buffer_is_host(src->buffer)) {\n cl_int err = mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr);\n if (err != CL_SUCCESS) {\n GGML_LOG_ERROR(\"%s: clEnqueueMemcpyINTEL (host-to-device) failed with error %d\\n\", __func__, err);\n return false;\n }\n return true;\n }\n // Can also copy from device to device if both are OpenVINO remote buffers\n if (ggml_backend_buffer_is_openvino(src->buffer)) {\n ggml_backend_openvino_buffer_context * src_ctx =\n (ggml_backend_openvino_buffer_context *) src->buffer->context;\n if (src_ctx->is_remote) {\n cl_int err =\n mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr);\n if (err != CL_SUCCESS) {\n GGML_LOG_ERROR(\"%s: clEnqueueMemcpyINTEL (device-to-device) failed with error %d\\n\", __func__,\n err);\n return false;\n }\n return true;\n }\n }\n return false;\n }\n\n // Host buffer - can copy from any host buffer\n if (ggml_backend_buffer_is_host(src->buffer)) {\n memcpy(dst->data, src->data, ggml_nbytes(src));\n return true;\n }\n return false;\n}\n\nstatic void ggml_backend_openvino_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n GGML_ASSERT(ctx->data != nullptr);\n if (ctx->is_remote) {\n cl_command_queue queue = ggml_openvino_get_cl_queue();\n auto mem_fill_fn = ggml_openvino_get_clEnqueueMemFillINTEL();\n if (queue != nullptr && mem_fill_fn != nullptr) {\n uint8_t pattern = value;\n cl_int err = mem_fill_fn(queue, ctx->data, &pattern, sizeof(pattern), ctx->size, 0, nullptr, nullptr);\n if (err != CL_SUCCESS) {\n GGML_LOG_WARN(\"%s: clEnqueueMemFillINTEL failed with error %d\\n\", __func__, err);\n }\n clFinish(queue);\n } else {\n GGML_LOG_WARN(\"%s: no OpenCL queue or clEnqueueMemFillINTEL not available for GPU buffer clear\\n\",\n __func__);\n }\n } else {\n memset(ctx->data, value, ctx->size);\n }\n}\n\nstatic const ggml_backend_buffer_i ggml_backend_openvino_buffer_interface = {\n /* .free_buffer = */ ggml_backend_openvino_buffer_free_buffer,\n /* .get_base = */ ggml_backend_openvino_buffer_get_base,\n /* .init_tensor = */ ggml_backend_openvino_buffer_init_tensor,\n /* .memset_tensor = */ ggml_backend_openvino_buffer_memset_tensor,\n /* .set_tensor = */ ggml_backend_openvino_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_openvino_buffer_get_tensor,\n /* .cpy_tensor = */ ggml_backend_openvino_buffer_cpy_tensor,\n /* .clear = */ ggml_backend_openvino_buffer_clear,\n /* .reset = */ NULL,\n};\n\n// Buffer type interface functions\nstatic const char * ggml_backend_openvino_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_openvino_buffer_type_context * ctx = (ggml_backend_openvino_buffer_type_context *) buft->context;\n return ctx->name.c_str();\n}\n\nstatic ggml_backend_buffer_t ggml_backend_openvino_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,\n size_t size) {\n ggml_backend_openvino_buffer_type_context * buft_ctx = (ggml_backend_openvino_buffer_type_context *) buft->context;\n\n // Create buffer context with contiguous memory allocation\n ggml_backend_openvino_buffer_context * ctx = new ggml_backend_openvino_buffer_context(buft_ctx->device, size);\n\n if (ctx->data == nullptr && size > 0) {\n GGML_LOG_ERROR(\"%s: failed to allocate buffer of size %zu\\n\", __func__, size);\n delete ctx;\n return nullptr;\n }\n\n return ggml_backend_buffer_init(buft, ggml_backend_openvino_buffer_interface, ctx, size);\n}\n\nstatic size_t ggml_backend_openvino_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n GGML_UNUSED(buft);\n return TENSOR_ALIGNMENT;\n}\n\nstatic size_t ggml_backend_openvino_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {\n GGML_UNUSED(buft);\n return SIZE_MAX;\n}\n\nstatic size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft,\n const ggml_tensor * tensor) {\n GGML_UNUSED(buft);\n\n // For quantized 2D tensors (weights), we need extra space for extracted data\n if (ggml_is_quantized(tensor->type) && tensor->ne[2] == 1 && tensor->ne[3] == 1) {\n ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor);\n if (layout.total_size > 0) {\n // GGML_LOG_DEBUG(\"%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\\n\",\n // __func__, tensor->name, layout.total_size, ggml_nbytes(tensor), layout.weights_size,\n // layout.scales_size, layout.zp_size);\n return layout.total_size;\n }\n }\n\n return ggml_nbytes(tensor);\n}\n\nstatic const ggml_backend_buffer_type_i ggml_backend_openvino_buffer_type_interface = {\n /* .get_name = */ ggml_backend_openvino_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_openvino_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_openvino_buffer_type_get_alignment,\n /* .get_max_size = */ ggml_backend_openvino_buffer_type_get_max_size,\n /* .get_alloc_size = */ ggml_backend_openvino_buffer_type_get_alloc_size,\n /* .is_host = */ nullptr,\n};\n\n// Get buffer type for a specific device\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_buffer_type(int device) {\n GGML_ASSERT(device >= 0 && device < ggml_backend_openvino_get_device_count());\n\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n static std::vector buffer_types;\n static std::vector buffer_type_contexts;\n\n if (buffer_types.empty()) {\n int device_count = ggml_backend_openvino_get_device_count();\n buffer_types.resize(device_count);\n buffer_type_contexts.resize(device_count);\n\n for (int i = 0; i < device_count; i++) {\n buffer_type_contexts[i].device = i;\n buffer_type_contexts[i].name = std::string(GGML_OPENVINO_NAME) + std::to_string(i);\n\n buffer_types[i] = ggml_backend_buffer_type{\n /* .iface = */ ggml_backend_openvino_buffer_type_interface,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_openvino_reg(), i),\n /* .context = */ &buffer_type_contexts[i],\n };\n }\n }\n\n return &buffer_types[device];\n}\n\n// =====================================================\n// OpenVINO Host Buffer Implementation\n// =====================================================\n\nstatic const char * ggml_backend_openvino_host_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_openvino_buffer_type_context * ctx = (ggml_backend_openvino_buffer_type_context *) buft->context;\n static std::string name;\n name = ctx->name + \"_HOST\";\n return name.c_str();\n}\n\nstatic bool ggml_backend_openvino_host_buffer_type_is_host(ggml_backend_buffer_type_t buft) {\n GGML_UNUSED(buft);\n return true;\n}\n\nstatic const ggml_backend_buffer_type_i ggml_backend_openvino_host_buffer_type_interface = {\n /* .get_name = */ ggml_backend_openvino_host_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_openvino_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_openvino_buffer_type_get_alignment,\n /* .get_max_size = */ ggml_backend_openvino_buffer_type_get_max_size,\n /* .get_alloc_size = */ ggml_backend_openvino_buffer_type_get_alloc_size,\n /* .is_host = */ ggml_backend_openvino_host_buffer_type_is_host,\n};\n\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_host_buffer_type(int device) {\n GGML_ASSERT(device >= 0 && device < ggml_backend_openvino_get_device_count());\n\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n static std::vector buffer_types;\n static std::vector buffer_type_contexts;\n\n if (buffer_types.empty()) {\n int device_count = ggml_backend_openvino_get_device_count();\n buffer_types.resize(device_count);\n buffer_type_contexts.resize(device_count);\n\n for (int i = 0; i < device_count; i++) {\n buffer_type_contexts[i].device = i;\n buffer_type_contexts[i].name = std::string(GGML_OPENVINO_NAME) + std::to_string(i);\n\n buffer_types[i] = ggml_backend_buffer_type{\n /* .iface = */ ggml_backend_openvino_host_buffer_type_interface,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_openvino_reg(), i),\n /* .context = */ &buffer_type_contexts[i],\n };\n }\n }\n\n return &buffer_types[device];\n}\n\nbool ggml_backend_buffer_is_openvino(ggml_backend_buffer_t buffer) {\n return buffer->iface.free_buffer == ggml_backend_openvino_buffer_free_buffer;\n}\n\nsize_t ggml_backend_openvino_buffer_get_ctx_id(ggml_backend_buffer_t buffer) {\n if (!ggml_backend_buffer_is_openvino(buffer)) {\n return 0;\n }\n ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;\n return ctx->id;\n}\n\nvoid ggml_openvino_buffer_register_extra(ggml_tensor * tensor, ggml_openvino_extra_base * extra) {\n GGML_ASSERT(tensor != nullptr);\n GGML_ASSERT(tensor->buffer != nullptr);\n GGML_ASSERT(ggml_backend_buffer_is_openvino(tensor->buffer));\n\n auto * ctx = static_cast(tensor->buffer->context);\n\n auto it = ctx->tensor_extras.find(tensor);\n if (it != ctx->tensor_extras.end()) {\n delete it->second;\n }\n\n ctx->tensor_extras[tensor] = extra;\n tensor->extra = extra;\n}\n\nbool ggml_backend_buft_is_openvino(ggml_backend_buffer_type_t buft) {\n return buft->iface.get_name == ggml_backend_openvino_buffer_type_get_name;\n}\n\nbool ggml_backend_buft_is_openvino_host(ggml_backend_buffer_type_t buft) {\n return buft->iface.get_name == ggml_backend_openvino_host_buffer_type_get_name;\n}\n\nstatic void ggml_backend_openvino_free(ggml_backend_t backend) {\n ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context;\n delete ctx;\n delete backend;\n}\n\nstatic const char * ggml_backend_openvino_get_name(ggml_backend_t backend) {\n return GGML_OPENVINO_NAME;\n GGML_UNUSED(backend);\n}\n\nstatic enum ggml_status ggml_backend_openvino_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {\n return ov_graph_compute(cgraph, backend);\n GGML_UNUSED(backend);\n}\n\nstatic const ggml_backend_i ggml_backend_openvino_interface = {\n /* .get_name = */ ggml_backend_openvino_get_name,\n /* .free = */ ggml_backend_openvino_free,\n /* .set_tensor_async = */ NULL,\n /* .get_tensor_async = */ NULL,\n /* .cpy_tensor_async = */ NULL,\n /* .synchronize = */ NULL,\n /* .graph_plan_create = */ NULL,\n /* .graph_plan_free = */ NULL,\n /* .graph_plan_update = */ NULL,\n /* .graph_plan_compute = */ NULL,\n /* .graph_compute = */ ggml_backend_openvino_graph_compute,\n /* .event_record = */ NULL,\n /* .event_wait = */ NULL,\n /* .graph_optimize = */ NULL,\n};\n\nint ggml_backend_openvino_get_device_count() {\n return 1;\n}\n\nstatic ggml_guid_t ggml_backend_openvino_guid(void) {\n static ggml_guid guid = {0x12, 0xa8, 0xae, 0xf4, 0xc0, 0x1e, 0x61, 0x97,\n 0x8f, 0xeb, 0x33, 0x04, 0xa1, 0x33, 0x51, 0x2d};\n return &guid;\n}\n\nstatic std::shared_ptr get_ov_runtime_context_ptr() {\n static std::shared_ptr r_ctx = std::make_shared();\n return r_ctx;\n}\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_openvino_init(int device) {\n if (device < 0 || device >= ggml_backend_openvino_get_device_count()) {\n GGML_LOG_ERROR(\"%s: invalid device %d\\n\", __func__, device);\n return nullptr;\n }\n\n ggml_backend_openvino_context * ctx = new ggml_backend_openvino_context;\n if (ctx == nullptr) {\n GGML_LOG_ERROR(\"%s: failed to allocate context\\n\", __func__);\n return nullptr;\n }\n\n ctx->runtime_context = get_ov_runtime_context_ptr();\n if (ctx->runtime_context == nullptr) {\n GGML_LOG_ERROR(\"%s: failed to allocate runtime context\\n\", __func__);\n delete ctx;\n return nullptr;\n }\n\n std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context);\n r_ctx->device = ggml_openvino_get_device_name();\n r_ctx->stateful = getenv(\"GGML_OPENVINO_STATEFUL_EXECUTION\") && !ggml_openvino_is_npu();\n\n ggml_backend_t openvino_backend = new ggml_backend{\n /* .guid = */ ggml_backend_openvino_guid(),\n /* .interface = */ ggml_backend_openvino_interface,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_openvino_reg(), device),\n /* .context = */ ctx,\n };\n\n return openvino_backend;\n}\n\nGGML_BACKEND_API bool ggml_backend_is_openvino(ggml_backend_t backend) {\n return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_openvino_guid());\n}\n\nstruct ggml_backend_openvino_device_context {\n int device;\n std::string name;\n std::string description;\n};\n\nstatic const char * ggml_backend_openvino_device_get_name(ggml_backend_dev_t dev) {\n ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;\n return ctx->name.c_str();\n}\n\nstatic const char * ggml_backend_openvino_device_get_description(ggml_backend_dev_t dev) {\n ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;\n return ctx->description.c_str();\n}\n\nstatic void ggml_backend_openvino_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n#ifdef _WIN32\n MEMORYSTATUSEX status;\n status.dwLength = sizeof(status);\n GlobalMemoryStatusEx(&status);\n *total = status.ullTotalPhys;\n *free = status.ullAvailPhys;\n#else\n long pages = sysconf(_SC_PHYS_PAGES);\n long page_size = sysconf(_SC_PAGE_SIZE);\n *total = pages * page_size;\n\n // \"free\" system memory is ill-defined, for practical purposes assume that all of it is free:\n *free = *total;\n#endif // _WIN32\n\n GGML_UNUSED(dev);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_openvino_device_get_type(ggml_backend_dev_t dev) {\n GGML_UNUSED(dev);\n return GGML_BACKEND_DEVICE_TYPE_GPU;\n}\n\nstatic void ggml_backend_openvino_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {\n props->name = ggml_backend_openvino_device_get_name(dev);\n props->description = ggml_backend_openvino_device_get_description(dev);\n props->type = ggml_backend_openvino_device_get_type(dev);\n ggml_backend_openvino_device_get_memory(dev, &props->memory_free, &props->memory_total);\n\n props->caps = {\n /* .async = */ false,\n /* .host_buffer = */ false,\n /* .buffer_from_host_ptr = */ false,\n /* .events = */ false,\n };\n}\n\nstatic ggml_backend_t ggml_backend_openvino_device_init(ggml_backend_dev_t dev, const char * params) {\n GGML_UNUSED(params);\n ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;\n return ggml_backend_openvino_init(ctx->device);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_openvino_device_get_buffer_type(ggml_backend_dev_t dev) {\n ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;\n return ggml_backend_openvino_buffer_type(ctx->device);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_openvino_device_get_host_buffer_type(ggml_backend_dev_t dev) {\n ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;\n return ggml_backend_openvino_host_buffer_type(ctx->device);\n}\n\nstatic bool has_view_op_input(const ggml_tensor * op) {\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (op->src[i] == nullptr) {\n break;\n }\n if (op->src[i]->op == GGML_OP_VIEW) {\n return true;\n }\n }\n return false;\n}\n\nstatic bool is_supported_flash_attn_pattern(const ggml_tensor * op) {\n // pattern of q,k,v should be q->op==PERMUTE, q->src[0]->op==VIEW, q->src[0]->src[0]->view_src==nullptr\n for (int i = 0; i < 3; i++) {\n const ggml_tensor * src = op->src[i];\n if (src->op != GGML_OP_PERMUTE || src->src[0] == nullptr || src->src[0]->op != GGML_OP_VIEW ||\n src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) {\n return false;\n }\n }\n return true;\n}\n\nstatic bool is_op_unsupported_case(const ggml_tensor * op) {\n switch (op->op) {\n case GGML_OP_GET_ROWS:\n case GGML_OP_SET_ROWS: {\n if (op->ne[3] != 1) {\n return true;\n }\n break;\n }\n case GGML_OP_ADD:\n case GGML_OP_MUL: {\n if (op->src[1]->op == GGML_OP_PERMUTE) {\n return true;\n }\n for (int i = 0; i < 4; i++) {\n if (op->src[0]->ne[i] != op->src[1]->ne[i] && (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1)) {\n return true;\n }\n }\n break;\n }\n case GGML_OP_SOFT_MAX: {\n if (op->src[2] != nullptr) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support SOFT_MAX with sinks\\n\");\n return true;\n }\n float scale = 1.0f;\n float max_bias = 0.0f;\n const auto * op_params = op->op_params;\n memcpy(&scale, (const float *) op_params + 0, sizeof(float));\n memcpy(&max_bias, (const float *) op_params + 1, sizeof(float));\n if (max_bias > 0) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support SOFT_MAX with max_bias > 0\\n\");\n return true;\n }\n break;\n }\n case GGML_OP_FLASH_ATTN_EXT: {\n if (op->src[4] != nullptr) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support FLASH_ATTN_EXT with sinks\\n\");\n return true;\n }\n if (!is_supported_flash_attn_pattern(op)) {\n return true;\n }\n float scale = 1.0f;\n float max_bias = 0.0f;\n float logit_softcap = 0.0f;\n const auto * op_params = op->op_params;\n memcpy(&scale, (const float *) op_params + 0, sizeof(float));\n memcpy(&max_bias, (const float *) op_params + 1, sizeof(float));\n memcpy(&logit_softcap, (const float *) op_params + 2, sizeof(float));\n if (max_bias > 0) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support FLASH_ATTN_EXT with max_bias > 0\\n\");\n return true;\n }\n if (logit_softcap != 0) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support FLASH_ATTN_EXT with logit_softcap != 0\\n\");\n return true;\n }\n break;\n }\n case GGML_OP_PERMUTE: {\n if (op->type == GGML_TYPE_BF16) {\n // err msg: [GPU] Could not find a suitable kernel for transpose\n // GGML_LOG_WARN(\"OpenVINO backend does not support PERMUTE with BF16 type\\n\");\n return true;\n }\n break;\n }\n case GGML_OP_CPY: {\n if (op->src[1] != op) {\n // GGML_LOG_WARN(\"OpenVINO backend only supports CPY that is a cast\\n\");\n return true;\n }\n break;\n }\n case GGML_OP_MUL_MAT: {\n if (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16) {\n // Has accuracy issue, try enabling this and see `test-backend-ops -o \"MUL_MAT\"`\n // GGML_LOG_WARN(\"OpenVINO backend does not support MUL_MAT with two F16 tensors\\n\");\n return true;\n }\n if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) {\n return true;\n }\n if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) {\n return true;\n }\n if (ggml_is_quantized(op->src[0]->type) && op->src[0]->ne[1] == 1) {\n // MUL_MAT(type_a=q4_0,type_b=f32,m=1,n=2048,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1)\n // triggers a bug in ov matmul_shape_inference.hpp\n return true;\n }\n if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) {\n return true;\n }\n break;\n }\n case GGML_OP_ROPE: {\n const int32_t * op_params = op->op_params;\n const int n_dims = op_params[1];\n const int mode = op_params[2];\n if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support ROPE with mode %d\\n\", mode);\n return true;\n }\n if (n_dims != 0.0f && n_dims != op->src[0]->ne[0]) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support ROPE with n_dims %d != src[0]->ne[0] %ld\\n\", n_dims,\n // op->src[0]->ne[0]);\n return true;\n }\n if (op->type != GGML_TYPE_F32) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support ROPE with type %s\\n\", ggml_type_name(op->type));\n return true;\n }\n float freq_scale;\n float ext_factor;\n memcpy(&freq_scale, op_params + 6, sizeof(float));\n memcpy(&ext_factor, op_params + 7, sizeof(float));\n if (ext_factor != 0.0f) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support ROPE with ext_factor %f != 0.0f\\n\", ext_factor);\n return true;\n }\n if (op->src[0]->op == GGML_OP_VIEW) {\n if (op->src[0]->view_src->ne[1] != op->src[0]->ne[2]) {\n // GGML_LOG_WARN(\n // \"OpenVINO backend does not support ROPE with src[0]->view_src->ne[1] %ld != src[0]->ne[2] \"\n // \"%ld\\n\",\n // op->src[0]->view_src->ne[1], op->src[0]->ne[2]);\n return true;\n }\n }\n break;\n }\n default:\n break;\n }\n if (op->op == GGML_OP_GET_ROWS) {\n if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K)) {\n // ERR = 0.000000306 > 0.000000100 GET_ROWS(type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0)\n // ERR = 0.000000197 > 0.000000100 GET_ROWS(type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0)\n return true;\n }\n }\n return false;\n}\n\nstatic bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {\n GGML_ASSERT(dev->reg != nullptr);\n\n static std::set supported_types{GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64,\n GGML_TYPE_I32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K,\n GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K};\n\n static const std::set supported_ops{GGML_OP_NONE, GGML_OP_ADD, GGML_OP_MUL, GGML_OP_MUL_MAT, GGML_OP_VIEW,\n /*GGML_OP_CONT,*/ GGML_OP_RESHAPE, GGML_OP_PERMUTE, GGML_OP_TRANSPOSE,\n GGML_OP_GET_ROWS, GGML_OP_ROPE, GGML_OP_RMS_NORM, GGML_OP_SCALE,\n // softmax is not updated due to replaced by flash_attn_ext\n // GGML_OP_SOFT_MAX,\n GGML_OP_SET_ROWS, GGML_OP_FLASH_ATTN_EXT, GGML_OP_CPY};\n static const std::set supported_unary_ops{\n GGML_UNARY_OP_SILU,\n };\n static const std::set supported_glu_ops{\n GGML_GLU_OP_SWIGLU,\n GGML_GLU_OP_GEGLU,\n };\n\n switch (op->op) {\n case GGML_OP_UNARY: {\n auto supported = supported_unary_ops.find(ggml_get_unary_op(op)) != supported_unary_ops.end();\n if (!supported) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support unary op %s\\n\", ggml_unary_op_name(ggml_get_unary_op(op)));\n return false;\n }\n if (has_view_op_input(op)) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support unary op %s with view input\\n\",\n // ggml_unary_op_name(ggml_get_unary_op(op)));\n return false;\n }\n break;\n }\n case GGML_OP_GLU: {\n auto supported = supported_glu_ops.find(ggml_get_glu_op(op)) != supported_glu_ops.end();\n if (!supported) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support GLU op %s\\n\", ggml_glu_op_name(ggml_get_glu_op(op)));\n return false;\n }\n if (has_view_op_input(op)) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support unary op %s with view input\\n\",\n // ggml_glu_op_name(ggml_get_glu_op(op)));\n return false;\n }\n if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) {\n // triggers bug in ov gpu\n return false;\n }\n break;\n }\n default: {\n auto supported = supported_ops.find(op->op) != supported_ops.end();\n if (!supported) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support op %s\\n\", ggml_op_name(op->op));\n return false;\n }\n static std::set ops_not_support_view_input{\n GGML_OP_GET_ROWS,\n GGML_OP_RMS_NORM,\n };\n if (ops_not_support_view_input.find(op->op) != ops_not_support_view_input.end() && has_view_op_input(op)) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support op %s with view input\\n\", ggml_op_name(op->op));\n return false;\n }\n }\n }\n\n if (supported_types.find(op->type) == supported_types.end()) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support tensor type %s\\n\", ggml_type_name(op->type));\n return false;\n }\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n auto * src = op->src[i];\n if (src == nullptr) {\n break;\n }\n if (supported_types.find(src->type) == supported_types.end()) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support tensor type %s\\n\", ggml_type_name(src->type));\n return false;\n }\n if (ggml_is_quantized(src->type) && src->ne[2] != 1) {\n // GGML_LOG_WARN(\"OpenVINO backend does not support 3D quantized tensors\\n\");\n return false;\n }\n }\n\n if (is_op_unsupported_case(op)) {\n return false;\n }\n return true;\n}\n\nstatic bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n return ggml_backend_buft_is_openvino(buft) || ggml_backend_buft_is_host(buft);\n GGML_UNUSED(dev);\n}\n\nstatic const struct ggml_backend_device_i ggml_backend_openvino_device_interface = {\n /* .get_name = */ ggml_backend_openvino_device_get_name,\n /* .get_description = */ ggml_backend_openvino_device_get_description,\n /* .get_memory = */ ggml_backend_openvino_device_get_memory,\n /* .get_type = */ ggml_backend_openvino_device_get_type,\n /* .get_props = */ ggml_backend_openvino_device_get_props,\n /* .init_backend = */ ggml_backend_openvino_device_init,\n /* .get_buffer_type = */ ggml_backend_openvino_device_get_buffer_type,\n /* .get_host_buffer_type = */ ggml_backend_openvino_device_get_host_buffer_type,\n /* .buffer_from_host_ptr = */ NULL,\n /* .supports_op = */ ggml_backend_openvino_device_supports_op,\n /* .supports_buft = */ ggml_backend_openvino_device_supports_buft,\n /* .offload_op = */ NULL,\n /* .event_new = */ NULL,\n /* .event_free = */ NULL,\n /* .event_synchronize = */ NULL,\n};\n\nstruct ggml_backend_openvino_reg_context {\n std::vector devices;\n};\n\nstatic const char * ggml_backend_openvino_reg_get_name(ggml_backend_reg_t reg) {\n return GGML_OPENVINO_NAME;\n GGML_UNUSED(reg);\n}\n\nstatic size_t ggml_backend_openvino_reg_get_device_count(ggml_backend_reg_t reg) {\n GGML_UNUSED(reg);\n return (size_t) ggml_backend_openvino_get_device_count();\n}\n\nstatic ggml_backend_dev_t ggml_backend_openvino_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n ggml_backend_openvino_reg_context * ctx = (ggml_backend_openvino_reg_context *) reg->context;\n GGML_ASSERT(index < ctx->devices.size());\n return ctx->devices[index];\n}\n\nstatic const struct ggml_backend_reg_i ggml_backend_openvino_reg_interface = {\n /* .get_name = */ ggml_backend_openvino_reg_get_name,\n /* .get_device_count = */ ggml_backend_openvino_reg_get_device_count,\n /* .get_device = */ ggml_backend_openvino_reg_get_device,\n /* .get_proc_address = */ NULL,\n};\n\nstatic void ggml_openvino_init() {\n // Initialize device config singleton from env var\n ggml_openvino_init_device_config();\n GGML_LOG_INFO(\"OpenVINO: using device %s\\n\", ggml_openvino_get_device_name().c_str());\n}\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_openvino_reg(void) {\n static ggml_backend_reg reg;\n\n static bool initialized = false;\n {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n if (!initialized) {\n ggml_openvino_init();\n\n ggml_backend_openvino_reg_context * ctx = new ggml_backend_openvino_reg_context;\n\n for (int i = 0; i < ggml_backend_openvino_get_device_count(); i++) {\n ggml_backend_openvino_device_context * dev_ctx = new ggml_backend_openvino_device_context;\n dev_ctx->device = i;\n dev_ctx->name = GGML_OPENVINO_NAME + std::to_string(i);\n\n dev_ctx->description = ov::get_openvino_version().description;\n\n ggml_backend_dev_t dev =\n new ggml_backend_device{/* .interface = */ ggml_backend_openvino_device_interface,\n /* .reg = */ ®,\n /* .context = */ dev_ctx};\n ctx->devices.push_back(dev);\n }\n\n reg = ggml_backend_reg{/* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_openvino_reg_interface,\n /* .context = */ ctx};\n }\n\n initialized = true;\n }\n\n return ®\n}\n" }, { "path": "src/ggml-openvino/ggml-quants.cpp", "content": "#include \"ggml-quants.h\"\n\n#include \"ggml-common.h\"\n#include \"ggml-impl.h\"\n#include \"ggml.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nvoid unpack_32_4(const uint8_t * data, uint8_t * dst) {\n std::fill_n(dst, 16, 0);\n for (int j = 0; j < 16; ++j) {\n uint8_t x = (data[j] & 0x0F);\n uint8_t y = (data[j] >> 4);\n if (j % 2 != 0) {\n x <<= 4;\n y <<= 4;\n }\n dst[j / 2] |= x;\n dst[8 + j / 2] |= y; // Last 16 weights are in the higher bits\n }\n}\n\n// Extracts (weight, scales, zp) from Q4_0 tensors.\n// Data layout is: |16 bit scale|32 x 4bit weights|.\nvoid extract_q4_0_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr) {\n const uint64_t bytes_per_block = 18; // 2 bytes scale, 32x0.5 byte weights\n\n auto * data = static_cast(tensor->data);\n auto * weights = static_cast(weights_arr.data());\n auto * scales = scales_arr.data::value_type>();\n auto * zp = static_cast(zp_arr.data());\n\n bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization\n\n // For Q4_0, zero point is always 8\n if (is_scalar_zp) {\n zp[0] = 8 | (8 << 4); // Pack two 4-bit values\n }\n\n ov::parallel_for(scales_arr.get_size(), [&](size_t i) {\n scales[i] = ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block)));\n // For asymmetric quantization, compute per-block zero points\n if (!is_scalar_zp) {\n // Pack two 4-bit zero points per byte\n if (i % 2 == 0) {\n zp[i / 2] = 8; // Lower nibble\n } else {\n zp[i / 2] |= (8 << 4); // Upper nibble\n }\n }\n unpack_32_4(data + i * bytes_per_block + 2, weights + i * 16);\n });\n}\n\n// Extracts (weight, scales, zp) from Q4_1 tensors.\n// Data layout is: |16 bit scale|16 bit min|32 x 4bit weights|.\nvoid extract_q4_1_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n bool use_bias) {\n const uint64_t bytes_per_block = 20; // 2 bytes scale, 2 bytes min, 32x0.5 byte weights\n\n auto * data = static_cast(tensor->data);\n auto * weights = static_cast(weights_arr.data());\n auto * scales = scales_arr.data::value_type>();\n\n if (use_bias) {\n // Store bias (min) directly as f16 instead of computing u4 zero points\n auto * bias = zp_arr.data::value_type>();\n ov::parallel_for(scales_arr.get_size(), [&](size_t i) {\n float scale = static_cast(ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block))));\n float min = static_cast(ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block + 2))));\n scales[i] = ov::float16(scale);\n bias[i] = ov::float16(min); // bias = min, dequant: w*s + bias\n unpack_32_4(data + i * bytes_per_block + 4, weights + i * 16);\n });\n } else {\n auto * zp = static_cast(zp_arr.data());\n ov::parallel_for(scales_arr.get_size(), [&](size_t i) {\n float scale = static_cast(ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block))));\n float min = static_cast(ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block + 2))));\n scales[i] = ov::float16(scale);\n // zp = -min / scale (bias = min, so zp = -bias/scale)\n uint8_t zp_val = (scale != 0.0f) ? (uint8_t) std::round(-min / scale) : 0;\n // Pack two 4-bit zero points per byte\n if (i % 2 == 0) {\n zp[i / 2] = zp_val & 0x0F; // Lower nibble\n } else {\n zp[i / 2] |= (zp_val << 4); // Upper nibble\n }\n unpack_32_4(data + i * bytes_per_block + 4, weights + i * 16);\n });\n }\n}\n\n// Extracts (weight, scales, zp) from Q8_0 tensors.\n// Data layout is: |16 bit scale|32 x 8bit weights|.\nvoid extract_q8_0_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr) {\n const uint64_t weights_per_block = 32;\n const uint64_t bytes_per_block = 34; // 2 bytes scale, 32x1 byte weights\n\n auto * data = static_cast(tensor->data);\n auto * weights = static_cast(weights_arr.data());\n auto * scales = scales_arr.data::value_type>();\n auto * zp = static_cast(zp_arr.data());\n\n bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization\n\n // For Q8_0, zero point is always 128\n if (is_scalar_zp) {\n zp[0] = 128;\n }\n\n ov::parallel_for(scales_arr.get_size(), [&](size_t i) {\n uint8_t * block_data = data + i * bytes_per_block;\n scales[i] = ov::float16::from_bits(*(uint16_t *) block_data);\n // For asymmetric quantization, store per-block zero points\n if (!is_scalar_zp) {\n zp[i] = 128;\n }\n for (size_t j = 0; j < weights_per_block; ++j) {\n uint8_t x = block_data[j + 2]; // j+2 to skip the scale bytes.\n // Original data is in int8_t, so we add a bias of -128 and invert the first bit.\n x ^= 1 << 7;\n weights[i * weights_per_block + j] = x;\n }\n });\n}\n\nvoid unpack_256_4(const uint8_t * data, uint8_t * dst) {\n // Initialize the output array with zeros\n std::fill_n(dst, 128, 0);\n\n for (size_t i = 0; i < 4; ++i) {\n for (int j = 0; j < 32; ++j) {\n uint8_t x = (data[i * 32 + j] & 0x0F);\n uint8_t y = (data[i * 32 + j] >> 4);\n if (j % 2 != 0) {\n x <<= 4;\n y <<= 4;\n }\n dst[i * 32 + j / 2] |= x;\n dst[i * 32 + 16 + j / 2] |= y; // Last 16 weights are in the higher bits\n }\n }\n}\n\nvoid extract_q4_k_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n bool use_bias) {\n const uint64_t bytes_per_block = 2 + 2 + 12 + 128;\n const uint64_t n_super_block = tensor->nb[3] / bytes_per_block;\n\n auto * data = static_cast(tensor->data);\n auto * weights = static_cast(weights_arr.data());\n auto * scales = scales_arr.data::value_type>();\n\n // For bias path, zp_arr holds f16 bias values; for zp path, it holds packed u4 zero points\n auto * zp_u4 = use_bias ? nullptr : static_cast(zp_arr.data());\n auto * bias_f16 = use_bias ? zp_arr.data::value_type>() : nullptr;\n\n ov::parallel_for(n_super_block, [&](size_t i) {\n uint8_t * block_data = data + i * bytes_per_block;\n\n // Extract scale factors and offsets\n float scale_scales = static_cast(ov::float16::from_bits(*((uint16_t *) block_data)));\n float scale_mins = static_cast(ov::float16::from_bits(*((uint16_t *) block_data + 1)));\n\n // Extract qs1 and qs2\n uint8_t * qs1 = block_data + 4;\n\n // Calculate scales\n float scale_vals[8];\n scale_vals[0] = scale_scales * static_cast((*(qs1) & 0b111111));\n scale_vals[1] = scale_scales * static_cast((*(qs1 + 1) & 0b111111));\n scale_vals[2] = scale_scales * static_cast((*(qs1 + 2) & 0b111111));\n scale_vals[3] = scale_scales * static_cast((*(qs1 + 3) & 0b111111));\n scale_vals[4] = scale_scales * static_cast((*(qs1 + 8) & 0b00001111) | ((*(qs1) >> 6) << 4));\n scale_vals[5] = scale_scales * static_cast((*(qs1 + 9) & 0b00001111) | ((*(qs1 + 1) >> 6) << 4));\n scale_vals[6] = scale_scales * static_cast((*(qs1 + 10) & 0b00001111) | ((*(qs1 + 2) >> 6) << 4));\n scale_vals[7] = scale_scales * static_cast((*(qs1 + 11) & 0b00001111) | ((*(qs1 + 3) >> 6) << 4));\n\n // Calculate min values (bias = -min)\n float min_vals[8];\n min_vals[0] = scale_mins * static_cast((*(qs1 + 4) & 0b111111));\n min_vals[1] = scale_mins * static_cast((*(qs1 + 5) & 0b111111));\n min_vals[2] = scale_mins * static_cast((*(qs1 + 6) & 0b111111));\n min_vals[3] = scale_mins * static_cast((*(qs1 + 7) & 0b111111));\n min_vals[4] = scale_mins * static_cast((*(qs1 + 8) >> 4) | ((*(qs1 + 4) >> 6) << 4));\n min_vals[5] = scale_mins * static_cast((*(qs1 + 9) >> 4) | ((*(qs1 + 5) >> 6) << 4));\n min_vals[6] = scale_mins * static_cast((*(qs1 + 10) >> 4) | ((*(qs1 + 6) >> 6) << 4));\n min_vals[7] = scale_mins * static_cast((*(qs1 + 11) >> 4) | ((*(qs1 + 7) >> 6) << 4));\n\n // Store scales and compute zero points or bias\n for (int j = 0; j < 8; j++) {\n scales[i * 8 + j] = ov::float16(scale_vals[j]);\n if (use_bias) {\n // Store bias = -min directly as f16, dequant: w*s + bias\n bias_f16[i * 8 + j] = ov::float16(-min_vals[j]);\n } else {\n // zp = min / scale (since bias = -min and zp = -bias/scale)\n uint8_t zp_val = (scale_vals[j] != 0.0f) ? (uint8_t) std::round(min_vals[j] / scale_vals[j]) : 0;\n // Pack two 4-bit zero points per byte\n size_t idx = i * 8 + j;\n if (idx % 2 == 0) {\n zp_u4[idx / 2] = zp_val & 0x0F;\n } else {\n zp_u4[idx / 2] |= (zp_val << 4);\n }\n }\n }\n unpack_256_4(block_data + 16, weights + i * 128);\n });\n}\n\nvoid extract_q6_k_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr) {\n const uint64_t bytes_per_block = 128 + 64 + 16 + 2;\n const uint64_t n_super_block = tensor->nb[3] / bytes_per_block;\n\n auto * data = static_cast(tensor->data);\n auto * weights = static_cast(weights_arr.data());\n auto * scales = scales_arr.data::value_type>();\n auto * zp = static_cast(zp_arr.data());\n\n bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization\n\n // For Q6_K, zero point is always 32\n if (is_scalar_zp) {\n zp[0] = 32;\n }\n\n ov::parallel_for(n_super_block, [&](size_t i) {\n uint8_t * block_data = data + i * bytes_per_block;\n\n float scale_factor =\n static_cast(ov::float16::from_bits(*((uint16_t *) block_data + 104))); // (128+64+16)/2\n\n for (size_t j = 0; j < 16; j++) {\n scales[j + i * 16] =\n ov::float16(scale_factor * static_cast(*((int8_t *) (block_data + 128 + 64 + j))));\n // For asymmetric quantization, store per-block zero points\n if (!is_scalar_zp) {\n zp[j + i * 16] = 32;\n }\n }\n\n uint8_t * ql = block_data;\n uint8_t * qh = block_data + 128;\n\n for (int64_t j = 0; j < 32; ++j) {\n weights[i * 256 + j] = (ql[j] & 0xF) | (((qh[j] >> 0) & 3) << 4);\n weights[i * 256 + j + 32] = (ql[32 + j] & 0xF) | (((qh[j] >> 2) & 3) << 4);\n weights[i * 256 + j + 64] = (ql[j] >> 4) | (((qh[j] >> 4) & 3) << 4);\n weights[i * 256 + j + 96] = (ql[32 + j] >> 4) | (((qh[j] >> 6) & 3) << 4);\n weights[i * 256 + j + 128] = (ql[64 + j] & 0xF) | (((qh[32 + j] >> 0) & 3) << 4);\n weights[i * 256 + j + 160] = (ql[96 + j] & 0xF) | (((qh[32 + j] >> 2) & 3) << 4);\n weights[i * 256 + j + 192] = (ql[64 + j] >> 4) | (((qh[32 + j] >> 4) & 3) << 4);\n weights[i * 256 + j + 224] = (ql[96 + j] >> 4) | (((qh[32 + j] >> 6) & 3) << 4);\n }\n });\n}\n\nstatic inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) {\n if (j < 4) {\n *d = q[j] & 63;\n *m = q[j + 4] & 63;\n } else {\n *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);\n *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);\n }\n}\n\nvoid extract_q5_k_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n bool use_bias) {\n const uint64_t bytes_per_block = 4 + 12 + 32 + 128;\n const uint64_t n_super_block = tensor->nb[3] / bytes_per_block;\n\n auto * data = static_cast(tensor->data);\n auto * weights = static_cast(weights_arr.data());\n auto * scales = scales_arr.data::value_type>();\n\n // For bias path, zp_arr holds f16 bias values; for zp path, it holds u8 zero points\n auto * zp_u8 = use_bias ? nullptr : static_cast(zp_arr.data());\n auto * bias_f16 = use_bias ? zp_arr.data::value_type>() : nullptr;\n\n ov::parallel_for(n_super_block, [&](size_t i) {\n uint8_t * block_data = data + i * bytes_per_block;\n\n const float d = static_cast(ov::float16::from_bits(*((uint16_t *) block_data)));\n const float min_factor = static_cast(ov::float16::from_bits(*((uint16_t *) block_data + 1)));\n\n const uint8_t * scales_data = block_data + 4; // 12 bytes of scales\n const uint8_t * qh = block_data + 4 + 12; // 32 bytes of high bits\n const uint8_t * ql = block_data + 4 + 12 + 32; // 128 bytes of low bits\n\n int is = 0;\n uint8_t u1 = 1;\n uint8_t u2 = 2;\n\n // Process 2 blocks in one iteration\n for (int j = 0; j < 256; j += 64) { // 256 = QK_K, so 4 iterations of 64\n uint8_t sc;\n uint8_t m;\n\n // Get scale and min for first 32 elements\n get_scale_min_k4(is + 0, scales_data, &sc, &m);\n const float d1 = d * sc;\n const float m1 = min_factor * m;\n\n // Get scale and min for second 32 elements\n get_scale_min_k4(is + 1, scales_data, &sc, &m);\n const float d2 = d * sc;\n const float m2 = min_factor * m;\n\n scales[i * 8 + is] = ov::float16(d1);\n scales[i * 8 + is + 1] = ov::float16(d2);\n if (use_bias) {\n // Store bias = -min directly as f16, dequant: w*s + bias\n bias_f16[i * 8 + is] = ov::float16(-m1);\n bias_f16[i * 8 + is + 1] = ov::float16(-m2);\n } else {\n // zp = min / scale (since bias = -min and zp = -bias/scale)\n zp_u8[i * 8 + is] = (d1 != 0.0f) ? (uint8_t) std::round(m1 / d1) : 0;\n zp_u8[i * 8 + is + 1] = (d2 != 0.0f) ? (uint8_t) std::round(m2 / d2) : 0;\n }\n\n // Extract weights for first 32 elements (matching deq formula exactly)\n for (int l = 0; l < 32; ++l) {\n weights[i * 256 + j + l] = (ql[l] & 0xF) + ((qh[l] & u1) ? 16 : 0);\n }\n\n // Extract weights for second 32 elements\n for (int l = 0; l < 32; ++l) {\n weights[i * 256 + j + l + 32] = (ql[l] >> 4) + ((qh[l] & u2) ? 16 : 0);\n }\n\n ql += 32;\n is += 2;\n u1 <<= 2;\n u2 <<= 2;\n }\n });\n}\n\n// TODO Reorder for make_intX_weights\n\nov::Output make_int8_weights(ov::Tensor & weight,\n ov::Tensor & scales,\n ov::Tensor & zp,\n size_t group_size,\n bool use_bias) {\n ov::Shape orig_shape = weight.get_shape();\n\n // Expand dimensions for scales and zp/bias\n auto scale_shape = scales.get_shape();\n auto zp_shape = zp.get_shape();\n bool is_scalar_zp = zp_shape.empty(); // Symmetric quantization\n\n ov::Shape packed_shape = {orig_shape[0], orig_shape[1] / group_size, group_size};\n\n if (packed_shape[1] == 1) {\n // Requantized channel-wise case\n packed_shape.erase(packed_shape.begin() + 1);\n } else {\n scale_shape.push_back(1);\n scales.set_shape(scale_shape);\n // For symmetric quantization, zp remains scalar (don't resize)\n if (!is_scalar_zp) {\n zp_shape.push_back(1);\n zp.set_shape(zp_shape);\n }\n }\n\n // Create graph nodes\n auto weights_node = std::make_shared(ov::element::u8, packed_shape,\n static_cast(weight.data()), nullptr);\n weights_node->get_rt_info()[\"__gguf_tensor_holder\"] = weight;\n auto scales_f16 = std::make_shared(scales);\n auto weights_f16 = std::make_shared(weights_node, ov::element::f16);\n\n ov::Output result;\n if (use_bias && !is_scalar_zp) {\n // Bias path: w * s + b (zp tensor holds f16 bias values)\n auto bias_f16 = std::make_shared(zp);\n auto w_s = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);\n result = std::make_shared(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY);\n } else {\n // Zero point path: (w - zp) * s\n auto zero_point = std::make_shared(zp);\n float zp_value;\n if (ov::op::util::get_single_value(zero_point, zp_value)) {\n zero_point = ov::op::v0::Constant::create(zero_point->get_element_type(), {}, {zp_value});\n }\n auto zero_point_f16 = std::make_shared(zero_point, ov::element::f16);\n auto w_zp =\n std::make_shared(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY);\n result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);\n }\n\n if (packed_shape.size() != 2) {\n // If not requantized channel-wise case, reshape back to original shape\n auto final_shape =\n std::make_shared(ov::element::i64, ov::Shape{orig_shape.size()}, orig_shape);\n result = std::make_shared(result, final_shape, false);\n }\n\n return std::make_shared(result, ov::element::f32);\n}\n\nov::Output make_int4_weights(ov::Tensor & weight,\n ov::Tensor & scales,\n ov::Tensor & zp,\n size_t group_size,\n bool use_bias) {\n ov::Shape orig_weight_shape = weight.get_shape();\n\n // Expand dimensions for scales and zp/bias\n ov::Shape scale_shape = scales.get_shape();\n auto zp_shape = zp.get_shape();\n bool is_scalar_zp = zp_shape.empty(); // Symmetric quantization\n\n // Create INT4 weight tensor\n ov::Shape packed_shape = {orig_weight_shape[0], orig_weight_shape[1] / group_size, group_size};\n\n if (packed_shape[1] == 1) {\n // Requantized channel-wise case\n packed_shape.erase(packed_shape.begin() + 1);\n } else {\n scale_shape.push_back(1);\n scales.set_shape(scale_shape);\n // For symmetric quantization, zp remains scalar (don't resize)\n if (!is_scalar_zp) {\n zp_shape.push_back(1);\n zp.set_shape(zp_shape);\n }\n }\n\n auto weights_node = std::make_shared(ov::element::u4, packed_shape,\n static_cast(weight.data()), nullptr);\n weights_node->get_rt_info()[\"__gguf_tensor_holder\"] = weight;\n auto weights_f16 = std::make_shared(weights_node, ov::element::f16);\n auto scales_f16 = std::make_shared(scales);\n\n ov::Output result;\n if (use_bias && !is_scalar_zp) {\n // Bias path: w * s + b (zp tensor holds f16 bias values)\n auto bias_f16 = std::make_shared(zp);\n auto w_s = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY);\n result = std::make_shared(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY);\n } else {\n // Zero point path: (w - zp) * s\n auto zero_points_node = std::make_shared(zp);\n float zp_value;\n if (ov::op::util::get_single_value(zero_points_node, zp_value)) {\n zero_points_node = ov::op::v0::Constant::create(zero_points_node->get_element_type(), {}, {zp_value});\n }\n auto zero_points_f16 = std::make_shared(zero_points_node, ov::element::f16);\n auto w_zp =\n std::make_shared(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY);\n result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY);\n }\n\n if (packed_shape.size() != 2) {\n // If not requantized channel-wise case, reshape back to original shape\n auto final_shape = std::make_shared(ov::element::i64, ov::Shape{orig_weight_shape.size()},\n orig_weight_shape);\n result = std::make_shared(result, final_shape, false);\n }\n\n return std::make_shared(result, ov::element::f32);\n}\n\n// Extract quantized weights from tensor and create weight subgraph\nstd::shared_ptr extract_quantized_weights(const ggml_tensor * tensor,\n const void * data,\n ov::Tensor & weights,\n ov::Tensor & scales,\n ov::Tensor & zp,\n bool use_bias) {\n // Create a temporary tensor for extraction functions that read from tensor->data\n ggml_tensor temp_tensor = *tensor;\n temp_tensor.data = const_cast(data);\n\n // Determine block size based on tensor type\n int64_t weights_per_block;\n bool is_u4;\n switch (tensor->type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q4_K:\n is_u4 = true;\n weights_per_block = 32;\n break;\n case GGML_TYPE_Q8_0:\n case GGML_TYPE_Q5_K:\n is_u4 = false;\n weights_per_block = 32;\n break;\n case GGML_TYPE_Q6_K:\n is_u4 = false;\n weights_per_block = 16;\n break;\n default:\n throw std::runtime_error(\"Unsupported quantized type for extraction: \" +\n std::string(ggml_type_name(tensor->type)));\n }\n\n // Extract quantized data\n switch (tensor->type) {\n case GGML_TYPE_Q4_0:\n extract_q4_0_data(&temp_tensor, weights, scales, zp);\n break;\n case GGML_TYPE_Q4_1:\n extract_q4_1_data(&temp_tensor, weights, scales, zp, use_bias);\n break;\n case GGML_TYPE_Q4_K:\n extract_q4_k_data(&temp_tensor, weights, scales, zp, use_bias);\n break;\n case GGML_TYPE_Q8_0:\n extract_q8_0_data(&temp_tensor, weights, scales, zp);\n break;\n case GGML_TYPE_Q6_K:\n extract_q6_k_data(&temp_tensor, weights, scales, zp);\n break;\n case GGML_TYPE_Q5_K:\n extract_q5_k_data(&temp_tensor, weights, scales, zp, use_bias);\n break;\n default:\n throw std::runtime_error(\"Unsupported quantized type: \" + std::string(ggml_type_name(tensor->type)));\n }\n\n // Create the OpenVINO weight subgraph\n ov::Output weight_node;\n if (is_u4) {\n weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias);\n } else {\n weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias);\n }\n\n auto result = weight_node.get_node_shared_ptr();\n result->set_friendly_name(tensor->name);\n return result;\n}\n\n// Requantize weights to target format, writing to provided buffers\nstd::shared_ptr requantize_to_buffers(const ggml_tensor * tensor,\n const void * data,\n ExtraQuantType requant_type,\n int64_t block_size,\n ov::Tensor & weights,\n ov::Tensor & scales,\n ov::Tensor & zp) {\n int64_t n_elements = ggml_nelements(tensor);\n\n // First dequantize to F32\n std::vector weights_f32(n_elements);\n ggml_get_type_traits(tensor->type)->to_float(data, weights_f32.data(), n_elements);\n\n // Handle F16 case - just convert and create constant\n if (requant_type == ExtraQuantType::F16) {\n ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements);\n auto result = std::make_shared(weights);\n result->set_friendly_name(tensor->name);\n return result;\n }\n\n // Requantize to target quantized format\n bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128);\n\n if (is_u4) {\n quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);\n } else if (requant_type == ExtraQuantType::Q8_1_C) {\n quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size);\n } else {\n quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);\n }\n\n // Create the OpenVINO weight subgraph\n ov::Output weight_node;\n if (is_u4) {\n weight_node = make_int4_weights(weights, scales, zp, block_size);\n } else {\n weight_node = make_int8_weights(weights, scales, zp, block_size);\n }\n\n auto result = weight_node.get_node_shared_ptr();\n result->set_friendly_name(tensor->name);\n return result;\n}\n\nOvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias) {\n GGML_ASSERT(tensor != nullptr);\n GGML_ASSERT(data != nullptr);\n\n OvWeight result;\n\n // Get 2D shape for weights [rows, cols]\n ov::Shape node_shape = {static_cast(tensor->ne[1]), static_cast(tensor->ne[0])};\n\n // Handle F16/F32/BF16 weights\n if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {\n ov::element::Type element_type;\n switch (tensor->type) {\n case GGML_TYPE_F32:\n element_type = ov::element::f32;\n break;\n case GGML_TYPE_F16:\n element_type = ov::element::f16;\n break;\n case GGML_TYPE_BF16:\n element_type = ov::element::bf16;\n break;\n default:\n OPENVINO_THROW(\"Unexpected tensor type in F16/F32/BF16 path\");\n }\n\n if (output_base_ptr && output_base_ptr != data) {\n // Using external buffer - copy data and create shared-memory constant\n size_t tensor_bytes = ggml_nbytes(tensor);\n memcpy(output_base_ptr, data, tensor_bytes);\n result.weights = ov::Tensor(element_type, node_shape, output_base_ptr);\n } else {\n result.weights = ov::Tensor(element_type, node_shape, data);\n }\n result.weight_node = std::make_shared(result.weights);\n return result;\n }\n\n // Handle quantized weights\n if (!ggml_is_quantized(tensor->type)) {\n OPENVINO_THROW(\"Unsupported weight tensor type: \", ggml_type_name(tensor->type));\n }\n\n result.layout = ggml_openvino_get_extracted_layout(tensor, use_bias);\n const auto & layout = result.layout;\n if (layout.total_size == 0) {\n OPENVINO_THROW(\"Unsupported quantized type: \", ggml_type_name(tensor->type));\n }\n\n if (use_bias) {\n OPENVINO_ASSERT(!layout.is_requant,\n \"use_bias is only used for test-backend-ops, which should not have requantization\");\n // bias node will be created on the fly and not use backend buffer\n output_base_ptr = nullptr;\n }\n\n // F16 requant path - no separate scales/zp needed in result\n if (layout.is_requant && layout.requant_type.has_value() && layout.requant_type.value() == ExtraQuantType::F16) {\n if (output_base_ptr) {\n result.weights = ov::Tensor(ov::element::f16, node_shape,\n static_cast(output_base_ptr) + layout.weights_offset);\n } else {\n result.weights = ov::Tensor(ov::element::f16, node_shape);\n }\n ov::Tensor dummy_scales, dummy_zp; // Not used for F16\n result.weight_node =\n requantize_to_buffers(tensor, data, ExtraQuantType::F16, 0, result.weights, dummy_scales, dummy_zp);\n return result;\n }\n\n // Quantized path (normal extraction or quantized requant)\n // Create weight/scale/zp tensors - shared between both paths\n ov::element::Type weight_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;\n ov::Shape scale_shape = {node_shape[0], node_shape[1] / layout.weights_per_block};\n ov::Shape zp_shape = layout.is_symmetric ? ov::Shape{} : scale_shape;\n\n if (output_base_ptr) {\n uint8_t * buf_base = static_cast(output_base_ptr);\n result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset);\n result.scales = ov::Tensor(ov::element::f16, scale_shape, buf_base + layout.scales_offset);\n result.zp = ov::Tensor(weight_type, zp_shape, buf_base + layout.zp_offset);\n } else {\n result.weights = ov::Tensor(weight_type, node_shape);\n result.scales = ov::Tensor(ov::element::f16, scale_shape);\n if (use_bias && !layout.is_symmetric) {\n // bias only has effect for asymmetric quant\n result.zp = ov::Tensor(ov::element::f16, zp_shape);\n } else {\n result.zp = ov::Tensor(weight_type, zp_shape);\n }\n }\n\n if (layout.is_requant && layout.requant_type.has_value()) {\n result.weight_node = requantize_to_buffers(tensor, data, layout.requant_type.value(), layout.weights_per_block,\n result.weights, result.scales, result.zp);\n } else {\n result.weight_node =\n extract_quantized_weights(tensor, data, result.weights, result.scales, result.zp, use_bias);\n }\n\n return result;\n}\n\nvoid quantize_q4_0(const float * x,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n int64_t k,\n int64_t qk) {\n assert(k % qk == 0);\n const int nb = k / qk;\n\n auto * weights = static_cast(weights_arr.data());\n auto * scales = scales_arr.data::value_type>();\n auto * zp = static_cast(zp_arr.data());\n bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization\n\n // For Q4_0, zero point is always 8\n if (is_scalar_zp) {\n zp[0] = 8 | (8 << 4); // Pack two 4-bit values\n }\n\n for (int i = 0; i < nb; i++) {\n float amax = 0.0f; // absolute max\n float max = 0.0f;\n\n for (int j = 0; j < qk; j++) {\n const float v = x[i * qk + j];\n if (amax < fabsf(v)) {\n amax = fabsf(v);\n max = v;\n }\n }\n\n const float d = max / -8;\n\n if (d == 0) {\n scales[i] = ov::float16(1.0f);\n // zp is already set to 8 for symmetric, or set per-block for asymmetric\n if (!is_scalar_zp) {\n if (i % 2 == 0) {\n zp[i / 2] = 8;\n } else {\n zp[i / 2] |= (8 << 4);\n }\n }\n memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2);\n continue;\n }\n\n const float id = 1.0f / d;\n scales[i] = ov::float16(d);\n // For asymmetric quantization, store per-block zero points\n if (!is_scalar_zp) {\n if (i % 2 == 0) {\n zp[i / 2] = 8;\n } else {\n zp[i / 2] |= (8 << 4);\n }\n }\n\n for (int j = 0; j < qk / 2; ++j) {\n const float x0 = x[i * qk + 2 * j] * id;\n const float x1 = x[i * qk + 2 * j + 1] * id;\n const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f));\n const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f));\n weights[i * qk / 2 + j] = xi0 | (xi1 << 4);\n }\n }\n}\n\nvoid quantize_q8_0(const float * x,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n int64_t k,\n int64_t qk) {\n assert(k % qk == 0);\n const int nb = k / qk;\n\n auto * weights = static_cast(weights_arr.data());\n auto * scales = scales_arr.data::value_type>();\n auto * zp = static_cast(zp_arr.data());\n bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization\n\n // For Q8_0, zero point is always 128\n if (is_scalar_zp) {\n zp[0] = 128;\n }\n\n for (int i = 0; i < nb; i++) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < qk; j++) {\n const float v = x[i * qk + j];\n if (amax < fabsf(v)) {\n amax = fabsf(v);\n }\n }\n\n const float d = amax / 127.0f;\n const float id = d ? 1.0f / d : 0.0f;\n scales[i] = ov::float16(d);\n // For asymmetric quantization, store per-block zero points\n if (!is_scalar_zp) {\n zp[i] = 128;\n }\n\n for (int j = 0; j < qk; ++j) {\n const float x0 = x[i * qk + j] * id;\n const int8_t xi0 = roundf(x0);\n weights[i * qk + j] = (uint8_t) (xi0 + 128);\n }\n }\n}\n\nvoid quantize_q8_1(const float * x,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n int64_t k,\n int64_t qk) {\n assert(k % qk == 0);\n const int nb = k / qk;\n\n auto * weights = static_cast(weights_arr.data());\n auto * scales = scales_arr.data::value_type>();\n auto * zp = static_cast(zp_arr.data());\n for (int i = 0; i < nb; i++) {\n float min = std::numeric_limits::max();\n float max = std::numeric_limits::lowest();\n\n for (int j = 0; j < qk; j++) {\n const float v = x[i * qk + j];\n if (v < min) {\n min = v;\n }\n if (v > max) {\n max = v;\n }\n }\n\n const float d = (max - min) / ((1 << 8) - 1);\n const float id = d ? 1.0f / d : 0.0f;\n scales[i] = ov::float16(d);\n // zp = -min / scale (Q8_1 is asymmetric)\n zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0;\n\n for (int j = 0; j < qk; ++j) {\n const float x0 = (x[i * qk + j] - min) * id;\n const uint8_t xi0 = roundf(x0);\n weights[i * qk + j] = xi0;\n }\n }\n}\n" }, { "path": "src/ggml-openvino/ggml-quants.h", "content": "#pragma once\n#include \"ggml-openvino-extra.h\" // For ExtraQuantType\n#include \"ggml.h\"\n\n#include \n#include \n#include \n\nvoid unpack_32_4(const uint8_t* data, uint8_t* dst);\n\nvoid extract_q4_0_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr);\n\nvoid extract_q4_1_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n bool use_bias = false);\n\nvoid extract_q8_0_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr);\n\nvoid unpack_256_4(const uint8_t* data, uint8_t* dst);\n\nvoid extract_q4_k_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n bool use_bias = false);\n\nvoid extract_q5_k_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n bool use_bias = false);\n\nvoid extract_q6_k_data(const ggml_tensor * tensor,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr);\n\nstatic constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32;\n\nov::Output make_int8_weights(ov::Tensor & weight,\n ov::Tensor & scales,\n ov::Tensor & zp,\n size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,\n bool use_bias = false);\n\nov::Output make_int4_weights(ov::Tensor & weight,\n ov::Tensor & scales,\n ov::Tensor & zp,\n size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,\n bool use_bias = false);\n\n// Extract quantized weights from tensor and create weight subgraph\n// If weights/scales/zp are provided (non-empty), uses them as output buffers\n// Otherwise allocates new ov::Tensors internally\n// Returns the weight node (make_int4_weights or make_int8_weights result)\nstd::shared_ptr extract_quantized_weights(\n const ggml_tensor * tensor,\n const void * data, // Source data pointer (may differ from tensor->data)\n ov::Tensor & weights,\n ov::Tensor & scales,\n ov::Tensor & zp,\n bool use_bias = false); // Use fp bias instead of quantized zero_point (for test-backend-ops)\n\n// Requantize weights from tensor to target format, writing to provided buffers\n// For F16 target, only weights buffer is used (scales/zp ignored)\n// Returns the weight node\nstd::shared_ptr requantize_to_buffers(const ggml_tensor * tensor,\n const void * data, // Source data pointer\n ExtraQuantType requant_type,\n int64_t block_size,\n ov::Tensor & weights,\n ov::Tensor & scales,\n ov::Tensor & zp);\n\ninline const char * extra_quant_type_name(ExtraQuantType t) {\n switch (t) {\n case ExtraQuantType::F16:\n return \"F16\";\n case ExtraQuantType::Q4_0_C:\n return \"Q4_0_C\";\n case ExtraQuantType::Q4_0_128:\n return \"Q4_0_128\";\n case ExtraQuantType::Q8_0_C:\n return \"Q8_0_C\";\n case ExtraQuantType::Q8_0_32:\n return \"Q8_0_32\";\n case ExtraQuantType::Q8_1_C:\n return \"Q8_1_C\";\n default:\n return \"unknown\";\n }\n}\n\n// Result from process_weight_tensor containing the weight node and tensors.\n// For quantized weights, also contains the extracted layout and scale/zp tensors.\nstruct OvWeight {\n std::shared_ptr weight_node;\n ggml_openvino_extracted_layout layout; // Only meaningful for quantized (layout.total_size > 0)\n ov::Tensor weights;\n ov::Tensor scales;\n ov::Tensor zp;\n\n bool is_quantized() const { return layout.scales_size > 0; }\n};\n\n// Process weight tensor and create an OpenVINO weight node\n// Handles F16/F32/BF16 and quantized weights, with optional requantization\n// If output_base_ptr is nullptr, allocates internal buffers (for decoder use)\n// If output_base_ptr is provided, uses pre-allocated buffers at specified offsets (for backend buffer use)\n// Returns OvWeight with the weight node and optional quantized tensors\nOvWeight process_weight_tensor(\n const ggml_tensor * tensor,\n const void * data, // Source data pointer (may differ from tensor->data)\n void * output_base_ptr = nullptr, // Base pointer for output buffers (or nullptr for internal allocation)\n bool use_bias = false); // Use fp bias instead of quantized zero_point, only used in test-backend-ops\n\nvoid quantize_q4_0(const float * x,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n int64_t k,\n int64_t qk);\nvoid quantize_q8_1(const float * x,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n int64_t k,\n int64_t qk);\nvoid quantize_q8_0(const float * x,\n ov::Tensor & weights_arr,\n ov::Tensor & scales_arr,\n ov::Tensor & zp_arr,\n int64_t k,\n int64_t qk);\n\nnamespace ov {\nnamespace op {\nnamespace util {\n// From /src/common/transformations/include/transformations/utils/utils.hpp\nbool get_single_value(const std::shared_ptr& const_node,\n float& value,\n bool check_value_range = true);\n} // namespace util\n} // namespace op\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/decoder.h", "content": "#pragma once\n\n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nclass GgmlDecoder : public DecoderBase {\npublic:\n virtual ov::Any get_attribute(const std::string& name) const = 0;\n\n virtual PartialShape get_input_shape(int node_idx, const std::string& name) const = 0;\n\n virtual std::vector get_input_stride(int node_idx, const std::string& name) const = 0;\n\n virtual element::Type get_input_type(int node_idx, const std::string& name) const = 0;\n\n virtual size_t get_input_size() const = 0;\n\n virtual size_t get_input_size(int node_idx) const = 0;\n\n virtual void get_input_node(size_t input_port_idx,\n std::string& producer_name,\n std::string& producer_output_port_name,\n size_t& producer_output_port_index) const = 0;\n\n virtual std::vector get_input_names(int node_idx) const = 0;\n\n virtual PartialShape get_output_shape(int node_idx) const = 0;\n\n virtual element::Type get_output_type(const int node_idx) const = 0;\n\n virtual int32_t* get_input_op_params(int node_idx, const std::string& name) const = 0;\n\n virtual int32_t * get_output_op_params(int node_idx) const = 0;\n\n virtual std::vector get_output_names(int node_idx) const = 0;\n\n virtual const std::string& get_op_type() const = 0;\n\n virtual const std::string& get_op_type(int node_idx) const = 0;\n\n virtual const std::string& get_op_name() const = 0;\n\n virtual const std::string& get_op_name(int node_idx) const = 0;\n\n virtual void visit_subgraph(std::function, int node_idx)> node_visitor) const = 0;\n\n virtual int get_op_case(int node_idx) const = 0;\n\n virtual const std::map>& get_model_inputs() const = 0;\n virtual const std::map>& get_model_extra_inputs() const = 0;\n virtual const std::map>& get_model_weights() const = 0;\n virtual std::vector get_model_output_names() const = 0;\n\n virtual int32_t* get_rope_params() const = 0;\n\n virtual std::map get_kv_param_res_names() const = 0;\n\n virtual bool is_static() const = 0;\n\n virtual bool is_stateful() const = 0;\n\n virtual int is_swa_layer(int layer) const = 0;\n};\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/frontend.cpp", "content": "#include \"frontend.h\"\n\n#include \"input_model.h\"\n#include \"op_table.h\"\n#include \"translate_session.h\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nFrontEnd::FrontEnd() {}\n\nstd::shared_ptr FrontEnd::convert(const InputModel::Ptr & model, bool naive) {\n auto ggml_model = std::dynamic_pointer_cast(model);\n FRONT_END_GENERAL_CHECK(ggml_model, \"Invalid input model\");\n std::shared_ptr converted_model;\n const auto & supported_ops = get_supported_ops();\n {\n TranslateSession translate_session(model, supported_ops, naive);\n converted_model = translate_session.get_converted_model();\n }\n return converted_model;\n}\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/frontend.h", "content": "// Copyright (C) 2018-2024 Intel Corporation\n// SPDX-License-Identifier: Apache-2.0\n//\n\n#pragma once\n\n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nclass FrontEnd {\npublic:\n using Ptr = std::shared_ptr;\n FrontEnd();\n\n static std::shared_ptr convert(const InputModel::Ptr& model, bool naive = false);\n};\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/input_model.cpp", "content": "#include \"input_model.h\"\n\n#include \"decoder.h\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nInputModel::InputModel(const std::shared_ptr & gdecoder) : m_decoder(gdecoder) {}\n\nconst std::shared_ptr & InputModel::get_model_decoder() const {\n return m_decoder;\n}\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/input_model.h", "content": "#pragma once\n\n#include \n\n#include \"decoder.h\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nclass FrontEnd;\nclass GgmlDecoder;\nusing ov::frontend::ggml::GgmlDecoder;\n\nclass InputModel : public ov::frontend::InputModel {\n friend class ::ov::frontend::ggml::FrontEnd;\n\npublic:\n explicit InputModel(const std::shared_ptr& gdecoder);\n\n const std::shared_ptr& get_model_decoder() const;\n\nprivate:\n std::shared_ptr m_decoder;\n};\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/node_context.h", "content": "#pragma once\n\n#include \n#include \n#include \n\n#include \"decoder.h\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nclass TranslateSession;\n\ntypedef std::map> TensorMap;\n\nclass NodeContext : public frontend::NodeContext {\npublic:\n NodeContext(const std::shared_ptr& decoder,\n std::shared_ptr& tensor_map,\n int node_idx,\n TranslateSession* translate_session = nullptr)\n : ov::frontend::NodeContext(decoder->get_op_type(node_idx)),\n m_decoder(decoder),\n m_tensor_map(tensor_map),\n m_node_idx(node_idx),\n m_translate_session(translate_session) {\n m_input_names = decoder->get_input_names(m_node_idx);\n m_output_names = decoder->get_output_names(m_node_idx);\n }\n\n TranslateSession* get_translate_session() const {\n return m_translate_session;\n }\n\n const std::vector& get_input_names() const { return m_input_names; }\n\n size_t get_input_size() const override {\n return m_decoder->get_input_size(m_node_idx);\n }\n\n ov::element::Type get_input_type(size_t index) const {\n return m_decoder->get_input_type(m_node_idx, m_input_names[index]);\n }\n\n PartialShape get_input_shape(size_t input_index) const {\n return m_decoder->get_input_shape(m_node_idx, m_input_names[input_index]);\n }\n\n std::vector get_input_stride(size_t index) const {\n return m_decoder->get_input_stride(m_node_idx, m_input_names[index]);\n }\n\n std::string get_output_name() const { return m_output_names[0]; }\n\n PartialShape get_output_shape() const { return m_decoder->get_output_shape(m_node_idx); }\n\n int32_t* get_input_op_params(size_t index) const {\n return m_decoder->get_input_op_params(m_node_idx, m_input_names[index]);\n }\n\n int32_t * get_output_op_params() const { return m_decoder->get_output_op_params(m_node_idx); }\n\n ov::element::Type get_output_type() const {\n return m_decoder->get_output_type(m_node_idx);\n }\n\n Output get_input(int idx) const override {\n return m_tensor_map->at(m_input_names[idx]);\n }\n\n Output get_input(const std::string& name) const override {\n if (m_tensor_map->find(name) == m_tensor_map->end()) {\n throw std::runtime_error(\"'\" + name + \"' not found in tensor map.\");\n }\n return m_tensor_map->at(name);\n }\n\n bool has_input(const std::string& name) const {\n return m_tensor_map->find(name) != m_tensor_map->end();\n }\n\n const std::string& get_name() const override {\n return m_decoder->get_op_name(m_node_idx);\n }\n\n ov::Any get_attribute_as_any(const std::string& name) const override {\n return m_decoder->get_attribute(name);\n }\n\n int get_op_case() const {\n return m_decoder->get_op_case(m_node_idx);\n }\n\n bool is_static() const { return m_decoder->is_static(); }\n\n bool is_stateful() const { return m_decoder->is_stateful(); }\n\nprivate:\n std::shared_ptr m_decoder;\n std::shared_ptr& m_tensor_map;\n int m_node_idx;\n TranslateSession* m_translate_session;\n std::vector m_input_names;\n std::vector m_output_names;\n};\n\nusing CreatorFunction = std::function;\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/cont.cpp", "content": "\n#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_cont(const NodeContext & context) {\n num_inputs_check(context, 1, 1);\n\n int op_case = context.get_op_case();\n FRONT_END_CHECK_IMPLEMENTED(op_case == 1 || op_case == 2 || op_case == 3, \"Unsupported CONT case\");\n\n auto src_shape = context.get_input_shape(0).to_shape();\n auto dst_shape = context.get_output_shape().to_shape();\n ov::Output res;\n\n if (op_case == 1) {\n // The input comes from a PERMUTE\n throw std::runtime_error(\"Code of this case might be outdated\");\n dst_shape[1] = -1;\n res = std::make_shared(\n context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {dst_shape.size()}, dst_shape), false);\n } else if (op_case == 2) {\n // The input comes from a TRANSPOSE\n return {context.get_input(0)};\n } else {\n // The input comes from a VIEW\n res = process_view_input(context, 0);\n }\n\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/cpy.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_cpy(const NodeContext & context) {\n auto res = std::make_shared(context.get_input(0), context.get_output_type());\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/flash_attn_ext.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_flash_attn_ext(const NodeContext & context) {\n num_inputs_check(context, 4, 4);\n auto q_f32 = context.get_input(0);\n auto k = context.get_input(1);\n auto v = context.get_input(2);\n auto mask = context.get_input(3);\n\n float * params = reinterpret_cast(context.get_output_op_params());\n float scale = params[0];\n // float max_bias = params[1];\n // float logit_softcap = params[2];\n\n auto q = std::make_shared(q_f32, ov::element::f16);\n auto scale_node = std::make_shared(ov::element::f16, ov::Shape{}, std::vector{scale});\n\n ov::Output mask_sliced, res;\n std::string mask_name = \"KQ_mask_sliced\";\n if (context.get_input_names()[3].find(\"swa\") != std::string::npos) {\n mask_name = \"KQ_mask_swa_sliced\";\n }\n if (context.has_input(mask_name)) {\n mask_sliced = context.get_input(mask_name);\n } else {\n auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});\n auto token_len = get_dimensions(q, {2});\n mask_sliced = std::make_shared(mask, zero, token_len, one, two);\n }\n\n if (mask_sliced.get_element_type() != ov::element::f16) {\n mask_sliced = std::make_shared(mask_sliced, ov::element::f16);\n }\n\n auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output kv) {\n int64_t factor = num_heads / num_heads_kv;\n if (factor > 1 && num_heads_kv > 1) {\n ov::Output kv_broadcast_shape, kv_unsqueezed, new_kv_shape;\n auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2});\n kv_unsqueezed = std::make_shared(kv, unsqueeze_axes);\n\n kv_broadcast_shape = ov::op::v0::Constant::create(\n ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1});\n new_kv_shape =\n ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, num_heads, (int64_t) -1, head_size});\n\n kv = std::make_shared(kv_unsqueezed, kv_broadcast_shape,\n ov::op::BroadcastType::BIDIRECTIONAL);\n kv = std::make_shared(kv, new_kv_shape, true);\n }\n return kv;\n };\n\n auto q_shape = context.get_input_shape(0).to_shape();\n auto k_shape = context.get_input_shape(1).to_shape();\n k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k);\n v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v);\n\n auto sdpa = std::make_shared(q, k, v, mask_sliced, scale_node, false);\n res = std::make_shared(sdpa,\n ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));\n res = std::make_shared(res, ov::element::f32);\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/get_rows.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_get_rows(const NodeContext & context) {\n num_inputs_check(context, 2, 2);\n\n int op_case = context.get_op_case();\n\n Output res;\n auto data = context.get_input(0);\n auto indices = context.get_input(1);\n\n if (op_case == 2) {\n // The input comes from a VIEW\n indices = process_view_input(context, 1);\n }\n\n // data[1,b,x,y] ind[1,1,b,x'] test-backend-ops case\n // data[x,y] ind[1,1,1,x'] normal case\n indices =\n std::make_shared(indices, ov::op::v0::Constant::create(ov::element::i64, {2}, {0, 1}));\n if (data.get_partial_shape().rank() == 4) {\n if (!(data.get_partial_shape()[1].is_dynamic()) && data.get_partial_shape()[1].get_length() == 1) {\n // Work-around for a bug in ov cpu plugin for test-backend-ops\n data = std::make_shared(data,\n ov::op::v0::Constant::create(ov::element::i64, {2}, {0, 1}));\n auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});\n res = std::make_shared(data, indices, axis);\n } else {\n auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1});\n data =\n std::make_shared(data, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));\n res = std::make_shared(data, indices, axis, 1);\n }\n } else if (context.is_stateful() && data.get_partial_shape().rank() == 3) {\n auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1});\n res = std::make_shared(data, indices, axis, 1);\n } else {\n auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});\n res = std::make_shared(data, indices, axis);\n }\n\n if (res.get_element_type() != context.get_output_type()) {\n res = std::make_shared(res, context.get_output_type());\n }\n if (!(context.is_stateful())) {\n res = std::make_shared(res, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));\n }\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/glu_geglu.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_glu_geglu(const NodeContext & context) {\n num_inputs_check(context, 1, 2);\n\n ov::Output src0;\n ov::Output src1;\n if (context.get_input_size() == 2) {\n src0 = context.get_input(0);\n src1 = context.get_input(1);\n } else {\n // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2.\n // Both halves are nc elements; if the dimension is odd, the last element is dropped.\n // Use Slice instead of Split to handle odd dimensions correctly.\n auto combined = context.get_input(0);\n auto combined_shape = combined.get_partial_shape();\n int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length();\n int64_t nc = last_dim_val / 2;\n\n auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});\n auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});\n auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});\n auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});\n\n src0 = std::make_shared(combined, start0, stop0, step, axis);\n src1 = std::make_shared(combined, start1, stop1, step, axis);\n }\n\n int32_t * params = context.get_output_op_params();\n const int32_t swapped = params[1];\n if (swapped) {\n std::swap(src0, src1);\n }\n\n auto gelu = std::make_shared(src0);\n auto res = std::make_shared(gelu, src1);\n\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/glu_swiglu.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_glu_swiglu(const NodeContext & context) {\n num_inputs_check(context, 1, 2);\n\n ov::Output src0;\n ov::Output src1;\n if (context.get_input_size() == 2) {\n src0 = context.get_input(0);\n src1 = context.get_input(1);\n } else {\n // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2.\n // Both halves are nc elements; if the dimension is odd, the last element is dropped.\n // Use Slice instead of Split to handle odd dimensions correctly.\n auto combined = context.get_input(0);\n auto combined_shape = combined.get_partial_shape();\n int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length();\n int64_t nc = last_dim_val / 2;\n\n auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});\n auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});\n auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});\n auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});\n\n src0 = std::make_shared(combined, start0, stop0, step, axis);\n src1 = std::make_shared(combined, start1, stop1, step, axis);\n }\n\n int32_t * params = context.get_output_op_params();\n const int32_t swapped = params[1];\n if (swapped) {\n std::swap(src0, src1);\n }\n\n auto sigmoid = std::make_shared(src0);\n auto silu = std::make_shared(src0, sigmoid);\n auto res = std::make_shared(silu, src1);\n\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/mulmat.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_mulmat(const NodeContext & context) {\n num_inputs_check(context, 2, 2);\n\n int op_case = context.get_op_case();\n\n ov::Output res;\n ov::Output B = context.get_input(0);\n ov::Output A = context.get_input(1);\n\n bool transpose_b = true;\n if (op_case == 2) {\n B = B.get_node_shared_ptr()->input_value(0);\n transpose_b = false;\n } else if (op_case == 3) {\n B = process_view_input(context, 0);\n A = process_view_input(context, 1);\n }\n if (A.get_element_type() != B.get_element_type()) {\n B = std::make_shared(context.get_input(0), context.get_input_type(1));\n }\n\n auto B_shape = context.get_input_shape(0).to_shape();\n auto A_shape = context.get_input_shape(1).to_shape();\n int64_t A_batch = A_shape[1];\n int64_t B_batch = B_shape[1];\n\n auto A_batch_larger = A_batch > B_batch;\n auto batch_large = A_batch_larger ? A_batch : B_batch;\n auto batch_small = A_batch_larger ? B_batch : A_batch;\n\n Output Z = A_batch_larger ? B : A;\n int64_t factor = batch_large / batch_small;\n if (factor > 1 && batch_small > 1) {\n auto batch_large_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{batch_large});\n auto batch_small_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{batch_small});\n auto factor_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{factor});\n\n auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2});\n auto Z_unsqueezed = std::make_shared(Z, unsqueeze_axes);\n\n auto broadcast_shape = ov::op::v0::Constant::create(\n ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1});\n auto new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4},\n {(int64_t) 0, batch_large, (int64_t) -1, (int64_t) A_shape[3]});\n\n auto Z_broadcasted = std::make_shared(Z_unsqueezed, broadcast_shape,\n ov::op::BroadcastType::BIDIRECTIONAL);\n Z = std::make_shared(Z_broadcasted, new_Z_shape, true);\n }\n if (A_batch_larger) {\n B = Z;\n } else {\n A = Z;\n }\n\n res = std::make_shared(A, B, false, transpose_b);\n\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/permute.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_permute(const NodeContext & context) {\n num_inputs_check(context, 1, 1);\n\n int op_case = context.get_op_case();\n FRONT_END_CHECK_IMPLEMENTED(op_case == 1 || op_case == 2 || op_case == 3 || op_case == 4,\n \"Unsupported PERMUTE case\");\n\n ov::Output res;\n auto src = context.get_input(0);\n auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3});\n\n if (op_case == 1 || context.is_stateful()) {\n res = std::make_shared(src, perm);\n } else if (op_case == 4) {\n auto output_shape = context.get_output_shape().to_shape();\n auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]});\n auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]});\n auto n_seq_active = context.has_input(\"n_seq_active\") ?\n context.get_input(\"n_seq_active\") :\n ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[0]});\n auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});\n\n auto new_shape =\n std::make_shared(ov::OutputVector{n_seq_active, neg_one, n_heads, head_size}, 0);\n\n // // Alternative\n // auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n // auto new_shape = std::make_shared(ov::OutputVector{n_seq_active, neg_one, zero, zero}, 0);\n\n auto reshaped = std::make_shared(src, new_shape, true);\n res = std::make_shared(reshaped, perm);\n } else {\n auto cache_shape = src.get_partial_shape();\n auto output_shape = context.get_output_shape().to_shape();\n int64_t head_size = output_shape[3];\n int64_t n_heads = output_shape[1];\n int64_t ctx_per_seq = cache_shape[2].is_static() ? cache_shape[2].get_length() : -1;\n int64_t n_seq = cache_shape[1].get_length();\n\n Output attention_size;\n if (!context.has_input(\"attention_size\")) {\n attention_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[2]});\n } else if (op_case == 2) {\n attention_size = context.get_input(\"attention_size\");\n } else {\n attention_size = context.get_input(\"attention_size_swa\");\n }\n\n Output seq_active_start;\n Output seq_active_end;\n if (context.has_input(\"seq_active_start\")) {\n seq_active_start = context.get_input(\"seq_active_start\");\n seq_active_end = context.get_input(\"seq_active_end\");\n } else {\n int64_t n_seq_active = output_shape[0];\n size_t offset = *((size_t *) context.get_input_op_params(0));\n int64_t seq_active_start_val = offset / context.get_input_stride(0)[0];\n int64_t seq_active_end_val = seq_active_start_val + n_seq_active;\n seq_active_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {seq_active_start_val});\n seq_active_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {seq_active_end_val});\n }\n\n // 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size]\n // 2. slice out the active sequences\n // 3. slice out the attention part in each sequence\n // 4. permute\n auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n\n auto src_reshaped = std::make_shared(\n src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), false);\n auto slice1 = std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero);\n auto slice2 = std::make_shared(slice1, zero, attention_size, one, one);\n res = std::make_shared(slice2, perm);\n }\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/reshape.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_reshape(const NodeContext & context) {\n num_inputs_check(context, 1, 1);\n if (context.get_input_shape(0) == context.get_output_shape()) {\n return {context.get_input(0)};\n }\n\n int op_case = context.get_op_case();\n FRONT_END_CHECK_IMPLEMENTED(\n op_case == 1 || op_case == 2 || op_case == 3 || op_case == 4 || op_case == 5 || op_case == 6,\n \"Unsupported RESHAPE case\");\n\n auto output_shape = context.get_output_shape().to_shape();\n std::shared_ptr new_shape_node;\n if (op_case == 1) {\n if (context.is_stateful()) {\n new_shape_node = ov::op::v0::Constant::create(\n ov::element::i64, {3},\n std::vector{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]});\n } else {\n new_shape_node = ov::op::v0::Constant::create(\n ov::element::i64, {4},\n std::vector{(int64_t) output_shape[0], -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});\n }\n } else if (op_case == 2) {\n new_shape_node = ov::op::v0::Constant::create(\n ov::element::i64, {4},\n std::vector{(int64_t) output_shape[0], (int64_t) output_shape[1], -1, (int64_t) output_shape[3]});\n\n } else if (op_case == 3) {\n throw std::runtime_error(\"might be outdated RESHAPE case\");\n new_shape_node = ov::op::v0::Constant::create(\n ov::element::i64, {4}, std::vector{(int64_t) output_shape[0], (int64_t) output_shape[1], -1, 1});\n\n } else if (op_case == 4) {\n return {context.get_input(0).get_node_shared_ptr()->input_value(0)};\n\n } else if (op_case == 5) {\n if (context.is_stateful()) {\n std::vector shape_vec = {1, -1, (int64_t) context.get_output_shape().to_shape()[3]};\n new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {3}, shape_vec);\n } else {\n std::vector shape_vec = {1, 1, -1, (int64_t) context.get_output_shape().to_shape()[3]};\n new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {4}, shape_vec);\n }\n\n // // Alternative\n // auto token_len = context.get_input(\"token_len\");\n // auto emb_size =\n // ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) context.get_output_shape().to_shape()[3]});\n // auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n // new_shape_node = std::make_shared(ov::OutputVector{one, one, token_len, emb_size}, 0);\n\n } else if (op_case == 6) {\n new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {4}, context.get_output_shape().to_shape());\n }\n auto res = std::make_shared(context.get_input(0), new_shape_node, false);\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/rms_norm.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_rms_norm(const NodeContext & context) {\n num_inputs_check(context, 1, 1);\n\n auto input_node = context.get_input(0);\n auto square = std::make_shared(\n input_node, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f}));\n\n auto mean = std::make_shared(\n square, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true);\n\n float eps;\n memcpy(&eps, context.get_output_op_params(), sizeof(float));\n\n auto rms = std::make_shared(\n std::make_shared(mean, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps})));\n\n auto reciprocal =\n std::make_shared(ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {1.0f}), rms);\n\n auto res = std::make_shared(input_node, reciprocal);\n\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/rope.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_rope(const NodeContext & context) {\n num_inputs_check(context, 2, 3);\n\n int op_case = context.get_op_case();\n\n ov::Output res;\n\n auto data_node = context.get_input(0).get_node_shared_ptr();\n auto output_shape = context.get_output_shape().to_shape();\n int32_t * op_params = context.get_output_op_params();\n\n Output cos_theta_node;\n Output sin_theta_node;\n if (context.has_input(\"rope_cos\")) {\n cos_theta_node = context.get_input(\"rope_cos\");\n sin_theta_node = context.get_input(\"rope_sin\");\n } else {\n auto inp_pos = context.get_input(1).get_node_shared_ptr();\n std::shared_ptr rope_freqs_weight;\n if (context.get_input_size() == 3) {\n rope_freqs_weight = context.get_input(2).get_node_shared_ptr();\n }\n auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight);\n sin_theta_node = sin_cos.first;\n cos_theta_node = sin_cos.second;\n }\n\n if (op_case == 2) {\n // The input comes from a VIEW\n int slice_len = output_shape[2] * output_shape[3];\n data_node = process_view_input(context, 0, slice_len).get_node_shared_ptr();\n if (context.is_stateful()) {\n auto data_shape = ov::op::v0::Constant::create(\n ov::element::i64, {3}, std::vector{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]});\n data_node = std::make_shared(data_node, data_shape, false);\n } else {\n auto data_shape = ov::op::v0::Constant::create(\n ov::element::i64, {4}, std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});\n data_node = std::make_shared(data_node, data_shape, false);\n }\n }\n\n const int mode = op_params[2];\n constexpr int ROPE_TYPE_NORMAL = 0;\n constexpr int ROPE_TYPE_NEOX = 2;\n\n if (mode == ROPE_TYPE_NORMAL) {\n auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});\n auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});\n auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]});\n Output even_slice;\n Output odd_slice;\n int32_t unsqueeze_dim = context.is_stateful() ? 3 : 4;\n even_slice = std::make_shared(data_node, zero, end, two, neg_one);\n odd_slice = std::make_shared(data_node, one, end, two, neg_one);\n\n Output first_half =\n std::make_shared(std::make_shared(even_slice, cos_theta_node),\n std::make_shared(odd_slice, sin_theta_node));\n Output second_half =\n std::make_shared(std::make_shared(even_slice, sin_theta_node),\n std::make_shared(odd_slice, cos_theta_node));\n\n first_half = std::make_shared(first_half,\n ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim}));\n second_half = std::make_shared(second_half,\n ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim}));\n auto stack = std::make_shared(OutputVector{first_half, second_half}, unsqueeze_dim);\n\n auto data_shape = ov::op::v0::Constant::create(\n ov::element::i64, {4}, std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});\n res = std::make_shared(stack, data_shape, false);\n } else if (mode == ROPE_TYPE_NEOX) {\n auto data_split = std::make_shared(\n data_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}), 2);\n Output slice_data_node_0 = data_split->outputs()[0];\n Output slice_data_node_1 = data_split->outputs()[1];\n\n auto first_half_node = std::make_shared(\n std::make_shared(slice_data_node_0, cos_theta_node),\n std::make_shared(slice_data_node_1, sin_theta_node));\n\n auto second_half_node = std::make_shared(\n std::make_shared(slice_data_node_0, sin_theta_node),\n std::make_shared(slice_data_node_1, cos_theta_node));\n\n res = std::make_shared(ov::OutputVector{first_half_node, second_half_node}, -1);\n }\n\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/scale.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_scale(const NodeContext & context) {\n num_inputs_check(context, 1, 1);\n\n float scale;\n float bias;\n memcpy(&scale, (float *) context.get_output_op_params() + 0, sizeof(float));\n memcpy(&bias, (float *) context.get_output_op_params() + 1, sizeof(float));\n\n auto scale_node = std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale});\n auto scaled = std::make_shared(context.get_input(0), scale_node);\n\n std::shared_ptr res;\n if (bias != 0.0f) {\n auto bias_node =\n std::make_shared(ov::element::f32, ov::Shape{}, std::vector{bias});\n res = std::make_shared(scaled, bias_node);\n } else {\n res = scaled;\n }\n\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/set_rows.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_set_rows(const NodeContext & context) {\n num_inputs_check(context, 3, 3);\n\n auto data = context.get_input(0);\n auto indices = context.get_input(1);\n auto dst = context.get_input(2);\n\n data = std::make_shared(data, context.get_output_type());\n\n auto dst_shape = context.get_output_shape().to_shape();\n\n auto ind_squeezed =\n std::make_shared(indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2}));\n auto data_reshaped = std::make_shared(\n data,\n ov::op::v0::Constant::create(ov::element::i64, {4},\n {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) dst_shape[3]}),\n false);\n auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2});\n\n Output res;\n if (context.is_stateful()) {\n int concat_axis = 1;\n int64_t dim2 = dst.get_partial_shape()[2].get_length();\n int64_t dim3 = dst.get_partial_shape()[3].get_length();\n data = std::make_shared(\n data, ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, dim2, dim3}), false);\n res = std::make_shared(OutputVector{dst, data}, concat_axis);\n } else {\n res = std::make_shared(dst, ind_squeezed, data_reshaped, axes);\n }\n\n if (auto dst_reshape = std::dynamic_pointer_cast(dst.get_node_shared_ptr())) {\n // Fix the case of multiple sequences, reshape back to original shape [1, n_seq, ctx_per_seq, emb]\n // ctx_per_seq is not fixed due to llama-bench compatibility\n auto dst_shape_partial = dst_reshape->get_input_partial_shape(0);\n std::vector dst_shape = {dst_shape_partial[0].get_length(), dst_shape_partial[1].get_length(),\n dst_shape_partial[2].is_static() ? dst_shape_partial[2].get_length() : -1,\n dst_shape_partial[3].get_length()};\n res = std::make_shared(res, ov::op::v0::Constant::create(ov::element::i64, {4}, dst_shape),\n false);\n }\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/softmax.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_soft_max(const NodeContext & context) {\n // TODO code is outdated\n num_inputs_check(context, 1, 2);\n\n auto input_node = context.get_input(0).get_node_shared_ptr();\n ov::Output res;\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n auto * op_params = context.get_output_op_params();\n memcpy(&scale, (float *) op_params + 0, sizeof(float));\n memcpy(&max_bias, (float *) op_params + 1, sizeof(float));\n auto src0_shape = context.get_input_shape(0).get_shape();\n const uint32_t h = src0_shape[2];\n const uint32_t n_head = src0_shape[0];\n const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n\n const float m0 = powf(2.0f, -(max_bias) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n const float slope =\n (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2 * (h - n_head_log2) + 1) : 1.0f;\n\n auto scale_node = std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale});\n auto scaled_input = std::make_shared(input_node, scale_node);\n\n if (context.get_input_size() < 2) {\n res = std::make_shared(scaled_input, 2);\n return rename_outputs_with_suffix({res}, context.get_name());\n }\n\n ov::Output mask_node_sliced;\n if (context.has_input(\"KQ_mask_sliced\")) {\n mask_node_sliced = context.get_input(\"KQ_mask_sliced\");\n } else {\n auto token_len = get_dimensions(input_node, {1});\n auto mask_node = context.get_input(1);\n auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n mask_node_sliced = std::make_shared(mask_node, zero, token_len, one, one);\n }\n\n if (mask_node_sliced.get_element_type() != context.get_output_type()) {\n mask_node_sliced = std::make_shared(mask_node_sliced, context.get_output_type());\n }\n\n Output slope_mask;\n if (slope != 1.0f) {\n auto slope_node =\n std::make_shared(ov::element::f32, ov::Shape{}, std::vector{slope});\n slope_mask = std::make_shared(mask_node_sliced, slope_node);\n throw std::runtime_error(\"Slope != 1.0f in softmax has not been tested, verify it before use.\");\n }\n slope_mask = mask_node_sliced;\n\n auto input_slope_mask_node = std::make_shared(scaled_input, slope_mask);\n\n res = std::make_shared(input_slope_mask_node, 2);\n\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/transpose.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_transpose(const NodeContext & context) {\n num_inputs_check(context, 1, 1);\n\n auto res = std::make_shared(\n context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 1, 3, 2}));\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/unary_silu.cpp", "content": "#include \"../node_context.h\"\n#include \"../op_table.h\"\n#include \"../utils.h\"\n\n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_unary_silu(const NodeContext & context) {\n num_inputs_check(context, 1, 1);\n\n auto input = context.get_input(0);\n auto sigmoid = std::make_shared(input);\n auto res = std::make_shared(input, sigmoid);\n\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op/view.cpp", "content": "#include \"../op_table.h\"\n#include \"../utils.h\"\n#include \nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace op {\n\nOutputVector translate_view(const NodeContext & context) {\n num_inputs_check(context, 1, 1);\n\n if (context.get_op_case() == 2) {\n auto dst_shape = context.get_output_shape().to_shape();\n return rename_outputs_with_suffix({process_view_input(context, 0, dst_shape[2] * dst_shape[3])},\n context.get_name());\n }\n // op_case 3\n if (context.get_op_case() == 3) {\n auto input = context.get_input(0);\n auto input_ov_shape = input.get_partial_shape();\n\n auto input_llama_shape = context.get_input_shape(0).to_shape();\n\n // if the input ov shape size is different from the input llama shape size, it means the input is already reshaped and we need to reshape it back to the original shape before slicing\n if (input_ov_shape.size() != input_llama_shape.size()) {\n input = std::make_shared(input, ov::op::v0::Constant::create(ov::element::i64, {input_llama_shape.size()}, input_llama_shape), false);\n }\n\n auto dst_shape = context.get_output_shape().to_shape();\n\n // find the index of dst_shape that is different from input shape, and use that index to slice the input\n int slice_dim = -1;\n for (size_t i = 0; i < dst_shape.size(); ++i) {\n if (dst_shape[i] != input_llama_shape[i]) {\n slice_dim = i;\n break;\n }\n }\n\n auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {dst_shape[slice_dim]});\n auto stride = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim});\n auto sliced = std::make_shared(input, begin, end, stride, axes);\n return {sliced};\n }\n return {context.get_input(0)};\n}\n\n} // namespace op\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op_table.cpp", "content": "#include \"op_table.h\"\n\n#include \"utils.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nstd::unordered_map get_supported_ops() {\n using namespace ov::op;\n return {\n {\"GGML_OP_ADD\", op::translate_1to1_match_2_inputs },\n {\"GGML_OP_ADD1\", op::translate_1to1_match_2_inputs },\n {\"GGML_OP_CONT\", op::translate_cont },\n {\"GGML_OP_DIV\", op::translate_1to1_match_2_inputs },\n {\"GGML_OP_GET_ROWS\", op::translate_get_rows },\n {\"GGML_OP_MUL\", op::translate_1to1_match_2_inputs},\n {\"GGML_OP_MUL_MAT\", op::translate_mulmat },\n {\"GGML_OP_PERMUTE\", op::translate_permute },\n {\"GGML_OP_RESHAPE\", op::translate_reshape },\n {\"GGML_OP_RMS_NORM\", op::translate_rms_norm },\n {\"GGML_OP_ROPE\", op::translate_rope },\n {\"GGML_OP_SCALE\", op::translate_scale },\n {\"GGML_OP_SOFT_MAX\", op::translate_soft_max },\n {\"GGML_OP_SUB\", op::translate_1to1_match_2_inputs},\n {\"GGML_OP_TRANSPOSE\", op::translate_transpose },\n {\"GGML_UNARY_OP_SILU\", op::translate_unary_silu },\n {\"GGML_OP_VIEW\", op::translate_view },\n {\"GGML_GLU_OP_SWIGLU\", op::translate_glu_swiglu },\n {\"GGML_GLU_OP_GEGLU\", op::translate_glu_geglu },\n {\"GGML_OP_SET_ROWS\", op::translate_set_rows },\n {\"GGML_OP_CPY\", op::translate_cpy },\n {\"GGML_OP_FLASH_ATTN_EXT\", op::translate_flash_attn_ext },\n };\n}\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/op_table.h", "content": "#pragma once\n\n#include \"node_context.h\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nnamespace op {\n\n#define GGML_OP_CONVERTER(op) OutputVector op(const NodeContext& context)\n\nGGML_OP_CONVERTER(translate_add);\nGGML_OP_CONVERTER(translate_cont);\nGGML_OP_CONVERTER(translate_get_rows);\nGGML_OP_CONVERTER(translate_mul);\nGGML_OP_CONVERTER(translate_mulmat);\nGGML_OP_CONVERTER(translate_permute);\nGGML_OP_CONVERTER(translate_reshape);\nGGML_OP_CONVERTER(translate_rms_norm);\nGGML_OP_CONVERTER(translate_rope);\nGGML_OP_CONVERTER(translate_scale);\nGGML_OP_CONVERTER(translate_unary_silu);\nGGML_OP_CONVERTER(translate_soft_max);\nGGML_OP_CONVERTER(translate_transpose);\nGGML_OP_CONVERTER(translate_view);\nGGML_OP_CONVERTER(translate_glu_swiglu);\nGGML_OP_CONVERTER(translate_glu_geglu);\nGGML_OP_CONVERTER(translate_set_rows);\nGGML_OP_CONVERTER(translate_cpy);\nGGML_OP_CONVERTER(translate_flash_attn_ext);\n\n} // namespace op\n\nstd::unordered_map get_supported_ops();\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/pass/eliminate_zp.cpp", "content": "#include \"eliminate_zp.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace pass {\n\nEliminateZeroPoints::EliminateZeroPoints() {\n // Find pattern:\n // (Multiply Any(scale)\n // (Subtract (Convert Constant(data)))\n // (Convert Constant(zero_point)))\n // where zero_point is a scalar\n // If data is u4 and zp value is 8 (q4_0), Replace the Subtract with an i4 Constant whose value is data - zp_val\n // If data is u8 and zp value is 128 (q8_0) or 32 (q6_k), Replace the Subtract with an i8 Constant\n\n auto m_data_constant = ov::pass::pattern::wrap_type();\n auto m_data_convert = ov::pass::pattern::wrap_type({m_data_constant});\n\n auto m_zp_constant = ov::pass::pattern::wrap_type();\n auto m_zp_convert = ov::pass::pattern::wrap_type({m_zp_constant});\n\n auto m_subtract = ov::pass::pattern::wrap_type({m_data_convert, m_zp_convert});\n auto m_scale = ov::pass::pattern::any_input();\n auto m_multiply = ov::pass::pattern::wrap_type({m_scale, m_subtract});\n\n const auto callback = [=](ov::pass::pattern::Matcher & m) {\n const auto & pattern_map = m.get_pattern_value_map();\n\n auto multiply_node =\n std::dynamic_pointer_cast(pattern_map.at(m_multiply).get_node_shared_ptr());\n auto subtract_node =\n std::dynamic_pointer_cast(pattern_map.at(m_subtract).get_node_shared_ptr());\n auto data_constant =\n std::dynamic_pointer_cast(pattern_map.at(m_data_constant).get_node_shared_ptr());\n auto zp_constant =\n std::dynamic_pointer_cast(pattern_map.at(m_zp_constant).get_node_shared_ptr());\n\n if (!multiply_node || !subtract_node || !data_constant || !zp_constant) {\n return false;\n }\n\n if (ov::shape_size(zp_constant->get_shape()) != 1) {\n return false;\n }\n\n auto data_type = data_constant->get_element_type();\n auto zp_data = zp_constant->cast_vector();\n\n if (zp_data.empty()) {\n return false;\n }\n\n int zp_value = zp_data[0];\n\n bool should_eliminate = false;\n ov::element::Type target_type;\n\n if (data_type == ov::element::u4 && zp_value == 8) {\n should_eliminate = true;\n target_type = ov::element::i4;\n } else if (data_type == ov::element::u8 && (zp_value == 128 || zp_value == 32)) {\n should_eliminate = true;\n target_type = ov::element::i8;\n }\n\n if (!should_eliminate) {\n return false;\n }\n\n auto data_shape = data_constant->get_shape();\n size_t total_elements = ov::shape_size(data_shape);\n\n std::shared_ptr new_constant;\n\n // TODO improve performance\n if (data_type == ov::element::u4) {\n auto data_values = data_constant->cast_vector();\n std::vector adjusted_values(total_elements);\n\n ov::parallel_for(total_elements, [&](size_t i) {\n adjusted_values[i] = static_cast(static_cast(data_values[i]) - 8);\n });\n\n new_constant = std::make_shared(target_type, data_shape, adjusted_values);\n } else if (data_type == ov::element::u8) {\n auto data_values = data_constant->cast_vector();\n std::vector adjusted_values(total_elements);\n\n ov::parallel_for(total_elements, [&, zp_value](size_t i) {\n adjusted_values[i] = static_cast(static_cast(data_values[i]) - zp_value);\n });\n\n new_constant = std::make_shared(target_type, data_shape, adjusted_values);\n }\n\n auto new_convert =\n std::make_shared(new_constant, subtract_node->get_output_element_type(0));\n ov::replace_node(subtract_node, new_convert);\n\n return true;\n };\n\n register_matcher(\n std::make_shared(m_multiply, \"ov::frontend::ggml::pass::EliminateZeroPoints\"),\n callback);\n}\n\n} // namespace pass\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/pass/eliminate_zp.h", "content": "#include \"openvino/pass/matcher_pass.hpp\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace pass {\n\nclass EliminateZeroPoints : public ov::pass::MatcherPass {\npublic:\n OPENVINO_MATCHER_PASS_RTTI(\"ov::frontend::ggml::pass::EliminateZeroPoints\")\n EliminateZeroPoints();\n};\n\n} // namespace pass\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp", "content": "#include \"fuse_to_sdpa.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace pass {\n\nFuseToSDPA::FuseToSDPA() {\n // Not maintained since FLASH_ATTN_EXT has replaced this pattern\n const auto m_k = ov::pass::pattern::any_input();\n const auto m_q = ov::pass::pattern::any_input();\n const auto m_qk = ov::pass::pattern::wrap_type({m_q, m_k});\n const auto m_qk_f32 = ov::pass::pattern::wrap_type({m_qk});\n const auto m_scale = ov::pass::pattern::any_input();\n const auto m_scaled_qk = ov::pass::pattern::wrap_type({m_qk_f32, m_scale});\n const auto m_mask = ov::pass::pattern::any_input();\n const auto m_masked_qk = ov::pass::pattern::wrap_type({m_scaled_qk, m_mask});\n const auto m_softmax_qk = ov::pass::pattern::wrap_type({m_masked_qk});\n const auto m_softmax_qk_f16 = ov::pass::pattern::wrap_type({m_softmax_qk});\n const auto m_v = ov::pass::pattern::any_input();\n const auto m_qkv = ov::pass::pattern::wrap_type({m_softmax_qk_f16, m_v});\n\n const auto callback = [=](ov::pass::pattern::Matcher & m) {\n auto & pattern_to_output = m.get_pattern_value_map();\n auto k = pattern_to_output[m_k];\n auto q = pattern_to_output[m_q];\n auto v = pattern_to_output[m_v];\n auto mask = pattern_to_output[m_mask];\n auto scale = pattern_to_output[m_scale];\n\n auto mask_f16 = register_new_node(mask, ov::element::f16);\n auto scale_f16 = register_new_node(scale, ov::element::f16);\n auto sdpa = std::make_shared(q, k, v, mask_f16, scale_f16, false);\n\n ov::replace_node(m.get_match_root(), sdpa);\n ov::copy_runtime_info(m.get_matched_nodes(), sdpa);\n\n return true;\n };\n register_matcher(std::make_shared(m_qkv, \"ov::frontend::ggml::pass::FuseToSDPA\"),\n callback);\n}\n\n} // namespace pass\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/pass/fuse_to_sdpa.h", "content": "#include \"openvino/pass/matcher_pass.hpp\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace pass {\n\nclass FuseToSDPA : public ov::pass::MatcherPass {\npublic:\n OPENVINO_MATCHER_PASS_RTTI(\"ov::frontend::ggml::pass::FuseToSDPA\")\n FuseToSDPA();\n};\n\n} // namespace pass\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h", "content": "#pragma once\n\n#include \"mark_decompression_convert_constant_folding.h\"\n#include \"openvino/pass/matcher_pass.hpp\"\n#include \"openvino/core/visibility.hpp\"\n\n#ifdef OPENVINO_STATIC_LIBRARY\n# define TRANSFORMATIONS_API\n#else\n# ifdef IMPLEMENT_OPENVINO_API\n# define TRANSFORMATIONS_API OPENVINO_CORE_EXPORTS\n# else\n# define TRANSFORMATIONS_API OPENVINO_CORE_IMPORTS\n# endif // IMPLEMENT_OPENVINO_API\n#endif // OPENVINO_STATIC_LIBRARY\n\nnamespace ov {\nnamespace pass {\n\nclass TRANSFORMATIONS_API MarkCompressedFloatConstants;\n\n} // namespace pass\n} // namespace ov\n\nclass ov::pass::MarkCompressedFloatConstants : public MatcherPass {\npublic:\n OPENVINO_MATCHER_PASS_RTTI(\"MarkCompressedFloatConstants\")\n MarkCompressedFloatConstants();\n};\n" }, { "path": "src/ggml-openvino/openvino/pass/squeeze_matmul.cpp", "content": "#include \"squeeze_matmul.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace opp = ov::pass::pattern;\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace pass {\n\n// For quantized models, NPUW expects the activation to be 3d in DQ(DynamicQuantization) opt, e.g. DQMatMulGQ2i\nSqueezeMatmul::SqueezeMatmul() {\n auto m_act = opp::any_input();\n auto m_wei = opp::any_input();\n auto m_matmul = opp::wrap_type({m_act, m_wei});\n\n const auto callback = [=](ov::pass::pattern::Matcher & m) {\n const auto & pattern_map = m.get_pattern_value_map();\n auto matmul_node =\n std::dynamic_pointer_cast(pattern_map.at(m_matmul).get_node_shared_ptr());\n auto act = pattern_map.at(m_act);\n auto wei = pattern_map.at(m_wei);\n auto act_shape = act.get_partial_shape();\n auto wei_shape = wei.get_partial_shape();\n if (act_shape.rank().is_dynamic() || wei_shape.rank().is_dynamic()) {\n return false;\n }\n if (act_shape.rank().get_length() == 4 && wei_shape.rank().get_length() == 2) {\n auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {0});\n auto squeezed_act = std::make_shared(act, axis);\n auto new_matmul = std::make_shared(squeezed_act, wei, matmul_node->get_transpose_a(),\n matmul_node->get_transpose_b());\n auto unsqueezed_output = std::make_shared(new_matmul, axis);\n unsqueezed_output->set_friendly_name(matmul_node->get_friendly_name());\n ov::copy_runtime_info(matmul_node, {squeezed_act, new_matmul, unsqueezed_output});\n ov::replace_node(matmul_node, unsqueezed_output);\n return true;\n }\n return false;\n };\n\n register_matcher(std::make_shared(m_matmul, \"ov::frontend::ggml::pass::SqueezeMatmul\"),\n callback);\n}\n\n} // namespace pass\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/pass/squeeze_matmul.h", "content": "#include \"openvino/pass/matcher_pass.hpp\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\nnamespace pass {\n\nclass SqueezeMatmul : public ov::pass::MatcherPass {\npublic:\n OPENVINO_MATCHER_PASS_RTTI(\"ov::frontend::ggml::pass::SqueezeMatmul\")\n SqueezeMatmul();\n};\n\n} // namespace pass\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/translate_session.cpp", "content": "#include \"translate_session.h\"\n\n#include \"ggml-openvino/openvino/node_context.h\"\n#include \"ggml-openvino/openvino/utils.h\"\n#include \"input_model.h\"\n#include \"pass/eliminate_zp.h\"\n#include \"pass/mark_decompression_convert_constant_folding.h\"\n#include \"pass/squeeze_matmul.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nusing namespace ov::op;\n\nnamespace {\n\nov::pass::MakeStateful::ParamResPairs get_kv_param_res_pairs(\n const std::shared_ptr & model,\n const std::map & kv_param_res_names) {\n ov::pass::MakeStateful::ParamResPairs pairs;\n const auto & params = model->get_parameters();\n const auto & results = model->get_results();\n\n for (const auto & param_res : kv_param_res_names) {\n const auto & param_name = param_res.first;\n const auto & res_name = param_res.second;\n\n auto param_it = std::find_if(params.begin(), params.end(), [&](const std::shared_ptr & node) {\n return node->get_friendly_name() == param_name;\n });\n\n OPENVINO_ASSERT(param_it != params.end(), \"The tensor name \", param_name,\n \" is not associated with any of \"\n \"Parameters in the network.\");\n\n auto res_it = std::find_if(results.begin(), results.end(), [&](const std::shared_ptr & node) {\n return node->get_friendly_name() == res_name;\n });\n\n OPENVINO_ASSERT(res_it != results.end(), \"The tensor name \", res_name,\n \" is not associated with any of \"\n \"Results in the network.\");\n\n std::shared_ptr param = *param_it;\n std::shared_ptr res = *res_it;\n pairs.emplace_back(param, res);\n }\n return pairs;\n}\n\nvoid add_sliced_mask(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) {\n\n auto create_sliced_mask = [&](const std::string & mask_name, const std::string & sliced_name, bool is_static) {\n if ((tensor_map.find(mask_name) != tensor_map.end()) &&\n (tensor_map.find(\"token_len_per_seq\") != tensor_map.end())) {\n auto token_len_per_seq = tensor_map.at(\"token_len_per_seq\").get_node_shared_ptr();\n auto mask = tensor_map.at(mask_name).get_node_shared_ptr();\n std::shared_ptr mask_sliced;\n if (is_static) {\n mask_sliced = mask;\n } else if (ggml_model_decoder.is_stateful()) {\n auto zero_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {0,0});\n auto one_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,1});\n auto zero_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n auto three_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});\n auto neg_one_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});\n auto axes = ov::op::v0::Constant::create(ov::element::i64, {2}, {-2,-1});\n auto inp_pos = tensor_map.at(\"inp_pos\").get_node_shared_ptr();\n auto gather_inp_pos = std::make_shared(inp_pos, neg_one_1d, three_1d);\n auto reshaped_inp_pos = std::make_shared(gather_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false);\n auto inp_pos_incremented = std::make_shared(reshaped_inp_pos, ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {1}));\n auto stop = std::make_shared(ov::OutputVector{token_len_per_seq, std::make_shared(inp_pos_incremented, token_len_per_seq)}, 0);\n mask_sliced =\n std::make_shared(mask, zero_2d, stop, one_2d, axes);\n mask_sliced = std::make_shared(mask_sliced, ov::element::f16);\n mask_sliced->set_friendly_name(sliced_name);\n } else {\n auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});\n auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});\n mask_sliced = std::make_shared(mask, zero, token_len_per_seq, one, two);\n mask_sliced = std::make_shared(mask_sliced, ov::element::f16);\n mask_sliced->set_friendly_name(sliced_name);\n }\n tensor_map.insert({sliced_name, mask_sliced->output(0)});\n }\n };\n\n create_sliced_mask(\"self_kq_mask\", \"KQ_mask_sliced\", ggml_model_decoder.is_static());\n create_sliced_mask(\"self_kq_mask_swa\", \"KQ_mask_swa_sliced\", ggml_model_decoder.is_static());\n}\n\nvoid add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) {\n int32_t * rope_params = ggml_model_decoder.get_rope_params();\n if (tensor_map.find(\"inp_pos\") == tensor_map.end() || rope_params == nullptr) {\n return;\n }\n auto inp_pos = tensor_map.at(\"inp_pos\").get_node_shared_ptr();\n std::shared_ptr rope_freqs_weight;\n if (tensor_map.find(\"rope_freqs.weight\") != tensor_map.end()) {\n rope_freqs_weight = tensor_map.at(\"rope_freqs.weight\").get_node_shared_ptr();\n }\n\n auto sin_cos = make_sin_cos(rope_params, inp_pos, rope_freqs_weight);\n auto sin_theta = sin_cos.first;\n auto cos_theta = sin_cos.second;\n\n cos_theta.get_node_shared_ptr()->set_friendly_name(\"rope_cos\");\n sin_theta.get_node_shared_ptr()->set_friendly_name(\"rope_sin\");\n tensor_map.insert({\"rope_cos\", cos_theta});\n tensor_map.insert({\"rope_sin\", sin_theta});\n}\n\n// Create common patterns\nvoid preprocess(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) {\n add_sliced_mask(tensor_map, ggml_model_decoder);\n add_rope_sin_cos(tensor_map, ggml_model_decoder);\n}\n\n} // namespace\n\nTranslateSession::TranslateSession(const frontend::InputModel::Ptr & input_model,\n const std::unordered_map & translator_map,\n bool naive) :\n m_input_model(input_model),\n m_translator_map(translator_map),\n m_ov_model(nullptr),\n m_naive(naive) {}\n\nstd::shared_ptr TranslateSession::get_converted_model() {\n if (m_ov_model) {\n return m_ov_model;\n }\n m_ov_model = translate_graph(m_input_model);\n return m_ov_model;\n}\n\nstd::shared_ptr TranslateSession::translate_graph(const frontend::InputModel::Ptr & input_model) {\n ov::ParameterVector params;\n ov::ResultVector results;\n auto tensor_map = std::make_shared();\n std::shared_ptr resulting_model;\n\n const auto & ggml_model = std::dynamic_pointer_cast(input_model);\n std::shared_ptr ggml_model_decoder = ggml_model->get_model_decoder();\n\n for (const auto & it : ggml_model_decoder->get_model_inputs()) {\n params.push_back(std::dynamic_pointer_cast(it.second));\n (*tensor_map)[it.first] = it.second;\n }\n\n for (const auto & it : ggml_model_decoder->get_model_extra_inputs()) {\n if (std::dynamic_pointer_cast(it.second)) {\n params.push_back(std::dynamic_pointer_cast(it.second));\n }\n (*tensor_map)[it.first] = it.second;\n }\n\n for (const auto & it : ggml_model_decoder->get_model_weights()) {\n (*tensor_map)[it.first] = it.second;\n }\n\n auto node_visitor = [&](std::shared_ptr decoder, int node_idx) {\n auto operation_type = decoder->get_op_type(node_idx);\n if (operation_type == \"GGML_OP_NONE\") {\n return;\n }\n\n ov::OutputVector converted_outputs;\n auto it = m_translator_map.find(operation_type);\n FRONT_END_OP_CONVERSION_CHECK(it != m_translator_map.end(), \"Translation for operation type \", operation_type,\n \" is not implemented.\");\n NodeContext node_context(decoder, tensor_map, node_idx, this);\n converted_outputs = it->second(node_context);\n\n const auto & node_output_names = decoder->get_output_names(node_idx);\n FRONT_END_OP_CONVERSION_CHECK(node_output_names.size() == converted_outputs.size(), \"Number of \",\n operation_type, \" outputs greater than number of converted outputs, which are \",\n node_output_names.size(), \" and \", converted_outputs.size(), \" respectively.\");\n\n for (size_t i = 0; i < node_output_names.size(); ++i) {\n auto output_name = node_output_names[i];\n if (i < converted_outputs.size() && converted_outputs[i].get_node_shared_ptr() != nullptr) {\n (*tensor_map)[output_name] = converted_outputs[i];\n }\n }\n };\n\n if (!m_naive) {\n preprocess(*tensor_map, *ggml_model_decoder);\n }\n ggml_model_decoder->visit_subgraph(node_visitor);\n\n for (const auto & name : ggml_model_decoder->get_model_output_names()) {\n FRONT_END_GENERAL_CHECK(tensor_map->find(name) != tensor_map->end(),\n \"Output name not found in tensor map: \", name);\n auto result = std::make_shared(tensor_map->at(name));\n result->set_friendly_name(name);\n results.push_back(result);\n }\n\n ov::ParameterVector used_params;\n for (const auto & param : params) {\n if (!param->output(0).get_target_inputs().empty()) {\n used_params.push_back(param);\n }\n }\n // if (auto diff = params.size() - used_params.size()) {\n // GGML_LOG_INFO(\"%zu parameters are not used in the model.\", diff);\n // }\n resulting_model = std::make_shared(results, used_params);\n\n apply_transformations(resulting_model);\n return resulting_model;\n}\n\nstd::shared_ptr TranslateSession::apply_transformations(std::shared_ptr model) {\n auto ggml_model_decoder = std::dynamic_pointer_cast(m_input_model)->get_model_decoder();\n {\n ov::pass::Manager manager;\n manager.set_per_pass_validation(true);\n manager.register_pass();\n\n if (ggml_model_decoder->is_stateful()) {\n const auto kv_param_res_names = ggml_model_decoder->get_kv_param_res_names();\n const auto kv_param_res_pairs = get_kv_param_res_pairs(model, kv_param_res_names);\n manager.register_pass(kv_param_res_pairs);\n }\n\n if (ggml_model_decoder->is_static()) {\n manager.register_pass();\n manager.register_pass();\n }\n manager.run_passes(model);\n if (ggml_model_decoder->is_stateful()) {\n auto output_names = ggml_model_decoder->get_model_output_names();\n std::map model_output_indexes;\n for (size_t i=0; iget_output_size(); i++) {\n auto output_friendly_name = model->output(i).get_node_shared_ptr()->get_friendly_name();\n auto output_id = model_output_indexes[output_friendly_name];\n auto model_output_shape = model->output(i).get_partial_shape();\n auto decoder_output_shape = ggml_model_decoder->get_output_shape(output_id);\n if (model_output_shape.rank().is_static() && decoder_output_shape.rank().is_static()\n && model_output_shape.rank().get_length() + 1 == decoder_output_shape.rank().get_length()\n && decoder_output_shape[0].is_static() && decoder_output_shape[0].get_length() == 1) {\n ppp.output(i).postprocess().custom([](const ov::Output& node) {\n auto axes = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {0});\n return std::make_shared(node, axes);\n });\n }\n }\n model = ppp.build();\n }\n }\n return model;\n}\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/translate_session.h", "content": "#pragma once\n\n#include \"input_model.h\"\n#include \"node_context.h\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nclass TranslateSession {\npublic:\n TranslateSession(const frontend::InputModel::Ptr& input_model,\n const std::unordered_map& translator_map, bool naive = false);\n\n std::shared_ptr get_converted_model();\n std::shared_ptr translate_graph(const frontend::InputModel::Ptr& input_model);\n\nprivate:\n std::shared_ptr apply_transformations(std::shared_ptr model);\n const frontend::InputModel::Ptr m_input_model;\n const std::unordered_map& m_translator_map;\n std::shared_ptr m_ov_model;\n bool m_naive;\n};\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/utils.cpp", "content": "#include \"utils.h\"\n\n#include \"ggml-impl.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nstd::string getCurrentTime() {\n std::time_t now = std::time(nullptr);\n char buf[100];\n std::strftime(buf, sizeof(buf), \"%Y-%m-%d %H:%M:%S\", std::localtime(&now));\n return buf;\n}\n\nvoid num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs) {\n auto input_size = context.get_input_size();\n FRONT_END_OP_CONVERSION_CHECK(input_size >= min_inputs, \"Got less inputs than expected\");\n FRONT_END_OP_CONVERSION_CHECK(input_size <= max_inputs, \"Got more inputs than expected\");\n}\n\nint non_cont_dim(std::vector ne, std::vector nb) {\n int dim = nb.size() - 1;\n size_t bytes = nb[dim];\n for (int i = dim; i > 0; i--) {\n bytes *= ne[i];\n if (bytes != nb[i - 1]) {\n return i;\n }\n }\n return 0;\n}\n\nstd::shared_ptr get_dimensions(const std::shared_ptr & shape,\n const std::vector & dims) {\n using namespace ov::op;\n const auto zero = v0::Constant::create(ov::element::i32, ov::Shape{}, {0});\n const auto dims_const = v0::Constant::create(ov::element::i32, ov::Shape{dims.size()}, dims);\n return std::make_shared(shape, dims_const, zero);\n}\n\nstd::shared_ptr get_dimensions(const std::shared_ptr & node, const std::vector & dims) {\n return get_dimensions(std::make_shared(node), dims);\n}\n\nOutputVector rename_outputs_with_suffix(const OutputVector & outputs, const std::string & suffix) {\n for (const auto & output : outputs) {\n auto node = output.get_node_shared_ptr();\n std::string name = node->get_friendly_name();\n name += \"_\";\n name += suffix;\n node->set_friendly_name(name);\n // std::cout << name << \" \" << output.get_partial_shape() << std::endl;\n }\n return outputs;\n}\n\nnamespace {\nov::Output rope_yarn_ramp_mix(int n_dims, const float corr_dims[2], float ext_factor) {\n int half_n_dims = n_dims / 2;\n std::vector dim_ids_vec(half_n_dims);\n std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0);\n auto dim_ids = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, (size_t) half_n_dims}, dim_ids_vec);\n auto corr_low = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[0]});\n auto corr_high = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[1]});\n auto denom = std::make_shared(\n std::make_shared(corr_high, corr_low),\n ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {0.001f}));\n auto ramp_y =\n std::make_shared(std::make_shared(dim_ids, corr_low), denom);\n auto ramp_clamped = std::make_shared(ramp_y, 0.0f, 1.0f);\n auto ext_factor_node = ov::op::v0::Constant::create(ov::element::f32, Shape{}, {ext_factor});\n auto ramp_mix = std::make_shared(ramp_clamped, ext_factor_node);\n return ramp_mix;\n}\n\nfloat ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) {\n#ifndef M_PI\n# define M_PI 3.14159265358979323846\n#endif\n return n_dims * logf(n_ctx_orig / (n_rot * 2 * (float) M_PI)) / (2 * logf(base));\n}\n\nvoid ggml_rope_yarn_corr_dims(int n_dims,\n int n_ctx_orig,\n float freq_base,\n float beta_fast,\n float beta_slow,\n float dims[2]) {\n float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base));\n float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base));\n dims[0] = std::max(0.0f, start);\n dims[1] = std::min(static_cast(n_dims - 1), end);\n}\n} // namespace\n\nstd::pair, ov::Output> make_sin_cos(int32_t * rope_params,\n std::shared_ptr inp_pos,\n std::shared_ptr rope_freqs_weight,\n bool stateful) {\n if (stateful) {\n inp_pos = std::make_shared(inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));\n inp_pos = std::make_shared(inp_pos, ov::element::f32);\n auto pos_perm =\n std::make_shared(ov::element::i64, ov::Shape{3}, std::vector{2, 1, 0});\n inp_pos = std::make_shared(inp_pos, pos_perm);\n } else {\n inp_pos = std::make_shared(inp_pos, ov::element::f32);\n auto pos_perm =\n std::make_shared(ov::element::i64, ov::Shape{4}, std::vector{0, 3, 1, 2});\n inp_pos = std::make_shared(inp_pos, pos_perm);\n }\n\n float freq_base;\n float freq_scale;\n float ext_factor;\n float attn_factor;\n float beta_fast;\n float beta_slow;\n const int n_dims = rope_params[1];\n const int n_ctx_orig = rope_params[4];\n memcpy(&freq_base, rope_params + 5, sizeof(float));\n memcpy(&freq_scale, rope_params + 6, sizeof(float));\n memcpy(&ext_factor, rope_params + 7, sizeof(float));\n memcpy(&attn_factor, rope_params + 8, sizeof(float));\n memcpy(&beta_fast, rope_params + 9, sizeof(float));\n memcpy(&beta_slow, rope_params + 10, sizeof(float));\n\n const float theta_scale = powf(freq_base, -2.0f / n_dims);\n\n float corr_dims[2];\n ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);\n\n std::vector factor(n_dims / 2);\n factor[0] = 1.0f;\n for (size_t i = 1; i < factor.size(); i++) {\n factor[i] = theta_scale * factor[i - 1];\n }\n\n Output freq_factors;\n if (stateful) {\n freq_factors =\n std::make_shared(ov::element::f32, ov::Shape{1, 1, factor.size()}, factor);\n } else {\n freq_factors =\n std::make_shared(ov::element::f32, ov::Shape{1, 1, 1, factor.size()}, factor);\n }\n if (rope_freqs_weight) {\n freq_factors = std::make_shared(freq_factors, rope_freqs_weight);\n }\n\n auto theta_extrap = std::make_shared(freq_factors, inp_pos);\n auto theta_interp = std::make_shared(\n theta_extrap, ov::op::v0::Constant::create(ov::element::f32, {1}, {freq_scale}));\n\n Output theta;\n float mscale = attn_factor;\n if (ext_factor == 0.0f) {\n theta = theta_interp;\n } else {\n auto ramp_mix = rope_yarn_ramp_mix(n_dims, corr_dims, ext_factor);\n Output one;\n if (stateful) {\n one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1}, {1.0f});\n } else {\n one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f});\n }\n auto one_minus_ramp = std::make_shared(one, ramp_mix);\n\n theta = std::make_shared(std::make_shared(theta_interp, one_minus_ramp),\n std::make_shared(theta_extrap, ramp_mix));\n mscale *= (1.0f + 0.1f * std::log(1.0f / freq_scale));\n }\n\n Output cos_theta = std::make_shared(theta);\n Output sin_theta = std::make_shared(theta);\n\n auto mscale_node = ov::op::v0::Constant::create(ov::element::f32, Shape{}, {mscale});\n\n cos_theta = std::make_shared(cos_theta, mscale_node);\n sin_theta = std::make_shared(sin_theta, mscale_node);\n return std::make_pair(sin_theta, cos_theta);\n}\n\nov::Output process_view_input(const NodeContext & context, int input_index, int slice_len) {\n // Only works for VIEW operations that slice at the lowest dimension\n // If the VIEW also reshape the result, `slice_len` should be provided\n auto input = context.get_input(input_index);\n auto * op_params = (size_t *) context.get_input_op_params(input_index);\n auto src1_stride = context.get_input_stride(input_index);\n\n int64_t split_addr = op_params[0] / src1_stride[3];\n if (slice_len == 0) {\n slice_len = context.get_input_shape(input_index)[3].get_length();\n }\n int64_t slice_end = split_addr + slice_len;\n\n auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {split_addr});\n auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_end});\n auto stride = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});\n auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {context.is_stateful() ? 2 : 3});\n auto sliced = std::make_shared(input, begin, end, stride, axes);\n return sliced;\n}\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/openvino/utils.h", "content": "#pragma once\n\n#include \n#include \n#include \n#include \n#include \n\n#include \"node_context.h\"\n\nnamespace ov {\nnamespace frontend {\nnamespace ggml {\n\nstd::string getCurrentTime();\n\nvoid dump_ov_model(std::shared_ptr model);\n\nvoid num_inputs_check(const NodeContext& context, size_t min_inputs, size_t max_inputs);\n\nint non_cont_dim(std::vector ne, std::vector nb);\n\ntemplate \nstd::vector argsort_descend(const std::vector& v) {\n std::vector idx(v.size());\n std::iota(idx.begin(), idx.end(), 0);\n std::sort(idx.begin(), idx.end(), [&v](int i1, int i2) {\n return v[i1] > v[i2];\n });\n return idx;\n}\n\ntemplate \nstd::vector sorted_descend(std::vector v) {\n std::sort(v.begin(), v.end(), [](T a, T b) {\n return a > b;\n });\n return v;\n}\n\ntemplate \nbool is_permuted(const std::vector& strides) {\n for (size_t i = 0; i < strides.size() - 1; ++i) {\n if (strides[i] < strides[i + 1]) {\n return true;\n }\n }\n return false;\n}\n\ntemplate \nstd::vector permute(const std::vector& x, const std::vector& perm) {\n std::vector result;\n result.reserve(perm.size());\n for (int i : perm) {\n result.push_back(x[i]);\n }\n return result;\n}\n\nstd::shared_ptr get_dimensions(const std::shared_ptr& shape,\n const std::vector& dims);\nstd::shared_ptr get_dimensions(const std::shared_ptr& node, const std::vector& dims);\n\nOutputVector rename_outputs_with_suffix(const OutputVector& outputs, const std::string& suffix);\n\nstd::pair, ov::Output> make_sin_cos(int32_t* rope_params,\n std::shared_ptr inp_pos,\n std::shared_ptr rope_freqs_weight = nullptr,\n bool stateful = false);\n\nov::Output process_view_input(const NodeContext& context, int input_index, int slice_len = 0);\n\nnamespace op {\ntemplate \nOutputVector translate_1to1_match_2_inputs(const NodeContext& context) {\n num_inputs_check(context, 2, 2);\n auto res = std::make_shared(context.get_input(0), context.get_input(1));\n return rename_outputs_with_suffix({res}, context.get_name());\n}\n} // namespace op\n\n} // namespace ggml\n} // namespace frontend\n} // namespace ov\n" }, { "path": "src/ggml-openvino/utils.cpp", "content": "#include \"utils.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-openvino-extra.h\"\n#include \"ggml-openvino/ggml-decoder.h\"\n#include \"ggml.h\"\n#include \"openvino/frontend.h\"\n#include \"openvino/input_model.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n// Suppress deprecation warning for ov::Tensor::data()\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wdeprecated-declarations\"\n\nenum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) {\n ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context;\n try {\n if (getenv(\"GGML_OPENVINO_DUMP_CGRAPH\")) {\n std::string filename = \"cgraph_ov.txt\";\n GgmlOvDecoder::dump_cgraph(cgraph, filename);\n }\n\n const auto is_static = ggml_openvino_is_npu();\n\n GGML_ASSERT(ctx->runtime_context != nullptr);\n std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context);\n\n return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx);\n } catch (const ov::Exception & e) {\n GGML_LOG_ERROR(\"GGML OpenVINO backend ov::Exception: %s\\n\", e.what());\n return GGML_STATUS_FAILED;\n } catch (const std::exception & e) {\n GGML_LOG_ERROR(\"GGML OpenVINO backend std::exception: %s\\n\", e.what());\n return GGML_STATUS_FAILED;\n } catch (...) {\n GGML_LOG_ERROR(\"GGML OpenVINO backend unknown exception\\n\");\n return GGML_STATUS_FAILED;\n }\n}\n\nov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder,\n std::shared_ptr infer_request,\n int output_index,\n const ggml_tensor * ggml_tensor) {\n auto output_type = ggml_decoder->get_ov_type(ggml_tensor);\n ov::Shape output_shape;\n if (ggml_decoder->is_static()) {\n output_shape = infer_request->get_output_tensor(output_index).get_shape();\n } else {\n output_shape = ggml_decoder->get_shape(ggml_tensor);\n }\n\n ov::Tensor output_tensor(output_type, output_shape, ggml_tensor->data);\n return output_tensor;\n}\n\nenum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr r_ctx) {\n auto & core = ov_singleton_core();\n const auto & config = ggml_openvino_get_compile_config();\n auto device = r_ctx->device;\n bool stateful = r_ctx->stateful;\n static auto is_static = false;\n\n if (is_naive(cgraph)) {\n return naive_compute(cgraph, core, device, config);\n }\n\n auto start_time = ggml_time_us();\n\n std::shared_ptr ggml_decoder;\n std::shared_ptr infer_request;\n ModelParams m_params;\n ComputeParams c_params;\n std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static);\n\n graph_key key(cgraph);\n bool cache_hit;\n\n int64_t decoder_end_time;\n int64_t conversion_end_time;\n int64_t compile_end_time;\n int64_t infer_end_time;\n\n {\n std::lock_guard lock(r_ctx->ov_compute_mutex);\n\n auto it = r_ctx->decoder_cache.find(key);\n\n cache_hit = it != r_ctx->decoder_cache.end();\n ModelParams old_m_params;\n if (cache_hit) {\n ggml_decoder = it->second;\n old_m_params = ggml_decoder->get_model_params();\n cache_hit = old_m_params.can_reuse_dynamically(m_params);\n }\n\n if (cache_hit) {\n std::map> model_weights;\n ggml_decoder->set_compute_params(c_params);\n ggml_decoder->set_model_params(m_params);\n if (old_m_params.kv_buffer_changed(m_params)) {\n ggml_decoder->update_io(cgraph);\n }\n ggml_decoder->add_extra_inputs();\n infer_request = r_ctx->infer_request_cache.at(key);\n\n if (stateful) {\n const auto * inp_pos = get_inp_pos_tensor(cgraph);\n int32_t * pos_data = (int32_t *) inp_pos->data;\n auto pos_shape = ggml_decoder->get_shape(inp_pos);\n if (pos_data[0] == 0) {\n infer_request->reset_state();\n r_ctx->stateful_kv_size = pos_shape[3];\n } else if (r_ctx->stateful_kv_size == static_cast(pos_data[0])) {\n r_ctx->stateful_kv_size += pos_shape[3];\n } else {\n auto states = infer_request->query_state();\n for (auto state : states) {\n auto state_tensor = state.get_state();\n auto state_tensor_shape = state_tensor.get_shape();\n if (static_cast(pos_data[0]) > r_ctx->stateful_kv_size) {\n std::string state_name;\n try {\n state_name = r_ctx->kv_state_input_name_map.at(state.get_name());\n } catch (...) {\n GGML_LOG_ERROR(\"GGML OpenVINO backend stateful inference failed: no input found for the state\\n\");\n return GGML_STATUS_FAILED;\n }\n auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name);\n kv_tensor.set_shape({state_tensor_shape[0], kv_tensor.get_shape()[2],\n state_tensor_shape[2], state_tensor_shape[3]});\n state_tensor = kv_tensor;\n state_tensor_shape = state_tensor.get_shape();\n }\n ov::Coordinate begin = {0, 0, 0, 0};\n ov::Coordinate end = {state_tensor_shape[0], static_cast(pos_data[0]),\n state_tensor_shape[2], state_tensor_shape[3]};\n ov::Tensor new_state_tensor(state_tensor, begin, end);\n state.set_state(new_state_tensor);\n }\n r_ctx->stateful_kv_size = pos_data[0] + 1;\n }\n }\n\n decoder_end_time = ggml_time_us();\n conversion_end_time = decoder_end_time;\n compile_end_time = decoder_end_time;\n } else {\n r_ctx->infer_request_cache.erase(key);\n\n std::shared_ptr model;\n auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph);\n\n ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, stateful);\n decoder_end_time = ggml_time_us();\n\n auto input_model = std::make_shared(ggml_decoder);\n model = ov::frontend::ggml::FrontEnd::convert(input_model);\n ggml_decoder->clear_model_weights();\n conversion_end_time = ggml_time_us();\n\n if (getenv(\"GGML_OPENVINO_DUMP_IR\")) {\n char timestamped_filename[64];\n auto timestamp = (long long) ggml_time_us();\n snprintf(timestamped_filename, sizeof(timestamped_filename), \"model_%lld.xml\", timestamp);\n ov::serialize(model, timestamped_filename);\n }\n\n ov::CompiledModel compiled_model;\n auto remote_context = ggml_openvino_get_remote_context();\n if (remote_context.has_value()) {\n compiled_model = core.compile_model(model, remote_context.value(), config);\n } else {\n compiled_model = core.compile_model(model, device, config);\n }\n compile_end_time = ggml_time_us();\n infer_request = std::make_shared(compiled_model.create_infer_request());\n r_ctx->infer_request_cache[key] = infer_request;\n r_ctx->decoder_cache[key] = ggml_decoder;\n\n std::vector ov_input_names;\n std::vector ov_output_names;\n for (const auto & ov_param : model->get_parameters()) {\n ov_input_names.push_back(ov_param->get_friendly_name());\n }\n for (const auto & ov_output : model->get_results()) {\n ov_output_names.push_back(ov_output->get_friendly_name());\n }\n r_ctx->ov_input_names_cache[key] = std::move(ov_input_names);\n r_ctx->ov_output_names_cache[key] = std::move(ov_output_names);\n\n if (stateful) {\n const auto * inp_pos = get_inp_pos_tensor(cgraph);\n auto pos_shape = ggml_decoder->get_shape(inp_pos);\n r_ctx->stateful_kv_size = pos_shape[3];\n const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names();\n for (const auto& pair : kv_param_res_names) {\n r_ctx->kv_state_input_name_map[pair.first+pair.second] = pair.first;\n }\n }\n }\n\n auto ov_input_names = r_ctx->ov_input_names_cache[key];\n auto ov_output_names = r_ctx->ov_output_names_cache[key];\n\n for (size_t i = 0; i < ov_input_names.size(); i++) {\n auto param_name = ov_input_names[i];\n auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name);\n infer_request->set_input_tensor(i, input_tensor);\n\n if (getenv(\"GGML_OPENVINO_DEBUG_INPUT\")) {\n print_input_tensor_info(param_name, input_tensor);\n }\n }\n\n for (size_t i = 0; i < ov_output_names.size(); i++) {\n auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names[i]);\n auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);\n infer_request->set_output_tensor(i, output_tensor);\n }\n\n infer_request->infer();\n infer_end_time = ggml_time_us();\n\n if (getenv(\"GGML_OPENVINO_DEBUG_OUTPUT\")) {\n for (size_t i = 0; i < ov_output_names.size(); i++) {\n const auto output_tensor = infer_request->get_output_tensor(i);\n print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data());\n }\n }\n\n if (getenv(\"GGML_OPENVINO_PROFILING\")) {\n GGML_LOG_INFO(\"\\nGGML OpenVINO Backend: \\n\");\n GGML_LOG_INFO(\" - Graph decoder time: %ld ms \\n\", (decoder_end_time - start_time) / 1000);\n if (!cache_hit) {\n GGML_LOG_INFO(\" - Graph conversion time: %ld ms \\n\", (conversion_end_time - decoder_end_time) / 1000);\n GGML_LOG_INFO(\" - Graph compile time: %ld ms \\n\", (compile_end_time - conversion_end_time) / 1000);\n }\n GGML_LOG_INFO(\" - Graph inference time: %ld ms \\n\", (infer_end_time - compile_end_time) / 1000);\n }\n }\n\n return GGML_STATUS_SUCCESS;\n}\n\nenum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr r_ctx) {\n auto & core = ov_singleton_core();\n\n auto get_prefill_chunk_size = [] {\n const char * chunk_size_str = getenv(\"GGML_OPENVINO_PREFILL_CHUNK_SIZE\");\n if (chunk_size_str && atoi(chunk_size_str) > 0) {\n return atoi(chunk_size_str);\n }\n return 256;\n };\n\n static std::string device = \"NPU\";\n static auto is_static = true;\n static auto stateful = false;\n static auto prefill_chunk_size = get_prefill_chunk_size();\n const auto & config = ggml_openvino_get_compile_config();\n\n if (is_naive(cgraph)) {\n return naive_compute(cgraph, core, device, config);\n }\n\n auto start_time = ggml_time_us();\n\n std::shared_ptr ggml_decoder;\n std::shared_ptr infer_request;\n ModelParams m_params;\n ComputeParams c_params;\n std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static);\n\n const auto * inp_pos = get_inp_pos_tensor(cgraph);\n const auto is_prefill = get_is_prefill(inp_pos);\n graph_key key(cgraph);\n bool cache_hit;\n\n int64_t decoder_end_time;\n int64_t conversion_end_time;\n int64_t compile_end_time;\n int64_t infer_end_time;\n\n auto it = r_ctx->decoder_cache.find(key);\n\n cache_hit = it != r_ctx->decoder_cache.end();\n ModelParams old_m_params;\n if (cache_hit) {\n ggml_decoder = it->second;\n old_m_params = ggml_decoder->get_model_params();\n cache_hit = old_m_params.can_reuse_statically(m_params);\n }\n\n if (cache_hit) {\n std::map> model_weights;\n ggml_decoder->m_is_prefill = is_prefill;\n ggml_decoder->set_model_params(m_params);\n ggml_decoder->set_compute_params(c_params);\n if (old_m_params.kv_buffer_changed(m_params)) {\n ggml_decoder->update_io(cgraph);\n }\n ggml_decoder->add_extra_inputs();\n infer_request = is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key);\n\n decoder_end_time = ggml_time_us();\n conversion_end_time = decoder_end_time;\n compile_end_time = decoder_end_time;\n } else {\n r_ctx->infer_request_cache.erase(key);\n r_ctx->infer_request_cache_prefill.erase(key);\n\n std::shared_ptr model;\n auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph);\n\n auto ggml_decoder_prefill = std::make_shared(cgraph, m_params, c_params, model_weights,\n is_static, stateful, true, prefill_chunk_size);\n auto ggml_decoder_decode = std::make_shared(cgraph, m_params, c_params, model_weights, is_static,\n stateful, false, prefill_chunk_size);\n decoder_end_time = ggml_time_us();\n\n auto input_model_prefill = std::make_shared(ggml_decoder_prefill);\n auto input_model_decode = std::make_shared(ggml_decoder_decode);\n\n auto model_prefill = ov::frontend::ggml::FrontEnd::convert(input_model_prefill);\n ggml_decoder_prefill->clear_model_weights();\n auto model_decode = ov::frontend::ggml::FrontEnd::convert(input_model_decode);\n ggml_decoder_decode->clear_model_weights();\n conversion_end_time = ggml_time_us();\n\n if (getenv(\"GGML_OPENVINO_DUMP_IR\")) {\n char timestamped_filename[64];\n auto timestamp = (long long) ggml_time_us();\n snprintf(timestamped_filename, sizeof(timestamped_filename), \"model_prefill_%lld.xml\", timestamp);\n ov::serialize(model_prefill, timestamped_filename);\n snprintf(timestamped_filename, sizeof(timestamped_filename), \"model_decode_%lld.xml\", timestamp);\n ov::serialize(model_decode, timestamped_filename);\n }\n\n ov::CompiledModel compiled_model_prefill;\n ov::CompiledModel compiled_model_decode;\n auto remote_context = ggml_openvino_get_remote_context();\n if (remote_context.has_value()) {\n compiled_model_prefill = core.compile_model(model_prefill, remote_context.value(), config);\n compiled_model_decode = core.compile_model(model_decode, remote_context.value(), config);\n } else {\n compiled_model_prefill = core.compile_model(model_prefill, device, config);\n compiled_model_decode = core.compile_model(model_decode, device, config);\n }\n\n r_ctx->infer_request_cache_prefill[key] =\n std::make_shared(compiled_model_prefill.create_infer_request());\n r_ctx->infer_request_cache[key] =\n std::make_shared(compiled_model_decode.create_infer_request());\n compile_end_time = ggml_time_us();\n\n model = is_prefill ? model_prefill : model_decode;\n ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode;\n infer_request = is_prefill ? r_ctx->infer_request_cache_prefill[key] : r_ctx->infer_request_cache[key];\n r_ctx->decoder_cache[key] = ggml_decoder;\n\n std::vector ov_input_names;\n std::vector ov_output_names;\n for (const auto & ov_param : model->get_parameters()) {\n ov_input_names.push_back(ov_param->get_friendly_name());\n }\n for (const auto & ov_output : model->get_results()) {\n ov_output_names.push_back(ov_output->get_friendly_name());\n }\n r_ctx->ov_input_names_cache[key] = std::move(ov_input_names);\n r_ctx->ov_output_names_cache[key] = std::move(ov_output_names);\n }\n\n auto ov_input_names = r_ctx->ov_input_names_cache[key];\n auto ov_output_names = r_ctx->ov_output_names_cache[key];\n\n if (is_prefill) {\n auto inp_len = inp_pos->ne[0];\n for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) {\n for (size_t i = 0; i < ov_input_names.size(); i++) {\n auto param_name = ov_input_names[i];\n auto input_tensor = get_ov_input_tensor_static_prefill(ggml_decoder, param_name, chunk_index);\n infer_request->set_input_tensor(i, input_tensor);\n\n if (getenv(\"GGML_OPENVINO_DEBUG_INPUT\")) {\n const auto input_tensor = infer_request->get_input_tensor(i);\n print_input_tensor_info(param_name, input_tensor);\n }\n }\n\n for (size_t i = 0; i < ov_output_names.size(); i++) {\n auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names[i]);\n auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);\n infer_request->set_output_tensor(i, output_tensor);\n }\n\n infer_request->infer();\n\n if (getenv(\"GGML_OPENVINO_DEBUG_OUTPUT\")) {\n for (size_t i = 0; i < ov_output_names.size(); i++) {\n const auto output_tensor = infer_request->get_output_tensor(i);\n print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data());\n }\n }\n }\n infer_end_time = ggml_time_us();\n } else {\n for (size_t i = 0; i < ov_input_names.size(); i++) {\n auto param_name = ov_input_names[i];\n auto input_tensor = get_ov_input_tensor_static_decode(ggml_decoder, param_name);\n infer_request->set_input_tensor(i, input_tensor);\n\n if (getenv(\"GGML_OPENVINO_DEBUG_INPUT\")) {\n const auto input_tensor = infer_request->get_input_tensor(i);\n print_input_tensor_info(param_name, input_tensor);\n }\n }\n\n for (size_t i = 0; i < ov_output_names.size(); i++) {\n auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names[i]);\n auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);\n infer_request->set_output_tensor(i, output_tensor);\n }\n\n infer_request->infer();\n infer_end_time = ggml_time_us();\n\n if (getenv(\"GGML_OPENVINO_DEBUG_OUTPUT\")) {\n for (size_t i = 0; i < ov_output_names.size(); i++) {\n const auto output_tensor = infer_request->get_output_tensor(i);\n print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data());\n }\n }\n }\n\n if (getenv(\"GGML_OPENVINO_PROFILING\")) {\n GGML_LOG_INFO(\"\\nGGML OpenVINO Backend: \\n\");\n GGML_LOG_INFO(\" - Graph decoder time: %ld ms \\n\", (decoder_end_time - start_time) / 1000);\n if (!cache_hit) {\n GGML_LOG_INFO(\" - Graph conversion time: %ld ms \\n\", (conversion_end_time - decoder_end_time) / 1000);\n GGML_LOG_INFO(\" - Graph compile time: %ld ms \\n\", (compile_end_time - conversion_end_time) / 1000);\n }\n GGML_LOG_INFO(\" - Graph inference time: %ld ms \\n\", (infer_end_time - compile_end_time) / 1000);\n }\n\n return GGML_STATUS_SUCCESS;\n}\n\nbool is_naive(ggml_cgraph * cgraph) {\n constexpr int naive_graph_size_threshold = 20;\n int count = 0;\n for (int i = 0; i < cgraph->n_nodes; i++) {\n if (cgraph->nodes[i]->op != GGML_OP_NONE) {\n count++;\n }\n }\n return count < naive_graph_size_threshold;\n}\n\nenum ggml_status naive_compute(ggml_cgraph * cgraph,\n ov::Core & core,\n const std::string & device,\n const ov::AnyMap & config) {\n if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) {\n return GGML_STATUS_SUCCESS;\n }\n\n bool naive = true;\n auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive);\n auto decoder = std::make_shared(cgraph, model_weights);\n auto input_model = std::make_shared(decoder);\n auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive);\n if (getenv(\"GGML_OPENVINO_DUMP_IR\")) {\n ov::serialize(model, \"IR_naive.xml\");\n }\n\n std::shared_ptr infer_request;\n auto remote_context = ggml_openvino_get_remote_context();\n if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) {\n // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved\n core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::PERFORMANCE));\n } else {\n core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY));\n }\n if (remote_context.has_value()) {\n infer_request = std::make_shared(\n core.compile_model(model, remote_context.value(), config).create_infer_request());\n } else {\n infer_request =\n std::make_shared(core.compile_model(model, device, config).create_infer_request());\n }\n\n auto ov_params = model->get_parameters();\n for (size_t i = 0; i < ov_params.size(); i++) {\n auto param_name = ov_params[i]->get_friendly_name();\n auto input_tensor = get_ov_input_tensor(decoder, param_name);\n infer_request->set_input_tensor(i, input_tensor);\n }\n\n auto ov_results = model->get_results();\n for (size_t i = 0; i < ov_results.size(); i++) {\n auto * ggml_tensor = decoder->get_model_outputs().at(ov_results[i]->get_friendly_name());\n auto output_tensor = create_ov_output_tensor(decoder, infer_request, i, ggml_tensor);\n infer_request->set_output_tensor(i, output_tensor);\n }\n\n infer_request->infer();\n return GGML_STATUS_SUCCESS;\n}\n\nnamespace {\nov::Tensor convert_ggml_input_to_ov(std::shared_ptr ggml_decoder, const std::string & name) {\n const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name);\n\n if (ggml_tensor->extra != nullptr) {\n // GGML_LOG_DEBUG(\"Using ggml_tensor->extra as ov::Tensor for input: %s\\n\", name.c_str());\n auto * extra_base = static_cast(ggml_tensor->extra);\n if (extra_base->type != ggml_openvino_extra_base::Type::TENSOR) {\n throw std::runtime_error(\"ggml tensor extra is not of type TENSOR for input: \" + name);\n }\n auto * tensor_extra = static_cast(extra_base);\n return *tensor_extra->tensor;\n }\n\n // GGML_LOG_DEBUG(\"Converting ggml tensor to ov::Tensor for input: %s\\n\", name.c_str());\n auto * input_data = ggml_tensor->data;\n ov::Shape input_shape;\n if (ggml_tensor->op == GGML_OP_VIEW) {\n // This case is added to make test-backend-ops work\n input_shape = ggml_decoder->get_shape(ggml_tensor->view_src);\n } else {\n input_shape = ggml_decoder->get_shape(ggml_tensor);\n }\n auto input_tensor = ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape, input_data);\n return input_tensor;\n}\n} // namespace\n\nov::Tensor get_ov_input_tensor(std::shared_ptr ggml_decoder, const std::string & param_name) {\n ov::Tensor input_tensor;\n if (ggml_decoder->get_model_extra_inputs().find(param_name) != ggml_decoder->get_model_extra_inputs().end()) {\n input_tensor = *ggml_decoder->get_model_extra_input_values().at(param_name);\n } else {\n input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name);\n }\n return input_tensor;\n}\n\nov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr ggml_decoder,\n const std::string & param_name) {\n // NPU decoding stage\n const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name);\n const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);\n\n if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ||\n GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) {\n assert(ggml_tensor->ne[0] == 1);\n ov::Shape input_shape = {1, 1, 1, 1};\n ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);\n if (ggml_tensor->type == GGML_TYPE_I32) {\n *input_tensor.data() = *((int32_t *) ggml_tensor->data);\n } else if (ggml_tensor->type == GGML_TYPE_I64) {\n *input_tensor.data() = *((int64_t *) ggml_tensor->data);\n } else {\n throw std::runtime_error(\"Unexpected tensor type for \" + param_name);\n }\n return input_tensor;\n }\n\n if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) {\n ov::Shape input_shape = {1, 1, 1, 1};\n ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);\n int32_t inp_out_id = *((int32_t *) ggml_tensor->data);\n assert(ggml_tensor->ne[0] == 1);\n assert(inp_out_id == 0);\n *input_tensor.data() = inp_out_id;\n return input_tensor;\n }\n\n if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) {\n size_t context_size = ggml_decoder->get_ctx_size();\n std::vector padded_data = pad_input(ggml_tensor, 1, context_size, -INFINITY);\n ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size});\n auto * data_ptr = input_tensor.data();\n std::copy(padded_data.begin(), padded_data.begin() + context_size, data_ptr);\n return input_tensor;\n }\n\n return get_ov_input_tensor(ggml_decoder, param_name);\n}\n\nov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr ggml_decoder,\n const std::string & param_name,\n int chunk_index) {\n // NPU prompt processing stage\n const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name);\n const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);\n\n const size_t input_len = ggml_decoder->get_input_len();\n const size_t chunk_size = ggml_decoder->m_prefill_chunk_size;\n const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size);\n const size_t chunk_pad_size = chunk_size - chunk_valid_size;\n\n if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ||\n GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) {\n ov::Shape input_shape = {1, 1, 1, chunk_size};\n ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);\n // copy the chunk_index-th chunk from ggml_tensor\n size_t element_size = ggml_type_size(ggml_tensor->type);\n void * input_data = (char *) ggml_tensor->data + chunk_index * chunk_size * element_size;\n std::memcpy(input_tensor.data(), input_data, chunk_valid_size * element_size);\n // pad the rest with last_value + 1, so that kv's of padded positions are inserted\n // to the next row after the valids row in the kvcache\n if (chunk_pad_size > 0) {\n if (ggml_tensor->type == GGML_TYPE_I32) {\n int32_t last_value =\n *((int32_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1));\n int32_t * output_data = input_tensor.data();\n std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1);\n } else if (ggml_tensor->type == GGML_TYPE_I64) {\n int64_t last_value =\n *((int64_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1));\n int64_t * output_data = input_tensor.data();\n std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1);\n } else {\n throw std::runtime_error(\"Unexpected tensor type for \" + param_name);\n }\n }\n return input_tensor;\n }\n\n if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) {\n size_t output_len = ggml_decoder->get_compute_params().output_len;\n ov::Shape input_shape = {1, 1, 1, output_len};\n ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);\n if (ggml_tensor->ne[0] == 0) {\n *input_tensor.data() = 0;\n } else {\n auto * data_addr = input_tensor.data();\n for (size_t i = 0; i < output_len; i++) {\n data_addr[i] = ((int32_t *) ggml_tensor->data)[i] % chunk_size;\n }\n }\n return input_tensor;\n }\n\n if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) {\n size_t cols = ggml_tensor->ne[0];\n size_t rows = ggml_tensor->ne[1];\n float * ggml_data = (float *) ggml_tensor->data + chunk_index * chunk_size * cols;\n size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size);\n size_t context_size = ggml_decoder->get_ctx_size();\n std::vector padded_data =\n pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY);\n set_zero_diagonal(padded_data, chunk_size, context_size);\n ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, context_size});\n auto * data_ptr = input_tensor.data();\n std::copy(padded_data.begin(), padded_data.begin() + chunk_size * context_size, data_ptr);\n return input_tensor;\n }\n\n return get_ov_input_tensor(ggml_decoder, param_name);\n}\n\nsize_t checksum(const void * data, size_t size) {\n const uint8_t * bytes = static_cast(data);\n size_t sum = 0;\n for (size_t i = 0; i < size; ++i) {\n sum += (uint8_t) i;\n sum += bytes[i];\n }\n return sum;\n}\n\nvoid print_input_tensor_info(const std::string & name, const ov::Tensor & tensor) {\n std::cout << \"Input name: \" << name << \", Input shape: \" << tensor.get_shape() << \", Address: \" << tensor.data()\n << std::endl;\n switch (tensor.get_element_type()) {\n case ov::element::f32: {\n if (name.find(\"self_kq_mask\") == std::string::npos) {\n std::cout << *(tensor.data()) << std::endl;\n } else {\n size_t rows = tensor.get_shape()[2];\n size_t cols = tensor.get_shape()[3];\n auto * data = tensor.data();\n for (size_t i = 0; i < rows; ++i) {\n for (size_t j = 0; j < cols; ++j) {\n float val = data[i * cols + j];\n if (std::isinf(val) && val < 0) {\n std::cout << std::setw(5) << \"-inf\";\n } else {\n std::cout << std::setw(5) << val;\n }\n }\n std::cout << std::endl;\n }\n }\n\n break;\n }\n case ov::element::f16:\n std::cout << *(tensor.data()) << std::endl;\n break;\n case ov::element::i32:\n for (size_t i = 0; i < tensor.get_size(); ++i) {\n std::cout << tensor.data()[i] << \" \";\n }\n std::cout << std::endl;\n break;\n case ov::element::i64:\n for (size_t i = 0; i < tensor.get_size(); ++i) {\n std::cout << tensor.data()[i] << \" \";\n }\n std::cout << std::endl;\n break;\n default:\n break;\n }\n}\n\nvoid print_output_tensor_info(const std::string & name, const ov::Tensor & tensor, const void * output_dst) {\n std::cout << \"Output name: \" << name << \", Output shape: \" << tensor.get_shape() << \", Address: \" << output_dst\n << std::endl;\n\n auto print_float_stats = [](const std::string & type_name, size_t size, auto get_value) {\n if (size == 0) {\n return;\n }\n\n float first = get_value(0);\n float min = first;\n float max = first;\n double sum = first;\n\n for (size_t i = 1; i < size; ++i) {\n float v = get_value(i);\n if (v < min) {\n min = v;\n }\n if (v > max) {\n max = v;\n }\n sum += v;\n }\n double mean = sum / size;\n\n std::cout << std::right << std::setw(6) << type_name << std::right << std::setw(12) << \"First\" << std::setw(12)\n << \"Min\" << std::setw(12) << \"Max\" << std::setw(12) << \"Mean\" << std::endl;\n std::cout << std::right << std::setw(6) << \"\" << std::right << std::setw(12) << first << std::setw(12) << min\n << std::setw(12) << max << std::setw(12) << mean << std::endl;\n };\n\n switch (tensor.get_element_type()) {\n case ov::element::f32: {\n const float * data = tensor.data();\n size_t size = tensor.get_size();\n print_float_stats(\"[f32]\", size, [data](size_t i) { return data[i]; });\n break;\n }\n case ov::element::f16: {\n const ov::float16 * data = tensor.data();\n size_t size = tensor.get_size();\n print_float_stats(\"[f16]\", size, [data](size_t i) { return static_cast(data[i]); });\n break;\n }\n default:\n break;\n }\n}\n\nvoid set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols) {\n for (size_t i = 0; i < rows; ++i) {\n size_t diag_col = std::min(i, cols - 1);\n matrix[i * cols + diag_col] = 0.0f;\n }\n}\n\nconst ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) {\n for (int i = 0; i < cgraph->n_nodes; ++i) {\n auto * op = cgraph->nodes[i];\n for (int j = 0; j < GGML_MAX_SRC; ++j) {\n auto * src = op->src[j];\n if (src == nullptr) {\n break;\n }\n if (GgmlOvDecoder::is_inp_pos(src, op)) {\n return src;\n }\n }\n }\n GGML_LOG_ERROR(\"get_inp_pos_tensor: inp_pos not found in cgraph\");\n throw std::runtime_error(\"get_inp_pos_tensor: inp_pos not found in cgraph\");\n}\n\nbool get_is_prefill(const ggml_tensor * inp_pos) {\n return inp_pos->ne[0] > 1;\n}\n\n#pragma GCC diagnostic pop\n" }, { "path": "src/ggml-openvino/utils.h", "content": "#include \"ggml-backend-impl.h\"\n#include \"ggml-decoder.h\"\n#include \"ggml-impl.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nstruct graph_key {\n int n_nodes;\n std::string first_node_name;\n std::string last_node_name;\n\n graph_key(const ggml_cgraph * cgraph) : n_nodes(cgraph->n_nodes) {\n if (n_nodes > 0) {\n first_node_name = cgraph->nodes[0]->name;\n last_node_name = cgraph->nodes[n_nodes - 1]->name;\n }\n }\n\n bool operator==(const graph_key & other) const {\n return n_nodes == other.n_nodes && first_node_name == other.first_node_name &&\n last_node_name == other.last_node_name;\n }\n};\n\nstruct graph_key_hash {\n size_t operator()(const graph_key & key) const {\n size_t h = std::hash{}(key.n_nodes);\n if (key.n_nodes > 0) {\n h ^= std::hash{}(key.first_node_name) + 0x9e3779b9 + (h << 6) + (h >> 2);\n h ^= std::hash{}(key.last_node_name) + 0x9e3779b9 + (h << 6) + (h >> 2);\n }\n return h;\n }\n};\n\nstruct ov_runtime_context {\n std::mutex ov_compute_mutex;\n std::string device;\n bool stateful;\n std::unordered_map, graph_key_hash> decoder_cache;\n std::unordered_map, graph_key_hash> infer_request_cache;\n std::unordered_map, graph_key_hash> infer_request_cache_prefill;\n std::unordered_map, graph_key_hash> ov_input_names_cache;\n std::unordered_map, graph_key_hash> ov_output_names_cache;\n //TODO: Stateful is only supported for single request at a time.\n // Simultanous stateful inference request support to be added.\n size_t stateful_kv_size;\n std::map kv_state_input_name_map;\n\n ov_runtime_context() :\n device(\"CPU\"),\n stateful(false),\n stateful_kv_size(0) {}\n};\n\nenum ggml_status ov_graph_compute(struct ggml_cgraph * cgraph, ggml_backend_t backend);\n\nenum ggml_status ov_graph_compute_dynamic(struct ggml_cgraph * cgraph, std::shared_ptr r_ctx);\nenum ggml_status ov_graph_compute_static(struct ggml_cgraph * cgraph, std::shared_ptr r_ctx);\n\nsize_t checksum(const void * data, size_t size);\n\nvoid print_input_tensor_info(const std::string & name, const ov::Tensor & tensor);\n\nvoid print_output_tensor_info(const std::string & name, const ov::Tensor & tensor, const void * output_dst);\n\ntemplate \nstd::vector pad_input(const T * data,\n size_t rows,\n size_t cols,\n size_t padded_rows,\n size_t padded_cols,\n T pad_value) {\n std::vector padded(padded_rows * padded_cols, pad_value);\n\n for (size_t i = 0; i < std::min(rows, padded_rows); ++i) {\n for (size_t j = 0; j < std::min(cols, padded_cols); ++j) {\n padded[i * padded_cols + j] = data[i * cols + j];\n }\n }\n\n return padded;\n}\n\ntemplate \nstd::vector pad_input(const ggml_tensor * tensor, size_t padded_rows, size_t padded_cols, T pad_value) {\n return pad_input(reinterpret_cast(tensor->data),\n static_cast(tensor->ne[1]), // rows\n static_cast(tensor->ne[0]), // cols\n padded_rows, padded_cols, pad_value);\n}\n\nvoid set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols);\n\nconst ggml_tensor * get_inp_pos_tensor(struct ggml_cgraph * cgraph);\n\nbool get_is_prefill(const ggml_tensor * inp_pos);\n\nov::Tensor get_ov_input_tensor(std::shared_ptr ggml_decoder, const std::string & param_name);\nov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr ggml_decoder,\n const std::string & param_name);\nov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr ggml_decoder,\n const std::string & param_name,\n int chunk_index);\n\nov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder,\n std::shared_ptr infer_request,\n int output_index,\n const ggml_tensor * ggml_tensor);\n\nbool is_naive(struct ggml_cgraph * cgraph);\n\nenum ggml_status naive_compute(struct ggml_cgraph * cgraph,\n ov::Core & core,\n const std::string & device,\n const ov::AnyMap & config);\n" }, { "path": "src/ggml-opt.cpp", "content": "#include \"ggml-opt.h\"\n\n#include \"ggml.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-impl.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nstruct ggml_opt_dataset {\n struct ggml_context * ctx = nullptr;\n ggml_backend_buffer_t buf = nullptr;\n struct ggml_tensor * data = nullptr;\n struct ggml_tensor * labels = nullptr;\n\n int64_t ndata = -1;\n int64_t ndata_shard = -1;\n size_t nbs_data = -1;\n size_t nbs_labels = -1;\n\n std::vector permutation;\n};\n\nstruct ggml_opt_context {\n ggml_backend_sched_t backend_sched = nullptr;\n ggml_cgraph * allocated_graph = nullptr;\n ggml_cgraph * allocated_graph_copy = nullptr;\n struct ggml_context * ctx_static = nullptr;\n struct ggml_context * ctx_cpu = nullptr;\n struct ggml_context * ctx_compute = nullptr;\n struct ggml_context * ctx_copy = nullptr;\n ggml_backend_buffer_t buf_static = nullptr;\n ggml_backend_buffer_t buf_cpu = nullptr;\n std::mt19937 rng;\n enum ggml_opt_loss_type loss_type;\n enum ggml_opt_build_type build_type;\n enum ggml_opt_build_type build_type_alloc;\n\n struct ggml_tensor * inputs = nullptr;\n struct ggml_tensor * outputs = nullptr;\n struct ggml_tensor * labels = nullptr;\n\n struct ggml_tensor * loss = nullptr;\n struct ggml_tensor * pred = nullptr;\n struct ggml_tensor * ncorrect = nullptr;\n\n struct ggml_cgraph * gf = nullptr;\n struct ggml_cgraph * gb_grad = nullptr;\n struct ggml_cgraph * gb_opt = nullptr;\n bool static_graphs = false;\n bool eval_ready = false;\n std::vector grad_accs;\n std::vector grad_m;\n std::vector grad_v;\n\n int64_t iter = 1;\n int32_t opt_period = 1;\n int32_t opt_i = 0;\n bool loss_per_datapoint = false;\n\n ggml_opt_get_optimizer_params get_opt_pars = nullptr;\n void * get_opt_pars_ud = nullptr;\n struct ggml_tensor * opt_step_params = nullptr; // Stores output of get_opt_pars.\n\n enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;\n};\n\nstruct ggml_opt_result {\n int64_t ndata = 0;\n std::vector loss;\n std::vector pred;\n int64_t ncorrect = 0;\n\n int64_t opt_period = -1;\n bool loss_per_datapoint = false;\n};\n\n// ====== Dataset ======\n\nggml_opt_dataset_t ggml_opt_dataset_init(\n enum ggml_type type_data,\n enum ggml_type type_label,\n int64_t ne_datapoint,\n int64_t ne_label,\n int64_t ndata,\n int64_t ndata_shard) {\n GGML_ASSERT(ne_datapoint > 0);\n GGML_ASSERT(ne_label >= 0);\n GGML_ASSERT(ndata > 0);\n GGML_ASSERT(ndata_shard > 0);\n\n ggml_opt_dataset_t result = new ggml_opt_dataset;\n result->ndata = ndata;\n result->ndata_shard = ndata_shard;\n\n {\n struct ggml_init_params params = {\n /*.mem_size =*/ 2*ggml_tensor_overhead(),\n /*.mem_buffer =*/ nullptr,\n /*.no_alloc =*/ true,\n };\n result->ctx = ggml_init(params);\n }\n\n result->data = ggml_new_tensor_2d(result->ctx, type_data, ne_datapoint, ndata);\n result->nbs_data = ggml_nbytes(result->data) * ndata_shard/ndata;\n\n if (ne_label > 0) {\n result->labels = ggml_new_tensor_2d(result->ctx, type_label, ne_label, ndata);\n result->nbs_labels = ggml_nbytes(result->labels) * ndata_shard/ndata;\n } else {\n result->labels = nullptr;\n result->nbs_labels = 0;\n }\n\n result->buf = ggml_backend_alloc_ctx_tensors_from_buft(result->ctx, ggml_backend_cpu_buffer_type());\n\n const int64_t nshards = ndata/ndata_shard;\n result->permutation.resize(nshards);\n for (int64_t i = 0; i < nshards; ++i) {\n result->permutation[i] = i;\n }\n return result;\n}\n\nvoid ggml_opt_dataset_free(ggml_opt_dataset_t dataset) {\n ggml_backend_buffer_free(dataset->buf);\n ggml_free(dataset->ctx);\n delete dataset;\n}\n\nint64_t ggml_opt_dataset_ndata(ggml_opt_dataset_t dataset) {\n return dataset->ndata;\n}\n\nstruct ggml_tensor * ggml_opt_dataset_data(ggml_opt_dataset_t dataset) {\n return dataset->data;\n}\n\nstruct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset) {\n return dataset->labels;\n}\n\nvoid ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata) {\n GGML_ASSERT(idata <= dataset->ndata);\n\n if (idata < 0) {\n std::shuffle(dataset->permutation.begin(), dataset->permutation.end(), opt_ctx->rng);\n return;\n }\n\n GGML_ASSERT(idata % dataset->ndata_shard == 0);\n const int64_t ishard_max = idata / dataset->ndata_shard;\n std::shuffle(dataset->permutation.begin(), dataset->permutation.begin() + ishard_max, opt_ctx->rng);\n}\n\nvoid ggml_opt_dataset_get_batch(ggml_opt_dataset_t dataset, struct ggml_tensor * data_batch, struct ggml_tensor * labels_batch, int64_t ibatch) {\n GGML_ASSERT( data_batch && ggml_is_contiguous(data_batch));\n GGML_ASSERT(!labels_batch || ggml_is_contiguous(labels_batch));\n GGML_ASSERT((labels_batch == nullptr) == (dataset->labels == nullptr));\n GGML_ASSERT( data_batch->type == dataset->data->type);\n GGML_ASSERT(!labels_batch || labels_batch->type == dataset->labels->type);\n\n const size_t nb_data_batch = ggml_nbytes(data_batch);\n GGML_ASSERT(nb_data_batch % dataset->nbs_data == 0);\n const int64_t shards_per_batch = nb_data_batch / dataset->nbs_data;\n\n if (labels_batch) {\n const size_t nb_labels_batch = ggml_nbytes(labels_batch);\n GGML_ASSERT(nb_labels_batch == shards_per_batch*dataset->nbs_labels);\n }\n\n GGML_ASSERT((ibatch + 1)*shards_per_batch <= int64_t(dataset->permutation.size()));\n\n for (int64_t ishard_batch = 0; ishard_batch < shards_per_batch; ++ishard_batch) {\n const int64_t ishard = dataset->permutation[ibatch*shards_per_batch + ishard_batch];\n\n const char * ptr_data = (const char *) dataset->data->data + ishard*dataset->nbs_data;\n ggml_backend_tensor_set(data_batch, ptr_data, ishard_batch*dataset->nbs_data, dataset->nbs_data);\n\n if (!labels_batch) {\n continue;\n }\n\n const char * ptr_labels = (const char *) dataset->labels->data + ishard*dataset->nbs_labels;\n ggml_backend_tensor_set(labels_batch, ptr_labels, ishard_batch*dataset->nbs_labels, dataset->nbs_labels);\n }\n}\n\nvoid ggml_opt_dataset_get_batch_host(ggml_opt_dataset_t dataset, void * data_batch, size_t nb_data_batch, void * labels_batch, int64_t ibatch) {\n GGML_ASSERT((labels_batch == nullptr) == (dataset->labels == nullptr));\n GGML_ASSERT(nb_data_batch % dataset->nbs_data == 0);\n\n const int64_t shards_per_batch = nb_data_batch / dataset->nbs_data;\n\n GGML_ASSERT((ibatch + 1)*shards_per_batch <= int64_t(dataset->permutation.size()));\n\n for (int64_t ishard_batch = 0; ishard_batch < shards_per_batch; ++ishard_batch) {\n const int64_t ishard = dataset->permutation[ibatch*shards_per_batch + ishard_batch];\n\n const char * ptr_data = (const char *) dataset->data->data + ishard *dataset->nbs_data;\n char * ptr_data_batch = (char *) data_batch + ishard_batch*dataset->nbs_data;\n memcpy(ptr_data_batch, ptr_data, dataset->nbs_data);\n\n if (!labels_batch) {\n continue;\n }\n\n const char * ptr_labels = (const char *) dataset->labels->data + ishard *dataset->nbs_labels;\n char * ptr_labels_batch = (char *) labels_batch + ishard_batch*dataset->nbs_labels;\n memcpy(ptr_labels_batch, ptr_labels, dataset->nbs_labels);\n }\n}\n\n// ====== Model / Context ======\n\nstruct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata) {\n GGML_UNUSED(userdata);\n\n ggml_opt_optimizer_params result;\n\n result.adamw.alpha = 0.001f;\n result.adamw.beta1 = 0.9f;\n result.adamw.beta2 = 0.999f;\n result.adamw.eps = 1e-8f;\n result.adamw.wd = 0.0f;\n\n result.sgd.alpha = 1e-3f;\n result.sgd.wd = 0.0f;\n\n return result;\n}\n\n\nstruct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata) {\n return *((struct ggml_opt_optimizer_params *) userdata);\n}\n\nstruct ggml_opt_params ggml_opt_default_params(\n ggml_backend_sched_t backend_sched,\n enum ggml_opt_loss_type loss_type) {\n return {\n /*backend_sched =*/ backend_sched,\n /*ctx_compute =*/ nullptr,\n /*inputs =*/ nullptr,\n /*logits =*/ nullptr,\n /*loss_type =*/ loss_type,\n /*build_type =*/ GGML_OPT_BUILD_TYPE_OPT,\n /*opt_period =*/ 1,\n /*get_opt_pars =*/ ggml_opt_get_default_optimizer_params,\n /*get_opt_pars_ud =*/ nullptr,\n /*optimizer =*/ GGML_OPT_OPTIMIZER_TYPE_ADAMW,\n };\n}\n\nstatic ggml_tensor * map_tensor(std::map & tensor_map, ggml_context * ctx, ggml_tensor * tensor) {\n if (!tensor) {\n return nullptr;\n }\n\n if (tensor_map.find(tensor) != tensor_map.end()) {\n return tensor_map[tensor];\n }\n\n ggml_tensor * new_tensor = ggml_dup_tensor(ctx, tensor);\n tensor_map[tensor] = new_tensor;\n\n new_tensor->op = tensor->op;\n for (int i = 0; i < GGML_MAX_DIMS; i++) {\n new_tensor->nb[i] = tensor->nb[i];\n }\n new_tensor->flags = tensor->flags;\n memcpy(new_tensor->op_params, tensor->op_params, sizeof(tensor->op_params));\n strcpy(new_tensor->name, tensor->name);\n new_tensor->data = tensor->data;\n new_tensor->buffer = tensor->buffer;\n new_tensor->extra = tensor->extra;\n new_tensor->view_offs = tensor->view_offs;\n new_tensor->view_src = map_tensor(tensor_map, ctx, tensor->view_src);\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n new_tensor->src[i] = map_tensor(tensor_map, ctx, tensor->src[i]);\n }\n\n return new_tensor;\n}\n\nstatic ggml_cgraph * dup_graph(ggml_context * ctx, ggml_cgraph * src) {\n std::map tensor_map;\n\n ggml_cgraph * dst = ggml_new_graph_custom(ctx, src->size, /*grads =*/ true);\n\n for (int i = 0; i < src->n_leafs; i++) {\n ggml_build_forward_expand(dst, map_tensor(tensor_map, ctx, src->leafs[i]));\n }\n GGML_ASSERT(dst->n_leafs == src->n_leafs);\n for (int i = 0; i < src->n_nodes; i++) {\n ggml_build_forward_expand(dst, map_tensor(tensor_map, ctx, src->nodes[i]));\n }\n GGML_ASSERT(dst->n_nodes == src->n_nodes);\n for (int i = 0; i < src->n_nodes; ++i) {\n const size_t igrad_src = ggml_hash_find(&src->visited_hash_set, src->nodes[i]);\n const size_t igrad_dst = ggml_hash_find(&dst->visited_hash_set, dst->nodes[i]);\n\n GGML_ASSERT(igrad_src != GGML_HASHSET_FULL);\n GGML_ASSERT(ggml_bitset_get(src->visited_hash_set.used, igrad_src));\n GGML_ASSERT(igrad_dst != GGML_HASHSET_FULL);\n GGML_ASSERT(ggml_bitset_get(dst->visited_hash_set.used, igrad_dst));\n\n dst->grads[igrad_dst] = src->grads[igrad_src];\n dst->grad_accs[igrad_dst] = src->grad_accs[igrad_src];\n }\n\n return dst;\n}\n\nstatic void ggml_opt_build(ggml_opt_context_t opt_ctx) {\n GGML_ASSERT(opt_ctx->ctx_compute && \"no compute context set, either use static graphs or set one with ggml_opt_prepare_alloc\");\n GGML_ASSERT((!opt_ctx->static_graphs || opt_ctx->inputs->data) && \"when using static graphs the inputs must be allocated statically\");\n\n const enum ggml_opt_optimizer_type optimizer = opt_ctx->optimizer;\n\n const bool accumulate = opt_ctx->build_type_alloc >= GGML_OPT_BUILD_TYPE_GRAD &&\n !(opt_ctx->static_graphs && opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_OPT && opt_ctx->opt_period == 1);\n\n const bool need_momenta = opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_OPT &&\n opt_ctx->optimizer == GGML_OPT_OPTIMIZER_TYPE_ADAMW;\n\n ggml_set_input(opt_ctx->inputs);\n ggml_set_output(opt_ctx->outputs);\n\n int n_param = 0;\n for (int i = 0; i < opt_ctx->gf->n_nodes; ++i) {\n const struct ggml_tensor * node = opt_ctx->gf->nodes[i];\n if (node->flags & GGML_TENSOR_FLAG_PARAM) {\n n_param++;\n }\n GGML_ASSERT(!(node->flags & GGML_TENSOR_FLAG_LOSS) && \"support for extra loss terms not implemented\");\n }\n\n if (!opt_ctx->ctx_static) {\n // The static context is used for:\n // - gradients (1 per loss, 1 tensor per param if using gradient accumulation)\n // - optimizer momenta (2 tensors per param)\n // - labels (if using static graphs)\n // - loss (if using static graphs, up to 5 tensors)\n // - pred (if using static graphs)\n // - ncorrect (if using static graphs, 2 tensors).\n constexpr size_t n_loss = 1;\n const size_t tensors_per_param = (accumulate ? 1 : 0) + (need_momenta ? 2 : 0);\n const size_t tensors_const = opt_ctx->static_graphs ? 9 : 0;\n const size_t size_meta = (n_loss + tensors_per_param*n_param + tensors_const) * ggml_tensor_overhead();\n struct ggml_init_params params = {\n /*.mem_size =*/ size_meta,\n /*.mem_buffer =*/ nullptr,\n /*.no_alloc =*/ true,\n };\n opt_ctx->ctx_static = ggml_init(params);\n }\n GGML_ASSERT(opt_ctx->build_type <= opt_ctx->build_type_alloc);\n\n {\n // The cpu context is allocated statically if using static graphs, dynamically otherwise.\n // It is used for:\n // - optimizer parameters (1 shared for all optimizer invocations)\n const size_t size_meta = 1 * ggml_tensor_overhead();\n struct ggml_init_params params = {\n /*.mem_size =*/ size_meta,\n /*.mem_buffer =*/ nullptr,\n /*.no_alloc =*/ true,\n };\n ggml_free(opt_ctx->ctx_cpu);\n opt_ctx->ctx_cpu = ggml_init(params);\n\n ggml_backend_buffer_free(opt_ctx->buf_cpu);\n opt_ctx->buf_cpu = nullptr;\n }\n\n struct ggml_context * ctx_results = opt_ctx->static_graphs ? opt_ctx->ctx_static : opt_ctx->ctx_compute;\n\n switch (opt_ctx->loss_type) {\n case GGML_OPT_LOSS_TYPE_MEAN: {\n opt_ctx->loss = ggml_sum(ctx_results, opt_ctx->outputs);\n ggml_set_name(opt_ctx->loss, \"loss_sum\");\n const float scale = 1.0f / (opt_ctx->opt_period * ggml_nelements(opt_ctx->outputs));\n opt_ctx->loss = ggml_scale(ctx_results, opt_ctx->loss, scale);\n ggml_set_name(opt_ctx->loss, \"loss_mean\");\n opt_ctx->loss_per_datapoint = true;\n break;\n }\n case GGML_OPT_LOSS_TYPE_SUM: {\n opt_ctx->loss = ggml_sum(ctx_results, opt_ctx->outputs);\n ggml_set_name(opt_ctx->loss, \"loss_sum\");\n opt_ctx->loss_per_datapoint = false;\n break;\n }\n case GGML_OPT_LOSS_TYPE_CROSS_ENTROPY: {\n opt_ctx->labels = ggml_dup_tensor(ctx_results, opt_ctx->outputs);\n ggml_set_input(opt_ctx->labels);\n ggml_set_name(opt_ctx->labels, \"labels\");\n opt_ctx->loss = ggml_cross_entropy_loss(ctx_results, opt_ctx->outputs, opt_ctx->labels);\n ggml_set_name(opt_ctx->loss, \"loss_cross_entropy\");\n if (opt_ctx->opt_period > 1) {\n opt_ctx->loss = ggml_scale(ctx_results, opt_ctx->loss, 1.0f / opt_ctx->opt_period);\n ggml_set_name(opt_ctx->loss, \"loss_cross_entropy_scaled\");\n }\n opt_ctx->loss_per_datapoint = true;\n break;\n }\n case GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR: {\n opt_ctx->labels = ggml_dup_tensor(ctx_results, opt_ctx->outputs);\n ggml_set_input(opt_ctx->labels);\n ggml_set_name(opt_ctx->labels, \"labels\");\n opt_ctx->loss = ggml_sub(ctx_results, opt_ctx->outputs, opt_ctx->labels);\n ggml_set_name(opt_ctx->loss, \"loss_error\");\n opt_ctx->loss = ggml_sqr(ctx_results, opt_ctx->loss);\n ggml_set_name(opt_ctx->loss, \"loss_squared_error\");\n opt_ctx->loss = ggml_sum(ctx_results, opt_ctx->loss);\n ggml_set_name(opt_ctx->loss, \"loss_sum_squared_error\");\n const float scale = 1.0f / (opt_ctx->opt_period * ggml_nelements(opt_ctx->outputs));\n opt_ctx->loss = ggml_scale(ctx_results, opt_ctx->loss, scale);\n ggml_set_name(opt_ctx->loss, \"loss_mean_squared_error\");\n opt_ctx->loss_per_datapoint = true;\n break;\n }\n }\n ggml_set_output(opt_ctx->loss);\n ggml_set_loss(opt_ctx->loss);\n ggml_build_forward_expand(opt_ctx->gf, opt_ctx->loss);\n\n if (opt_ctx->loss_type == GGML_OPT_LOSS_TYPE_CROSS_ENTROPY) {\n opt_ctx->pred = ggml_argmax(ctx_results, opt_ctx->outputs);\n ggml_set_name(opt_ctx->pred, \"pred\");\n ggml_set_output(opt_ctx->pred);\n ggml_build_forward_expand(opt_ctx->gf, opt_ctx->pred);\n\n opt_ctx->ncorrect = ggml_count_equal(ctx_results, opt_ctx->pred, ggml_argmax(ctx_results, opt_ctx->labels));\n ggml_set_name(opt_ctx->ncorrect, \"ncorrect\");\n ggml_set_output(opt_ctx->ncorrect);\n ggml_build_forward_expand(opt_ctx->gf, opt_ctx->ncorrect);\n }\n\n if (opt_ctx->buf_static) {\n if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_FORWARD) {\n return;\n }\n } else if (opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_FORWARD) {\n opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors(\n opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0));\n return;\n }\n\n if (opt_ctx->grad_accs.empty()) {\n GGML_ASSERT(opt_ctx->build_type_alloc >= GGML_OPT_BUILD_TYPE_GRAD);\n\n const int n_nodes = opt_ctx->gf->n_nodes;\n opt_ctx->grad_accs.resize(n_nodes);\n for (int i = 0; i < n_nodes; ++i) {\n ggml_tensor * node = opt_ctx->gf->nodes[i];\n if ((accumulate && (node->flags & GGML_TENSOR_FLAG_PARAM)) || (node->flags & GGML_TENSOR_FLAG_LOSS)) {\n opt_ctx->grad_accs[i] = ggml_new_tensor(opt_ctx->ctx_static, GGML_TYPE_F32, GGML_MAX_DIMS, node->ne);\n } else {\n opt_ctx->grad_accs[i] = nullptr;\n }\n }\n\n if (need_momenta && opt_ctx->build_type_alloc >= GGML_OPT_BUILD_TYPE_OPT) {\n opt_ctx->grad_m.resize(n_nodes);\n opt_ctx->grad_v.resize(n_nodes);\n for (int i = 0; i < n_nodes; ++i) {\n ggml_tensor * node = opt_ctx->gf->nodes[i];\n if (node->flags & GGML_TENSOR_FLAG_PARAM) {\n opt_ctx->grad_m[i] = ggml_new_tensor(opt_ctx->ctx_static, GGML_TYPE_F32, GGML_MAX_DIMS, node->ne);\n opt_ctx->grad_v[i] = ggml_new_tensor(opt_ctx->ctx_static, GGML_TYPE_F32, GGML_MAX_DIMS, node->ne);\n } else {\n opt_ctx->grad_m[i] = nullptr;\n opt_ctx->grad_v[i] = nullptr;\n }\n }\n }\n }\n\n // gb_grad == graph backward gradients, forward pass, then backward pass to calculate gradients.\n opt_ctx->gb_grad = ggml_graph_dup(opt_ctx->ctx_compute, opt_ctx->gf, /*force_grads =*/ true);\n ggml_build_backward_expand(opt_ctx->ctx_compute, opt_ctx->gb_grad, opt_ctx->grad_accs.data());\n\n if (opt_ctx->buf_static) {\n if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_GRAD) {\n return;\n }\n } else if (opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_GRAD) {\n opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors(opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0));\n ggml_graph_reset(opt_ctx->gb_grad);\n }\n\n GGML_ASSERT(opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_OPT);\n\n // gb_opt == graph backward optimize, forward pass, then backward pass to calculate gradients, then optimizer step.\n opt_ctx->gb_opt = ggml_graph_dup(opt_ctx->ctx_compute, opt_ctx->gb_grad, /*force_grads =*/ true);\n\n opt_ctx->opt_step_params = ggml_new_tensor_1d(opt_ctx->ctx_cpu, GGML_TYPE_F32, need_momenta ? 7 : 2);\n ggml_tensor * adamw_params = opt_ctx->opt_step_params;\n ggml_set_input(adamw_params);\n const char * optimizer_name = ggml_opt_optimizer_name(opt_ctx->optimizer);\n ggml_format_name(adamw_params, \"%s_params\", optimizer_name);\n for (int i = opt_ctx->gf->n_nodes-1; i >= 0; --i) {\n struct ggml_tensor * node = opt_ctx->gb_opt->nodes[i];\n struct ggml_tensor * grad = ggml_graph_get_grad(opt_ctx->gb_opt, node);\n\n if (grad && (node->flags & GGML_TENSOR_FLAG_PARAM)) {\n struct ggml_tensor * m = nullptr;\n struct ggml_tensor * v = nullptr;\n if (need_momenta) {\n m = opt_ctx->grad_m[i];\n v = opt_ctx->grad_v[i];\n ggml_format_name(m, \"AdamW m for %s\", node->name);\n ggml_format_name(v, \"AdamW v for %s\", node->name);\n }\n struct ggml_tensor * opt_step;\n switch (optimizer) {\n case GGML_OPT_OPTIMIZER_TYPE_ADAMW:\n opt_step = ggml_opt_step_adamw(opt_ctx->ctx_compute, node, grad, m, v, adamw_params);\n break;\n case GGML_OPT_OPTIMIZER_TYPE_SGD:\n opt_step = ggml_opt_step_sgd(opt_ctx->ctx_compute, node, grad, adamw_params);\n break;\n default:\n GGML_ABORT(\"fatal error\");\n }\n ggml_format_name(opt_step, \"%s step for %s\", optimizer_name, node->name);\n ggml_build_forward_expand(opt_ctx->gb_opt, opt_step);\n }\n }\n\n if (!opt_ctx->buf_static) {\n opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors(\n opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0));\n ggml_graph_reset(opt_ctx->gb_opt);\n }\n\n opt_ctx->buf_cpu = ggml_backend_alloc_ctx_tensors_from_buft(opt_ctx->ctx_cpu, ggml_backend_cpu_buffer_type());\n}\n\nggml_opt_context_t ggml_opt_init(struct ggml_opt_params params) {\n ggml_opt_context_t result = new struct ggml_opt_context;\n result->backend_sched = params.backend_sched;\n result->ctx_compute = params.ctx_compute;\n result->loss_type = params.loss_type;\n result->build_type = params.build_type;\n result->build_type_alloc = params.build_type;\n result->inputs = params.inputs;\n result->outputs = params.outputs;\n result->opt_period = params.opt_period;\n result->get_opt_pars = params.get_opt_pars;\n result->get_opt_pars_ud = params.get_opt_pars_ud;\n result->optimizer = params.optimizer;\n\n GGML_ASSERT(result->opt_period >= 1);\n\n result->static_graphs = result->ctx_compute;\n\n if (!result->static_graphs) {\n GGML_ASSERT(!result->inputs);\n GGML_ASSERT(!result->outputs);\n return result;\n }\n\n GGML_ASSERT(result->inputs);\n GGML_ASSERT(result->outputs);\n\n result->gf = ggml_new_graph_custom(result->ctx_compute, GGML_DEFAULT_GRAPH_SIZE, /*grads =*/ true); // Forward pass.\n ggml_build_forward_expand(result->gf, result->outputs);\n\n ggml_opt_build(result);\n\n return result;\n}\n\nvoid ggml_opt_free(ggml_opt_context_t opt_ctx) {\n if (opt_ctx == nullptr) {\n return;\n }\n ggml_backend_buffer_free(opt_ctx->buf_static);\n ggml_backend_buffer_free(opt_ctx->buf_cpu);\n ggml_free(opt_ctx->ctx_static);\n ggml_free(opt_ctx->ctx_cpu);\n delete opt_ctx;\n}\n\nvoid ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer) {\n if (optimizer) {\n ggml_graph_reset(opt_ctx->gb_opt);\n opt_ctx->iter = 1;\n } else {\n ggml_graph_reset(opt_ctx->gb_grad);\n }\n}\n\nbool ggml_opt_static_graphs(ggml_opt_context_t opt_ctx) {\n return opt_ctx->static_graphs;\n}\n\nstruct ggml_tensor * ggml_opt_inputs(ggml_opt_context_t opt_ctx) {\n return opt_ctx->inputs;\n}\n\nstruct ggml_tensor * ggml_opt_outputs(ggml_opt_context_t opt_ctx) {\n return opt_ctx->outputs;\n}\n\nstruct ggml_tensor * ggml_opt_labels(ggml_opt_context_t opt_ctx) {\n return opt_ctx->labels;\n}\n\nstruct ggml_tensor * ggml_opt_loss(ggml_opt_context_t opt_ctx) {\n return opt_ctx->loss;\n}\n\nstruct ggml_tensor * ggml_opt_pred(ggml_opt_context_t opt_ctx) {\n return opt_ctx->pred;\n}\n\nstruct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx) {\n return opt_ctx->ncorrect;\n}\n\nstruct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node) {\n return ggml_graph_get_grad_acc(opt_ctx->gb_opt, node);\n}\n\n// ====== Optimization Result ======\n\nggml_opt_result_t ggml_opt_result_init() {\n return new ggml_opt_result;\n}\n\nvoid ggml_opt_result_free(ggml_opt_result_t result) {\n delete result;\n}\n\nvoid ggml_opt_result_reset(ggml_opt_result_t result) {\n result->ndata = 0;\n result->loss.clear();\n result->pred.clear();\n result->ncorrect = 0;\n}\n\nvoid ggml_opt_result_ndata(ggml_opt_result_t result, int64_t * ndata) {\n *ndata = result->ndata;\n}\n\nvoid ggml_opt_result_loss(ggml_opt_result_t result, double * loss, double * unc) {\n const int64_t nbatches = result->loss.size(); // Number of physical batches.\n\n if (nbatches == 0) {\n *loss = 0.0;\n *unc = NAN;\n return;\n }\n\n double sum = 0.0;\n double sum_squared = 0.0;\n\n for (const float & loss : result->loss) {\n // If the loss is per datapoint it was scaled by 1.0f/opt_period for each physical batch.\n const float loss_scaled = result->loss_per_datapoint ? loss*result->opt_period : loss;\n sum += loss_scaled;\n sum_squared += loss_scaled*loss_scaled;\n }\n\n const double mean = sum/nbatches;\n *loss = result->loss_per_datapoint ? mean : sum;\n\n if (!unc) {\n return;\n }\n\n if (nbatches < 2) {\n *unc = NAN;\n return;\n }\n\n const double var_sum = sum_squared/nbatches - mean*mean; // variance without Bessel's correction, i.e. nbatches/(nbatches-1)\n *unc = result->loss_per_datapoint ? sqrt(var_sum / (nbatches - 1)) : sqrt(var_sum * nbatches/(nbatches - 1));\n}\n\nvoid ggml_opt_result_pred(ggml_opt_result_t result, int32_t * pred) {\n for (size_t i = 0; i < result->pred.size(); ++i) {\n pred[i] = result->pred[i];\n }\n}\n\nvoid ggml_opt_result_accuracy(ggml_opt_result_t result, double * accuracy, double * unc) {\n *accuracy = result->ncorrect >= 0 ? double(result->ncorrect) / double(result->ndata) : NAN;\n\n if (!unc) {\n return;\n }\n\n *unc = result->ncorrect >= 0 && result->ndata >= 2 ?\n sqrt((*accuracy) * (1.0 - (*accuracy)) / double(result->ndata - 1)) : NAN;\n}\n\n// ====== Computation ======\n\nvoid ggml_opt_prepare_alloc(\n ggml_opt_context_t opt_ctx,\n struct ggml_context * ctx_compute,\n struct ggml_cgraph * gf,\n struct ggml_tensor * inputs,\n struct ggml_tensor * outputs) {\n GGML_ASSERT(!opt_ctx->static_graphs);\n opt_ctx->ctx_compute = ctx_compute;\n opt_ctx->gf = gf;\n opt_ctx->inputs = inputs;\n opt_ctx->outputs = outputs;\n}\n\nvoid ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward) {\n GGML_ASSERT(!opt_ctx->eval_ready);\n if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_OPT && opt_ctx->opt_period > 1 && opt_ctx->opt_i == 0) {\n ggml_graph_reset(opt_ctx->gb_grad);\n }\n if (backward) {\n const int32_t opt_i_next = (opt_ctx->opt_i + 1) % opt_ctx->opt_period;\n opt_ctx->build_type = opt_i_next == 0 ? GGML_OPT_BUILD_TYPE_OPT : GGML_OPT_BUILD_TYPE_GRAD;\n } else {\n opt_ctx->build_type = GGML_OPT_BUILD_TYPE_FORWARD;\n }\n\n if (!opt_ctx->static_graphs) {\n ggml_opt_build(opt_ctx);\n }\n\n struct ggml_cgraph * graph = nullptr;\n switch (opt_ctx->build_type) {\n case GGML_OPT_BUILD_TYPE_FORWARD: {\n graph = opt_ctx->gf;\n } break;\n case GGML_OPT_BUILD_TYPE_GRAD: {\n graph = opt_ctx->gb_grad;\n } break;\n case GGML_OPT_BUILD_TYPE_OPT: {\n graph = opt_ctx->gb_opt;\n } break;\n }\n GGML_ASSERT(graph);\n\n if (opt_ctx->allocated_graph == graph) {\n opt_ctx->eval_ready = true;\n return;\n }\n\n ggml_backend_sched_reset(opt_ctx->backend_sched); // clear allocation of previous graph\n\n if (opt_ctx->static_graphs) {\n ggml_init_params params = {\n /*.mem_size =*/ graph->size*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph->size, graph->grads),\n /*.mem_buffer =*/ nullptr,\n /*.no_alloc =*/ true,\n };\n ggml_free(opt_ctx->ctx_copy);\n opt_ctx->ctx_copy = ggml_init(params);\n\n opt_ctx->allocated_graph_copy = dup_graph(opt_ctx->ctx_copy, graph);\n } else {\n opt_ctx->allocated_graph_copy = graph;\n }\n\n ggml_backend_sched_alloc_graph(opt_ctx->backend_sched, opt_ctx->allocated_graph_copy);\n opt_ctx->allocated_graph = graph;\n\n opt_ctx->eval_ready = true;\n}\n\nvoid ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result) {\n GGML_ASSERT(opt_ctx->eval_ready);\n if (opt_ctx->allocated_graph == opt_ctx->gb_opt) {\n const ggml_opt_optimizer_params & opt_pars = opt_ctx->get_opt_pars(opt_ctx->get_opt_pars_ud);\n\n switch (opt_ctx->optimizer) {\n case GGML_OPT_OPTIMIZER_TYPE_ADAMW: {\n GGML_ASSERT(opt_pars.adamw.alpha > 0.0f);\n GGML_ASSERT(opt_pars.adamw.beta1 >= 0.0f);\n GGML_ASSERT(opt_pars.adamw.beta1 <= 1.0f);\n GGML_ASSERT(opt_pars.adamw.beta2 >= 0.0f);\n GGML_ASSERT(opt_pars.adamw.beta2 <= 1.0f);\n GGML_ASSERT(opt_pars.adamw.eps >= 0.0f);\n GGML_ASSERT(opt_pars.adamw.wd >= 0.0f);\n GGML_ASSERT(opt_pars.adamw.wd <= 1.0f);\n\n // beta1, beta2 after applying warmup\n const float beta1h = 1.0f / (1.0f - powf(opt_pars.adamw.beta1, opt_ctx->iter));\n const float beta2h = 1.0f / (1.0f - powf(opt_pars.adamw.beta2, opt_ctx->iter));\n\n float * adamw_par_data = ggml_get_data_f32(opt_ctx->opt_step_params);\n adamw_par_data[0] = opt_pars.adamw.alpha;\n adamw_par_data[1] = opt_pars.adamw.beta1;\n adamw_par_data[2] = opt_pars.adamw.beta2;\n adamw_par_data[3] = opt_pars.adamw.eps;\n adamw_par_data[4] = opt_pars.adamw.wd;\n adamw_par_data[5] = beta1h;\n adamw_par_data[6] = beta2h;\n } break;\n case GGML_OPT_OPTIMIZER_TYPE_SGD: {\n GGML_ASSERT(opt_pars.sgd.alpha > 0.0f);\n GGML_ASSERT(opt_pars.sgd.wd >= 0.0f);\n GGML_ASSERT(opt_pars.sgd.wd <= 1.0f);\n float * sgd = ggml_get_data_f32(opt_ctx->opt_step_params);\n sgd[0] = opt_pars.sgd.alpha;\n sgd[1] = opt_pars.sgd.wd;\n } break;\n default:\n GGML_ABORT(\"fatal error\");\n }\n }\n\n ggml_backend_sched_graph_compute(opt_ctx->backend_sched, opt_ctx->allocated_graph_copy);\n opt_ctx->iter += opt_ctx->allocated_graph == opt_ctx->gb_opt;\n opt_ctx->opt_i = (opt_ctx->opt_i + 1) % opt_ctx->opt_period;\n\n if (!opt_ctx->static_graphs) {\n opt_ctx->gf = nullptr;\n opt_ctx->gb_grad = nullptr;\n opt_ctx->gb_opt = nullptr;\n opt_ctx->allocated_graph = nullptr;\n opt_ctx->allocated_graph_copy = nullptr;\n }\n\n opt_ctx->eval_ready = false;\n\n if (!result) {\n return;\n }\n\n if (result->ndata == 0) {\n result->loss_per_datapoint = opt_ctx->loss_per_datapoint;\n result->opt_period = opt_ctx->opt_period;\n } else {\n GGML_ASSERT(result->loss_per_datapoint == opt_ctx->loss_per_datapoint);\n GGML_ASSERT(result->opt_period == opt_ctx->opt_period);\n }\n\n const int64_t ndata = opt_ctx->outputs->ne[1];\n GGML_ASSERT(result->ndata == ndata*int64_t(result->loss.size()) && \"varying batch size not supported\");\n result->ndata += ndata;\n\n GGML_ASSERT(ggml_is_scalar(opt_ctx->loss));\n GGML_ASSERT(opt_ctx->loss->type == GGML_TYPE_F32);\n float loss;\n ggml_backend_tensor_get(opt_ctx->loss, &loss, 0, ggml_nbytes(opt_ctx->loss));\n result->loss.push_back(loss);\n\n if (opt_ctx->pred) {\n GGML_ASSERT(opt_ctx->pred->type == GGML_TYPE_I32);\n std::vector pred(ndata);\n ggml_backend_tensor_get(opt_ctx->pred, pred.data(), 0, ggml_nbytes(opt_ctx->pred));\n result->pred.insert(result->pred.end(), pred.begin(), pred.end());\n }\n\n if (!opt_ctx->ncorrect || result->ncorrect < 0) {\n result->ncorrect = -1;\n return;\n }\n\n GGML_ASSERT(ggml_is_scalar(opt_ctx->ncorrect));\n GGML_ASSERT(opt_ctx->ncorrect->type == GGML_TYPE_I64);\n int64_t ncorrect;\n ggml_backend_tensor_get(opt_ctx->ncorrect, &ncorrect, 0, ggml_nbytes(opt_ctx->ncorrect));\n result->ncorrect += ncorrect;\n}\n\n// ====== High-Level Functions ======\n\nvoid ggml_opt_epoch(\n ggml_opt_context_t opt_ctx,\n ggml_opt_dataset_t dataset,\n ggml_opt_result_t result_train,\n ggml_opt_result_t result_eval,\n int64_t idata_split,\n ggml_opt_epoch_callback callback_train,\n ggml_opt_epoch_callback callback_eval) {\n GGML_ASSERT(ggml_opt_static_graphs(opt_ctx) && \"ggml_opt_epoch requires static graphs\");\n struct ggml_tensor * inputs = ggml_opt_inputs(opt_ctx);\n struct ggml_tensor * labels = ggml_opt_labels(opt_ctx);\n struct ggml_tensor * data = ggml_opt_dataset_data(dataset);\n GGML_ASSERT(data->ne[0] == inputs->ne[0]);\n\n const int64_t ndata = data->ne[1];\n const int64_t ndata_batch = inputs->ne[1];\n\n GGML_ASSERT(data->ne[1] % inputs->ne[1] == 0);\n const int64_t nbatches = ndata/ndata_batch;\n\n idata_split = idata_split < 0 ? ndata : idata_split;\n GGML_ASSERT(idata_split % ndata_batch == 0);\n const int64_t ibatch_split = idata_split / ndata_batch;\n\n int64_t ibatch = 0;\n int64_t t_loop_start = ggml_time_us();\n for (; ibatch < ibatch_split; ++ibatch) {\n ggml_opt_alloc(opt_ctx, /*backward =*/ true);\n ggml_opt_dataset_get_batch(dataset, inputs, labels, ibatch);\n ggml_opt_eval(opt_ctx, result_train);\n if (callback_train) {\n callback_train(true, opt_ctx, dataset, result_train, ibatch+1, ibatch_split, t_loop_start);\n }\n }\n t_loop_start = ggml_time_us();\n for (; ibatch < nbatches; ++ibatch) {\n ggml_opt_alloc(opt_ctx, /*backward =*/ false);\n ggml_opt_dataset_get_batch(dataset, inputs, labels, ibatch);\n ggml_opt_eval(opt_ctx, result_eval);\n if (callback_eval) {\n callback_eval(false, opt_ctx, dataset, result_eval, ibatch+1-ibatch_split, nbatches-ibatch_split, t_loop_start);\n }\n }\n}\n\nvoid ggml_opt_epoch_callback_progress_bar(\n bool train,\n ggml_opt_context_t opt_ctx,\n ggml_opt_dataset_t dataset,\n ggml_opt_result_t result,\n int64_t ibatch,\n int64_t ibatch_max,\n int64_t t_start_us) {\n fprintf(stderr, \"%s[\", train ? \"train: \" : \"val: \");\n\n // The progress bar consists of partially filled blocks, unicode has 8 separate fill levels.\n constexpr int64_t bar_length = 8;\n const int64_t ibatch8 = 8 * ibatch;\n for (int64_t j = 0; j < bar_length; ++j) {\n if (ibatch_max * (8*j + 8) / bar_length < ibatch8) {\n fprintf(stderr, \"\\u2588\"); // full block\n } else if (ibatch_max * (8*j + 7) / bar_length < ibatch8) {\n fprintf(stderr, \"\\u2589\"); // 7/8 filled\n } else if (ibatch_max * (8*j + 6) / bar_length < ibatch8) {\n fprintf(stderr, \"\\u258A\"); // 6/8 filled\n } else if (ibatch_max * (8*j + 5) / bar_length < ibatch8) {\n fprintf(stderr, \"\\u258B\"); // 5/8 filled\n } else if (ibatch_max * (8*j + 4) / bar_length < ibatch8) {\n fprintf(stderr, \"\\u258C\"); // 4/8 filled\n } else if (ibatch_max * (8*j + 3) / bar_length < ibatch8) {\n fprintf(stderr, \"\\u258D\"); // 3/8 filled\n } else if (ibatch_max * (8*j + 2) / bar_length < ibatch8) {\n fprintf(stderr, \"\\u258E\"); // 2/8 filled\n } else if (ibatch_max * (8*j + 1) / bar_length < ibatch8) {\n fprintf(stderr, \"\\u258F\"); // 1/8 filled\n } else {\n fprintf(stderr, \" \");\n }\n }\n\n const int64_t batch_size = ggml_opt_inputs(opt_ctx)->ne[1];\n const int64_t idata = ibatch*batch_size;\n const int64_t idata_max = ibatch_max*batch_size;\n\n double loss;\n double loss_unc;\n ggml_opt_result_loss(result, &loss, &loss_unc);\n\n double accuracy;\n double accuracy_unc;\n ggml_opt_result_accuracy(result, &accuracy, &accuracy_unc);\n\n const int64_t t_ibatch_us = ggml_time_us() - t_start_us;\n int64_t t_ibatch_s = t_ibatch_us / 1000000;\n const int64_t t_ibatch_h = t_ibatch_s / 3600;\n t_ibatch_s -= t_ibatch_h * 3600;\n const int64_t t_ibatch_m = t_ibatch_s / 60;\n t_ibatch_s -= t_ibatch_m * 60;\n\n const int64_t t_eta_us = t_ibatch_us * (ibatch_max - ibatch)/ibatch;\n int64_t t_eta_s = t_eta_us / 1000000;\n const int64_t t_eta_h = t_eta_s / 3600;\n t_eta_s -= t_eta_h * 3600;\n const int64_t t_eta_m = t_eta_s / 60;\n t_eta_s -= t_eta_m * 60;\n\n fprintf(stderr, \"] data=%07\" PRId64 \"/%07\" PRId64 \" loss=%.5lf±%.5lf acc=%.2lf±%.2lf%% \"\n \"t=%02\" PRId64 \":%02\" PRId64 \":%02\" PRId64 \" ETA=%02\" PRId64 \":%02\" PRId64 \":%02\" PRId64 \" \\r\",\n idata, idata_max, loss, loss_unc, 100.0*accuracy, 100.0*accuracy_unc,\n t_ibatch_h, t_ibatch_m, t_ibatch_s, t_eta_h, t_eta_m, t_eta_s);\n if (ibatch == ibatch_max) {\n fprintf(stderr, \"\\n\");\n }\n fflush(stderr);\n\n GGML_UNUSED(dataset);\n}\n\nvoid ggml_opt_fit(\n ggml_backend_sched_t backend_sched,\n ggml_context * ctx_compute,\n ggml_tensor * inputs,\n ggml_tensor * outputs,\n ggml_opt_dataset_t dataset,\n enum ggml_opt_loss_type loss_type,\n enum ggml_opt_optimizer_type optimizer,\n ggml_opt_get_optimizer_params get_opt_pars,\n int64_t nepoch,\n int64_t nbatch_logical,\n float val_split,\n bool silent) {\n ggml_time_init();\n const int64_t t_start_us = ggml_time_us();\n\n const int64_t ndata = ggml_opt_dataset_data(dataset)->ne[1];\n const int64_t nbatch_physical = inputs->ne[1];\n GGML_ASSERT(ndata % nbatch_logical == 0);\n GGML_ASSERT(nbatch_logical % nbatch_physical == 0);\n\n const int64_t opt_period = nbatch_logical / nbatch_physical;\n const int64_t nbatches_logical = ndata / nbatch_logical;\n\n GGML_ASSERT(val_split >= 0.0f);\n GGML_ASSERT(val_split < 1.0f);\n const int64_t ibatch_split = int64_t(((1.0f - val_split) * nbatches_logical)) * opt_period; // train <-> val split index (physical)\n const int64_t idata_split = ibatch_split * nbatch_physical;\n\n int64_t epoch = 1;\n\n ggml_opt_params params = ggml_opt_default_params(backend_sched, loss_type);\n params.ctx_compute = ctx_compute;\n params.inputs = inputs;\n params.outputs = outputs;\n params.opt_period = opt_period;\n params.get_opt_pars = get_opt_pars;\n params.get_opt_pars_ud = &epoch;\n params.optimizer = optimizer;\n ggml_opt_context_t opt_ctx = ggml_opt_init(params);\n\n // Shuffling the data is generally useful but there is only a point if not all data is used in a single batch.\n if (nbatch_logical < ndata) {\n ggml_opt_dataset_shuffle(opt_ctx, dataset, -1); // Shuffle all data (train + validation).\n }\n\n ggml_opt_result_t result_train = ggml_opt_result_init();\n ggml_opt_result_t result_val = ggml_opt_result_init();\n\n ggml_opt_epoch_callback epoch_callback = silent ? nullptr : ggml_opt_epoch_callback_progress_bar;\n\n for (; epoch <= nepoch; ++epoch) {\n if (nbatch_logical < idata_split) {\n ggml_opt_dataset_shuffle(opt_ctx, dataset, idata_split);\n }\n\n ggml_opt_result_reset(result_train);\n ggml_opt_result_reset(result_val);\n\n if (!silent) {\n fprintf(stderr, \"%s: epoch %04\" PRId64 \"/%04\" PRId64 \":\\n\", __func__, epoch, nepoch);\n }\n ggml_opt_epoch(opt_ctx, dataset, result_train, result_val, idata_split, epoch_callback, epoch_callback);\n if (!silent) {\n fprintf(stderr, \"\\n\");\n }\n }\n\n if (!silent) {\n int64_t t_total_s = (ggml_time_us() - t_start_us) / 1000000;\n const int64_t t_total_h = t_total_s / 3600;\n t_total_s -= t_total_h * 3600;\n const int64_t t_total_m = t_total_s / 60;\n t_total_s -= t_total_m * 60;\n fprintf(stderr, \"%s: training took %02\" PRId64 \":%02\" PRId64 \":%02\" PRId64 \"\\n\", __func__, t_total_h, t_total_m, t_total_s);\n }\n\n ggml_opt_free(opt_ctx);\n ggml_opt_result_free(result_train);\n ggml_opt_result_free(result_val);\n}\n\nenum ggml_opt_optimizer_type ggml_opt_context_optimizer_type(ggml_opt_context_t c) {\n return c->optimizer;\n}\n\nGGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer_type o) {\n switch (o) {\n case GGML_OPT_OPTIMIZER_TYPE_ADAMW:\n return \"adamw\";\n case GGML_OPT_OPTIMIZER_TYPE_SGD:\n return \"sgd\";\n default:\n return \"undefined\";\n };\n}\n" }, { "path": "src/ggml-quants.c", "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu/ggml-cpu-impl.h\"\n#include \"ggml-cpu.h\"\n\n#include \n#include \n#include \n#include \n#include // for qsort\n#include // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\nstatic inline int best_index_int8(int n, const int8_t * val, float x) {\n if (x <= val[0]) return 0;\n if (x >= val[n-1]) return n-1;\n int ml = 0, mu = n-1;\n while (mu-ml > 1) {\n int mav = (ml+mu)/2;\n if (x < val[mav]) mu = mav; else ml = mav;\n }\n return x - val[mu-1] < val[mu] - x ? mu-1 : mu;\n}\n\n// reference implementation for deterministic creation of model files\nvoid quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK4_0;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n float amax = 0.0f; // absolute max\n float max = 0.0f;\n\n for (int j = 0; j < qk; j++) {\n const float v = x[i*qk + j];\n if (amax < fabsf(v)) {\n amax = fabsf(v);\n max = v;\n }\n }\n\n const float d = max / -8;\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_FP32_TO_FP16(d);\n\n for (int j = 0; j < qk/2; ++j) {\n const float x0 = x[i*qk + 0 + j]*id;\n const float x1 = x[i*qk + qk/2 + j]*id;\n\n const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f));\n const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f));\n\n y[i].qs[j] = xi0;\n y[i].qs[j] |= xi1 << 4;\n }\n }\n}\n\nvoid quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k) {\n const int qk = QK4_1;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n float min = FLT_MAX;\n float max = -FLT_MAX;\n\n for (int j = 0; j < qk; j++) {\n const float v = x[i*qk + j];\n\n if (v < min) min = v;\n if (v > max) max = v;\n }\n\n const float d = (max - min) / ((1 << 4) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_FP32_TO_FP16(d);\n y[i].m = GGML_FP32_TO_FP16(min);\n\n for (int j = 0; j < qk/2; ++j) {\n const float x0 = (x[i*qk + 0 + j] - min)*id;\n const float x1 = (x[i*qk + qk/2 + j] - min)*id;\n\n const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f));\n const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f));\n\n y[i].qs[j] = xi0;\n y[i].qs[j] |= xi1 << 4;\n }\n }\n}\n\nvoid quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK5_0;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n float amax = 0.0f; // absolute max\n float max = 0.0f;\n\n for (int j = 0; j < qk; j++) {\n const float v = x[i*qk + j];\n if (amax < fabsf(v)) {\n amax = fabsf(v);\n max = v;\n }\n }\n\n const float d = max / -16;\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_FP32_TO_FP16(d);\n\n uint32_t qh = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const float x0 = x[i*qk + 0 + j]*id;\n const float x1 = x[i*qk + qk/2 + j]*id;\n\n const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f));\n const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f));\n\n y[i].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);\n\n // get the 5-th bit and store it in qh at the right position\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + qk/2);\n }\n\n memcpy(&y[i].qh, &qh, sizeof(qh));\n }\n}\n\nvoid quantize_row_q5_1_ref(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k) {\n const int qk = QK5_1;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n float min = FLT_MAX;\n float max = -FLT_MAX;\n\n for (int j = 0; j < qk; j++) {\n const float v = x[i*qk + j];\n\n if (v < min) min = v;\n if (v > max) max = v;\n }\n\n const float d = (max - min) / ((1 << 5) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_FP32_TO_FP16(d);\n y[i].m = GGML_FP32_TO_FP16(min);\n\n uint32_t qh = 0;\n\n for (int j = 0; j < qk/2; ++j) {\n const float x0 = (x[i*qk + 0 + j] - min)*id;\n const float x1 = (x[i*qk + qk/2 + j] - min)*id;\n\n const uint8_t xi0 = (uint8_t)(x0 + 0.5f);\n const uint8_t xi1 = (uint8_t)(x1 + 0.5f);\n\n y[i].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);\n\n // get the 5-th bit and store it in qh at the right position\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + qk/2);\n }\n\n memcpy(&y[i].qh, &qh, sizeof(y[i].qh));\n }\n}\n\n// reference implementation for deterministic creation of model files\nvoid quantize_row_q8_0_ref(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k) {\n assert(k % QK8_0 == 0);\n const int nb = k / QK8_0;\n\n for (int i = 0; i < nb; i++) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK8_0; j++) {\n const float v = x[i*QK8_0 + j];\n amax = MAX(amax, fabsf(v));\n }\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_FP32_TO_FP16(d);\n\n for (int j = 0; j < QK8_0; ++j) {\n const float x0 = x[i*QK8_0 + j]*id;\n\n y[i].qs[j] = roundf(x0);\n }\n }\n}\n\n// reference implementation for deterministic creation of model files\nvoid quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k) {\n assert(QK8_1 == 32);\n assert(k % QK8_1 == 0);\n const int nb = k / QK8_1;\n\n for (int i = 0; i < nb; i++) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK8_1; j++) {\n const float v = x[i*QK8_1 + j];\n amax = MAX(amax, fabsf(v));\n }\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_FP32_TO_FP16(d);\n\n int sum = 0;\n\n for (int j = 0; j < QK8_1/2; ++j) {\n const float v0 = x[i*QK8_1 + j]*id;\n const float v1 = x[i*QK8_1 + QK8_1/2 + j]*id;\n\n y[i].qs[ j] = roundf(v0);\n y[i].qs[QK8_1/2 + j] = roundf(v1);\n\n sum += y[i].qs[ j];\n sum += y[i].qs[QK8_1/2 + j];\n }\n\n y[i].s = GGML_FP32_TO_FP16(sum*d);\n }\n}\n\nstatic inline int best_index_mxfp4(float x, float e) {\n int best_index = 0;\n float best_err = fabsf(kvalues_mxfp4[0]*e - x);\n for (int i = 1; i < 16; i++) {\n float err = fabsf(kvalues_mxfp4[i]*e - x);\n if (err < best_err) {\n best_index = i;\n best_err = err;\n }\n }\n return best_index;\n}\n\nvoid quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK_MXFP4;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < qk; j++) {\n const float v = x[i*qk + j];\n\n if (amax < fabsf(v)) {\n amax = fabsf(v);\n }\n }\n\n const uint8_t e = amax > 0.0f ? (uint8_t) (floorf(log2f(amax)) - 2 + 127) : 0;\n\n const float d = GGML_E8M0_TO_FP32_HALF(e);\n\n y[i].e = e;\n\n for (int j = 0; j < qk/2; ++j) {\n const uint8_t x0 = best_index_mxfp4(x[i*qk + 0 + j], d);\n const uint8_t x1 = best_index_mxfp4(x[i*qk + qk/2 + j], d);\n\n y[i].qs[j] = x0;\n y[i].qs[j] |= x1 << 4;\n }\n }\n}\n\nvoid quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK_NVFP4;\n static const int qk_sub = QK_NVFP4_SUB;\n static const int n_sub = QK_NVFP4 / QK_NVFP4_SUB;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n for (int s = 0; s < n_sub; s++) {\n const float * xb = x + i*qk + s*qk_sub;\n\n float amax = 0.0f;\n for (int j = 0; j < qk_sub; j++) {\n if (amax < fabsf(xb[j])) {\n amax = fabsf(xb[j]);\n }\n }\n\n // UE4M3 scale: amax / 6.0 maps the max E2M1 value (6.0) to amax\n const uint8_t ue = ggml_fp32_to_ue4m3(amax / 6.0f);\n y[i].d[s] = ue;\n const float d = ggml_ue4m3_to_fp32(ue);\n\n for (int j = 0; j < qk_sub/2; ++j) {\n const uint8_t x0 = best_index_mxfp4(xb[0 + j], d);\n const uint8_t x1 = best_index_mxfp4(xb[qk_sub/2 + j], d);\n\n y[i].qs[s*(qk_sub/2) + j] = x0 | (x1 << 4);\n }\n }\n }\n}\n\nvoid dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK4_0;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n const float d = GGML_FP16_TO_FP32(x[i].d);\n\n for (int j = 0; j < qk/2; ++j) {\n const int x0 = (x[i].qs[j] & 0x0F) - 8;\n const int x1 = (x[i].qs[j] >> 4) - 8;\n\n y[i*qk + j + 0 ] = x0*d;\n y[i*qk + j + qk/2] = x1*d;\n }\n }\n}\n\nvoid dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK4_1;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n const float d = GGML_FP16_TO_FP32(x[i].d);\n const float m = GGML_FP16_TO_FP32(x[i].m);\n\n for (int j = 0; j < qk/2; ++j) {\n const int x0 = (x[i].qs[j] & 0x0F);\n const int x1 = (x[i].qs[j] >> 4);\n\n y[i*qk + j + 0 ] = x0*d + m;\n y[i*qk + j + qk/2] = x1*d + m;\n }\n }\n}\n\nvoid dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK5_0;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n const float d = GGML_FP16_TO_FP32(x[i].d);\n\n uint32_t qh;\n memcpy(&qh, x[i].qh, sizeof(qh));\n\n for (int j = 0; j < qk/2; ++j) {\n const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;\n const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;\n\n const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;\n const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;\n\n y[i*qk + j + 0 ] = x0*d;\n y[i*qk + j + qk/2] = x1*d;\n }\n }\n}\n\nvoid dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK5_1;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n const float d = GGML_FP16_TO_FP32(x[i].d);\n const float m = GGML_FP16_TO_FP32(x[i].m);\n\n uint32_t qh;\n memcpy(&qh, x[i].qh, sizeof(qh));\n\n for (int j = 0; j < qk/2; ++j) {\n const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;\n const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;\n\n const int x0 = (x[i].qs[j] & 0x0F) | xh_0;\n const int x1 = (x[i].qs[j] >> 4) | xh_1;\n\n y[i*qk + j + 0 ] = x0*d + m;\n y[i*qk + j + qk/2] = x1*d + m;\n }\n }\n}\n\nvoid dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK8_0;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n const float d = GGML_FP16_TO_FP32(x[i].d);\n\n for (int j = 0; j < qk; ++j) {\n y[i*qk + j] = x[i].qs[j]*d;\n }\n }\n}\n\nvoid dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK_MXFP4;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n const float d = GGML_E8M0_TO_FP32_HALF(x[i].e);\n\n for (int j = 0; j < qk/2; ++j) {\n const int8_t x0 = kvalues_mxfp4[x[i].qs[j] & 0x0F];\n const int8_t x1 = kvalues_mxfp4[x[i].qs[j] >> 4];\n\n y[i*qk + j + 0 ] = x0*d;\n y[i*qk + j + qk/2] = x1*d;\n }\n }\n}\n\nvoid dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n static const int qk = QK_NVFP4;\n static const int qk_sub = QK_NVFP4_SUB;\n static const int n_sub = QK_NVFP4 / QK_NVFP4_SUB;\n\n assert(k % qk == 0);\n\n const int nb = k / qk;\n\n for (int i = 0; i < nb; i++) {\n for (int s = 0; s < n_sub; s++) {\n const float d = ggml_ue4m3_to_fp32(x[i].d[s]);\n float * yb = y + i*qk + s*qk_sub;\n\n for (int j = 0; j < qk_sub/2; ++j) {\n const int8_t v0 = kvalues_mxfp4[x[i].qs[s*(qk_sub/2) + j] & 0x0F];\n const int8_t v1 = kvalues_mxfp4[x[i].qs[s*(qk_sub/2) + j] >> 4];\n\n yb[j + 0 ] = v0*d;\n yb[j + qk_sub/2] = v1*d;\n }\n }\n }\n}\n\n//\n// 2-6 bit quantization in super-blocks\n//\n\n//\n// ===================== Helper functions\n//\nstatic inline int nearest_int(float fval) {\n assert(fabsf(fval) <= 4194303.f);\n float val = fval + 12582912.f;\n int i; memcpy(&i, &val, sizeof(int));\n return (i & 0x007fffff) - 0x00400000;\n}\n\nstatic float make_qx_quants(int n, int nmax, const float * GGML_RESTRICT x, int8_t * GGML_RESTRICT L, int rmse_type,\n const float * GGML_RESTRICT qw) {\n float max = 0;\n float amax = 0;\n for (int i = 0; i < n; ++i) {\n float ax = fabsf(x[i]);\n if (ax > amax) { amax = ax; max = x[i]; }\n }\n if (amax < GROUP_MAX_EPS) { // all zero\n for (int i = 0; i < n; ++i) {\n L[i] = 0;\n }\n return 0.f;\n }\n float iscale = -nmax / max;\n if (rmse_type == 0) {\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale * x[i]);\n L[i] = nmax + MAX(-nmax, MIN(nmax-1, l));\n }\n return 1/iscale;\n }\n bool return_early = false;\n if (rmse_type < 0) {\n rmse_type = -rmse_type;\n return_early = true;\n }\n float sumlx = 0;\n float suml2 = 0;\n#ifdef HAVE_BUGGY_APPLE_LINKER\n // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7\n for (volatile int i = 0; i < n; ++i) {\n#else\n for (int i = 0; i < n; ++i) {\n#endif\n int l = nearest_int(iscale * x[i]);\n l = MAX(-nmax, MIN(nmax-1, l));\n L[i] = l + nmax;\n float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i]));\n sumlx += w*x[i]*l;\n suml2 += w*l*l;\n }\n float scale = suml2 ? sumlx/suml2 : 0.0f;\n if (return_early) return suml2 > 0 ? 0.5f*(scale + 1/iscale) : 1/iscale;\n float best = scale * sumlx;\n for (int is = -9; is <= 9; ++is) {\n if (is == 0) {\n continue;\n }\n iscale = -(nmax + 0.1f*is) / max;\n sumlx = suml2 = 0;\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale * x[i]);\n l = MAX(-nmax, MIN(nmax-1, l));\n float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i]));\n sumlx += w*x[i]*l;\n suml2 += w*l*l;\n }\n if (suml2 > 0 && sumlx*sumlx > best*suml2) {\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale * x[i]);\n L[i] = nmax + MAX(-nmax, MIN(nmax-1, l));\n }\n scale = sumlx/suml2; best = scale*sumlx;\n }\n }\n return scale;\n}\n\nstatic float make_q3_quants(int n, int nmax, const float * GGML_RESTRICT x, int8_t * GGML_RESTRICT L, bool do_rmse) {\n float max = 0;\n float amax = 0;\n for (int i = 0; i < n; ++i) {\n float ax = fabsf(x[i]);\n if (ax > amax) { amax = ax; max = x[i]; }\n }\n if (amax < GROUP_MAX_EPS) { // all zero\n for (int i = 0; i < n; ++i) { L[i] = 0; }\n return 0.f;\n }\n float iscale = -nmax / max;\n if (do_rmse) {\n float sumlx = 0;\n float suml2 = 0;\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale * x[i]);\n l = MAX(-nmax, MIN(nmax-1, l));\n L[i] = l;\n float w = x[i]*x[i];\n sumlx += w*x[i]*l;\n suml2 += w*l*l;\n }\n for (int itry = 0; itry < 5; ++itry) {\n int n_changed = 0;\n for (int i = 0; i < n; ++i) {\n float w = x[i]*x[i];\n float slx = sumlx - w*x[i]*L[i];\n if (slx > 0) {\n float sl2 = suml2 - w*L[i]*L[i];\n int new_l = nearest_int(x[i] * sl2 / slx);\n new_l = MAX(-nmax, MIN(nmax-1, new_l));\n if (new_l != L[i]) {\n slx += w*x[i]*new_l;\n sl2 += w*new_l*new_l;\n if (sl2 > 0 && slx*slx*suml2 > sumlx*sumlx*sl2) {\n L[i] = new_l; sumlx = slx; suml2 = sl2;\n ++n_changed;\n }\n }\n }\n }\n if (!n_changed) {\n break;\n }\n }\n for (int i = 0; i < n; ++i) {\n L[i] += nmax;\n }\n return suml2 > 0.0f ? sumlx / suml2 : 0.0f;\n }\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale * x[i]);\n l = MAX(-nmax, MIN(nmax-1, l));\n L[i] = l + nmax;\n }\n return 1/iscale;\n}\n\nstatic float make_qkx1_quants(int n, int nmax, const float * GGML_RESTRICT x, uint8_t * GGML_RESTRICT L, float * GGML_RESTRICT the_min,\n int ntry, float alpha) {\n float min = x[0];\n float max = x[0];\n for (int i = 1; i < n; ++i) {\n if (x[i] < min) min = x[i];\n if (x[i] > max) max = x[i];\n }\n if (max == min) {\n for (int i = 0; i < n; ++i) L[i] = 0;\n *the_min = 0;\n return 0.f;\n }\n if (min > 0) min = 0;\n float iscale = nmax/(max - min);\n float scale = 1/iscale;\n for (int itry = 0; itry < ntry; ++itry) {\n float sumlx = 0; int suml2 = 0;\n bool did_change = false;\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale*(x[i] - min));\n l = MAX(0, MIN(nmax, l));\n if (l != L[i]) {\n L[i] = l;\n did_change = true;\n }\n sumlx += (x[i] - min)*l;\n suml2 += l*l;\n }\n scale = sumlx/suml2;\n float sum = 0;\n for (int i = 0; i < n; ++i) {\n sum += x[i] - scale*L[i];\n }\n min = alpha*min + (1 - alpha)*sum/n;\n if (min > 0) min = 0;\n iscale = 1/scale;\n if (!did_change) break;\n }\n *the_min = -min;\n return scale;\n}\n\nstatic float make_qkx2_quants(int n, int nmax, const float * GGML_RESTRICT x, const float * GGML_RESTRICT weights,\n uint8_t * GGML_RESTRICT L, float * GGML_RESTRICT the_min, uint8_t * GGML_RESTRICT Laux,\n float rmin, float rdelta, int nstep, bool use_mad) {\n float min = x[0];\n float max = x[0];\n float sum_w = weights[0];\n float sum_x = sum_w * x[0];\n#ifdef HAVE_BUGGY_APPLE_LINKER\n // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7\n for (volatile int i = 1; i < n; ++i) {\n#else\n for (int i = 1; i < n; ++i) {\n#endif\n if (x[i] < min) min = x[i];\n if (x[i] > max) max = x[i];\n float w = weights[i];\n sum_w += w;\n sum_x += w * x[i];\n }\n if (min > 0) min = 0;\n if (max == min) {\n for (int i = 0; i < n; ++i) L[i] = 0;\n *the_min = -min;\n return 0.f;\n }\n float iscale = nmax/(max - min);\n float scale = 1/iscale;\n float best_error = 0;\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale*(x[i] - min));\n L[i] = MAX(0, MIN(nmax, l));\n float diff = scale * L[i] + min - x[i];\n diff = use_mad ? fabsf(diff) : diff * diff;\n float w = weights[i];\n best_error += w * diff;\n }\n if (nstep < 1) {\n *the_min = -min;\n return scale;\n }\n for (int is = 0; is <= nstep; ++is) {\n iscale = (rmin + rdelta*is + nmax)/(max - min);\n float sum_l = 0, sum_l2 = 0, sum_xl = 0;\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale*(x[i] - min));\n l = MAX(0, MIN(nmax, l));\n Laux[i] = l;\n float w = weights[i];\n sum_l += w*l;\n sum_l2 += w*l*l;\n sum_xl += w*l*x[i];\n }\n float D = sum_w * sum_l2 - sum_l * sum_l;\n if (D > 0) {\n float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D;\n float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D;\n if (this_min > 0) {\n this_min = 0;\n this_scale = sum_xl / sum_l2;\n }\n float cur_error = 0;\n for (int i = 0; i < n; ++i) {\n float diff = this_scale * Laux[i] + this_min - x[i];\n diff = use_mad ? fabsf(diff) : diff * diff;\n float w = weights[i];\n cur_error += w * diff;\n }\n if (cur_error < best_error) {\n for (int i = 0; i < n; ++i) {\n L[i] = Laux[i];\n }\n best_error = cur_error;\n scale = this_scale;\n min = this_min;\n }\n }\n }\n *the_min = -min;\n return scale;\n}\n\nstatic inline void get_scale_min_k4(int j, const uint8_t * GGML_RESTRICT q, uint8_t * GGML_RESTRICT d, uint8_t * GGML_RESTRICT m) {\n if (j < 4) {\n *d = q[j] & 63; *m = q[j + 4] & 63;\n } else {\n *d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);\n *m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);\n }\n}\n\n//========================- 2-bit (de)-quantization\n\nvoid quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n uint8_t L[QK_K];\n uint8_t Laux[16];\n float weights[16];\n float mins[QK_K/16];\n float scales[QK_K/16];\n\n const float q4scale = 15.f;\n\n for (int i = 0; i < nb; i++) {\n float max_scale = 0; // as we are deducting the min, scales are always positive\n float max_min = 0;\n for (int j = 0; j < QK_K/16; ++j) {\n for (int l = 0; l < 16; ++l) weights[l] = fabsf(x[16*j + l]);\n scales[j] = make_qkx2_quants(16, 3, x + 16*j, weights, L + 16*j, &mins[j], Laux, -0.5f, 0.1f, 15, true);\n float scale = scales[j];\n if (scale > max_scale) {\n max_scale = scale;\n }\n float min = mins[j];\n if (min > max_min) {\n max_min = min;\n }\n }\n\n if (max_scale > 0) {\n float iscale = q4scale/max_scale;\n for (int j = 0; j < QK_K/16; ++j) {\n int l = nearest_int(iscale*scales[j]);\n y[i].scales[j] = l;\n }\n y[i].d = GGML_FP32_TO_FP16(max_scale/q4scale);\n } else {\n for (int j = 0; j < QK_K/16; ++j) y[i].scales[j] = 0;\n y[i].d = GGML_FP32_TO_FP16(0.f);\n }\n if (max_min > 0) {\n float iscale = q4scale/max_min;\n for (int j = 0; j < QK_K/16; ++j) {\n int l = nearest_int(iscale*mins[j]);\n y[i].scales[j] |= (l << 4);\n }\n y[i].dmin = GGML_FP32_TO_FP16(max_min/q4scale);\n } else {\n y[i].dmin = GGML_FP32_TO_FP16(0.f);\n }\n for (int j = 0; j < QK_K/16; ++j) {\n const float d = GGML_FP16_TO_FP32(y[i].d) * (y[i].scales[j] & 0xF);\n if (!d) continue;\n const float dm = GGML_FP16_TO_FP32(y[i].dmin) * (y[i].scales[j] >> 4);\n for (int ii = 0; ii < 16; ++ii) {\n int l = nearest_int((x[16*j + ii] + dm)/d);\n l = MAX(0, MIN(3, l));\n L[16*j + ii] = l;\n }\n }\n\n for (int j = 0; j < QK_K; j += 128) {\n for (int l = 0; l < 32; ++l) {\n y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);\n }\n }\n\n x += QK_K;\n }\n}\n\nvoid dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n for (int i = 0; i < nb; i++) {\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n const float min = GGML_FP16_TO_FP32(x[i].dmin);\n\n const uint8_t * q = x[i].qs;\n\n int is = 0;\n float dl, ml;\n for (int n = 0; n < QK_K; n += 128) {\n int shift = 0;\n for (int j = 0; j < 4; ++j) {\n\n uint8_t sc = x[i].scales[is++];\n dl = d * (sc & 0xF); ml = min * (sc >> 4);\n for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l] >> shift) & 3)) - ml;\n\n sc = x[i].scales[is++];\n dl = d * (sc & 0xF); ml = min * (sc >> 4);\n for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3)) - ml;\n\n shift += 2;\n }\n q += 32;\n }\n }\n}\n\nstatic float make_qkx3_quants(int n, int nmax, const float * GGML_RESTRICT x, const float * GGML_RESTRICT weights,\n uint8_t * GGML_RESTRICT L, float * GGML_RESTRICT the_min, uint8_t * GGML_RESTRICT Laux,\n float rmin, float rdelta, int nstep, bool use_mad) {\n float min = x[0];\n float max = x[0];\n float sum_w = weights ? weights[0] : x[0]*x[0];\n float sum_x = sum_w * x[0];\n#ifdef HAVE_BUGGY_APPLE_LINKER\n // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7\n for (volatile int i = 1; i < n; ++i) {\n#else\n for (int i = 1; i < n; ++i) {\n#endif\n if (x[i] < min) min = x[i];\n if (x[i] > max) max = x[i];\n float w = weights ? weights[i] : x[i]*x[i];\n sum_w += w;\n sum_x += w * x[i];\n }\n if (min > 0) {\n min = 0;\n }\n if (max <= min) {\n memset(L, 0, n);\n *the_min = -min;\n return 0.f;\n }\n float iscale = nmax/(max - min);\n float scale = 1/iscale;\n float best_mad = 0;\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale*(x[i] - min));\n L[i] = MAX(0, MIN(nmax, l));\n float diff = scale * L[i] + min - x[i];\n diff = use_mad ? fabsf(diff) : diff*diff;\n float w = weights ? weights[i] : x[i]*x[i];\n best_mad += w * diff;\n }\n if (nstep < 1) {\n *the_min = -min;\n return scale;\n }\n for (int is = 0; is <= nstep; ++is) {\n iscale = (rmin + rdelta*is + nmax)/(max - min);\n float sum_l = 0, sum_l2 = 0, sum_xl = 0;\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale*(x[i] - min));\n l = MAX(0, MIN(nmax, l));\n Laux[i] = l;\n float w = weights ? weights[i] : x[i]*x[i];\n sum_l += w*l;\n sum_l2 += w*l*l;\n sum_xl += w*l*x[i];\n }\n float D = sum_w * sum_l2 - sum_l * sum_l;\n if (D > 0) {\n float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D;\n float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D;\n if (this_min > 0) {\n this_min = 0;\n this_scale = sum_xl / sum_l2;\n }\n float mad = 0;\n for (int i = 0; i < n; ++i) {\n float diff = this_scale * Laux[i] + this_min - x[i];\n diff = use_mad ? fabsf(diff) : diff*diff;\n float w = weights ? weights[i] : x[i]*x[i];\n mad += w * diff;\n }\n if (mad < best_mad) {\n for (int i = 0; i < n; ++i) {\n L[i] = Laux[i];\n }\n best_mad = mad;\n scale = this_scale;\n min = this_min;\n }\n }\n }\n *the_min = -min;\n return scale;\n}\n\nstatic float make_qp_quants(int n, int nmax, const float * GGML_RESTRICT x, uint8_t * GGML_RESTRICT L, const float * quant_weights) {\n float max = 0;\n for (int i = 0; i < n; ++i) {\n max = MAX(max, x[i]);\n }\n if (max < GROUP_MAX_EPS) { // all zero\n for (int i = 0; i < n; ++i) { L[i] = 0; }\n return 0.f;\n }\n float iscale = nmax / max;\n for (int i = 0; i < n; ++i) {\n L[i] = nearest_int(iscale * x[i]);\n }\n float scale = 1/iscale;\n float best_mse = 0;\n for (int i = 0; i < n; ++i) {\n float diff = x[i] - scale*L[i];\n float w = quant_weights[i];\n best_mse += w*diff*diff;\n }\n for (int is = -4; is <= 4; ++is) {\n if (is == 0) continue;\n float iscale_is = (0.1f*is + nmax)/max;\n float scale_is = 1/iscale_is;\n float mse = 0;\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale_is*x[i]);\n l = MIN(nmax, l);\n float diff = x[i] - scale_is*l;\n float w = quant_weights[i];\n mse += w*diff*diff;\n }\n if (mse < best_mse) {\n best_mse = mse;\n iscale = iscale_is;\n }\n }\n float sumlx = 0;\n float suml2 = 0;\n for (int i = 0; i < n; ++i) {\n int l = nearest_int(iscale * x[i]);\n l = MIN(nmax, l);\n L[i] = l;\n float w = quant_weights[i];\n sumlx += w*x[i]*l;\n suml2 += w*l*l;\n }\n for (int itry = 0; itry < 5; ++itry) {\n int n_changed = 0;\n for (int i = 0; i < n; ++i) {\n float w = quant_weights[i];\n float slx = sumlx - w*x[i]*L[i];\n float sl2 = suml2 - w*L[i]*L[i];\n if (slx > 0 && sl2 > 0) {\n int new_l = nearest_int(x[i] * sl2 / slx);\n new_l = MIN(nmax, new_l);\n if (new_l != L[i]) {\n slx += w*x[i]*new_l;\n sl2 += w*new_l*new_l;\n if (slx*slx*suml2 > sumlx*sumlx*sl2) {\n L[i] = new_l; sumlx = slx; suml2 = sl2;\n ++n_changed;\n }\n }\n }\n }\n if (!n_changed) {\n break;\n }\n }\n return suml2 > 0.0f ? sumlx / suml2 : 0.0f;\n}\n\nstatic void quantize_row_q2_K_impl(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int k, const float * GGML_RESTRICT quant_weights) {\n GGML_ASSERT(quant_weights);\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n const bool requantize = true;\n\n uint8_t L[QK_K];\n uint8_t Laux[16];\n float mins[QK_K/16];\n float scales[QK_K/16];\n float sw[QK_K/16];\n float weight[16];\n uint8_t Ls[QK_K/16], Lm[QK_K/16];\n\n for (int i = 0; i < nb; i++) {\n memset(sw, 0, QK_K/16*sizeof(float));\n float sumx2 = 0;\n for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j];\n float sigma2 = sumx2/QK_K;\n for (int j = 0; j < QK_K/16; ++j) {\n const float * GGML_RESTRICT qw = quant_weights + QK_K * i + 16*j;\n for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]);\n for (int l = 0; l < QK_K/16; ++l) sw[j] += weight[l];\n scales[j] = make_qkx3_quants(16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);\n }\n\n float dm, mm;\n dm = make_qp_quants(QK_K/16, 15, scales, Ls, sw);\n mm = make_qp_quants(QK_K/16, 15, mins, Lm, sw);\n\n y[i].d = GGML_FP32_TO_FP16(dm);\n y[i].dmin = GGML_FP32_TO_FP16(mm);\n dm = GGML_FP16_TO_FP32(y[i].d);\n mm = GGML_FP16_TO_FP32(y[i].dmin);\n\n for (int j = 0; j < QK_K/16; ++j) {\n y[i].scales[j] = Ls[j] | (Lm[j] << 4);\n }\n\n if (requantize) {\n for (int j = 0; j < QK_K/16; ++j) {\n const float d = dm * (y[i].scales[j] & 0xF);\n if (!d) continue;\n const float m = mm * (y[i].scales[j] >> 4);\n for (int ii = 0; ii < 16; ++ii) {\n int l = nearest_int((x[16*j + ii] + m)/d);\n l = MAX(0, MIN(3, l));\n L[16*j + ii] = l;\n }\n }\n }\n\n for (int j = 0; j < QK_K; j += 128) {\n for (int l = 0; l < 32; ++l) {\n y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);\n }\n }\n\n x += QK_K;\n }\n}\n\nsize_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n size_t row_size = ggml_row_size(GGML_TYPE_Q2_K, n_per_row);\n if (!quant_weights) {\n quantize_row_q2_K_ref(src, dst, (int64_t)nrow*n_per_row);\n }\n else {\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_q2_K_impl(src, (block_q2_K*)qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += row_size;\n }\n }\n return nrow * row_size;\n}\n\n//========================= 3-bit (de)-quantization\n\nvoid quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n int8_t L[QK_K];\n float scales[QK_K / 16];\n\n for (int i = 0; i < nb; i++) {\n\n float max_scale = 0;\n float amax = 0;\n for (int j = 0; j < QK_K/16; ++j) {\n scales[j] = make_q3_quants(16, 4, x + 16*j, L + 16*j, true);\n float scale = fabsf(scales[j]);\n if (scale > amax) {\n amax = scale; max_scale = scales[j];\n }\n }\n\n memset(y[i].scales, 0, 12);\n if (max_scale) {\n float iscale = -32.f/max_scale;\n for (int j = 0; j < QK_K/16; ++j) {\n int8_t l = nearest_int(iscale*scales[j]);\n l = MAX(-32, MIN(31, l)) + 32;\n if (j < 8) {\n y[i].scales[j] = l & 0xF;\n } else {\n y[i].scales[j-8] |= ((l & 0xF) << 4);\n }\n l >>= 4;\n y[i].scales[j%4 + 8] |= (l << (2*(j/4)));\n }\n y[i].d = GGML_FP32_TO_FP16(1/iscale);\n } else {\n y[i].d = GGML_FP32_TO_FP16(0.f);\n }\n\n int8_t sc;\n for (int j = 0; j < QK_K/16; ++j) {\n sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4;\n sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32;\n float d = GGML_FP16_TO_FP32(y[i].d) * sc;\n if (!d) {\n continue;\n }\n for (int ii = 0; ii < 16; ++ii) {\n int l = nearest_int(x[16*j + ii]/d);\n l = MAX(-4, MIN(3, l));\n L[16*j + ii] = l + 4;\n }\n }\n\n memset(y[i].hmask, 0, QK_K/8);\n // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc.\n int m = 0;\n uint8_t hm = 1;\n for (int j = 0; j < QK_K; ++j) {\n if (L[j] > 3) {\n y[i].hmask[m] |= hm;\n L[j] -= 4;\n }\n if (++m == QK_K/8) {\n m = 0; hm <<= 1;\n }\n }\n for (int j = 0; j < QK_K; j += 128) {\n for (int l = 0; l < 32; ++l) {\n y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);\n }\n }\n\n x += QK_K;\n }\n}\n\nvoid dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n const uint32_t kmask1 = 0x03030303;\n const uint32_t kmask2 = 0x0f0f0f0f;\n\n uint32_t aux[4];\n const int8_t * scales = (const int8_t*)aux;\n\n for (int i = 0; i < nb; i++) {\n\n const float d_all = GGML_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT q = x[i].qs;\n const uint8_t * GGML_RESTRICT hm = x[i].hmask;\n uint8_t m = 1;\n\n memcpy(aux, x[i].scales, 12);\n uint32_t tmp = aux[2];\n aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);\n aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);\n aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);\n aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);\n\n int is = 0;\n float dl;\n for (int n = 0; n < QK_K; n += 128) {\n int shift = 0;\n for (int j = 0; j < 4; ++j) {\n\n dl = d_all * (scales[is++] - 32);\n for (int l = 0; l < 16; ++l) {\n *y++ = dl * ((int8_t)((q[l+ 0] >> shift) & 3) - ((hm[l+ 0] & m) ? 0 : 4));\n }\n\n dl = d_all * (scales[is++] - 32);\n for (int l = 0; l < 16; ++l) {\n *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3) - ((hm[l+16] & m) ? 0 : 4));\n }\n\n shift += 2;\n m <<= 1;\n }\n q += 32;\n }\n\n }\n}\n\nstatic void quantize_row_q3_K_impl(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t n_per_row, const float * GGML_RESTRICT quant_weights) {\n assert(n_per_row % QK_K == 0);\n const int nb = n_per_row / QK_K;\n\n int8_t L[QK_K];\n float scales[QK_K / 16];\n float weight[16];\n float sw[QK_K / 16];\n int8_t Ls[QK_K / 16];\n\n for (int i = 0; i < nb; i++) {\n\n float sumx2 = 0;\n for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j];\n float sigma2 = 2*sumx2/QK_K;\n\n for (int j = 0; j < QK_K/16; ++j) {\n if (quant_weights) {\n const float * qw = quant_weights + QK_K * i + 16*j;\n for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j+l]*x[16*j+l]);\n } else {\n for (int l = 0; l < 16; ++l) weight[l] = x[16*j+l]*x[16*j+l];\n }\n float sumw = 0;\n for (int l = 0; l < 16; ++l) sumw += weight[l];\n sw[j] = sumw;\n\n scales[j] = make_qx_quants(16, 4, x + 16*j, L + 16*j, 1, weight);\n\n }\n\n memset(y[i].scales, 0, 12);\n\n float d_block = make_qx_quants(QK_K/16, 32, scales, Ls, 1, sw);\n for (int j = 0; j < QK_K/16; ++j) {\n int l = Ls[j];\n if (j < 8) {\n y[i].scales[j] = l & 0xF;\n } else {\n y[i].scales[j-8] |= ((l & 0xF) << 4);\n }\n l >>= 4;\n y[i].scales[j%4 + 8] |= (l << (2*(j/4)));\n }\n y[i].d = GGML_FP32_TO_FP16(d_block);\n\n int8_t sc;\n for (int j = 0; j < QK_K/16; ++j) {\n sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4;\n sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32;\n float d = GGML_FP16_TO_FP32(y[i].d) * sc;\n if (!d) {\n continue;\n }\n for (int ii = 0; ii < 16; ++ii) {\n int l = nearest_int(x[16*j + ii]/d);\n l = MAX(-4, MIN(3, l));\n L[16*j + ii] = l + 4;\n }\n }\n\n memset(y[i].hmask, 0, QK_K/8);\n // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc.\n int m = 0;\n uint8_t hm = 1;\n for (int j = 0; j < QK_K; ++j) {\n if (L[j] > 3) {\n y[i].hmask[m] |= hm;\n L[j] -= 4;\n }\n if (++m == QK_K/8) {\n m = 0; hm <<= 1;\n }\n }\n for (int j = 0; j < QK_K; j += 128) {\n for (int l = 0; l < 32; ++l) {\n y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);\n }\n }\n\n x += QK_K;\n }\n}\n\nsize_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n size_t row_size = ggml_row_size(GGML_TYPE_Q3_K, n_per_row);\n if (!quant_weights) {\n quantize_row_q3_K_ref(src, dst, (int64_t)nrow*n_per_row);\n }\n else {\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_q3_K_impl(src, (block_q3_K*)qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += row_size;\n }\n }\n return nrow * row_size;\n}\n\n// ====================== 4-bit (de)-quantization\n\nvoid quantize_row_q4_K_ref(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n uint8_t L[QK_K];\n uint8_t Laux[32];\n float weights[32];\n float mins[QK_K/32];\n float scales[QK_K/32];\n\n for (int i = 0; i < nb; i++) {\n float max_scale = 0; // as we are deducting the min, scales are always positive\n float max_min = 0;\n for (int j = 0; j < QK_K/32; ++j) {\n //scales[j] = make_qkx1_quants(32, 15, x + 32*j, L + 32*j, &mins[j], 9, 0.5f);\n float sum_x2 = 0;\n for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l];\n float av_x = sqrtf(sum_x2/32);\n for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);\n scales[j] = make_qkx2_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -1.f, 0.1f, 20, false);\n float scale = scales[j];\n if (scale > max_scale) {\n max_scale = scale;\n }\n float min = mins[j];\n if (min > max_min) {\n max_min = min;\n }\n }\n\n float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f;\n float inv_min = max_min > 0 ? 63.f/max_min : 0.f;\n for (int j = 0; j < QK_K/32; ++j) {\n uint8_t ls = nearest_int(inv_scale*scales[j]);\n uint8_t lm = nearest_int(inv_min*mins[j]);\n ls = MIN(63, ls);\n lm = MIN(63, lm);\n if (j < 4) {\n y[i].scales[j] = ls;\n y[i].scales[j+4] = lm;\n } else {\n y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);\n y[i].scales[j-4] |= ((ls >> 4) << 6);\n y[i].scales[j-0] |= ((lm >> 4) << 6);\n }\n }\n y[i].d = GGML_FP32_TO_FP16(max_scale/63.f);\n y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f);\n\n uint8_t sc, m;\n for (int j = 0; j < QK_K/32; ++j) {\n get_scale_min_k4(j, y[i].scales, &sc, &m);\n const float d = GGML_FP16_TO_FP32(y[i].d) * sc;\n if (!d) continue;\n const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;\n for (int ii = 0; ii < 32; ++ii) {\n int l = nearest_int((x[32*j + ii] + dm)/d);\n l = MAX(0, MIN(15, l));\n L[32*j + ii] = l;\n }\n }\n\n uint8_t * q = y[i].qs;\n for (int j = 0; j < QK_K; j += 64) {\n for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4);\n q += 32;\n }\n\n x += QK_K;\n }\n}\n\nvoid dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int nb = k / QK_K;\n\n for (int i = 0; i < nb; i++) {\n const uint8_t * q = x[i].qs;\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n const float min = GGML_FP16_TO_FP32(x[i].dmin);\n\n int is = 0;\n uint8_t sc, m;\n for (int j = 0; j < QK_K; j += 64) {\n get_scale_min_k4(is + 0, x[i].scales, &sc, &m);\n const float d1 = d * sc; const float m1 = min * m;\n get_scale_min_k4(is + 1, x[i].scales, &sc, &m);\n const float d2 = d * sc; const float m2 = min * m;\n for (int l = 0; l < 32; ++l) *y++ = d1 * (q[l] & 0xF) - m1;\n for (int l = 0; l < 32; ++l) *y++ = d2 * (q[l] >> 4) - m2;\n q += 32; is += 2;\n }\n }\n}\n\nstatic void quantize_row_q4_K_impl(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {\n assert(n_per_row % QK_K == 0);\n const int64_t nb = n_per_row / QK_K;\n\n uint8_t L[QK_K];\n uint8_t Laux[32];\n uint8_t Ls[QK_K/32];\n uint8_t Lm[QK_K/32];\n float weights[32];\n float sw[QK_K/32];\n float mins[QK_K/32];\n float scales[QK_K/32];\n\n for (int i = 0; i < nb; i++) {\n\n float sum_x2 = 0;\n for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l];\n float sigma2 = 2*sum_x2/QK_K;\n float av_x = sqrtf(sigma2);\n\n for (int j = 0; j < QK_K/32; ++j) {\n if (quant_weights) {\n const float * qw = quant_weights + QK_K*i + 32*j;\n for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]);\n } else {\n for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);\n }\n float sumw = 0;\n for (int l = 0; l < 32; ++l) sumw += weights[l];\n sw[j] = sumw;\n scales[j] = make_qkx3_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);\n }\n\n float d_block = make_qp_quants(QK_K/32, 63, scales, Ls, sw);\n float m_block = make_qp_quants(QK_K/32, 63, mins, Lm, sw);\n for (int j = 0; j < QK_K/32; ++j) {\n uint8_t ls = Ls[j];\n uint8_t lm = Lm[j];\n if (j < 4) {\n y[i].scales[j] = ls;\n y[i].scales[j+4] = lm;\n } else {\n y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);\n y[i].scales[j-4] |= ((ls >> 4) << 6);\n y[i].scales[j-0] |= ((lm >> 4) << 6);\n }\n }\n y[i].d = GGML_FP32_TO_FP16(d_block);\n y[i].dmin = GGML_FP32_TO_FP16(m_block);\n\n uint8_t sc, m;\n for (int j = 0; j < QK_K/32; ++j) {\n get_scale_min_k4(j, y[i].scales, &sc, &m);\n const float d = GGML_FP16_TO_FP32(y[i].d) * sc;\n if (!d) continue;\n const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;\n for (int ii = 0; ii < 32; ++ii) {\n int l = nearest_int((x[32*j + ii] + dm)/d);\n l = MAX(0, MIN(15, l));\n L[32*j + ii] = l;\n }\n }\n uint8_t * q = y[i].qs;\n for (int j = 0; j < QK_K; j += 64) {\n for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4);\n q += 32;\n }\n\n x += QK_K;\n\n }\n}\n\nsize_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n size_t row_size = ggml_row_size(GGML_TYPE_Q4_K, n_per_row);\n if (!quant_weights) {\n quantize_row_q4_K_ref(src, dst, (int64_t)nrow*n_per_row);\n }\n else {\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_q4_K_impl(src, (block_q4_K*)qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += row_size;\n }\n }\n return nrow * row_size;\n}\n\n// ====================== 5-bit (de)-quantization\n\nvoid quantize_row_q5_K_ref(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n uint8_t L[QK_K];\n float mins[QK_K/32];\n float scales[QK_K/32];\n float weights[32];\n uint8_t Laux[32];\n\n for (int i = 0; i < nb; i++) {\n float max_scale = 0; // as we are deducting the min, scales are always positive\n float max_min = 0;\n for (int j = 0; j < QK_K/32; ++j) {\n //scales[j] = make_qkx1_quants(32, 31, x + 32*j, L + 32*j, &mins[j], 9, 0.5f);\n float sum_x2 = 0;\n for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l];\n float av_x = sqrtf(sum_x2/32);\n for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);\n scales[j] = make_qkx2_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.5f, 0.1f, 15, false);\n float scale = scales[j];\n if (scale > max_scale) {\n max_scale = scale;\n }\n float min = mins[j];\n if (min > max_min) {\n max_min = min;\n }\n }\n\n float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f;\n float inv_min = max_min > 0 ? 63.f/max_min : 0.f;\n for (int j = 0; j < QK_K/32; ++j) {\n uint8_t ls = nearest_int(inv_scale*scales[j]);\n uint8_t lm = nearest_int(inv_min*mins[j]);\n ls = MIN(63, ls);\n lm = MIN(63, lm);\n if (j < 4) {\n y[i].scales[j] = ls;\n y[i].scales[j+4] = lm;\n } else {\n y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);\n y[i].scales[j-4] |= ((ls >> 4) << 6);\n y[i].scales[j-0] |= ((lm >> 4) << 6);\n }\n }\n y[i].d = GGML_FP32_TO_FP16(max_scale/63.f);\n y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f);\n\n uint8_t sc, m;\n for (int j = 0; j < QK_K/32; ++j) {\n get_scale_min_k4(j, y[i].scales, &sc, &m);\n const float d = GGML_FP16_TO_FP32(y[i].d) * sc;\n if (!d) continue;\n const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;\n for (int ii = 0; ii < 32; ++ii) {\n int l = nearest_int((x[32*j + ii] + dm)/d);\n l = MAX(0, MIN(31, l));\n L[32*j + ii] = l;\n }\n }\n\n uint8_t * GGML_RESTRICT qh = y[i].qh;\n uint8_t * GGML_RESTRICT ql = y[i].qs;\n memset(qh, 0, QK_K/8);\n\n uint8_t m1 = 1, m2 = 2;\n for (int n = 0; n < QK_K; n += 64) {\n for (int j = 0; j < 32; ++j) {\n int l1 = L[n + j];\n if (l1 > 15) {\n l1 -= 16; qh[j] |= m1;\n }\n int l2 = L[n + j + 32];\n if (l2 > 15) {\n l2 -= 16; qh[j] |= m2;\n }\n ql[j] = l1 | (l2 << 4);\n }\n m1 <<= 2; m2 <<= 2;\n ql += 32;\n }\n\n x += QK_K;\n }\n}\n\nvoid dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int i = 0; i < nb; i++) {\n const uint8_t * ql = x[i].qs;\n const uint8_t * qh = x[i].qh;\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n const float min = GGML_FP16_TO_FP32(x[i].dmin);\n\n int is = 0;\n uint8_t sc, m;\n uint8_t u1 = 1, u2 = 2;\n for (int j = 0; j < QK_K; j += 64) {\n get_scale_min_k4(is + 0, x[i].scales, &sc, &m);\n const float d1 = d * sc; const float m1 = min * m;\n get_scale_min_k4(is + 1, x[i].scales, &sc, &m);\n const float d2 = d * sc; const float m2 = min * m;\n for (int l = 0; l < 32; ++l) *y++ = d1 * ((ql[l] & 0xF) + (qh[l] & u1 ? 16 : 0)) - m1;\n for (int l = 0; l < 32; ++l) *y++ = d2 * ((ql[l] >> 4) + (qh[l] & u2 ? 16 : 0)) - m2;\n ql += 32; is += 2;\n u1 <<= 2; u2 <<= 2;\n }\n }\n}\n\nstatic void quantize_row_q5_K_impl(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {\n assert(n_per_row % QK_K == 0);\n const int64_t nb = n_per_row / QK_K;\n\n uint8_t L[QK_K];\n uint8_t Laux[32];\n uint8_t Ls[QK_K/32];\n uint8_t Lm[QK_K/32];\n float mins[QK_K/32];\n float scales[QK_K/32];\n float sw[QK_K/32];\n float weights[32];\n\n for (int i = 0; i < nb; i++) {\n\n float sum_x2 = 0;\n for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l];\n float sigma2 = 2*sum_x2/QK_K;\n float av_x = sqrtf(sigma2);\n\n for (int j = 0; j < QK_K/32; ++j) {\n if (quant_weights) {\n const float * qw = quant_weights + QK_K*i + 32*j;\n for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]);\n } else {\n for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);\n }\n float sumw = 0;\n for (int l = 0; l < 32; ++l) sumw += weights[l];\n sw[j] = sumw;\n\n scales[j] = make_qkx3_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);\n }\n\n float d_block = make_qp_quants(QK_K/32, 63, scales, Ls, sw);\n float m_block = make_qp_quants(QK_K/32, 63, mins, Lm, sw);\n\n for (int j = 0; j < QK_K/32; ++j) {\n uint8_t ls = Ls[j];\n uint8_t lm = Lm[j];\n ls = MIN(63, ls);\n lm = MIN(63, lm);\n if (j < 4) {\n y[i].scales[j] = ls;\n y[i].scales[j+4] = lm;\n } else {\n y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);\n y[i].scales[j-4] |= ((ls >> 4) << 6);\n y[i].scales[j-0] |= ((lm >> 4) << 6);\n }\n }\n y[i].d = GGML_FP32_TO_FP16(d_block);\n y[i].dmin = GGML_FP32_TO_FP16(m_block);\n\n uint8_t sc, m;\n for (int j = 0; j < QK_K/32; ++j) {\n get_scale_min_k4(j, y[i].scales, &sc, &m);\n const float d = GGML_FP16_TO_FP32(y[i].d) * sc;\n if (!d) continue;\n const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;\n for (int ii = 0; ii < 32; ++ii) {\n int l = nearest_int((x[32*j + ii] + dm)/d);\n l = MAX(0, MIN(31, l));\n L[32*j + ii] = l;\n }\n }\n\n uint8_t * GGML_RESTRICT qh = y[i].qh;\n uint8_t * GGML_RESTRICT ql = y[i].qs;\n memset(qh, 0, QK_K/8);\n\n uint8_t m1 = 1, m2 = 2;\n for (int n = 0; n < QK_K; n += 64) {\n for (int j = 0; j < 32; ++j) {\n int l1 = L[n + j];\n if (l1 > 15) {\n l1 -= 16; qh[j] |= m1;\n }\n int l2 = L[n + j + 32];\n if (l2 > 15) {\n l2 -= 16; qh[j] |= m2;\n }\n ql[j] = l1 | (l2 << 4);\n }\n m1 <<= 2; m2 <<= 2;\n ql += 32;\n }\n\n x += QK_K;\n\n }\n}\n\nsize_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n size_t row_size = ggml_row_size(GGML_TYPE_Q5_K, n_per_row);\n if (!quant_weights) {\n quantize_row_q5_K_ref(src, dst, (int64_t)nrow*n_per_row);\n }\n else {\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_q5_K_impl(src, (block_q5_K*)qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += row_size;\n }\n }\n return nrow * row_size;\n}\n\n// ====================== 6-bit (de)-quantization\n\nvoid quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n int8_t L[QK_K];\n float scales[QK_K/16];\n\n for (int i = 0; i < nb; i++) {\n\n float max_scale = 0;\n float max_abs_scale = 0;\n\n for (int ib = 0; ib < QK_K/16; ++ib) {\n\n const float scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL);\n scales[ib] = scale;\n\n const float abs_scale = fabsf(scale);\n if (abs_scale > max_abs_scale) {\n max_abs_scale = abs_scale;\n max_scale = scale;\n }\n\n }\n\n if (max_abs_scale < GROUP_MAX_EPS) {\n memset(&y[i], 0, sizeof(block_q6_K));\n y[i].d = GGML_FP32_TO_FP16(0.f);\n x += QK_K;\n continue;\n }\n\n float iscale = -128.f/max_scale;\n y[i].d = GGML_FP32_TO_FP16(1/iscale);\n for (int ib = 0; ib < QK_K/16; ++ib) {\n y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]));\n }\n\n for (int j = 0; j < QK_K/16; ++j) {\n float d = GGML_FP16_TO_FP32(y[i].d) * y[i].scales[j];\n if (!d) {\n continue;\n }\n for (int ii = 0; ii < 16; ++ii) {\n int l = nearest_int(x[16*j + ii]/d);\n l = MAX(-32, MIN(31, l));\n L[16*j + ii] = l + 32;\n }\n }\n\n uint8_t * GGML_RESTRICT ql = y[i].ql;\n uint8_t * GGML_RESTRICT qh = y[i].qh;\n for (int j = 0; j < QK_K; j += 128) {\n for (int l = 0; l < 32; ++l) {\n const uint8_t q1 = L[j + l + 0] & 0xF;\n const uint8_t q2 = L[j + l + 32] & 0xF;\n const uint8_t q3 = L[j + l + 64] & 0xF;\n const uint8_t q4 = L[j + l + 96] & 0xF;\n ql[l+ 0] = q1 | (q3 << 4);\n ql[l+32] = q2 | (q4 << 4);\n qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6);\n }\n ql += 64;\n qh += 32;\n }\n\n x += QK_K;\n }\n}\n\nvoid dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int i = 0; i < nb; i++) {\n const float d = GGML_FP16_TO_FP32(x[i].d);\n\n const uint8_t * GGML_RESTRICT ql = x[i].ql;\n const uint8_t * GGML_RESTRICT qh = x[i].qh;\n const int8_t * GGML_RESTRICT sc = x[i].scales;\n\n for (int n = 0; n < QK_K; n += 128) {\n for (int l = 0; l < 32; ++l) {\n int is = l/16;\n const int8_t q1 = (int8_t)((ql[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;\n const int8_t q2 = (int8_t)((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;\n const int8_t q3 = (int8_t)((ql[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;\n const int8_t q4 = (int8_t)((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;\n y[l + 0] = d * sc[is + 0] * q1;\n y[l + 32] = d * sc[is + 2] * q2;\n y[l + 64] = d * sc[is + 4] * q3;\n y[l + 96] = d * sc[is + 6] * q4;\n }\n y += 128;\n ql += 64;\n qh += 32;\n sc += 8;\n }\n }\n}\n\nstatic void quantize_row_q6_K_impl(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {\n assert(n_per_row % QK_K == 0);\n const int64_t nb = n_per_row / QK_K;\n\n int8_t L[QK_K];\n float scales[QK_K/16];\n //float weights[16];\n\n for (int i = 0; i < nb; i++) {\n\n //float sum_x2 = 0;\n //for (int j = 0; j < QK_K; ++j) sum_x2 += x[j]*x[j];\n //float sigma2 = sum_x2/QK_K;\n\n float max_scale = 0;\n float max_abs_scale = 0;\n\n for (int ib = 0; ib < QK_K/16; ++ib) {\n\n float scale;\n if (quant_weights) {\n const float * qw = quant_weights + QK_K*i + 16*ib;\n //for (int j = 0; j < 16; ++j) weights[j] = qw[j] * sqrtf(sigma2 + x[16*ib + j]*x[16*ib + j]);\n //scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, weights);\n scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, qw);\n } else {\n scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL);\n }\n scales[ib] = scale;\n\n const float abs_scale = fabsf(scale);\n if (abs_scale > max_abs_scale) {\n max_abs_scale = abs_scale;\n max_scale = scale;\n }\n\n }\n\n if (max_abs_scale < GROUP_MAX_EPS) {\n memset(&y[i], 0, sizeof(block_q6_K));\n y[i].d = GGML_FP32_TO_FP16(0.f);\n x += QK_K;\n continue;\n }\n\n float iscale = -128.f/max_scale;\n y[i].d = GGML_FP32_TO_FP16(1/iscale);\n for (int ib = 0; ib < QK_K/16; ++ib) {\n y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]));\n }\n\n for (int j = 0; j < QK_K/16; ++j) {\n float d = GGML_FP16_TO_FP32(y[i].d) * y[i].scales[j];\n if (!d) {\n continue;\n }\n for (int ii = 0; ii < 16; ++ii) {\n int l = nearest_int(x[16*j + ii]/d);\n l = MAX(-32, MIN(31, l));\n L[16*j + ii] = l + 32;\n }\n }\n\n uint8_t * GGML_RESTRICT ql = y[i].ql;\n uint8_t * GGML_RESTRICT qh = y[i].qh;\n for (int j = 0; j < QK_K; j += 128) {\n for (int l = 0; l < 32; ++l) {\n const uint8_t q1 = L[j + l + 0] & 0xF;\n const uint8_t q2 = L[j + l + 32] & 0xF;\n const uint8_t q3 = L[j + l + 64] & 0xF;\n const uint8_t q4 = L[j + l + 96] & 0xF;\n ql[l+ 0] = q1 | (q3 << 4);\n ql[l+32] = q2 | (q4 << 4);\n qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6);\n }\n ql += 64;\n qh += 32;\n }\n\n x += QK_K;\n\n }\n}\n\nsize_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n size_t row_size = ggml_row_size(GGML_TYPE_Q6_K, n_per_row);\n if (!quant_weights) {\n quantize_row_q6_K_ref(src, dst, (int64_t)nrow*n_per_row);\n }\n else {\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_q6_K_impl(src, (block_q6_K*)qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += row_size;\n }\n }\n return nrow * row_size;\n}\n\nstatic void quantize_row_q4_0_impl(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {\n static_assert(QK4_0 == 32, \"QK4_0 must be 32\");\n\n if (!quant_weights) {\n quantize_row_q4_0_ref(x, y, n_per_row);\n return;\n }\n\n float weight[QK4_0];\n int8_t L[QK4_0];\n\n float sum_x2 = 0;\n for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];\n float sigma2 = sum_x2/n_per_row;\n\n const int64_t nb = n_per_row/QK4_0;\n for (int ib = 0; ib < nb; ++ib) {\n const float * xb = x + QK4_0 * ib;\n const float * qw = quant_weights + QK4_0 * ib;\n for (int j = 0; j < QK4_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);\n float d = make_qx_quants(QK4_0, 8, xb, L, 1, weight);\n y[ib].d = GGML_FP32_TO_FP16(d);\n for (int j = 0; j < 16; ++j) {\n y[ib].qs[j] = L[j] | (L[j+16] << 4);\n }\n }\n}\n\nsize_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n if (!quant_weights) {\n quantize_row_q4_0_ref(src, dst, (int64_t)nrow*n_per_row);\n return nrow * ggml_row_size(GGML_TYPE_Q4_0, n_per_row);\n }\n size_t row_size = ggml_row_size(GGML_TYPE_Q4_0, n_per_row);\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_q4_0_impl(src, (block_q4_0*)qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += row_size;\n }\n return nrow * row_size;\n}\n\nstatic void quantize_row_q4_1_impl(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {\n static_assert(QK4_1 == 32, \"QK4_1 must be 32\");\n\n if (!quant_weights) {\n quantize_row_q4_1_ref(x, y, n_per_row);\n return;\n }\n\n float weight[QK4_1];\n uint8_t L[QK4_1], Laux[QK4_1];\n\n float sum_x2 = 0;\n for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];\n float sigma2 = sum_x2/n_per_row;\n\n const int64_t nb = n_per_row/QK4_1;\n for (int ib = 0; ib < nb; ++ib) {\n const float * xb = x + QK4_1 * ib;\n const float * qw = quant_weights + QK4_1 * ib;\n for (int j = 0; j < QK4_1; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);\n float min;\n float d = make_qkx3_quants(QK4_1, 15, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false);\n y[ib].d = GGML_FP32_TO_FP16(d);\n y[ib].m = GGML_FP32_TO_FP16(-min);\n for (int j = 0; j < 16; ++j) {\n y[ib].qs[j] = L[j] | (L[j+16] << 4);\n }\n }\n}\n\nsize_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n if (!quant_weights) {\n quantize_row_q4_1_ref(src, dst, (int64_t)nrow*n_per_row);\n return nrow * ggml_row_size(GGML_TYPE_Q4_1, n_per_row);\n }\n size_t row_size = ggml_row_size(GGML_TYPE_Q4_1, n_per_row);\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_q4_1_impl(src, (block_q4_1*)qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += row_size;\n }\n return nrow * row_size;\n}\n\nstatic void quantize_row_q5_0_impl(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {\n static_assert(QK5_0 == 32, \"QK5_0 must be 32\");\n\n if (!quant_weights) {\n quantize_row_q5_0_ref(x, y, n_per_row);\n return;\n }\n\n float weight[QK5_0];\n int8_t L[QK5_0];\n\n float sum_x2 = 0;\n for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];\n float sigma2 = sum_x2/n_per_row;\n\n const int64_t nb = n_per_row/QK5_0;\n for (int ib = 0; ib < nb; ++ib) {\n const float * xb = x + QK5_0 * ib;\n const float * qw = quant_weights + QK5_0 * ib;\n for (int j = 0; j < QK5_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);\n float d = make_qx_quants(QK5_0, 16, xb, L, 1, weight);\n y[ib].d = GGML_FP32_TO_FP16(d);\n\n uint32_t qh = 0;\n\n for (int j = 0; j < 16; ++j) {\n const uint8_t xi0 = L[j];\n const uint8_t xi1 = L[j+16];\n y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);\n\n // get the 5-th bit and store it in qh at the right position\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);\n }\n\n memcpy(&y[ib].qh, &qh, sizeof(qh));\n }\n}\n\nsize_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n if (!quant_weights) {\n quantize_row_q5_0_ref(src, dst, (int64_t)nrow*n_per_row);\n return nrow * ggml_row_size(GGML_TYPE_Q5_0, n_per_row);\n }\n size_t row_size = ggml_row_size(GGML_TYPE_Q5_0, n_per_row);\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_q5_0_impl(src, (block_q5_0*)qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += row_size;\n }\n return nrow * row_size;\n}\n\nstatic void quantize_row_q5_1_impl(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) {\n static_assert(QK5_1 == 32, \"QK5_1 must be 32\");\n\n if (!quant_weights) {\n quantize_row_q5_1_ref(x, y, n_per_row);\n return;\n }\n\n float weight[QK5_1];\n uint8_t L[QK5_1], Laux[QK5_1];\n\n float sum_x2 = 0;\n for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];\n float sigma2 = sum_x2/n_per_row;\n\n const int64_t nb = n_per_row/QK5_1;\n for (int ib = 0; ib < nb; ++ib) {\n const float * xb = x + QK5_1 * ib;\n const float * qw = quant_weights + QK5_1 * ib;\n for (int j = 0; j < QK5_1; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);\n float min;\n float d = make_qkx3_quants(QK5_1, 31, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false);\n y[ib].d = GGML_FP32_TO_FP16(d);\n y[ib].m = GGML_FP32_TO_FP16(-min);\n\n uint32_t qh = 0;\n for (int j = 0; j < 16; ++j) {\n const uint8_t xi0 = L[j];\n const uint8_t xi1 = L[j+16];\n y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);\n // get the 5-th bit and store it in qh at the right position\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);\n }\n memcpy(&y[ib].qh, &qh, sizeof(qh));\n }\n}\n\nsize_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n if (!quant_weights) {\n quantize_row_q5_1_ref(src, dst, (int64_t)nrow*n_per_row);\n return nrow * ggml_row_size(GGML_TYPE_Q5_1, n_per_row);\n }\n size_t row_size = ggml_row_size(GGML_TYPE_Q5_1, n_per_row);\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_q5_1_impl(src, (block_q5_1*)qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += row_size;\n }\n return nrow * row_size;\n}\n\nsize_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n (void)quant_weights; // not used\n const size_t row_size = ggml_row_size(GGML_TYPE_Q8_0, n_per_row);\n quantize_row_q8_0_ref(src, dst, (int64_t)nrow*n_per_row);\n return nrow * row_size;\n}\n\nsize_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_UNUSED(quant_weights);\n quantize_row_mxfp4_ref(src, dst, (int64_t)nrow*n_per_row);\n return nrow * ggml_row_size(GGML_TYPE_MXFP4, n_per_row);\n}\n\nsize_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_UNUSED(quant_weights);\n quantize_row_nvfp4_ref(src, dst, (int64_t)nrow*n_per_row);\n return nrow * ggml_row_size(GGML_TYPE_NVFP4, n_per_row);\n}\n\n// ====================== Ternary (de)-quantization (BitNet b1.58 and TriLMs)\n\nvoid quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int64_t i = 0; i < nb; i++) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK_K; j++) {\n const float v = x[j];\n amax = MAX(amax, fabsf(v));\n }\n\n const float d = amax;\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_FP32_TO_FP16(d);\n\n // 5 elements per byte, along 32 bytes\n for (size_t j = 0; j < sizeof(y->qs) - sizeof(y->qs) % 32; j += 32) {\n for (size_t m = 0; m < 32; ++m) {\n uint8_t q = 0;\n for (size_t n = 0; n < 5; ++n) {\n int xi = lroundf(x[m + n*32] * id) + 1; // -1, 0, 1 -> 0, 1, 2\n q *= 3;\n q += xi;\n }\n // ceiling division (243 == pow(3, 5))\n q = ((uint16_t)q * 256 + (243 - 1)) / 243;\n y[i].qs[j + m] = q;\n }\n x += 5*32;\n }\n // along 16 bytes\n for (size_t j = sizeof(y->qs) - sizeof(y->qs) % 32; j < sizeof(y->qs); j += 16) {\n for (size_t m = 0; m < 16; ++m) {\n uint8_t q = 0;\n for (size_t n = 0; n < 5; ++n) {\n int xi = lroundf(x[m + n*16] * id) + 1; // -1, 0, 1 -> 0, 1, 2\n q *= 3;\n q += xi;\n }\n // ceiling division (243 == pow(3, 5))\n q = ((uint16_t)q * 256 + (243 - 1)) / 243;\n y[i].qs[j + m] = q;\n }\n x += 5*16;\n }\n // 4 elements per byte\n for (size_t j = 0; j < sizeof(y->qh); ++j) {\n uint8_t q = 0;\n for (size_t m = 0; m < 4; ++m) {\n // -1, 0, 1 -> 0, 1, 2\n int xi = lroundf(x[j + m*sizeof(y->qh)] * id) + 1;\n q *= 3;\n q += xi;\n }\n // shift the first value to the most significant trit\n q *= 3;\n // ceiling division (243 == pow(3, 5))\n q = ((uint16_t)q * 256 + (243 - 1)) / 243;\n y[i].qh[j] = q;\n }\n x += 4*sizeof(y->qh);\n }\n}\n\nvoid quantize_row_tq2_0_ref(const float * GGML_RESTRICT x, block_tq2_0 * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int64_t i = 0; i < nb; i++) {\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK_K; j++) {\n const float v = x[j];\n amax = MAX(amax, fabsf(v));\n }\n\n const float d = amax;\n const float id = d ? 1.0f/d : 0.0f;\n\n y[i].d = GGML_FP32_TO_FP16(d);\n\n for (size_t j = 0; j < sizeof(y->qs); j += 32) {\n for (size_t m = 0; m < 32; ++m) {\n uint8_t q = 0;\n for (size_t n = 0; n < 4; ++n) {\n // -1, 0, 1 -> 0, 1, 2\n int xi = lroundf(x[m + n*32] * id) + 1;\n q += (xi & 3) << (2*n);\n }\n y[i].qs[j + m] = q;\n }\n x += 4*32;\n }\n }\n}\n\nsize_t quantize_tq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n (void)quant_weights; // not used\n const size_t row_size = ggml_row_size(GGML_TYPE_TQ1_0, n_per_row);\n quantize_row_tq1_0_ref(src, dst, (int64_t)nrow*n_per_row);\n return nrow * row_size;\n}\n\nsize_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n (void)quant_weights; // not used\n const size_t row_size = ggml_row_size(GGML_TYPE_TQ2_0, n_per_row);\n quantize_row_tq2_0_ref(src, dst, (int64_t)nrow*n_per_row);\n return nrow * row_size;\n}\n\nvoid dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243};\n\n for (int64_t i = 0; i < nb; ++i) {\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n\n for (size_t j = 0; j < sizeof(x->qs) - sizeof(x->qs) % 32; j += 32) {\n for (size_t n = 0; n < 5; ++n) {\n for (size_t m = 0; m < 32; ++m) {\n uint8_t q = x[i].qs[j + m] * pow3[n];\n int16_t xi = ((uint16_t) q * 3) >> 8;\n *y++ = (float) (xi - 1) * d;\n }\n }\n }\n for (size_t j = sizeof(x->qs) - sizeof(x->qs) % 32; j < sizeof(x->qs); j += 16) {\n for (size_t n = 0; n < 5; ++n) {\n for (size_t m = 0; m < 16; ++m) {\n uint8_t q = x[i].qs[j + m] * pow3[n];\n int16_t xi = ((uint16_t) q * 3) >> 8;\n *y++ = (float) (xi - 1) * d;\n }\n }\n }\n\n for (size_t n = 0; n < 4; ++n) {\n for (size_t j = 0; j < sizeof(x->qh); ++j) {\n uint8_t q = x[i].qh[j] * pow3[n];\n int16_t xi = ((uint16_t) q * 3) >> 8;\n *y++ = (float) (xi - 1) * d;\n }\n }\n }\n}\n\nvoid dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int64_t i = 0; i < nb; ++i) {\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n\n for (size_t j = 0; j < sizeof(x->qs); j += 32) {\n for (size_t l = 0; l < 4; ++l) {\n for (size_t m = 0; m < 32; ++m) {\n int8_t q = (x[i].qs[j + m] >> (l*2)) & 3;\n *y++ = (float) (q - 1) * d;\n }\n }\n }\n }\n}\n\n// ====================== \"True\" 2-bit (de)-quantization\n\nvoid dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n uint32_t aux32[2];\n const uint8_t * aux8 = (const uint8_t *)aux32;\n\n for (int i = 0; i < nb; i++) {\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n\n for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n memcpy(aux32, x[i].qs + 4*ib32, 2*sizeof(uint32_t));\n const float db = d * (0.5f + (aux32[1] >> 28)) * 0.25f;\n for (int l = 0; l < 4; ++l) {\n const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);\n const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];\n for (int j = 0; j < 8; ++j) {\n y[j] = db * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n }\n y += 8;\n }\n }\n }\n}\n\n// ====================== 2.3125 bpw (de)-quantization\n\nvoid dequantize_row_iq2_xs(const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n float db[2];\n\n for (int i = 0; i < nb; i++) {\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n\n for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f;\n db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f;\n for (int l = 0; l < 4; ++l) {\n const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (x[i].qs[4*ib32 + l] & 511));\n const uint8_t signs = ksigns_iq2xs[x[i].qs[4*ib32 + l] >> 9];\n for (int j = 0; j < 8; ++j) {\n y[j] = db[l/2] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n }\n y += 8;\n }\n }\n }\n}\n\n// ====================== 2.5625 bpw (de)-quantization\n\nvoid dequantize_row_iq2_s(const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n float db[2];\n\n for (int i = 0; i < nb; i++) {\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n const uint8_t * qs = x[i].qs;\n const uint8_t * qh = x[i].qh;\n const uint8_t * signs = qs + QK_K/8;\n\n for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f;\n db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f;\n for (int l = 0; l < 4; ++l) {\n const float dl = db[l/2];\n const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));\n for (int j = 0; j < 8; ++j) {\n y[j] = dl * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1.f : 1.f);\n }\n y += 8;\n }\n qs += 4;\n signs += 4;\n }\n }\n}\n\n// ====================== 3.0625 bpw (de)-quantization\n\nvoid dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n uint32_t aux32;\n\n for (int i = 0; i < nb; i++) {\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n const uint8_t * qs = x[i].qs;\n const uint8_t * scales_and_signs = qs + QK_K/4;\n\n for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n memcpy(&aux32, scales_and_signs + 4*ib32, sizeof(uint32_t));\n const float db = d * (0.5f + (aux32 >> 28)) * 0.5f;\n for (int l = 0; l < 4; ++l) {\n const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];\n const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + qs[2*l+0]);\n const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + qs[2*l+1]);\n for (int j = 0; j < 4; ++j) {\n y[j+0] = db * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);\n y[j+4] = db * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);\n }\n y += 8;\n }\n qs += 8;\n }\n }\n}\n\n// ====================== 3.3125 bpw (de)-quantization\n\nvoid dequantize_row_iq3_s(const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int i = 0; i < nb; i++) {\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n const uint8_t * qs = x[i].qs;\n const uint8_t * qh = x[i].qh;\n const uint8_t * signs = x[i].signs;\n\n for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n const float db1 = d * (1 + 2*(x[i].scales[ib32/2] & 0xf));\n const float db2 = d * (1 + 2*(x[i].scales[ib32/2] >> 4));\n for (int l = 0; l < 4; ++l) {\n const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256)));\n const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256)));\n for (int j = 0; j < 4; ++j) {\n y[j+0] = db1 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f);\n y[j+4] = db1 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f);\n }\n y += 8;\n }\n qs += 8;\n signs += 4;\n for (int l = 0; l < 4; ++l) {\n const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[1] << (8-2*l)) & 256)));\n const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[1] << (7-2*l)) & 256)));\n for (int j = 0; j < 4; ++j) {\n y[j+0] = db2 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f);\n y[j+4] = db2 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f);\n }\n y += 8;\n }\n qh += 2;\n qs += 8;\n signs += 4;\n }\n }\n}\n\n// ====================== 1.5625 bpw (de)-quantization\n\nvoid dequantize_row_iq1_s(const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int i = 0; i < nb; i++) {\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n const uint8_t * qs = x[i].qs;\n const uint16_t * qh = x[i].qh;\n\n for (int ib = 0; ib < QK_K/32; ++ib) {\n const float dl = d * (2*((qh[ib] >> 12) & 7) + 1);\n const float delta = qh[ib] & 0x8000 ? -IQ1S_DELTA : IQ1S_DELTA;\n for (int l = 0; l < 4; ++l) {\n const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));\n for (int j = 0; j < 8; ++j) {\n y[j] = dl * (grid[j] + delta);\n }\n y += 8;\n }\n qs += 4;\n }\n }\n}\n\nvoid dequantize_row_iq1_m(const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n float delta[4];\n uint16_t idx[4];\n\n iq1m_scale_t scale;\n\n for (int i = 0; i < nb; i++) {\n\n const uint16_t * sc = (const uint16_t *)x[i].scales;\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n const float d = GGML_FP16_TO_FP32(scale.f16);\n\n const uint8_t * qs = x[i].qs;\n const uint8_t * qh = x[i].qh;\n\n for (int ib = 0; ib < QK_K/32; ++ib) {\n const float dl1 = d * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1);\n const float dl2 = d * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1);\n\n idx[0] = qs[0] | ((qh[0] << 8) & 0x700);\n idx[1] = qs[1] | ((qh[0] << 4) & 0x700);\n idx[2] = qs[2] | ((qh[1] << 8) & 0x700);\n idx[3] = qs[3] | ((qh[1] << 4) & 0x700);\n delta[0] = qh[0] & 0x08 ? -IQ1S_DELTA : IQ1S_DELTA;\n delta[1] = qh[0] & 0x80 ? -IQ1S_DELTA : IQ1S_DELTA;\n delta[2] = qh[1] & 0x08 ? -IQ1S_DELTA : IQ1S_DELTA;\n delta[3] = qh[1] & 0x80 ? -IQ1S_DELTA : IQ1S_DELTA;\n for (int l = 0; l < 2; ++l) {\n const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);\n for (int j = 0; j < 8; ++j) {\n y[j] = dl1 * (grid[j] + delta[l]);\n }\n y += 8;\n }\n for (int l = 2; l < 4; ++l) {\n const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);\n for (int j = 0; j < 8; ++j) {\n y[j] = dl2 * (grid[j] + delta[l]);\n }\n y += 8;\n }\n qs += 4;\n qh += 2;\n }\n }\n}\n\nvoid dequantize_row_iq4_nl(const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK4_NL == 0);\n const int64_t nb = k / QK4_NL;\n\n for (int i = 0; i < nb; i++) {\n\n const uint8_t * qs = x[i].qs;\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n for (int j = 0; j < QK4_NL/2; ++j) {\n y[j+ 0] = d * kvalues_iq4nl[qs[j] & 0xf];\n y[j+QK4_NL/2] = d * kvalues_iq4nl[qs[j] >> 4];\n }\n y += QK4_NL;\n qs += QK4_NL/2;\n }\n}\n\nvoid dequantize_row_iq4_xs(const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int i = 0; i < nb; i++) {\n\n const uint8_t * qs = x[i].qs;\n\n const float d = GGML_FP16_TO_FP32(x[i].d);\n\n for (int ib = 0; ib < QK_K/32; ++ib) {\n const int ls = ((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4);\n const float dl = d * (ls - 32);\n for (int j = 0; j < 16; ++j) {\n y[j+ 0] = dl * kvalues_iq4nl[qs[j] & 0xf];\n y[j+16] = dl * kvalues_iq4nl[qs[j] >> 4];\n }\n y += 32;\n qs += 16;\n }\n }\n}\n\n//===================================== Q8_K ==============================================\n\nvoid quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int i = 0; i < nb; i++) {\n\n float max = 0;\n float amax = 0;\n for (int j = 0; j < QK_K; ++j) {\n float ax = fabsf(x[j]);\n if (ax > amax) {\n amax = ax; max = x[j];\n }\n }\n if (!amax) {\n y[i].d = 0;\n memset(y[i].qs, 0, QK_K);\n x += QK_K;\n continue;\n }\n //const float iscale = -128.f/max;\n // We need this change for IQ2_XXS, else the AVX implementation becomes very awkward\n const float iscale = -127.f/max;\n for (int j = 0; j < QK_K; ++j) {\n int v = nearest_int(iscale*x[j]);\n y[i].qs[j] = MIN(127, v);\n }\n for (int j = 0; j < QK_K/16; ++j) {\n int sum = 0;\n for (int ii = 0; ii < 16; ++ii) {\n sum += y[i].qs[j*16 + ii];\n }\n y[i].bsums[j] = sum;\n }\n y[i].d = 1/iscale;\n x += QK_K;\n }\n}\n\nvoid dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n const int64_t nb = k / QK_K;\n\n for (int i = 0; i < nb; i++) {\n for (int j = 0; j < QK_K; ++j) {\n *y++ = x[i].d * x[i].qs[j];\n }\n }\n}\n\n// ================================ IQ2 quantization =============================================\n\ntypedef struct {\n uint64_t * grid;\n int * map;\n uint16_t * neighbours;\n} iq2_entry_t;\n\nstatic iq2_entry_t iq2_data[4] = {\n {NULL, NULL, NULL},\n {NULL, NULL, NULL},\n {NULL, NULL, NULL},\n {NULL, NULL, NULL},\n};\n\nstatic inline int iq2_data_index(enum ggml_type type) {\n GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S);\n return type == GGML_TYPE_IQ2_XXS ? 0 :\n type == GGML_TYPE_IQ2_XS ? 1 :\n type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 2 : 3;\n}\n\nstatic inline int iq2_grid_size(enum ggml_type type) {\n GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S);\n return type == GGML_TYPE_IQ2_XXS ? 256 :\n type == GGML_TYPE_IQ2_XS ? 512 :\n type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? NGRID_IQ1S : 1024;\n}\n\nstatic int iq2_compare_func(const void * left, const void * right) {\n const int * l = (const int *)left;\n const int * r = (const int *)right;\n return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;\n}\n\nvoid iq2xs_init_impl(enum ggml_type type) {\n const int gindex = iq2_data_index(type);\n const int grid_size = iq2_grid_size(type);\n if (iq2_data[gindex].grid) {\n return;\n }\n static const uint16_t kgrid_2bit_256[256] = {\n 0, 2, 5, 8, 10, 17, 20, 32, 34, 40, 42, 65, 68, 80, 88, 97,\n 100, 128, 130, 138, 162, 257, 260, 272, 277, 320, 388, 408, 512, 514, 546, 642,\n 1025, 1028, 1040, 1057, 1060, 1088, 1090, 1096, 1120, 1153, 1156, 1168, 1188, 1280, 1282, 1288,\n 1312, 1350, 1385, 1408, 1425, 1545, 1552, 1600, 1668, 1700, 2048, 2053, 2056, 2068, 2088, 2113,\n 2116, 2128, 2130, 2184, 2308, 2368, 2562, 2580, 4097, 4100, 4112, 4129, 4160, 4192, 4228, 4240,\n 4245, 4352, 4360, 4384, 4432, 4442, 4480, 4644, 4677, 5120, 5128, 5152, 5157, 5193, 5248, 5400,\n 5474, 5632, 5654, 6145, 6148, 6160, 6208, 6273, 6400, 6405, 6560, 6737, 8192, 8194, 8202, 8260,\n 8289, 8320, 8322, 8489, 8520, 8704, 8706, 9217, 9220, 9232, 9280, 9302, 9472, 9537, 9572, 9872,\n 10248, 10272, 10388, 10820, 16385, 16388, 16400, 16408, 16417, 16420, 16448, 16456, 16470, 16480, 16513, 16516,\n 16528, 16640, 16672, 16737, 16768, 16773, 16897, 16912, 16968, 16982, 17000, 17408, 17416, 17440, 17536, 17561,\n 17682, 17700, 17920, 18433, 18436, 18448, 18496, 18501, 18688, 18776, 18785, 18818, 19013, 19088, 20480, 20488,\n 20497, 20505, 20512, 20608, 20616, 20740, 20802, 20900, 21137, 21648, 21650, 21770, 22017, 22100, 22528, 22545,\n 22553, 22628, 22848, 23048, 24580, 24592, 24640, 24680, 24832, 24917, 25112, 25184, 25600, 25605, 25872, 25874,\n 25988, 26690, 32768, 32770, 32778, 32833, 32898, 33028, 33048, 33088, 33297, 33793, 33796, 33808, 33813, 33856,\n 33888, 34048, 34118, 34196, 34313, 34368, 34400, 34818, 35076, 35345, 36868, 36880, 36900, 36928, 37025, 37142,\n 37248, 37445, 37888, 37922, 37956, 38225, 39041, 39200, 40962, 41040, 41093, 41225, 41472, 42008, 43088, 43268,\n };\n static const uint16_t kgrid_2bit_512[512] = {\n 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,\n 73, 80, 82, 85, 88, 97, 100, 128, 130, 133, 136, 145, 148, 153, 160, 257,\n 260, 262, 265, 272, 274, 277, 280, 282, 289, 292, 320, 322, 325, 328, 337, 340,\n 352, 360, 385, 388, 400, 512, 514, 517, 520, 529, 532, 544, 577, 580, 592, 597,\n 640, 650, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1088, 1090, 1093, 1096,\n 1105, 1108, 1110, 1120, 1153, 1156, 1168, 1280, 1282, 1285, 1288, 1297, 1300, 1312, 1345, 1348,\n 1360, 1377, 1408, 1537, 1540, 1552, 1574, 1600, 1602, 1668, 2048, 2050, 2053, 2056, 2058, 2065,\n 2068, 2080, 2085, 2113, 2116, 2128, 2136, 2176, 2208, 2218, 2305, 2308, 2320, 2368, 2433, 2441,\n 2560, 2592, 2600, 2710, 2720, 4097, 4100, 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4160,\n 4162, 4165, 4168, 4177, 4180, 4192, 4202, 4225, 4228, 4240, 4352, 4354, 4357, 4360, 4369, 4372,\n 4384, 4417, 4420, 4432, 4480, 4500, 4502, 4609, 4612, 4614, 4624, 4672, 4704, 5120, 5122, 5125,\n 5128, 5137, 5140, 5152, 5185, 5188, 5193, 5200, 5220, 5248, 5377, 5380, 5392, 5440, 5632, 5652,\n 5705, 6145, 6148, 6160, 6162, 6208, 6228, 6278, 6400, 6405, 6502, 6737, 6825, 8192, 8194, 8197,\n 8200, 8202, 8209, 8212, 8224, 8257, 8260, 8272, 8320, 8352, 8449, 8452, 8464, 8512, 8520, 8549,\n 8704, 8738, 8832, 8872, 9217, 9220, 9232, 9257, 9280, 9472, 9537, 9554, 9625, 9729, 9754, 9894,\n 10240, 10248, 10250, 10272, 10325, 10376, 10402, 10600, 10640, 10760, 10784, 10882, 10888, 10890, 16385, 16388,\n 16390, 16393, 16400, 16402, 16405, 16408, 16417, 16420, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16480,\n 16485, 16513, 16516, 16528, 16640, 16642, 16645, 16648, 16657, 16660, 16672, 16705, 16708, 16720, 16768, 16773,\n 16802, 16897, 16900, 16912, 16914, 16937, 16960, 17408, 17410, 17413, 17416, 17425, 17428, 17433, 17440, 17473,\n 17476, 17488, 17536, 17556, 17665, 17668, 17680, 17700, 17728, 17818, 17920, 17930, 17988, 18000, 18433, 18436,\n 18448, 18496, 18501, 18516, 18530, 18688, 18705, 18756, 18768, 18793, 18948, 20480, 20482, 20485, 20488, 20497,\n 20500, 20512, 20520, 20545, 20548, 20560, 20608, 20737, 20740, 20752, 20757, 20800, 20802, 20992, 21060, 21162,\n 21505, 21508, 21520, 21537, 21568, 21600, 21633, 21665, 21760, 21768, 21888, 21896, 22049, 22120, 22177, 22528,\n 22548, 22593, 22608, 22681, 22810, 22848, 22850, 23173, 24577, 24580, 24592, 24640, 24660, 24674, 24710, 24745,\n 24832, 25124, 25162, 25234, 25600, 25622, 25872, 25920, 25925, 26020, 26625, 26730, 26917, 27142, 27220, 27234,\n 32768, 32770, 32773, 32776, 32785, 32788, 32800, 32810, 32833, 32836, 32848, 32896, 32898, 32936, 32938, 33025,\n 33028, 33030, 33040, 33088, 33105, 33113, 33280, 33312, 33408, 33410, 33440, 33448, 33793, 33796, 33808, 33810,\n 33813, 33856, 33888, 33929, 34048, 34116, 34213, 34328, 34410, 34816, 34824, 34853, 34906, 34944, 34946, 34984,\n 35078, 35362, 35456, 35464, 35478, 35496, 36865, 36868, 36880, 36928, 36950, 36996, 37120, 37154, 37220, 37462,\n 37513, 37888, 37893, 37956, 37968, 37976, 38185, 38288, 38290, 38465, 38993, 39078, 39241, 39445, 39520, 40960,\n 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048,\n 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690,\n };\n static const uint16_t kgrid_1bit_2048[NGRID_IQ1S] = {\n 0, 2, 5, 8, 10, 17, 21, 32, 34, 40, 42, 69, 81, 84, 86, 101,\n 128, 130, 136, 138, 149, 160, 162, 168, 170, 260, 261, 273, 276, 278, 281, 282,\n 293, 321, 326, 329, 338, 341, 346, 353, 356, 358, 360, 389, 401, 404, 406, 421,\n 512, 514, 520, 522, 533, 544, 546, 552, 554, 581, 593, 601, 612, 617, 640, 642,\n 648, 650, 657, 661, 665, 672, 674, 680, 682, 1041, 1044, 1046, 1061, 1089, 1097, 1109,\n 1114, 1124, 1125, 1169, 1177, 1189, 1281, 1284, 1285, 1286, 1301, 1304, 1306, 1321, 1344, 1349,\n 1354, 1360, 1361, 1364, 1365, 1366, 1369, 1376, 1378, 1381, 1384, 1386, 1409, 1425, 1429, 1432,\n 1434, 1441, 1444, 1445, 1446, 1449, 1556, 1561, 1601, 1604, 1616, 1618, 1621, 1624, 1632, 1633,\n 1638, 1641, 1669, 1681, 1684, 1689, 2048, 2050, 2056, 2058, 2069, 2080, 2082, 2088, 2090, 2117,\n 2129, 2134, 2149, 2176, 2178, 2184, 2186, 2197, 2208, 2210, 2216, 2218, 2309, 2321, 2324, 2329,\n 2340, 2341, 2369, 2384, 2385, 2389, 2401, 2404, 2409, 2449, 2452, 2454, 2457, 2469, 2560, 2562,\n 2568, 2570, 2581, 2592, 2594, 2600, 2602, 2629, 2641, 2649, 2657, 2661, 2688, 2690, 2693, 2696,\n 2698, 2709, 2720, 2722, 2728, 2730, 4112, 4113, 4116, 4121, 4132, 4133, 4161, 4164, 4176, 4181,\n 4184, 4193, 4196, 4197, 4201, 4241, 4244, 4246, 4257, 4261, 4353, 4356, 4358, 4361, 4368, 4370,\n 4373, 4376, 4385, 4388, 4393, 4421, 4426, 4432, 4433, 4434, 4436, 4437, 4438, 4441, 4448, 4453,\n 4484, 4498, 4501, 4513, 4516, 4625, 4628, 4630, 4645, 4672, 4678, 4681, 4690, 4693, 4696, 4698,\n 4708, 4710, 4741, 4753, 4756, 4758, 4773, 5121, 5126, 5129, 5140, 5141, 5144, 5145, 5153, 5158,\n 5185, 5189, 5190, 5192, 5194, 5201, 5204, 5205, 5206, 5209, 5218, 5221, 5224, 5252, 5257, 5264,\n 5268, 5269, 5272, 5273, 5274, 5281, 5284, 5285, 5289, 5378, 5381, 5386, 5393, 5396, 5397, 5398,\n 5401, 5408, 5410, 5413, 5416, 5418, 5441, 5444, 5445, 5446, 5457, 5458, 5460, 5461, 5462, 5465,\n 5466, 5473, 5476, 5477, 5478, 5481, 5504, 5506, 5508, 5509, 5512, 5514, 5520, 5521, 5524, 5525,\n 5526, 5529, 5530, 5536, 5538, 5541, 5633, 5636, 5637, 5638, 5653, 5654, 5656, 5658, 5665, 5670,\n 5696, 5698, 5700, 5701, 5704, 5706, 5713, 5717, 5718, 5720, 5721, 5729, 5732, 5733, 5736, 5737,\n 5738, 5766, 5770, 5778, 5781, 5796, 5801, 6161, 6166, 6181, 6209, 6212, 6214, 6217, 6224, 6229,\n 6232, 6234, 6240, 6241, 6244, 6246, 6249, 6277, 6289, 6292, 6309, 6416, 6418, 6421, 6426, 6433,\n 6437, 6466, 6468, 6469, 6472, 6481, 6484, 6485, 6486, 6489, 6490, 6496, 6501, 6506, 6537, 6545,\n 6546, 6549, 6552, 6561, 6566, 6569, 6665, 6678, 6692, 6694, 6724, 6726, 6729, 6736, 6738, 6741,\n 6744, 6753, 6758, 6761, 6789, 6801, 6806, 6810, 8192, 8194, 8200, 8202, 8213, 8224, 8226, 8229,\n 8232, 8234, 8261, 8273, 8281, 8289, 8293, 8320, 8322, 8328, 8330, 8341, 8352, 8354, 8357, 8360,\n 8362, 8453, 8465, 8468, 8473, 8485, 8514, 8516, 8521, 8533, 8536, 8538, 8545, 8548, 8549, 8550,\n 8581, 8592, 8598, 8601, 8613, 8705, 8712, 8714, 8721, 8725, 8736, 8738, 8744, 8746, 8773, 8785,\n 8790, 8793, 8805, 8833, 8840, 8842, 8849, 8853, 8864, 8866, 8872, 8874, 9221, 9236, 9238, 9241,\n 9253, 9284, 9285, 9286, 9289, 9298, 9301, 9304, 9306, 9318, 9349, 9361, 9364, 9369, 9377, 9381,\n 9481, 9493, 9505, 9513, 9536, 9541, 9544, 9553, 9556, 9557, 9561, 9570, 9573, 9576, 9609, 9616,\n 9620, 9621, 9624, 9626, 9633, 9636, 9638, 9641, 9733, 9744, 9746, 9753, 9765, 9793, 9801, 9813,\n 9824, 9825, 9833, 9860, 9862, 9872, 9882, 10240, 10242, 10248, 10250, 10261, 10272, 10274, 10280, 10282,\n 10309, 10321, 10324, 10341, 10368, 10370, 10376, 10378, 10400, 10402, 10408, 10410, 10505, 10513, 10516, 10521,\n 10533, 10566, 10569, 10578, 10581, 10593, 10596, 10598, 10601, 10629, 10640, 10646, 10649, 10660, 10661, 10752,\n 10754, 10760, 10762, 10784, 10786, 10792, 10794, 10821, 10833, 10838, 10841, 10853, 10880, 10882, 10888, 10890,\n 10901, 10912, 10914, 10920, 10922, 16389, 16401, 16406, 16421, 16457, 16466, 16469, 16472, 16474, 16481, 16484,\n 16486, 16532, 16537, 16545, 16550, 16640, 16641, 16644, 16646, 16649, 16658, 16661, 16662, 16664, 16666, 16673,\n 16678, 16681, 16709, 16712, 16714, 16721, 16724, 16725, 16726, 16729, 16730, 16741, 16744, 16746, 16769, 16772,\n 16774, 16784, 16786, 16789, 16800, 16801, 16802, 16901, 16913, 16916, 16918, 16933, 16961, 16978, 16981, 16986,\n 16996, 17001, 17033, 17044, 17061, 17409, 17429, 17433, 17449, 17477, 17480, 17482, 17489, 17492, 17493, 17494,\n 17505, 17506, 17509, 17512, 17514, 17537, 17542, 17545, 17552, 17554, 17557, 17568, 17569, 17577, 17665, 17666,\n 17669, 17674, 17681, 17684, 17685, 17686, 17689, 17696, 17701, 17706, 17729, 17732, 17733, 17734, 17737, 17744,\n 17745, 17748, 17749, 17750, 17752, 17753, 17761, 17764, 17765, 17766, 17769, 17794, 17796, 17797, 17800, 17809,\n 17812, 17813, 17814, 17817, 17818, 17829, 17832, 17834, 17921, 17925, 17929, 17940, 17941, 17944, 17946, 17953,\n 17956, 17961, 17984, 17986, 17989, 17992, 18000, 18001, 18002, 18005, 18006, 18009, 18018, 18021, 18024, 18049,\n 18053, 18058, 18068, 18069, 18081, 18084, 18086, 18437, 18449, 18453, 18458, 18469, 18498, 18505, 18512, 18517,\n 18520, 18529, 18532, 18534, 18537, 18565, 18577, 18580, 18582, 18585, 18597, 18689, 18693, 18694, 18698, 18704,\n 18708, 18709, 18712, 18721, 18724, 18726, 18752, 18757, 18762, 18769, 18770, 18772, 18773, 18774, 18777, 18784,\n 18786, 18789, 18790, 18794, 18822, 18825, 18834, 18837, 18838, 18840, 18849, 18852, 18854, 18857, 18966, 19012,\n 19014, 19017, 19029, 19032, 19034, 19044, 19049, 19092, 19109, 20481, 20484, 20485, 20486, 20489, 20498, 20501,\n 20506, 20513, 20516, 20521, 20544, 20549, 20552, 20561, 20564, 20565, 20566, 20569, 20581, 20584, 20614, 20617,\n 20629, 20632, 20640, 20641, 20646, 20649, 20741, 20744, 20745, 20746, 20753, 20756, 20757, 20758, 20760, 20761,\n 20768, 20773, 20774, 20776, 20778, 20801, 20804, 20805, 20806, 20809, 20816, 20817, 20818, 20820, 20821, 20822,\n 20824, 20825, 20826, 20833, 20836, 20837, 20838, 20841, 20866, 20869, 20881, 20884, 20885, 20886, 20889, 20896,\n 20901, 20906, 20993, 20998, 21010, 21013, 21018, 21025, 21028, 21058, 21061, 21066, 21073, 21076, 21077, 21078,\n 21081, 21090, 21093, 21125, 21136, 21138, 21141, 21145, 21146, 21156, 21508, 21509, 21521, 21524, 21525, 21526,\n 21528, 21529, 21537, 21541, 21544, 21546, 21569, 21572, 21573, 21574, 21577, 21578, 21584, 21585, 21588, 21589,\n 21590, 21592, 21593, 21594, 21601, 21602, 21604, 21605, 21606, 21609, 21632, 21640, 21642, 21649, 21652, 21653,\n 21654, 21657, 21665, 21668, 21669, 21674, 21761, 21762, 21764, 21765, 21766, 21769, 21776, 21777, 21778, 21780,\n 21781, 21782, 21785, 21786, 21793, 21796, 21797, 21798, 21801, 21824, 21825, 21826, 21828, 21829, 21830, 21832,\n 21833, 21840, 21841, 21842, 21844, 21845, 21846, 21848, 21849, 21850, 21856, 21857, 21860, 21861, 21862, 21864,\n 21865, 21866, 21889, 21892, 21893, 21897, 21898, 21904, 21905, 21908, 21909, 21910, 21912, 21913, 21921, 21924,\n 21925, 21926, 21929, 22016, 22017, 22018, 22020, 22022, 22024, 22025, 22033, 22036, 22037, 22040, 22041, 22048,\n 22049, 22050, 22052, 22053, 22054, 22056, 22057, 22081, 22085, 22086, 22088, 22089, 22090, 22096, 22097, 22098,\n 22100, 22101, 22102, 22104, 22105, 22106, 22113, 22116, 22117, 22121, 22146, 22149, 22150, 22152, 22153, 22154,\n 22161, 22165, 22170, 22178, 22181, 22182, 22184, 22185, 22532, 22533, 22534, 22537, 22544, 22549, 22552, 22561,\n 22570, 22597, 22600, 22602, 22609, 22612, 22613, 22614, 22616, 22617, 22624, 22626, 22628, 22629, 22658, 22665,\n 22672, 22674, 22677, 22680, 22689, 22697, 22785, 22786, 22789, 22794, 22801, 22804, 22805, 22806, 22809, 22821,\n 22849, 22852, 22853, 22854, 22857, 22864, 22865, 22866, 22868, 22869, 22870, 22872, 22873, 22874, 22881, 22884,\n 22885, 22886, 22889, 22913, 22917, 22921, 22929, 22932, 22933, 22934, 22936, 22937, 22949, 23044, 23048, 23061,\n 23066, 23072, 23077, 23078, 23081, 23109, 23112, 23113, 23121, 23125, 23126, 23128, 23129, 23138, 23141, 23144,\n 23146, 23169, 23178, 23186, 23189, 23190, 23192, 23194, 23201, 24581, 24596, 24598, 24601, 24613, 24644, 24656,\n 24661, 24662, 24664, 24666, 24673, 24676, 24678, 24681, 24705, 24726, 24741, 24833, 24836, 24838, 24841, 24850,\n 24853, 24865, 24866, 24870, 24873, 24901, 24905, 24913, 24917, 24918, 24921, 24933, 24934, 24938, 24964, 24970,\n 24978, 24981, 24993, 24998, 25001, 25105, 25110, 25113, 25152, 25153, 25158, 25173, 25174, 25176, 25184, 25221,\n 25233, 25238, 25253, 25617, 25618, 25621, 25622, 25626, 25633, 25638, 25641, 25664, 25666, 25669, 25672, 25674,\n 25681, 25684, 25685, 25686, 25689, 25690, 25696, 25698, 25701, 25732, 25733, 25737, 25744, 25746, 25748, 25749,\n 25750, 25752, 25754, 25761, 25764, 25769, 25861, 25864, 25866, 25873, 25877, 25878, 25881, 25924, 25925, 25926,\n 25929, 25936, 25937, 25940, 25941, 25942, 25945, 25953, 25956, 25957, 25958, 25961, 25990, 25993, 25994, 26001,\n 26005, 26006, 26009, 26010, 26018, 26021, 26022, 26024, 26114, 26121, 26133, 26144, 26150, 26152, 26153, 26176,\n 26181, 26184, 26186, 26193, 26196, 26197, 26198, 26200, 26202, 26208, 26213, 26216, 26240, 26242, 26245, 26250,\n 26260, 26262, 26264, 26265, 26272, 26276, 26278, 26282, 26646, 26649, 26661, 26689, 26706, 26709, 26714, 26721,\n 26729, 26757, 26769, 26776, 26790, 26881, 26884, 26896, 26901, 26913, 26916, 26918, 26921, 26944, 26945, 26949,\n 26950, 26952, 26961, 26964, 26965, 26966, 26969, 26976, 26981, 26986, 27010, 27012, 27018, 27029, 27041, 27044,\n 27045, 27049, 27153, 27158, 27160, 27201, 27204, 27209, 27216, 27221, 27224, 27226, 27236, 27237, 27241, 27270,\n 27284, 27288, 27290, 27302, 32768, 32770, 32776, 32778, 32800, 32802, 32808, 32810, 32837, 32848, 32849, 32852,\n 32854, 32857, 32869, 32896, 32898, 32904, 32906, 32917, 32928, 32930, 32936, 32938, 33029, 33041, 33044, 33046,\n 33049, 33061, 33089, 33092, 33097, 33104, 33106, 33109, 33110, 33112, 33113, 33124, 33126, 33129, 33157, 33161,\n 33172, 33174, 33177, 33189, 33280, 33282, 33288, 33290, 33301, 33312, 33314, 33320, 33322, 33361, 33364, 33369,\n 33381, 33408, 33410, 33416, 33418, 33429, 33440, 33442, 33448, 33450, 33812, 33817, 33857, 33860, 33873, 33877,\n 33882, 33889, 33892, 33897, 33940, 33945, 34049, 34057, 34066, 34069, 34074, 34086, 34089, 34112, 34113, 34117,\n 34120, 34129, 34132, 34133, 34134, 34137, 34138, 34149, 34150, 34152, 34154, 34177, 34180, 34182, 34185, 34192,\n 34194, 34197, 34200, 34214, 34321, 34326, 34329, 34341, 34369, 34372, 34377, 34378, 34384, 34389, 34393, 34394,\n 34401, 34406, 34410, 34437, 34449, 34458, 34468, 34816, 34818, 34824, 34826, 34837, 34848, 34850, 34856, 34858,\n 34881, 34885, 34897, 34900, 34905, 34917, 34921, 34944, 34946, 34952, 34954, 34965, 34976, 34978, 34984, 34986,\n 35077, 35078, 35089, 35092, 35094, 35109, 35137, 35140, 35142, 35145, 35152, 35154, 35157, 35162, 35169, 35172,\n 35205, 35222, 35225, 35237, 35328, 35330, 35336, 35338, 35349, 35360, 35362, 35368, 35370, 35397, 35409, 35412,\n 35414, 35456, 35458, 35464, 35466, 35477, 35488, 35490, 35496, 35498, 36869, 36881, 36886, 36888, 36889, 36901,\n 36929, 36934, 36937, 36949, 36952, 36954, 36969, 36970, 36997, 37009, 37012, 37014, 37017, 37029, 37121, 37124,\n 37126, 37129, 37136, 37141, 37144, 37146, 37153, 37156, 37158, 37161, 37184, 37189, 37200, 37201, 37204, 37205,\n 37206, 37209, 37218, 37221, 37252, 37254, 37266, 37269, 37272, 37281, 37284, 37286, 37289, 37381, 37393, 37396,\n 37401, 37413, 37444, 37446, 37449, 37456, 37458, 37461, 37464, 37478, 37481, 37509, 37524, 37526, 37545, 37889,\n 37892, 37894, 37904, 37909, 37912, 37926, 37952, 37962, 37969, 37972, 37973, 37974, 37976, 37977, 37984, 37985,\n 37986, 37989, 38020, 38022, 38034, 38036, 38037, 38040, 38049, 38057, 38144, 38149, 38152, 38154, 38160, 38161,\n 38164, 38165, 38166, 38169, 38177, 38181, 38185, 38186, 38209, 38212, 38213, 38214, 38217, 38224, 38225, 38226,\n 38228, 38229, 38230, 38232, 38233, 38234, 38241, 38244, 38245, 38246, 38249, 38273, 38277, 38280, 38289, 38290,\n 38292, 38293, 38294, 38297, 38298, 38304, 38306, 38309, 38312, 38314, 38401, 38404, 38416, 38421, 38425, 38432,\n 38438, 38441, 38469, 38472, 38473, 38481, 38482, 38485, 38486, 38489, 38501, 38504, 38530, 38532, 38537, 38538,\n 38546, 38548, 38549, 38564, 38566, 38569, 38917, 38934, 38937, 38949, 38977, 38982, 38992, 38994, 38997, 38998,\n 39002, 39012, 39013, 39045, 39057, 39062, 39065, 39077, 39172, 39174, 39177, 39184, 39186, 39189, 39192, 39194,\n 39200, 39201, 39204, 39206, 39232, 39234, 39237, 39240, 39242, 39249, 39252, 39253, 39254, 39257, 39266, 39269,\n 39270, 39274, 39297, 39300, 39312, 39314, 39317, 39322, 39329, 39334, 39429, 39445, 39461, 39492, 39494, 39497,\n 39504, 39509, 39512, 39521, 39557, 39569, 39572, 39573, 39574, 40960, 40962, 40968, 40970, 40981, 40992, 40994,\n 41000, 41002, 41029, 41041, 41044, 41046, 41049, 41088, 41090, 41096, 41098, 41109, 41120, 41122, 41128, 41130,\n 41221, 41225, 41233, 41236, 41238, 41241, 41242, 41286, 41289, 41297, 41301, 41304, 41306, 41313, 41316, 41349,\n 41360, 41362, 41366, 41369, 41474, 41480, 41482, 41488, 41497, 41506, 41512, 41514, 41541, 41553, 41558, 41561,\n 41573, 41600, 41602, 41608, 41610, 41621, 41632, 41634, 41640, 41642, 42009, 42021, 42049, 42052, 42064, 42068,\n 42069, 42072, 42074, 42081, 42085, 42086, 42088, 42089, 42117, 42246, 42249, 42256, 42258, 42261, 42264, 42278,\n 42281, 42306, 42309, 42321, 42324, 42325, 42326, 42329, 42341, 42346, 42369, 42372, 42373, 42374, 42377, 42386,\n 42389, 42392, 42501, 42513, 42518, 42522, 42529, 42533, 42564, 42566, 42570, 42578, 42581, 42582, 42584, 42592,\n 42594, 42630, 42640, 42645, 42646, 42649, 42657, 42660, 42662, 43008, 43010, 43016, 43018, 43040, 43042, 43048,\n 43050, 43089, 43092, 43094, 43097, 43136, 43138, 43144, 43146, 43157, 43168, 43170, 43176, 43178, 43269, 43284,\n 43289, 43297, 43301, 43329, 43344, 43349, 43354, 43361, 43366, 43369, 43408, 43414, 43520, 43522, 43528, 43530,\n 43552, 43554, 43560, 43562, 43601, 43604, 43606, 43648, 43650, 43656, 43658, 43669, 43680, 43682, 43688, 43690,\n };\n static const uint16_t kgrid_2bit_1024[1024] = {\n 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,\n 73, 80, 82, 85, 88, 97, 100, 102, 105, 128, 130, 133, 136, 145, 148, 160,\n 165, 170, 257, 260, 262, 265, 272, 274, 277, 280, 289, 292, 320, 322, 325, 328,\n 337, 340, 342, 345, 352, 357, 360, 385, 388, 400, 402, 405, 417, 420, 512, 514,\n 517, 520, 529, 532, 544, 554, 577, 580, 582, 585, 592, 597, 640, 645, 650, 660,\n 674, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1062, 1065, 1088, 1090, 1093,\n 1096, 1098, 1105, 1108, 1110, 1113, 1120, 1122, 1125, 1153, 1156, 1158, 1161, 1168, 1173, 1176,\n 1185, 1188, 1280, 1282, 1285, 1288, 1290, 1297, 1300, 1302, 1305, 1312, 1317, 1320, 1345, 1348,\n 1350, 1353, 1360, 1362, 1365, 1368, 1377, 1380, 1408, 1410, 1413, 1416, 1425, 1428, 1440, 1537,\n 1540, 1542, 1545, 1552, 1557, 1600, 1605, 1608, 1617, 1620, 1632, 1665, 1668, 1680, 2048, 2050,\n 2053, 2056, 2065, 2068, 2070, 2073, 2080, 2085, 2090, 2113, 2116, 2118, 2121, 2128, 2130, 2133,\n 2136, 2145, 2148, 2176, 2181, 2196, 2218, 2305, 2308, 2320, 2322, 2325, 2328, 2337, 2368, 2373,\n 2376, 2385, 2388, 2400, 2433, 2448, 2560, 2577, 2580, 2594, 2600, 2602, 2640, 2713, 4097, 4100,\n 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4134, 4160, 4162, 4165, 4168, 4177, 4180, 4182,\n 4185, 4192, 4194, 4197, 4200, 4225, 4228, 4230, 4240, 4245, 4248, 4257, 4260, 4352, 4354, 4357,\n 4360, 4362, 4369, 4372, 4374, 4377, 4384, 4386, 4389, 4392, 4417, 4420, 4422, 4425, 4432, 4434,\n 4437, 4440, 4449, 4452, 4480, 4482, 4485, 4488, 4497, 4500, 4609, 4612, 4617, 4624, 4629, 4641,\n 4644, 4672, 4677, 4689, 4692, 4737, 4740, 4752, 5120, 5122, 5125, 5128, 5137, 5140, 5142, 5145,\n 5152, 5157, 5160, 5185, 5188, 5190, 5193, 5200, 5202, 5205, 5208, 5217, 5220, 5248, 5250, 5253,\n 5256, 5265, 5268, 5280, 5377, 5380, 5382, 5385, 5392, 5394, 5397, 5400, 5409, 5412, 5440, 5442,\n 5445, 5448, 5457, 5460, 5472, 5505, 5508, 5520, 5632, 5637, 5640, 5649, 5652, 5664, 5697, 5700,\n 5712, 5760, 5802, 6145, 6148, 6150, 6153, 6160, 6165, 6168, 6177, 6208, 6210, 6213, 6216, 6225,\n 6228, 6240, 6273, 6276, 6400, 6402, 6405, 6408, 6417, 6420, 6432, 6465, 6468, 6480, 6505, 6562,\n 6660, 6672, 6720, 6742, 8192, 8194, 8197, 8200, 8209, 8212, 8214, 8217, 8224, 8229, 8234, 8257,\n 8260, 8272, 8274, 8277, 8292, 8320, 8330, 8340, 8362, 8449, 8452, 8464, 8466, 8469, 8481, 8512,\n 8514, 8517, 8529, 8532, 8544, 8577, 8580, 8592, 8704, 8714, 8738, 8744, 8746, 8772, 8784, 8840,\n 8842, 8872, 9217, 9220, 9222, 9225, 9232, 9237, 9240, 9249, 9252, 9280, 9282, 9285, 9288, 9297,\n 9300, 9312, 9345, 9348, 9360, 9472, 9477, 9480, 9489, 9492, 9504, 9537, 9540, 9552, 9574, 9600,\n 9729, 9732, 9744, 9792, 9817, 10240, 10245, 10257, 10260, 10305, 10308, 10320, 10378, 10410, 10497, 10500,\n 10512, 10645, 10762, 10786, 10852, 10888, 10890, 16385, 16388, 16390, 16393, 16400, 16402, 16405, 16408, 16410,\n 16417, 16420, 16422, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16470, 16473, 16480, 16482, 16485, 16513,\n 16516, 16528, 16533, 16536, 16545, 16548, 16640, 16642, 16645, 16648, 16657, 16660, 16662, 16665, 16672, 16674,\n 16677, 16705, 16708, 16710, 16713, 16720, 16722, 16725, 16728, 16737, 16740, 16768, 16770, 16773, 16776, 16785,\n 16788, 16800, 16897, 16900, 16912, 16914, 16917, 16920, 16932, 16960, 16965, 16968, 16977, 16980, 16992, 17025,\n 17028, 17408, 17410, 17413, 17416, 17418, 17425, 17428, 17430, 17433, 17440, 17442, 17445, 17448, 17473, 17476,\n 17478, 17481, 17488, 17490, 17493, 17496, 17505, 17508, 17536, 17538, 17541, 17544, 17553, 17556, 17568, 17665,\n 17668, 17670, 17673, 17680, 17682, 17685, 17688, 17697, 17700, 17728, 17730, 17733, 17736, 17745, 17748, 17760,\n 17770, 17793, 17796, 17808, 17920, 17922, 17925, 17928, 17937, 17940, 17952, 17985, 17988, 18000, 18048, 18085,\n 18433, 18436, 18441, 18448, 18450, 18453, 18456, 18465, 18468, 18496, 18498, 18501, 18504, 18513, 18516, 18528,\n 18564, 18576, 18688, 18690, 18693, 18696, 18705, 18708, 18720, 18753, 18756, 18768, 18816, 18838, 18945, 18948,\n 18960, 19008, 20480, 20482, 20485, 20488, 20497, 20500, 20502, 20505, 20512, 20514, 20517, 20520, 20545, 20548,\n 20550, 20553, 20560, 20562, 20565, 20568, 20577, 20580, 20608, 20610, 20613, 20616, 20625, 20628, 20737, 20740,\n 20742, 20745, 20752, 20754, 20757, 20760, 20769, 20772, 20800, 20802, 20805, 20808, 20817, 20820, 20832, 20865,\n 20868, 20880, 20992, 20997, 21000, 21009, 21012, 21024, 21057, 21060, 21072, 21097, 21120, 21505, 21508, 21510,\n 21513, 21520, 21522, 21525, 21528, 21537, 21540, 21568, 21570, 21573, 21576, 21585, 21588, 21600, 21633, 21636,\n 21648, 21760, 21762, 21765, 21768, 21777, 21780, 21792, 21825, 21828, 21840, 21888, 22017, 22020, 22032, 22054,\n 22080, 22528, 22530, 22533, 22536, 22545, 22548, 22560, 22593, 22596, 22608, 22618, 22656, 22785, 22788, 22800,\n 22848, 23040, 23065, 23173, 23208, 24577, 24580, 24582, 24592, 24594, 24597, 24600, 24609, 24612, 24640, 24645,\n 24648, 24657, 24660, 24672, 24708, 24720, 24832, 24834, 24837, 24840, 24849, 24852, 24864, 24897, 24900, 24912,\n 24960, 24985, 25092, 25104, 25152, 25174, 25249, 25600, 25605, 25608, 25617, 25620, 25632, 25665, 25668, 25680,\n 25728, 25857, 25860, 25872, 25920, 25930, 25960, 26002, 26112, 26260, 26625, 26628, 26640, 26725, 26776, 26880,\n 26922, 27202, 27297, 32768, 32770, 32773, 32776, 32785, 32788, 32793, 32800, 32805, 32833, 32836, 32848, 32850,\n 32853, 32856, 32865, 32896, 32901, 32913, 32916, 33025, 33028, 33033, 33040, 33042, 33045, 33048, 33057, 33060,\n 33088, 33090, 33093, 33096, 33105, 33108, 33153, 33156, 33168, 33193, 33280, 33285, 33290, 33297, 33300, 33345,\n 33348, 33360, 33793, 33796, 33798, 33801, 33808, 33810, 33813, 33816, 33825, 33856, 33858, 33861, 33864, 33873,\n 33876, 33888, 33921, 33924, 33936, 34048, 34050, 34053, 34056, 34065, 34068, 34080, 34113, 34116, 34128, 34176,\n 34186, 34305, 34308, 34320, 34345, 34368, 34816, 34821, 34833, 34836, 34881, 34884, 34896, 34978, 35073, 35076,\n 35136, 35173, 35362, 35416, 35418, 35458, 35490, 36865, 36868, 36873, 36880, 36882, 36885, 36888, 36900, 36928,\n 36930, 36933, 36936, 36945, 36948, 36960, 36993, 36996, 37008, 37120, 37125, 37137, 37140, 37185, 37188, 37200,\n 37210, 37377, 37380, 37392, 37440, 37542, 37888, 37890, 37893, 37896, 37905, 37908, 37920, 37953, 37956, 37968,\n 38016, 38038, 38145, 38148, 38160, 38208, 38296, 38305, 38400, 38470, 38500, 38913, 38916, 38928, 38950, 38976,\n 39081, 39168, 39241, 39250, 39568, 40960, 40965, 40970, 40980, 40994, 41002, 41025, 41028, 41040, 41122, 41130,\n 41280, 41317, 41474, 41482, 41506, 41512, 41514, 41602, 41608, 41610, 41640, 41985, 41988, 42000, 42048, 42121,\n 42148, 42240, 42265, 42577, 43018, 43048, 43170, 43348, 43398, 43528, 43530, 43552, 43554, 43560, 43656, 43690,\n };\n\n const int kmap_size = 43692;\n //const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2;\n const int nwant = type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 3 : type == GGML_TYPE_IQ2_S ? 1 : 2;\n const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 :\n type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 :\n type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? kgrid_1bit_2048 : kgrid_2bit_1024;\n uint64_t * kgrid_q2xs;\n int * kmap_q2xs;\n uint16_t * kneighbors_q2xs;\n\n //printf(\"================================================================= %s(grid_size = %d)\\n\", __func__, grid_size);\n uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t));\n for (int k = 0; k < grid_size; ++k) {\n int8_t * pos = (int8_t *)(the_grid + k);\n for (int i = 0; i < 8; ++i) {\n int l = (kgrid[k] >> 2*i) & 0x3;\n pos[i] = 2*l + 1;\n }\n }\n kgrid_q2xs = the_grid;\n iq2_data[gindex].grid = the_grid;\n kmap_q2xs = (int *)malloc(kmap_size*sizeof(int));\n iq2_data[gindex].map = kmap_q2xs;\n for (int i = 0; i < kmap_size; ++i) kmap_q2xs[i] = -1;\n uint64_t aux64;\n uint8_t * aux8 = (uint8_t *)&aux64;\n for (int i = 0; i < grid_size; ++i) {\n aux64 = kgrid_q2xs[i];\n uint16_t index = 0;\n for (int k=0; k<8; ++k) {\n uint16_t q = (aux8[k] - 1)/2;\n index |= (q << 2*k);\n }\n kmap_q2xs[index] = i;\n }\n int8_t pos[8];\n int * dist2 = (int *)malloc(2*grid_size*sizeof(int));\n int num_neighbors = 0, num_not_in_map = 0;\n for (int i = 0; i < kmap_size; ++i) {\n if (kmap_q2xs[i] >= 0) continue;\n ++num_not_in_map;\n for (int k = 0; k < 8; ++k) {\n int l = (i >> 2*k) & 0x3;\n pos[k] = 2*l + 1;\n }\n for (int j = 0; j < grid_size; ++j) {\n const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);\n int d2 = 0;\n for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);\n dist2[2*j+0] = d2;\n dist2[2*j+1] = j;\n }\n qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);\n int n = 0; int d2 = dist2[0];\n int nhave = 1;\n for (int j = 0; j < grid_size; ++j) {\n if (dist2[2*j] > d2) {\n if (nhave == nwant) break;\n d2 = dist2[2*j];\n ++nhave;\n }\n ++n;\n }\n num_neighbors += n;\n }\n //printf(\"%s: %d neighbours in total\\n\", __func__, num_neighbors);\n kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));\n iq2_data[gindex].neighbours = kneighbors_q2xs;\n int counter = 0;\n for (int i = 0; i < kmap_size; ++i) {\n if (kmap_q2xs[i] >= 0) continue;\n for (int k = 0; k < 8; ++k) {\n int l = (i >> 2*k) & 0x3;\n pos[k] = 2*l + 1;\n }\n for (int j = 0; j < grid_size; ++j) {\n const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);\n int d2 = 0;\n for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);\n dist2[2*j+0] = d2;\n dist2[2*j+1] = j;\n }\n qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);\n kmap_q2xs[i] = -(counter + 1);\n int d2 = dist2[0];\n uint16_t * start = &kneighbors_q2xs[counter++];\n int n = 0, nhave = 1;\n for (int j = 0; j < grid_size; ++j) {\n if (dist2[2*j] > d2) {\n if (nhave == nwant) break;\n d2 = dist2[2*j];\n ++nhave;\n }\n kneighbors_q2xs[counter++] = dist2[2*j+1];\n ++n;\n }\n *start = n;\n }\n free(dist2);\n}\n\nvoid iq2xs_free_impl(enum ggml_type type) {\n GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S);\n const int gindex = iq2_data_index(type);\n if (iq2_data[gindex].grid) {\n free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL;\n free(iq2_data[gindex].map); iq2_data[gindex].map = NULL;\n free(iq2_data[gindex].neighbours); iq2_data[gindex].neighbours = NULL;\n }\n}\n\nstatic int iq2_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,\n const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) {\n int num_neighbors = neighbours[0];\n GGML_ASSERT(num_neighbors > 0);\n float best_d2 = FLT_MAX;\n int grid_index = -1;\n for (int j = 1; j <= num_neighbors; ++j) {\n const int8_t * pg = (const int8_t *)(grid + neighbours[j]);\n float d2 = 0;\n for (int i = 0; i < 8; ++i) {\n float q = pg[i];\n float diff = scale*q - xval[i];\n d2 += weight[i]*diff*diff;\n }\n if (d2 < best_d2) {\n best_d2 = d2; grid_index = neighbours[j];\n }\n }\n GGML_ASSERT(grid_index >= 0);\n const int8_t * pg = (const int8_t *)(grid + grid_index);\n for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;\n return grid_index;\n}\n\nstatic void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {\n\n const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS);\n\n const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;\n const int * kmap_q2xs = iq2_data[gindex].map;\n const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;\n\n GGML_ASSERT(quant_weights && \"missing quantization weights\");\n GGML_ASSERT(kgrid_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kmap_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(n%QK_K == 0);\n\n const int kMaxQ = 3;\n\n const int64_t nbl = n/QK_K;\n\n block_iq2_xxs * y = vy;\n\n float scales[QK_K/32];\n float weight[32];\n float xval[32];\n int8_t L[32];\n int8_t Laux[32];\n float waux[32];\n uint8_t block_signs[4];\n uint32_t q2[2*(QK_K/32)];\n\n for (int ibl = 0; ibl < nbl; ++ibl) {\n\n y[ibl].d = GGML_FP32_TO_FP16(0.f);\n memset(q2, 0, QK_K/4);\n\n float max_scale = 0;\n\n const float * xbl = x + QK_K*ibl;\n float sumx2 = 0;\n for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];\n float sigma2 = sumx2/QK_K;\n\n for (int ib = 0; ib < QK_K/32; ++ib) {\n const float * xb = xbl + 32*ib;\n const float * qw = quant_weights + QK_K*ibl + 32*ib;\n for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);\n for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]);\n for (int k = 0; k < 4; ++k) {\n int nflip = 0;\n uint8_t s = 0;\n for (int i = 0; i < 8; ++i) {\n if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];\n else {\n xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);\n }\n }\n if (nflip%2) {\n int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];\n for (int i = 1; i < 8; ++i) {\n float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];\n if (ax < min) {\n min = ax; imin = i;\n }\n }\n xval[8*k+imin] = -xval[8*k+imin];\n s ^= (1 << imin);\n }\n block_signs[k] = s & 127;\n }\n float max = xval[0];\n for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);\n if (max < GROUP_MAX_EPS) {\n scales[ib] = 0;\n memset(L, 0, 32);\n continue;\n }\n float scale = make_qp_quants(32, kMaxQ+1, xval, (uint8_t*)L, weight);\n float eff_max = scale*kMaxQ;\n if (eff_max <= 0) {\n scales[ib] = 0;\n memset(L, 0, 32);\n continue;\n }\n float best = 0;\n for (int is = -6; is <= 6; ++is) {\n float id = (2*kMaxQ-1+is*0.1f)/eff_max;\n float this_scale = 1/id;\n for (int k = 0; k < 4; ++k) {\n for (int i = 0; i < 8; ++i) {\n int l = nearest_int(0.5f*(id*xval[8*k+i]-1));\n Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));\n }\n uint16_t u = 0;\n for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);\n int grid_index = kmap_q2xs[u];\n if (grid_index < 0) {\n const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;\n grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);\n }\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 32; ++i) {\n float w = weight[i];\n float q = 2*Laux[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {\n scale = sumqx/sumq2; best = scale*sumqx;\n memcpy(L, Laux, 32);\n }\n }\n if (scale > 0) {\n float id = 1/scale;\n for (int k = 0; k < 4; ++k) {\n uint16_t u = 0;\n for (int i = 0; i < 8; ++i) {\n int l = nearest_int(0.5f*(id*xval[8*k+i]-1));\n l = MAX(0, MIN(kMaxQ-1, l));\n u |= (l << 2*i);\n }\n int grid_index = kmap_q2xs[u];\n if (grid_index < 0) {\n const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;\n grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);\n }\n const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);\n for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 32; ++i) {\n float w = weight[i];\n float q = 2*L[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0) scale = sumqx/sumq2;\n }\n if (scale < 0) {\n // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)\n // and correspondingly flip quant signs.\n scale = -scale;\n for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127;\n }\n for (int k = 0; k < 4; ++k) {\n uint16_t u = 0;\n for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);\n int grid_index = kmap_q2xs[u];\n if (grid_index < 0) {\n printf(\"Oops: found point %u not on grid:\", u);\n for (int i = 0; i < 8; ++i) printf(\" %d\", L[8*k+i]);\n printf(\"\\n\");\n GGML_ABORT(\"fatal error\");\n }\n q2[2*ib+0] |= ((uint32_t) grid_index << 8*k);\n q2[2*ib+1] |= (block_signs[k] << 7*k);\n }\n GGML_ASSERT(scale >= 0);\n scales[ib] = scale;\n max_scale = MAX(max_scale, scale);\n }\n\n if (!max_scale) {\n memset(y[ibl].qs, 0, QK_K/4);\n continue;\n }\n\n float d = max_scale/31;\n y[ibl].d = GGML_FP32_TO_FP16(d);\n float id = 1/d;\n for (int ib = 0; ib < QK_K/32; ++ib) {\n int l = nearest_int(0.5f*(id*scales[ib]-1));\n l = MAX(0, MIN(15, l));\n q2[2*ib+1] |= ((uint32_t)l << 28);\n }\n memcpy(y[ibl].qs, q2, QK_K/4);\n }\n}\n\nstatic void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {\n\n const int gindex = iq2_data_index(GGML_TYPE_IQ2_XS);\n\n const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;\n const int * kmap_q2xs = iq2_data[gindex].map;\n const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;\n\n GGML_ASSERT(quant_weights && \"missing quantization weights\");\n GGML_ASSERT(kmap_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kgrid_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(n%QK_K == 0);\n\n const int kMaxQ = 3;\n\n const int64_t nbl = n/QK_K;\n\n block_iq2_xs * y = vy;\n\n float scales[QK_K/16];\n float weight[16];\n float xval[16];\n int8_t L[16];\n int8_t Laux[16];\n float waux[16];\n bool is_on_grid[2];\n bool is_on_grid_aux[2];\n uint8_t block_signs[2];\n uint16_t q2[2*(QK_K/16)];\n\n for (int ibl = 0; ibl < nbl; ++ibl) {\n\n y[ibl].d = GGML_FP32_TO_FP16(0.f);\n memset(q2, 0, QK_K/4);\n memset(y[ibl].scales, 0, QK_K/32);\n\n float max_scale = 0;\n\n const float * xbl = x + QK_K*ibl;\n float sumx2 = 0;\n for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];\n float sigma2 = sumx2/QK_K;\n\n for (int ib = 0; ib < QK_K/16; ++ib) {\n const float * xb = xbl + 16*ib;\n const float * qw = quant_weights + QK_K*ibl + 16*ib;\n for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);\n for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]);\n for (int k = 0; k < 2; ++k) {\n int nflip = 0;\n uint8_t s = 0;\n for (int i = 0; i < 8; ++i) {\n if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];\n else {\n xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);\n }\n }\n if (nflip%2) {\n int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];\n for (int i = 1; i < 8; ++i) {\n float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];\n if (ax < min) {\n min = ax; imin = i;\n }\n }\n xval[8*k+imin] = -xval[8*k+imin];\n s ^= (1 << imin);\n }\n block_signs[k] = s & 127;\n }\n float max = xval[0];\n for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);\n memset(L, 0, 16);\n if (max < GROUP_MAX_EPS) {\n scales[ib] = 0;\n continue;\n }\n float best = 0;\n float scale = max/(2*kMaxQ-1);\n is_on_grid[0] = is_on_grid[1] = true;\n for (int is = -9; is <= 9; ++is) {\n float id = (2*kMaxQ-1+is*0.1f)/max;\n float this_scale = 1/id;\n for (int k = 0; k < 2; ++k) {\n for (int i = 0; i < 8; ++i) {\n int l = nearest_int(0.5f*(id*xval[8*k+i]-1));\n Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));\n }\n uint16_t u = 0;\n for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);\n int grid_index = kmap_q2xs[u];\n is_on_grid_aux[k] = true;\n if (grid_index < 0) {\n is_on_grid_aux[k] = false;\n const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;\n grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);\n }\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 16; ++i) {\n float w = weight[i];\n float q = 2*Laux[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {\n scale = sumqx/sumq2; best = scale*sumqx;\n for (int i = 0; i < 16; ++i) L[i] = Laux[i];\n for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k];\n }\n }\n int n_not_ongrid = 0;\n for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid;\n if (n_not_ongrid > 0 && scale > 0) {\n float id = 1/scale;\n for (int k = 0; k < 2; ++k) {\n if (is_on_grid[k]) continue;\n uint16_t u = 0;\n for (int i = 0; i < 8; ++i) {\n int l = nearest_int(0.5f*(id*xval[8*k+i]-1));\n l = MAX(0, MIN(kMaxQ-1, l));\n u |= (l << 2*i);\n L[8*k + i] = l;\n }\n int grid_index = kmap_q2xs[u];\n if (grid_index < 0) {\n const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;\n grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);\n }\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 16; ++i) {\n float w = weight[i];\n float q = 2*L[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0) scale = sumqx/sumq2;\n }\n if (scale < 0) {\n scale = -scale;\n for (int k = 0; k < 2; ++k) block_signs[k] = (~block_signs[k]) & 127;\n }\n for (int k = 0; k < 2; ++k) {\n uint16_t u = 0;\n for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);\n int grid_index = kmap_q2xs[u];\n if (grid_index < 0) {\n printf(\"Oops: found point %u not on grid:\", u);\n for (int i = 0; i < 8; ++i) printf(\" %d\", L[8*k+i]);\n printf(\"\\n\");\n GGML_ABORT(\"fatal error\");\n }\n q2[2*ib+k] = grid_index | (block_signs[k] << 9);\n }\n GGML_ASSERT(scale >= 0);\n scales[ib] = scale;\n max_scale = MAX(max_scale, scale);\n }\n\n if (!max_scale) {\n memset(y[ibl].qs, 0, QK_K/4);\n continue;\n }\n\n float d = max_scale/31;\n y[ibl].d = GGML_FP32_TO_FP16(d);\n float id = 1/d;\n for (int ib = 0; ib < QK_K/16; ++ib) {\n int l = nearest_int(0.5f*(id*scales[ib]-1));\n l = MAX(0, MIN(15, l));\n if (ib%2 == 0) y[ibl].scales[ib/2] = l;\n else y[ibl].scales[ib/2] |= (l << 4);\n }\n memcpy(y[ibl].qs, q2, QK_K/4);\n\n }\n}\n\nsize_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_ASSERT(n_per_row%QK_K == 0);\n int64_t nblock = n_per_row/QK_K;\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_iq2_xxs_impl(src, qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += nblock*sizeof(block_iq2_xxs);\n }\n return nrow * nblock * sizeof(block_iq2_xxs);\n}\n\nsize_t quantize_iq2_xs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_ASSERT(n_per_row%QK_K == 0);\n int64_t nblock = n_per_row/QK_K;\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_iq2_xs_impl(src, qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += nblock*sizeof(block_iq2_xs);\n }\n return nrow * nblock * sizeof(block_iq2_xs);\n}\n\n//\n// ============================================= 3-bit using D4 lattice\n//\n\ntypedef struct {\n uint32_t * grid;\n int * map;\n uint16_t * neighbours;\n} iq3_entry_t;\n\nstatic iq3_entry_t iq3_data[2] = {\n {NULL, NULL, NULL},\n {NULL, NULL, NULL},\n};\n\nstatic inline int iq3_data_index(int grid_size) {\n (void)grid_size;\n GGML_ASSERT(grid_size == 256 || grid_size == 512);\n return grid_size == 256 ? 0 : 1;\n}\n\nstatic int iq3_compare_func(const void * left, const void * right) {\n const int * l = (const int *)left;\n const int * r = (const int *)right;\n return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;\n}\n\nvoid iq3xs_init_impl(int grid_size) {\n const int gindex = iq3_data_index(grid_size);\n if (iq3_data[gindex].grid) {\n return;\n }\n static const uint16_t kgrid_256[256] = {\n 0, 2, 4, 9, 11, 15, 16, 18, 25, 34, 59, 61, 65, 67, 72, 74,\n 81, 85, 88, 90, 97, 108, 120, 128, 130, 132, 137, 144, 146, 153, 155, 159,\n 169, 175, 189, 193, 199, 200, 202, 213, 248, 267, 287, 292, 303, 315, 317, 321,\n 327, 346, 362, 413, 436, 456, 460, 462, 483, 497, 513, 515, 520, 522, 529, 531,\n 536, 538, 540, 551, 552, 576, 578, 585, 592, 594, 641, 643, 648, 650, 657, 664,\n 698, 704, 706, 720, 729, 742, 758, 769, 773, 808, 848, 852, 870, 889, 901, 978,\n 992, 1024, 1026, 1033, 1035, 1040, 1042, 1046, 1049, 1058, 1089, 1091, 1093, 1096, 1098, 1105,\n 1112, 1139, 1143, 1144, 1152, 1154, 1161, 1167, 1168, 1170, 1183, 1184, 1197, 1217, 1224, 1228,\n 1272, 1276, 1309, 1323, 1347, 1367, 1377, 1404, 1473, 1475, 1486, 1509, 1537, 1544, 1546, 1553,\n 1555, 1576, 1589, 1594, 1600, 1602, 1616, 1625, 1636, 1638, 1665, 1667, 1672, 1685, 1706, 1722,\n 1737, 1755, 1816, 1831, 1850, 1856, 1862, 1874, 1901, 1932, 1950, 1971, 2011, 2032, 2052, 2063,\n 2077, 2079, 2091, 2095, 2172, 2192, 2207, 2208, 2224, 2230, 2247, 2277, 2308, 2345, 2356, 2389,\n 2403, 2424, 2501, 2504, 2506, 2520, 2570, 2593, 2616, 2624, 2630, 2646, 2669, 2700, 2714, 2746,\n 2754, 2795, 2824, 2835, 2839, 2874, 2882, 2905, 2984, 3028, 3042, 3092, 3108, 3110, 3124, 3153,\n 3185, 3215, 3252, 3288, 3294, 3364, 3397, 3434, 3483, 3523, 3537, 3587, 3589, 3591, 3592, 3610,\n 3626, 3670, 3680, 3722, 3749, 3754, 3776, 3789, 3803, 3824, 3857, 3873, 3904, 3906, 3924, 3992,\n };\n static const uint16_t kgrid_512[512] = {\n 0, 1, 2, 5, 7, 8, 9, 10, 12, 14, 16, 17, 21, 27, 32, 34,\n 37, 39, 41, 43, 48, 50, 57, 60, 63, 64, 65, 66, 68, 72, 73, 77,\n 80, 83, 87, 89, 93, 100, 113, 117, 122, 128, 129, 133, 135, 136, 139, 142,\n 145, 149, 152, 156, 162, 165, 167, 169, 171, 184, 187, 195, 201, 205, 208, 210,\n 217, 219, 222, 228, 232, 234, 247, 249, 253, 256, 267, 271, 273, 276, 282, 288,\n 291, 297, 312, 322, 324, 336, 338, 342, 347, 353, 357, 359, 374, 379, 390, 393,\n 395, 409, 426, 441, 448, 450, 452, 464, 466, 470, 475, 488, 492, 512, 513, 514,\n 516, 520, 521, 523, 525, 527, 528, 530, 537, 540, 542, 556, 558, 561, 570, 576,\n 577, 579, 582, 584, 588, 593, 600, 603, 609, 616, 618, 632, 638, 640, 650, 653,\n 655, 656, 660, 666, 672, 675, 685, 688, 698, 705, 708, 711, 712, 715, 721, 727,\n 728, 732, 737, 754, 760, 771, 773, 778, 780, 793, 795, 802, 806, 808, 812, 833,\n 840, 843, 849, 856, 858, 873, 912, 916, 919, 932, 934, 961, 963, 968, 970, 977,\n 989, 993, 1010, 1016, 1024, 1025, 1027, 1029, 1031, 1032, 1034, 1036, 1038, 1041, 1043, 1047,\n 1048, 1050, 1057, 1059, 1061, 1064, 1066, 1079, 1080, 1083, 1085, 1088, 1090, 1096, 1099, 1103,\n 1106, 1109, 1113, 1116, 1122, 1129, 1153, 1156, 1159, 1169, 1171, 1176, 1183, 1185, 1195, 1199,\n 1209, 1212, 1216, 1218, 1221, 1225, 1234, 1236, 1241, 1243, 1250, 1256, 1270, 1281, 1287, 1296,\n 1299, 1306, 1309, 1313, 1338, 1341, 1348, 1353, 1362, 1375, 1376, 1387, 1400, 1408, 1410, 1415,\n 1425, 1453, 1457, 1477, 1481, 1494, 1496, 1507, 1512, 1538, 1545, 1547, 1549, 1551, 1554, 1561,\n 1563, 1565, 1570, 1572, 1575, 1577, 1587, 1593, 1601, 1603, 1605, 1612, 1617, 1619, 1632, 1648,\n 1658, 1662, 1664, 1674, 1680, 1690, 1692, 1704, 1729, 1736, 1740, 1745, 1747, 1751, 1752, 1761,\n 1763, 1767, 1773, 1787, 1795, 1801, 1806, 1810, 1817, 1834, 1840, 1844, 1857, 1864, 1866, 1877,\n 1882, 1892, 1902, 1915, 1934, 1953, 1985, 1987, 2000, 2002, 2013, 2048, 2052, 2058, 2064, 2068,\n 2071, 2074, 2081, 2088, 2104, 2114, 2119, 2121, 2123, 2130, 2136, 2141, 2147, 2153, 2157, 2177,\n 2179, 2184, 2189, 2193, 2203, 2208, 2223, 2226, 2232, 2244, 2249, 2251, 2256, 2258, 2265, 2269,\n 2304, 2306, 2324, 2335, 2336, 2361, 2373, 2375, 2385, 2418, 2443, 2460, 2480, 2504, 2509, 2520,\n 2531, 2537, 2562, 2568, 2572, 2578, 2592, 2596, 2599, 2602, 2614, 2620, 2625, 2627, 2629, 2634,\n 2641, 2650, 2682, 2688, 2697, 2707, 2712, 2718, 2731, 2754, 2759, 2760, 2775, 2788, 2793, 2805,\n 2811, 2817, 2820, 2832, 2842, 2854, 2890, 2902, 2921, 2923, 2978, 3010, 3012, 3026, 3081, 3083,\n 3085, 3097, 3099, 3120, 3136, 3152, 3159, 3188, 3210, 3228, 3234, 3245, 3250, 3256, 3264, 3276,\n 3281, 3296, 3349, 3363, 3378, 3392, 3395, 3420, 3440, 3461, 3488, 3529, 3531, 3584, 3588, 3591,\n 3600, 3602, 3614, 3616, 3628, 3634, 3650, 3657, 3668, 3683, 3685, 3713, 3716, 3720, 3726, 3729,\n 3736, 3753, 3778, 3802, 3805, 3819, 3841, 3845, 3851, 3856, 3880, 3922, 3938, 3970, 3993, 4032,\n };\n\n const int kmap_size = 4096;\n const int nwant = grid_size == 256 ? 2 : 3;\n const uint16_t * kgrid = grid_size == 256 ? kgrid_256 : kgrid_512;\n uint32_t * kgrid_q3xs;\n int * kmap_q3xs;\n uint16_t * kneighbors_q3xs;\n\n //printf(\"================================================================= %s(grid_size = %d)\\n\", __func__, grid_size);\n uint32_t * the_grid = (uint32_t *)malloc(grid_size*sizeof(uint32_t));\n for (int k = 0; k < grid_size; ++k) {\n int8_t * pos = (int8_t *)(the_grid + k);\n for (int i = 0; i < 4; ++i) {\n int l = (kgrid[k] >> 3*i) & 0x7;\n pos[i] = 2*l + 1;\n }\n }\n kgrid_q3xs = the_grid;\n iq3_data[gindex].grid = the_grid;\n kmap_q3xs = (int *)malloc(kmap_size*sizeof(int));\n iq3_data[gindex].map = kmap_q3xs;\n for (int i = 0; i < kmap_size; ++i) kmap_q3xs[i] = -1;\n uint32_t aux32;\n uint8_t * aux8 = (uint8_t *)&aux32;\n for (int i = 0; i < grid_size; ++i) {\n aux32 = kgrid_q3xs[i];\n uint16_t index = 0;\n for (int k=0; k<4; ++k) {\n uint16_t q = (aux8[k] - 1)/2;\n index |= (q << 3*k);\n }\n kmap_q3xs[index] = i;\n }\n int8_t pos[4];\n int * dist2 = (int *)malloc(2*grid_size*sizeof(int));\n int num_neighbors = 0, num_not_in_map = 0;\n for (int i = 0; i < kmap_size; ++i) {\n if (kmap_q3xs[i] >= 0) continue;\n ++num_not_in_map;\n for (int k = 0; k < 4; ++k) {\n int l = (i >> 3*k) & 0x7;\n pos[k] = 2*l + 1;\n }\n for (int j = 0; j < grid_size; ++j) {\n const int8_t * pg = (const int8_t *)(kgrid_q3xs + j);\n int d2 = 0;\n for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);\n dist2[2*j+0] = d2;\n dist2[2*j+1] = j;\n }\n qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func);\n int n = 0; int d2 = dist2[0];\n int nhave = 1;\n for (int j = 0; j < grid_size; ++j) {\n if (dist2[2*j] > d2) {\n if (nhave == nwant) break;\n d2 = dist2[2*j];\n ++nhave;\n }\n ++n;\n }\n num_neighbors += n;\n }\n //printf(\"%s: %d neighbours in total\\n\", __func__, num_neighbors);\n kneighbors_q3xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));\n iq3_data[gindex].neighbours = kneighbors_q3xs;\n int counter = 0;\n for (int i = 0; i < kmap_size; ++i) {\n if (kmap_q3xs[i] >= 0) continue;\n for (int k = 0; k < 4; ++k) {\n int l = (i >> 3*k) & 0x7;\n pos[k] = 2*l + 1;\n }\n for (int j = 0; j < grid_size; ++j) {\n const int8_t * pg = (const int8_t *)(kgrid_q3xs + j);\n int d2 = 0;\n for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);\n dist2[2*j+0] = d2;\n dist2[2*j+1] = j;\n }\n qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func);\n kmap_q3xs[i] = -(counter + 1);\n int d2 = dist2[0];\n uint16_t * start = &kneighbors_q3xs[counter++];\n int n = 0, nhave = 1;\n for (int j = 0; j < grid_size; ++j) {\n if (dist2[2*j] > d2) {\n if (nhave == nwant) break;\n d2 = dist2[2*j];\n ++nhave;\n }\n kneighbors_q3xs[counter++] = dist2[2*j+1];\n ++n;\n }\n *start = n;\n }\n free(dist2);\n}\n\nvoid iq3xs_free_impl(int grid_size) {\n GGML_ASSERT(grid_size == 256 || grid_size == 512);\n const int gindex = iq3_data_index(grid_size);\n if (iq3_data[gindex].grid) {\n free(iq3_data[gindex].grid); iq3_data[gindex].grid = NULL;\n free(iq3_data[gindex].map); iq3_data[gindex].map = NULL;\n free(iq3_data[gindex].neighbours); iq3_data[gindex].neighbours = NULL;\n }\n}\n\nstatic int iq3_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, const uint32_t * GGML_RESTRICT grid,\n const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) {\n int num_neighbors = neighbours[0];\n GGML_ASSERT(num_neighbors > 0);\n float best_d2 = FLT_MAX;\n int grid_index = -1;\n for (int j = 1; j <= num_neighbors; ++j) {\n const int8_t * pg = (const int8_t *)(grid + neighbours[j]);\n float d2 = 0;\n for (int i = 0; i < 4; ++i) {\n float q = pg[i];\n float diff = scale*q - xval[i];\n d2 += weight[i]*diff*diff;\n }\n if (d2 < best_d2) {\n best_d2 = d2; grid_index = neighbours[j];\n }\n }\n GGML_ASSERT(grid_index >= 0);\n const int8_t * pg = (const int8_t *)(grid + grid_index);\n for (int i = 0; i < 4; ++i) L[i] = (pg[i] - 1)/2;\n return grid_index;\n}\n\nstatic void quantize_row_iq3_xxs_impl(int grid_size, const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n,\n const float * GGML_RESTRICT quant_weights) {\n\n const int gindex = iq3_data_index(grid_size);\n\n const uint32_t * kgrid_q3xs = iq3_data[gindex].grid;\n const int * kmap_q3xs = iq3_data[gindex].map;\n const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours;\n\n //GGML_ASSERT(quant_weights && \"missing quantization weights\");\n GGML_ASSERT(kgrid_q3xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kmap_q3xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kneighbors_q3xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(n%QK_K == 0);\n\n const int kMaxQ = 8;\n\n const int64_t nbl = n/QK_K;\n\n ggml_fp16_t * dh;\n uint8_t * qs;\n int block_size;\n if (grid_size == 256) {\n block_iq3_xxs * y = vy;\n dh = &y->d;\n qs = y->qs;\n block_size = sizeof(block_iq3_xxs);\n } else {\n block_iq3_s * y = vy;\n dh = &y->d;\n qs = y->qs;\n block_size = sizeof(block_iq3_s);\n }\n int quant_size = block_size - sizeof(ggml_fp16_t);\n\n float scales[QK_K/32];\n float weight[32];\n float xval[32];\n int8_t L[32];\n int8_t Laux[32];\n float waux[32];\n bool is_on_grid[8];\n bool is_on_grid_aux[8];\n uint8_t block_signs[8];\n uint8_t q3[3*(QK_K/8)+QK_K/32];\n uint32_t * scales_and_signs = (uint32_t *)(q3 + QK_K/4);\n uint8_t * qh = q3 + 3*(QK_K/8);\n\n for (int ibl = 0; ibl < nbl; ++ibl) {\n\n dh[0] = GGML_FP32_TO_FP16(0.f);\n memset(q3, 0, 3*QK_K/8+QK_K/32);\n\n float max_scale = 0;\n\n const float * xbl = x + QK_K*ibl;\n float sumx2 = 0;\n for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];\n float sigma2 = 2*sumx2/QK_K;\n\n for (int ib = 0; ib < QK_K/32; ++ib) {\n const float * xb = xbl + 32*ib;\n if (quant_weights) {\n const float * qw = quant_weights + QK_K*ibl + 32*ib;\n for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);\n } else {\n for (int i = 0; i < 32; ++i) weight[i] = xb[i]*xb[i];\n }\n for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]);\n for (int k = 0; k < 4; ++k) {\n int nflip = 0;\n uint8_t s = 0;\n for (int i = 0; i < 8; ++i) {\n if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];\n else {\n xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);\n }\n }\n if (nflip%2) {\n int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];\n for (int i = 1; i < 8; ++i) {\n float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];\n if (ax < min) {\n min = ax; imin = i;\n }\n }\n xval[8*k+imin] = -xval[8*k+imin];\n s ^= (1 << imin);\n }\n block_signs[k] = s & 127;\n }\n float max = xval[0];\n for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);\n memset(L, 0, 32);\n if (max < GROUP_MAX_EPS_IQ3_XXS) {\n scales[ib] = 0;\n continue;\n }\n float best = 0;\n float scale = max/(2*kMaxQ-1);\n for (int k = 0; k < 8; ++k) is_on_grid[k] = true;\n for (int is = -15; is <= 15; ++is) {\n float id = (2*kMaxQ-1+is*0.2f)/max;\n float this_scale = 1/id;\n for (int k = 0; k < 8; ++k) {\n for (int i = 0; i < 4; ++i) {\n int l = nearest_int(0.5f*(id*xval[4*k+i]-1));\n Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l));\n }\n uint16_t u = 0;\n for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i);\n int grid_index = kmap_q3xs[u];\n is_on_grid_aux[k] = true;\n if (grid_index < 0) {\n is_on_grid_aux[k] = false;\n const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;\n grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k);\n }\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 32; ++i) {\n float w = weight[i];\n float q = 2*Laux[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {\n scale = sumqx/sumq2; best = scale*sumqx;\n for (int i = 0; i < 32; ++i) L[i] = Laux[i];\n for (int k = 0; k < 8; ++k) is_on_grid[k] = is_on_grid_aux[k];\n }\n }\n int n_not_ongrid = 0;\n for (int k = 0; k < 8; ++k) if (!is_on_grid[k]) ++n_not_ongrid;\n if (n_not_ongrid > 0 && scale > 0) {\n float id = 1/scale;\n for (int k = 0; k < 8; ++k) {\n if (is_on_grid[k]) continue;\n uint16_t u = 0;\n for (int i = 0; i < 4; ++i) {\n int l = nearest_int(0.5f*(id*xval[4*k+i]-1));\n l = MAX(0, MIN(kMaxQ-1, l));\n u |= (l << 3*i);\n }\n int grid_index = kmap_q3xs[u];\n if (grid_index < 0) {\n const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;\n grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k);\n }\n const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index);\n for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2;\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 32; ++i) {\n float w = weight[i];\n float q = 2*L[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0) scale = sumqx/sumq2;\n }\n if (scale < 0) {\n // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)\n // and correspondingly flip quant signs.\n scale = -scale;\n for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127;\n }\n for (int k = 0; k < 8; ++k) {\n uint16_t u = 0;\n for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i);\n int grid_index = kmap_q3xs[u];\n if (grid_index < 0) {\n printf(\"Oops: found point %u not on grid:\", u);\n for (int i = 0; i < 4; ++i) printf(\" %d\", L[4*k+i]);\n printf(\"\\n\");\n GGML_ABORT(\"fatal error\");\n }\n if (grid_size == 256) {\n q3[8*ib+k] = grid_index;\n } else {\n q3[8*ib+k] = grid_index & 255;\n qh[ib] |= ((grid_index >> 8) << k);\n }\n\n }\n scales_and_signs[ib] = block_signs[0] | (block_signs[1] << 7) | (block_signs[2] << 14) | (block_signs[3] << 21);\n GGML_ASSERT(scale >= 0);\n scales[ib] = scale;\n max_scale = MAX(max_scale, scale);\n }\n\n if (!max_scale) {\n memset(qs, 0, quant_size);\n dh += block_size/sizeof(ggml_fp16_t);\n qs += block_size;\n continue;\n }\n\n float d = max_scale/31;\n dh[0] = GGML_FP32_TO_FP16(d * 1.0125f); // small improvement via this fudge factor\n float id = 1/d;\n for (int ib = 0; ib < QK_K/32; ++ib) {\n int l = nearest_int(0.5f*(id*scales[ib]-1));\n l = MAX(0, MIN(15, l));\n scales_and_signs[ib] |= ((uint32_t)l << 28);\n }\n memcpy(qs, q3, quant_size);\n\n dh += block_size/sizeof(ggml_fp16_t);\n qs += block_size;\n\n }\n}\n\nsize_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_ASSERT(n_per_row%QK_K == 0);\n int64_t nblock = n_per_row/QK_K;\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_iq3_xxs_impl(256, src, qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += nblock*sizeof(block_iq3_xxs);\n }\n return nrow * nblock * sizeof(block_iq3_xxs);\n}\n\nvoid quantize_row_iq3_xxs_ref(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n quantize_row_iq3_xxs_impl(256, x, y, k, NULL);\n}\n\nstatic void quantize_row_iq3_s_impl(int block_size, const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int n,\n const float * GGML_RESTRICT quant_weights,\n float * scales,\n float * weight,\n float * xval,\n int8_t * L,\n int8_t * Laux,\n float * waux,\n bool * is_on_grid,\n bool * is_on_grid_aux,\n uint8_t * block_signs) {\n\n const int gindex = iq3_data_index(512);\n\n const uint32_t * kgrid_q3xs = iq3_data[gindex].grid;\n const int * kmap_q3xs = iq3_data[gindex].map;\n const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours;\n\n //GGML_ASSERT(quant_weights && \"missing quantization weights\");\n GGML_ASSERT(kgrid_q3xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kmap_q3xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kneighbors_q3xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(n%QK_K == 0);\n\n const int kMaxQ = 8;\n\n const int64_t nbl = n/QK_K;\n\n block_iq3_s * y = vy;\n\n const int bs4 = block_size/4;\n const int bs8 = block_size/8;\n\n for (int ibl = 0; ibl < nbl; ++ibl) {\n\n memset(&y[ibl], 0, sizeof(block_iq3_s));\n y[ibl].d = GGML_FP32_TO_FP16(0.f);\n\n uint8_t * qs = y[ibl].qs;\n uint8_t * qh = y[ibl].qh;\n uint8_t * signs = y[ibl].signs;\n\n float max_scale = 0;\n\n const float * xbl = x + QK_K*ibl;\n float sumx2 = 0;\n for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];\n float sigma2 = 2*sumx2/QK_K;\n\n for (int ib = 0; ib < QK_K/block_size; ++ib) {\n const float * xb = xbl + block_size*ib;\n if (quant_weights) {\n const float * qw = quant_weights + QK_K*ibl + block_size*ib;\n for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);\n } else {\n for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i];\n }\n for (int i = 0; i < block_size; ++i) waux[i] = sqrtf(weight[i]);\n for (int k = 0; k < bs8; ++k) {\n uint8_t s = 0;\n for (int i = 0; i < 8; ++i) {\n if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];\n else {\n xval[8*k + i] = -xb[8*k + i]; s |= (1 << i);\n }\n }\n block_signs[k] = s;\n }\n float max = xval[0];\n for (int i = 1; i < block_size; ++i) max = MAX(max, xval[i]);\n memset(L, 0, block_size);\n if (!max) {\n scales[ib] = 0;\n continue;\n }\n float best = 0;\n float scale = max/(2*kMaxQ-1);\n for (int k = 0; k < bs4; ++k) is_on_grid[k] = false;\n for (int is = -9; is <= 9; ++is) {\n float id = (2*kMaxQ-1+is*0.2f)/max;\n float this_scale = 1/id;\n for (int k = 0; k < bs4; ++k) {\n for (int i = 0; i < 4; ++i) {\n int l = nearest_int(0.5f*(id*xval[4*k+i]-1));\n Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l));\n }\n uint16_t u = 0;\n for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i);\n int grid_index = kmap_q3xs[u];\n is_on_grid_aux[k] = true;\n if (grid_index < 0) {\n is_on_grid_aux[k] = false;\n const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;\n grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k);\n }\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < block_size; ++i) {\n float w = weight[i];\n float q = 2*Laux[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {\n scale = sumqx/sumq2; best = scale*sumqx;\n for (int i = 0; i < block_size; ++i) L[i] = Laux[i];\n for (int k = 0; k < bs4; ++k) is_on_grid[k] = is_on_grid_aux[k];\n }\n }\n int n_not_ongrid = 0;\n for (int k = 0; k < bs4; ++k) if (!is_on_grid[k]) ++n_not_ongrid;\n if (n_not_ongrid > 0 && scale > 0) {\n float id = 1/scale;\n for (int k = 0; k < bs4; ++k) {\n //if (is_on_grid[k]) continue;\n uint16_t u = 0;\n for (int i = 0; i < 4; ++i) {\n int l = nearest_int(0.5f*(id*xval[4*k+i]-1));\n l = MAX(0, MIN(kMaxQ-1, l));\n u |= (l << 3*i);\n }\n int grid_index = kmap_q3xs[u];\n if (grid_index < 0) {\n const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;\n grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k);\n }\n const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index);\n for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2;\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < block_size; ++i) {\n float w = weight[i];\n float q = 2*L[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0) scale = sumqx/sumq2;\n }\n if (scale < 0) {\n // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)\n // and correspondingly flip quant signs.\n scale = -scale;\n for (int k = 0; k < bs8; ++k) block_signs[k] = ~block_signs[k];\n }\n for (int k = 0; k < bs4; ++k) {\n uint16_t u = 0;\n for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i);\n int grid_index = kmap_q3xs[u];\n if (grid_index < 0) {\n printf(\"Oops: found point %u not on grid:\", u);\n for (int i = 0; i < 4; ++i) printf(\" %d\", L[4*k+i]);\n printf(\"\\n\");\n GGML_ABORT(\"fatal error\");\n }\n qs[k] = grid_index & 255;\n qh[(ib*bs4+k)/8] |= ((grid_index >> 8) << ((ib*bs4+k)%8));\n }\n qs += bs4;\n for (int k = 0; k < bs8; ++k) signs[k] = block_signs[k];\n signs += bs8;\n GGML_ASSERT(scale >= 0);\n scales[ib] = scale;\n max_scale = MAX(max_scale, scale);\n }\n\n if (!max_scale) {\n continue;\n }\n\n float d = max_scale/31;\n y[ibl].d = GGML_FP32_TO_FP16(d * 1.033f);\n float id = 1/d;\n for (int ib = 0; ib < QK_K/block_size; ib += 2) {\n int l1 = nearest_int(0.5f*(id*scales[ib+0]-1));\n l1 = MAX(0, MIN(15, l1));\n int l2 = nearest_int(0.5f*(id*scales[ib+1]-1));\n l2 = MAX(0, MIN(15, l2));\n y[ibl].scales[ib/2] = l1 | (l2 << 4);\n }\n\n }\n}\n\n#define IQ3S_BLOCK_SIZE 32\nsize_t quantize_iq3_s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_ASSERT(n_per_row%QK_K == 0);\n int64_t nblock = n_per_row/QK_K;\n float scales[QK_K/IQ3S_BLOCK_SIZE];\n float weight[IQ3S_BLOCK_SIZE];\n float xval[IQ3S_BLOCK_SIZE];\n int8_t L[IQ3S_BLOCK_SIZE];\n int8_t Laux[IQ3S_BLOCK_SIZE];\n float waux[IQ3S_BLOCK_SIZE];\n bool is_on_grid[IQ3S_BLOCK_SIZE/4];\n bool is_on_grid_aux[IQ3S_BLOCK_SIZE/4];\n uint8_t block_signs[IQ3S_BLOCK_SIZE/8];\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_iq3_s_impl(IQ3S_BLOCK_SIZE, src, qrow, n_per_row, quant_weights,\n scales, weight, xval, L, Laux, waux, is_on_grid, is_on_grid_aux, block_signs);\n src += n_per_row;\n qrow += nblock*sizeof(block_iq3_s);\n }\n return nrow * nblock * sizeof(block_iq3_s);\n}\n\nvoid quantize_row_iq3_s_ref(const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n quantize_iq3_s(x, y, 1, k, NULL);\n}\n\n\n// =================================== 1.5 bpw ===================================================\n\nstatic int iq1_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,\n const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float * scale, int8_t * GGML_RESTRICT L, int ngrid) {\n int num_neighbors = neighbours[0];\n GGML_ASSERT(num_neighbors > 0);\n float best_score = -FLT_MAX;\n int grid_index = -1;\n for (int j = 1; j <= num_neighbors; ++j) {\n const int8_t * pg = (const int8_t *)(grid + neighbours[j]);\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 8; ++i) {\n float q = (pg[i] - 3)/2;\n float w = weight[i];\n sumqx += w*q*xval[i];\n sumq2 += w*q*q;\n }\n if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {\n *scale = sumqx/sumq2; best_score = *scale * sumqx;\n grid_index = neighbours[j];\n }\n }\n if (grid_index < 0) {\n for (int i = 0; i < ngrid; ++i) {\n const int8_t * grid_i = (const int8_t *)(grid + i);\n float sumqx = 0, sumq2 = 0;\n for (int j = 0; j < 8; ++j) {\n float w = weight[j];\n float q = (grid_i[j] - 3)/2;\n sumqx += w*q*xval[j];\n sumq2 += w*q*q;\n }\n if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {\n *scale = sumqx/sumq2; best_score = *scale*sumqx;\n grid_index = i;\n }\n }\n }\n if (grid_index < 0) {\n printf(\"Oops, did not find grid point\\n\");\n printf(\"Have %d neighbours\\n\", num_neighbors);\n for (int j = 1; j <= num_neighbors; ++j) {\n const int8_t * pg = (const int8_t *)(grid + neighbours[j]);\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 8; ++i) {\n float q = (pg[i] - 3)/2;\n float w = weight[i];\n sumqx += w*q*xval[i];\n sumq2 += w*q*q;\n }\n printf(\" neighbour %d: sumqx = %g sumq2 = %g\\n\", j, (double)sumqx, (double)sumq2);\n }\n }\n GGML_ASSERT(grid_index >= 0);\n //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n *scale *= 1.05f; // This is a fudge factor. Don't ask me why it improves the result.\n //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n const int8_t * pg = (const int8_t *)(grid + grid_index);\n for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;\n return grid_index;\n}\n\nstatic int iq1_find_best_neighbour2(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,\n const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, const float * GGML_RESTRICT xg, int8_t * GGML_RESTRICT L, int ngrid) {\n int num_neighbors = neighbours[0];\n GGML_ASSERT(num_neighbors > 0);\n float best_score = FLT_MAX;\n int grid_index = -1;\n for (int j = 1; j <= num_neighbors; ++j) {\n const int8_t * pg = (const int8_t *)(grid + neighbours[j]);\n float d2 = 0;\n for (int i = 0; i < 8; ++i) {\n float q = xg[(pg[i] - 1)/2];\n float w = weight[i];\n float diff = scale*q - xval[i];\n d2 += w*diff*diff;\n }\n if (d2 < best_score) {\n best_score = d2;\n grid_index = neighbours[j];\n }\n }\n if (grid_index < 0) {\n for (int i = 0; i < ngrid; ++i) {\n const int8_t * grid_i = (const int8_t *)(grid + i);\n float d2 = 0;\n for (int j = 0; j < 8; ++j) {\n float w = weight[j];\n float q = xg[(grid_i[j] - 1)/2];\n float diff = scale*q - xval[i];\n d2 += w*diff*diff;\n }\n if (d2 < best_score) {\n best_score = d2;\n grid_index = i;\n }\n }\n }\n if (grid_index < 0) {\n printf(\"Oops, did not find grid point\\n\");\n printf(\"Have %d neighbours\\n\", num_neighbors);\n for (int j = 1; j <= num_neighbors; ++j) {\n const int8_t * pg = (const int8_t *)(grid + neighbours[j]);\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 8; ++i) {\n float q = xg[(pg[i] - 1)/2];\n float w = weight[i];\n sumqx += w*q*xval[i];\n sumq2 += w*q*q;\n }\n printf(\" neighbour %d: sumqx = %g sumq2 = %g\\n\", j, (double)sumqx, (double)sumq2);\n }\n }\n GGML_ASSERT(grid_index >= 0);\n const int8_t * pg = (const int8_t *)(grid + grid_index);\n for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;\n return grid_index;\n}\n\nstatic int iq1_sort_helper(const void * left, const void * right) {\n const float * l = left;\n const float * r = right;\n return *l < *r ? -1 : *l > *r ? 1 : 0;\n}\n\n#define IQ1S_BLOCK_SIZE 32\n#define IQ1M_BLOCK_SIZE 16\nstatic void quantize_row_iq1_s_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights,\n float * scales,\n float * weight,\n float * sumx,\n float * sumw,\n float * pairs,\n int8_t * L,\n uint16_t * index,\n int8_t * shifts) {\n\n const int gindex = iq2_data_index(GGML_TYPE_IQ1_S);\n\n const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;\n const int * kmap_q2xs = iq2_data[gindex].map;\n const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;\n\n GGML_ASSERT(quant_weights && \"missing quantization weights\");\n GGML_ASSERT(kgrid_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kmap_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(n%QK_K == 0);\n\n block_iq1_s * y = vy;\n\n const int64_t nbl = n/QK_K;\n\n const int block_size = IQ1S_BLOCK_SIZE;\n\n const float x_p[3] = {-1 + IQ1S_DELTA, IQ1S_DELTA, 1 + IQ1S_DELTA};\n const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA};\n\n\n int * idx = (int *)(pairs + 1);\n\n for (int ibl = 0; ibl < nbl; ++ibl) {\n\n y[ibl].d = GGML_FP32_TO_FP16(0.f);\n memset(y[ibl].qs, 0, QK_K/8);\n memset(y[ibl].qh, 0, QK_K/16);\n\n float max_scale = 0;\n\n const float * xbl = x + QK_K*ibl;\n float sumx2 = 0;\n for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];\n float sigma2 = 2*sumx2/QK_K;\n\n for (int ib = 0; ib < QK_K/block_size; ++ib) {\n const float * xb = xbl + block_size*ib;\n const float * qw = quant_weights + QK_K*ibl + block_size*ib;\n for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);\n float max = fabsf(xb[0]);\n for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));\n if (max < GROUP_MAX_EPS_IQ1_S) {\n scales[ib] = 0;\n shifts[ib] = 1;\n memset(L, 1, block_size);\n continue;\n }\n // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem.\n // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two\n // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights\n // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and\n // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale\n // for each possible and score for each split.\n for (int j = 0; j < block_size; ++j) {\n pairs[2*j] = xb[j];\n idx[2*j] = j;\n }\n qsort(pairs, block_size, 2*sizeof(float), iq1_sort_helper);\n {\n sumx[0] = sumw[0] = 0;\n for (int j = 0; j < block_size; ++j) {\n int i = idx[2*j];\n sumx[j+1] = sumx[j] + weight[i]*xb[i];\n sumw[j+1] = sumw[j] + weight[i];\n }\n }\n float best_score = -FLT_MAX, scale = max;\n int besti1 = -1, besti2 = -1, best_shift = 0;\n for (int i1 = 0; i1 <= block_size; ++i1) {\n for (int i2 = i1; i2 <= block_size; ++i2) {\n float sumqx = (sumx[i1] - sumx[0])*x_p[0] + (sumx[i2] - sumx[i1])*x_p[1] + (sumx[block_size] - sumx[i2])*x_p[2];\n float sumq2 = (sumw[i1] - sumw[0])*x_p[0]*x_p[0] + (sumw[i2] - sumw[i1])*x_p[1]*x_p[1] + (sumw[block_size] - sumw[i2])*x_p[2]*x_p[2];\n if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {\n scale = sumqx/sumq2; best_score = scale*sumqx;\n besti1 = i1; besti2 = i2; best_shift = 1;\n }\n sumqx = (sumx[i1] - sumx[0])*x_m[0] + (sumx[i2] - sumx[i1])*x_m[1] + (sumx[block_size] - sumx[i2])*x_m[2];\n sumq2 = (sumw[i1] - sumw[0])*x_m[0]*x_m[0] + (sumw[i2] - sumw[i1])*x_m[1]*x_m[1] + (sumw[block_size] - sumw[i2])*x_m[2]*x_m[2];\n if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {\n scale = sumqx/sumq2; best_score = scale*sumqx;\n besti1 = i1; besti2 = i2; best_shift = -1;\n }\n }\n }\n if (besti1 < 0 || besti2 < 0 || best_shift == 0) {\n scales[ib] = 0;\n shifts[ib] = 1;\n memset(L, 1, block_size);\n continue;\n }\n for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;\n for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;\n for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2;\n if (scale < 0) {\n for (int j = 0; j < block_size; ++j) L[j] = 2 - L[j];\n scale = -scale; best_shift = -best_shift;\n }\n bool all_on_grid = true;\n const float * xx = best_shift == 1 ? x_p : x_m;\n for (int k = 0; k < block_size/8; ++k) {\n uint16_t u = 0;\n for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j);\n int grid_index = kmap_q2xs[u];\n if (grid_index < 0) {\n all_on_grid = false;\n const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;\n grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S);\n GGML_ASSERT(grid_index >= 0);\n }\n index[k] = grid_index;\n }\n if (!all_on_grid) {\n float sumqx = 0, sumq2 = 0;\n for (int k = 0; k < block_size/8; ++k) {\n const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]);\n for (int j = 0; j < 8; ++j) {\n float w = weight[8*k + j];\n float q = xx[(pg[j] - 1)/2];\n sumqx += w*q*xb[8*k+j];\n sumq2 += w*q*q;\n }\n }\n if (sumqx > 0 && sumq2 > 0) scale = sumqx/sumq2;\n }\n uint16_t h = 0;\n for (int k = 0; k < block_size/8; ++k) {\n y[ibl].qs[(block_size/8)*ib + k] = index[k] & 255;\n h |= (index[k] >> 8) << 3*k;\n }\n y[ibl].qh[ib] = h;\n GGML_ASSERT(scale >= 0);\n scales[ib] = scale;\n shifts[ib] = best_shift;\n max_scale = MAX(max_scale, scale);\n }\n\n if (!max_scale) {\n continue;\n }\n\n float d = max_scale/15;\n y[ibl].d = GGML_FP32_TO_FP16(d*1.125f); // 1.125f is another fudge factor. Don't ask me why it is needed.\n float id = 1/d;\n for (int ib = 0; ib < QK_K/block_size; ++ib) {\n int l = nearest_int(0.5f*(id*scales[ib]-1));\n l = MAX(0, MIN(7, l));\n if (shifts[ib] == -1) l |= 8;\n y[ibl].qh[ib] |= (l << 12);\n }\n }\n}\n\nsize_t quantize_iq1_s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_ASSERT(n_per_row%QK_K == 0);\n float scales[QK_K/IQ1S_BLOCK_SIZE];\n float weight[IQ1S_BLOCK_SIZE];\n int8_t L[IQ1S_BLOCK_SIZE];\n float sumx[IQ1S_BLOCK_SIZE+1];\n float sumw[IQ1S_BLOCK_SIZE+1];\n float pairs[2*IQ1S_BLOCK_SIZE];\n uint16_t index[IQ1S_BLOCK_SIZE/8];\n int8_t shifts[QK_K/IQ1S_BLOCK_SIZE];\n int64_t nblock = n_per_row/QK_K;\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_iq1_s_impl(src, qrow, n_per_row, quant_weights, scales, weight, sumx, sumw, pairs, L, index, shifts);\n src += n_per_row;\n qrow += nblock*sizeof(block_iq1_s);\n }\n return nrow * nblock * sizeof(block_iq1_s);\n}\n\nstatic void quantize_row_iq1_m_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights,\n float * scales,\n float * weight,\n float * pairs,\n int8_t * L,\n uint16_t * index,\n int8_t * shifts) {\n\n const int gindex = iq2_data_index(GGML_TYPE_IQ1_M);\n\n const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;\n const int * kmap_q2xs = iq2_data[gindex].map;\n const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;\n\n //GGML_ASSERT(quant_weights && \"missing quantization weights\");\n GGML_ASSERT(kgrid_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kmap_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(n%QK_K == 0);\n\n block_iq1_m * y = vy;\n\n const int64_t nbl = n/QK_K;\n\n const int block_size = IQ1M_BLOCK_SIZE;\n\n const float x_p[3] = {-1 + IQ1M_DELTA, IQ1M_DELTA, 1 + IQ1M_DELTA};\n const float x_m[3] = {-1 - IQ1M_DELTA, -IQ1M_DELTA, 1 - IQ1M_DELTA};\n const uint8_t masks[4] = {0x00, 0x80, 0x08, 0x88};\n\n int * idx = (int *)(pairs + 1);\n\n float sumqx[4], sumq2[4];\n\n iq1m_scale_t s;\n const float * xx;\n\n for (int ibl = 0; ibl < nbl; ++ibl) {\n memset(y[ibl].qs, 0, QK_K/8);\n memset(y[ibl].qh, 0, QK_K/16);\n memset(y[ibl].scales, 0, QK_K/32);\n\n float max_scale = 0;\n\n const float * xbl = x + QK_K*ibl;\n float sumx2 = 0;\n for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];\n float sigma2 = 2*sumx2/QK_K;\n\n for (int ib = 0; ib < QK_K/block_size; ++ib) {\n const float * xb = xbl + block_size*ib;\n if (quant_weights) {\n const float * qw = quant_weights + QK_K*ibl + block_size*ib;\n for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);\n } else {\n for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i];\n }\n float max = fabsf(xb[0]);\n for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));\n if (max < GROUP_MAX_EPS_IQ1_M) {\n scales[ib] = 0;\n shifts[ib] = 0;\n memset(L, 1, block_size);\n continue;\n }\n // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem.\n // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two\n // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights\n // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and\n // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale\n // for each possible and score for each split.\n for (int j = 0; j < block_size; ++j) {\n pairs[2*j] = xb[j];\n idx[2*j] = j;\n }\n qsort(pairs, block_size, 2*sizeof(float), iq1_sort_helper);\n float best_score = -FLT_MAX, scale = max;\n int besti1 = -1, besti2 = -1, best_k = -1;\n // 0: +, +\n // 1: +, -\n // 2: -, +\n // 3: -, -\n for (int i1 = 0; i1 <= block_size; ++i1) {\n for (int i2 = i1; i2 <= block_size; ++i2) {\n memset(sumqx, 0, 4*sizeof(float));\n memset(sumq2, 0, 4*sizeof(float));\n for (int j = 0; j < i1; ++j) {\n int i = idx[2*j];\n if (i < block_size/2) {\n sumqx[0] += weight[i]*x_p[0]*xb[i];\n sumqx[1] += weight[i]*x_p[0]*xb[i];\n sumqx[2] += weight[i]*x_m[0]*xb[i];\n sumqx[3] += weight[i]*x_m[0]*xb[i];\n sumq2[0] += weight[i]*x_p[0]*x_p[0];\n sumq2[1] += weight[i]*x_p[0]*x_p[0];\n sumq2[2] += weight[i]*x_m[0]*x_m[0];\n sumq2[3] += weight[i]*x_m[0]*x_m[0];\n } else {\n sumqx[0] += weight[i]*x_p[0]*xb[i];\n sumqx[2] += weight[i]*x_p[0]*xb[i];\n sumqx[1] += weight[i]*x_m[0]*xb[i];\n sumqx[3] += weight[i]*x_m[0]*xb[i];\n sumq2[0] += weight[i]*x_p[0]*x_p[0];\n sumq2[2] += weight[i]*x_p[0]*x_p[0];\n sumq2[1] += weight[i]*x_m[0]*x_m[0];\n sumq2[3] += weight[i]*x_m[0]*x_m[0];\n }\n }\n for (int j = i1; j < i2; ++j) {\n int i = idx[2*j];\n if (i < block_size/2) {\n sumqx[0] += weight[i]*x_p[1]*xb[i];\n sumqx[1] += weight[i]*x_p[1]*xb[i];\n sumqx[2] += weight[i]*x_m[1]*xb[i];\n sumqx[3] += weight[i]*x_m[1]*xb[i];\n sumq2[0] += weight[i]*x_p[1]*x_p[1];\n sumq2[1] += weight[i]*x_p[1]*x_p[1];\n sumq2[2] += weight[i]*x_m[1]*x_m[1];\n sumq2[3] += weight[i]*x_m[1]*x_m[1];\n } else {\n sumqx[0] += weight[i]*x_p[1]*xb[i];\n sumqx[2] += weight[i]*x_p[1]*xb[i];\n sumqx[1] += weight[i]*x_m[1]*xb[i];\n sumqx[3] += weight[i]*x_m[1]*xb[i];\n sumq2[0] += weight[i]*x_p[1]*x_p[1];\n sumq2[2] += weight[i]*x_p[1]*x_p[1];\n sumq2[1] += weight[i]*x_m[1]*x_m[1];\n sumq2[3] += weight[i]*x_m[1]*x_m[1];\n }\n }\n for (int j = i2; j < block_size; ++j) {\n int i = idx[2*j];\n if (i < block_size/2) {\n sumqx[0] += weight[i]*x_p[2]*xb[i];\n sumqx[1] += weight[i]*x_p[2]*xb[i];\n sumqx[2] += weight[i]*x_m[2]*xb[i];\n sumqx[3] += weight[i]*x_m[2]*xb[i];\n sumq2[0] += weight[i]*x_p[2]*x_p[2];\n sumq2[1] += weight[i]*x_p[2]*x_p[2];\n sumq2[2] += weight[i]*x_m[2]*x_m[2];\n sumq2[3] += weight[i]*x_m[2]*x_m[2];\n } else {\n sumqx[0] += weight[i]*x_p[2]*xb[i];\n sumqx[2] += weight[i]*x_p[2]*xb[i];\n sumqx[1] += weight[i]*x_m[2]*xb[i];\n sumqx[3] += weight[i]*x_m[2]*xb[i];\n sumq2[0] += weight[i]*x_p[2]*x_p[2];\n sumq2[2] += weight[i]*x_p[2]*x_p[2];\n sumq2[1] += weight[i]*x_m[2]*x_m[2];\n sumq2[3] += weight[i]*x_m[2]*x_m[2];\n }\n }\n for (int k = 0; k < 4; ++k) {\n if (sumq2[k] > 0 && sumqx[k]*sumqx[k] > best_score*sumq2[k]) {\n scale = sumqx[k]/sumq2[k]; best_score = scale*sumqx[k];\n besti1 = i1; besti2 = i2; best_k = k;\n }\n }\n }\n }\n if (besti1 < 0 || besti2 < 0 || best_k < 0) {\n scales[ib] = 0;\n shifts[ib] = 0;\n memset(L, 1, block_size);\n continue;\n }\n for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;\n for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;\n for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2;\n if (scale < 0) {\n for (int j = 0; j < block_size; ++j) L[j] = 2 - L[j];\n scale = -scale;\n best_k = best_k == 0 ? 3 : best_k == 1 ? 2 : best_k == 2 ? 1 : 0;\n }\n bool all_on_grid = true;\n for (int k = 0; k < block_size/8; ++k) {\n if (k == 0) xx = best_k < 2 ? x_p : x_m;\n else xx = best_k%2 == 0 ? x_p : x_m;\n uint16_t u = 0;\n for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j);\n int grid_index = kmap_q2xs[u];\n if (grid_index < 0) {\n all_on_grid = false;\n const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;\n grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S);\n GGML_ASSERT(grid_index >= 0);\n }\n index[k] = grid_index;\n }\n if (!all_on_grid) {\n float sumqx_f = 0, sumq2_f = 0;\n for (int k = 0; k < block_size/8; ++k) {\n if (k == 0) xx = best_k < 2 ? x_p : x_m;\n else xx = best_k%2 == 0 ? x_p : x_m;\n const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]);\n for (int j = 0; j < 8; ++j) {\n float w = weight[8*k + j];\n float q = xx[(pg[j] - 1)/2];\n sumqx_f += w*q*xb[8*k+j];\n sumq2_f += w*q*q;\n }\n }\n if (sumqx_f > 0 && sumq2_f > 0) scale = sumqx_f/sumq2_f;\n }\n y[ibl].qs[2*ib + 0] = index[0] & 255;\n y[ibl].qs[2*ib + 1] = index[1] & 255;\n y[ibl].qh[ib] = (index[0] >> 8) | ((index[1] >> 8) << 4);\n GGML_ASSERT(scale >= 0);\n scales[ib] = scale;\n shifts[ib] = best_k;\n max_scale = MAX(max_scale, scale);\n }\n\n if (!max_scale) {\n continue;\n }\n\n uint16_t * sc = (uint16_t *)y[ibl].scales;\n float d = max_scale/15;\n float id = 1/d;\n float sumqx_f = 0, sumq2_f = 0;\n for (int ib = 0; ib < QK_K/block_size; ++ib) {\n int l = nearest_int(0.5f*(id*scales[ib+0]-1));\n l = MAX(0, MIN(7, l));\n sc[ib/4] |= (l << 3*(ib%4));\n y[ibl].qh[ib] |= masks[shifts[ib]];\n const float * xb = xbl + block_size*ib;\n if (quant_weights) {\n const float * qw = quant_weights + QK_K*ibl + block_size*ib;\n for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);\n } else {\n for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i];\n }\n for (int k = 0; k < block_size/8; ++k) {\n if (k == 0) xx = shifts[ib] < 2 ? x_p : x_m;\n else xx = shifts[ib]%2 == 0 ? x_p : x_m;\n const int8_t * pg = (const int8_t *)(kgrid_q2xs + y[ibl].qs[2*ib+k] + ((y[ibl].qh[ib] << (8 - 4*k)) & 0x700));\n for (int j = 0; j < 8; ++j) {\n float w = weight[8*k + j];\n float q = xx[(pg[j] - 1)/2]*(2*l+1);\n sumqx_f += w*q*xb[8*k+j];\n sumq2_f += w*q*q;\n }\n }\n }\n if (sumq2_f > 0) d = sumqx_f/sumq2_f;\n s.f16 = GGML_FP32_TO_FP16(d*1.1125f); // 1.1125f is another fudge factor. Don't ask me why it is needed.\n sc[0] |= ((s.u16 & 0x000f) << 12);\n sc[1] |= ((s.u16 & 0x00f0) << 8);\n sc[2] |= ((s.u16 & 0x0f00) << 4);\n sc[3] |= ((s.u16 & 0xf000) << 0);\n }\n}\n\nsize_t quantize_iq1_m(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_ASSERT(n_per_row%QK_K == 0);\n float scales[QK_K/IQ1M_BLOCK_SIZE];\n float weight[IQ1M_BLOCK_SIZE];\n int8_t L[IQ1M_BLOCK_SIZE];\n float pairs[2*IQ1M_BLOCK_SIZE];\n uint16_t index[IQ1M_BLOCK_SIZE/8];\n int8_t shifts[QK_K/IQ1M_BLOCK_SIZE];\n int64_t nblock = n_per_row/QK_K;\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_iq1_m_impl(src, qrow, n_per_row, quant_weights, scales, weight, pairs, L, index, shifts);\n src += n_per_row;\n qrow += nblock*sizeof(block_iq1_m);\n }\n return nrow * nblock * sizeof(block_iq1_m);\n}\n\n// ============================ 4-bit non-linear quants\n\nstatic void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, const float * GGML_RESTRICT x,\n ggml_fp16_t * dh, uint8_t * q4, uint16_t * scales_h, uint8_t * scales_l,\n float * scales, float * weight, uint8_t * L,\n const int8_t * values,\n const float * quant_weights,\n const int ntry) {\n\n float sigma2 = 0;\n for (int j = 0; j < super_block_size; ++j) sigma2 += x[j]*x[j];\n sigma2 *= 2.f/super_block_size;\n\n memset(q4, 0, super_block_size/2);\n dh[0] = GGML_FP32_TO_FP16(0.f);\n\n float max_scale = 0, amax_scale = 0;\n for (int ib = 0; ib < super_block_size/block_size; ++ib) {\n const float * xb = x + ib*block_size;\n uint8_t * Lb = L + ib*block_size;\n if (quant_weights) {\n const float * qw = quant_weights + ib*block_size;\n for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);\n } else {\n for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j];\n }\n float amax = 0, max = 0;\n for (int j = 0; j < block_size; ++j) {\n float ax = fabsf(xb[j]);\n if (ax > amax) {\n amax = ax; max = xb[j];\n }\n }\n if (amax < GROUP_MAX_EPS) {\n scales[ib] = 0;\n continue;\n }\n float d = ntry > 0 ? -max/values[0] : max/values[0];\n float id = 1/d;\n float sumqx = 0, sumq2 = 0;\n for (int j = 0; j < block_size; ++j) {\n float al = id*xb[j];\n int l = best_index_int8(16, values, al);\n Lb[j] = l;\n float q = values[l];\n float w = weight[j];\n sumqx += w*q*xb[j];\n sumq2 += w*q*q;\n }\n d = sumq2 > 0 ? sumqx/sumq2 : 0.f;\n float best = d*sumqx;\n for (int itry = -ntry; itry <= ntry; ++itry) {\n id = (itry + values[0])/max;\n sumqx = sumq2 = 0;\n for (int j = 0; j < block_size; ++j) {\n float al = id*xb[j];\n int l = best_index_int8(16, values, al);\n float q = values[l];\n float w = weight[j];\n sumqx += w*q*xb[j];\n sumq2 += w*q*q;\n }\n if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {\n d = sumqx/sumq2; best = d * sumqx;\n }\n }\n scales[ib] = d;\n float abs_d = fabsf(d);\n if (abs_d > amax_scale) {\n amax_scale = abs_d; max_scale = d;\n }\n }\n\n if (super_block_size/block_size > 1) {\n int nb = super_block_size/block_size;\n memset(scales_h, 0, ((nb+7)/8)*sizeof(uint16_t));\n float d = -max_scale/32;\n dh[0] = GGML_FP32_TO_FP16(d);\n float id = d ? 1/d : 0.f;\n for (int ib = 0; ib < super_block_size/block_size; ++ib) {\n int l = nearest_int(id*scales[ib]);\n l = MAX(-32, MIN(31, l));\n float dl = d * l;\n float idl = dl ? 1/dl : 0.f;\n uint8_t * Lb = L + ib*block_size;\n const float * xb = x + ib*block_size;\n for (int j = 0; j < block_size; ++j) {\n Lb[j] = best_index_int8(16, values, idl*xb[j]);\n }\n l += 32;\n uint8_t l_l = l & 0xf;\n uint8_t l_h = l >> 4;\n if (ib%2 == 0) scales_l[ib/2] = l_l;\n else scales_l[ib/2] |= (l_l << 4);\n scales_h[ib/8] |= (l_h << 2*(ib%8));\n }\n } else {\n dh[0] = GGML_FP32_TO_FP16(scales[0]);\n if (ntry > 0) {\n float id = scales[0] ? 1/scales[0] : 0;\n for (int j = 0; j < super_block_size; ++j) {\n L[j] = best_index_int8(16, values, id*x[j]);\n }\n }\n }\n\n for (int i = 0; i < super_block_size/32; ++i) {\n for (int j = 0; j < 16; ++j) {\n q4[16*i + j] = L[32*i + j] | (L[32*i + 16 + j] << 4);\n }\n }\n}\n\nsize_t quantize_iq4_nl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_ASSERT(n_per_row%QK4_NL == 0);\n int64_t nblock = n_per_row/QK4_NL;\n char * qrow = (char *)dst;\n uint8_t L[QK4_NL];\n float weight[QK4_NL];\n uint16_t unused_h;\n uint8_t * unused_l = NULL;\n float scale;\n for (int64_t row = 0; row < nrow; ++row) {\n block_iq4_nl * iq4 = (block_iq4_nl *)qrow;\n for (int ibl = 0; ibl < nblock; ++ibl) {\n const float * qw = quant_weights ? quant_weights + QK4_NL*ibl : NULL;\n quantize_row_iq4_nl_impl(QK4_NL, 32, src + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l,\n &scale, weight, L, kvalues_iq4nl, qw, 7);\n }\n src += n_per_row;\n qrow += nblock*sizeof(block_iq4_nl);\n }\n return nrow * nblock * sizeof(block_iq4_nl);\n}\n\n//void quantize_row_iq4_nl_ref(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\nvoid quantize_row_iq4_nl_ref(const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k) {\n GGML_ASSERT(k%QK4_NL == 0);\n int64_t nblock = k/QK4_NL;\n uint8_t L[QK4_NL];\n float weight[QK4_NL];\n uint16_t unused_h;\n uint8_t * unused_l = NULL;\n float scale;\n block_iq4_nl * iq4 = y;\n for (int ibl = 0; ibl < nblock; ++ibl) {\n quantize_row_iq4_nl_impl(QK4_NL, 32, x + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l,\n &scale, weight, L, kvalues_iq4nl, NULL, -1);\n }\n}\n\nsize_t quantize_iq4_xs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_ASSERT(n_per_row%QK_K == 0);\n int64_t nblock = n_per_row/QK_K;\n char * qrow = (char *)dst;\n uint8_t L[QK_K];\n float weight[32];\n float scales[QK_K/32];\n for (int64_t row = 0; row < nrow; ++row) {\n block_iq4_xs * iq4 = (block_iq4_xs *)qrow;\n for (int ibl = 0; ibl < nblock; ++ibl) {\n const float * qw = quant_weights ? quant_weights + QK_K*ibl : NULL;\n quantize_row_iq4_nl_impl(QK_K, 32, src + QK_K*ibl, &iq4[ibl].d, iq4[ibl].qs, &iq4[ibl].scales_h, iq4[ibl].scales_l,\n scales, weight, L, kvalues_iq4nl, qw, 7);\n }\n src += n_per_row;\n qrow += nblock*sizeof(block_iq4_xs);\n }\n return nrow * nblock * sizeof(block_iq4_xs);\n}\n\nvoid quantize_row_iq4_xs_ref(const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n quantize_iq4_xs(x, y, 1, k, NULL);\n}\n\n// =============================== 2.5625 bpw\n\nstatic void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {\n\n const int gindex = iq2_data_index(GGML_TYPE_IQ2_S);\n\n const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;\n const int * kmap_q2xs = iq2_data[gindex].map;\n const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;\n\n GGML_ASSERT(kmap_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kgrid_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\");\n GGML_ASSERT(n%QK_K == 0);\n\n const int kMaxQ = 3;\n\n const int64_t nbl = n/QK_K;\n\n block_iq2_s * y = vy;\n\n float scales[QK_K/16];\n float weight[16];\n float xval[16];\n int8_t L[16];\n int8_t Laux[16];\n float waux[16];\n bool is_on_grid[2];\n bool is_on_grid_aux[2];\n uint8_t block_signs[2];\n\n for (int ibl = 0; ibl < nbl; ++ibl) {\n\n memset(&y[ibl], 0, sizeof(block_iq2_s));\n y[ibl].d = GGML_FP32_TO_FP16(0.f);\n\n float max_scale = 0;\n\n const float * xbl = x + QK_K*ibl;\n float sumx2 = 0;\n for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];\n float sigma2 = 2*sumx2/QK_K;\n\n for (int ib = 0; ib < QK_K/16; ++ib) {\n const float * xb = xbl + 16*ib;\n if (quant_weights) {\n const float * qw = quant_weights + QK_K*ibl + 16*ib;\n for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);\n } else {\n for (int i = 0; i < 16; ++i) weight[i] = 0.25f*sigma2 + xb[i]*xb[i];\n }\n for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]);\n for (int k = 0; k < 2; ++k) {\n uint8_t s = 0;\n for (int i = 0; i < 8; ++i) {\n if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];\n else {\n xval[8*k + i] = -xb[8*k + i]; s |= (1 << i);\n }\n }\n block_signs[k] = s;\n }\n float max = xval[0];\n for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);\n memset(L, 0, 16);\n if (max < GROUP_MAX_EPS_IQ2_S) {\n scales[ib] = 0;\n continue;\n }\n float best = 0;\n float scale = max/(2*kMaxQ-1);\n is_on_grid[0] = is_on_grid[1] = true;\n for (int is = -9; is <= 9; ++is) {\n float id = (2*kMaxQ-1+is*0.1f)/max;\n float this_scale = 1/id;\n for (int k = 0; k < 2; ++k) {\n for (int i = 0; i < 8; ++i) {\n int l = nearest_int(0.5f*(id*xval[8*k+i]-1));\n Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));\n }\n uint16_t u = 0;\n for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);\n int grid_index = kmap_q2xs[u];\n is_on_grid_aux[k] = true;\n if (grid_index < 0) {\n is_on_grid_aux[k] = false;\n const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;\n grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);\n }\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 16; ++i) {\n float w = weight[i];\n float q = 2*Laux[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {\n scale = sumqx/sumq2; best = scale*sumqx;\n for (int i = 0; i < 16; ++i) L[i] = Laux[i];\n for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k];\n }\n }\n int n_not_ongrid = 0;\n for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid;\n if (n_not_ongrid > 0 && scale > 0) {\n float id = 1/scale;\n for (int k = 0; k < 2; ++k) {\n if (is_on_grid[k]) continue;\n uint16_t u = 0;\n for (int i = 0; i < 8; ++i) {\n int l = nearest_int(0.5f*(id*xval[8*k+i]-1));\n l = MAX(0, MIN(kMaxQ-1, l));\n u |= (l << 2*i);\n L[8*k + i] = l;\n }\n int grid_index = kmap_q2xs[u];\n if (grid_index < 0) {\n const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;\n grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);\n }\n }\n float sumqx = 0, sumq2 = 0;\n for (int i = 0; i < 16; ++i) {\n float w = weight[i];\n float q = 2*L[i] + 1;\n sumqx += w*xval[i]*q;\n sumq2 += w*q*q;\n }\n if (sumq2 > 0) scale = sumqx/sumq2;\n }\n if (scale < 0) {\n scale = -scale;\n for (int k = 0; k < 2; ++k) block_signs[k] = ~block_signs[k];\n }\n for (int k = 0; k < 2; ++k) {\n uint16_t u = 0;\n for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);\n int grid_index = kmap_q2xs[u];\n if (grid_index < 0) {\n printf(\"Oops: found point %u not on grid:\", u);\n for (int i = 0; i < 8; ++i) printf(\" %d\", L[8*k+i]);\n printf(\"\\n\");\n GGML_ABORT(\"fatal error\");\n }\n const int i8 = 2*ib + k;\n y[ibl].qs[i8] = grid_index & 255;\n y[ibl].qh[i8/4] |= ((grid_index >> 8) << 2*(i8%4));\n y[ibl].qs[QK_K/8 + i8] = block_signs[k];\n }\n GGML_ASSERT(scale >= 0);\n scales[ib] = scale;\n max_scale = MAX(max_scale, scale);\n }\n\n if (!max_scale) {\n continue;\n }\n\n float d = max_scale/31;\n y[ibl].d = GGML_FP32_TO_FP16(d * 0.9875f);\n float id = 1/d;\n for (int ib = 0; ib < QK_K/16; ++ib) {\n int l = nearest_int(0.5f*(id*scales[ib]-1));\n l = MAX(0, MIN(15, l));\n if (ib%2 == 0) y[ibl].scales[ib/2] = l;\n else y[ibl].scales[ib/2] |= (l << 4);\n }\n }\n}\n\nsize_t quantize_iq2_s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {\n GGML_ASSERT(n_per_row%QK_K == 0);\n int64_t nblock = n_per_row/QK_K;\n char * qrow = (char *)dst;\n for (int64_t row = 0; row < nrow; ++row) {\n quantize_row_iq2_s_impl(src, qrow, n_per_row, quant_weights);\n src += n_per_row;\n qrow += nblock*sizeof(block_iq2_s);\n }\n return nrow * nblock * sizeof(block_iq2_s);\n}\n\nvoid quantize_row_iq2_s_ref(const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k) {\n assert(k % QK_K == 0);\n quantize_iq2_s(x, y, 1, k, NULL);\n}\n\n// =============================== data validation\n\nstatic bool validate_float(float f, size_t i) {\n if (isinf(f)) {\n fprintf(stderr, \"ggml_validate_row_data: found inf value at block %zu\\n\", i);\n return false;\n }\n\n if (isnan(f)) {\n fprintf(stderr, \"ggml_validate_row_data: found nan value at block %zu\\n\", i);\n return false;\n }\n\n return true;\n}\n\nstatic bool isinf_fp16(ggml_fp16_t f) {\n return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) == 0;\n}\n\nstatic bool isnan_fp16(ggml_fp16_t f) {\n return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) != 0;\n}\n\nstatic bool validate_fp16(ggml_fp16_t f, size_t i) {\n if (isinf_fp16(f)) {\n fprintf(stderr, \"ggml_validate_row_data: found inf value at block %zu\\n\", i);\n return false;\n }\n\n if (isnan_fp16(f)) {\n fprintf(stderr, \"ggml_validate_row_data: found nan value at block %zu\\n\", i);\n return false;\n }\n\n return true;\n}\n\nstatic bool validate_e_e8m0(uint8_t e, size_t i) {\n if (e == 0xff) {\n fprintf(stderr, \"ggml_validate_row_data: found invalid e value %d at block %zu\\n\", e, i);\n return false;\n }\n\n return true;\n}\n\n#define VALIDATE_ROW_DATA_D_F16_IMPL(type, data, nb) \\\n const type * q = (const type *) (data); \\\n for (size_t i = 0; i < (nb); ++i) { \\\n if (!validate_fp16(q[i].d, i)) { \\\n return false; \\\n } \\\n }\n\n#define VALIDATE_ROW_DATA_DM_F16_IMPL(type, data, nb, d, m) \\\n const type * q = (const type *) (data); \\\n for (size_t i = 0; i < (nb); ++i) { \\\n if (!validate_fp16(q[i].d, i) || !validate_fp16(q[i].m, i)) { \\\n return false; \\\n } \\\n }\n\n#define VALIDATE_ROW_DATA_E_E8M0_IMPL(type, data, nb) \\\n const type * q = (const type *) (data); \\\n for (size_t i = 0; i < (nb); ++i) { \\\n if (!validate_e_e8m0(q[i].e, i)) { \\\n return false; \\\n } \\\n }\n\n#define VALIDATE_ROW_DATA_DVEC_F16_IMPL(type, data, nb, nr) \\\n const type * q = (const type *) (data); \\\n for (size_t i = 0; i < (nb); ++i) { \\\n for (size_t j = 0; j < (nr); ++j) { \\\n if (!validate_fp16(q[i].d[j], i)) { \\\n return false; \\\n } \\\n } \\\n }\n\nbool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes) {\n if (type < 0 || type >= GGML_TYPE_COUNT) {\n fprintf(stderr, \"%s: invalid type %d\\n\", __func__, type);\n return false;\n }\n\n if (nbytes % ggml_type_size(type) != 0) {\n fprintf(stderr, \"%s: invalid size %zu for type %s (type size = %zu)\\n\", __func__, nbytes, ggml_type_name(type), ggml_type_size(type));\n return false;\n }\n\n const size_t nb = nbytes/ggml_type_size(type);\n\n switch (type) {\n case GGML_TYPE_BF16:\n {\n int nans = 0;\n int infs = 0;\n const unsigned short * f = (const unsigned short *) data;\n for (size_t i = 0; i < nb; ++i) {\n nans += (f[i] & 0x7fff) > 0x7f80;\n infs += (f[i] & 0x7fff) == 0x7f80;\n }\n if (nans) {\n fprintf(stderr, \"%s: found %d NaNs in row of %zu BF16 values\\n\", __func__, nans, nb);\n return false;\n }\n if (infs) {\n fprintf(stderr, \"%s: found %d infinities in row of %zu BF16 values\\n\", __func__, infs, nb);\n return false;\n }\n } break;\n case GGML_TYPE_F16:\n {\n const ggml_fp16_t * f = (const ggml_fp16_t *) data;\n size_t i = 0;\n#if defined(__AVX2__)\n for (; i + 15 < nb; i += 16) {\n __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));\n __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi16(0x7c00));\n __m256i cmp = _mm256_cmpeq_epi16(vexp, _mm256_set1_epi16(0x7c00));\n int mask = _mm256_movemask_epi8(cmp);\n if (mask) {\n for (size_t j = 0; j < 16; ++j) {\n if (!validate_fp16(f[i + j], i + j)) {\n return false;\n }\n }\n GGML_UNREACHABLE();\n }\n }\n#elif defined(__ARM_NEON)\n for (; i + 7 < nb; i += 8) {\n uint16x8_t v = vld1q_u16(f + i);\n uint16x8_t vexp = vandq_u16(v, vdupq_n_u16(0x7c00));\n uint16x8_t cmp = vceqq_u16(vexp, vdupq_n_u16(0x7c00));\n uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(cmp, 4)), 0);\n if (mask) {\n for (size_t j = 0; j < 8; ++j) {\n if (!validate_fp16(f[i + j], i + j)) {\n return false;\n }\n }\n GGML_UNREACHABLE();\n }\n }\n#endif\n for (; i < nb; ++i) {\n if (!validate_fp16(f[i], i)) {\n return false;\n }\n }\n } break;\n case GGML_TYPE_F32:\n {\n const float * f = (const float *) data;\n size_t i = 0;\n#if defined(__AVX2__)\n for (; i + 7 < nb; i += 8) {\n __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));\n __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi32(0x7f800000));\n __m256i cmp = _mm256_cmpeq_epi32(vexp, _mm256_set1_epi32(0x7f800000));\n int mask = _mm256_movemask_epi8(cmp);\n if (mask) {\n for (size_t j = 0; j < 8; ++j) {\n if (!validate_float(f[i + j], i + j)) {\n return false;\n }\n }\n GGML_UNREACHABLE();\n }\n }\n#elif defined(__ARM_NEON)\n for (; i + 3 < nb; i += 4) {\n uint32x4_t v = vld1q_u32((const uint32_t *)f + i);\n uint32x4_t vexp = vandq_u32(v, vdupq_n_u32(0x7f800000));\n uint32x4_t cmp = vceqq_u32(vexp, vdupq_n_u32(0x7f800000));\n uint64_t mask = vget_lane_u64(vreinterpret_u64_u16(vshrn_n_u32(cmp, 8)), 0);\n if (mask) {\n for (size_t j = 0; j < 4; ++j) {\n if (!validate_float(f[i + j], i + j)) {\n return false;\n }\n }\n GGML_UNREACHABLE();\n }\n }\n#endif\n for (; i < nb; ++i) {\n if (!validate_float(f[i], i)) {\n return false;\n }\n }\n } break;\n case GGML_TYPE_F64:\n {\n const double * f = (const double *) data;\n for (size_t i = 0; i < nb; ++i) {\n if (!validate_float(f[i], i)) {\n return false;\n }\n }\n } break;\n case GGML_TYPE_Q4_0:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb);\n } break;\n case GGML_TYPE_Q4_1:\n {\n VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_1, data, nb, d, m);\n } break;\n case GGML_TYPE_Q5_0:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_q5_0, data, nb);\n } break;\n case GGML_TYPE_Q5_1:\n {\n VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_1, data, nb, d, m);\n } break;\n case GGML_TYPE_Q8_0:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_q8_0, data, nb);\n } break;\n case GGML_TYPE_MXFP4:\n {\n VALIDATE_ROW_DATA_E_E8M0_IMPL(block_mxfp4, data, nb);\n } break;\n case GGML_TYPE_NVFP4:\n {\n // UE4M3 scales are uint8_t — all byte values are valid\n GGML_UNUSED(data);\n GGML_UNUSED(nb);\n } break;\n case GGML_TYPE_Q2_K:\n {\n VALIDATE_ROW_DATA_DM_F16_IMPL(block_q2_K, data, nb, d, dmin);\n } break;\n case GGML_TYPE_Q3_K:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_q3_K, data, nb);\n } break;\n case GGML_TYPE_Q4_K:\n {\n VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d, dmin);\n } break;\n case GGML_TYPE_Q5_K:\n {\n VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_K, data, nb, d, dmin);\n } break;\n case GGML_TYPE_Q6_K:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_q6_K, data, nb);\n } break;\n case GGML_TYPE_Q8_K:\n {\n const block_q8_K * q = (const block_q8_K *) data;\n for (size_t i = 0; i < nb; ++i) {\n if (!validate_float(q[i].d, i)) {\n return false;\n }\n }\n } break;\n case GGML_TYPE_TQ1_0:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_tq1_0, data, nb);\n } break;\n case GGML_TYPE_TQ2_0:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_tq2_0, data, nb);\n } break;\n case GGML_TYPE_IQ1_S:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_s, data, nb);\n } break;\n case GGML_TYPE_IQ1_M:\n {\n const block_iq1_m * q = (const block_iq1_m *) data;\n for (size_t i = 0; i < nb; ++i) {\n iq1m_scale_t scale;\n const uint16_t * sc = (const uint16_t *)q[i].scales;\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n if (!validate_fp16(scale.f16, i)) {\n return false;\n }\n }\n } break;\n case GGML_TYPE_IQ2_XXS:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xxs, data, nb);\n } break;\n case GGML_TYPE_IQ2_XS:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xs, data, nb);\n } break;\n case GGML_TYPE_IQ2_S:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_s, data, nb);\n } break;\n case GGML_TYPE_IQ3_XXS:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_xxs, data, nb);\n } break;\n\n case GGML_TYPE_IQ3_S:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_s, data, nb);\n } break;\n case GGML_TYPE_IQ4_XS:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_xs, data, nb);\n } break;\n case GGML_TYPE_IQ4_NL:\n {\n VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb);\n } break;\n\n case GGML_TYPE_I8:\n case GGML_TYPE_I16:\n case GGML_TYPE_I32:\n case GGML_TYPE_I64:\n // nothing to validate\n break;\n default:\n {\n fprintf(stderr, \"%s: invalid type %d\\n\", __func__, type);\n return false;\n }\n }\n\n return true;\n}\n" }, { "path": "src/ggml-quants.h", "content": "#pragma once\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n\n#include \"ggml.h\"\n\n// GGML internal header\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// NOTE: these functions are defined as GGML_API because they used by the CPU backend\n\n// Quantization\nGGML_API void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q5_1_ref(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q8_0_ref(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);\n\nGGML_API void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RESTRICT y, int64_t k);\n\nGGML_API void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q4_K_ref(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q5_K_ref(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);\n\nGGML_API void quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_tq2_0_ref(const float * GGML_RESTRICT x, block_tq2_0 * GGML_RESTRICT y, int64_t k);\n\nGGML_API void quantize_row_iq3_xxs_ref(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_iq4_nl_ref (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_iq4_xs_ref (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_iq3_s_ref (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);\nGGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);\n\n// Dequantization\nGGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\n//GGML_API void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\n\nGGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\n\nGGML_API void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\n\nGGML_API void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\n\nGGML_API void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\nGGML_API void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\n\n// Quantization utilizing an importance matrix (a.k.a. \"Activation aWare Quantization\")\nGGML_API size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_iq1_m (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\n\nGGML_API size_t quantize_tq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\n\nGGML_API size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\n\nGGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\nGGML_API size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);\n\nGGML_API void iq2xs_init_impl(enum ggml_type type);\nGGML_API void iq2xs_free_impl(enum ggml_type type);\nGGML_API void iq3xs_init_impl(int grid_size);\nGGML_API void iq3xs_free_impl(int grid_size);\n\n#ifdef __cplusplus\n}\n#endif\n" }, { "path": "src/ggml-rpc/CMakeLists.txt", "content": "message(STATUS \"Using RPC backend\")\n\nggml_add_backend_library(ggml-rpc\n ggml-rpc.cpp\n )\n\nif (WIN32)\n target_link_libraries(ggml-rpc PRIVATE ws2_32)\nendif()\n" }, { "path": "src/ggml-rpc/ggml-rpc.cpp", "content": "#include \"ggml-rpc.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-cpp.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#ifdef _WIN32\n# define WIN32_LEAN_AND_MEAN\n# ifndef NOMINMAX\n# define NOMINMAX\n# endif\n# include \n# include \n#else\n# include \n# include \n# include \n# include \n# include \n# include \n# include \n#endif\n#include \n#include \n#include \n#include \n\nstatic const char * RPC_DEBUG = std::getenv(\"GGML_RPC_DEBUG\");\n\n#define LOG_DBG(...) \\\n do { if (RPC_DEBUG) GGML_LOG_DEBUG(__VA_ARGS__); } while (0)\n\n\nnamespace fs = std::filesystem;\n\nstatic constexpr size_t MAX_CHUNK_SIZE = 1024ull * 1024ull * 1024ull; // 1 GiB\n\n#ifdef _WIN32\ntypedef SOCKET sockfd_t;\nusing ssize_t = __int64;\n#else\ntypedef int sockfd_t;\n#endif\n\n// cross-platform socket\nstruct socket_t {\n sockfd_t fd;\n socket_t(sockfd_t fd) : fd(fd) {}\n ~socket_t() {\n LOG_DBG(\"[%s] closing socket %d\\n\", __func__, this->fd);\n#ifdef _WIN32\n closesocket(this->fd);\n#else\n close(this->fd);\n#endif\n }\n};\n\n// macro for nicer error messages on server crash\n#define RPC_STATUS_ASSERT(x) if (!(x)) GGML_ABORT(\"Remote RPC server crashed or returned malformed response\")\n\n// all RPC structures must be packed\n#pragma pack(push, 1)\n// ggml_tensor is serialized into rpc_tensor\nstruct rpc_tensor {\n uint64_t id;\n uint32_t type;\n uint64_t buffer;\n uint32_t ne[GGML_MAX_DIMS];\n uint32_t nb[GGML_MAX_DIMS];\n uint32_t op;\n int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];\n int32_t flags;\n uint64_t src[GGML_MAX_SRC];\n uint64_t view_src;\n uint64_t view_offs;\n uint64_t data;\n char name[GGML_MAX_NAME];\n\n char padding[4];\n};\n\nstatic_assert(sizeof(rpc_tensor) % 8 == 0, \"rpc_tensor size must be multiple of 8\");\n\n// RPC commands\nenum rpc_cmd {\n RPC_CMD_ALLOC_BUFFER = 0,\n RPC_CMD_GET_ALIGNMENT,\n RPC_CMD_GET_MAX_SIZE,\n RPC_CMD_BUFFER_GET_BASE,\n RPC_CMD_FREE_BUFFER,\n RPC_CMD_BUFFER_CLEAR,\n RPC_CMD_SET_TENSOR,\n RPC_CMD_SET_TENSOR_HASH,\n RPC_CMD_GET_TENSOR,\n RPC_CMD_COPY_TENSOR,\n RPC_CMD_GRAPH_COMPUTE,\n RPC_CMD_GET_DEVICE_MEMORY,\n RPC_CMD_INIT_TENSOR,\n RPC_CMD_GET_ALLOC_SIZE,\n RPC_CMD_HELLO,\n RPC_CMD_DEVICE_COUNT,\n RPC_CMD_GRAPH_RECOMPUTE,\n RPC_CMD_COUNT,\n};\n\nstatic_assert(RPC_CMD_HELLO == 14, \"RPC_CMD_HELLO must be always 14\");\n\n// Try RPC_CMD_SET_TENSOR_HASH first when data size is larger than this threshold\nconst size_t HASH_THRESHOLD = 10 * 1024 * 1024;\n\nstruct rpc_msg_hello_rsp {\n uint8_t major;\n uint8_t minor;\n uint8_t patch;\n};\n\nstruct rpc_msg_device_count_rsp {\n uint32_t device_count;\n};\n\nstruct rpc_msg_get_alloc_size_req {\n uint32_t device;\n rpc_tensor tensor;\n rpc_tensor srcs[GGML_MAX_SRC];\n};\n\nstruct rpc_msg_get_alloc_size_rsp {\n uint64_t alloc_size;\n};\n\nstruct rpc_msg_init_tensor_req {\n rpc_tensor tensor;\n};\n\nstruct rpc_msg_alloc_buffer_req {\n uint32_t device;\n uint64_t size;\n};\n\nstruct rpc_msg_alloc_buffer_rsp {\n uint64_t remote_ptr;\n uint64_t remote_size;\n};\n\nstruct rpc_msg_get_alignment_req {\n uint32_t device;\n};\n\nstruct rpc_msg_get_alignment_rsp {\n uint64_t alignment;\n};\n\nstruct rpc_msg_get_max_size_req {\n uint32_t device;\n};\n\nstruct rpc_msg_get_max_size_rsp {\n uint64_t max_size;\n};\n\nstruct rpc_msg_buffer_get_base_req {\n uint64_t remote_ptr;\n};\n\nstruct rpc_msg_buffer_get_base_rsp {\n uint64_t base_ptr;\n};\n\nstruct rpc_msg_free_buffer_req {\n uint64_t remote_ptr;\n};\n\nstruct rpc_msg_buffer_clear_req {\n uint64_t remote_ptr;\n uint8_t value;\n};\n\nstruct rpc_msg_set_tensor_hash_req {\n rpc_tensor tensor;\n uint64_t offset;\n uint64_t hash;\n};\n\nstruct rpc_msg_set_tensor_hash_rsp {\n uint8_t result;\n};\n\nstruct rpc_msg_get_tensor_req {\n rpc_tensor tensor;\n uint64_t offset;\n uint64_t size;\n};\n\nstruct rpc_msg_copy_tensor_req {\n rpc_tensor src;\n rpc_tensor dst;\n};\n\nstruct rpc_msg_copy_tensor_rsp {\n uint8_t result;\n};\n\nstruct rpc_msg_get_device_memory_req {\n uint32_t device;\n};\n\nstruct rpc_msg_get_device_memory_rsp {\n uint64_t free_mem;\n uint64_t total_mem;\n};\n\nstruct rpc_msg_graph_recompute_req {\n uint32_t device;\n};\n\n#pragma pack(pop)\n\n// RPC data structures\n\nstatic ggml_guid_t ggml_backend_rpc_guid() {\n static ggml_guid guid = {0x99, 0x68, 0x5b, 0x6c, 0xd2, 0x83, 0x3d, 0x24, 0x25, 0x36, 0x72, 0xe1, 0x5b, 0x0e, 0x14, 0x03};\n return &guid;\n}\n\nstruct ggml_backend_rpc_buffer_type_context {\n std::string endpoint;\n uint32_t device;\n std::string name;\n size_t alignment;\n size_t max_size;\n};\n\nstruct graph_cache {\n\n bool is_cached(const ggml_cgraph * cgraph) {\n if ((int)last_graph.size() != cgraph->n_nodes) {\n return false;\n }\n for (int i = 0; i < cgraph->n_nodes; i++) {\n if (memcmp(&last_graph[i], cgraph->nodes[i], sizeof(ggml_tensor)) != 0) {\n return false;\n }\n }\n return true;\n }\n\n void add(const ggml_cgraph * cgraph) {\n last_graph.resize(cgraph->n_nodes);\n for (int i = 0; i < cgraph->n_nodes; i++) {\n memcpy(&last_graph[i], cgraph->nodes[i], sizeof(ggml_tensor));\n }\n }\n\n std::vector last_graph;\n};\n\nstruct ggml_backend_rpc_context {\n std::string endpoint;\n uint32_t device;\n std::string name;\n graph_cache gc;\n};\n\nstruct ggml_backend_rpc_buffer_context {\n std::shared_ptr sock;\n void * base_ptr;\n uint64_t remote_ptr;\n};\n\n// RPC helper functions\n\n// Computes FNV-1a hash of the data\nstatic uint64_t fnv_hash(const uint8_t * data, size_t len) {\n const uint64_t fnv_prime = 0x100000001b3ULL;\n uint64_t hash = 0xcbf29ce484222325ULL;\n\n for (size_t i = 0; i < len; ++i) {\n hash ^= data[i];\n hash *= fnv_prime;\n }\n return hash;\n}\n\nstatic std::shared_ptr make_socket(sockfd_t fd) {\n#ifdef _WIN32\n if (fd == INVALID_SOCKET) {\n return nullptr;\n }\n#else\n if (fd < 0) {\n return nullptr;\n }\n#endif\n return std::make_shared(fd);\n}\n\nstatic bool set_no_delay(sockfd_t sockfd) {\n int flag = 1;\n // set TCP_NODELAY to disable Nagle's algorithm\n int ret = setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(int));\n return ret == 0;\n}\n\nstatic bool set_reuse_addr(sockfd_t sockfd) {\n int flag = 1;\n int ret = setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof(int));\n return ret == 0;\n}\n\nstatic std::shared_ptr socket_connect(const char * host, int port) {\n struct sockaddr_in addr;\n auto sockfd = socket(AF_INET, SOCK_STREAM, 0);\n auto sock_ptr = make_socket(sockfd);\n if (sock_ptr == nullptr) {\n return nullptr;\n }\n if (!set_no_delay(sockfd)) {\n GGML_LOG_ERROR(\"Failed to set TCP_NODELAY\\n\");\n return nullptr;\n }\n addr.sin_family = AF_INET;\n addr.sin_port = htons(port);\n struct hostent * server = gethostbyname(host);\n if (server == NULL) {\n GGML_LOG_ERROR(\"Cannot resolve host '%s'\\n\", host);\n return nullptr;\n }\n memcpy(&addr.sin_addr.s_addr, server->h_addr, server->h_length);\n if (connect(sock_ptr->fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {\n return nullptr;\n }\n return sock_ptr;\n}\n\nstatic std::shared_ptr socket_accept(sockfd_t srv_sockfd) {\n auto client_socket_fd = accept(srv_sockfd, NULL, NULL);\n auto client_socket = make_socket(client_socket_fd);\n if (client_socket == nullptr) {\n return nullptr;\n }\n if (!set_no_delay(client_socket_fd)) {\n GGML_LOG_ERROR(\"Failed to set TCP_NODELAY\\n\");\n return nullptr;\n }\n return client_socket;\n}\n\nstatic std::shared_ptr create_server_socket(const char * host, int port) {\n auto sockfd = socket(AF_INET, SOCK_STREAM, 0);\n auto sock = make_socket(sockfd);\n if (sock == nullptr) {\n return nullptr;\n }\n if (!set_reuse_addr(sockfd)) {\n GGML_LOG_ERROR(\"Failed to set SO_REUSEADDR\\n\");\n return nullptr;\n }\n if (inet_addr(host) == INADDR_NONE) {\n GGML_LOG_ERROR(\"Invalid host address: %s\\n\", host);\n return nullptr;\n }\n struct sockaddr_in serv_addr;\n serv_addr.sin_family = AF_INET;\n serv_addr.sin_addr.s_addr = inet_addr(host);\n serv_addr.sin_port = htons(port);\n\n if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) {\n return nullptr;\n }\n if (listen(sockfd, 1) < 0) {\n return nullptr;\n }\n return sock;\n}\n\nstatic bool send_data(sockfd_t sockfd, const void * data, size_t size) {\n size_t bytes_sent = 0;\n while (bytes_sent < size) {\n size_t size_to_send = std::min(size - bytes_sent, MAX_CHUNK_SIZE);\n ssize_t n = send(sockfd, (const char *)data + bytes_sent, size_to_send, 0);\n if (n < 0) {\n GGML_LOG_ERROR(\"send failed (bytes_sent=%zu, size_to_send=%zu)\\n\",\n bytes_sent, size_to_send);\n return false;\n }\n bytes_sent += (size_t)n;\n }\n return true;\n}\n\nstatic bool recv_data(sockfd_t sockfd, void * data, size_t size) {\n size_t bytes_recv = 0;\n while (bytes_recv < size) {\n size_t size_to_recv = std::min(size - bytes_recv, MAX_CHUNK_SIZE);\n ssize_t n = recv(sockfd, (char *)data + bytes_recv, size_to_recv, 0);\n if (n < 0) {\n GGML_LOG_ERROR(\"recv failed (bytes_recv=%zu, size_to_recv=%zu)\\n\",\n bytes_recv, size_to_recv);\n return false;\n }\n if (n == 0) {\n LOG_DBG(\"recv returned 0 (peer closed?)\\n\");\n return false;\n }\n bytes_recv += (size_t)n;\n }\n return true;\n}\n\nstatic bool send_msg(sockfd_t sockfd, const void * msg, size_t msg_size) {\n if (!send_data(sockfd, &msg_size, sizeof(msg_size))) {\n return false;\n }\n return send_data(sockfd, msg, msg_size);\n}\n\nstatic bool recv_msg(sockfd_t sockfd, void * msg, size_t msg_size) {\n uint64_t size;\n if (!recv_data(sockfd, &size, sizeof(size))) {\n return false;\n }\n if (size != msg_size) {\n return false;\n }\n return recv_data(sockfd, msg, msg_size);\n}\n\nstatic bool recv_msg(sockfd_t sockfd, std::vector & input) {\n uint64_t size;\n if (!recv_data(sockfd, &size, sizeof(size))) {\n return false;\n }\n try {\n input.resize(size);\n } catch (const std::bad_alloc & e) {\n GGML_LOG_ERROR(\"Failed to allocate input buffer of size %\" PRIu64 \"\\n\", size);\n return false;\n }\n return recv_data(sockfd, input.data(), size);\n}\n\nstatic bool parse_endpoint(const std::string & endpoint, std::string & host, int & port) {\n size_t pos = endpoint.find(':');\n if (pos == std::string::npos) {\n return false;\n }\n host = endpoint.substr(0, pos);\n port = std::stoi(endpoint.substr(pos + 1));\n return true;\n}\n\n// RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) |\n// No response\nstatic bool send_rpc_cmd(const std::shared_ptr & sock, enum rpc_cmd cmd, const void * input, size_t input_size) {\n uint8_t cmd_byte = cmd;\n if (!send_data(sock->fd, &cmd_byte, sizeof(cmd_byte))) {\n return false;\n }\n if (!send_data(sock->fd, &input_size, sizeof(input_size))) {\n return false;\n }\n if (!send_data(sock->fd, input, input_size)) {\n return false;\n }\n return true;\n}\n\n// RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) |\n// RPC response: | response_size (8 bytes) | response_data (response_size bytes) |\nstatic bool send_rpc_cmd(const std::shared_ptr & sock, enum rpc_cmd cmd, const void * input, size_t input_size, void * output, size_t output_size) {\n if (!send_rpc_cmd(sock, cmd, input, input_size)) {\n return false;\n }\n // TODO: currently the output_size is always known, do we need support for commands with variable output size?\n // even if we do, we can skip sending output_size from the server for commands with known output size\n uint64_t out_size;\n if (!recv_data(sock->fd, &out_size, sizeof(out_size))) {\n return false;\n }\n if (out_size != output_size) {\n return false;\n }\n if (!recv_data(sock->fd, output, output_size)) {\n return false;\n }\n return true;\n}\n\n// RPC client-side implementation\n\nstatic bool check_server_version(const std::shared_ptr & sock) {\n rpc_msg_hello_rsp response;\n bool status = send_rpc_cmd(sock, RPC_CMD_HELLO, nullptr, 0, &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n if (response.major != RPC_PROTO_MAJOR_VERSION || response.minor > RPC_PROTO_MINOR_VERSION) {\n GGML_LOG_ERROR(\"RPC server version mismatch: %d.%d.%d\\n\", response.major, response.minor, response.patch);\n return false;\n }\n if (response.minor != RPC_PROTO_MINOR_VERSION || response.patch != RPC_PROTO_PATCH_VERSION) {\n GGML_LOG_INFO(\"WARNING: RPC server version mismatch: %d.%d.%d\\n\", response.major, response.minor, response.patch);\n }\n return true;\n}\n\nstatic std::shared_ptr get_socket(const std::string & endpoint) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n static std::unordered_map> sockets;\n static bool initialized = false;\n\n auto it = sockets.find(endpoint);\n if (it != sockets.end()) {\n if (auto sock = it->second.lock()) {\n return sock;\n }\n }\n std::string host;\n int port;\n if (!parse_endpoint(endpoint, host, port)) {\n GGML_LOG_ERROR(\"Failed to parse endpoint: %s\\n\", endpoint.c_str());\n return nullptr;\n }\n#ifdef _WIN32\n if (!initialized) {\n WSADATA wsaData;\n int res = WSAStartup(MAKEWORD(2, 2), &wsaData);\n if (res != 0) {\n return nullptr;\n }\n initialized = true;\n }\n#else\n GGML_UNUSED(initialized);\n#endif\n auto sock = socket_connect(host.c_str(), port);\n if (sock == nullptr) {\n return nullptr;\n }\n if (!check_server_version(sock)) {\n return nullptr;\n }\n LOG_DBG(\"[%s] connected to %s, sockfd=%d\\n\", __func__, endpoint.c_str(), sock->fd);\n sockets[endpoint] = sock;\n return sock;\n}\n\nstatic void ggml_backend_rpc_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;\n rpc_msg_free_buffer_req request = {ctx->remote_ptr};\n bool status = send_rpc_cmd(ctx->sock, RPC_CMD_FREE_BUFFER, &request, sizeof(request), nullptr, 0);\n RPC_STATUS_ASSERT(status);\n delete ctx;\n}\n\nstatic void * ggml_backend_rpc_buffer_get_base(ggml_backend_buffer_t buffer) {\n ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;\n if (ctx->base_ptr != nullptr) {\n return ctx->base_ptr;\n }\n rpc_msg_buffer_get_base_req request = {ctx->remote_ptr};\n rpc_msg_buffer_get_base_rsp response;\n bool status = send_rpc_cmd(ctx->sock, RPC_CMD_BUFFER_GET_BASE, &request, sizeof(request), &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n ctx->base_ptr = reinterpret_cast(response.base_ptr);\n return ctx->base_ptr;\n}\n\nstatic bool ggml_backend_buffer_is_rpc(ggml_backend_buffer_t buffer) {\n return buffer->iface.free_buffer == ggml_backend_rpc_buffer_free_buffer;\n}\n\nstatic rpc_tensor serialize_tensor(const ggml_tensor * tensor) {\n rpc_tensor result;\n if (!tensor) {\n memset(&result, 0, sizeof(result));\n return result;\n }\n\n result.id = reinterpret_cast(tensor);\n result.type = tensor->type;\n if (tensor->buffer && ggml_backend_buffer_is_rpc(tensor->buffer)) {\n ggml_backend_buffer_t buffer = tensor->buffer;\n ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;\n result.buffer = ctx != nullptr ? ctx->remote_ptr : 0;\n } else {\n result.buffer = 0;\n }\n for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) {\n result.ne[i] = tensor->ne[i];\n result.nb[i] = tensor->nb[i];\n }\n result.op = tensor->op;\n for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) {\n result.op_params[i] = tensor->op_params[i];\n }\n result.flags = tensor->flags;\n for (uint32_t i = 0; i < GGML_MAX_SRC; i++) {\n result.src[i] = reinterpret_cast(tensor->src[i]);\n }\n result.view_src = reinterpret_cast(tensor->view_src);\n result.view_offs = tensor->view_offs;\n result.data = reinterpret_cast(tensor->data);\n\n // Avoid sending uninitialized data over the wire\n memset(result.name, 0, sizeof(result.name));\n memset(result.padding, 0, sizeof(result.padding));\n\n snprintf(result.name, GGML_MAX_NAME, \"%s\", tensor->name);\n return result;\n}\n\nstatic enum ggml_status ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {\n ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;\n\n // CUDA backend on the server pads everything to 512 due to CUDA limitations.\n // Due to bandwidth constraints, we only call the server init tensor functions if necessary.\n // In particular, only quantized tensors need padding\n if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr)) {\n rpc_msg_init_tensor_req request;\n\n request.tensor = serialize_tensor(tensor);\n\n bool status = send_rpc_cmd(ctx->sock, RPC_CMD_INIT_TENSOR, &request, sizeof(request), nullptr, 0);\n RPC_STATUS_ASSERT(status);\n }\n return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;\n rpc_tensor rpc_tensor = serialize_tensor(tensor);\n if (size > HASH_THRESHOLD) {\n rpc_msg_set_tensor_hash_req request;\n request.tensor = rpc_tensor;\n request.offset = offset;\n request.hash = fnv_hash((const uint8_t*)data, size);\n rpc_msg_set_tensor_hash_rsp response;\n bool status = send_rpc_cmd(ctx->sock, RPC_CMD_SET_TENSOR_HASH, &request, sizeof(request), &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n if (response.result) {\n // the server has the same data, no need to send it\n return;\n }\n }\n // input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes)\n size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size;\n std::vector input(input_size, 0);\n memcpy(input.data(), &rpc_tensor, sizeof(rpc_tensor));\n memcpy(input.data() + sizeof(rpc_tensor), &offset, sizeof(offset));\n memcpy(input.data() + sizeof(rpc_tensor) + sizeof(offset), data, size);\n bool status = send_rpc_cmd(ctx->sock, RPC_CMD_SET_TENSOR, input.data(), input.size());\n RPC_STATUS_ASSERT(status);\n}\n\nstatic void ggml_backend_rpc_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;\n rpc_msg_get_tensor_req request;\n request.tensor = serialize_tensor(tensor);\n request.offset = offset;\n request.size = size;\n bool status = send_rpc_cmd(ctx->sock, RPC_CMD_GET_TENSOR, &request, sizeof(request), data, size);\n RPC_STATUS_ASSERT(status);\n}\n\nstatic bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {\n if (ggml_backend_buffer_is_rpc(src->buffer)) {\n // check if src and dst are on the same server\n ggml_backend_buffer_t src_buffer = src->buffer;\n ggml_backend_rpc_buffer_context * src_ctx = (ggml_backend_rpc_buffer_context *)src_buffer->context;\n ggml_backend_buffer_t dst_buffer = dst->buffer;\n ggml_backend_rpc_buffer_context * dst_ctx = (ggml_backend_rpc_buffer_context *)dst_buffer->context;\n if (src_ctx->sock != dst_ctx->sock) {\n return false;\n }\n ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;\n rpc_msg_copy_tensor_req request;\n request.src = serialize_tensor(src);\n request.dst = serialize_tensor(dst);\n rpc_msg_copy_tensor_rsp response;\n bool status = send_rpc_cmd(ctx->sock, RPC_CMD_COPY_TENSOR, &request, sizeof(request), &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n return response.result;\n }\n return false;\n}\n\nstatic void ggml_backend_rpc_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;\n rpc_msg_buffer_clear_req request = {ctx->remote_ptr, value};\n bool status = send_rpc_cmd(ctx->sock, RPC_CMD_BUFFER_CLEAR, &request, sizeof(request), nullptr, 0);\n RPC_STATUS_ASSERT(status);\n}\n\nstatic ggml_backend_buffer_i ggml_backend_rpc_buffer_interface = {\n /* .free_buffer = */ ggml_backend_rpc_buffer_free_buffer,\n /* .get_base = */ ggml_backend_rpc_buffer_get_base,\n /* .init_tensor = */ ggml_backend_rpc_buffer_init_tensor,\n /* .memset_tensor = */ NULL,\n /* .set_tensor = */ ggml_backend_rpc_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_rpc_buffer_get_tensor,\n /* .cpy_tensor = */ ggml_backend_rpc_buffer_cpy_tensor,\n /* .clear = */ ggml_backend_rpc_buffer_clear,\n /* .reset = */ NULL,\n};\n\nstatic const char * ggml_backend_rpc_buffer_type_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;\n return buft_ctx->name.c_str();\n}\n\nstatic ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;\n rpc_msg_alloc_buffer_req request = {buft_ctx->device, size};\n rpc_msg_alloc_buffer_rsp response;\n auto sock = get_socket(buft_ctx->endpoint);\n bool status = send_rpc_cmd(sock, RPC_CMD_ALLOC_BUFFER, &request, sizeof(request), &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n if (response.remote_ptr != 0) {\n ggml_backend_buffer_t buffer = ggml_backend_buffer_init(buft,\n ggml_backend_rpc_buffer_interface,\n new ggml_backend_rpc_buffer_context{sock, nullptr, response.remote_ptr},\n response.remote_size);\n return buffer;\n } else {\n return nullptr;\n }\n}\n\nstatic size_t get_alignment(const std::shared_ptr & sock, uint32_t device) {\n rpc_msg_get_alignment_req request = {device};\n rpc_msg_get_alignment_rsp response;\n bool status = send_rpc_cmd(sock, RPC_CMD_GET_ALIGNMENT, &request, sizeof(request), &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n return response.alignment;\n}\n\nstatic size_t ggml_backend_rpc_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;\n return buft_ctx->alignment;\n}\n\nstatic size_t get_max_size(const std::shared_ptr & sock, uint32_t device) {\n rpc_msg_get_max_size_req request = {device};\n rpc_msg_get_max_size_rsp response;\n bool status = send_rpc_cmd(sock, RPC_CMD_GET_MAX_SIZE, &request, sizeof(request), &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n return response.max_size;\n}\n\nstatic size_t ggml_backend_rpc_get_max_size(ggml_backend_buffer_type_t buft) {\n ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;\n return buft_ctx->max_size;\n}\n\nstatic size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n // should we query the remote server for the actual size\n bool rpc_get = false;\n\n // See comments in init_tensor.\n rpc_get |= ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr);\n\n // ops that require additional memory for fleeting data on certain backends\n // ref: https://github.com/ggml-org/llama.cpp/pull/15966\n rpc_get |= tensor->op == GGML_OP_FLASH_ATTN_EXT;\n rpc_get |= tensor->op == GGML_OP_MUL_MAT_ID;\n\n if (rpc_get) {\n ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;\n auto sock = get_socket(buft_ctx->endpoint);\n\n rpc_msg_get_alloc_size_req request = {\n /*.device =*/ buft_ctx->device,\n /*.tensor =*/ serialize_tensor(tensor),\n /*.srcs =*/ {},\n };\n\n // .get_alloc_size could be a function of the tensor's srcs, so we must serialize them as well\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n request.srcs[i] = serialize_tensor(tensor->src[i]);\n }\n\n // TODO: cache the alloc responses to avoid extra RPC calls?\n rpc_msg_get_alloc_size_rsp response;\n bool status = send_rpc_cmd(sock, RPC_CMD_GET_ALLOC_SIZE, &request, sizeof(request), &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n\n return response.alloc_size;\n }\n\n return ggml_nbytes(tensor);\n}\n\nstatic ggml_backend_buffer_type_i ggml_backend_rpc_buffer_type_interface = {\n /* .get_name = */ ggml_backend_rpc_buffer_type_name,\n /* .alloc_buffer = */ ggml_backend_rpc_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_rpc_buffer_type_get_alignment,\n /* .get_max_size = */ ggml_backend_rpc_get_max_size,\n /* .get_alloc_size = */ ggml_backend_rpc_buffer_type_get_alloc_size,\n /* .is_host = */ NULL,\n};\n\nstatic const char * ggml_backend_rpc_name(ggml_backend_t backend) {\n ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;\n\n return rpc_ctx->name.c_str();\n}\n\nstatic void ggml_backend_rpc_free(ggml_backend_t backend) {\n ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;\n delete rpc_ctx;\n delete backend;\n}\n\nstatic void ggml_backend_rpc_synchronize(ggml_backend_t backend) {\n GGML_UNUSED(backend);\n // this is no-op because we don't have any async operations\n}\n\nstatic void add_tensor(ggml_tensor * tensor, std::vector & tensors, std::unordered_set & visited) {\n if (tensor == nullptr) {\n return;\n }\n if (visited.find(tensor) != visited.end()) {\n return;\n }\n visited.insert(tensor);\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n add_tensor(tensor->src[i], tensors, visited);\n }\n add_tensor(tensor->view_src, tensors, visited);\n tensors.push_back(serialize_tensor(tensor));\n}\n\nstatic void serialize_graph(uint32_t device, const ggml_cgraph * cgraph, std::vector & output) {\n uint32_t n_nodes = cgraph->n_nodes;\n std::vector tensors;\n std::unordered_set visited;\n for (uint32_t i = 0; i < n_nodes; i++) {\n add_tensor(cgraph->nodes[i], tensors, visited);\n }\n // serialization format:\n // | device (4 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) |\n uint32_t n_tensors = tensors.size();\n int output_size = 2*sizeof(uint32_t) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t) + n_tensors * sizeof(rpc_tensor);\n output.resize(output_size, 0);\n uint8_t * dest = output.data();\n memcpy(dest, &device, sizeof(device));\n dest += sizeof(device);\n memcpy(dest, &n_nodes, sizeof(n_nodes));\n dest += sizeof(n_nodes);\n for (uint32_t i = 0; i < n_nodes; i++) {\n memcpy(dest + i * sizeof(uint64_t), &cgraph->nodes[i], sizeof(uint64_t));\n }\n dest += n_nodes * sizeof(uint64_t);\n memcpy(dest, &n_tensors, sizeof(n_tensors));\n dest += sizeof(n_tensors);\n rpc_tensor * out_tensors = (rpc_tensor *)dest;\n memcpy(out_tensors, tensors.data(), n_tensors * sizeof(rpc_tensor));\n}\n\nstatic enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {\n ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;\n\n GGML_ASSERT(cgraph->n_nodes > 0);\n bool reuse = rpc_ctx->gc.is_cached(cgraph);\n if (reuse) {\n rpc_msg_graph_recompute_req request;\n request.device = rpc_ctx->device;\n auto sock = get_socket(rpc_ctx->endpoint);\n bool status = send_rpc_cmd(sock, RPC_CMD_GRAPH_RECOMPUTE, &request, sizeof(request));\n RPC_STATUS_ASSERT(status);\n } else {\n rpc_ctx->gc.add(cgraph);\n std::vector input;\n serialize_graph(rpc_ctx->device, cgraph, input);\n auto sock = get_socket(rpc_ctx->endpoint);\n bool status = send_rpc_cmd(sock, RPC_CMD_GRAPH_COMPUTE, input.data(), input.size());\n RPC_STATUS_ASSERT(status);\n }\n return GGML_STATUS_SUCCESS;\n}\n\nstatic ggml_backend_i ggml_backend_rpc_interface = {\n /* .get_name = */ ggml_backend_rpc_name,\n /* .free = */ ggml_backend_rpc_free,\n /* .set_tensor_async = */ NULL,\n /* .get_tensor_async = */ NULL,\n /* .cpy_tensor_async = */ NULL,\n /* .synchronize = */ ggml_backend_rpc_synchronize,\n /* .graph_plan_create = */ NULL,\n /* .graph_plan_free = */ NULL,\n /* .graph_plan_update = */ NULL,\n /* .graph_plan_compute = */ NULL,\n /* .graph_compute = */ ggml_backend_rpc_graph_compute,\n /* .event_record = */ NULL,\n /* .event_wait = */ NULL,\n /* .graph_optimize = */ NULL,\n};\n\nggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, uint32_t device) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n std::string buft_name = \"RPC\" + std::to_string(device) + \"[\" + std::string(endpoint) + \"]\";\n // NOTE: buffer types are allocated and never freed; this is by design\n static std::unordered_map buft_map;\n auto it = buft_map.find(buft_name);\n if (it != buft_map.end()) {\n return it->second;\n }\n auto sock = get_socket(endpoint);\n if (sock == nullptr) {\n GGML_LOG_ERROR(\"Failed to connect to %s\\n\", endpoint);\n return nullptr;\n }\n size_t alignment = get_alignment(sock, device);\n size_t max_size = get_max_size(sock, device);\n ggml_backend_rpc_buffer_type_context * buft_ctx = new ggml_backend_rpc_buffer_type_context {\n /* .endpoint = */ endpoint,\n /* .device = */ device,\n /* .name = */ buft_name,\n /* .alignment = */ alignment,\n /* .max_size = */ max_size\n };\n auto reg = ggml_backend_rpc_add_server(endpoint);\n ggml_backend_buffer_type_t buft = new ggml_backend_buffer_type {\n /* .iface = */ ggml_backend_rpc_buffer_type_interface,\n /* .device = */ ggml_backend_reg_dev_get(reg, device),\n /* .context = */ buft_ctx\n };\n buft_map[buft_name] = buft;\n return buft;\n}\n\nggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device) {\n std::string dev_name = \"RPC\" + std::to_string(device) + \"[\" + std::string(endpoint) + \"]\";\n ggml_backend_rpc_context * ctx = new ggml_backend_rpc_context {\n /* .endpoint = */ endpoint,\n /* .device = */ device,\n /* .name = */ dev_name,\n /* .gc = */ {},\n };\n auto reg = ggml_backend_rpc_add_server(endpoint);\n ggml_backend_t backend = new ggml_backend {\n /* .guid = */ ggml_backend_rpc_guid(),\n /* .iface = */ ggml_backend_rpc_interface,\n /* .device = */ ggml_backend_reg_dev_get(reg, device),\n /* .context = */ ctx\n };\n return backend;\n}\n\nbool ggml_backend_is_rpc(ggml_backend_t backend) {\n return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_rpc_guid());\n}\n\nstatic void get_device_memory(const std::shared_ptr & sock, uint32_t device, size_t * free, size_t * total) {\n rpc_msg_get_device_memory_req request;\n request.device = device;\n rpc_msg_get_device_memory_rsp response;\n bool status = send_rpc_cmd(sock, RPC_CMD_GET_DEVICE_MEMORY, &request, sizeof(request), &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n *free = response.free_mem;\n *total = response.total_mem;\n}\n\nvoid ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total) {\n auto sock = get_socket(endpoint);\n if (sock == nullptr) {\n *free = 0;\n *total = 0;\n return;\n }\n get_device_memory(sock, device, free, total);\n}\n\n// RPC server-side implementation\n\nclass rpc_server {\npublic:\n rpc_server(std::vector all_backends, const char * cache_dir)\n : backends(std::move(all_backends)), cache_dir(cache_dir) {\n stored_graphs.resize(backends.size());\n }\n ~rpc_server();\n\n void hello(rpc_msg_hello_rsp & response);\n bool alloc_buffer(const rpc_msg_alloc_buffer_req & request, rpc_msg_alloc_buffer_rsp & response);\n bool get_alignment(const rpc_msg_get_alignment_req & request, rpc_msg_get_alignment_rsp & response);\n bool get_max_size(const rpc_msg_get_max_size_req & request, rpc_msg_get_max_size_rsp & response);\n bool buffer_get_base(const rpc_msg_buffer_get_base_req & request, rpc_msg_buffer_get_base_rsp & response);\n bool free_buffer(const rpc_msg_free_buffer_req & request);\n bool buffer_clear(const rpc_msg_buffer_clear_req & request);\n bool set_tensor(const std::vector & input);\n bool set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response);\n bool get_tensor(const rpc_msg_get_tensor_req & request, std::vector & response);\n bool copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response);\n bool graph_compute(const std::vector & input);\n bool graph_recompute(const rpc_msg_graph_recompute_req & request);\n bool init_tensor(const rpc_msg_init_tensor_req & request);\n bool get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response);\n bool get_device_memory(const rpc_msg_get_device_memory_req & request, rpc_msg_get_device_memory_rsp & response);\n\n struct stored_graph {\n ggml_context_ptr ctx_ptr;\n ggml_cgraph * graph;\n };\n\nprivate:\n bool get_cached_file(uint64_t hash, std::vector & data);\n ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor);\n ggml_tensor * create_node(uint64_t id,\n struct ggml_context * ctx,\n const std::unordered_map & tensor_ptrs,\n std::unordered_map & tensor_map);\n\n\n std::vector backends;\n const char * cache_dir;\n std::unordered_set buffers;\n // store the last computed graph for each backend\n std::vector stored_graphs;\n};\n\nvoid rpc_server::hello(rpc_msg_hello_rsp & response) {\n response.major = RPC_PROTO_MAJOR_VERSION;\n response.minor = RPC_PROTO_MINOR_VERSION;\n response.patch = RPC_PROTO_PATCH_VERSION;\n LOG_DBG(\"[%s] version: %d.%d.%d\\n\", __func__, response.major, response.minor, response.patch);\n}\n\nbool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response) {\n uint32_t dev_id = request.device;\n if (dev_id >= backends.size()) {\n return false;\n }\n ggml_backend_buffer_type_t buft;\n struct ggml_init_params params {\n /*.mem_size =*/ ggml_tensor_overhead()*(1 + GGML_MAX_SRC),\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n\n ggml_context_ptr ctx_ptr { ggml_init(params) };\n GGML_ASSERT(ctx_ptr != nullptr);\n ggml_context * ctx = ctx_ptr.get();\n\n ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);\n if (tensor == nullptr) {\n GGML_LOG_ERROR(\"Null tensor pointer passed to server get_alloc_size function.\\n\");\n return false;\n }\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n if (request.srcs[i].id != 0) {\n tensor->src[i] = deserialize_tensor(ctx, &request.srcs[i]);\n }\n }\n\n LOG_DBG(\"[%s] device: %d, buffer: %p, data: %p\\n\", __func__, dev_id, (void*)tensor->buffer, tensor->data);\n if (tensor->buffer == nullptr) {\n //No buffer allocated.\n buft = ggml_backend_get_default_buffer_type(backends[dev_id]);\n } else {\n buft = tensor->buffer->buft;\n }\n\n response.alloc_size = ggml_backend_buft_get_alloc_size(buft, tensor);\n\n return true;\n}\n\nbool rpc_server::alloc_buffer(const rpc_msg_alloc_buffer_req & request, rpc_msg_alloc_buffer_rsp & response) {\n uint32_t dev_id = request.device;\n if (dev_id >= backends.size()) {\n return false;\n }\n ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]);\n ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, request.size);\n response.remote_ptr = 0;\n response.remote_size = 0;\n if (buffer != nullptr) {\n response.remote_ptr = reinterpret_cast(buffer);\n response.remote_size = buffer->size;\n LOG_DBG(\"[%s] device: %d, size: %\" PRIu64 \" -> remote_ptr: %\" PRIx64 \", remote_size: %\" PRIu64 \"\\n\",\n __func__, dev_id, request.size, response.remote_ptr, response.remote_size);\n buffers.insert(buffer);\n } else {\n LOG_DBG(\"[%s] device: %d, size: %\" PRIu64 \" -> failed\\n\", __func__, dev_id, request.size);\n }\n return true;\n}\n\nbool rpc_server::get_alignment(const rpc_msg_get_alignment_req & request, rpc_msg_get_alignment_rsp & response) {\n uint32_t dev_id = request.device;\n if (dev_id >= backends.size()) {\n return false;\n }\n ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]);\n size_t alignment = ggml_backend_buft_get_alignment(buft);\n LOG_DBG(\"[%s] device: %d, alignment: %lu\\n\", __func__, dev_id, alignment);\n response.alignment = alignment;\n return true;\n}\n\nbool rpc_server::get_max_size(const rpc_msg_get_max_size_req & request, rpc_msg_get_max_size_rsp & response) {\n uint32_t dev_id = request.device;\n if (dev_id >= backends.size()) {\n return false;\n }\n ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]);\n size_t max_size = ggml_backend_buft_get_max_size(buft);\n LOG_DBG(\"[%s] device: %d, max_size: %lu\\n\", __func__, dev_id, max_size);\n response.max_size = max_size;\n return true;\n}\n\nbool rpc_server::buffer_get_base(const rpc_msg_buffer_get_base_req & request, rpc_msg_buffer_get_base_rsp & response) {\n LOG_DBG(\"[%s] remote_ptr: %\" PRIx64 \"\\n\", __func__, request.remote_ptr);\n ggml_backend_buffer_t buffer = reinterpret_cast(request.remote_ptr);\n if (buffers.find(buffer) == buffers.end()) {\n GGML_LOG_ERROR(\"[%s] buffer not found\\n\", __func__);\n return false;\n }\n void * base = ggml_backend_buffer_get_base(buffer);\n response.base_ptr = reinterpret_cast(base);\n return true;\n}\n\nbool rpc_server::free_buffer(const rpc_msg_free_buffer_req & request) {\n LOG_DBG(\"[%s] remote_ptr: %\" PRIx64 \"\\n\", __func__, request.remote_ptr);\n ggml_backend_buffer_t buffer = reinterpret_cast(request.remote_ptr);\n if (buffers.find(buffer) == buffers.end()) {\n GGML_LOG_ERROR(\"[%s] buffer not found\\n\", __func__);\n return false;\n }\n ggml_backend_buffer_free(buffer);\n buffers.erase(buffer);\n return true;\n}\n\nbool rpc_server::buffer_clear(const rpc_msg_buffer_clear_req & request) {\n LOG_DBG(\"[%s] remote_ptr: %\" PRIx64 \", value: %u\\n\", __func__, request.remote_ptr, request.value);\n ggml_backend_buffer_t buffer = reinterpret_cast(request.remote_ptr);\n if (buffers.find(buffer) == buffers.end()) {\n GGML_LOG_ERROR(\"[%s] buffer not found\\n\", __func__);\n return false;\n }\n ggml_backend_buffer_clear(buffer, request.value);\n return true;\n}\n\nggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor) {\n // Validate tensor type before using it\n if (tensor->type >= GGML_TYPE_COUNT) {\n GGML_LOG_ERROR(\"[%s] invalid tensor type received: %u\\n\", __func__, tensor->type);\n return nullptr;\n }\n\n ggml_tensor * result = ggml_new_tensor_4d(ctx, (ggml_type) tensor->type,\n tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);\n\n // ggml_new_tensor_4d might fail if dimensions are invalid, although less likely to crash than invalid type\n if (result == nullptr) {\n GGML_LOG_ERROR(\"[%s] ggml_new_tensor_4d failed for type %u\\\\n\", __func__, tensor->type);\n return nullptr;\n }\n\n for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) {\n result->nb[i] = tensor->nb[i];\n }\n result->buffer = reinterpret_cast(tensor->buffer);\n if (result->buffer && buffers.find(result->buffer) == buffers.end()) {\n result->buffer = nullptr;\n }\n\n if (result->buffer) {\n // require that the tensor data does not go beyond the buffer end\n uint64_t tensor_size = (uint64_t) ggml_nbytes(result);\n uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(result->buffer);\n uint64_t buffer_size = (uint64_t) ggml_backend_buffer_get_size(result->buffer);\n GGML_ASSERT(tensor->data + tensor_size >= tensor->data); // check for overflow\n GGML_ASSERT(tensor->data >= buffer_start && tensor->data + tensor_size <= buffer_start + buffer_size);\n }\n\n result->op = (ggml_op) tensor->op;\n for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) {\n result->op_params[i] = tensor->op_params[i];\n }\n result->flags = tensor->flags;\n result->data = reinterpret_cast(tensor->data);\n ggml_set_name(result, tensor->name);\n return result;\n}\n\n\nbool rpc_server::set_tensor(const std::vector & input) {\n // serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) |\n if (input.size() < sizeof(rpc_tensor) + sizeof(uint64_t)) {\n return false;\n }\n const rpc_tensor * in_tensor = (const rpc_tensor *)input.data();\n uint64_t offset;\n memcpy(&offset, input.data() + sizeof(rpc_tensor), sizeof(offset));\n const size_t size = input.size() - sizeof(rpc_tensor) - sizeof(offset);\n\n struct ggml_init_params params {\n /*.mem_size =*/ ggml_tensor_overhead(),\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n ggml_context_ptr ctx_ptr { ggml_init(params) };\n GGML_ASSERT(ctx_ptr != nullptr);\n ggml_context * ctx = ctx_ptr.get();\n ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor);\n if (tensor == nullptr || tensor->buffer == nullptr) {\n GGML_LOG_ERROR(\"[%s] error deserializing tensor\\n\", __func__);\n return false;\n }\n LOG_DBG(\"[%s] buffer: %p, data: %p, offset: %\" PRIu64 \", size: %zu\\n\", __func__, (void*)tensor->buffer, tensor->data, offset, size);\n\n // sanitize tensor->data\n {\n const size_t p0 = (size_t) ggml_backend_buffer_get_base(tensor->buffer);\n const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer);\n\n if (in_tensor->data + offset < p0 || in_tensor->data + offset >= p1 || size > (p1 - in_tensor->data - offset)) {\n GGML_LOG_ERROR(\"[%s] tensor data region (data=0x%\" PRIx64 \", offset=%\" PRIu64 \", size=%zu) out of buffer bounds [0x%zx, 0x%zx)\\n\",\n __func__, in_tensor->data, offset, size, p0, p1);\n return false;\n }\n }\n\n const void * data = input.data() + sizeof(rpc_tensor) + sizeof(offset);\n if (cache_dir && size > HASH_THRESHOLD) {\n uint64_t hash = fnv_hash((const uint8_t*)data, size);\n char hash_str[17];\n snprintf(hash_str, sizeof(hash_str), \"%016\" PRIx64, hash);\n // save to cache_dir/hash_str\n fs::path cache_file = fs::path(cache_dir) / hash_str;\n std::ofstream ofs(cache_file, std::ios::binary);\n ofs.write((const char *)data, size);\n GGML_LOG_INFO(\"[%s] saved to '%s'\\n\", __func__, cache_file.c_str());\n }\n ggml_backend_tensor_set(tensor, data, offset, size);\n return true;\n}\n\nbool rpc_server::get_cached_file(uint64_t hash, std::vector & data) {\n if (!cache_dir) {\n return false;\n }\n char hash_str[17];\n snprintf(hash_str, sizeof(hash_str), \"%016\" PRIx64, hash);\n fs::path cache_file = fs::path(cache_dir) / hash_str;\n std::error_code ec;\n if (!fs::exists(cache_file, ec)) {\n return false;\n }\n std::ifstream ifs(cache_file, std::ios::binary);\n ifs.seekg(0, std::ios::end);\n size_t size = ifs.tellg();\n ifs.seekg(0, std::ios::beg);\n data.resize(size);\n ifs.read((char *)data.data(), size);\n return true;\n}\n\nbool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response)\n{\n std::vector cached_file;\n if (!get_cached_file(request.hash, cached_file)) {\n response.result = 0;\n return true;\n }\n size_t size = cached_file.size();\n struct ggml_init_params params {\n /*.mem_size =*/ ggml_tensor_overhead(),\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n ggml_context_ptr ctx_ptr { ggml_init(params) };\n GGML_ASSERT(ctx_ptr != nullptr);\n ggml_context * ctx = ctx_ptr.get();\n ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);\n if (tensor == nullptr || tensor->buffer == nullptr) {\n GGML_LOG_ERROR(\"[%s] error deserializing tensor\\n\", __func__);\n return false;\n }\n LOG_DBG(\"[%s] buffer: %p, data: %p, offset: %\" PRIu64 \", size: %zu, hash: %\" PRIx64 \"\\n\",\n __func__, (void*)tensor->buffer, tensor->data, request.offset, size, request.hash);\n\n // sanitize tensor->data\n {\n const size_t p0 = (size_t) ggml_backend_buffer_get_base(tensor->buffer);\n const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer);\n\n if (request.tensor.data + request.offset < p0\n || request.tensor.data + request.offset >= p1\n || size > (p1 - request.tensor.data - request.offset)) {\n GGML_LOG_ERROR(\"[%s] tensor data region (data=0x%\" PRIx64 \", offset=%\" PRIu64 \", size=%zu, hash=0x%\" PRIx64 \") out of buffer bounds [0x%zx, 0x%zx)\\n\",\n __func__, request.tensor.data, request.offset, size, request.hash, p0, p1);\n return false;\n }\n }\n ggml_backend_tensor_set(tensor, cached_file.data(), request.offset, size);\n response.result = 1;\n return true;\n}\n\nbool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) {\n struct ggml_init_params params {\n /*.mem_size =*/ ggml_tensor_overhead(),\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n ggml_context_ptr ctx_ptr { ggml_init(params) };\n GGML_ASSERT(ctx_ptr != nullptr);\n ggml_context * ctx = ctx_ptr.get();\n ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);\n if (tensor == nullptr) {\n GGML_LOG_ERROR(\"Null tensor pointer passed to server init_tensor function.\\n\");\n return false;\n }\n LOG_DBG(\"[%s] buffer: %p, data: %p\\n\", __func__, (void*)tensor->buffer, tensor->data);\n // Call the backend's buffer_init_tensor function\n ggml_backend_buffer_t buffer = tensor->buffer;\n if (buffer && buffer->iface.init_tensor) {\n buffer->iface.init_tensor(buffer, tensor);\n } else {\n GGML_LOG_ERROR(\"Null buffer for tensor passed to init_tensor function\\n\");\n }\n\n if (tensor->extra != nullptr) {\n // This pointer can either be passed around client/server, or probably better stored server-side and kept track of.\n // Currently unimplemented.\n GGML_LOG_ERROR(\"tensor->extra populated by the backend, this is currently unsupported.\\n\");\n return false;\n }\n\n return true;\n}\n\nbool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector & response) {\n struct ggml_init_params params {\n /*.mem_size =*/ ggml_tensor_overhead(),\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n ggml_context_ptr ctx_ptr { ggml_init(params) };\n GGML_ASSERT(ctx_ptr != nullptr);\n ggml_context * ctx = ctx_ptr.get();\n ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);\n if (tensor == nullptr || tensor->buffer == nullptr) {\n GGML_LOG_ERROR(\"[%s] error deserializing tensor\\n\", __func__);\n return false;\n }\n LOG_DBG(\"[%s] buffer: %p, data: %p, offset: %\" PRIu64 \", size: %\" PRIu64 \"\\n\", __func__, (void*)tensor->buffer, tensor->data, request.offset, request.size);\n\n // sanitize tensor->data\n {\n const size_t p0 = (size_t) ggml_backend_buffer_get_base(tensor->buffer);\n const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer);\n\n if (request.tensor.data + request.offset < p0 ||\n request.tensor.data + request.offset >= p1 ||\n request.size > (p1 - request.tensor.data - request.offset)) {\n GGML_LOG_ERROR(\"[%s] requested tensor region (data=0x%\" PRIx64 \", offset=%\" PRIu64 \", size=%\" PRIu64 \") out of buffer bounds [0x%zx, 0x%zx)\\n\",\n __func__, request.tensor.data, request.offset, request.size, p0, p1);\n return false;\n }\n }\n\n response.resize(request.size, 0);\n ggml_backend_tensor_get(tensor, response.data(), request.offset, request.size);\n return true;\n}\n\nbool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response) {\n struct ggml_init_params params {\n /*.mem_size =*/ 2*ggml_tensor_overhead(),\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n ggml_context_ptr ctx_ptr { ggml_init(params) };\n GGML_ASSERT(ctx_ptr != nullptr);\n ggml_context * ctx = ctx_ptr.get();\n\n ggml_tensor * src = deserialize_tensor(ctx, &request.src);\n ggml_tensor * dst = deserialize_tensor(ctx, &request.dst);\n if (src == nullptr || dst == nullptr || src->buffer == nullptr || dst->buffer == nullptr) {\n GGML_LOG_ERROR(\"[%s] error deserializing tensors\\n\", __func__);\n return false;\n }\n\n uint64_t src_size = (uint64_t) ggml_nbytes(src);\n uint64_t dst_data = (uint64_t) dst->data;\n uint64_t dst_base = (uint64_t) ggml_backend_buffer_get_base(dst->buffer);\n uint64_t dst_buf_sz = (uint64_t) ggml_backend_buffer_get_size(dst->buffer);\n\n if (dst_data + src_size > dst_base + dst_buf_sz) {\n GGML_LOG_ERROR(\"[%s] out-of-bounds write in rpc_server::copy_tensor:\\n\"\n \" write range : [0x%\" PRIx64 \", 0x%\" PRIx64 \"]\\n\"\n \" buffer base: [0x%\" PRIx64 \", 0x%\" PRIx64 \"]\\n\",\n __func__,\n dst_data,\n dst_data + src_size,\n dst_base,\n dst_base + dst_buf_sz);\n return false;\n }\n\n LOG_DBG(\"[%s] src->buffer: %p, dst->buffer: %p\\n\",\n __func__, (void*) src->buffer, (void*) dst->buffer);\n\n response.result = ggml_backend_buffer_copy_tensor(src, dst);\n return true;\n}\n\nggml_tensor * rpc_server::create_node(uint64_t id,\n struct ggml_context * ctx,\n const std::unordered_map & tensor_ptrs,\n std::unordered_map & tensor_map) {\n if (tensor_map.find(id) != tensor_map.end()) {\n return tensor_map[id];\n }\n // Safely find the tensor pointer\n auto it_ptr = tensor_ptrs.find(id);\n if (it_ptr == tensor_ptrs.end()) {\n return nullptr;\n }\n const rpc_tensor * tensor = it_ptr->second;\n\n struct ggml_tensor * result = deserialize_tensor(ctx, tensor);\n if (result == nullptr) {\n return nullptr;\n }\n tensor_map[id] = result;\n for (int i = 0; i < GGML_MAX_SRC; i++) {\n // Check if the source ID is 0 before calling create_node recursively\n if (tensor->src[i] == 0) {\n result->src[i] = nullptr;\n } else {\n result->src[i] = create_node(tensor->src[i], ctx, tensor_ptrs, tensor_map);\n // If the recursive call failed for a non-zero ID, propagate the error\n if (result->src[i] == nullptr) {\n GGML_LOG_ERROR(\"[%s] failed to create source node %d (src_id=%\" PRIu64 \") for node id %\" PRIu64 \"\\n\",\n __func__, i, tensor->src[i], id);\n // Must return nullptr to signal failure up the call stack\n return nullptr;\n }\n }\n }\n\n // Handle view_src similarly\n if (tensor->view_src == 0) {\n result->view_src = nullptr;\n } else {\n result->view_src = create_node(tensor->view_src, ctx, tensor_ptrs, tensor_map);\n // If the recursive call failed for a non-zero ID, propagate the error\n if (result->view_src == nullptr) {\n GGML_LOG_ERROR(\"[%s] failed to create view_src node (view_src_id=%\" PRIu64 \") for node id %\" PRIu64 \"\\n\",\n __func__, tensor->view_src, id);\n // Must return nullptr to signal failure up the call stack\n return nullptr;\n }\n }\n result->view_offs = tensor->view_offs;\n return result;\n}\n\nbool rpc_server::graph_compute(const std::vector & input) {\n // serialization format:\n // | device (4 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) |\n if (input.size() < 2*sizeof(uint32_t)) {\n return false;\n }\n const uint8_t * src = input.data();\n uint32_t device;\n memcpy(&device, src, sizeof(device));\n src += sizeof(device);\n if (device >= backends.size()) {\n return false;\n }\n uint32_t n_nodes;\n memcpy(&n_nodes, src, sizeof(n_nodes));\n src += sizeof(n_nodes);\n if (input.size() < 2*sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t)) {\n return false;\n }\n const uint64_t * nodes = (const uint64_t *)src;\n src += n_nodes*sizeof(uint64_t);\n uint32_t n_tensors;\n memcpy(&n_tensors, src, sizeof(n_tensors));\n src += sizeof(n_tensors);\n if (input.size() < 2*sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t) + n_tensors*sizeof(rpc_tensor)) {\n return false;\n }\n const rpc_tensor * tensors = (const rpc_tensor *)src;\n LOG_DBG(\"[%s] device: %u, n_nodes: %u, n_tensors: %u\\n\", __func__, device, n_nodes, n_tensors);\n\n size_t buf_size = ggml_tensor_overhead()*(n_nodes + n_tensors) + ggml_graph_overhead_custom(n_nodes, false);\n\n struct ggml_init_params params = {\n /*.mem_size =*/ buf_size,\n /*.mem_buffer =*/ NULL,\n /*.no_alloc =*/ true,\n };\n ggml_context_ptr ctx_ptr { ggml_init(params) };\n GGML_ASSERT(ctx_ptr != nullptr);\n ggml_context * ctx = ctx_ptr.get();\n struct ggml_cgraph * graph = ggml_new_graph_custom(ctx, n_nodes, false);\n graph->n_nodes = n_nodes;\n std::unordered_map tensor_ptrs;\n tensor_ptrs.reserve(n_tensors);\n for (uint32_t i = 0; i < n_tensors; i++) {\n tensor_ptrs.emplace(tensors[i].id, &tensors[i]);\n }\n std::unordered_map tensor_map;\n tensor_map.reserve(n_nodes);\n for (uint32_t i = 0; i < n_nodes; i++) {\n int64_t id;\n memcpy(&id, &nodes[i], sizeof(id));\n graph->nodes[i] = create_node(id, ctx, tensor_ptrs, tensor_map);\n\n // Check if create_node failed for a *non-zero* ID.\n // If id was 0, create_node returning nullptr is expected.\n // If id was non-zero and create_node returned nullptr, it indicates a deserialization error.\n if (graph->nodes[i] == nullptr && id != 0) {\n GGML_LOG_ERROR(\"[%s] failed to create graph node %d (id=%\" PRId64 \")\\n\", __func__, i, id);\n return false;\n }\n }\n ggml_status status = ggml_backend_graph_compute(backends[device], graph);\n GGML_ASSERT(status == GGML_STATUS_SUCCESS && \"Unsuccessful graph computations are not supported with RPC\");\n stored_graphs[device].ctx_ptr.swap(ctx_ptr);\n stored_graphs[device].graph = graph;\n return true;\n}\n\nbool rpc_server::graph_recompute(const rpc_msg_graph_recompute_req & request) {\n uint32_t device = request.device;\n if (device >= backends.size()) {\n return false;\n }\n if (stored_graphs[device].graph == nullptr) {\n return false;\n }\n ggml_cgraph * graph = stored_graphs[device].graph;\n LOG_DBG(\"[%s] device: %u\\n\", __func__, device);\n ggml_status status = ggml_backend_graph_compute(backends[device], graph);\n GGML_ASSERT(status == GGML_STATUS_SUCCESS && \"Unsuccessful graph computations are not supported with RPC\");\n return true;\n}\n\nbool rpc_server::get_device_memory(const rpc_msg_get_device_memory_req & request, rpc_msg_get_device_memory_rsp & response) {\n uint32_t dev_id = request.device;\n if (dev_id >= backends.size()) {\n return false;\n }\n size_t free, total;\n ggml_backend_dev_t dev = ggml_backend_get_device(backends[dev_id]);\n ggml_backend_dev_memory(dev, &free, &total);\n response.free_mem = free;\n response.total_mem = total;\n LOG_DBG(\"[%s] device: %u, free_mem: %\" PRIu64 \", total_mem: %\" PRIu64 \"\\n\", __func__, dev_id, response.free_mem, response.total_mem);\n return true;\n}\n\nrpc_server::~rpc_server() {\n for (auto buffer : buffers) {\n ggml_backend_buffer_free(buffer);\n }\n}\n\nstatic void rpc_serve_client(const std::vector & backends, const char * cache_dir,\n sockfd_t sockfd) {\n rpc_server server(backends, cache_dir);\n uint8_t cmd;\n if (!recv_data(sockfd, &cmd, 1)) {\n return;\n }\n // the first command sent by the client must be HELLO\n if (cmd != RPC_CMD_HELLO) {\n GGML_LOG_ERROR(\"Expected HELLO command, update client\\n\");\n return;\n }\n if (!recv_msg(sockfd, nullptr, 0)) {\n return;\n }\n rpc_msg_hello_rsp response;\n server.hello(response);\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n while (true) {\n if (!recv_data(sockfd, &cmd, 1)) {\n break;\n }\n if (cmd >= RPC_CMD_COUNT) {\n // fail fast if the command is invalid\n GGML_LOG_ERROR(\"Unknown command: %d\\n\", cmd);\n break;\n }\n switch (cmd) {\n case RPC_CMD_HELLO: {\n // HELLO command is handled above\n return;\n }\n case RPC_CMD_DEVICE_COUNT: {\n if (!recv_msg(sockfd, nullptr, 0)) {\n return;\n }\n rpc_msg_device_count_rsp response;\n response.device_count = backends.size();\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n break;\n }\n case RPC_CMD_ALLOC_BUFFER: {\n rpc_msg_alloc_buffer_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n rpc_msg_alloc_buffer_rsp response;\n if (!server.alloc_buffer(request, response)) {\n return;\n }\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n break;\n }\n case RPC_CMD_GET_ALLOC_SIZE: {\n rpc_msg_get_alloc_size_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n rpc_msg_get_alloc_size_rsp response;\n if (!server.get_alloc_size(request, response)) {\n return;\n }\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n break;\n }\n case RPC_CMD_GET_ALIGNMENT: {\n rpc_msg_get_alignment_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n rpc_msg_get_alignment_rsp response;\n if (!server.get_alignment(request, response)) {\n return;\n }\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n break;\n }\n case RPC_CMD_GET_MAX_SIZE: {\n rpc_msg_get_max_size_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n rpc_msg_get_max_size_rsp response;\n if (!server.get_max_size(request, response)) {\n return;\n }\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n break;\n }\n case RPC_CMD_BUFFER_GET_BASE: {\n rpc_msg_buffer_get_base_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n rpc_msg_buffer_get_base_rsp response;\n if (!server.buffer_get_base(request, response)) {\n return;\n }\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n break;\n }\n case RPC_CMD_FREE_BUFFER: {\n rpc_msg_free_buffer_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n if (!server.free_buffer(request)) {\n return;\n }\n if (!send_msg(sockfd, nullptr, 0)) {\n return;\n }\n break;\n }\n case RPC_CMD_BUFFER_CLEAR: {\n rpc_msg_buffer_clear_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n if (!server.buffer_clear(request)) {\n return;\n }\n if (!send_msg(sockfd, nullptr, 0)) {\n return;\n }\n break;\n }\n case RPC_CMD_SET_TENSOR: {\n std::vector input;\n if (!recv_msg(sockfd, input)) {\n return;\n }\n if (!server.set_tensor(input)) {\n return;\n }\n break;\n }\n case RPC_CMD_SET_TENSOR_HASH: {\n rpc_msg_set_tensor_hash_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n rpc_msg_set_tensor_hash_rsp response;\n if (!server.set_tensor_hash(request, response)) {\n return;\n }\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n break;\n }\n case RPC_CMD_INIT_TENSOR: {\n rpc_msg_init_tensor_req request;\n if (!recv_msg(sockfd, &request,sizeof(request))) {\n return;\n }\n if (!server.init_tensor(request)) {\n return;\n }\n if (!send_msg(sockfd, nullptr, 0)) {\n return;\n }\n break;\n }\n case RPC_CMD_GET_TENSOR: {\n rpc_msg_get_tensor_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n std::vector response;\n if (!server.get_tensor(request, response)) {\n return;\n }\n if (!send_msg(sockfd, response.data(), response.size())) {\n return;\n }\n break;\n }\n case RPC_CMD_COPY_TENSOR: {\n rpc_msg_copy_tensor_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n rpc_msg_copy_tensor_rsp response;\n if (!server.copy_tensor(request, response)) {\n return;\n }\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n break;\n }\n case RPC_CMD_GRAPH_COMPUTE: {\n std::vector input;\n if (!recv_msg(sockfd, input)) {\n return;\n }\n if (!server.graph_compute(input)) {\n return;\n }\n break;\n }\n case RPC_CMD_GRAPH_RECOMPUTE: {\n rpc_msg_graph_recompute_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n if (!server.graph_recompute(request)) {\n return;\n }\n break;\n }\n case RPC_CMD_GET_DEVICE_MEMORY: {\n rpc_msg_get_device_memory_req request;\n if (!recv_msg(sockfd, &request, sizeof(request))) {\n return;\n }\n rpc_msg_get_device_memory_rsp response;\n if (!server.get_device_memory(request, response)) {\n return;\n }\n if (!send_msg(sockfd, &response, sizeof(response))) {\n return;\n }\n break;\n }\n default: {\n GGML_LOG_ERROR(\"Unknown command: %d\\n\", cmd);\n return;\n }\n }\n }\n}\n\nvoid ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir,\n size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices) {\n if (n_devices == 0 || devices == nullptr) {\n fprintf(stderr, \"Invalid arguments to ggml_backend_rpc_start_server\\n\");\n return;\n }\n std::vector backends;\n printf(\"Starting RPC server v%d.%d.%d\\n\",\n RPC_PROTO_MAJOR_VERSION,\n RPC_PROTO_MINOR_VERSION,\n RPC_PROTO_PATCH_VERSION);\n printf(\" endpoint : %s\\n\", endpoint);\n printf(\" local cache : %s\\n\", cache_dir ? cache_dir : \"n/a\");\n printf(\"Devices:\\n\");\n for (size_t i = 0; i < n_devices; i++) {\n auto dev = devices[i];\n size_t free, total;\n ggml_backend_dev_memory(dev, &free, &total);\n printf(\" %s: %s (%zu MiB, %zu MiB free)\\n\", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),\n total / 1024 / 1024, free / 1024 / 1024);\n auto backend = ggml_backend_dev_init(dev, nullptr);\n if (!backend) {\n fprintf(stderr, \"Failed to create backend for device %s\\n\", dev->iface.get_name(dev));\n return;\n }\n backends.push_back(backend);\n ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;\n if (reg) {\n auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, \"ggml_backend_set_n_threads\");\n if (ggml_backend_set_n_threads_fn) {\n ggml_backend_set_n_threads_fn(backend, n_threads);\n }\n }\n }\n\n std::string host;\n int port;\n if (!parse_endpoint(endpoint, host, port)) {\n return;\n }\n#ifdef _WIN32\n {\n WSADATA wsaData;\n int res = WSAStartup(MAKEWORD(2, 2), &wsaData);\n if (res != 0) {\n fprintf(stderr, \"WSAStartup failed: %d\\n\", res);\n return;\n }\n }\n#endif\n auto server_socket = create_server_socket(host.c_str(), port);\n if (server_socket == nullptr) {\n fprintf(stderr, \"Failed to create server socket\\n\");\n return;\n }\n while (true) {\n auto client_socket = socket_accept(server_socket->fd);\n if (client_socket == nullptr) {\n fprintf(stderr, \"Failed to accept client connection\\n\");\n return;\n }\n printf(\"Accepted client connection\\n\");\n fflush(stdout);\n rpc_serve_client(backends, cache_dir, client_socket->fd);\n printf(\"Client connection closed\\n\");\n fflush(stdout);\n }\n#ifdef _WIN32\n WSACleanup();\n#endif\n for (auto backend : backends) {\n ggml_backend_free(backend);\n }\n}\n\n// device interface\n\nstruct ggml_backend_rpc_device_context {\n std::string endpoint;\n uint32_t device;\n std::string name;\n std::string description;\n};\n\nstatic const char * ggml_backend_rpc_device_get_name(ggml_backend_dev_t dev) {\n ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;\n\n return ctx->name.c_str();\n}\n\nstatic const char * ggml_backend_rpc_device_get_description(ggml_backend_dev_t dev) {\n ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;\n\n return ctx->description.c_str();\n}\n\nstatic void ggml_backend_rpc_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;\n\n ggml_backend_rpc_get_device_memory(ctx->endpoint.c_str(), ctx->device, free, total);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_rpc_device_get_type(ggml_backend_dev_t dev) {\n // TODO: obtain value from the server\n return GGML_BACKEND_DEVICE_TYPE_GPU;\n\n GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_rpc_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {\n props->name = ggml_backend_rpc_device_get_name(dev);\n props->description = ggml_backend_rpc_device_get_description(dev);\n props->type = ggml_backend_rpc_device_get_type(dev);\n ggml_backend_rpc_device_get_memory(dev, &props->memory_free, &props->memory_total);\n props->caps = {\n /* .async = */ false,\n /* .host_buffer = */ false,\n /* .buffer_from_host_ptr = */ false,\n /* .events = */ false,\n };\n}\n\nstatic ggml_backend_t ggml_backend_rpc_device_init(ggml_backend_dev_t dev, const char * params) {\n ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;\n\n return ggml_backend_rpc_init(ctx->endpoint.c_str(), ctx->device);\n\n GGML_UNUSED(params);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_rpc_device_get_buffer_type(ggml_backend_dev_t dev) {\n ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;\n\n return ggml_backend_rpc_buffer_type(ctx->endpoint.c_str(), ctx->device);\n\n GGML_UNUSED(dev);\n}\n\nstatic bool ggml_backend_rpc_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {\n GGML_UNUSED(dev);\n GGML_UNUSED(op);\n //TODO: call the remote backend and cache the results\n return true;\n}\n\nstatic bool ggml_backend_rpc_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n if (!buft || buft->iface.get_name != ggml_backend_rpc_buffer_type_name) {\n return false;\n }\n ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;\n ggml_backend_rpc_device_context * dev_ctx = (ggml_backend_rpc_device_context *)dev->context;\n return buft_ctx->endpoint == dev_ctx->endpoint && buft_ctx->device == dev_ctx->device;\n}\n\nstatic const struct ggml_backend_device_i ggml_backend_rpc_device_i = {\n /* .get_name = */ ggml_backend_rpc_device_get_name,\n /* .get_description = */ ggml_backend_rpc_device_get_description,\n /* .get_memory = */ ggml_backend_rpc_device_get_memory,\n /* .get_type = */ ggml_backend_rpc_device_get_type,\n /* .get_props = */ ggml_backend_rpc_device_get_props,\n /* .init_backend = */ ggml_backend_rpc_device_init,\n /* .get_buffer_type = */ ggml_backend_rpc_device_get_buffer_type,\n /* .get_host_buffer_type = */ NULL,\n /* .buffer_from_host_ptr = */ NULL,\n /* .supports_op = */ ggml_backend_rpc_device_supports_op,\n /* .supports_buft = */ ggml_backend_rpc_device_supports_buft,\n /* .offload_op = */ NULL,\n /* .event_new = */ NULL,\n /* .event_free = */ NULL,\n /* .event_synchronize = */ NULL,\n};\n\n// backend reg interface\n\nstruct ggml_backend_rpc_reg_context {\n std::string name;\n std::vector devices;\n};\n\nstatic const char * ggml_backend_rpc_reg_get_name(ggml_backend_reg_t reg) {\n ggml_backend_rpc_reg_context * ctx = (ggml_backend_rpc_reg_context *)reg->context;\n return ctx ? ctx->name.c_str() : \"RPC\";\n}\n\nstatic size_t ggml_backend_rpc_reg_get_device_count(ggml_backend_reg_t reg) {\n ggml_backend_rpc_reg_context * ctx = (ggml_backend_rpc_reg_context *)reg->context;\n return ctx ? ctx->devices.size() : 0;\n}\n\nstatic ggml_backend_dev_t ggml_backend_rpc_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n ggml_backend_rpc_reg_context * ctx = (ggml_backend_rpc_reg_context *)reg->context;\n if (ctx == nullptr) {\n GGML_ABORT(\"The RPC backend does not have enumerated devices - use ggml_backend_rpc_add_server instead\");\n } else {\n GGML_ASSERT(index < ctx->devices.size());\n return ctx->devices[index];\n }\n}\n\nstatic void * ggml_backend_rpc_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n if (std::strcmp(name, \"ggml_backend_rpc_add_server\") == 0) {\n return (void *)ggml_backend_rpc_add_server;\n }\n if (std::strcmp(name, \"ggml_backend_rpc_start_server\") == 0) {\n return (void *)ggml_backend_rpc_start_server;\n }\n return NULL;\n\n GGML_UNUSED(reg);\n}\n\nstatic const struct ggml_backend_reg_i ggml_backend_rpc_reg_i = {\n /* .get_name = */ ggml_backend_rpc_reg_get_name,\n /* .get_device_count = */ ggml_backend_rpc_reg_get_device_count,\n /* .get_device = */ ggml_backend_rpc_reg_get_device,\n /* .get_proc_address = */ ggml_backend_rpc_get_proc_address,\n};\n\nggml_backend_reg_t ggml_backend_rpc_reg(void) {\n static struct ggml_backend_reg ggml_backend_rpc_reg = {\n /* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_rpc_reg_i,\n /* .context = */ NULL,\n };\n\n return &ggml_backend_rpc_reg;\n}\n\nstatic uint32_t ggml_backend_rpc_get_device_count(const char * endpoint) {\n auto sock = get_socket(endpoint);\n if (sock == nullptr) {\n GGML_LOG_ERROR(\"Failed to connect to %s\\n\", endpoint);\n return 0;\n }\n rpc_msg_device_count_rsp response;\n bool status = send_rpc_cmd(sock, RPC_CMD_DEVICE_COUNT, nullptr, 0, &response, sizeof(response));\n RPC_STATUS_ASSERT(status);\n return response.device_count;\n}\n\nstatic const ggml_backend_reg_i ggml_backend_rpc_reg_interface = {\n /* .get_name = */ ggml_backend_rpc_reg_get_name,\n /* .get_device_count = */ ggml_backend_rpc_reg_get_device_count,\n /* .get_device = */ ggml_backend_rpc_reg_get_device,\n /* .get_proc_address = */ ggml_backend_rpc_get_proc_address,\n};\n\nggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint) {\n static std::unordered_map reg_map;\n static std::mutex mutex;\n static uint32_t dev_id = 0;\n std::lock_guard lock(mutex);\n if (reg_map.find(endpoint) != reg_map.end()) {\n return reg_map[endpoint];\n }\n uint32_t dev_count = ggml_backend_rpc_get_device_count(endpoint);\n if (dev_count == 0) {\n return nullptr;\n }\n ggml_backend_rpc_reg_context * ctx = new ggml_backend_rpc_reg_context;\n ctx->name = \"RPC[\" + std::string(endpoint) + \"]\";\n for (uint32_t ind = 0; ind < dev_count; ind++) {\n std::string dev_name = \"RPC\" + std::to_string(dev_id);\n std::string dev_desc = std::string(endpoint);\n ggml_backend_rpc_device_context * dev_ctx = new ggml_backend_rpc_device_context {\n /* .endpoint = */ endpoint,\n /* .device = */ ind,\n /* .name = */ dev_name,\n /* .description = */ dev_desc\n };\n\n ggml_backend_dev_t dev = new ggml_backend_device {\n /* .iface = */ ggml_backend_rpc_device_i,\n /* .reg = */ ggml_backend_rpc_reg(),\n /* .context = */ dev_ctx,\n };\n ctx->devices.push_back(dev);\n dev_id++;\n }\n ggml_backend_reg_t reg = new ggml_backend_reg {\n /* .api_version = */ GGML_BACKEND_API_VERSION,\n /* .iface = */ ggml_backend_rpc_reg_interface,\n /* .context = */ ctx\n };\n reg_map[endpoint] = reg;\n return reg;\n}\n\n\nGGML_BACKEND_DL_IMPL(ggml_backend_rpc_reg)\n" }, { "path": "src/ggml-sycl/CMakeLists.txt", "content": "message(STATUS \"GGML_SYCL_TARGET=${GGML_SYCL_TARGET}\")\n\nif (NOT GGML_SYCL_TARGET MATCHES \"^(INTEL)$\")\n message(FATAL_ERROR \"GGML_SYCL_TARGET: Invalid target, the supported options are [INTEL]\")\nendif()\n\ncheck_cxx_compiler_flag(\"-fsycl\" SUPPORTS_SYCL)\n\nif (DEFINED ENV{ONEAPI_ROOT})\n message(STATUS \"Using oneAPI Release SYCL compiler (icpx).\")\nelseif(SUPPORTS_SYCL)\n message(WARNING \"Using open-source SYCL compiler (clang++). Didn't detect ENV {ONEAPI_ROOT}.\n If you expected the oneAPI Release compiler, please install oneAPI & source it, like:\n source /opt/intel/oneapi/setvars.sh\")\nelse()\n message(FATAL_ERROR \"C++ compiler lacks SYCL support.\")\nendif()\nmessage(STATUS \"SYCL found\")\n#todo: AOT\n\nggml_add_backend_library(ggml-sycl\n ggml-sycl.cpp\n ../../include/ggml-sycl.h\n )\n\nfile(GLOB GGML_HEADERS_SYCL \"*.hpp\")\nfile(GLOB GGML_SOURCES_SYCL \"*.cpp\")\nfile(GLOB SRCS \"template-instances/fattn-tile*.cpp\")\nlist(APPEND GGML_SOURCES_SYCL ${SRCS})\nfile(GLOB SRCS \"template-instances/fattn-vec*.cpp\")\nlist(APPEND GGML_SOURCES_SYCL ${SRCS})\n\ntarget_sources(ggml-sycl PRIVATE ${GGML_HEADERS_SYCL} ${GGML_SOURCES_SYCL})\n\nif (WIN32)\n # To generate a Visual Studio solution, using Intel C++ Compiler for ggml-sycl is mandatory\n if( ${CMAKE_GENERATOR} MATCHES \"Visual Studio\" AND NOT (${CMAKE_GENERATOR_TOOLSET} MATCHES \"Intel C\"))\n set_target_properties(ggml-sycl PROPERTIES VS_PLATFORM_TOOLSET \"Intel C++ Compiler 2025\")\n set(CMAKE_CXX_COMPILER \"icx\")\n set(CMAKE_CXX_COMPILER_ID \"IntelLLVM\")\n endif()\nendif()\n\nmacro(detect_and_find_package package_name)\n set(test_source \"\n cmake_minimum_required(VERSION ${CMAKE_VERSION})\n project(check_package LANGUAGES CXX)\n find_package(${package_name} QUIET)\n \")\n\n set(test_dir \"${CMAKE_CURRENT_BINARY_DIR}/check_package_${package_name}\")\n file(WRITE \"${test_dir}/CMakeLists.txt\" \"${test_source}\")\n\n set(cmake_args \"\")\n if(CMAKE_GENERATOR)\n list(APPEND cmake_args \"-G\" \"${CMAKE_GENERATOR}\")\n endif()\n if(CMAKE_GENERATOR_PLATFORM)\n list(APPEND cmake_args \"-A\" \"${CMAKE_GENERATOR_PLATFORM}\")\n endif()\n if(CMAKE_GENERATOR_TOOLSET)\n list(APPEND cmake_args \"-T\" \"${CMAKE_GENERATOR_TOOLSET}\")\n endif()\n if(CMAKE_CXX_COMPILER)\n list(APPEND cmake_args \"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}\")\n endif()\n\n execute_process(\n COMMAND ${CMAKE_COMMAND} ${cmake_args} .\n WORKING_DIRECTORY \"${test_dir}\"\n RESULT_VARIABLE result\n OUTPUT_QUIET\n ERROR_QUIET\n )\n\n if(result EQUAL 0)\n find_package(${package_name} ${ARGN})\n else()\n message(WARNING \"Detection of ${package_name} failed. The package might be broken or incompatible.\")\n set(${package_name}_FOUND FALSE)\n endif()\nendmacro()\n\ndetect_and_find_package(IntelSYCL)\nif (IntelSYCL_FOUND)\n # Use oneAPI CMake when possible\n target_link_libraries(ggml-sycl PRIVATE IntelSYCL::SYCL_CXX)\nelse()\n # Fallback to the simplest way of enabling SYCL when using intel/llvm nightly for instance\n target_compile_options(ggml-sycl PRIVATE \"-fsycl\")\n target_link_options(ggml-sycl PRIVATE \"-fsycl\")\nendif()\n\ntarget_compile_options(ggml-sycl PRIVATE \"-Wno-narrowing\")\n\n# Link against oneDNN\nset(GGML_SYCL_DNNL 0)\nif(GGML_SYCL_DNN)\n find_package(DNNL)\n if(DNNL_FOUND)\n if (NOT DEFINED DNNL_GPU_VENDOR)\n # default to intel target\n set(DNNL_GPU_VENDOR \"INTEL\")\n if(NOT \"${GGML_SYCL_TARGET}\" STREQUAL \"INTEL\")\n message(WARNING \"oneDNN builds bundled with oneapi release only support INTEL target\")\n endif()\n endif()\n\n # Verify oneDNN was compiled for the same target as llama\n if(\"${GGML_SYCL_TARGET}\" STREQUAL \"${DNNL_GPU_VENDOR}\")\n target_link_libraries(ggml-sycl PRIVATE DNNL::dnnl)\n set(GGML_SYCL_DNNL 1)\n get_target_property(CONFIGS DNNL::dnnl IMPORTED_CONFIGURATIONS)\n foreach(CONFIG ${CONFIGS})\n get_target_property(DNNL_LIB DNNL::dnnl IMPORTED_LOCATION_${CONFIG})\n message(STATUS \"Found oneDNN: ${DNNL_LIB}\")\n endforeach()\n else()\n message(WARNING\n \"oneDNN must be compiled for the same target as llama.cpp.\n llama.cpp: ${GGML_SYCL_TARGET}, oneDNN: ${DNNL_GPU_VENDOR}.\n Disabling oneDNN support.\")\n endif()\n else()\n message(STATUS \"oneDNN not found, disabling oneDNN support\")\n endif()\nelse()\n message(STATUS \"oneDNN support disabled by the user\")\nendif()\ntarget_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_DNNL=${GGML_SYCL_DNNL})\n\nif (GGML_SYCL_F16)\n add_compile_definitions(GGML_SYCL_F16)\nendif()\n\nif (GGML_SYCL_TARGET STREQUAL \"INTEL\")\n add_compile_definitions(GGML_SYCL_WARP_SIZE=16)\n target_link_options(ggml-sycl PRIVATE -Xs -ze-intel-greater-than-4GB-buffer-required)\n\n # Link against Intel oneMKL\n if (CMAKE_CXX_COMPILER_ID STREQUAL \"Clang\")\n set(SYCL_COMPILER ON)\n endif()\n find_package(MKL REQUIRED)\n target_link_libraries(ggml-sycl PRIVATE MKL::MKL_SYCL::BLAS)\nelse()\n # default for other target\n message(FATAL_ERROR \"GGML_SYCL_TARGET is not supported\")\n add_compile_definitions(GGML_SYCL_WARP_SIZE=32)\nendif()\n\nif (GGML_SYCL_GRAPH)\n message(STATUS \"find GGML_SYCL_GRAPH\")\n target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_GRAPH)\nendif()\n\nif (GGML_SYCL_DEVICE_ARCH)\n target_compile_options(ggml-sycl PRIVATE -Xsycl-target-backend --offload-arch=${GGML_SYCL_DEVICE_ARCH})\n target_link_options(ggml-sycl PRIVATE -Xsycl-target-backend --offload-arch=${GGML_SYCL_DEVICE_ARCH})\nendif()\n\n" }, { "path": "src/ggml-sycl/add-id.cpp", "content": "#include \n#include \"common.hpp\"\n#include \"add-id.hpp\"\n\nstatic void add_id_kernel(\n const float* src0,\n const float* src1,\n const int32_t* src2,\n float* dst,\n int64_t ne0,\n int64_t ne1,\n size_t nb01,\n size_t nb02,\n size_t nb11,\n size_t nb21,\n sycl::nd_item<3> item_ct1) {\n const int64_t i1 = item_ct1.get_group(2);\n const int64_t i2 = item_ct1.get_group(1);\n\n const int i11 =\n *(const int32_t*)((const char*)src2 + i1 * sizeof(int32_t) + i2 * nb21);\n\n const size_t nb1 = ne0 * sizeof(float);\n const size_t nb2 = ne1 * nb1;\n\n float* dst_row = (float*)((char*)dst + i1 * nb1 + i2 * nb2);\n const float* src0_row =\n (const float*)((const char*)src0 + i1 * nb01 + i2 * nb02);\n const float* src1_row = (const float*)((const char*)src1 + i11 * nb11);\n\n for (int64_t i0 = item_ct1.get_local_id(2); i0 < ne0;\n i0 += item_ct1.get_local_range(2)) {\n dst_row[i0] = src0_row[i0] + src1_row[i0];\n }\n}\n\nvoid ggml_sycl_add_id(ggml_backend_sycl_context& ctx, ggml_tensor* dst) {\n const ggml_tensor* src0 = dst->src[0];\n const ggml_tensor* src1 = dst->src[1];\n const ggml_tensor* src2 = dst->src[2];\n\n GGML_TENSOR_TERNARY_OP_LOCALS\n\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT(src2->type == GGML_TYPE_I32);\n\n GGML_ASSERT(nb00 == sizeof(float));\n GGML_ASSERT(nb10 == sizeof(float));\n GGML_ASSERT(nb20 == sizeof(int32_t));\n\n const float* src0_d = (const float*)src0->data;\n const float* src1_d = (const float*)src1->data;\n const int32_t* src2_d = (const int32_t*)src2->data;\n float* dst_d = (float*)dst->data;\n\n const unsigned int max_work_group_size = ggml_sycl_info().max_work_group_sizes[ctx.device];\n assert(work_group_size % (WARP_SIZE * WARP_SIZE) == 0);\n\n int threads = std::min((unsigned int)ne00, max_work_group_size); // cols\n\n ctx.stream()->parallel_for(\n sycl::nd_range<3>(\n sycl::range<3>(1, ne02, ne01) * sycl::range<3>(1, 1, threads),\n sycl::range<3>(1, 1, threads)),\n [=](sycl::nd_item<3> item_ct1) {\n add_id_kernel(\n src0_d,\n src1_d,\n src2_d,\n dst_d,\n ne0,\n ne1,\n nb01,\n nb02,\n nb11,\n nb21,\n item_ct1);\n });\n}\n" }, { "path": "src/ggml-sycl/add-id.hpp", "content": "#ifndef GGML_SYCL_ADD_ID_HPP\n#define GGML_SYCL_ADD_ID_HPP\n\n#include \"common.hpp\"\n\nvoid ggml_sycl_add_id(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\n#endif // GGML_SYCL_ADD_ID_HPP\n" }, { "path": "src/ggml-sycl/backend.hpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_BACKEND_HPP\n#define GGML_SYCL_BACKEND_HPP\n\n#include \"binbcast.hpp\"\n#include \"common.hpp\"\n#include \"concat.hpp\"\n#include \"conv.hpp\"\n#include \"convert.hpp\"\n#include \"count-equal.hpp\"\n#include \"cpy.hpp\"\n#include \"dequantize.hpp\"\n#include \"dmmv.hpp\"\n#include \"element_wise.hpp\"\n#include \"fattn.hpp\"\n#include \"gla.hpp\"\n#include \"im2col.hpp\"\n#include \"mmq.hpp\"\n#include \"mmvq.hpp\"\n#include \"norm.hpp\"\n#include \"outprod.hpp\"\n#include \"pad.hpp\"\n#include \"quantize.hpp\"\n#include \"quants.hpp\"\n#include \"roll.hpp\"\n#include \"rope.hpp\"\n#include \"set_rows.hpp\"\n#include \"ssm_conv.hpp\"\n#include \"softmax.hpp\"\n#include \"tsembd.hpp\"\n#include \"wkv.hpp\"\n#include \"pad_reflect_1d.hpp\"\n\n\n#endif // GGML_SYCL_BACKEND_HPP\n" }, { "path": "src/ggml-sycl/binbcast.cpp", "content": "#include \"binbcast.hpp\"\n\n#include \n#include \n#include \n\n#include \"ggml.h\"\n\ntemplate\nstatic void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst,\n int ne0, int ne1, int ne2, int ne3,\n int ne10, int ne11, int ne12, int ne13,\n /*int s0, */ int s1, int s2, int s3,\n int s00, int s01, int s02, int s03,\n int s10, int s11, int s12, int s13,\n const sycl::nd_item<3> &item_ct1) {\n const int i0s = item_ct1.get_local_range(2) * item_ct1.get_group(2) +\n item_ct1.get_local_id(2);\n const int i1 = (item_ct1.get_local_range(1) * item_ct1.get_group(1) +\n item_ct1.get_local_id(1));\n const int i2 = (item_ct1.get_local_range(0) * item_ct1.get_group(0) +\n item_ct1.get_local_id(0)) /\n ne3;\n const int i3 = (item_ct1.get_local_range(0) * item_ct1.get_group(0) +\n item_ct1.get_local_id(0)) %\n ne3;\n\n if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {\n return;\n }\n\n const int i11 = i1 % ne11;\n const int i12 = i2 % ne12;\n const int i13 = i3 % ne13;\n\n const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;\n const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;\n const size_t i_dst = i3*s3 + i2*s2 + i1*s1;\n\n const src0_t * src0_row = src0 + i_src0;\n const src1_t * src1_row = src1 + i_src1;\n dst_t * dst_row = dst + i_dst;\n\n for (int i0 = i0s; i0 < ne0;\n i0 += item_ct1.get_local_range(2) * item_ct1.get_group_range(2)) {\n const int i10 = i0 % ne10;\n dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0*s00] : 0.0f, (float)src1_row[i10*s10]);\n }\n}\n\ntemplate\nstatic void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t * dst,\n int ne0, int ne1, int ne2, int ne3,\n int ne10, int ne11, int ne12, int ne13,\n /*int s0, */ int s1, int s2, int s3,\n int s00, int s01, int s02, int s03,\n int s10, int s11, int s12, int s13,\n const sycl::nd_item<3> &item_ct1) {\n\n const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +\n item_ct1.get_local_id(2);\n\n const int i3 = i/(ne2*ne1*ne0);\n const int i2 = (i/(ne1*ne0)) % ne2;\n const int i1 = (i/ne0) % ne1;\n const int i0 = i % ne0;\n\n if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {\n return;\n }\n\n const int i11 = i1 % ne11;\n const int i12 = i2 % ne12;\n const int i13 = i3 % ne13;\n\n const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;\n const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;\n const size_t i_dst = i3*s3 + i2*s2 + i1*s1;\n\n const src0_t * src0_row = src0 + i_src0;\n const src1_t * src1_row = src1 + i_src1;\n dst_t * dst_row = dst + i_dst;\n\n const int i10 = i0 % ne10;\n dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0*s00] : 0.0f, (float)src1_row[i10*s10]);\n}\n\n\ntemplate\nstruct bin_bcast_sycl {\n template \n void operator()(const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd, const int64_t ne00,\n const int64_t ne01, const int64_t ne02, const int64_t ne03, const int64_t ne10, const int64_t ne11,\n const int64_t ne12, const int64_t ne13, const int64_t ne0, const int64_t ne1, const int64_t ne2,\n const int64_t ne3, const size_t nb00, const size_t nb01, const size_t nb02, const size_t nb03,\n const size_t nb10, const size_t nb11, const size_t nb12, const size_t nb13, const size_t nb0,\n const size_t nb1, const size_t nb2, const size_t nb3, const bool src0_is_contiguous,\n const bool src1_is_contiguous, const bool src0_is_permuted, const bool src1_is_permuted,\n queue_ptr stream) {\n int nr0 = ne10 / ne0;\n int nr1 = ne11/ne1;\n int nr2 = ne12/ne2;\n int nr3 = ne13/ne3;\n\n int nr[4] = { nr0, nr1, nr2, nr3 };\n\n // collapse dimensions until first broadcast dimension\n int64_t cne[] = {ne0, ne1, ne2, ne3};\n int64_t cne0[] = {ne00, ne01, ne02, ne03};\n int64_t cne1[] = {ne10, ne11, ne12, ne13};\n size_t cnb[] = {nb0, nb1, nb2, nb3};\n size_t cnb0[] = {nb00, nb01, nb02, nb03};\n size_t cnb1[] = {nb10, nb11, nb12, nb13};\n auto collapse = [](int64_t cne[]) {\n cne[0] *= cne[1];\n cne[1] = cne[2];\n cne[2] = cne[3];\n cne[3] = 1;\n };\n\n auto collapse_nb = [](size_t cnb[], int64_t cne[]) {\n cnb[1] *= cne[1];\n cnb[2] *= cne[2];\n cnb[3] *= cne[3];\n };\n\n if (src0_is_contiguous && src1_is_contiguous && !src0_is_permuted && !src1_is_permuted) {\n for (int i = 0; i < 4; i++) {\n if (nr[i] != 1) {\n break;\n }\n if (i > 0) {\n collapse_nb(cnb, cne);\n collapse_nb(cnb0, cne0);\n collapse_nb(cnb1, cne1);\n collapse(cne);\n collapse(cne0);\n collapse(cne1);\n }\n }\n }\n {\n int64_t ne0 = cne[0];\n int64_t ne1 = cne[1];\n int64_t ne2 = cne[2];\n int64_t ne3 = cne[3];\n\n int64_t ne10 = cne1[0];\n int64_t ne11 = cne1[1];\n int64_t ne12 = cne1[2];\n int64_t ne13 = cne1[3];\n\n size_t nb0 = cnb[0];\n size_t nb1 = cnb[1];\n size_t nb2 = cnb[2];\n size_t nb3 = cnb[3];\n\n size_t nb00 = cnb0[0];\n size_t nb01 = cnb0[1];\n size_t nb02 = cnb0[2];\n size_t nb03 = cnb0[3];\n\n size_t nb10 = cnb1[0];\n size_t nb11 = cnb1[1];\n size_t nb12 = cnb1[2];\n size_t nb13 = cnb1[3];\n\n // size_t s0 = nb0 / sizeof(dst_t);\n size_t s1 = nb1 / sizeof(dst_t);\n size_t s2 = nb2 / sizeof(dst_t);\n size_t s3 = nb3 / sizeof(dst_t);\n\n size_t s10 = nb10 / sizeof(src1_t);\n size_t s11 = nb11 / sizeof(src1_t);\n size_t s12 = nb12 / sizeof(src1_t);\n size_t s13 = nb13 / sizeof(src1_t);\n\n size_t s00 = nb00 / sizeof(src0_t);\n size_t s01 = nb01 / sizeof(src0_t);\n size_t s02 = nb02 / sizeof(src0_t);\n size_t s03 = nb03 / sizeof(src0_t);\n\n GGML_UNUSED(s00);\n\n GGML_ASSERT(nb0 % sizeof(dst_t) == 0);\n GGML_ASSERT(nb1 % sizeof(dst_t) == 0);\n GGML_ASSERT(nb2 % sizeof(dst_t) == 0);\n GGML_ASSERT(nb3 % sizeof(dst_t) == 0);\n\n GGML_ASSERT(nb00 % sizeof(src0_t) == 0);\n GGML_ASSERT(nb01 % sizeof(src0_t) == 0);\n GGML_ASSERT(nb02 % sizeof(src0_t) == 0);\n GGML_ASSERT(nb03 % sizeof(src0_t) == 0);\n\n GGML_ASSERT(nb10 % sizeof(src1_t) == 0);\n GGML_ASSERT(nb11 % sizeof(src1_t) == 0);\n GGML_ASSERT(nb12 % sizeof(src1_t) == 0);\n GGML_ASSERT(nb13 % sizeof(src1_t) == 0);\n\n const int block_size = 128;\n\n int64_t hne0 = std::max(ne0/2LL, 1LL);\n\n sycl::range<3> block_dims(1, 1, 1);\n block_dims[2] = std::min(hne0, block_size);\n block_dims[1] = std::min(\n ne1, block_size / (unsigned int)block_dims[2]);\n block_dims[0] = std::min(\n std::min(\n ne2 * ne3, block_size / (unsigned int)block_dims[2] /\n (unsigned int)block_dims[1]),\n 64U);\n\n sycl::range<3> block_nums(\n (ne2 * ne3 + block_dims[0] - 1) / block_dims[0],\n (ne1 + block_dims[1] - 1) / block_dims[1],\n (hne0 + block_dims[2] - 1) / block_dims[2]);\n\n if (block_nums[0] > 65535) {\n // this is the maximum number of blocks in z direction, fallback to 1D grid kernel\n int block_num = (ne0*ne1*ne2*ne3 + block_size - 1) / block_size;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, block_num) *\n sycl::range<3>(1, 1, block_size),\n sycl::range<3>(1, 1, block_size)),\n [=](sycl::nd_item<3> item_ct1) {\n k_bin_bcast_unravel(\n src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3,\n ne10, ne11, ne12, ne13, s1, s2, s3, s00, s01, s02,\n s03, s10, s11, s12, s13, item_ct1);\n });\n }\n } else {\n /*\n DPCT1049:16: The work-group size passed to the SYCL kernel may\n exceed the limit. To get the device limit, query\n info::device::max_work_group_size. Adjust the work-group size if\n needed.\n */\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) {\n k_bin_bcast(src0_dd, src1_dd, dst_dd, ne0, ne1,\n ne2, ne3, ne10, ne11, ne12, ne13,\n s1, s2, s3, s00, s01, s02, s03, s10, s11, s12, s13,\n item_ct1);\n });\n }\n }\n }\n};\n\ntemplate \ninline void ggml_sycl_op_bin_bcast(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1,\n ggml_tensor * dst) {\n dpct::queue_ptr main_stream = ctx.stream();\n GGML_TENSOR_BINARY_OP_LOCALS\n\n if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n op()((const float *) src0->data, (const float *) src1->data, (float *) dst->data, ne00, ne01, ne02, ne03, ne10,\n ne11, ne12, ne13, ne0, ne1, ne2, ne3, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb0, nb1, nb2, nb3,\n ggml_is_contiguous(src0), ggml_is_contiguous(src1), ggml_is_permuted(src0), ggml_is_permuted(src1), main_stream);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {\n op()((const sycl::half *) src0->data, (const sycl::half *) src1->data, (sycl::half *) dst->data, ne00, ne01,\n ne02, ne03, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13,\n nb0, nb1, nb2, nb3, ggml_is_contiguous(src0), ggml_is_contiguous(src1), ggml_is_permuted(src0), ggml_is_permuted(src1),\n main_stream);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) {\n op()((const sycl::half *) src0->data, (const float *) src1->data, (sycl::half *) dst->data, ne00, ne01, ne02,\n ne03, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb0, nb1,\n nb2, nb3, ggml_is_contiguous(src0), ggml_is_contiguous(src1), ggml_is_permuted(src0), ggml_is_permuted(src1),\n main_stream);\n } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_I32) {\n op()((const int32_t *) src0->data, (const int32_t *) src1->data, (int32_t *) dst->data, ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb0, nb1, nb2,\n nb3, ggml_is_contiguous(src0), ggml_is_contiguous(src1), ggml_is_permuted(src0), ggml_is_permuted(src1),\n main_stream);\n } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16 && dst->type == GGML_TYPE_I16) {\n op()((const int16_t *) src0->data, (const int16_t *) src1->data, (int16_t *) dst->data, ne00, ne01, ne02, ne03,\n ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb0, nb1, nb2,\n nb3, ggml_is_contiguous(src0), ggml_is_contiguous(src1), ggml_is_permuted(src0), ggml_is_permuted(src1),\n main_stream);\n } else {\n fprintf(stderr, \"%s: unsupported types: dst: %s, src0: %s, src1: %s\\n\", __func__, ggml_type_name(dst->type),\n ggml_type_name(src0->type), ggml_type_name(src1->type));\n GGML_ABORT(\"fatal error\");\n }\n}\n\ninline void ggml_sycl_op_add(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n\n ggml_sycl_op_bin_bcast>(ctx, dst->src[0], dst->src[1], dst);\n}\n\ninline void ggml_sycl_op_sub(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n\n ggml_sycl_op_bin_bcast>(ctx, dst->src[0], dst->src[1], dst);\n}\n\ninline void ggml_sycl_op_mul(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n\n ggml_sycl_op_bin_bcast>(ctx, dst->src[0], dst->src[1], dst);\n}\n\ninline void ggml_sycl_op_div(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n\n ggml_sycl_op_bin_bcast>(ctx, dst->src[0], dst->src[1], dst);\n}\n\ninline void ggml_sycl_op_repeat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n ggml_sycl_op_bin_bcast>(ctx, dst, dst->src[0], dst);\n}\n\n\nvoid ggml_sycl_add(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);\n ggml_sycl_op_add(ctx, dst);\n}\n\nvoid ggml_sycl_sub(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);\n ggml_sycl_op_sub(ctx, dst);\n}\n\nvoid ggml_sycl_mul(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);\n ggml_sycl_op_mul(ctx, dst);\n}\n\nvoid ggml_sycl_div(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);\n ggml_sycl_op_div(ctx, dst);\n}\n\nvoid ggml_sycl_repeat(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_repeat(ctx, dst);\n}\n\n" }, { "path": "src/ggml-sycl/binbcast.hpp", "content": "#ifndef GGML_SYCL_BINBCAST_HPP\n#define GGML_SYCL_BINBCAST_HPP\n#include \"common.hpp\"\n\n\nstatic __dpct_inline__ float op_repeat(const float a, const float b) {\n return b;\n GGML_UNUSED(a);\n}\n\nstatic __dpct_inline__ float op_add(const float a, const float b) {\n return a + b;\n}\n\nstatic __dpct_inline__ float op_sub(const float a, const float b) {\n return a - b;\n}\n\nstatic __dpct_inline__ float op_mul(const float a, const float b) {\n return a * b;\n}\n\nstatic __dpct_inline__ float op_div(const float a, const float b) {\n return a / b;\n}\n\nvoid ggml_sycl_add(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_sub(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_mul(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_div(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_repeat(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\n\n#endif //GGML_SYCL_BINBCAST_HPP\n\n" }, { "path": "src/ggml-sycl/common.cpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#include \"common.hpp\"\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-impl.h\"\n\nint get_current_device_id() {\n return dpct::dev_mgr::instance().current_device_id();\n}\n\nvoid* ggml_sycl_host_malloc(size_t size) try {\n if (getenv(\"GGML_SYCL_NO_PINNED\") != nullptr) {\n return nullptr;\n }\n\n void* ptr = nullptr;\n // allow to use dpct::get_in_order_queue() for host malloc\n dpct::err0 err = CHECK_TRY_ERROR(\n ptr = (void*)sycl::malloc_host(size, dpct::get_in_order_queue()));\n\n if (err != 0) {\n // clear the error\n GGML_LOG_ERROR(\"WARNING: failed to allocate %.2f MB of pinned memory: %s\\n\", size / 1024.0 / 1024.0, \"syclGetErrorString is not supported\");\n return nullptr;\n }\n\n return ptr;\n} catch (sycl::exception const& exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nvoid ggml_sycl_host_free(void* ptr) try {\n // allow to use dpct::get_in_order_queue() for host malloc\n SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, dpct::get_in_order_queue())));\n} catch (sycl::exception const& exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nbool gpu_has_xmx(sycl::device &dev) {\n return dev.has(sycl::aspect::ext_intel_matrix);\n}\n\nint64_t downsample_sycl_global_range(int64_t accumulate_block_num, int64_t block_size) {\n const int64_t max_range = std::numeric_limits::max();\n int64_t sycl_down_blk_size = block_size;\n int64_t global_range = accumulate_block_num * sycl_down_blk_size;\n while(global_range > max_range) {\n sycl_down_blk_size /= 2;\n global_range = accumulate_block_num * sycl_down_blk_size;\n }\n return sycl_down_blk_size;\n}\n\nvoid release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector streams) {\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n for (int64_t is = 0; is < GGML_SYCL_MAX_STREAMS; ++is) {\n if (extra->events[i][is] != nullptr) {\n SYCL_CHECK(CHECK_TRY_ERROR(dpct::destroy_event(extra->events[i][is])));\n }\n }\n if (extra->data_device[i] != nullptr && streams.size()>0) {\n ggml_sycl_set_device(i);\n SYCL_CHECK(\n CHECK_TRY_ERROR(sycl::free(extra->data_device[i], *(streams[i]))));\n }\n }\n delete extra;\n}\n" }, { "path": "src/ggml-sycl/common.hpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_COMMON_HPP\n#define GGML_SYCL_COMMON_HPP\n\n#include \n#include \n#include \n#include \n\n#include \"dpct/helper.hpp\"\n#include \"ggml.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-sycl.h\"\n#include \"presets.hpp\"\n#include \"sycl_hw.hpp\"\n\nnamespace syclexp = sycl::ext::oneapi::experimental;\n\n#if GGML_SYCL_DNNL\n#include \"dnnl.hpp\"\n#include \"dnnl_sycl.hpp\"\n#endif\n\n#define GGML_COMMON_DECL_SYCL\n#define GGML_COMMON_IMPL_SYCL\n#define SYCL_FLASH_ATTN //remove it to disable FLASH_ATTENTION in building.\n#define SYCL_FAST_FP16 //don't change. remove it will break fattn-tile.hpp building\n\n/* suppress warning spam */\n#pragma clang diagnostic push\n#pragma clang diagnostic ignored \"-Wnested-anon-types\"\n#include \"ggml-common.h\"\n#pragma clang diagnostic pop\n#include \"ggml-impl.h\"\n\nvoid* ggml_sycl_host_malloc(size_t size);\nvoid ggml_sycl_host_free(void* ptr);\n\n\nextern int g_ggml_sycl_debug;\nextern int g_ggml_sycl_disable_optimize;\nextern int g_ggml_sycl_prioritize_dmmv;\nextern int g_ggml_sycl_enable_flash_attention;\n\n\n#if defined(__clang__) && __has_builtin(__builtin_expect)\n// Hint the optimizer to pipeline the more likely following instruction in branches\n# define LIKELY(expr) __builtin_expect(expr, true)\n# define UNLIKELY(expr) __builtin_expect(expr, false)\n#else\n# define LIKELY(expr) (expr)\n# define UNLIKELY(expr) (expr)\n#endif\n\n#define GGML_SYCL_DEBUG(...) \\\n do { \\\n if (UNLIKELY(g_ggml_sycl_debug)) \\\n fprintf(stderr, __VA_ARGS__); \\\n } while (0)\n\n#define CHECK_TRY_ERROR(expr) \\\n [&]() { \\\n try { \\\n expr; \\\n return dpct::success; \\\n } catch (std::exception const& e) { \\\n std::cerr << e.what() << \"\\nException caught at file:\" << __FILE__ \\\n << \", line:\" << __LINE__ << \", func:\" << __func__ \\\n << std::endl; \\\n return dpct::default_error; \\\n } \\\n }()\n\n\n#define __SYCL_ARCH__ DPCT_COMPATIBILITY_TEMP\n#define VER_4VEC 610 // todo for hardware optimize.\n#define VER_GEN9 700 // todo for hardware optimize.\n#define VER_GEN12 1000000 // todo for hardware optimize.\n#define VER_GEN13 (VER_GEN12 + 1030) // todo for hardware optimize.\n\n#define GGML_SYCL_MAX_NODES 8192 // TODO: adapt to hardwares\n\n// define for XMX in Intel GPU\n// TODO: currently, it's not used for XMX really.\n#if !defined(GGML_SYCL_FORCE_MMQ)\n #define SYCL_USE_XMX\n#endif\n\n// max batch size to use MMQ kernels when tensor cores are available\n#define MMQ_MAX_BATCH_SIZE 32\n\n// dmmv = dequantize_mul_mat_vec\n#ifndef GGML_SYCL_DMMV_X\n#define GGML_SYCL_DMMV_X 32\n#endif\n#ifndef GGML_SYCL_MMV_Y\n#define GGML_SYCL_MMV_Y 1\n#endif\n\ntypedef sycl::queue *queue_ptr;\n\nenum ggml_sycl_backend_gpu_mode {\n SYCL_UNSET_GPU_MODE = -1,\n SYCL_SINGLE_GPU_MODE = 0,\n SYCL_MUL_GPU_MODE\n};\n\nstatic_assert(sizeof(sycl::half) == sizeof(ggml_fp16_t), \"wrong fp16 size\");\n\nstatic void crash() {\n int* ptr = NULL;\n *ptr = 0;\n}\n\n[[noreturn]] static void ggml_sycl_error(\n const char* stmt,\n const char* func,\n const char* file,\n const int line,\n const char* msg) {\n fprintf(stderr, \"SYCL error: %s: %s\\n\", stmt, msg);\n fprintf(stderr, \" in function %s at %s:%d\\n\", func, file, line);\n GGML_ABORT(\"SYCL error\");\n}\n\n#define SYCL_CHECK(err) \\\n do { \\\n auto err_ = (err); \\\n if (err_ != 0) \\\n ggml_sycl_error(#err, __func__, __FILE__, __LINE__, \"Exception caught in this line of code.\"); \\\n } while (0)\n\n#if DPCT_COMPAT_RT_VERSION >= 11100\n#define GGML_SYCL_ASSUME(x) __builtin_assume(x)\n#else\n#define GGML_SYCL_ASSUME(x)\n#endif // DPCT_COMPAT_RT_VERSION >= 11100\n\n#ifdef GGML_SYCL_F16\ntypedef sycl::half dfloat; // dequantize float\ntypedef sycl::half2 dfloat2;\n#else\ntypedef float dfloat; // dequantize float\ntypedef sycl::float2 dfloat2;\n#endif // GGML_SYCL_F16\n\n#define MMVQ_MAX_BATCH_SIZE 8\n\nstatic int g_all_sycl_device_count = -1;\nstatic bool g_ggml_backend_sycl_buffer_type_initialized = false;\n\nstatic ggml_sycl_backend_gpu_mode g_ggml_sycl_backend_gpu_mode =\n SYCL_UNSET_GPU_MODE;\n\nstatic void* g_scratch_buffer = nullptr;\nstatic size_t g_scratch_size = 0; // disabled by default\nstatic size_t g_scratch_offset = 0;\n\n[[noreturn]] static inline void bad_arch(const sycl::stream& stream_ct1) {\n stream_ct1 << \"ERROR: ggml-sycl was compiled without support for the \"\n \"current GPU architecture.\\n\";\n // __trap();\n std::exit(1);\n\n (void)bad_arch; // suppress unused function warning\n}\n\nint get_current_device_id();\n\ninline int ggml_sycl_get_device() {\n return get_current_device_id();\n}\n\ninline dpct::err0 ggml_sycl_set_device(const int device) try {\n int current_device_id;\n SYCL_CHECK(CHECK_TRY_ERROR(current_device_id = get_current_device_id()));\n\n // GGML_SYCL_DEBUG(\"ggml_sycl_set_device device_id=%d,\n // current_device_id=%d\\n\", device, current_device);\n if (device == current_device_id) {\n return 0;\n }\n\n return CHECK_TRY_ERROR(dpct::select_device(device));\n} catch (sycl::exception const& exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n crash();\n std::exit(1);\n}\n\n//////////////////////\nstruct optimize_feature {\n bool reorder=false;\n};\n\nstruct sycl_device_info {\n int cc; // compute capability\n int nsm; // number of streaming multiprocessors (CUDA) maps to the maximum\n // number of compute units on a SYCL device.\n // size_t smpb; // max. shared memory per block\n size_t smpbo; // max. shared memory per block (with opt-in)\n int warp_size; // WARP_SIZE(16)|WARP_32_SIZE(32)|WARP_16_SIZE(16). For Intel GPU, 16 is better in most cases. Some OP support 32 only.\n int max_wg_per_cu; // max work groups per compute unit - refer to\n // cudaOccupancyMaxActiveBlocksPerMultiprocessor\n bool vmm; // virtual memory support\n size_t total_vram;\n //sycl_hw_info hw_info; \\\\ device id and aarch, currently not used\n optimize_feature opt_feature;\n};\n\n\nstruct ggml_sycl_device_info {\n int device_count;\n\n sycl_device_info devices[GGML_SYCL_MAX_DEVICES] = {};\n\n std::array default_tensor_split = {};\n\n int max_work_group_sizes[GGML_SYCL_MAX_DEVICES] = {0};\n};\n\nconst ggml_sycl_device_info & ggml_sycl_info();\n\nstruct ggml_sycl_pool {\n virtual ~ggml_sycl_pool() = default;\n\n virtual void * alloc(size_t size, size_t * actual_size) = 0;\n virtual void free(void * ptr, size_t size) = 0;\n};\n\ntemplate\nstruct ggml_sycl_pool_alloc {\n ggml_sycl_pool * pool = nullptr;\n T * ptr = nullptr;\n size_t actual_size = 0;\n\n explicit ggml_sycl_pool_alloc(ggml_sycl_pool & pool) : pool(&pool) {\n }\n\n ggml_sycl_pool_alloc(ggml_sycl_pool & pool, size_t size) : pool(&pool) {\n alloc(size);\n }\n\n ~ggml_sycl_pool_alloc() {\n if (ptr != nullptr) {\n pool->free(ptr, actual_size);\n }\n }\n\n T * realloc(size_t size) {\n GGML_ASSERT(pool != nullptr);\n if (ptr)\n pool->free(ptr, actual_size);\n ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size);\n return ptr;\n }\n\n // size is in number of elements\n T * alloc(size_t size) {\n GGML_ASSERT(pool != nullptr);\n GGML_ASSERT(ptr == nullptr);\n ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size);\n return ptr;\n }\n\n T * alloc(ggml_sycl_pool & pool, size_t size) {\n this->pool = &pool;\n return alloc(size);\n }\n\n T * get() {\n return ptr;\n }\n\n ggml_sycl_pool_alloc() = default;\n ggml_sycl_pool_alloc(const ggml_sycl_pool_alloc &) = delete;\n ggml_sycl_pool_alloc(ggml_sycl_pool_alloc &&) = delete;\n ggml_sycl_pool_alloc& operator=(const ggml_sycl_pool_alloc &) = delete;\n ggml_sycl_pool_alloc& operator=(ggml_sycl_pool_alloc &&) = delete;\n};\n\n// backend interface\n\nstruct ggml_tensor_extra_gpu {\n void* data_device[GGML_SYCL_MAX_DEVICES]; // 1 pointer for each device for split\n // tensors\n dpct::event_ptr events[GGML_SYCL_MAX_DEVICES]\n [GGML_SYCL_MAX_STREAMS]; // events for synchronizing multiple GPUs\n optimize_feature optimized_feature;\n};\n\nvoid release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector streams={});\n\nnamespace sycl_ex = sycl::ext::oneapi::experimental;\nstruct ggml_backend_sycl_context {\n int device;\n std::string name;\n optimize_feature opt_feature;\n\n queue_ptr qptrs[GGML_SYCL_MAX_DEVICES][GGML_SYCL_MAX_STREAMS] = { { nullptr } };\n\n explicit ggml_backend_sycl_context(int device) :\n device(device),\n name(GGML_SYCL_NAME + std::to_string(device)) {\n opt_feature = ggml_sycl_info().devices[device].opt_feature;\n }\n\n queue_ptr stream(int device, int stream) {\n if (qptrs[device][stream] == nullptr) {\n qptrs[device][stream] = &(dpct::get_device(device).default_queue());\n }\n return qptrs[device][stream];\n }\n\n queue_ptr stream() {\n return stream(device, 0);\n }\n\n#if GGML_SYCL_DNNL\n dnnl::engine make_engine(sycl::queue* q) {\n // Get the device associated with the queue\n sycl::device dev = q->get_device();\n // Get the context associated with the queue\n sycl::context ctx = q->get_context();\n const dnnl::engine eng = dnnl::sycl_interop::make_engine(dev, ctx);\n return eng;\n }\n\n std::unordered_map stream_map;\n std::unordered_map engine_map;\n dnnl::stream stream_dnnl(int device, int _stream) {\n auto q = stream(device, _stream);\n return stream_dnnl(q);\n }\n dnnl::engine engine_dnnl(sycl::queue* qptr) {\n auto it = engine_map.find(qptr);\n if (it == engine_map.end()) {\n auto eng = make_engine(qptr);\n engine_map[qptr] = eng;\n return eng;\n }\n else\n {\n return it->second;\n }\n }\n dnnl::stream stream_dnnl(sycl::queue* qptr) {\n auto it = stream_map.find(qptr);\n if (it == stream_map.end()) {\n auto eng = engine_dnnl(qptr);\n auto stream = dnnl::sycl_interop::make_stream(eng, *qptr);\n stream_map[qptr] = stream;\n return stream;\n }\n else\n {\n return it->second;\n }\n }\n dnnl::stream stream_dnnl() {\n return stream_dnnl(device, 0);\n }\n dnnl::memory get_scratchpad_mem(const dnnl::memory::desc & scratchpad_md,\n const dnnl::engine & eng, const queue_ptr q) {\n ggml_sycl_pool_alloc * pool;\n auto it = scratchpad_map.find(q);\n if (it == scratchpad_map.end()) {\n scratchpad_map[q] = std::make_unique>(this->pool());\n pool = scratchpad_map[q].get();\n } else {\n pool = it->second.get();\n }\n\n size_t scratchpad_size = scratchpad_md.get_size();\n if (scratchpad_size > pool->actual_size) {\n pool->realloc(scratchpad_size);\n }\n void * mem_ptr = pool->get();\n return dnnl::memory(scratchpad_md, eng, mem_ptr);\n }\n#endif\n\n // pool\n std::unique_ptr pools[GGML_SYCL_MAX_DEVICES];\n std::unordered_map>> scratchpad_map;\n\n std::unique_ptr host_pools[GGML_SYCL_MAX_DEVICES];\n\n static std::unique_ptr new_pool_for_device(queue_ptr qptr, int device);\n\n static std::unique_ptr new_pool_for_host(queue_ptr qptr, int device);\n\n ggml_sycl_pool & pool(int device) {\n if (pools[device] == nullptr) {\n pools[device] = new_pool_for_device(stream(device,0), device);\n }\n return *pools[device];\n }\n\n ggml_sycl_pool & pool() {\n return pool(device);\n }\n\n#ifdef GGML_SYCL_GRAPH\n std::unique_ptr> exec_graph = nullptr;\n#endif\n\n ggml_sycl_pool & host_pool(int device) {\n if (host_pools[device] == nullptr) {\n host_pools[device] = new_pool_for_host(stream(device, 0), device);\n }\n return *host_pools[device];\n }\n\n ggml_sycl_pool & host_pool() { return host_pool(device); }\n};\n\n// common device functions\n\nstatic __dpct_inline__ float warp_reduce_sum(float x,\n const sycl::nd_item<3>& item_ct1) {\n#pragma unroll\n for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {\n x += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), x, mask);\n }\n return x;\n}\n\nstatic __dpct_inline__ sycl::float2\nwarp_reduce_sum(sycl::float2 a, const sycl::nd_item<3>& item_ct1) {\n#pragma unroll\n for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {\n a.x() += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), a.x(),\n mask);\n a.y() += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), a.y(),\n mask);\n }\n return a;\n}\n\n/* use WARP_SIZE or WARP_32_SIZE*/\ntemplate \nstatic __dpct_inline__ int warp_reduce_sum(int x) {\n return sycl::reduce_over_group(\n sycl::ext::oneapi::this_work_item::get_sub_group(), x, sycl::plus<>());\n}\n\n/* use WARP_SIZE or WARP_32_SIZE*/\ntemplate \nstatic __dpct_inline__ float warp_reduce_sum(float x) {\n#pragma unroll\n for (int offset = width / 2; offset > 0; offset >>= 1) {\n x += dpct::permute_sub_group_by_xor(\n sycl::ext::oneapi::this_work_item::get_sub_group(), x, offset, width);\n }\n return x;\n}\n\n/* use WARP_SIZE or WARP_32_SIZE*/\ntemplate \nstatic __dpct_inline__ float warp_reduce_sum(float x, const sycl::nd_item<3>& item_ct1) {\n#pragma unroll\n for (int offset = width / 2; offset > 0; offset >>= 1) {\n x += dpct::permute_sub_group_by_xor(\n item_ct1.get_sub_group(), x, offset);\n }\n return x;\n}\n\n/* use WARP_SIZE or WARP_32_SIZE*/\ntemplate \nstatic __dpct_inline__ sycl::float2 warp_reduce_sum(sycl::float2 a) {\n#pragma unroll\n for (int offset = width / 2; offset > 0; offset >>= 1) {\n a.x() += dpct::permute_sub_group_by_xor(\n sycl::ext::oneapi::this_work_item::get_sub_group(), a.x(), offset,\n width);\n a.y() += dpct::permute_sub_group_by_xor(\n sycl::ext::oneapi::this_work_item::get_sub_group(), a.y(), offset,\n width);\n }\n return a;\n}\n\n/* use WARP_SIZE or WARP_32_SIZE*/\ntemplate \nstatic __dpct_inline__ sycl::half2 warp_reduce_sum(sycl::half2 a) {\n#pragma unroll\n for (int offset = width / 2; offset > 0; offset >>= 1) {\n a = a + dpct::permute_sub_group_by_xor(\n sycl::ext::oneapi::this_work_item::get_sub_group(), a, offset,\n width);\n }\n return a;\n}\n\nstatic constexpr int ggml_sycl_get_physical_warp_size() {\n // todo: for old iGPU + dGPU case, need to be changed.\n return WARP_SIZE;\n}\n\n/* use WARP_SIZE or WARP_32_SIZE*/\ntemplate \nstatic __dpct_inline__ int warp_reduce_all(int x) {\n if (width == ggml_sycl_get_physical_warp_size()) {\n return sycl::all_of_group(\n sycl::ext::oneapi::this_work_item::get_sub_group(),\n (~0xffffffff &\n (0x1 << sycl::ext::oneapi::this_work_item::get_sub_group()\n .get_local_linear_id())) ||\n x);\n } else {\n#pragma unroll\n for (int offset = width / 2; offset > 0; offset >>= 1) {\n x = dpct::permute_sub_group_by_xor(\n sycl::ext::oneapi::this_work_item::get_sub_group(), x,\n offset, width) &&\n x;\n }\n return x;\n }\n}\n\n/* use WARP_SIZE or WARP_32_SIZE*/\ntemplate \nstatic __dpct_inline__ int warp_reduce_any(int x) {\n if (width == ggml_sycl_get_physical_warp_size()) {\n return sycl::any_of_group(\n sycl::ext::oneapi::this_work_item::get_sub_group(),\n (0xffffffff &\n (0x1 << sycl::ext::oneapi::this_work_item::get_sub_group()\n .get_local_linear_id())) &&\n x);\n } else {\n#pragma unroll\n for (int offset = width / 2; offset > 0; offset >>= 1) {\n x = dpct::permute_sub_group_by_xor(\n sycl::ext::oneapi::this_work_item::get_sub_group(), x,\n offset, width) ||\n x;\n }\n return x;\n }\n}\n\n/* use WARP_SIZE or WARP_32_SIZE*/\ntemplate \nstatic __dpct_inline__ float warp_reduce_max(float x) {\n#pragma unroll\n for (int offset = width / 2; offset > 0; offset >>= 1) {\n x = sycl::fmax(x, dpct::permute_sub_group_by_xor(\n sycl::ext::oneapi::this_work_item::get_sub_group(), x,\n offset, width));\n }\n return x;\n}\n\nstatic __dpct_inline__ float warp_reduce_max(float x,\n const sycl::nd_item<3>& item_ct1) {\n#pragma unroll\n for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {\n x = sycl::fmax(x, dpct::permute_sub_group_by_xor(\n item_ct1.get_sub_group(), x, mask));\n }\n return x;\n}\n\n/* Helper for Computing the linear offset of a ggml_tensor given\nper-dimension sizes, strides, and indices */\ntemplate\n__dpct_inline__ size_t calculate_offset(const std::array & strides, const std::array & indices) {\n size_t offset = 0;\n#pragma unroll\n for (int i = 0; i < N; i++) {\n auto index_i = indices[i];\n offset += strides[i] * index_i;\n }\n return offset;\n}\n\n// Helper for vec loading aligned data\ntemplate \ninline sycl::vec vec_aligned_load(const Tp* aligned_ptr) {\n return *reinterpret_cast*>(aligned_ptr);\n}\n\n// Helper for accessing pointers with no warnings\ntemplate \nstatic __dpct_inline__ Tp* get_pointer(sycl::local_accessor acc) {\n return acc.template get_multi_ptr().get();\n}\n\nint64_t downsample_sycl_global_range(int64_t accumulate_block_num, int64_t block_size);\n\nconstexpr size_t ceil_div(const size_t m, const size_t n) {\n return (m + n - 1) / n;\n}\n\nbool gpu_has_xmx(sycl::device &dev);\n\ntemplate std::string debug_get_array_str(const std::string & prefix, const T array[N]) {\n if (LIKELY(!g_ggml_sycl_debug)) {\n return \"\";\n }\n std::stringstream ss;\n ss << prefix << \"=[\";\n for (std::size_t i = 0; i < N - 1; ++i) {\n ss << array[i] << \", \";\n }\n if constexpr (N > 0) {\n ss << array[N - 1];\n }\n ss << \"]\";\n return ss.str();\n}\n\ninline std::string debug_get_tensor_str(const std::string &prefix,\n const ggml_tensor *tensor, const std::string &suffix = \"\") {\n std::stringstream ss;\n if (LIKELY(!g_ggml_sycl_debug)) { return ss.str(); }\n ss << prefix.c_str() << \"=\";\n if (tensor) {\n ss << \"'\" << tensor->name << \"':type=\" << ggml_type_name(tensor->type);\n ss << debug_get_array_str(\";ne\", tensor->ne);\n ss << debug_get_array_str(\";nb\", tensor->nb);\n\n if (!ggml_is_contiguous(tensor)) { ss << \";strided\"; }\n if (ggml_is_permuted(tensor)) { ss << \";permuted\"; }\n } else {\n ss << \"nullptr\";\n }\n ss << suffix;\n return ss.str();\n}\n\n// Use scope_op_debug_print to log operations coming from running a model\nstruct scope_op_debug_print {\n // Use string_views to avoid the cost of creating a string and concatenating them\n // string_views must be alive for as long as the object is alive\n // scope_op_debug_print are used with string literals in practice which are stored in constant space so always accessible\n scope_op_debug_print(const std::string_view & func, const std::string_view & func_suffix, const ggml_tensor * dst,\n std::size_t num_src, const std::string_view & suffix = \"\") :\n func(func),\n func_suffix(func_suffix) {\n if (LIKELY(!g_ggml_sycl_debug)) {\n return;\n }\n GGML_SYCL_DEBUG(\"[SYCL][OP] call %s%s:\", func.data(), func_suffix.data());\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\" dst\", dst).c_str());\n if (dst) {\n for (std::size_t i = 0; i < num_src; ++i) {\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\"\\tsrc\" + std::to_string(i), dst->src[i]).c_str());\n }\n }\n GGML_SYCL_DEBUG(\"%s\\n\", suffix.data());\n }\n\n scope_op_debug_print(const std::string_view & func, const ggml_tensor * dst, std::size_t num_src,\n const std::string_view & suffix = \"\") :\n scope_op_debug_print(func, \"\", dst, num_src, suffix) {}\n\n ~scope_op_debug_print() { GGML_SYCL_DEBUG(\"[SYCL][OP] call %s%s done\\n\", func.data(), func_suffix.data()); }\n\n private:\n std::string_view func;\n std::string_view func_suffix;\n};\n\nstatic __dpct_inline__ float get_alibi_slope(const float max_bias,\n const uint32_t h,\n const uint32_t n_head_log2,\n const float m0,\n const float m1) {\n if (max_bias <= 0.0f) {\n return 1.0f;\n }\n const float base = h < n_head_log2 ? m0 : m1;\n const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n return dpct::pow(base, exph);\n}\n\nstatic const sycl::uint3 init_fastdiv_values(uint32_t d) {\n GGML_ASSERT(d != 0);\n\n uint32_t L = 0;\n while (L < 32 && (uint32_t{ 1 } << L) < d) {\n L++;\n }\n\n uint32_t mp = (uint32_t) ((uint64_t{ 1 } << 32) * ((uint64_t{ 1 } << L) - d) / d + 1);\n return sycl::uint3(mp, L, d);\n}\n\n// Maximum number of bytes that can be copied in a single instruction.\n// Set by test result.\nstatic constexpr int ggml_sycl_get_max_cpy_bytes() {\n return 16;\n}\n\n// Aligned memory transfers of 8/16 bytes can be faster than 2 transfers with 4 bytes.\ntemplate \nstatic __dpct_inline__ void ggml_sycl_memcpy_1(void * dst, const void * src) {\n if constexpr (alignment != 0) {\n static_assert(nbytes % alignment == 0, \"bad alignment\");\n }\n constexpr int nb_per_cpy = alignment == 0 ? nbytes : alignment;\n\n#pragma unroll\n for (int i = 0; i < nbytes/nb_per_cpy; ++i) {\n if constexpr (nb_per_cpy == 1) {\n ((char *) dst)[i] = ((const char *) src)[i];\n } else if constexpr (nb_per_cpy == 2) {\n ((short *) dst)[i] = ((const short *) src)[i];\n } else if constexpr (nb_per_cpy == 4) {\n ((int *) dst)[i] = ((const int *) src)[i];\n } else if constexpr (nb_per_cpy == 8) {\n ((sycl::int2 *) dst)[i] = ((const sycl::int2 *) src)[i];\n } else if constexpr (nb_per_cpy == 16) {\n ((sycl::int4 *) dst)[i] = ((const sycl::int4 *) src)[i];\n } else {\n static_assert(nbytes == 0 && nbytes == -1, \"bad nbytes\");\n }\n }\n}\ntemplate \nsycl::half2 __dpct_inline__ make_half2( T x, T y) {\n sycl::half2 res(static_cast(x),static_cast(y));\n return res;\n}\n\nstatic __dpct_inline__ uint32_t fastdiv(uint32_t n, const sycl::uint3 fastdiv_values) {\n const uint32_t hi = sycl::mul_hi(n, fastdiv_values.x());\n return (hi + n) >> fastdiv_values.y();\n}\n\n\ntemplate \nsycl::float2 __dpct_inline__ make_float2( T x, T y) {\n sycl::float2 res(static_cast(x),static_cast(y));\n return res;\n}\n\nsycl::float2 __dpct_inline__ __half22float2(sycl::half2 &H) {\n sycl::float2 float2_value(static_cast(H.x()), static_cast(H.y()));\n return float2_value;\n}\n\nstatic __dpct_inline__ sycl::uint2 fast_div_modulo(uint32_t n, const sycl::uint3 fastdiv_values) {\n const uint32_t div_val = fastdiv(n, fastdiv_values);\n const uint32_t mod_val = n - div_val * fastdiv_values.z();\n return sycl::uint2(div_val, mod_val);\n}\n\nstatic __dpct_inline__ int ggml_sycl_dp4a(const int a, const int b, int c) {\n return dpct::dp4a(a, b, c);\n}\n\nstatic __dpct_inline__ float ggml_sycl_e8m0_to_fp32(uint8_t x) {\n uint32_t bits;\n if (x == 0) {\n bits = 0x00400000;\n } else {\n bits = (uint32_t) x << 23;\n }\n\n float result;\n memcpy(&result, &bits, sizeof(float));\n return result;\n}\n\nsycl::float2 __dpct_inline__ __half22float2(const sycl::half2 &H) {\n sycl::float2 float2_value(static_cast(H.x()), static_cast(H.y()));\n return float2_value;\n}\n\nfloat __dpct_inline__ __half2float(sycl::half H) {\n return static_cast(H);\n}\n\nstatic __dpct_inline__ void ggml_sycl_mad(float & acc, const float v, const float u) {\n acc += v*u;\n}\n\nstatic __dpct_inline__ void ggml_sycl_mad(float & acc, const sycl::float2 v, const sycl::float2 u) {\n acc += v.x() * u.x();\n acc += v.y() * u.y();\n}\n\nstatic __dpct_inline__ void ggml_sycl_mad(float & acc, const sycl::half2 v, const sycl::half2 u) {\n#ifdef GGML_SYCL_F16\n const sycl::float2 tmp = (v * u).template convert();\n acc += tmp.x() + tmp.y();\n#else\n const sycl::float2 tmpv = __half22float2(v);\n const sycl::float2 tmpu = __half22float2(u);\n acc += tmpv.x() * tmpu.x();\n acc += tmpv.y() * tmpu.y();\n#endif // GGML_SYCL_F16\n}\n\nstatic __dpct_inline__ void ggml_sycl_mad(sycl::half2 & acc, const sycl::half2 v, const sycl::half2 u) {\n#ifdef GGML_SYCL_F16\n acc += v*u;\n#else\n const sycl::float2 tmpv = __half22float2(v);\n const sycl::float2 tmpu = __half22float2(u);\n sycl::float2 tmpacc = __half22float2(acc);\n // tmpacc.x += tmpv.x() * tmpu.x();\n // tmpacc.y += tmpv.y() * tmpu.y();\n sycl::float2 tmp1(tmpacc.x() + tmpv.x() * tmpu.x(), tmpacc.y() + tmpv.y() * tmpu.y());\n acc = make_half2(tmp1.x(), tmp1.y());\n#endif // GGML_SYCL_F16\n}\n\ntemplate \nstruct ggml_sycl_unroll {\n template \n void operator()(const Func & f, Args... args) const {\n f(n - 1, args...);\n ggml_sycl_unroll{}(f, args...);\n }\n};\n\ntemplate <>\nstruct ggml_sycl_unroll<1> {\n template \n void operator()(const Func & f, Args... args) const {\n f(0, args...);\n }\n};\n\nstatic __dpct_inline__ sycl::half2 ggml_sycl_hmax2(const sycl::half2 a, const sycl::half2 b) {\n sycl::half2 ret;\n reinterpret_cast(ret.x()) =\n sycl::vec(sycl::fmax(a[0], b[0])).convert()[0];\n reinterpret_cast(ret.y()) =\n sycl::vec(sycl::fmax(a[1], b[1])).convert()[0];\n return ret;\n}\n\nstatic __dpct_inline__ sycl::half ggml_sycl_hmax(const sycl::half a, const sycl::half b) {\n return sycl::vec(\n sycl::fmax(sycl::vec(a).convert()[0],\n sycl::vec(b).convert()[0]))\n .convert()[0];\n}\n\nstatic __dpct_inline__ uint32_t __hgt2_mask(const sycl::half2 a, const sycl::half2 b) {\n const uint32_t mask_low = 0x0000FFFF * (float(a[0]) > float(b[0]));\n const uint32_t mask_high = 0xFFFF0000 * (float(a[1]) > float(b[1]));\n return mask_low | mask_high;\n}\n\nstatic __dpct_inline__ uint32_t fastmodulo(uint32_t n, const sycl::uint3 fastdiv_values) {\n // expects fastdiv_values to contain in (see init_fastdiv_values)\n return n - fastdiv(n, fastdiv_values) * fastdiv_values.z();\n}\n\nstatic bool fast_fp16_available(const int cc) {\n GGML_UNUSED(cc);\n return true; //Intel GPUs always support FP16.\n}\n\nenum class block_reduce_method {\n MAX,\n SUM,\n};\n\ntemplate\nstruct block_reduce_policy;\n\ntemplate \ninline constexpr bool is_any = (std::is_same_v || ...);\n\ntemplate\ninline constexpr bool ggml_sycl_dependent_false_v = false;\n\n#define WARP_32_SIZE 32\n\ntemplate struct block_reduce_policy {\n static T reduce(T val) {\n if constexpr (is_any) {\n return warp_reduce_sum(val);\n } else {\n static_assert(ggml_sycl_dependent_false_v, \"Unsupported type for block reduce sum\");\n }\n }\n\n static T sentinel() {\n if constexpr (std::is_same_v) {\n return 0.0f;\n } else if constexpr (std::is_same_v) {\n return sycl::float2(0.0f, 0.0f);\n } else if constexpr (std::is_same_v) {\n return sycl::half2(0.0f, 0.0f);\n } else if constexpr (std::is_same_v) {\n return 0;\n } else {\n static_assert(ggml_sycl_dependent_false_v, \"Unsupported type for block reduce sum\");\n }\n }\n};\n\ntemplate struct block_reduce_policy {\n static T reduce(T val) {\n if constexpr (is_any) {\n return warp_reduce_max(val);\n } else {\n static_assert(ggml_sycl_dependent_false_v, \"Unsupported type for block reduce max\");\n }\n }\n\n static T sentinel() {\n if constexpr (std::is_same_v) {\n return -INFINITY;\n } else if constexpr (std::is_same_v) {\n return sycl::half2(-INFINITY, -INFINITY);\n } else {\n static_assert(ggml_sycl_dependent_false_v, \"Unsupported type for block reduce max\");\n }\n }\n};\n\n\ntemplate \nstatic T block_reduce(T val, T * shared_vals, int block_size_template) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n val = block_reduce_policy::reduce(val);\n const int block_size = block_size_template == 0 ? item_ct1.get_local_range(2) : block_size_template;\n const int nthreads = item_ct1.get_local_range(2);\n const int nwarps = nthreads / WARP_SIZE;\n\n if (block_size > warp_size) {\n assert((block_size <= 1024) && (block_size % warp_size) == 0);\n const int warp_id = item_ct1.get_local_id(2) / warp_size;\n const int lane_id = item_ct1.get_local_id(2) % warp_size;\n if (lane_id == 0) {\n shared_vals[warp_id] = val;\n }\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n size_t nreduce = nwarps / WARP_SIZE;\n float tmp = 0.f;\n if (lane_id < (static_cast(block_size) / warp_size)) {\n for (size_t i = 0; i < nreduce; i += 1)\n {\n tmp += shared_vals[lane_id + i * WARP_SIZE];\n }\n }\n return block_reduce_policy::reduce(tmp);\n }\n return val;\n}\n\n#endif // GGML_SYCL_COMMON_HPP\n" }, { "path": "src/ggml-sycl/concat.cpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#include \"concat.hpp\"\n\nstatic inline size_t elem_size(ggml_type t) {\n return ggml_type_size(t) / ggml_blck_size(t);\n}\n\ntemplate \nstatic void concat_T_dim0(const T *x, const T *y, T *dst,\n const int ne0, const int ne00,\n const sycl::nd_item<3> &item_ct1) {\n int nidx = item_ct1.get_local_id(2) +\n item_ct1.get_group(2) * item_ct1.get_local_range(2);\n if (nidx >= ne0) {\n return;\n }\n // operation\n int offset_dst = nidx + item_ct1.get_group(1) * ne0 +\n item_ct1.get_group(0) * ne0 * item_ct1.get_group_range(1);\n if (nidx < ne00) { // src0\n int offset_src = nidx + item_ct1.get_group(1) * ne00 +\n item_ct1.get_group(0) * ne00 * item_ct1.get_group_range(1);\n dst[offset_dst] = x[offset_src];\n } else {\n int offset_src =\n nidx - ne00 + item_ct1.get_group(1) * (ne0 - ne00) +\n item_ct1.get_group(0) * (ne0 - ne00) * item_ct1.get_group_range(1);\n dst[offset_dst] = y[offset_src];\n }\n}\n\ntemplate \nstatic void concat_T_dim1(const T *x, const T *y, T *dst,\n const int ne0, const int ne01,\n const sycl::nd_item<3> &item_ct1) {\n int nidx = item_ct1.get_local_id(2) +\n item_ct1.get_group(2) * item_ct1.get_local_range(2);\n if (nidx >= ne0) {\n return;\n }\n // operation\n int offset_dst = nidx + item_ct1.get_group(1) * ne0 +\n item_ct1.get_group(0) * ne0 * item_ct1.get_group_range(1);\n if (item_ct1.get_group(1) < (size_t) ne01) { // src0\n int offset_src =\n nidx + item_ct1.get_group(1) * ne0 + item_ct1.get_group(0) * ne0 * ne01;\n dst[offset_dst] = x[offset_src];\n } else {\n int offset_src =\n nidx + (item_ct1.get_group(1) - ne01) * ne0 +\n item_ct1.get_group(0) * ne0 * (item_ct1.get_group_range(1) - ne01);\n dst[offset_dst] = y[offset_src];\n }\n}\n\ntemplate \nstatic void concat_T_dim2(const T *x, const T *y, T *dst,\n const int ne0, const int ne02,\n const sycl::nd_item<3> &item_ct1) {\n int nidx = item_ct1.get_local_id(2) +\n item_ct1.get_group(2) * item_ct1.get_local_range(2);\n if (nidx >= ne0) {\n return;\n }\n // operation\n int offset_dst = nidx + item_ct1.get_group(1) * ne0 +\n item_ct1.get_group(0) * ne0 * item_ct1.get_group_range(1);\n if (item_ct1.get_group(0) < (size_t) ne02) { // src0\n int offset_src = nidx + item_ct1.get_group(1) * ne0 +\n item_ct1.get_group(0) * ne0 * item_ct1.get_group_range(1);\n dst[offset_dst] = x[offset_src];\n } else {\n int offset_src =\n nidx + item_ct1.get_group(1) * ne0 +\n (item_ct1.get_group(0) - ne02) * ne0 * item_ct1.get_group_range(1);\n dst[offset_dst] = y[offset_src];\n }\n}\n\ntemplate \nstatic void concat_T_sycl(const T *x, const T *y, T *dst,\n int ne00, int ne01, int ne02, int ne0, int ne1,\n int ne2, int dim, queue_ptr stream) {\n int num_blocks = (ne0 + SYCL_CONCAT_BLOCK_SIZE - 1) / SYCL_CONCAT_BLOCK_SIZE;\n sycl::range<3> gridDim(ne2, ne1, num_blocks);\n switch (dim) {\n case 0:\n stream->parallel_for(sycl::nd_range<3>(gridDim * sycl::range<3>(1, 1, SYCL_CONCAT_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CONCAT_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) { concat_T_dim0(x, y, dst, ne0, ne00, item_ct1); });\n break;\n case 1:\n stream->parallel_for(sycl::nd_range<3>(gridDim * sycl::range<3>(1, 1, SYCL_CONCAT_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CONCAT_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) { concat_T_dim1(x, y, dst, ne0, ne01, item_ct1); });\n break;\n // dim >=2 will be dispatched to the default path\n default:\n stream->parallel_for(sycl::nd_range<3>(gridDim * sycl::range<3>(1, 1, SYCL_CONCAT_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CONCAT_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) { concat_T_dim2(x, y, dst, ne0, ne02, item_ct1); });\n break;\n }\n}\n\n// non-contiguous kernel (slow)\ntemplate\nstatic void concat_T_sycl_non_cont(\n queue_ptr stream, const char *src0, const char *src1, char *dst,\n int64_t ne00, int64_t ne01, int64_t ne02, int64_t ne03, uint64_t nb00,\n uint64_t nb01, uint64_t nb02, uint64_t nb03, int64_t /*ne10*/,\n int64_t /*ne11*/, int64_t /*ne12*/, int64_t /*ne13*/, uint64_t nb10,\n uint64_t nb11, uint64_t nb12, uint64_t nb13, int64_t ne0, int64_t ne1,\n int64_t ne2, int64_t ne3, uint64_t nb0, uint64_t nb1, uint64_t nb2,\n uint64_t nb3, int32_t dim) {\n sycl::range<3> gridDim(ne3, ne2, ne1);\n stream->parallel_for(sycl::nd_range<3>(gridDim, sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) {\n int64_t i3 = item_ct1.get_group(0);\n int64_t i2 = item_ct1.get_group(1);\n int64_t i1 = item_ct1.get_group(2);\n\n int64_t o[4] = { 0, 0, 0, 0 };\n o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));\n\n const T * x;\n\n for (int i0 = item_ct1.get_local_id(2); i0 < ne0; i0 += item_ct1.get_local_range(2)) {\n if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n x = (const T *) (src0 + (i3) *nb03 + (i2) *nb02 + (i1) *nb01 + (i0) *nb00);\n } else {\n x = (const T *) (src1 + (i3 - o[3]) * nb13 + (i2 - o[2]) * nb12 + (i1 - o[1]) * nb11 +\n (i0 - o[0]) * nb10);\n }\n\n T *y = (T *)(dst + i3 * nb3 + i2 * nb2 + i1 * nb1 + i0 * nb0);\n\n *y = *x;\n }\n });\n}\n\ntemplate \nvoid concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n queue_ptr stream = ctx.stream();\n\n const int32_t dim = ((int32_t *) dst->op_params)[0];\n\n if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {\n const T * src0_d = (const T *) src0->data;\n const T * src1_d = (const T *) src1->data;\n T * dst_d = (T *) dst->data;\n size_t type_size = elem_size(dst->type);\n if (dim != 3) {\n for (int i3 = 0; i3 < dst->ne[3]; i3++) {\n concat_T_sycl(src0_d + i3 * (src0->nb[3] / type_size), src1_d + i3 * (src1->nb[3] / type_size),\n dst_d + i3 * (dst->nb[3] / type_size), src0->ne[0], src0->ne[1], src0->ne[2], dst->ne[0],\n dst->ne[1], dst->ne[2], dim, stream);\n }\n } else {\n const size_t size0 = ggml_nbytes(src0);\n const size_t size1 = ggml_nbytes(src1);\n\n SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0).wait()));\n SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d + size0 / type_size, src1_d, size1).wait()));\n }\n } else {\n concat_T_sycl_non_cont(stream, (const char *) src0->data, (const char *) src1->data, (char *) dst->data,\n src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0->nb[0], src0->nb[1],\n src0->nb[2], src0->nb[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],\n src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], dst->ne[0], dst->ne[1], dst->ne[2],\n dst->ne[3], dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);\n }\n}\n\nvoid ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n\n switch (dst->type) {\n case GGML_TYPE_F32:\n concat_impl_sycl(ctx, dst);\n break;\n case GGML_TYPE_I32:\n concat_impl_sycl(ctx, dst);\n break;\n default:\n GGML_ASSERT(false && \"ggml_sycl_op_concat: unsupported type\");\n break;\n }\n}\n" }, { "path": "src/ggml-sycl/concat.hpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_CONCAT_HPP\n#define GGML_SYCL_CONCAT_HPP\n\n#include \"common.hpp\"\n\nvoid ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst);\n\n#endif // GGML_SYCL_CONCAT_HPP\n" }, { "path": "src/ggml-sycl/conv.cpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#include \"conv.hpp\"\n\nstatic void conv_transpose_1d_kernel(\n const int s0, const int output_size,\n const int src0_ne0, const int src0_ne1, const int src0_ne2,\n const int src1_ne0, const int dst_ne0,\n const float * src0, const float * src1, float * dst,\n const sycl::nd_item<3> &item_ct1) {\n int global_index = item_ct1.get_local_id(2) +\n item_ct1.get_group(2) * item_ct1.get_local_range(2);\n if (global_index >= output_size) {\n return;\n }\n\n int out_index = global_index / dst_ne0;\n\n float accumulator = 0;\n\n for (int c = 0; c < src0_ne2; c++) {\n int idx = global_index % dst_ne0;\n\n int kernel_offset = (src0_ne0 * src0_ne1 * c) + (out_index * src0_ne0);\n int input_offset = src1_ne0 * c;\n\n for (int i = 0; i < src1_ne0; i++) {\n if (!(idx >= i*s0 && idx < i*s0 + src0_ne0)) {\n continue;\n }\n int weight_idx = idx - i*s0;\n\n float kernel_weight = src0[kernel_offset + weight_idx];\n float input_value = src1[input_offset+i];\n\n accumulator += kernel_weight * input_value;\n }\n }\n dst[global_index] = accumulator;\n}\n\nstatic void conv_transpose_1d_f32_f32_sycl(\n const int s0, const int output_size,\n const int src0_ne0, const int src0_ne1, const int src0_ne2,\n const int src1_ne0, const int dst_ne0,\n const float *src0, const float *src1, float *dst,\n const queue_ptr& stream) {\n\n const int num_blocks = (output_size + SYCL_CONV_TRANPOSE_1D_BLOCK_SIZE - 1) / SYCL_CONV_TRANPOSE_1D_BLOCK_SIZE;\n const sycl::range<3> block_dims(1, 1, SYCL_CONV_TRANPOSE_1D_BLOCK_SIZE);\n const sycl::range<3> block_nums(1, 1, num_blocks);\n stream->parallel_for(\n sycl::nd_range<3>(\n block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) {\n conv_transpose_1d_kernel(\n s0, output_size,\n src0_ne0, src0_ne1, src0_ne2,\n src1_ne0, dst_ne0,\n src0, src1, dst, item_ct1);\n });\n}\n\nvoid ggml_sycl_op_conv_transpose_1d(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);\n const ggml_tensor *src0 = dst->src[0];\n const ggml_tensor *src1 = dst->src[1];\n const float * src0_d = (const float *)src0->data;\n const float * src1_d = (const float *)src1->data;\n\n float * dst_d = (float *)dst->data;\n dpct::queue_ptr stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src1));\n\n const int32_t * opts = (const int32_t *)dst->op_params;\n\n const int s0 = opts[0];\n\n const int64_t output_size = ggml_nelements(dst);\n\n conv_transpose_1d_f32_f32_sycl(s0, output_size,\n src0->ne[0], src0->ne[1], src0->ne[2],\n src1->ne[0], dst->ne[0],\n src0_d, src1_d, dst_d, stream);\n}\n\n" }, { "path": "src/ggml-sycl/conv.hpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_CONV_HPP\n#define GGML_SYCL_CONV_HPP\n\n#include \"common.hpp\"\n\nvoid ggml_sycl_op_conv_transpose_1d(ggml_backend_sycl_context & ctx, ggml_tensor *dst);\n\n#endif // GGML_SYCL_CONV_HPP\n" }, { "path": "src/ggml-sycl/convert.cpp", "content": "#include \"convert.hpp\"\n#include \"dequantize.hpp\"\n#include \"presets.hpp\"\n\n#if defined(__INTEL_LLVM_COMPILER)\n #if __has_include()\n #include \n #define GGML_SYCL_HAS_BF16\n #endif\n#endif\n\ntemplate \nstatic void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k,\n const sycl::nd_item<3> &item_ct1) {\n const int64_t i = 2 * (item_ct1.get_local_range(2) * item_ct1.get_group(2) +\n item_ct1.get_local_id(2));\n\n if (i >= k) {\n return;\n }\n\n const int64_t ib = i/qk; // block index\n const int64_t iqs = (i%qk)/qr; // quant index\n const int64_t iybs = i - i%qk; // y block start index\n const int64_t y_offset = qr == 1 ? 1 : qk/2;\n\n // dequantize\n dfloat2 v;\n dequantize_kernel(vx, ib, iqs, v);\n\n y[iybs + iqs + 0] = v.x();\n y[iybs + iqs + y_offset] = v.y();\n}\n\ntemplate \nstatic void dequantize_block_sycl(const void *__restrict__ vx,\n dst_t *__restrict__ y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t num_blocks = (k + 2*SYCL_DEQUANTIZE_BLOCK_SIZE - 1) / (2*SYCL_DEQUANTIZE_BLOCK_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n stream->parallel_for(\n sycl::nd_range<3>(\n sycl::range<3>(1, 1, num_blocks) *\n sycl::range<3>(1, 1, SYCL_DEQUANTIZE_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_DEQUANTIZE_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block(vx, y, k, item_ct1);\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_q2_K_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n#if QK_K == 256\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 64),\n sycl::range<3>(1, 1, 64)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q2_K(vx, y, item_ct1);\n });\n }\n#else\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q2_K(vx, y, item_ct1);\n });\n }\n\n#endif\n}\n\ntemplate \nstatic void dequantize_row_q3_K_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n#if QK_K == 256\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 64),\n sycl::range<3>(1, 1, 64)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q3_K(vx, y, item_ct1);\n });\n }\n#else\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q3_K(vx, y, item_ct1);\n });\n }\n#endif\n}\n\ntemplate \nstatic void dequantize_row_q4_0_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb32 = k / 32;\n const int64_t nb = (k + 255) / 256;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q4_0(vx, y, nb32, item_ct1);\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n int constexpr WARP_K = WARP_SIZE * QK4_0;\n const int n_warp = (k + WARP_K - 1) / WARP_K;\n GGML_ASSERT(k % 2 == 0);\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) *\n sycl::range<3>(1, 1, WARP_SIZE),\n sycl::range<3>(1, 1, WARP_SIZE)),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{\n dequantize_block_q4_0_reorder(vx, y, k, item_ct1);\n });\n\n}\n\ntemplate \nstatic void dequantize_row_q4_1_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb32 = k / 32;\n const int64_t nb = (k + 255) / 256;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q4_1(vx, y, nb32, item_ct1);\n });\n }\n}\n\n\ntemplate \nstatic void dequantize_row_q4_K_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n sycl::local_accessor scale_local_acc(sycl::range<1>(12), cgh);\n cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q4_K(vx, y, get_pointer(scale_local_acc), item_ct1);\n });\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_q4_K_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n const size_t local_size = 32;\n const size_t global_size = nb * local_size;\n\n dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });\n\n stream->submit([&](sycl::handler & cgh) {\n sycl::local_accessor scale_local_acc(sycl::range<1>(12), cgh);\n\n cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(global_size), sycl::range<1>(local_size)),\n [=](sycl::nd_item<1> item_ct1) {\n dequantize_block_q4_K_reorder(vx, y, get_pointer(scale_local_acc), item_ct1, nb);\n });\n });\n}\n\ntemplate \nstatic void dequantize_row_q5_K_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n#if QK_K == 256\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 64),\n sycl::range<3>(1, 1, 64)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q5_K(vx, y, item_ct1);\n });\n }\n#else\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q5_K(vx, y, item_ct1);\n });\n }\n\n#endif\n}\n\ntemplate \nstatic void dequantize_row_q6_K_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n#if QK_K == 256\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 64),\n sycl::range<3>(1, 1, 64)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q6_K(vx, y, item_ct1);\n });\n }\n#else\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_q6_K(vx, y, item_ct1);\n });\n }\n\n#endif\n}\n\ntemplate \nstatic void dequantize_row_q6_K_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n\n dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });\n\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)),\n [=](sycl::nd_item<3> item_ct1) { dequantize_block_q6_K_reorder(vx, y, item_ct1, nb); });\n}\n\ntemplate \nstatic void dequantize_row_iq1_s_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_iq1_s(\n vx, y, item_ct1, iq1s_grid_gpu\n );\n });\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_iq1_m_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_iq1_m(\n vx, y, item_ct1, iq1s_grid_gpu\n );\n });\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_iq2_xxs_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_iq2_xxs(\n vx, y, item_ct1, iq2xxs_grid,\n ksigns_iq2xs, kmask_iq2xs);\n });\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_iq2_xs_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_iq2_xs(\n vx, y, item_ct1, iq2xs_grid,\n ksigns_iq2xs, kmask_iq2xs);\n });\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_iq2_s_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_iq2_s(vx, y, item_ct1);\n });\n });\n }\n}\n\n\ntemplate \nstatic void dequantize_row_iq3_xxs_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_iq3_xxs(\n vx, y, item_ct1, iq3xxs_grid,\n ksigns_iq2xs, kmask_iq2xs);\n });\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_iq3_s_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = k / QK_K;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_iq3_s(\n vx, y, item_ct1, kmask_iq2xs, iq3s_grid);\n });\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_iq4_xs_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = (k + QK_K - 1) / QK_K;\n#if QK_K == 64\n dequantize_row_iq4_nl_sycl(vx, y, k, stream);\n#else\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_iq4_xs(vx, y, item_ct1);\n });\n });\n }\n#endif\n}\n\ntemplate \nstatic void dequantize_row_iq4_nl_sycl(const void *vx, dst_t *y, const int64_t k,\n dpct::queue_ptr stream) {\n const int64_t nb = (k + QK_K - 1) / QK_K;\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *\n sycl::range<3>(1, 1, 32),\n sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_iq4_nl(vx, y, item_ct1);\n });\n });\n }\n}\n\ntemplate \nstatic void dequantize_row_mxfp4_sycl(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {\n const int nb = (k + QK_K - 1) / QK_K;\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 32), sycl::range<3>(1, 1, 32)),\n [=](sycl::nd_item<3> item_ct1) {\n dequantize_block_mxfp4(vx, y, item_ct1);\n });\n}\n\ntemplate \nstatic void dequantize_block_nc(const void * __restrict__ vx, dst_t * __restrict__ y,\n const int64_t ne00, const int64_t ne01, const int64_t ne02,\n const int64_t s01, const int64_t s02, const int64_t s03) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const int64_t i00 = 2 * (int64_t(item_ct1.get_local_range(2)) * item_ct1.get_group(2) + item_ct1.get_local_id(2));\n\n if (i00 >= ne00) {\n return;\n }\n\n const int64_t i01 = item_ct1.get_group(1);\n const int64_t i02 = item_ct1.get_group(0) % ne02;\n const int64_t i03 = item_ct1.get_group(0) / ne02;\n\n const int64_t ibx0 = i03*s03 + i02*s02 + i01*s01;\n\n const int64_t ib = ibx0 + i00/qk; // block index\n const int64_t iqs = (i00%qk)/qr; // quant index\n const int64_t iybs = i00 - i00%qk; // y block start index\n const int64_t y_offset = qr == 1 ? 1 : qk/2;\n\n // dequantize\n #ifdef GGML_SYCL_F16\n sycl::half2 v;\n #else\n sycl::float2 v;\n #endif\n\n dequantize_kernel(vx, ib, iqs, v);\n\n const int64_t iy0 = ((i03*ne02 + i02)*ne01 + i01)*ne00 + iybs + iqs;\n y[iy0 + 0] = ggml_sycl_cast(v.x());\n y[iy0 + y_offset] = ggml_sycl_cast(v.y());\n}\n\n\ntemplate \nstatic void dequantize_block_nc_sycl(const void * vx,\n dst_t * y,\n const int64_t ne00,\n const int64_t ne01,\n const int64_t ne02,\n const int64_t ne03,\n const int64_t s01,\n const int64_t s02,\n const int64_t s03,\n dpct::queue_ptr stream) {\n const dpct::dim3 num_blocks((ne00 + 2 * SYCL_DEQUANTIZE_BLOCK_SIZE - 1) / (2 * SYCL_DEQUANTIZE_BLOCK_SIZE), ne01,\n ne02 * ne03);\n stream->parallel_for(sycl::nd_range<3>(num_blocks * sycl::range<3>(1, 1, SYCL_DEQUANTIZE_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_DEQUANTIZE_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n GGML_UNUSED(item_ct1);\n dequantize_block_nc(vx, y, ne00, ne01, ne02, s01, s02, s03);\n });\n}\ntemplate \nstatic void convert_unary_nc(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t ne00, const int64_t ne01,\n const int64_t ne02, const int64_t s01, const int64_t s02, const int64_t s03,\n const sycl::nd_item<3> & item_ct1) {\n\n const int64_t work_group_size = item_ct1.get_local_range(2);\n const int64_t global_id = item_ct1.get_local_id(2) + work_group_size * item_ct1.get_group(2);\n\n const int64_t i01 = item_ct1.get_group(1);\n const int64_t i02 = item_ct1.get_group(0) % ne02;\n const int64_t i03 = item_ct1.get_group(0) / ne02;\n\n // make each work-item deal with more elements since sycl global range can not exceed max int\n const src_t * x = static_cast(vx);\n const int64_t ix = i03 * s03 + i02 * s02 + i01 * s01;\n const int64_t iy = ((i03 * ne02 + i02) * ne01 + i01) * ne00;\n\n#pragma unroll\n for (int64_t i00 = global_id; i00 < ne00; i00 += work_group_size * item_ct1.get_group_range(2)) {\n y[iy + i00] = static_cast(x[ix + i00]);\n }\n}\n\ntemplate \nstatic void convert_unary_nc_sycl(const void * __restrict__ vx, dst_t * __restrict__ y,\n const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n const int64_t s01, const int64_t s02, const int64_t s03, dpct::queue_ptr queue) {\n dpct::has_capability_or_fail(queue->get_device(), { sycl::aspect::fp16 });\n\n sycl::range<3> global_size(ne02 * ne03, ne01, ceil_div(ne00, SYCL_DEQUANTIZE_BLOCK_SIZE));\n\n // decrease global range when it exceeds the max int\n // TODO: Downsample logic is separated from the kernel, a rewrite is desirable\n int64_t downsized_workgroup = downsample_sycl_global_range(global_size[0], SYCL_DEQUANTIZE_BLOCK_SIZE);\n sycl::range<3> workgroup_size(1, 1, downsized_workgroup);\n\n queue->parallel_for(sycl::nd_range<3>(global_size * workgroup_size, workgroup_size), [=](sycl::nd_item<3> item_ct1) {\n convert_unary_nc(vx, y, ne00, ne01, ne02, s01, s02, s03, item_ct1);\n });\n}\n\ntemplate \nstatic void convert_unary_sycl(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr queue) {\n convert_unary_nc_sycl(vx, y, k, 1, 1, 1, k, k, k, queue);\n}\n\n\nto_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {\n switch (type) {\n case GGML_TYPE_Q4_0:\n if (dst->src[0]->extra &&\n ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {\n return dequantize_row_q4_0_sycl_reorder;\n } else {\n return dequantize_block_sycl;\n }\n case GGML_TYPE_Q4_1:\n return dequantize_block_sycl;\n case GGML_TYPE_Q5_0:\n return dequantize_block_sycl;\n case GGML_TYPE_Q5_1:\n return dequantize_block_sycl;\n case GGML_TYPE_Q8_0:\n return dequantize_block_sycl;\n case GGML_TYPE_Q2_K:\n return dequantize_row_q2_K_sycl;\n case GGML_TYPE_Q3_K:\n return dequantize_row_q3_K_sycl;\n case GGML_TYPE_Q4_K:\n if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {\n return dequantize_row_q4_K_sycl_reorder;\n } else {\n return dequantize_row_q4_K_sycl;\n }\n case GGML_TYPE_Q5_K:\n return dequantize_row_q5_K_sycl;\n case GGML_TYPE_Q6_K:\n if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {\n return dequantize_row_q6_K_sycl_reorder;\n } else {\n return dequantize_row_q6_K_sycl;\n }\n case GGML_TYPE_IQ1_S:\n return dequantize_row_iq1_s_sycl;\n case GGML_TYPE_IQ1_M:\n return dequantize_row_iq1_m_sycl;\n case GGML_TYPE_IQ2_XXS:\n return dequantize_row_iq2_xxs_sycl;\n case GGML_TYPE_IQ2_XS:\n return dequantize_row_iq2_xs_sycl;\n case GGML_TYPE_IQ2_S:\n return dequantize_row_iq2_s_sycl;\n case GGML_TYPE_IQ3_XXS:\n return dequantize_row_iq3_xxs_sycl;\n case GGML_TYPE_IQ3_S:\n return dequantize_row_iq3_s_sycl;\n case GGML_TYPE_IQ4_XS:\n return dequantize_row_iq4_xs_sycl;\n case GGML_TYPE_IQ4_NL:\n return dequantize_row_iq4_nl_sycl;\n case GGML_TYPE_MXFP4:\n return dequantize_row_mxfp4_sycl;\n case GGML_TYPE_F32:\n return convert_unary_sycl;\n#ifdef GGML_SYCL_HAS_BF16\n case GGML_TYPE_BF16:\n return convert_unary_sycl;\n#endif\n default:\n return nullptr;\n }\n}\n\nto_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) {\n switch (type) {\n case GGML_TYPE_Q4_0:\n if (dst->src[0]->extra &&\n ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {\n return dequantize_row_q4_0_sycl_reorder;\n } else {\n return dequantize_row_q4_0_sycl;\n }\n case GGML_TYPE_Q4_1:\n return dequantize_row_q4_1_sycl;\n case GGML_TYPE_Q5_0:\n return dequantize_block_sycl;\n case GGML_TYPE_Q5_1:\n return dequantize_block_sycl;\n case GGML_TYPE_Q8_0:\n return dequantize_block_sycl;\n case GGML_TYPE_Q2_K:\n return dequantize_row_q2_K_sycl;\n case GGML_TYPE_Q3_K:\n return dequantize_row_q3_K_sycl;\n case GGML_TYPE_Q4_K:\n if (dst->src[0]->extra &&\n ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {\n return dequantize_row_q4_K_sycl_reorder;\n } else {\n return dequantize_row_q4_K_sycl;\n }\n case GGML_TYPE_Q5_K:\n return dequantize_row_q5_K_sycl;\n case GGML_TYPE_Q6_K:\n if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {\n return dequantize_row_q6_K_sycl_reorder;\n } else {\n return dequantize_row_q6_K_sycl;\n }\n case GGML_TYPE_IQ1_S:\n return dequantize_row_iq1_s_sycl;\n case GGML_TYPE_IQ1_M:\n return dequantize_row_iq1_m_sycl;\n case GGML_TYPE_IQ2_XXS:\n return dequantize_row_iq2_xxs_sycl;\n case GGML_TYPE_IQ2_XS:\n return dequantize_row_iq2_xs_sycl;\n case GGML_TYPE_IQ2_S:\n return dequantize_row_iq2_s_sycl;\n case GGML_TYPE_IQ3_XXS:\n return dequantize_row_iq3_xxs_sycl;\n case GGML_TYPE_IQ3_S:\n return dequantize_row_iq3_s_sycl;\n case GGML_TYPE_IQ4_XS:\n return dequantize_row_iq4_xs_sycl;\n case GGML_TYPE_IQ4_NL:\n return dequantize_row_iq4_nl_sycl;\n case GGML_TYPE_MXFP4:\n return dequantize_row_mxfp4_sycl;\n case GGML_TYPE_F16:\n return convert_unary_sycl;\n#ifdef GGML_SYCL_HAS_BF16\n case GGML_TYPE_BF16:\n return convert_unary_sycl;\n#endif\n default:\n return nullptr;\n }\n}\n\n\nto_fp16_nc_sycl_t ggml_get_to_fp16_nc_sycl(ggml_type type) {\n switch (type) {\n case GGML_TYPE_F32:\n return convert_unary_nc_sycl;\n#ifdef GGML_SYCL_HAS_BF16\n case GGML_TYPE_BF16:\n return convert_unary_nc_sycl;\n#endif\n case GGML_TYPE_Q4_0:\n return dequantize_block_nc_sycl;\n case GGML_TYPE_Q4_1:\n return dequantize_block_nc_sycl;\n case GGML_TYPE_Q5_0:\n return dequantize_block_nc_sycl;\n case GGML_TYPE_Q5_1:\n return dequantize_block_nc_sycl;\n case GGML_TYPE_Q8_0:\n return dequantize_block_nc_sycl;\n default:\n return nullptr;\n }\n}\n" }, { "path": "src/ggml-sycl/convert.hpp", "content": "//\n// MIT license\n// Copyright (C) 2025 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_CONVERT_HPP\n#define GGML_SYCL_CONVERT_HPP\n\n#include \"common.hpp\"\n\ntemplate \nusing to_t_sycl_t = void (*)(const void * __restrict__ x, T * __restrict__ y, int64_t k, dpct::queue_ptr stream);\ntypedef to_t_sycl_t to_fp32_sycl_t;\ntypedef to_t_sycl_t to_fp16_sycl_t;\n\nto_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst);\nto_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor * dst);\n\n// Nc = Non-contiguous\ntemplate \nusing to_t_nc_sycl_t = void (*)(const void * x, T * y, int64_t ne00, int64_t ne01, int64_t ne02, int64_t ne03,\n int64_t s01, int64_t s02, int64_t s03, dpct::queue_ptr queue);\n\ntypedef to_t_nc_sycl_t to_fp16_nc_sycl_t;\nto_fp16_nc_sycl_t ggml_get_to_fp16_nc_sycl(ggml_type type);\n\ntemplate\n inline dst_t ggml_sycl_cast(src_t x) {\n if constexpr (std::is_same_v) {\n return x;\n } else if constexpr (std::is_same_v) {\n return sycl::ext::oneapi::bfloat16(float(x));\n } else if constexpr (std::is_same_v) {\n return static_cast(x);\n } else if constexpr (std::is_same_v && std::is_same_v) {\n return x.template convert();\n } else if constexpr (std::is_same_v &&\n std::is_same_v>) {\n return {x.x, x.y};\n } else if constexpr(std::is_same_v) {\n return int32_t(x);\n } else {\n return float(x);\n }\n}\n\n\n#endif // GGML_SYCL_CONVERT_HPP\n" }, { "path": "src/ggml-sycl/count-equal.cpp", "content": "#include \"count-equal.hpp\"\n\n#include \n\ntemplate \nstatic void count_equal(const T *__restrict__ x, const T *__restrict__ y,\n int64_t *__restrict__ dst, const int64_t dk,\n const int64_t k) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const int64_t i0 = (int64_t)item_ct1.get_group(2) * dk;\n const int64_t i1 = sycl::min(i0 + dk, k);\n\n int nequal = 0;\n\n for (int64_t i = i0 + item_ct1.get_local_id(2); i < i1; i += WARP_SIZE) {\n const T xi = x[i];\n const T yi = y[i];\n nequal += xi == yi;\n }\n\n nequal = warp_reduce_sum(nequal);\n\n if (item_ct1.get_local_id(2) != 0) {\n return;\n }\n\n dpct::atomic_fetch_add(\n (int *)dst, nequal);\n}\n\nvoid ggml_sycl_count_equal(ggml_backend_sycl_context &ctx, ggml_tensor *dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n GGML_ASSERT(src0->type == src1->type);\n GGML_ASSERT( dst->type == GGML_TYPE_I64);\n\n GGML_ASSERT(ggml_are_same_shape(src0, src1));\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(src1));\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n int64_t * dst_d = (int64_t *) dst->data;\n\n dpct::queue_ptr stream = ctx.stream();\n const int id = get_current_device_id();\n const int nsm = ggml_sycl_info().devices[id].nsm;\n\n const int64_t ne = ggml_nelements(src0);\n GGML_ASSERT(ne < (1 << 30) && \"atomicAdd implementation only supports int\");\n const int64_t dne =\n GGML_PAD((ne + 4 * nsm - 1) / (4 * nsm), SYCL_COUNT_EQUAL_CHUNK_SIZE);\n\n SYCL_CHECK(CHECK_TRY_ERROR(stream->memset(dst_d, 0, ggml_nbytes(dst))));\n\n const dpct::dim3 block_dims(WARP_SIZE, 1, 1);\n const dpct::dim3 block_nums(\n std::min((int64_t)4 * nsm, (ne + SYCL_COUNT_EQUAL_CHUNK_SIZE - 1) /\n SYCL_COUNT_EQUAL_CHUNK_SIZE),\n 1, 1);\n\n switch (src0->type) {\n case GGML_TYPE_I32: {\n const int *src0_d = (const int *)src0->data;\n const int *src1_d = (const int *)src1->data;\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) {\n count_equal(src0_d, src1_d, dst_d, dne, ne);\n GGML_UNUSED(item_ct1);\n });\n\n } break;\n default:\n GGML_ASSERT(false);\n break;\n }\n}\n" }, { "path": "src/ggml-sycl/count-equal.hpp", "content": "#ifndef GGML_SYCL_COUNT_EQUAL_HPP\n#define GGML_SYCL_COUNT_EQUAL_HPP\n#include \"common.hpp\"\n\n#define SYCL_COUNT_EQUAL_CHUNK_SIZE 128\n\nvoid ggml_sycl_count_equal(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\n#endif //GGML_SYCL_COUNT_EQUAL_HPP\n" }, { "path": "src/ggml-sycl/cpy.cpp", "content": "#include \"cpy.hpp\"\n\n#include \n\n#include \"dequantize.hpp\"\n#include \"ggml-sycl/common.hpp\"\n#include \"ggml-sycl/presets.hpp\"\n#include \"ggml.h\"\n\n\nstatic void cpy_1_f32_f32(const char * cxi, char * cdsti) {\n const float * xi = (const float *) cxi;\n float * dsti = (float *) cdsti;\n\n *dsti = *xi;\n}\n\nstatic void cpy_1_f32_f16(const char * cxi, char * cdsti) {\n const float * xi = (const float *) cxi;\n sycl::half * dsti = (sycl::half *) cdsti;\n\n *dsti = sycl::vec(*xi).convert()[0];\n}\n\nstatic void cpy_1_f16_f16(const char * cxi, char * cdsti) {\n const sycl::half * xi = (const sycl::half *) cxi;\n sycl::half * dsti = (sycl::half *) cdsti;\n\n *dsti = *xi;\n}\n\nstatic void cpy_1_f16_f32(const char * cxi, char * cdsti) {\n const sycl::half * xi = (const sycl::half *) cxi;\n float * dsti = (float *) cdsti;\n\n *dsti = *xi;\n}\n\nstatic void cpy_1_i16_i16(const char * cxi, char * cdsti) {\n const int16_t * xi = (const int16_t *) cxi;\n int16_t * dsti = (int16_t *) cdsti;\n\n *dsti = *xi;\n}\n\nstatic void cpy_1_i32_i32(const char * cxi, char * cdsti) {\n const int32_t * xi = (const int32_t *) cxi;\n int32_t * dsti = (int32_t *) cdsti;\n\n *dsti = *xi;\n}\n\ntemplate \nstatic void cpy_f32_f16(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02,\n const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11,\n const int ne12, const int nb10, const int nb11, const int nb12, const int nb13,\n const sycl::nd_item<3> & item_ct1) {\n const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2);\n\n if (i >= ne) {\n return;\n }\n\n // determine indices i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor\n // then combine those indices with the corresponding byte offsets to get the total offsets\n const int i03 = i / (ne00 * ne01 * ne02);\n const int i02 = (i - i03 * ne00 * ne01 * ne02) / (ne00 * ne01);\n const int i01 = (i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00) / ne00;\n const int i00 = i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00 - i01 * ne00;\n const int x_offset = i00 * nb00 + i01 * nb01 + i02 * nb02 + i03 * nb03;\n\n const int i13 = i / (ne10 * ne11 * ne12);\n const int i12 = (i - i13 * ne10 * ne11 * ne12) / (ne10 * ne11);\n const int i11 = (i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11) / ne10;\n const int i10 = i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11 - i11 * ne10;\n const int dst_offset = i10 * nb10 + i11 * nb11 + i12 * nb12 + i13 * nb13;\n\n cpy_1(cx + x_offset, cdst + dst_offset);\n}\n\n\n/* quantized type same copy */\ntemplate\nstatic void cpy_blck_q_q(const char * cxi, char * cdsti) {\n const T * xi = (const T *) cxi;\n T * dsti = (T *) cdsti;\n *dsti = *xi;\n}\n\n\nstatic void cpy_blck_q8_0_f32(const char * cxi, char * cdsti) {\n float * cdstf = (float *) (cdsti);\n\n for (int j = 0; j < QK8_0; j += 2) {\n dfloat2 dq;\n dequantize_q8_0(cxi, 0, j, dq);\n *(cdstf + j) = dq.x();\n *(cdstf + j + 1) = dq.y();\n }\n}\n\n\n\ntemplate static void cpy_blck_q_f32(const char * cxi, char * cdsti) {\n float * cdstf = (float *) (cdsti);\n\n for (int j = 0; j < qk / 2; j++) {\n dfloat2 dq;\n dequant(cxi, 0, j, dq);\n *(cdstf + j) = dq.x();\n *(cdstf + j + qk / 2) = dq.y();\n }\n}\n\n\ntemplate \nstatic void cpy_q_q(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02,\n const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11,\n const int ne12, const int nb10, const int nb11, const int nb12, const int nb13,\n const sycl::nd_item<3> & item_ct1) {\n const int i = (item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2)) * qk;\n\n if (i >= ne) {\n return;\n }\n\n const int i03 = i / (ne00 * ne01 * ne02);\n const int i02 = (i - i03 * ne00 * ne01 * ne02) / (ne00 * ne01);\n const int i01 = (i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00) / ne00;\n const int i00 = i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00 - i01 * ne00;\n const int x_offset = (i00 / qk) * nb00 + i01 * nb01 + i02 * nb02 + i03 * nb03;\n\n\n const int i13 = i / (ne10 * ne11 * ne12);\n const int i12 = (i - i13 * ne10 * ne11 * ne12) / (ne10 * ne11);\n const int i11 = (i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11) / ne10;\n const int i10 = i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11 - i11 * ne10;\n const int dst_offset = (i10 / qk) * nb10 + i11 * nb11 + i12 * nb12 + i13 * nb13;\n\n cpy_blck_q_q(cx + x_offset, cdst + dst_offset);\n}\n\ntemplate \nstatic void cpy_f32_q(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02,\n const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11,\n const int ne12, const int nb10, const int nb11, const int nb12, const int nb13,\n const sycl::nd_item<3> & item_ct1) {\n const int i = (item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2)) * qk;\n\n if (i >= ne) {\n return;\n }\n\n\n const int i03 = i / (ne00 * ne01 * ne02);\n const int i02 = (i - i03 * ne00 * ne01 * ne02) / (ne00 * ne01);\n const int i01 = (i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00) / ne00;\n const int i00 = i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00 - i01 * ne00;\n const int x_offset = i00 * nb00 + i01 * nb01 + i02 * nb02 + i03 * nb03;\n\n const int i13 = i / (ne10 * ne11 * ne12);\n const int i12 = (i - i13 * ne10 * ne11 * ne12) / (ne10 * ne11);\n const int i11 = (i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11) / ne10;\n const int i10 = i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11 - i11 * ne10;\n const int dst_offset = (i10 / qk) * nb10 + i11 * nb11 + i12 * nb12 + i13 * nb13;\n\n cpy_blck(cx + x_offset, cdst + dst_offset);\n}\n\ntemplate \nstatic void cpy_q_f32(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02,\n const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11,\n const int ne12, const int nb10, const int nb11, const int nb12, const int nb13,\n const sycl::nd_item<3> & item_ct1) {\n const int i = (item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2)) * qk;\n\n if (i >= ne) {\n return;\n }\n\n const int i03 = i / (ne00 * ne01 * ne02);\n const int i02 = (i - i03 * ne00 * ne01 * ne02) / (ne00 * ne01);\n const int i01 = (i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00) / ne00;\n const int i00 = i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00 - i01 * ne00;\n const int x_offset = (i00 / qk) * nb00 + i01 * nb01 + i02 * nb02 + i03 * nb03;\n\n const int i13 = i / (ne10 * ne11 * ne12);\n const int i12 = (i - i13 * ne10 * ne11 * ne12) / (ne10 * ne11);\n const int i11 = (i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11) / ne10;\n const int i10 = i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11 - i11 * ne10;\n const int dst_offset = i10 * nb10 + i11 * nb11 + i12 * nb12 + i13 * nb13;\n\n cpy_blck(cx + x_offset, cdst + dst_offset);\n}\n\nstatic void ggml_cpy_f16_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE;\n {\n dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });\n\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12,\n nb10, nb11, nb12, nb13, item_ct1);\n });\n }\n}\n\nstatic void ggml_cpy_f32_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE;\n {\n dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });\n\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12,\n nb10, nb11, nb12, nb13, item_ct1);\n });\n }\n}\n\nstatic void ggml_cpy_f32_f16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE;\n {\n dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });\n\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12,\n nb10, nb11, nb12, nb13, item_ct1);\n });\n }\n}\n\nstatic void ggml_cpy_f32_q8_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n GGML_ASSERT(ne % QK8_0 == 0);\n const int num_blocks = ne / QK8_0;\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\nstatic void ggml_cpy_q8_0_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = ne;\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_q_f32(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\nstatic void ggml_cpy_f32_q4_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n GGML_ASSERT(ne % QK4_0 == 0);\n const int num_blocks = ne / QK4_0;\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\nstatic void ggml_cpy_q4_0_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = ne;\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) {\n cpy_q_f32, QK4_0>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02,\n nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13,\n item_ct1);\n });\n}\n\nstatic void ggml_cpy_f32_q4_1_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n GGML_ASSERT(ne % QK4_1 == 0);\n const int num_blocks = ne / QK4_1;\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\nstatic void ggml_cpy_q4_1_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = ne;\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) {\n cpy_q_f32, QK4_1>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02,\n nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13,\n item_ct1);\n });\n}\n\nstatic void ggml_cpy_f32_q5_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n GGML_ASSERT(ne % QK5_0 == 0);\n const int num_blocks = ne / QK5_0;\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\nstatic void ggml_cpy_q5_0_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = ne;\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) {\n cpy_q_f32, QK5_0>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02,\n nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13,\n item_ct1);\n });\n}\n\nstatic void ggml_cpy_f32_q5_1_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n GGML_ASSERT(ne % QK5_1 == 0);\n const int num_blocks = ne / QK5_1;\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\nstatic void ggml_cpy_q5_1_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = ne;\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) {\n cpy_q_f32, QK5_1>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02,\n nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13,\n item_ct1);\n });\n}\n\nstatic void ggml_cpy_f32_iq4_nl_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n GGML_ASSERT(ne % QK4_NL == 0);\n const int num_blocks = ne / QK4_NL;\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11,\n ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\nstatic void ggml_cpy_f16_f16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE;\n {\n dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });\n\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12,\n nb10, nb11, nb12, nb13, item_ct1);\n });\n }\n}\n\nstatic void ggml_cpy_i16_i16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE;\n {\n // dpct::has_capability_or_fail(stream->get_device(),\n // {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12,\n nb10, nb11, nb12, nb13, item_ct1);\n });\n }\n}\n\nstatic void ggml_cpy_i32_i32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE;\n {\n // dpct::has_capability_or_fail(stream->get_device(),\n // {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12,\n nb10, nb11, nb12, nb13, item_ct1);\n });\n }\n}\n\nstatic void ggml_cpy_q8_0_q8_0(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) {\n cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\n\nstatic void ggml_cpy_q5_0_q5_0(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) {\n cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\n\nstatic void ggml_cpy_q5_1_q5_1(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE);\n\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) {\n cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\n\nstatic void ggml_cpy_q4_0_q4_0(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) {\n cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\n\nstatic void ggml_cpy_q4_1_q4_1(const char * cx, char * cdst, const int ne, const int ne00, const int ne01,\n const int ne02, const int nb00, const int nb01, const int nb02, const int nb03,\n const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,\n const int nb12, const int nb13, queue_ptr stream) {\n\n const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) {\n cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1);\n });\n}\n\nvoid ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1) try {\n // Unlike other operators ggml_sycl_cpy takes 2 distinct tensors instead of a dst ggml_tensor and rely on its src field\n scope_op_debug_print scope_dbg_print(__func__, src1, /*num_src=*/0, debug_get_tensor_str(\"\\tsrc0\", src0));\n const int64_t ne = ggml_nelements(src0);\n GGML_ASSERT(ne == ggml_nelements(src1));\n\n GGML_TENSOR_BINARY_OP_LOCALS01;\n\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n queue_ptr main_stream = ctx.stream();\n\n char * src0_ddc = (char *) src0->data;\n char * src1_ddc = (char *) src1->data;\n if ((src0->type == src1->type) && (ggml_is_contiguous(src0) && ggml_is_contiguous(src1))) {\n GGML_SYCL_DEBUG(\"%s: memcpy path\\n\", __func__);\n main_stream->memcpy(src1_ddc, src0_ddc, ggml_nbytes(src0));\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_f32_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {\n ggml_cpy_f32_f16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {\n ggml_cpy_f32_q8_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {\n ggml_cpy_f32_q4_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {\n ggml_cpy_f32_q4_1_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_f16_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {\n ggml_cpy_f16_f16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) {\n ggml_cpy_i16_i16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32) {\n ggml_cpy_i32_i32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q4_0 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q4_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q4_1 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q4_1_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q8_0 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q8_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) {\n ggml_cpy_f32_q5_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q5_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {\n ggml_cpy_f32_q5_1_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q5_1 && src1->type == GGML_TYPE_F32) {\n ggml_cpy_q5_1_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10,\n nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {\n ggml_cpy_f32_iq4_nl_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12,\n nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q8_0 && src1->type == GGML_TYPE_Q8_0) {\n ggml_cpy_q8_0_q8_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_Q5_0) {\n ggml_cpy_q5_0_q5_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q5_1 && src1->type == GGML_TYPE_Q5_1) {\n ggml_cpy_q5_1_q5_1(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q4_0 && src1->type == GGML_TYPE_Q4_0) {\n ggml_cpy_q4_0_q4_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else if (src0->type == GGML_TYPE_Q4_1 && src1->type == GGML_TYPE_Q4_1) {\n ggml_cpy_q4_1_q4_1(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n } else {\n GGML_LOG_ERROR(\"%s: unsupported type combination (%s to %s)\\n\", __func__, ggml_type_name(src0->type),\n ggml_type_name(src1->type));\n GGML_ABORT(\"fatal error\");\n }\n} catch (const sycl::exception & exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__ << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nvoid ggml_sycl_dup(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_cpy(ctx, dst->src[0], dst);\n}\n" }, { "path": "src/ggml-sycl/cpy.hpp", "content": "#ifndef GGML_SYCL_CPY_HPP\n#define GGML_SYCL_CPY_HPP\n\n#include \"common.hpp\"\n#include \n\ntypedef void (*cpy_kernel_t)(const char * cx, char * cdst);\n\n__dpct_inline__ int best_index_int8(int n, const int8_t * val, float x) {\n if (x <= val[0]) {\n return 0;\n }\n if (x >= val[n - 1]) {\n return n - 1;\n }\n int ml = 0, mu = n - 1;\n while (mu - ml > 1) {\n int mav = (ml + mu) / 2;\n if (x < val[mav]) {\n mu = mav;\n } else {\n ml = mav;\n }\n }\n return x - val[mu - 1] < val[mu] - x ? mu - 1 : mu;\n}\n\ninline void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) {\n const float * xi = (const float *) cxi;\n block_q8_0 * dsti = (block_q8_0 *) cdsti;\n\n float amax = 0.0f; // absolute max\n\n for (int j = 0; j < QK8_0; j++) {\n const float v = xi[j];\n amax = sycl::fmax(amax, sycl::fabs((float) v));\n }\n\n const float d = amax / ((1 << 7) - 1);\n const float id = d ? 1.0f / d : 0.0f;\n\n dsti->d = d;\n\n for (int j = 0; j < QK8_0; ++j) {\n const float x0 = xi[j] * id;\n\n dsti->qs[j] = sycl::round((float) x0);\n }\n}\n\ninline void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {\n const float * xi = (const float *) cxi;\n block_q4_0 * dsti = (block_q4_0 *) cdsti;\n\n float amax = 0.0f;\n float vmax = 0.0f;\n\n for (int j = 0; j < QK4_0; ++j) {\n const float v = xi[j];\n if (amax < sycl::fabs((float) v)) {\n amax = sycl::fabs((float) v);\n vmax = v;\n }\n }\n\n const float d = vmax / -8;\n const float id = d ? 1.0f / d : 0.0f;\n\n dsti->d = d;\n\n for (int j = 0; j < QK4_0 / 2; ++j) {\n const float x0 = xi[0 + j] * id;\n const float x1 = xi[QK4_0 / 2 + j] * id;\n\n const uint8_t xi0 = dpct::min(15, (int8_t) (x0 + 8.5f));\n const uint8_t xi1 = dpct::min(15, (int8_t) (x1 + 8.5f));\n\n dsti->qs[j] = xi0;\n dsti->qs[j] |= xi1 << 4;\n }\n}\n\ninline void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {\n const float * xi = (const float *) cxi;\n block_q4_1 * dsti = (block_q4_1 *) cdsti;\n\n float vmin = FLT_MAX;\n float vmax = -FLT_MAX;\n\n for (int j = 0; j < QK4_1; ++j) {\n const float v = xi[j];\n\n vmin = sycl::min(v, vmin);\n vmax = sycl::max(v, vmax);\n }\n\n const float d = (vmax - vmin) / ((1 << 4) - 1);\n const float id = d ? 1.0f / d : 0.0f;\n\n dsti->dm.x() = d;\n dsti->dm.y() = vmin;\n\n for (int j = 0; j < QK4_1 / 2; ++j) {\n const float x0 = (xi[0 + j] - vmin) * id;\n const float x1 = (xi[QK4_1 / 2 + j] - vmin) * id;\n\n const uint8_t xi0 = dpct::min(15, (int8_t) (x0 + 0.5f));\n const uint8_t xi1 = dpct::min(15, (int8_t) (x1 + 0.5f));\n\n dsti->qs[j] = xi0;\n dsti->qs[j] |= xi1 << 4;\n }\n}\n\ninline void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {\n const float * xi = (const float *) cxi;\n block_q5_0 * dsti = (block_q5_0 *) cdsti;\n\n float amax = 0.0f;\n float vmax = 0.0f;\n\n for (int j = 0; j < QK5_0; ++j) {\n const float v = xi[j];\n if (amax < sycl::fabs((float) v)) {\n amax = sycl::fabs((float) v);\n vmax = v;\n }\n }\n\n const float d = vmax / -16;\n const float id = d ? 1.0f / d : 0.0f;\n\n dsti->d = d;\n\n uint32_t qh = 0;\n for (int j = 0; j < QK5_0 / 2; ++j) {\n const float x0 = xi[0 + j] * id;\n const float x1 = xi[QK5_0 / 2 + j] * id;\n\n const uint8_t xi0 = dpct::min(31, (int8_t) (x0 + 16.5f));\n const uint8_t xi1 = dpct::min(31, (int8_t) (x1 + 16.5f));\n\n dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0 / 2);\n }\n memcpy(dsti->qh, &qh, sizeof(qh));\n}\n\ninline void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) {\n const float * xi = (const float *) cxi;\n block_q5_1 * dsti = (block_q5_1 *) cdsti;\n\n float min = xi[0];\n float max = xi[0];\n\n for (int j = 1; j < QK5_1; ++j) {\n const float v = xi[j];\n min = v < min ? v : min;\n max = v > max ? v : max;\n }\n\n const float d = (max - min) / 31;\n const float id = d ? 1.0f / d : 0.0f;\n\n dsti->dm.x() = d;\n dsti->dm.y() = min;\n\n uint32_t qh = 0;\n for (int j = 0; j < QK5_1 / 2; ++j) {\n const float x0 = (xi[0 + j] - min) * id;\n const float x1 = (xi[QK5_1 / 2 + j] - min) * id;\n\n const uint8_t xi0 = (uint8_t) (x0 + 0.5f);\n const uint8_t xi1 = (uint8_t) (x1 + 0.5f);\n\n dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);\n qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1 / 2);\n }\n memcpy(dsti->qh, &qh, sizeof(qh));\n}\n\ninline void cpy_blck_f32_iq4_nl(const char * cxi, char * cdsti) {\n const float * xi = (const float *) cxi;\n block_iq4_nl * dsti = (block_iq4_nl *) cdsti;\n\n float amax = 0.0f;\n float vmax = 0.0f;\n\n for (int j = 0; j < QK4_NL; ++j) {\n const float v = xi[j];\n if (amax < sycl::fabs((float) v)) {\n amax = sycl::fabs((float) v);\n vmax = v;\n }\n }\n\n float d = vmax / kvalues_iq4nl[0];\n const float id = d ? 1.0f / d : 0.0f;\n\n float sumqx = 0, sumq2 = 0;\n for (int j = 0; j < QK4_NL / 2; ++j) {\n const float x0 = xi[0 + j] * id;\n const float x1 = xi[QK4_NL / 2 + j] * id;\n const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl, x0);\n const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl, x1);\n dsti->qs[j] = xi0 | (xi1 << 4);\n const float v0 = kvalues_iq4nl[xi0];\n const float v1 = kvalues_iq4nl[xi1];\n const float w0 = xi[0 + j] * xi[0 + j];\n const float w1 = xi[QK4_NL / 2 + j] * xi[QK4_NL / 2 + j];\n sumqx += w0 * v0 * xi[j] + w1 * v1 * xi[QK4_NL / 2 + j];\n sumq2 += w0 * v0 * v0 + w1 * v1 * v1;\n }\n\n dsti->d = sumq2 > 0 ? sumqx / sumq2 : d;\n}\n\nvoid ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1);\nvoid ggml_sycl_dup(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\n#endif // GGML_SYCL_CPY_HPP\n" }, { "path": "src/ggml-sycl/dequantize.hpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_DEQUANTIZE_HPP\n#define GGML_SYCL_DEQUANTIZE_HPP\n\n#include \"common.hpp\"\n\ntypedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, dfloat2 & v);\ntypedef void (*dequantize_kernel_t_reorder)(const void *d, const int64_t ib, const void *qs,\n const int iqs, dfloat2 &v);\n\nstatic __dpct_inline__ void dequantize_q4_0(const void *vx, const int64_t ib,\n const int iqs, dfloat2 &v) {\n const block_q4_0 * x = (const block_q4_0 *) vx;\n\n const dfloat d = x[ib].d;\n\n const int vui = x[ib].qs[iqs];\n\n v.x() = vui & 0xF;\n v.y() = vui >> 4;\n\n#ifdef GGML_SYCL_F16\n // v = v - {8.0f, 8.0f};\n // v = v * {d, d};\n v.s0() = (v.s0() - 8.0f) * d;\n v.s1() = (v.s1() - 8.0f) * d;\n\n#else\n v.x() = (v.x() - 8.0f) * d;\n v.y() = (v.y() - 8.0f) * d;\n#endif // GGML_SYCL_F16\n}\n\nstatic __dpct_inline__ void dequantize_q4_0_reorder(const void *d_ptr, const int64_t ib, const void *qs,\n const int iqs, dfloat2 &v) {\n // const block_q4_0 * x = (const block_q4_0 *) vx;\n\n const dfloat d = (const dfloat)*((const sycl::half*)d_ptr+ib);\n\n const int vui = *((const uint8_t *)qs+iqs);\n\n v.x() = vui & 0xF;\n v.y() = vui >> 4;\n\n#ifdef GGML_SYCL_F16\n // v = v - {8.0f, 8.0f};\n // v = v * {d, d};\n v.s0() = (v.s0() - 8.0f) * d;\n v.s1() = (v.s1() - 8.0f) * d;\n\n#else\n v.x() = (v.x() - 8.0f) * d;\n v.y() = (v.y() - 8.0f) * d;\n#endif // GGML_SYCL_F16\n}\n\nstatic __dpct_inline__ void dequantize_q4_1(const void *vx, const int64_t ib,\n const int iqs, dfloat2 &v) {\n const block_q4_1 * x = (const block_q4_1 *) vx;\n\n const dfloat d = x[ib].dm[0];\n const dfloat m = x[ib].dm[1];\n\n const int vui = x[ib].qs[iqs];\n\n v.x() = vui & 0xF;\n v.y() = vui >> 4;\n\n#ifdef GGML_SYCL_F16\n // v = v * {d, d};\n // v = v + {m, m};\n v.s0() = sycl::fma(v.s0(), d, m);\n v.s1() = sycl::fma(v.s1(), d, m);\n\n#else\n v.x() = sycl::fma(v.x(), d, m);\n v.y() = sycl::fma(v.y(), d, m);\n#endif // GGML_SYCL_F16\n}\n\nstatic __dpct_inline__ void dequantize_q5_0(const void *vx, const int64_t ib,\n const int iqs, dfloat2 &v) {\n const block_q5_0 * x = (const block_q5_0 *) vx;\n\n const dfloat d = x[ib].d;\n\n uint32_t qh;\n memcpy(&qh, x[ib].qh, sizeof(qh));\n\n const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;\n const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;\n\n v.x() = ((x[ib].qs[iqs] & 0xf) | xh_0);\n v.y() = ((x[ib].qs[iqs] >> 4) | xh_1);\n\n#ifdef GGML_SYCL_F16\n // v = v - {16.0f, 16.0f};\n // v = v * {d, d};\n v.s0() = (v.s0() - 16.0f) * d;\n v.s1() = (v.s1() - 16.0f) * d;\n\n#else\n v.x() = (v.x() - 16.0f) * d;\n v.y() = (v.y() - 16.0f) * d;\n#endif // GGML_SYCL_F16\n}\n\nstatic __dpct_inline__ void dequantize_q5_1(const void *vx, const int64_t ib,\n const int iqs, dfloat2 &v) {\n const block_q5_1 * x = (const block_q5_1 *) vx;\n\n const dfloat d = x[ib].dm[0];\n const dfloat m = x[ib].dm[1];\n\n uint32_t qh;\n memcpy(&qh, x[ib].qh, sizeof(qh));\n\n const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;\n const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;\n\n v.x() = ((x[ib].qs[iqs] & 0xf) | xh_0);\n v.y() = ((x[ib].qs[iqs] >> 4) | xh_1);\n\n#ifdef GGML_SYCL_F16\n // v = v * {d, d};\n // v = v + {m, m};\n v.s0() = sycl::fma(v.s0(), d, m);\n v.s1() = sycl::fma(v.s1(), d, m);\n#else\n v.x() = sycl::fma(v.x(), d, m);\n v.y() = sycl::fma(v.y(), d, m);\n#endif // GGML_SYCL_F16\n}\n\nstatic __dpct_inline__ void dequantize_q8_0(const void *vx, const int64_t ib,\n const int iqs, dfloat2 &v) {\n const block_q8_0 * x = (const block_q8_0 *) vx;\n\n const dfloat d = x[ib].d;\n\n v.x() = x[ib].qs[iqs + 0];\n v.y() = x[ib].qs[iqs + 1];\n\n#ifdef GGML_SYCL_F16\n // v = v * {d, d};\n v.s0() *= d;\n v.s1() *= d;\n#else\n v.x() *= d;\n v.y() *= d;\n#endif // GGML_SYCL_F16\n}\n\ntemplate\nstatic void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32,\n const sycl::nd_item<3> &item_ct1) {\n\n const int64_t i = item_ct1.get_group(2);\n\n // assume 32 threads\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t il = tid/8;\n const int64_t ir = tid%8;\n const int64_t ib = 8*i + ir;\n if (ib >= nb32) {\n return;\n }\n\n dst_t * y = yy + 256*i + 32*ir + 4*il;\n\n const block_q4_0 * x = (const block_q4_0 *)vx + ib;\n const float d = sycl::vec(x->d)\n .convert()[0];\n const float dm = -8*d;\n\n const uint8_t * q = x->qs + 4*il;\n\n for (int l = 0; l < 4; ++l) {\n y[l+ 0] = d * (q[l] & 0xF) + dm;\n y[l+16] = d * (q[l] >> 4) + dm;\n }\n}\n\ntemplate\nstatic void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32,\n const sycl::nd_item<3> &item_ct1) {\n\n const int64_t i = item_ct1.get_group(2);\n auto k=nb32;\n // assume 32 threads\n const int64_t tid = item_ct1.get_local_id(2);\n const int lane_ib = i * WARP_SIZE + tid;\n\n if (lane_ib >= k / QK4_0) {\n return;\n }\n\n dst_t * y_ptr = yy + lane_ib * QK4_0;\n\n auto qs = (const uint8_t*)vx + lane_ib * QK4_0 / 2;\n auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k / 2) + lane_ib;\n\n const float d = float(*s_ptr);\n\n#pragma unroll\n for (int l = 0; l < QK4_0 / 2; ++l) {\n int vq = qs[l];\n y_ptr[l + 0] = d * ((vq & 0xF) - 8);\n y_ptr[l + 16] = d * ((vq >> 4) - 8);\n }\n\n}\n\ntemplate\nstatic void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32,\n const sycl::nd_item<3> &item_ct1) {\n\n const int64_t i = item_ct1.get_group(2);\n\n // assume 32 threads\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t il = tid/8;\n const int64_t ir = tid%8;\n const int64_t ib = 8*i + ir;\n if (ib >= nb32) {\n return;\n }\n\n dst_t * y = yy + 256*i + 32*ir + 4*il;\n\n const block_q4_1 * x = (const block_q4_1 *)vx + ib;\n const sycl::float2 d =\n x->dm.convert();\n\n const uint8_t * q = x->qs + 4*il;\n\n for (int l = 0; l < 4; ++l) {\n y[l + 0] = d.x() * (q[l] & 0xF) + d.y();\n y[l + 16] = d.x() * (q[l] >> 4) + d.y();\n }\n}\n\n\n//================================== k-quants\n\ntemplate\nstatic void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy,\n const sycl::nd_item<3> &item_ct1) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_q2_K * x = (const block_q2_K *) vx;\n\n const int64_t tid = item_ct1.get_local_id(2);\n#if QK_K == 256\n const int64_t n = tid/32;\n const int64_t l = tid - 32*n;\n const int64_t is = 8*n + l/16;\n\n const uint8_t q = x[i].qs[32*n + l];\n dst_t * y = yy + i*QK_K + 128*n;\n\n float dall = x[i].dm[0];\n float dmin = x[i].dm[1];\n y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);\n y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4);\n y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);\n y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);\n#else\n const int64_t is = tid/16; // 0 or 1\n const int64_t il = tid%16; // 0...15\n const uint8_t q = x[i].qs[il] >> (2*is);\n dst_t * y = yy + i*QK_K + 16*is + il;\n\n float dall = x[i].dm[0];\n float dmin = x[i].dm[1];\n y[ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);\n y[32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+2] >> 4);\n#endif\n\n}\n\ntemplate\nstatic void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy,\n const sycl::nd_item<3> &item_ct1) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_q3_K * x = (const block_q3_K *) vx;\n\n#if QK_K == 256\n const int64_t r = item_ct1.get_local_id(2) / 4;\n const int64_t tid = r/2;\n const int64_t is0 = r%2;\n const int64_t l0 = 16 * is0 + 4 * (item_ct1.get_local_id(2) % 4);\n const int64_t n = tid / 4;\n const int64_t j = tid - 4*n;\n\n uint8_t m = 1 << (4*n + j);\n int64_t is = 8*n + 2*j + is0;\n int shift = 2*j;\n\n int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :\n is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :\n is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :\n (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);\n float d_all = x[i].d;\n float dl = d_all * (us - 32);\n\n dst_t * y = yy + i*QK_K + 128*n + 32*j;\n const uint8_t * q = x[i].qs + 32*n;\n const uint8_t * hm = x[i].hmask;\n\n for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));\n#else\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t is = tid/16; // 0 or 1\n const int64_t il = tid%16; // 0...15\n const int64_t im = il/8; // 0...1\n const int64_t in = il%8; // 0...7\n\n dst_t * y = yy + i*QK_K + 16*is + il;\n\n const uint8_t q = x[i].qs[il] >> (2*is);\n const uint8_t h = x[i].hmask[in] >> (2*is + im);\n const float d = (float)x[i].d;\n\n if (is == 0) {\n y[ 0] = d * ((x[i].scales[0] & 0xF) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));\n y[32] = d * ((x[i].scales[1] & 0xF) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));\n } else {\n y[ 0] = d * ((x[i].scales[0] >> 4) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));\n y[32] = d * ((x[i].scales[1] >> 4) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));\n }\n#endif\n\n}\n\n#if QK_K == 256\nstatic inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {\n if (j < 4) {\n d = q[j] & 63;\n m = q[j + 4] & 63;\n } else {\n d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);\n m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);\n }\n}\n#endif\n\ntemplate \ninline void dequantize_q4_K_common(dst_t * __restrict__ y, const uint8_t * __restrict__ qs_ptr, const float dall,\n const float dmin, uint8_t * __restrict__ scales_local, int il, int ir) {\n const int is = 2 * il;\n constexpr int n = 4;\n\n uint8_t sc, m;\n get_scale_min_k4(is + 0, scales_local, sc, m);\n const float d1 = dall * sc;\n const float m1 = dmin * m;\n\n get_scale_min_k4(is + 1, scales_local, sc, m);\n const float d2 = dall * sc;\n const float m2 = dmin * m;\n\n sycl::vec q_vec = vec_aligned_load(qs_ptr + 32 * il + n * ir);\n for (int l = 0; l < n; ++l) {\n y[l + 0] = d1 * (q_vec[l] & 0xF) - m1;\n y[l + 32] = d2 * (q_vec[l] >> 4) - m2;\n }\n}\n\ntemplate\nstatic void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy,\n uint8_t* scales_local, const sycl::nd_item<3> &item_ct1) {\n const block_q4_K * x = (const block_q4_K *) vx;\n\n const int64_t i = item_ct1.get_group(2);\n\n#if QK_K == 256\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t il = tid / 8;\n const int64_t ir = tid % 8;\n\n dst_t * y = yy + i * QK_K + 64 * il + 4 * ir;\n\n const sycl::half2 dm = x[i].dm;\n const float dall = dm[0];\n const float dmin = dm[1];\n\n if (tid < 12) {\n scales_local[tid] = x[i].scales[tid];\n }\n\n item_ct1.barrier(sycl::access::fence_space::local_space);\n dequantize_q4_K_common(y, x[i].qs, dall, dmin, scales_local, il, ir);\n#else\n const int64_t tid = item_ct1.get_local_id(2);\n const uint8_t * q = x[i].qs;\n dst_t * y = yy + i*QK_K;\n const float d = (float)x[i].dm[0];\n const float m = (float)x[i].dm[1];\n y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4);\n y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >> 4) - m * (x[i].scales[1] >> 4);\n#endif\n}\n\ntemplate \nstatic void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, uint8_t * scales_local,\n const sycl::nd_item<1> & item_ct1, int64_t nb) {\n const int64_t i = item_ct1.get_group(0); // block index\n const int64_t tid = item_ct1.get_local_id(0); // thread index within block\n const int64_t il = tid / 8;\n const int64_t ir = tid % 8;\n\n dst_t * y = yy + i * QK_K + 64 * il + 4 * ir;\n\n const uint8_t * base = static_cast(vx);\n const size_t qs_offset = i * (QK_K / 2);\n const size_t scales_offset = nb * (QK_K / 2) + i * K_SCALE_SIZE;\n const size_t dm_offset = nb * (QK_K / 2) + nb * K_SCALE_SIZE + i * sizeof(ggml_half2);\n\n const uint8_t * qs_ptr = base + qs_offset;\n const uint8_t * scales_ptr = base + scales_offset;\n ggml_half2 dm_values = *reinterpret_cast(base + dm_offset);\n\n const float dall = dm_values.x();\n const float dmin = dm_values.y();\n\n if (tid < 12) {\n scales_local[tid] = scales_ptr[tid];\n }\n\n item_ct1.barrier(sycl::access::fence_space::local_space);\n dequantize_q4_K_common(y, qs_ptr, dall, dmin, scales_local, il, ir);\n}\n\ntemplate\nstatic void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy,\n const sycl::nd_item<3> &item_ct1) {\n const block_q5_K * x = (const block_q5_K *) vx;\n\n const int64_t i = item_ct1.get_group(2);\n\n#if QK_K == 256\n // assume 64 threads - this is very slightly better than the one below\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t il = tid/16; // il is in 0...3\n const int64_t ir = tid%16; // ir is in 0...15\n const int64_t is = 2*il; // is is in 0...6\n\n dst_t * y = yy + i*QK_K + 64*il + 2*ir;\n\n const float dall = x[i].dm[0];\n const float dmin = x[i].dm[1];\n\n const uint8_t * ql = x[i].qs + 32*il + 2*ir;\n const uint8_t * qh = x[i].qh + 2*ir;\n\n uint8_t sc, m;\n get_scale_min_k4(is + 0, x[i].scales, sc, m);\n const float d1 = dall * sc; const float m1 = dmin * m;\n get_scale_min_k4(is + 1, x[i].scales, sc, m);\n const float d2 = dall * sc; const float m2 = dmin * m;\n\n uint8_t hm = 1 << (2*il);\n y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;\n y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;\n hm <<= 1;\n y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;\n y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;\n#else\n const int64_t tid = item_ct1.get_local_id(2);\n const uint8_t q = x[i].qs[tid];\n const int64_t im = tid/8; // 0...3\n const int64_t in = tid%8; // 0...7\n const int64_t is = tid/16; // 0 or 1\n const uint8_t h = x[i].qh[in] >> im;\n const float d = x[i].d;\n dst_t * y = yy + i*QK_K + tid;\n y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16));\n y[32] = d * x[i].scales[is+2] * ((q >> 4) - ((h >> 4) & 1 ? 0 : 16));\n#endif\n}\n\ntemplate\nstatic void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy,\n const sycl::nd_item<3> &item_ct1) {\n const block_q6_K * x = (const block_q6_K *) vx;\n\n const int64_t i = item_ct1.get_group(2);\n#if QK_K == 256\n\n // assume 64 threads - this is very slightly better than the one below\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t ip = tid/32; // ip is 0 or 1\n const int64_t il = tid - 32*ip; // 0...32\n const int64_t is = 8*ip + il/16;\n\n dst_t * y = yy + i*QK_K + 128*ip + il;\n\n const float d = x[i].d;\n\n const uint8_t * ql = x[i].ql + 64*ip + il;\n const uint8_t qh = x[i].qh[32*ip + il];\n const int8_t * sc = x[i].scales + is;\n\n y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);\n y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);\n y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);\n y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);\n#else\n\n // assume 32 threads\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t ip = tid/16; // 0 or 1\n const int64_t il = tid - 16*ip; // 0...15\n\n dst_t * y = yy + i*QK_K + 16*ip + il;\n\n const float d = x[i].d;\n\n const uint8_t ql = x[i].ql[16*ip + il];\n const uint8_t qh = x[i].qh[il] >> (2*ip);\n const int8_t * sc = x[i].scales;\n\n y[ 0] = d * sc[ip+0] * ((int8_t)((ql & 0xF) | (((qh >> 0) & 3) << 4)) - 32);\n y[32] = d * sc[ip+2] * ((int8_t)((ql >> 4) | (((qh >> 4) & 3) << 4)) - 32);\n#endif\n}\n\ntemplate \nstatic void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy,\n const sycl::nd_item<3> & item_ct1, int64_t n_blocks) {\n const int64_t ib = item_ct1.get_group(2);\n\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t ip = tid / 32; // ip is 0 or 1\n const int64_t il = tid - 32 * ip; // 0...32\n const int64_t is = 8 * ip + il / 16;\n\n const uint8_t * base_ptr = static_cast(vx);\n const auto ql_offset = ib * (QK_K / 2);\n const auto qh_offset = (QK_K / 2) * n_blocks + (QK_K / 4) * ib;\n const auto base_scales_offset = (QK_K / 2) * n_blocks + (QK_K / 4) * n_blocks + (QK_K / 16) * ib;\n const auto base_d_offset = ((QK_K / 2) + (QK_K / 4) + (QK_K / 16)) * n_blocks;\n const uint8_t * ql_ptr = base_ptr + ql_offset;\n const uint8_t * qh_ptr = base_ptr + qh_offset;\n const uint8_t * scales_ptr = base_ptr + base_scales_offset;\n const ggml_half * d = (const ggml_half *) (base_ptr + base_d_offset) + ib;\n\n dst_t * y = yy + ib * QK_K + 128 * ip + il;\n\n const uint8_t * ql = ql_ptr + 64 * ip + il;\n const uint8_t qh = *(qh_ptr + 32 * ip + il);\n const int8_t * sc = reinterpret_cast(scales_ptr + is);\n\n y[0] = *d * sc[0] * ((int8_t) ((ql[0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);\n y[32] = *d * sc[2] * ((int8_t) ((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);\n y[64] = *d * sc[4] * ((int8_t) ((ql[0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);\n y[96] = *d * sc[6] * ((int8_t) ((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);\n}\n\ntemplate\nstatic void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy,\n const sycl::nd_item<3> &item_ct1,\n const uint64_t *iq2xxs_grid_ptr,\n const uint8_t *ksigns_iq2xs_ptr,\n const uint8_t *kmask_iq2xs_ptr) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_iq2_xxs * x = (const block_iq2_xxs *) vx;\n\n const int64_t tid = item_ct1.get_local_id(2);\n#if QK_K == 256\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint16_t * q2 = x[i].qs + 4*ib;\n const uint8_t * aux8 = (const uint8_t *)q2;\n const uint8_t * grid = (const uint8_t *)(iq2xxs_grid_ptr + aux8[il]);\n const uint32_t aux32 = q2[2] | (q2[3] << 16);\n const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f;\n const uint8_t signs = ksigns_iq2xs_ptr[(aux32 >> 7*il) & 127];\n for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs_ptr[j] ? -1.f : 1.f);\n#else\n assert(false);\n#endif\n\n}\n\ntemplate\nstatic void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy,\n const sycl::nd_item<3> &item_ct1,\n const uint64_t *iq2xs_grid,\n const uint8_t *ksigns_iq2xs,\n const uint8_t *kmask_iq2xs) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_iq2_xs * x = (const block_iq2_xs *) vx;\n\n const int64_t tid = item_ct1.get_local_id(2);\n#if QK_K == 256\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint16_t * q2 = x[i].qs + 4*ib;\n const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));\n const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;\n const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];\n for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n#else\n assert(false);\n#endif\n\n}\n\ntemplate \n__dpct_inline__ static void\ndequantize_block_iq2_s(const void *__restrict__ vx, dst_t *__restrict__ yy,\n const sycl::nd_item<3> &item_ct1) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_iq2_s * x = (const block_iq2_s *) vx;\n\n const int64_t tid = item_ct1.get_local_id(2);\n#if QK_K == 256\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));\n const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;\n const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];\n#pragma unroll\n for (int j = 0; j < 8; ++j)\n y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n#else\n assert(false);\n\n#endif\n\n}\n\ntemplate\nstatic void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy,\n const sycl::nd_item<3> &item_ct1,\n const uint32_t *iq3xxs_grid,\n const uint8_t *ksigns_iq2xs,\n const uint8_t *kmask_iq2xs) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_iq3_xxs * x = (const block_iq3_xxs *) vx;\n\n const int64_t tid = item_ct1.get_local_id(2);\n#if QK_K == 256\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint8_t * q3 = x[i].qs + 8*ib;\n const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;\n const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);\n const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);\n const uint32_t aux32 = gas[0] | (gas[1] << 16);\n const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f;\n const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];\n for (int j = 0; j < 4; ++j) {\n y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);\n y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);\n }\n#else\n assert(false);\n#endif\n\n}\n\ntemplate \n__dpct_inline__ static void\ndequantize_block_iq3_s(const void *__restrict__ vx, dst_t *__restrict__ yy,\n const sycl::nd_item<3> &item_ct1,\n const uint8_t *kmask_iq2xs, const uint32_t *iq3s_grid) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_iq3_s * x = (const block_iq3_s *) vx;\n\n const int64_t tid = item_ct1.get_local_id(2);\n#if QK_K == 256\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint8_t * qs = x[i].qs + 8*ib;\n const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));\n const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));\n const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf));\n const uint8_t signs = x[i].signs[4*ib + il];\n#pragma unroll\n for (int j = 0; j < 4; ++j) {\n y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);\n y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);\n }\n#else\n assert(false);\n#endif\n\n}\n\ntemplate \n__dpct_inline__ static void\ndequantize_block_iq1_s(const void *__restrict__ vx, dst_t *__restrict__ yy,\n const sycl::nd_item<3> &item_ct1,\n const uint32_t *iq1s_grid_gpu) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_iq1_s * x = (const block_iq1_s *) vx;\n\n const int64_t tid = item_ct1.get_local_id(2);\n#if QK_K == 256\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;\n const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1);\n uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;\n grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)];\n grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;\n grid32[0] &= 0x0f0f0f0f;\n#pragma unroll\n for (int j = 0; j < 8; ++j) {\n y[j] = d * (q[j] + delta);\n }\n#else\n assert(false);\n#endif\n\n}\n\ntemplate \n__dpct_inline__ static void\ndequantize_block_iq1_m(const void *__restrict__ vx, dst_t *__restrict__ yy,\n const sycl::nd_item<3> &item_ct1,\n const uint32_t *iq1s_grid_gpu) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_iq1_m * x = (const block_iq1_m *) vx;\n\n const int64_t tid = item_ct1.get_local_id(2);\n#if QK_K == 256\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n const uint16_t * sc = (const uint16_t *)x[i].scales;\n iq1m_scale_t scale;\n scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n const int ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);\n const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);\n const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;\n uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;\n grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];\n grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;\n grid32[0] &= 0x0f0f0f0f;\n#pragma unroll\n for (int j = 0; j < 8; ++j) {\n y[j] = d * (q[j] + delta);\n }\n#else\n assert(false);\n#endif\n\n}\n\ntemplate \n__dpct_inline__ static void\ndequantize_block_iq4_nl(const void *__restrict__ vx, dst_t *__restrict__ yy,\n const sycl::nd_item<3> &item_ct1) {\n\n const int64_t i = item_ct1.get_group(2);\n const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);\n\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 4*il;\n const uint8_t * q4 = x[ib].qs + 4*il;\n const float d = (float)x[ib].d;\n#pragma unroll\n for (int j = 0; j < 4; ++j) {\n y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];\n y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];\n }\n\n}\n\n\ntemplate \n__dpct_inline__ static void\ndequantize_block_iq4_xs(const void *__restrict__ vx, dst_t *__restrict__ yy,\n const sycl::nd_item<3> &item_ct1) {\n const int64_t i = item_ct1.get_group(2);\n const block_iq4_xs * x = (const block_iq4_xs *)vx;\n\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 4*il;\n const uint8_t * q4 = x[i].qs + 16*ib + 4*il;\n const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);\n#pragma unroll\n for (int j = 0; j < 4; ++j) {\n y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];\n y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];\n }\n}\n\ntemplate\nstatic void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy,\n const sycl::nd_item<3> &item_ct1) {\n // auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const int64_t i = item_ct1.get_group(2);\n const block_mxfp4 * x = (const block_mxfp4 *) vx + i*(QK_K/QK_MXFP4);\n\n const int64_t tid = item_ct1.get_local_id(2);\n const int64_t il = tid/8; // 0...3\n const int64_t ib = tid%8; // 0...7\n dst_t * y = yy + i*QK_K + 32*ib + 4*il;\n const uint8_t * q4 = x[ib].qs + 4*il;\n const float d = ggml_sycl_e8m0_to_fp32(x[ib].e);\n for (int j = 0; j < 4; ++j) {\n y[j+ 0] = d * kvalues_mxfp4[q4[j] & 0xf]*0.5f;\n y[j+16] = d * kvalues_mxfp4[q4[j] >> 4]*0.5f;\n }\n}\n\n#endif // GGML_SYCL_DEQUANTIZE_HPP\n" }, { "path": "src/ggml-sycl/dmmv.cpp", "content": "#include \"convert.hpp\"\n#include \"dmmv.hpp\"\n#include \"dequantize.hpp\"\n#include \"presets.hpp\"\n\nstatic void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){\n const sycl::half *x = (const sycl::half *)vx;\n\n // automatic half -> float type cast if dfloat == float\n v.x() = x[ib + iqs + 0];\n v.y() = x[ib + iqs + 1];\n}\n\nstatic void convert_f32(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){\n const float * x = (const float *) vx;\n\n // automatic half -> float type cast if dfloat == float\n v.x() = x[ib + iqs + 0];\n v.y() = x[ib + iqs + 1];\n}\n\ntemplate \nstatic void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows,\n const sycl::nd_item<3> &item_ct1) {\n // qk = quantized weights per x block\n // qr = number of quantized weights per data value in x block\n const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +\n item_ct1.get_local_id(1);\n\n if (row >= nrows) {\n return;\n }\n\n const int tid = item_ct1.get_local_id(2);\n\n const int iter_stride = 2*GGML_SYCL_DMMV_X;\n const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter\n const int y_offset = qr == 1 ? 1 : qk/2;\n\n// partial sum for each thread\n#ifdef GGML_SYCL_F16\n sycl::half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics\n#else\n float tmp = 0.0f;\n#endif // GGML_SYCL_F16\n\n for (int i = 0; i < ncols; i += iter_stride) {\n const int col = i + vals_per_iter*tid;\n const int ib = (row*ncols + col)/qk; // x block index\n const int iqs = (col%qk)/qr; // x quant index\n const int iybs = col - col%qk; // y block start index\n\n// processing >2 values per i iter is faster for fast GPUs\n#pragma unroll\n for (int j = 0; j < vals_per_iter; j += 2) {\n // process 2 vals per j iter\n\n // dequantize\n // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val\n dfloat2 v;\n dequantize_kernel(vx, ib, iqs + j/qr, v);\n\n // matrix multiplication\n // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2\n#ifdef GGML_SYCL_F16\n dfloat2 t1{y[iybs + iqs + j / qr + 0],\n y[iybs + iqs + j / qr + y_offset]};\n\n tmp += v * t1;\n#else\n tmp += v.x() * y[iybs + iqs + j / qr + 0];\n tmp += v.y() * y[iybs + iqs + j / qr + y_offset];\n#endif // GGML_SYCL_F16\n }\n }\n\n // sum up partial sums and write back result\n const int mask_start = ncols > GGML_SYCL_DMMV_X ? WARP_SIZE >> 1 : WARP_SIZE >> 2;\n for (int mask = mask_start; mask > 0; mask >>= 1) {\n tmp +=\n dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);\n }\n\n if (tid == 0) {\n#ifdef GGML_SYCL_F16\n dst[row] = tmp.x() + tmp.y();\n#else\n dst[row] = tmp;\n#endif // GGML_SYCL_F16\n }\n}\n\ntemplate \nstatic void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows,\n const sycl::nd_item<3> &item_ct1) {\n // qk = quantized weights per x block\n // qr = number of quantized weights per data value in x block\n const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +\n item_ct1.get_local_id(1);\n\n if (row >= nrows) {\n return;\n }\n\n const int tid = item_ct1.get_local_id(2);\n\n\n const int ncols_left = ncols % (QK4_0*WARP_SIZE);\n const int ncols_align = ncols - ncols_left;\n const int iter_stride = 8*2*GGML_SYCL_DMMV_X;\n const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter //64/16=4, 512/16/2= 16\n const int y_offset = qr == 1 ? 1 : qk/2;\n\n// partial sum for each thread\n#ifdef GGML_SYCL_F16\n sycl::half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics\n#else\n float tmp = 0.0f;\n#endif // GGML_SYCL_F16\n const char *d_ptr = (const char*)vx+ncols*nrows/2;\n int i=0;\n for (i = 0; i < ncols_align; i += iter_stride) {\n const int col = i + vals_per_iter*tid;\n const int ib = (row*ncols + col)/qk; // x block index\n const int iqs = (col%qk)/qr; // x quant index\n const int iybs = col - col%qk; // y block start index\n\n// processing >2 values per i iter is faster for fast GPUs\n#pragma unroll\n for (int j = 0; j < vals_per_iter; j += 2) {\n // process 2 vals per j iter\n\n // dequantize\n // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val\n dfloat2 v;\n dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v);\n\n // matrix multiplication\n // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2\n#ifdef GGML_SYCL_F16\n dfloat2 t1{y[iybs + iqs + j / qr + 0],\n y[iybs + iqs + j / qr + y_offset]};\n\n tmp += v * t1;\n#else\n tmp += v.x() * y[iybs + iqs + j / qr + 0];\n tmp += v.y() * y[iybs + iqs + j / qr + y_offset];\n#endif // GGML_SYCL_F16\n }\n }\n\n for (; i < ncols; i += iter_stride) {\n if (tid>=ncols_left/QK4_0) continue;\n const int col = i + vals_per_iter*tid;\n const int ib = (row*ncols + col)/qk; // x block index\n const int iqs = (col%qk)/qr; // x quant index\n const int iybs = col - col%qk; // y block start index\n\n// processing >2 values per i iter is faster for fast GPUs\n#pragma unroll\n for (int j = 0; j < vals_per_iter; j += 2) {\n // process 2 vals per j iter\n\n // dequantize\n // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val\n dfloat2 v;\n dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v);\n\n // matrix multiplication\n // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2\n#ifdef GGML_SYCL_F16\n dfloat2 t1{y[iybs + iqs + j / qr + 0],\n y[iybs + iqs + j / qr + y_offset]};\n\n tmp += v * t1;\n#else\n tmp += v.x() * y[iybs + iqs + j / qr + 0];\n tmp += v.y() * y[iybs + iqs + j / qr + y_offset];\n#endif // GGML_SYCL_F16\n }\n }\n\n // sum up partial sums and write back result\n const int mask_start = ncols > GGML_SYCL_DMMV_X ? WARP_SIZE >> 1 : WARP_SIZE >> 2;\n for (int mask = mask_start; mask > 0; mask >>= 1) {\n tmp +=\n dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);\n }\n\n if (tid == 0) {\n#ifdef GGML_SYCL_F16\n dst[row] = tmp.x() + tmp.y();\n#else\n dst[row] = tmp;\n#endif // GGML_SYCL_F16\n }\n}\n\nstatic void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);\n const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n dequantize_mul_mat_vec<1, 1, convert_f16>(vx, y, dst, ncols,\n nrows, item_ct1);\n });\n }\n}\n\n/*\nDPCT1110:4: The total declared local variable size in device function\ndequantize_mul_mat_vec_q2_k exceeds 128 bytes and may cause high register\npressure. Consult with your hardware vendor to find the total register size\navailable and adjust the code, or use smaller sub-group size to avoid high\nregister pressure.\n*/\nstatic void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx,\n const float *__restrict__ yy,\n float *__restrict__ dst,\n const int ncols, int nrows,\n const sycl::nd_item<3> &item_ct1) {\n\n static_assert(16%K_QUANTS_PER_ITERATION == 0, \"16 must be divisible by K_QUANTS_PER_ITERATION\");\n\n const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +\n item_ct1.get_local_id(1);\n if (row > nrows) return;\n\n const int num_blocks_per_row = ncols / QK_K;\n const int ib0 = row*num_blocks_per_row;\n\n const block_q2_K * x = (const block_q2_K *)vx + ib0;\n\n float tmp = 0; // partial sum for thread in warp\n\n#if QK_K == 256\n const int tid =\n item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...15\n const int ix =\n item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1\n\n const int step = 16/K_QUANTS_PER_ITERATION;\n\n const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...\n const int in = tid - step*im; // 0...15 or 0...7\n\n const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2\n const int q_offset = 32*im + l0;\n const int s_offset = 8*im;\n const int y_offset = 128*im + l0;\n\n uint32_t aux[4];\n const uint8_t * d = (const uint8_t *)aux;\n const uint8_t * m = (const uint8_t *)(aux + 2);\n\n for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {\n\n const float * y = yy + i * QK_K + y_offset;\n const uint8_t * q = x[i].qs + q_offset;\n\n const float dall = x[i].dm[0];\n const float dmin = x[i].dm[1];\n\n const uint32_t * a = (const uint32_t *)(x[i].scales + s_offset);\n aux[0] = a[0] & 0x0f0f0f0f;\n aux[1] = a[1] & 0x0f0f0f0f;\n aux[2] = (a[0] >> 4) & 0x0f0f0f0f;\n aux[3] = (a[1] >> 4) & 0x0f0f0f0f;\n\n float sum1 = 0, sum2 = 0;\n for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {\n sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3)\n + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3)\n + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3)\n + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3)\n + y[l+16] * d[1] * ((q[l+16] >> 0) & 3)\n + y[l+48] * d[3] * ((q[l+16] >> 2) & 3)\n + y[l+80] * d[5] * ((q[l+16] >> 4) & 3)\n +y[l+112] * d[7] * ((q[l+16] >> 6) & 3);\n sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6]\n + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7];\n\n }\n tmp += dall * sum1 - dmin * sum2;\n\n }\n#else\n const int tid = item_ct1.get_local_id(2) /\n (2 * K_QUANTS_PER_ITERATION); // 0...15 or 0...7\n const int ix = item_ct1.get_local_id(2) %\n (2 * K_QUANTS_PER_ITERATION); // 0....1 or 0...3\n const int offset = tid * K_QUANTS_PER_ITERATION;\n\n uint32_t uaux[2];\n const uint8_t * d = (const uint8_t *)uaux;\n\n\n for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {\n\n const float * y = yy + i * QK_K + offset;\n const uint8_t * q = x[i].qs + offset;\n const uint32_t * s = (const uint32_t *)x[i].scales;\n\n uaux[0] = s[0] & 0x0f0f0f0f;\n uaux[1] = (s[0] >> 4) & 0x0f0f0f0f;\n\n const sycl::float2 dall =\n x[i].dm.convert();\n\n float sum1 = 0, sum2 = 0;\n for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {\n const uint8_t ql = q[l];\n sum1 += y[l+ 0] * d[0] * ((ql >> 0) & 3)\n + y[l+16] * d[1] * ((ql >> 2) & 3)\n + y[l+32] * d[2] * ((ql >> 4) & 3)\n + y[l+48] * d[3] * ((ql >> 6) & 3);\n sum2 += y[l+0] * d[4] + y[l+16] * d[5] + y[l+32] * d[6] + y[l+48] * d[7];\n }\n tmp += dall.x() * sum1 - dall.y() * sum2;\n }\n\n#endif\n\n // sum up partial sums and write back result\n#pragma unroll\n for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) {\n tmp +=\n dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);\n }\n\n if (item_ct1.get_local_id(2) == 0) {\n dst[row] = tmp;\n }\n}\n\n/*\nDPCT1110:5: The total declared local variable size in device function\ndequantize_mul_mat_vec_q3_k exceeds 128 bytes and may cause high register\npressure. Consult with your hardware vendor to find the total register size\navailable and adjust the code, or use smaller sub-group size to avoid high\nregister pressure.\n*/\nstatic void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx,\n const float *__restrict__ yy,\n float *__restrict__ dst,\n const int ncols, int nrows,\n const sycl::nd_item<3> &item_ct1) {\n\n const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +\n item_ct1.get_local_id(1);\n if (row > nrows) return;\n\n const int num_blocks_per_row = ncols / QK_K;\n const int ib0 = row*num_blocks_per_row;\n\n const block_q3_K * x = (const block_q3_K *)vx + ib0;\n\n float tmp = 0; // partial sum for thread in warp\n\n#if QK_K == 256\n\n const uint16_t kmask1 = 0x0303;\n const uint16_t kmask2 = 0x0f0f;\n\n const int tid =\n item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...16\n const int ix =\n item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1\n\n const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop\n const int step = 16/K_QUANTS_PER_ITERATION;\n const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...\n const int in = tid - step*im; // 0....15 or 0...7\n\n const uint8_t m = 1 << (4*im);\n\n const int l0 = n*in; // 0...15 or 0...14 in steps of 2\n const int q_offset = 32*im + l0;\n const int y_offset = 128*im + l0;\n\n uint16_t utmp[4];\n const int8_t * s = (const int8_t *)utmp;\n\n const uint16_t s_shift = 4*im;\n\n for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {\n\n const float * y = yy + i * QK_K + y_offset;\n const uint8_t * q = x[i].qs + q_offset;\n const uint8_t * h = x[i].hmask + l0;\n\n const uint16_t * a = (const uint16_t *)x[i].scales;\n utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4);\n utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4);\n utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4);\n utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4);\n\n const float d = x[i].d;\n\n float sum = 0;\n for (int l = 0; l < n; ++l) {\n sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4))\n + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4))\n + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4))\n + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4));\n sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4))\n + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4))\n + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4))\n + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4));\n }\n tmp += d * sum;\n\n }\n#else\n\n const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7\n const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3\n const int offset = tid * K_QUANTS_PER_ITERATION; // 0...15 or 0...14\n const int in = offset/8; // 0 or 1\n const int im = offset%8; // 0...7\n\n for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {\n\n const float * y = yy + i * QK_K + offset;\n const uint8_t * q = x[i].qs + offset;\n const uint8_t * s = x[i].scales;\n\n const float dall = (float)x[i].d;\n\n float sum = 0;\n for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {\n const uint8_t hl = x[i].hmask[im+l] >> in;\n const uint8_t ql = q[l];\n sum += y[l+ 0] * dall * ((s[0] & 0xF) - 8) * ((int8_t)((ql >> 0) & 3) - ((hl >> 0) & 1 ? 0 : 4))\n + y[l+16] * dall * ((s[0] >> 4) - 8) * ((int8_t)((ql >> 2) & 3) - ((hl >> 2) & 1 ? 0 : 4))\n + y[l+32] * dall * ((s[1] & 0xF) - 8) * ((int8_t)((ql >> 4) & 3) - ((hl >> 4) & 1 ? 0 : 4))\n + y[l+48] * dall * ((s[1] >> 4) - 8) * ((int8_t)((ql >> 6) & 3) - ((hl >> 6) & 1 ? 0 : 4));\n }\n tmp += sum;\n }\n#endif\n\n // sum up partial sums and write back result\n#pragma unroll\n for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) {\n tmp +=\n dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);\n }\n\n if (item_ct1.get_local_id(2) == 0) {\n dst[row] = tmp;\n }\n}\n\n/*\nDPCT1110:6: The total declared local variable size in device function\ndequantize_mul_mat_vec_q4_k exceeds 128 bytes and may cause high register\npressure. Consult with your hardware vendor to find the total register size\navailable and adjust the code, or use smaller sub-group size to avoid high\nregister pressure.\n*/\nstatic void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx,\n const float *__restrict__ yy,\n float *__restrict__ dst,\n const int ncols, int nrows,\n const sycl::nd_item<3> &item_ct1) {\n\n const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +\n item_ct1.get_local_id(1);\n if (row > nrows) return;\n const int num_blocks_per_row = ncols / QK_K;\n const int ib0 = row*num_blocks_per_row;\n\n const block_q4_K * x = (const block_q4_K *)vx + ib0;\n\n#if QK_K == 256\n const uint16_t kmask1 = 0x3f3f;\n const uint16_t kmask2 = 0x0f0f;\n const uint16_t kmask3 = 0xc0c0;\n\n const int tid =\n item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...16\n const int ix =\n item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1\n\n const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4\n\n const int il = tid/step; // 0...3\n const int ir = tid - step*il; // 0...7 or 0...3\n const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4\n\n const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224\n const int in = il%2;\n\n const int l0 = n*(2*ir + in);\n const int q_offset = 32*im + l0;\n const int y_offset = 64*im + l0;\n\n uint16_t aux[4];\n const uint8_t * sc = (const uint8_t *)aux;\n\n#if K_QUANTS_PER_ITERATION == 2\n uint32_t q32[4];\n const uint8_t * q4 = (const uint8_t *)q32;\n#else\n uint16_t q16[4];\n const uint8_t * q4 = (const uint8_t *)q16;\n#endif\n\n float tmp = 0; // partial sum for thread in warp\n\n for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {\n\n const float * y1 = yy + i*QK_K + y_offset;\n const float * y2 = y1 + 128;\n\n const float dall = x[i].dm[0];\n const float dmin = x[i].dm[1];\n\n const uint16_t * a = (const uint16_t *)x[i].scales;\n aux[0] = a[im+0] & kmask1;\n aux[1] = a[im+2] & kmask1;\n aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);\n aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);\n\n#if K_QUANTS_PER_ITERATION == 2\n const uint32_t * q1 = (const uint32_t *)(x[i].qs + q_offset);\n const uint32_t * q2 = q1 + 16;\n\n q32[0] = q1[0] & 0x0f0f0f0f;\n q32[1] = q1[0] & 0xf0f0f0f0;\n q32[2] = q2[0] & 0x0f0f0f0f;\n q32[3] = q2[0] & 0xf0f0f0f0;\n\n sycl::float4 s = {0.f, 0.f, 0.f, 0.f};\n float smin = 0;\n for (int l = 0; l < 4; ++l) {\n s.x() += y1[l] * q4[l + 0]; s.y() += y1[l + 32] * q4[l + 4];\n s.z() += y2[l] * q4[l + 8]; s.w() += y2[l + 32] * q4[l + 12];\n smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];\n }\n tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f / 16.f +\n s.z() * sc[4] + s.w() * sc[5] * 1.f / 16.f) -\n dmin * smin;\n#else\n const uint16_t * q1 = (const uint16_t *)(x[i].qs + q_offset);\n const uint16_t * q2 = q1 + 32;\n\n q16[0] = q1[0] & 0x0f0f;\n q16[1] = q1[0] & 0xf0f0;\n q16[2] = q2[0] & 0x0f0f;\n q16[3] = q2[0] & 0xf0f0;\n\n float4 s = {0.f, 0.f, 0.f, 0.f};\n float smin = 0;\n for (int l = 0; l < 2; ++l) {\n s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2];\n s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6];\n smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];\n }\n tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;\n#endif\n\n }\n#else\n const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15\n const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION);\n\n const int step = tid * K_QUANTS_PER_ITERATION;\n\n uint16_t aux16[2];\n const uint8_t * s = (const uint8_t *)aux16;\n\n float tmp = 0;\n\n for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {\n const uint8_t * q = x[i].qs + step;\n const float * y = yy + i*QK_K + step;\n const uint16_t * a = (const uint16_t *)x[i].scales;\n aux16[0] = a[0] & 0x0f0f;\n aux16[1] = (a[0] >> 4) & 0x0f0f;\n const float d = (float)x[i].dm[0];\n const float m = (float)x[i].dm[1];\n float sum = 0.f;\n for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {\n sum += y[j+ 0] * (d * s[0] * (q[j+ 0] & 0xF) - m * s[2])\n + y[j+16] * (d * s[0] * (q[j+16] & 0xF) - m * s[2])\n + y[j+32] * (d * s[1] * (q[j+ 0] >> 4) - m * s[3])\n + y[j+48] * (d * s[1] * (q[j+16] >> 4) - m * s[3]);\n }\n tmp += sum;\n }\n\n#endif\n\n // sum up partial sums and write back result\n#pragma unroll\n for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) {\n tmp +=\n dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);\n }\n\n if (tid == 0) {\n dst[row] = tmp;\n }\n}\n\n/*\nDPCT1110:7: The total declared local variable size in device function\ndequantize_mul_mat_vec_q5_k exceeds 128 bytes and may cause high register\npressure. Consult with your hardware vendor to find the total register size\navailable and adjust the code, or use smaller sub-group size to avoid high\nregister pressure.\n*/\nstatic void dequantize_mul_mat_vec_q5_k(const void *__restrict__ vx,\n const float *__restrict__ yy,\n float *__restrict__ dst,\n const int ncols,\n const sycl::nd_item<3> &item_ct1) {\n\n const int row = item_ct1.get_group(2);\n const int num_blocks_per_row = ncols / QK_K;\n const int ib0 = row*num_blocks_per_row;\n\n const block_q5_K * x = (const block_q5_K *)vx + ib0;\n\n float tmp = 0; // partial sum for thread in warp\n\n#if QK_K == 256\n const uint16_t kmask1 = 0x3f3f;\n const uint16_t kmask2 = 0x0f0f;\n const uint16_t kmask3 = 0xc0c0;\n\n const int tid = item_ct1.get_local_id(2) / 2; // 0...15\n const int ix = item_ct1.get_local_id(2) % 2;\n\n const int il = tid/4; // 0...3\n const int ir = tid - 4*il;// 0...3\n const int n = 2;\n\n const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224\n const int in = il%2;\n\n const int l0 = n*(2*ir + in);\n const int q_offset = 32*im + l0;\n const int y_offset = 64*im + l0;\n\n const uint8_t hm1 = 1 << (2*im);\n const uint8_t hm2 = hm1 << 4;\n\n uint16_t aux[4];\n const uint8_t * sc = (const uint8_t *)aux;\n\n uint16_t q16[8];\n const uint8_t * q4 = (const uint8_t *)q16;\n\n for (int i = ix; i < num_blocks_per_row; i += 2) {\n\n const uint8_t * ql1 = x[i].qs + q_offset;\n const uint8_t * qh = x[i].qh + l0;\n const float * y1 = yy + i*QK_K + y_offset;\n const float * y2 = y1 + 128;\n\n const float dall = x[i].dm[0];\n const float dmin = x[i].dm[1];\n\n const uint16_t * a = (const uint16_t *)x[i].scales;\n aux[0] = a[im+0] & kmask1;\n aux[1] = a[im+2] & kmask1;\n aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);\n aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);\n\n sycl::float4 sum = {0.f, 0.f, 0.f, 0.f};\n float smin = 0;\n const uint16_t * q1 = (const uint16_t *)ql1;\n const uint16_t * q2 = q1 + 32;\n q16[0] = q1[0] & 0x0f0f;\n q16[1] = q1[8] & 0x0f0f;\n q16[2] = (q1[0] >> 4) & 0x0f0f;\n q16[3] = (q1[8] >> 4) & 0x0f0f;\n q16[4] = q2[0] & 0x0f0f;\n q16[5] = q2[8] & 0x0f0f;\n q16[6] = (q2[0] >> 4) & 0x0f0f;\n q16[7] = (q2[8] >> 4) & 0x0f0f;\n for (int l = 0; l < n; ++l) {\n sum.x() +=\n y1[l + 0] * (q4[l + 0] + (qh[l + 0] & (hm1 << 0) ? 16 : 0)) +\n y1[l + 16] * (q4[l + 2] + (qh[l + 16] & (hm1 << 0) ? 16 : 0));\n sum.y() +=\n y1[l + 32] * (q4[l + 4] + (qh[l + 0] & (hm1 << 1) ? 16 : 0)) +\n y1[l + 48] * (q4[l + 6] + (qh[l + 16] & (hm1 << 1) ? 16 : 0));\n sum.z() +=\n y2[l + 0] * (q4[l + 8] + (qh[l + 0] & (hm2 << 0) ? 16 : 0)) +\n y2[l + 16] * (q4[l + 10] + (qh[l + 16] & (hm2 << 0) ? 16 : 0));\n sum.w() +=\n y2[l + 32] * (q4[l + 12] + (qh[l + 0] & (hm2 << 1) ? 16 : 0)) +\n y2[l + 48] * (q4[l + 14] + (qh[l + 16] & (hm2 << 1) ? 16 : 0));\n smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3]\n + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7];\n }\n tmp += dall * (sum.x() * sc[0] + sum.y() * sc[1] + sum.z() * sc[4] +\n sum.w() * sc[5]) -\n dmin * smin;\n }\n\n#else\n const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15\n const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION);\n const int step = tid * K_QUANTS_PER_ITERATION;\n const int im = step/8;\n const int in = step%8;\n\n for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {\n const uint8_t * q = x[i].qs + step;\n const int8_t * s = x[i].scales;\n const float * y = yy + i*QK_K + step;\n const float d = x[i].d;\n float sum = 0.f;\n for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {\n const uint8_t h = x[i].qh[in+j] >> im;\n sum += y[j+ 0] * d * s[0] * ((q[j+ 0] & 0xF) - ((h >> 0) & 1 ? 0 : 16))\n + y[j+16] * d * s[1] * ((q[j+16] & 0xF) - ((h >> 2) & 1 ? 0 : 16))\n + y[j+32] * d * s[2] * ((q[j+ 0] >> 4) - ((h >> 4) & 1 ? 0 : 16))\n + y[j+48] * d * s[3] * ((q[j+16] >> 4) - ((h >> 6) & 1 ? 0 : 16));\n }\n tmp += sum;\n }\n#endif\n\n // sum up partial sums and write back result\n#pragma unroll\n for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) {\n tmp +=\n dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);\n }\n\n if (item_ct1.get_local_id(2) == 0) {\n dst[row] = tmp;\n }\n}\n\nstatic void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows,\n const sycl::nd_item<3> &item_ct1) {\n\n static_assert(16%K_QUANTS_PER_ITERATION == 0, \"16 must be divisible by K_QUANTS_PER_ITERATION\");\n\n const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +\n item_ct1.get_local_id(1);\n if (row > nrows) return;\n\n const int num_blocks_per_row = ncols / QK_K;\n const int ib0 = row*num_blocks_per_row;\n\n const block_q6_K * x = (const block_q6_K *)vx + ib0;\n\n#if QK_K == 256\n\n const int tid =\n item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...16\n const int ix =\n item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0, 1\n\n const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8\n\n const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...\n const int in = tid - step*im; // 0...15 or 0...7\n\n#if K_QUANTS_PER_ITERATION == 1\n const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15\n const int is = 0;\n#else\n const int l0 = 4 * in; // 0, 4, 8, ..., 28\n const int is = in / 4;\n#endif\n const int ql_offset = 64*im + l0;\n const int qh_offset = 32*im + l0;\n const int s_offset = 8*im + is;\n const int y_offset = 128*im + l0;\n\n float tmp = 0; // partial sum for thread in warp\n\n for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {\n\n const float * y = yy + i * QK_K + y_offset;\n const uint8_t * ql = x[i].ql + ql_offset;\n const uint8_t * qh = x[i].qh + qh_offset;\n const int8_t * s = x[i].scales + s_offset;\n\n const float d = x[i].d;\n\n#if K_QUANTS_PER_ITERATION == 1\n float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32)\n + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32)\n + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32)\n + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32)\n + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32)\n + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32)\n + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32)\n +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32);\n tmp += sum;\n#else\n float sum = 0;\n for (int l = 0; l < 4; ++l) {\n sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32)\n + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32)\n + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32)\n + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32);\n }\n tmp += sum;\n#endif\n\n }\n\n#else\n\n const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...7\n const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); // 0...3\n\n const int step = tid * K_QUANTS_PER_ITERATION;\n\n float tmp = 0; // partial sum for thread in warp\n\n for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {\n\n const float * y = yy + i * QK_K + step;\n const uint8_t * ql = x[i].ql + step;\n const uint8_t * qh = x[i].qh + step;\n const int8_t * s = x[i].scales;\n\n const float d = x[i+0].d;\n\n float sum = 0;\n for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {\n sum += y[j+ 0] * s[0] * d * ((int8_t)((ql[j+ 0] & 0xF) | ((qh[j] & 0x03) << 4)) - 32)\n + y[j+16] * s[1] * d * ((int8_t)((ql[j+16] & 0xF) | ((qh[j] & 0x0c) << 2)) - 32)\n + y[j+32] * s[2] * d * ((int8_t)((ql[j+ 0] >> 4) | ((qh[j] & 0x30) >> 0)) - 32)\n + y[j+48] * s[3] * d * ((int8_t)((ql[j+16] >> 4) | ((qh[j] & 0xc0) >> 2)) - 32);\n }\n tmp += sum;\n\n }\n\n#endif\n\n // sum up partial sums and write back result\n#pragma unroll\n for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) {\n tmp +=\n dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);\n }\n\n if (tid == 0) {\n dst[row] = tmp;\n }\n}\n\nstatic void dequantize_mul_mat_vec_q4_0_sycl_reorder(const void *vx, const dfloat *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);\n const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;\n // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n dequantize_mul_mat_vec_reorder(\n vx, y, dst, ncols, nrows, item_ct1);\n });\n }\n}\n\n\nstatic void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);\n const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;\n // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n dequantize_mul_mat_vec(\n vx, y, dst, ncols, nrows, item_ct1);\n });\n }\n}\n\nstatic void dequantize_mul_mat_vec_q4_1_sycl(const void *vx, const dfloat *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);\n const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n dequantize_mul_mat_vec(\n vx, y, dst, ncols, nrows, item_ct1);\n });\n }\n}\n\nstatic void dequantize_mul_mat_vec_q5_0_sycl(const void *vx, const dfloat *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);\n const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n dequantize_mul_mat_vec(\n vx, y, dst, ncols, nrows, item_ct1);\n });\n }\n}\n\nstatic void dequantize_mul_mat_vec_q5_1_sycl(const void *vx, const dfloat *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);\n const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n dequantize_mul_mat_vec(\n vx, y, dst, ncols, nrows, item_ct1);\n });\n }\n}\n\nstatic void dequantize_mul_mat_vec_q8_0_sycl(const void *vx, const dfloat *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);\n const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n dequantize_mul_mat_vec(\n vx, y, dst, ncols, nrows, item_ct1);\n });\n }\n}\n\nstatic void dequantize_mul_mat_vec_q2_K_sycl(const void *vx, const float *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % QK_K == 0);\n const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2\n const int block_num_y = (nrows + ny - 1) / ny;\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE);\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] {\n dequantize_mul_mat_vec_q2_k(vx, y, dst, ncols, nrows, item_ct1);\n });\n}\n\nstatic void dequantize_mul_mat_vec_q3_K_sycl(const void *vx, const float *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % QK_K == 0);\n const int ny = 2 / K_QUANTS_PER_ITERATION;\n const int block_num_y = (nrows + ny - 1) / ny;\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE);\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] {\n dequantize_mul_mat_vec_q3_k(vx, y, dst, ncols, nrows, item_ct1);\n });\n}\n\nstatic void dequantize_mul_mat_vec_q4_K_sycl(const void *vx, const float *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % QK_K == 0);\n const int ny = 2 / K_QUANTS_PER_ITERATION;\n const int block_num_y = (nrows + ny - 1) / ny;\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE);\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] {\n dequantize_mul_mat_vec_q4_k(vx, y, dst, ncols, nrows, item_ct1);\n });\n}\n\nstatic void dequantize_mul_mat_vec_q5_K_sycl(const void *vx, const float *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % QK_K == 0);\n const sycl::range<3> block_dims(1, 1, QK_WARP_SIZE);\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] {\n dequantize_mul_mat_vec_q5_k(vx, y, dst, ncols, item_ct1);\n });\n}\n\nstatic void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y,\n float *dst, const int ncols,\n const int nrows,\n dpct::queue_ptr stream) {\n GGML_ASSERT(ncols % QK_K == 0);\n const int ny = 2 / K_QUANTS_PER_ITERATION;\n const int block_num_y = (nrows + ny - 1) / ny;\n const sycl::range<3> block_nums(1, 1, block_num_y);\n const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE);\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] {\n dequantize_mul_mat_vec_q6_k(vx, y, dst, ncols, nrows, item_ct1);\n });\n}\n\nvoid ggml_sycl_op_dequantize_mul_mat_vec(\n ggml_backend_sycl_context & ctx,\n const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst,\n const char *src0_dd_i, const float *src1_ddf_i, const char *src1_ddq_i,\n float *dst_dd_i, const int64_t row_low, const int64_t row_high,\n const int64_t src1_ncols, const int64_t src1_padded_row_size,\n const dpct::queue_ptr &stream) {\n\n const int64_t ne00 = src0->ne[0];\n const int64_t row_diff = row_high - row_low;\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics\n#ifdef GGML_SYCL_F16\n ggml_sycl_pool_alloc src1_dfloat_a(ctx.pool());\n sycl::half *src1_dfloat = nullptr; // dfloat == half\n\n bool src1_convert_f16 =\n src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||\n src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 ||\n src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16;\n\n if (src1_convert_f16) {\n scope_op_debug_print scope_dbg_print(__func__, \"/to_fp16_sycl\", dst, /*num_src=*/2,\n \" : converting src1 to fp16\");\n src1_dfloat = src1_dfloat_a.alloc(ne00);\n const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst);\n GGML_ASSERT(to_fp16_sycl != nullptr);\n to_fp16_sycl(src1_ddf_i, src1_dfloat, ne00, stream);\n }\n#else\n const dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion\n#endif // GGML_SYCL_F16\n\n switch (src0->type) {\n case GGML_TYPE_Q4_0:\n if ((ggml_tensor_extra_gpu*)dst->src[0]->extra &&\n ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {\n dequantize_mul_mat_vec_q4_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);\n } else {\n dequantize_mul_mat_vec_q4_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);\n }\n break;\n case GGML_TYPE_Q4_1:\n dequantize_mul_mat_vec_q4_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);\n break;\n case GGML_TYPE_Q5_0:\n dequantize_mul_mat_vec_q5_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);\n break;\n case GGML_TYPE_Q5_1:\n dequantize_mul_mat_vec_q5_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);\n break;\n case GGML_TYPE_Q8_0:\n dequantize_mul_mat_vec_q8_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);\n break;\n case GGML_TYPE_Q2_K:\n dequantize_mul_mat_vec_q2_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);\n break;\n case GGML_TYPE_Q3_K:\n dequantize_mul_mat_vec_q3_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);\n break;\n case GGML_TYPE_Q4_K:\n if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&\n ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {\n // reorder is currently not supported for dmmv\n GGML_ABORT(\"Unimplemented dequantize case case for q4_k reorder\");\n } else {\n dequantize_mul_mat_vec_q4_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);\n }\n break;\n case GGML_TYPE_Q5_K:\n dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);\n break;\n case GGML_TYPE_Q6_K:\n dequantize_mul_mat_vec_q6_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);\n break;\n case GGML_TYPE_F16:\n convert_mul_mat_vec_f16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);\n break;\n default:\n printf(\"ggml_sycl_op_dequantize_mul_mat_vec unsupported GGML_TYPE %d\\n\", src0->type);\n GGML_ABORT(\"fatal error\");\n }\n\n GGML_UNUSED(src1);\n GGML_UNUSED(dst);\n GGML_UNUSED(src1_ddq_i);\n GGML_UNUSED(src1_ncols);\n GGML_UNUSED(src1_padded_row_size);\n GGML_UNUSED(ctx);\n}\n" }, { "path": "src/ggml-sycl/dmmv.hpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_DMMV_HPP\n#define GGML_SYCL_DMMV_HPP\n\n#include \"common.hpp\"\n\n\nvoid ggml_sycl_op_dequantize_mul_mat_vec(\n ggml_backend_sycl_context & ctx,\n const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst,\n const char *src0_dd_i, const float *src1_ddf_i, const char *src1_ddq_i,\n float *dst_dd_i, const int64_t row_low, const int64_t row_high,\n const int64_t src1_ncols, const int64_t src1_padded_row_size,\n const dpct::queue_ptr &stream);\n\n#endif // GGML_SYCL_DMMV_HPP\n" }, { "path": "src/ggml-sycl/dpct/helper.hpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_DPCT_HELPER_HPP\n#define GGML_SYCL_DPCT_HELPER_HPP\n\n#include \n#include \n#include \n\n#include \n\n#include \"ggml.h\"\n\n#if defined(__linux__)\n#include \n#elif defined(_WIN64)\n#ifndef NOMINMAX\n#define NOMINMAX\n#endif\n#include \n#else\n#error \"Only support Windows and Linux.\"\n#endif\n\n#if defined(__linux__)\n#include \n#include \n#endif\n#if defined(_WIN64)\n#ifndef NOMINMAX\n#define NOMINMAX\n#endif\n#include \n#endif\n\n#define DPCT_COMPATIBILITY_TEMP (900)\n\n#if defined(_MSC_VER)\n#define __dpct_align__(n) __declspec(align(n))\n#define __dpct_inline__ __forceinline\n#else\n#define __dpct_align__(n) __attribute__((aligned(n)))\n#define __dpct_inline__ __inline__ __attribute__((always_inline))\n#endif\n\n#if defined(_MSC_VER)\n#define __dpct_noinline__ __declspec(noinline)\n#else\n#define __dpct_noinline__ __attribute__((noinline))\n#endif\n\ninline std::string get_device_type_name(const sycl::device &Device) {\n auto DeviceType = Device.get_info();\n switch (DeviceType) {\n case sycl::info::device_type::cpu:\n return \"cpu\";\n case sycl::info::device_type::gpu:\n return \"gpu\";\n case sycl::info::device_type::host:\n return \"host\";\n case sycl::info::device_type::accelerator:\n return \"acc\";\n default:\n return \"unknown\";\n }\n}\n\ninline std::string get_device_backend_and_type(const sycl::device &device) {\n std::stringstream device_type;\n sycl::backend backend = device.get_backend();\n device_type << backend << \":\" << get_device_type_name(device);\n return device_type.str();\n}\n\ntemplate struct matrix_info_t {\n oneapi::mkl::transpose transpose_info[2];\n Ts value_info[2];\n std::int64_t size_info[3];\n std::int64_t ld_info[3];\n std::int64_t groupsize_info;\n};\n\nnamespace dpct\n{\n typedef sycl::queue *queue_ptr;\n typedef sycl::event *event_ptr;\n typedef char *device_ptr;\n typedef uint8_t byte_t;\n typedef sycl::buffer buffer_t;\n\n /// SYCL default exception handler\n inline auto exception_handler = [](sycl::exception_list exceptions)\n {\n for (std::exception_ptr const &e : exceptions)\n {\n try\n {\n std::rethrow_exception(e);\n }\n catch (sycl::exception const &e)\n {\n std::cerr << \"Caught asynchronous SYCL exception:\" << std::endl\n << e.what() << std::endl\n << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n }\n }\n };\n\n enum error_code\n {\n success = 0,\n default_error = 999\n };\n\n enum memcpy_direction\n {\n host_to_host,\n host_to_device,\n device_to_host,\n device_to_device,\n automatic\n };\n\n enum memory_region\n {\n global = 0, // device global memory\n constant, // device constant memory\n local, // device local memory\n shared, // memory which can be accessed by host and device\n };\n\n enum class library_data_t : unsigned char\n {\n real_float = 0,\n complex_float,\n real_double,\n complex_double,\n real_half,\n complex_half,\n real_bfloat16,\n complex_bfloat16,\n real_int4,\n complex_int4,\n real_uint4,\n complex_uint4,\n real_int8,\n complex_int8,\n real_uint8,\n complex_uint8,\n real_int16,\n complex_int16,\n real_uint16,\n complex_uint16,\n real_int32,\n complex_int32,\n real_uint32,\n complex_uint32,\n real_int64,\n complex_int64,\n real_uint64,\n complex_uint64,\n real_int8_4,\n real_int8_32,\n real_uint8_4,\n library_data_t_size\n };\n\n template \n struct DataType\n {\n using T2 = T;\n };\n template \n struct DataType>\n {\n using T2 = std::complex;\n };\n\n static void destroy_event(event_ptr event)\n {\n delete event;\n }\n\n static inline unsigned int get_tid()\n {\n#if defined(__linux__)\n return syscall(SYS_gettid);\n#elif defined(_WIN64)\n return GetCurrentThreadId();\n#else\n#error \"Only support Windows and Linux.\"\n#endif\n }\n\n namespace detail\n {\n static void get_version(const sycl::device &dev, int &major, int &minor)\n {\n // Version string has the following format:\n // a. OpenCL\n // b. \n // c. e.g gfx1030\n std::string ver;\n ver = dev.get_info();\n std::string::size_type i = 0;\n while (i < ver.size()) {\n if (isdigit(ver[i]))\n break;\n i++;\n }\n major = std::stoi(&(ver[i]));\n while (i < ver.size()) {\n if (ver[i] == '.')\n break;\n i++;\n }\n if (i < ver.size()) {\n // a. and b.\n i++;\n minor = std::stoi(&(ver[i]));\n } else {\n // c.\n minor = 0;\n }\n }\n\n template \n class generic_error_type\n {\n public:\n generic_error_type() = default;\n generic_error_type(T value) : value{value} {}\n operator T() const { return value; }\n\n private:\n T value;\n };\n\n } // namespace detail\n\n // COPY from DPCT head files\n /// dim3 is used to store 3 component dimensions.\n class dim3 {\n public:\n unsigned x, y, z;\n\n constexpr dim3(unsigned x = 1, unsigned y = 1, unsigned z = 1)\n : x(x), y(y), z(z) {}\n\n dim3(const sycl::id<3> &r) : dim3(r[2], r[1], r[0]) {}\n\n operator sycl::range<3>() const { return sycl::range<3>(z, y, x); }\n }; // namespace dim3\n\n inline dim3 operator*(const dim3 &a, const dim3 &b) {\n return dim3{a.x * b.x, a.y * b.y, a.z * b.z};\n }\n // COPY from DPCT head files\n\n\n /// Pitched 2D/3D memory data.\n class pitched_data\n {\n public:\n pitched_data() : pitched_data(nullptr, 0, 0, 0) {}\n pitched_data(void *data, size_t pitch, size_t x, size_t y)\n : _data(data), _pitch(pitch), _x(x), _y(y) {}\n\n void *get_data_ptr() { return _data; }\n void set_data_ptr(void *data) { _data = data; }\n\n size_t get_pitch() { return _pitch; }\n void set_pitch(size_t pitch) { _pitch = pitch; }\n\n size_t get_x() { return _x; }\n void set_x(size_t x) { _x = x; }\n\n size_t get_y() { return _y; }\n void set_y(size_t y) { _y = y; }\n\n private:\n void *_data;\n size_t _pitch, _x, _y;\n };\n\n class device_info\n {\n public:\n // get interface\n const char *get_name() const { return _name; }\n char *get_name() { return _name; }\n template ,\n std::enable_if_t> ||\n std::is_same_v,\n int> = 0>\n auto get_max_work_item_sizes() const\n {\n if constexpr (std::is_same_v>)\n return sycl::range<3>(_max_work_item_sizes_i[0],\n _max_work_item_sizes_i[1],\n _max_work_item_sizes_i[2]);\n else\n {\n return _max_work_item_sizes_i;\n }\n }\n template ,\n std::enable_if_t> ||\n std::is_same_v,\n int> = 0>\n auto get_max_work_item_sizes()\n {\n if constexpr (std::is_same_v>)\n return sycl::range<3>(_max_work_item_sizes_i[0],\n _max_work_item_sizes_i[1],\n _max_work_item_sizes_i[2]);\n else\n {\n return _max_work_item_sizes_i;\n }\n }\n bool get_host_unified_memory() const { return _host_unified_memory; }\n int get_major_version() const { return _major; }\n int get_minor_version() const { return _minor; }\n int get_integrated() const { return _integrated; }\n int get_max_clock_frequency() const { return _frequency; }\n int get_max_compute_units() const { return _max_compute_units; }\n int get_max_work_group_size() const { return _max_work_group_size; }\n int get_max_sub_group_size() const { return _max_sub_group_size; }\n int get_max_work_items_per_compute_unit() const\n {\n return _max_work_items_per_compute_unit;\n }\n int get_max_register_size_per_work_group() const\n {\n return _max_register_size_per_work_group;\n }\n template ||\n std::is_same_v,\n int> = 0>\n auto get_max_nd_range_size() const\n {\n if constexpr (std::is_same_v)\n return _max_nd_range_size;\n else\n return _max_nd_range_size_i;\n }\n template ||\n std::is_same_v,\n int> = 0>\n auto get_max_nd_range_size()\n {\n if constexpr (std::is_same_v)\n return _max_nd_range_size;\n else\n return _max_nd_range_size_i;\n }\n size_t get_global_mem_size() const { return _global_mem_size; }\n size_t get_local_mem_size() const { return _local_mem_size; }\n size_t get_max_mem_alloc_size() const { return _max_mem_alloc_size; }\n /// Returns the maximum clock rate of device's global memory in kHz. If\n /// compiler does not support this API then returns default value 3200000 kHz.\n unsigned int get_memory_clock_rate() const { return _memory_clock_rate; }\n /// Returns the maximum bus width between device and memory in bits. If\n /// compiler does not support this API then returns default value 64 bits.\n unsigned int get_memory_bus_width() const { return _memory_bus_width; }\n uint32_t get_device_id() const { return _device_id; }\n std::array get_uuid() const { return _uuid; }\n /// Returns global memory cache size in bytes.\n unsigned int get_global_mem_cache_size() const\n {\n return _global_mem_cache_size;\n }\n\n // set interface\n void set_name(const char *name)\n {\n size_t length = strlen(name);\n if (length < 256)\n {\n std::memcpy(_name, name, length + 1);\n }\n else\n {\n std::memcpy(_name, name, 255);\n _name[255] = '\\0';\n }\n }\n void set_max_work_item_sizes(const sycl::range<3> max_work_item_sizes)\n {\n for (int i = 0; i < 3; ++i)\n _max_work_item_sizes_i[i] = max_work_item_sizes[i];\n }\n [[deprecated]] void\n set_max_work_item_sizes(const sycl::id<3> max_work_item_sizes)\n {\n for (int i = 0; i < 3; ++i)\n {\n _max_work_item_sizes_i[i] = max_work_item_sizes[i];\n }\n }\n void set_host_unified_memory(bool host_unified_memory)\n {\n _host_unified_memory = host_unified_memory;\n }\n void set_major_version(int major) { _major = major; }\n void set_minor_version(int minor) { _minor = minor; }\n void set_integrated(int integrated) { _integrated = integrated; }\n void set_max_clock_frequency(int frequency) { _frequency = frequency; }\n void set_max_compute_units(int max_compute_units)\n {\n _max_compute_units = max_compute_units;\n }\n void set_global_mem_size(size_t global_mem_size)\n {\n _global_mem_size = global_mem_size;\n }\n void set_local_mem_size(size_t local_mem_size)\n {\n _local_mem_size = local_mem_size;\n }\n void set_max_mem_alloc_size(size_t max_mem_alloc_size)\n {\n _max_mem_alloc_size = max_mem_alloc_size;\n }\n void set_max_work_group_size(int max_work_group_size)\n {\n _max_work_group_size = max_work_group_size;\n }\n void set_max_sub_group_size(int max_sub_group_size)\n {\n _max_sub_group_size = max_sub_group_size;\n }\n void\n set_max_work_items_per_compute_unit(int max_work_items_per_compute_unit)\n {\n _max_work_items_per_compute_unit = max_work_items_per_compute_unit;\n }\n void set_max_nd_range_size(int max_nd_range_size[])\n {\n for (int i = 0; i < 3; i++)\n {\n _max_nd_range_size[i] = max_nd_range_size[i];\n _max_nd_range_size_i[i] = max_nd_range_size[i];\n }\n }\n void set_memory_clock_rate(unsigned int memory_clock_rate)\n {\n _memory_clock_rate = memory_clock_rate;\n }\n void set_memory_bus_width(unsigned int memory_bus_width)\n {\n _memory_bus_width = memory_bus_width;\n }\n void\n set_max_register_size_per_work_group(int max_register_size_per_work_group)\n {\n _max_register_size_per_work_group = max_register_size_per_work_group;\n }\n void set_device_id(uint32_t device_id)\n {\n _device_id = device_id;\n }\n void set_uuid(std::array uuid)\n {\n _uuid = std::move(uuid);\n }\n void set_global_mem_cache_size(unsigned int global_mem_cache_size)\n {\n _global_mem_cache_size = global_mem_cache_size;\n }\n\n private:\n char _name[256];\n int _max_work_item_sizes_i[3];\n bool _host_unified_memory = false;\n int _major;\n int _minor;\n int _integrated = 0;\n int _frequency;\n // Set estimated value 3200000 kHz as default value.\n unsigned int _memory_clock_rate = 3200000;\n // Set estimated value 64 bits as default value.\n unsigned int _memory_bus_width = 64;\n unsigned int _global_mem_cache_size;\n int _max_compute_units;\n int _max_work_group_size;\n int _max_sub_group_size;\n int _max_work_items_per_compute_unit;\n int _max_register_size_per_work_group;\n size_t _global_mem_size;\n size_t _local_mem_size;\n size_t _max_mem_alloc_size;\n size_t _max_nd_range_size[3];\n int _max_nd_range_size_i[3];\n uint32_t _device_id;\n std::array _uuid;\n };\n\n static int get_major_version(const sycl::device &dev)\n {\n int major, minor;\n detail::get_version(dev, major, minor);\n return major;\n }\n\n static int get_minor_version(const sycl::device &dev)\n {\n int major, minor;\n detail::get_version(dev, major, minor);\n return minor;\n }\n\n static void get_device_info(device_info &out, const sycl::device &dev)\n {\n device_info prop;\n prop.set_name(dev.get_info().c_str());\n\n int major, minor;\n detail::get_version(dev, major, minor);\n prop.set_major_version(major);\n prop.set_minor_version(minor);\n\n prop.set_max_work_item_sizes(\n#if (__SYCL_COMPILER_VERSION && __SYCL_COMPILER_VERSION < 20220902)\n // oneAPI DPC++ compiler older than 2022/09/02, where max_work_item_sizes\n // is an enum class element\n dev.get_info());\n#else\n // SYCL 2020-conformant code, max_work_item_sizes is a struct templated by\n // an int\n dev.get_info>());\n#endif\n prop.set_host_unified_memory(dev.has(sycl::aspect::usm_host_allocations));\n\n prop.set_max_clock_frequency(\n dev.get_info() * 1000);\n\n prop.set_max_compute_units(\n dev.get_info());\n prop.set_max_work_group_size(\n dev.get_info());\n prop.set_global_mem_size(dev.get_info());\n prop.set_local_mem_size(dev.get_info());\n prop.set_max_mem_alloc_size(dev.get_info());\n\n#if (defined(SYCL_EXT_INTEL_DEVICE_INFO) && SYCL_EXT_INTEL_DEVICE_INFO >= 6)\n if (dev.has(sycl::aspect::ext_intel_memory_clock_rate))\n {\n unsigned int tmp =\n dev.get_info();\n if (tmp != 0)\n prop.set_memory_clock_rate(1000 * tmp);\n }\n if (dev.has(sycl::aspect::ext_intel_memory_bus_width))\n {\n prop.set_memory_bus_width(\n dev.get_info());\n }\n if (dev.has(sycl::aspect::ext_intel_device_id))\n {\n prop.set_device_id(\n dev.get_info());\n }\n if (dev.has(sycl::aspect::ext_intel_device_info_uuid))\n {\n prop.set_uuid(dev.get_info());\n }\n#elif defined(_MSC_VER) && !defined(__clang__)\n#pragma message(\"get_device_info: querying memory_clock_rate and \\\n memory_bus_width are not supported by the compiler used. \\\n Use 3200000 kHz as memory_clock_rate default value. \\\n Use 64 bits as memory_bus_width default value.\")\n#else\n#warning \"get_device_info: querying memory_clock_rate and \\\n memory_bus_width are not supported by the compiler used. \\\n Use 3200000 kHz as memory_clock_rate default value. \\\n Use 64 bits as memory_bus_width default value.\"\n#endif\n\n size_t max_sub_group_size = 1;\n std::vector sub_group_sizes =\n dev.get_info();\n\n for (const auto &sub_group_size : sub_group_sizes)\n {\n if (max_sub_group_size < sub_group_size)\n max_sub_group_size = sub_group_size;\n }\n\n prop.set_max_sub_group_size(max_sub_group_size);\n\n prop.set_max_work_items_per_compute_unit(\n dev.get_info());\n int max_nd_range_size[] = {0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF};\n prop.set_max_nd_range_size(max_nd_range_size);\n\n // Estimates max register size per work group, feel free to update the value\n // according to device properties.\n prop.set_max_register_size_per_work_group(65536);\n\n prop.set_global_mem_cache_size(\n dev.get_info());\n out = prop;\n }\n\n /// dpct device extension\n class device_ext : public sycl::device {\n typedef std::mutex mutex_type;\n\n public:\n device_ext() : sycl::device() {}\n ~device_ext() {\n std::lock_guard lock(m_mutex);\n clear_queues();\n }\n device_ext(const sycl::device &base) : sycl::device(base) {\n std::lock_guard lock(m_mutex);\n init_queues();\n }\n\n int is_native_atomic_supported() { return 0; }\n int get_major_version() const { return dpct::get_major_version(*this); }\n\n int get_minor_version() const { return dpct::get_minor_version(*this); }\n\n int get_max_compute_units() const {\n return get_device_info().get_max_compute_units();\n }\n\n /// Return the maximum clock frequency of this device in KHz.\n int get_max_clock_frequency() const {\n return get_device_info().get_max_clock_frequency();\n }\n\n int get_integrated() const { return get_device_info().get_integrated(); }\n\n int get_max_sub_group_size() const {\n return get_device_info().get_max_sub_group_size();\n }\n\n int get_max_register_size_per_work_group() const {\n return get_device_info().get_max_register_size_per_work_group();\n }\n\n int get_max_work_group_size() const {\n return get_device_info().get_max_work_group_size();\n }\n\n int get_mem_base_addr_align() const {\n return get_info();\n }\n\n size_t get_global_mem_size() const {\n return get_device_info().get_global_mem_size();\n }\n\n size_t get_max_mem_alloc_size() const {\n return get_device_info().get_max_mem_alloc_size();\n }\n\n /// Get the number of bytes of free and total memory on the SYCL device.\n /// \\param [out] free_memory The number of bytes of free memory on the\n /// SYCL device. \\param [out] total_memory The number of bytes of total\n /// memory on the SYCL device.\n void get_memory_info(size_t &free_memory, size_t &total_memory) {\n total_memory = get_device_info().get_global_mem_size();\n const char *warning_info =\n \"get_memory_info: [warning] ext_intel_free_memory is not \"\n \"supported (export/set ZES_ENABLE_SYSMAN=1 to support), \"\n \"use total memory as free memory\";\n#if (defined(__SYCL_COMPILER_VERSION) && __SYCL_COMPILER_VERSION >= 20221105)\n if (!has(sycl::aspect::ext_intel_free_memory)) {\n std::cerr << warning_info << std::endl;\n free_memory = total_memory;\n } else {\n free_memory = get_info();\n }\n#else\n std::cerr << warning_info << std::endl;\n free_memory = total_memory;\n#if defined(_MSC_VER) && !defined(__clang__)\n#pragma message(\"Querying the number of bytes of free memory is not supported\")\n#else\n#warning \"Querying the number of bytes of free memory is not supported\"\n#endif\n#endif\n }\n\n void get_device_info(device_info &out) const {\n dpct::get_device_info(out, *this);\n }\n\n device_info get_device_info() const {\n device_info prop;\n dpct::get_device_info(prop, *this);\n return prop;\n }\n\n void reset() {\n std::lock_guard lock(m_mutex);\n clear_queues();\n init_queues();\n }\n\n sycl::queue &in_order_queue() { return _q_in_order; }\n\n sycl::queue &out_of_order_queue() { return _q_out_of_order; }\n\n sycl::queue &default_queue() { return in_order_queue(); }\n\n void queues_wait_and_throw() {\n std::unique_lock lock(m_mutex);\n lock.unlock();\n for (auto &q : _queues) {\n q.wait_and_throw();\n }\n // Guard the destruct of current_queues to make sure the ref count is\n // safe.\n lock.lock();\n }\n\n sycl::queue create_queue(bool enable_exception_handler = false) {\n return create_in_order_queue(enable_exception_handler);\n }\n\n sycl::queue create_queue(sycl::device device,\n bool enable_exception_handler = false) {\n return create_in_order_queue(device, enable_exception_handler);\n }\n\n sycl::queue create_in_order_queue(bool enable_exception_handler = false) {\n std::lock_guard lock(m_mutex);\n return create_queue_impl(enable_exception_handler,\n sycl::property::queue::in_order());\n }\n\n sycl::queue create_in_order_queue(sycl::device device,\n bool enable_exception_handler = false) {\n std::lock_guard lock(m_mutex);\n return create_queue_impl(device, enable_exception_handler,\n sycl::property::queue::in_order());\n }\n\n sycl::queue create_out_of_order_queue(\n bool enable_exception_handler = false) {\n std::lock_guard lock(m_mutex);\n return create_queue_impl(enable_exception_handler);\n }\n\n void destroy_queue(sycl::queue queue) {\n std::lock_guard lock(m_mutex);\n _queues.erase(std::remove_if(_queues.begin(), _queues.end(),\n [=](const sycl::queue &q) -> bool\n {\n return q == queue;\n }),\n _queues.end());\n }\n void set_saved_queue(sycl::queue q) {\n std::lock_guard lock(m_mutex);\n _saved_queue = q;\n }\n sycl::queue get_saved_queue() const {\n std::lock_guard lock(m_mutex);\n return _saved_queue;\n }\n\n private:\n void clear_queues() { _queues.clear(); }\n\n void init_queues() {\n _q_in_order =\n create_queue_impl(true, sycl::property::queue::in_order());\n _q_out_of_order = create_queue_impl(true);\n _saved_queue = default_queue();\n }\n\n /// Caller should acquire resource \\p m_mutex before calling this\n /// function.\n template \n sycl::queue create_queue_impl(bool enable_exception_handler,\n Properties... properties) {\n sycl::async_handler eh = {};\n if (enable_exception_handler) {\n eh = exception_handler;\n }\n _queues.push_back(sycl::queue(\n *this, eh,\n sycl::property_list(\n#ifdef DPCT_PROFILING_ENABLED\n sycl::property::queue::enable_profiling(),\n#endif\n properties...)));\n\n return _queues.back();\n }\n\n template \n sycl::queue create_queue_impl(sycl::device device,\n bool enable_exception_handler,\n Properties... properties) {\n sycl::async_handler eh = {};\n if (enable_exception_handler) {\n eh = exception_handler;\n }\n _queues.push_back(sycl::queue(\n device, eh,\n sycl::property_list(\n#ifdef DPCT_PROFILING_ENABLED\n sycl::property::queue::enable_profiling(),\n#endif\n properties...)));\n\n return _queues.back();\n }\n\n void get_version(int &major, int &minor) const {\n detail::get_version(*this, major, minor);\n }\n sycl::queue _q_in_order, _q_out_of_order;\n sycl::queue _saved_queue;\n std::vector _queues;\n mutable mutex_type m_mutex;\n };\n\n\n /// device manager\n class dev_mgr\n {\n public:\n device_ext ¤t_device()\n {\n unsigned int dev_id = current_device_id();\n check_id(dev_id);\n return *_devs[dev_id];\n }\n device_ext &cpu_device() const\n {\n std::lock_guard lock(m_mutex);\n if (_cpu_device == -1)\n {\n throw std::runtime_error(\"no valid cpu device\");\n }\n else\n {\n return *_devs[_cpu_device];\n }\n }\n device_ext &get_device(unsigned int id) const\n {\n std::lock_guard lock(m_mutex);\n check_id(id);\n return *_devs[id];\n }\n unsigned int current_device_id() const\n {\n std::lock_guard lock(m_mutex);\n auto it = _thread2dev_map.find(get_tid());\n if (it != _thread2dev_map.end())\n return it->second;\n return DEFAULT_DEVICE_ID;\n }\n\n /// Select device with a device ID.\n /// \\param [in] id The id of the device which can\n /// be obtained through get_device_id(const sycl::device).\n void select_device(unsigned int id)\n {\n std::lock_guard lock(m_mutex);\n check_id(id);\n _thread2dev_map[get_tid()] = id;\n }\n unsigned int device_count() { return _devs.size(); }\n\n unsigned int get_device_id(const sycl::device &dev)\n {\n unsigned int id = 0;\n for (auto &dev_item : _devs)\n {\n if (*dev_item == dev)\n {\n return id;\n }\n id++;\n }\n return -1;\n }\n\n inline std::string get_preferred_gpu_platform_name() {\n std::string result;\n\n std::string filter = \"\";\n char* env = getenv(\"ONEAPI_DEVICE_SELECTOR\");\n if (env) {\n if (std::strstr(env, \"level_zero\")) {\n filter = \"level-zero\";\n }\n else if (std::strstr(env, \"opencl\")) {\n filter = \"opencl\";\n }\n else if (std::strstr(env, \"cuda\")) {\n filter = \"cuda\";\n }\n else if (std::strstr(env, \"hip\")) {\n filter = \"hip\";\n }\n else {\n throw std::runtime_error(\"invalid device filter: \" + std::string(env));\n }\n } else {\n auto default_device = sycl::device(sycl::default_selector_v);\n auto default_platform_name = default_device.get_platform().get_info();\n\n if (std::strstr(default_platform_name.c_str(), \"Level-Zero\") || default_device.is_cpu()) {\n filter = \"level-zero\";\n }\n else if (std::strstr(default_platform_name.c_str(), \"CUDA\")) {\n filter = \"cuda\";\n }\n else if (std::strstr(default_platform_name.c_str(), \"HIP\")) {\n filter = \"hip\";\n }\n }\n\n auto platform_list = sycl::platform::get_platforms();\n\n for (const auto& platform : platform_list) {\n auto devices = platform.get_devices();\n auto gpu_dev = std::find_if(devices.begin(), devices.end(), [](const sycl::device& d) {\n return d.is_gpu();\n });\n\n if (gpu_dev == devices.end()) {\n // cout << \"platform [\" << platform_name\n // << \"] does not contain GPU devices, skipping\\n\";\n continue;\n }\n\n auto platform_name = platform.get_info();\n std::string platform_name_low_case;\n platform_name_low_case.resize(platform_name.size());\n\n std::transform(\n platform_name.begin(), platform_name.end(), platform_name_low_case.begin(), ::tolower);\n\n if (platform_name_low_case.find(filter) == std::string::npos) {\n // cout << \"platform [\" << platform_name\n // << \"] does not match with requested \"\n // << filter << \", skipping\\n\";\n continue;\n }\n\n result = platform_name;\n }\n\n if (result.empty())\n throw std::runtime_error(\"can not find preferred GPU platform\");\n\n return result;\n }\n\n template \n std::enable_if_t<\n std::is_invocable_r_v>\n select_device(const DeviceSelector &selector = sycl::gpu_selector_v)\n {\n sycl::device selected_device = sycl::device(selector);\n unsigned int selected_device_id = get_device_id(selected_device);\n select_device(selected_device_id);\n }\n\n /// Returns the instance of device manager singleton.\n static dev_mgr &instance()\n {\n static dev_mgr d_m;\n return d_m;\n }\n dev_mgr(const dev_mgr &) = delete;\n dev_mgr &operator=(const dev_mgr &) = delete;\n dev_mgr(dev_mgr &&) = delete;\n dev_mgr &operator=(dev_mgr &&) = delete;\n\n private:\n mutable std::recursive_mutex m_mutex;\n static bool compare_dev(sycl::device &device1, sycl::device &device2)\n {\n sycl::backend backend1 = device1.get_backend();\n sycl::backend backend2 = device2.get_backend();\n // levelzero backends always come first\n if(backend1 == sycl::backend::ext_oneapi_level_zero && backend2 != sycl::backend::ext_oneapi_level_zero) return true;\n if(backend1 != sycl::backend::ext_oneapi_level_zero && backend2 == sycl::backend::ext_oneapi_level_zero) return false;\n dpct::device_info prop1;\n dpct::get_device_info(prop1, device1);\n dpct::device_info prop2;\n dpct::get_device_info(prop2, device2);\n return prop1.get_max_compute_units() > prop2.get_max_compute_units();\n }\n static int convert_backend_index(std::string & backend) {\n if (backend == \"ext_oneapi_level_zero:gpu\") return 0;\n if (backend == \"opencl:gpu\") return 1;\n if (backend == \"ext_oneapi_cuda:gpu\") return 2;\n if (backend == \"ext_oneapi_hip:gpu\") return 3;\n if (backend == \"opencl:cpu\") return 4;\n if (backend == \"opencl:acc\") return 5;\n printf(\"convert_backend_index: can't handle backend=%s\\n\", backend.c_str());\n GGML_ABORT(\"fatal error\");\n }\n static bool compare_backend(std::string &backend1, std::string &backend2) {\n return convert_backend_index(backend1) < convert_backend_index(backend2);\n }\n dev_mgr()\n {\n sycl::device default_device =\n sycl::device(sycl::default_selector_v);\n _devs.push_back(std::make_shared(default_device));\n\n std::vector sycl_all_devs;\n // Collect other devices except for the default device.\n if (default_device.is_cpu())\n _cpu_device = 0;\n\n auto Platforms = sycl::platform::get_platforms();\n // Keep track of the number of devices per backend\n std::map DeviceNums;\n std::map> backend_devices;\n auto preferred_platform_name = get_preferred_gpu_platform_name();\n\n while (!Platforms.empty()) {\n auto Platform = Platforms.back();\n Platforms.pop_back();\n auto platform_name = Platform.get_info();\n if (platform_name.compare(preferred_platform_name) != 0) {\n continue;\n }\n auto devices = Platform.get_devices();\n std::string backend_type = get_device_backend_and_type(devices[0]);\n for (const auto &device : devices) {\n backend_devices[backend_type].push_back(device);\n }\n }\n\n std::vector keys;\n for(auto it = backend_devices.begin(); it != backend_devices.end(); ++it) {\n keys.push_back(it->first);\n }\n std::sort(keys.begin(), keys.end(), compare_backend);\n\n for (auto &key : keys) {\n std::vector devs = backend_devices[key];\n std::sort(devs.begin(), devs.end(), compare_dev);\n for (const auto &dev : devs) {\n sycl_all_devs.push_back(dev);\n }\n }\n\n for (auto &dev : sycl_all_devs)\n {\n if (dev == default_device)\n {\n continue;\n }\n _devs.push_back(std::make_shared(dev));\n if (_cpu_device == -1 && dev.is_cpu())\n {\n _cpu_device = _devs.size() - 1;\n }\n }\n }\n void check_id(unsigned int id) const\n {\n if (id >= _devs.size())\n {\n throw std::runtime_error(\"invalid device id\");\n }\n }\n std::vector> _devs;\n /// DEFAULT_DEVICE_ID is used, if current_device_id() can not find current\n /// thread id in _thread2dev_map, which means default device should be used\n /// for the current thread.\n const unsigned int DEFAULT_DEVICE_ID = 0;\n /// thread-id to device-id map.\n std::map _thread2dev_map;\n int _cpu_device = -1;\n };\n\n static inline sycl::queue &get_default_queue()\n {\n return dev_mgr::instance().current_device().default_queue();\n }\n\n namespace detail\n {\n enum class pointer_access_attribute\n {\n host_only = 0,\n device_only,\n host_device,\n end\n };\n\n static pointer_access_attribute get_pointer_attribute(sycl::queue &q,\n const void *ptr)\n {\n switch (sycl::get_pointer_type(ptr, q.get_context()))\n {\n case sycl::usm::alloc::unknown:\n return pointer_access_attribute::host_only;\n case sycl::usm::alloc::device:\n return pointer_access_attribute::device_only;\n case sycl::usm::alloc::shared:\n case sycl::usm::alloc::host:\n return pointer_access_attribute::host_device;\n }\n }\n\n template \n inline constexpr std::uint64_t get_type_combination_id(ArgT Val)\n {\n static_assert((unsigned char)library_data_t::library_data_t_size <=\n std::numeric_limits::max() &&\n \"library_data_t size exceeds limit.\");\n static_assert(std::is_same_v, \"Unsupported ArgT\");\n return (std::uint64_t)Val;\n }\n\n template \n inline constexpr std::uint64_t get_type_combination_id(FirstT FirstVal,\n RestT... RestVal)\n {\n static_assert((std::uint8_t)library_data_t::library_data_t_size <=\n std::numeric_limits::max() &&\n \"library_data_t size exceeds limit.\");\n static_assert(sizeof...(RestT) <= 8 && \"Too many parameters\");\n static_assert(std::is_same_v, \"Unsupported FirstT\");\n return get_type_combination_id(RestVal...) << 8 | ((std::uint64_t)FirstVal);\n }\n\n class mem_mgr\n {\n mem_mgr()\n {\n // Reserved address space, no real memory allocation happens here.\n#if defined(__linux__)\n mapped_address_space =\n (byte_t *)mmap(nullptr, mapped_region_size, PROT_NONE,\n MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);\n#elif defined(_WIN64)\n mapped_address_space = (byte_t *)VirtualAlloc(\n NULL, // NULL specified as the base address parameter\n mapped_region_size, // Size of allocation\n MEM_RESERVE, // Allocate reserved pages\n PAGE_NOACCESS); // Protection = no access\n#else\n#error \"Only support Windows and Linux.\"\n#endif\n next_free = mapped_address_space;\n }\n\n public:\n using buffer_id_t = int;\n\n struct allocation\n {\n buffer_t buffer;\n byte_t *alloc_ptr;\n size_t size;\n };\n\n ~mem_mgr()\n {\n#if defined(__linux__)\n munmap(mapped_address_space, mapped_region_size);\n#elif defined(_WIN64)\n VirtualFree(mapped_address_space, 0, MEM_RELEASE);\n#else\n#error \"Only support Windows and Linux.\"\n#endif\n }\n\n mem_mgr(const mem_mgr &) = delete;\n mem_mgr &operator=(const mem_mgr &) = delete;\n mem_mgr(mem_mgr &&) = delete;\n mem_mgr &operator=(mem_mgr &&) = delete;\n\n /// Allocate\n void *mem_alloc(size_t size)\n {\n if (!size)\n return nullptr;\n std::lock_guard lock(m_mutex);\n if (next_free + size > mapped_address_space + mapped_region_size)\n {\n throw std::runtime_error(\"dpct_malloc: out of memory for virtual memory pool\");\n }\n // Allocation\n sycl::range<1> r(size);\n buffer_t buf(r);\n allocation A{buf, next_free, size};\n // Map allocation to device pointer\n void *result = next_free;\n m_map.emplace(next_free + size, A);\n // Update pointer to the next free space.\n next_free += (size + extra_padding + alignment - 1) & ~(alignment - 1);\n\n return result;\n }\n\n /// Deallocate\n void mem_free(const void *ptr)\n {\n if (!ptr)\n return;\n std::lock_guard lock(m_mutex);\n auto it = get_map_iterator(ptr);\n m_map.erase(it);\n }\n\n /// map: device pointer -> allocation(buffer, alloc_ptr, size)\n allocation translate_ptr(const void *ptr)\n {\n std::lock_guard lock(m_mutex);\n auto it = get_map_iterator(ptr);\n return it->second;\n }\n\n /// Check if the pointer represents device pointer or not.\n bool is_device_ptr(const void *ptr) const\n {\n std::lock_guard lock(m_mutex);\n return (mapped_address_space <= ptr) &&\n (ptr < mapped_address_space + mapped_region_size);\n }\n\n /// Returns the instance of memory manager singleton.\n static mem_mgr &instance()\n {\n static mem_mgr m;\n return m;\n }\n\n private:\n std::map m_map;\n mutable std::mutex m_mutex;\n byte_t *mapped_address_space;\n byte_t *next_free;\n const size_t mapped_region_size = 128ull * 1024 * 1024 * 1024;\n const size_t alignment = 256;\n /// This padding may be defined to some positive value to debug\n /// out of bound accesses.\n const size_t extra_padding = 0;\n\n std::map::iterator get_map_iterator(const void *ptr)\n {\n auto it = m_map.upper_bound(const_cast(reinterpret_cast(ptr)));\n if (it == m_map.end())\n {\n // Not a virtual pointer.\n throw std::runtime_error(\"can not get buffer from non-virtual pointer\");\n }\n const allocation &alloc = it->second;\n if (ptr < alloc.alloc_ptr)\n {\n // Out of bound.\n // This may happen if there's a gap between allocations due to alignment\n // or extra padding and pointer points to this gap.\n throw std::runtime_error(\"invalid virtual pointer\");\n }\n return it;\n }\n };\n\n template \n class accessor;\n template \n class memory_traits\n {\n public:\n static constexpr sycl::access::target target =\n sycl::access::target::device;\n static constexpr sycl::access_mode mode =\n (Memory == constant) ? sycl::access_mode::read\n : sycl::access_mode::read_write;\n static constexpr size_t type_size = sizeof(T);\n using element_t =\n typename std::conditional::type;\n using value_t = typename std::remove_cv::type;\n template \n using accessor_t = typename std::conditional<\n Memory == local, sycl::local_accessor,\n sycl::accessor>::type;\n using pointer_t = T *;\n };\n\n static inline void *dpct_malloc(size_t size, sycl::queue &q)\n {\n return sycl::malloc_device(size, q.get_device(), q.get_context());\n }\n\n#define PITCH_DEFAULT_ALIGN(x) (((x) + 31) & ~(0x1F))\n static inline void *dpct_malloc(size_t &pitch, size_t x, size_t y, size_t z,\n sycl::queue &q)\n {\n pitch = PITCH_DEFAULT_ALIGN(x);\n return dpct_malloc(pitch * y * z, q);\n }\n\n /**\n * @brief Sets \\p value to the first \\p size elements starting from \\p dev_ptr in \\p q.\n * @tparam valueT The type of the element to be set.\n * @param [in] q The queue in which the operation is done.\n * @param [in] dev_ptr Pointer to the virtual device memory address.\n * @param [in] value The value to be set.\n * @param [in] size Number of elements to be set to the value.\n * @return An event representing the memset operation.\n */\n template \n static inline sycl::event dpct_memset(sycl::queue &q, void *dev_ptr,\n valueT value, size_t size)\n {\n return q.fill(dev_ptr, value, size);\n }\n\n /**\n * @brief Sets \\p value to the 3D memory region pointed by \\p data in \\p q.\n * @tparam valueT The type of the element to be set.\n * @param [in] q The queue in which the operation is done.\n * @param [in] data Pointer to the pitched device memory region.\n * @param [in] value The value to be set.\n * @param [in] size 3D memory region by number of elements.\n * @return An event list representing the memset operations.\n */\n template \n static inline std::vector\n dpct_memset(sycl::queue &q, pitched_data data, valueT value,\n sycl::range<3> size)\n {\n std::vector event_list;\n size_t slice = data.get_pitch() * data.get_y();\n unsigned char *data_surface = (unsigned char *)data.get_data_ptr();\n for (size_t z = 0; z < size.get(2); ++z)\n {\n unsigned char *data_ptr = data_surface;\n for (size_t y = 0; y < size.get(1); ++y)\n {\n event_list.push_back(dpct_memset(q, data_ptr, value, size.get(0)));\n data_ptr += data.get_pitch();\n }\n data_surface += slice;\n }\n return event_list;\n }\n\n /**\n * @brief Sets \\p val to the pitched 2D memory region pointed by \\p ptr in \\p q.\n * @tparam valueT The type of the element to be set.\n * @param [in] q The queue in which the operation is done.\n * @param [in] ptr Pointer to the virtual device memory.\n * @param [in] pitch The pitch size by number of elements, including padding.\n * @param [in] val The value to be set.\n * @param [in] x The width of memory region by number of elements.\n * @param [in] y The height of memory region by number of elements.\n * @return An event list representing the memset operations.\n */\n template \n static inline std::vector\n dpct_memset(sycl::queue &q, void *ptr, size_t pitch, valueT val, size_t x,\n size_t y)\n {\n return dpct_memset(q, pitched_data(ptr, pitch, x, 1), val,\n sycl::range<3>(x, y, 1));\n }\n\n static memcpy_direction deduce_memcpy_direction(sycl::queue &q, void *to_ptr,\n const void *from_ptr,\n memcpy_direction dir)\n {\n switch (dir)\n {\n case memcpy_direction::host_to_host:\n case memcpy_direction::host_to_device:\n case memcpy_direction::device_to_host:\n case memcpy_direction::device_to_device:\n return dir;\n case memcpy_direction::automatic:\n {\n // table[to_attribute][from_attribute]\n static const memcpy_direction\n direction_table[static_cast(pointer_access_attribute::end)]\n [static_cast(pointer_access_attribute::end)] =\n {{memcpy_direction::host_to_host,\n memcpy_direction::device_to_host,\n memcpy_direction::host_to_host},\n {memcpy_direction::host_to_device,\n memcpy_direction::device_to_device,\n memcpy_direction::device_to_device},\n {memcpy_direction::host_to_host,\n memcpy_direction::device_to_device,\n memcpy_direction::device_to_device}};\n return direction_table[static_cast(get_pointer_attribute(\n q, to_ptr))][static_cast(get_pointer_attribute(q, from_ptr))];\n }\n default:\n throw std::runtime_error(\"dpct_memcpy: invalid direction value\");\n }\n }\n\n static sycl::event\n dpct_memcpy(sycl::queue &q, void *to_ptr, const void *from_ptr, size_t size,\n memcpy_direction direction,\n const std::vector &dep_events = {})\n {\n if (!size)\n return sycl::event{};\n return q.memcpy(to_ptr, from_ptr, size, dep_events);\n GGML_UNUSED(direction);\n }\n\n // Get actual copy range and make sure it will not exceed range.\n static inline size_t get_copy_range(sycl::range<3> size, size_t slice,\n size_t pitch)\n {\n return slice * (size.get(2) - 1) + pitch * (size.get(1) - 1) + size.get(0);\n }\n\n static inline size_t get_offset(sycl::id<3> id, size_t slice,\n size_t pitch)\n {\n return slice * id.get(2) + pitch * id.get(1) + id.get(0);\n }\n\n /// copy 3D matrix specified by \\p size from 3D matrix specified by \\p from_ptr\n /// and \\p from_range to another specified by \\p to_ptr and \\p to_range.\n static inline std::vector\n dpct_memcpy(sycl::queue &q, void *to_ptr, const void *from_ptr,\n sycl::range<3> to_range, sycl::range<3> from_range,\n sycl::id<3> to_id, sycl::id<3> from_id,\n sycl::range<3> size, memcpy_direction direction,\n const std::vector &dep_events = {})\n {\n // RAII for host pointer\n class host_buffer\n {\n void *_buf;\n size_t _size;\n sycl::queue &_q;\n const std::vector &_deps; // free operation depends\n\n public:\n host_buffer(size_t size, sycl::queue &q,\n const std::vector &deps)\n : _buf(std::malloc(size)), _size(size), _q(q), _deps(deps) {}\n void *get_ptr() const { return _buf; }\n size_t get_size() const { return _size; }\n ~host_buffer()\n {\n if (_buf)\n {\n _q.submit([&](sycl::handler &cgh)\n {\n cgh.depends_on(_deps);\n cgh.host_task([buf = _buf] { std::free(buf); }); });\n }\n }\n };\n std::vector event_list;\n\n size_t to_slice = to_range.get(1) * to_range.get(0),\n from_slice = from_range.get(1) * from_range.get(0);\n unsigned char *to_surface =\n (unsigned char *)to_ptr + get_offset(to_id, to_slice, to_range.get(0));\n const unsigned char *from_surface =\n (const unsigned char *)from_ptr +\n get_offset(from_id, from_slice, from_range.get(0));\n\n if (to_slice == from_slice && to_slice == size.get(1) * size.get(0))\n {\n return {dpct_memcpy(q, to_surface, from_surface, to_slice * size.get(2),\n direction, dep_events)};\n }\n direction = deduce_memcpy_direction(q, to_ptr, from_ptr, direction);\n size_t size_slice = size.get(1) * size.get(0);\n switch (direction)\n {\n case host_to_host:\n for (size_t z = 0; z < size.get(2); ++z)\n {\n unsigned char *to_ptr = to_surface;\n const unsigned char *from_ptr = from_surface;\n if (to_range.get(0) == from_range.get(0) &&\n to_range.get(0) == size.get(0))\n {\n event_list.push_back(dpct_memcpy(q, to_ptr, from_ptr, size_slice,\n direction, dep_events));\n }\n else\n {\n for (size_t y = 0; y < size.get(1); ++y)\n {\n event_list.push_back(dpct_memcpy(q, to_ptr, from_ptr, size.get(0),\n direction, dep_events));\n to_ptr += to_range.get(0);\n from_ptr += from_range.get(0);\n }\n }\n to_surface += to_slice;\n from_surface += from_slice;\n }\n break;\n case host_to_device:\n {\n host_buffer buf(get_copy_range(size, to_slice, to_range.get(0)), q,\n event_list);\n std::vector host_events;\n if (to_slice == size_slice)\n {\n // Copy host data to a temp host buffer with the shape of target.\n host_events =\n dpct_memcpy(q, buf.get_ptr(), from_surface, to_range, from_range,\n sycl::id<3>(0, 0, 0), sycl::id<3>(0, 0, 0), size,\n host_to_host, dep_events);\n }\n else\n {\n // Copy host data to a temp host buffer with the shape of target.\n host_events = dpct_memcpy(\n q, buf.get_ptr(), from_surface, to_range, from_range,\n sycl::id<3>(0, 0, 0), sycl::id<3>(0, 0, 0), size, host_to_host,\n // If has padding data, not sure whether it is useless. So fill temp\n // buffer with it.\n std::vector{\n dpct_memcpy(q, buf.get_ptr(), to_surface, buf.get_size(),\n device_to_host, dep_events)});\n }\n // Copy from temp host buffer to device with only one submit.\n event_list.push_back(dpct_memcpy(q, to_surface, buf.get_ptr(),\n buf.get_size(), host_to_device,\n host_events));\n break;\n }\n case device_to_host:\n {\n host_buffer buf(get_copy_range(size, from_slice, from_range.get(0)), q,\n event_list);\n // Copy from host temp buffer to host target with reshaping.\n event_list = dpct_memcpy(\n q, to_surface, buf.get_ptr(), to_range, from_range, sycl::id<3>(0, 0, 0),\n sycl::id<3>(0, 0, 0), size, host_to_host,\n // Copy from device to temp host buffer with only one submit.\n std::vector{dpct_memcpy(q, buf.get_ptr(), from_surface,\n buf.get_size(),\n device_to_host, dep_events)});\n break;\n }\n case device_to_device:\n event_list.push_back(q.submit([&](sycl::handler &cgh){\n cgh.depends_on(dep_events);\n cgh.parallel_for(\n size,\n [=](sycl::id<3> id) {\n to_surface[get_offset(id, to_slice, to_range.get(0))] =\n from_surface[get_offset(id, from_slice, from_range.get(0))];\n }); }));\n break;\n default:\n throw std::runtime_error(\"dpct_memcpy: invalid direction value\");\n }\n return event_list;\n }\n\n /// memcpy 2D/3D matrix specified by pitched_data.\n static inline std::vector\n dpct_memcpy(sycl::queue &q, pitched_data to, sycl::id<3> to_id,\n pitched_data from, sycl::id<3> from_id, sycl::range<3> size,\n memcpy_direction direction = automatic)\n {\n return dpct_memcpy(q, to.get_data_ptr(), from.get_data_ptr(),\n sycl::range<3>(to.get_pitch(), to.get_y(), 1),\n sycl::range<3>(from.get_pitch(), from.get_y(), 1), to_id, from_id,\n size, direction);\n }\n\n /// memcpy 2D matrix with pitch.\n static inline std::vector\n dpct_memcpy(sycl::queue &q, void *to_ptr, const void *from_ptr,\n size_t to_pitch, size_t from_pitch, size_t x, size_t y,\n memcpy_direction direction = automatic)\n {\n return dpct_memcpy(q, to_ptr, from_ptr, sycl::range<3>(to_pitch, y, 1),\n sycl::range<3>(from_pitch, y, 1),\n sycl::id<3>(0, 0, 0), sycl::id<3>(0, 0, 0),\n sycl::range<3>(x, y, 1), direction);\n }\n\n namespace deprecated\n {\n\n template \n class usm_allocator\n {\n private:\n using Alloc = sycl::usm_allocator;\n Alloc _impl;\n\n public:\n using value_type = typename std::allocator_traits::value_type;\n using pointer = typename std::allocator_traits::pointer;\n using const_pointer = typename std::allocator_traits::const_pointer;\n using void_pointer = typename std::allocator_traits::void_pointer;\n using const_void_pointer =\n typename std::allocator_traits::const_void_pointer;\n using reference = typename std::allocator_traits::value_type &;\n using const_reference =\n const typename std::allocator_traits::value_type &;\n using difference_type =\n typename std::allocator_traits::difference_type;\n using size_type = typename std::allocator_traits::size_type;\n using propagate_on_container_copy_assignment = typename std::allocator_traits<\n Alloc>::propagate_on_container_copy_assignment;\n using propagate_on_container_move_assignment = typename std::allocator_traits<\n Alloc>::propagate_on_container_move_assignment;\n using propagate_on_container_swap =\n typename std::allocator_traits::propagate_on_container_swap;\n using is_always_equal =\n typename std::allocator_traits::is_always_equal;\n\n template \n struct rebind\n {\n typedef usm_allocator other;\n };\n\n usm_allocator() : _impl(dpct::get_default_queue()) {}\n ~usm_allocator() {}\n usm_allocator(const usm_allocator &other) : _impl(other._impl) {}\n usm_allocator(usm_allocator &&other) : _impl(std::move(other._impl)) {}\n pointer address(reference r) { return &r; }\n const_pointer address(const_reference r) { return &r; }\n pointer allocate(size_type cnt, const_void_pointer hint = nullptr)\n {\n return std::allocator_traits::allocate(_impl, cnt, hint);\n }\n void deallocate(pointer p, size_type cnt)\n {\n std::allocator_traits::deallocate(_impl, p, cnt);\n }\n size_type max_size() const\n {\n return std::allocator_traits::max_size(_impl);\n }\n bool operator==(const usm_allocator &other) const { return _impl == other._impl; }\n bool operator!=(const usm_allocator &other) const { return _impl != other._impl; }\n };\n\n } // namespace deprecated\n\n inline void dpct_free(void *ptr,\n const sycl::queue &q)\n {\n if (ptr)\n {\n sycl::free(ptr, q.get_context());\n }\n }\n\n template \n inline auto get_memory(const void *x)\n {\n T *new_x = reinterpret_cast(const_cast(x));\n return new_x;\n }\n\n template \n inline typename DataType::T2 get_value(const T *s, sycl::queue &q)\n {\n using Ty = typename DataType::T2;\n Ty s_h;\n if (get_pointer_attribute(q, s) == pointer_access_attribute::device_only)\n detail::dpct_memcpy(q, (void *)&s_h, (const void *)s, sizeof(T), device_to_host)\n .wait();\n else\n s_h = *reinterpret_cast(s);\n return s_h;\n }\n\n } // namespace detail\n\n template \n inline auto get_value(const T *s, sycl::queue &q)\n {\n return detail::get_value(s, q);\n }\n\n namespace detail\n {\n template \n inline void gemm_impl(sycl::queue & q, oneapi::mkl::transpose a_trans, oneapi::mkl::transpose b_trans, int m,\n int n, int k, const void * alpha, const void * a, int lda, const void * b, int ldb,\n const void * beta, void * c, int ldc) {\n Ts alpha_value = dpct::get_value(reinterpret_cast(alpha), q);\n Ts beta_value = dpct::get_value(reinterpret_cast(beta), q);\n auto data_a = get_memory(a);\n auto data_b = get_memory(b);\n auto data_c = get_memory(c);\n oneapi::mkl::blas::column_major::gemm(q, a_trans, b_trans, m, n, k, alpha_value, data_a,\n lda, data_b, ldb, beta_value, data_c, ldc);\n }\n\n template \n class vectorized_binary\n {\n public:\n inline VecT operator()(VecT a, VecT b, const BinaryOperation binary_op)\n {\n VecT v4;\n for (size_t i = 0; i < v4.size(); ++i)\n {\n v4[i] = binary_op(a[i], b[i]);\n }\n return v4;\n }\n };\n\n template \n class vectorized_binary<\n VecT, BinaryOperation,\n std::void_t>>\n {\n public:\n inline VecT operator()(VecT a, VecT b, const BinaryOperation binary_op)\n {\n return binary_op(a, b).template as();\n }\n };\n\n template \n inline void gemm_batch_impl(sycl::queue & q, oneapi::mkl::transpose a_trans, oneapi::mkl::transpose b_trans,\n int m, int n, int k, const void * alpha, const void ** a, int lda, const void ** b,\n int ldb, const void * beta, void ** c, int ldc, int batch_size,\n matrix_info_t * matrix_info) {\n Ts alpha_value = dpct::get_value(reinterpret_cast(alpha), q);\n Ts beta_value = dpct::get_value(reinterpret_cast(beta), q);\n\n matrix_info->transpose_info[0] = a_trans;\n matrix_info->transpose_info[1] = b_trans;\n matrix_info->value_info[0] = alpha_value;\n matrix_info->value_info[1] = beta_value;\n matrix_info->size_info[0] = m;\n matrix_info->size_info[1] = n;\n matrix_info->size_info[2] = k;\n matrix_info->ld_info[0] = lda;\n matrix_info->ld_info[1] = ldb;\n matrix_info->ld_info[2] = ldc;\n matrix_info->groupsize_info = batch_size;\n\n sycl::event e = oneapi::mkl::blas::column_major::gemm_batch(\n q, matrix_info->transpose_info, matrix_info->transpose_info + 1,\n matrix_info->size_info, matrix_info->size_info + 1, matrix_info->size_info + 2,\n reinterpret_cast(matrix_info->value_info), reinterpret_cast(a), matrix_info->ld_info,\n reinterpret_cast(b), matrix_info->ld_info + 1,\n reinterpret_cast(matrix_info->value_info + 1), reinterpret_cast(c),\n matrix_info->ld_info + 2, 1, &(matrix_info->groupsize_info));\n }\n\n template \n inline void gemm_batch_impl(sycl::queue & q, oneapi::mkl::transpose a_trans, oneapi::mkl::transpose b_trans,\n int m, int n, int k, const void * alpha, const void * a, int lda,\n long long int stride_a, const void * b, int ldb, long long int stride_b,\n const void * beta, void * c, int ldc, long long int stride_c, int batch_size) {\n Ts alpha_value = dpct::get_value(reinterpret_cast(alpha), q);\n Ts beta_value = dpct::get_value(reinterpret_cast(beta), q);\n auto data_a = get_memory(a);\n auto data_b = get_memory(b);\n auto data_c = get_memory(c);\n oneapi::mkl::blas::column_major::gemm_batch(q, a_trans, b_trans, m, n, k, alpha_value,\n data_a, lda, stride_a, data_b, ldb, stride_b, beta_value,\n data_c, ldc, stride_c, batch_size);\n }\n\n } // namespace detail\n\n template \n inline unsigned vectorized_binary(unsigned a, unsigned b,\n const BinaryOperation binary_op)\n {\n sycl::vec v0{a}, v1{b};\n auto v2 = v0.as();\n auto v3 = v1.as();\n auto v4 =\n detail::vectorized_binary()(v2, v3, binary_op);\n v0 = v4.template as>();\n return v0;\n }\n\n static void async_dpct_memcpy(void *to_ptr, const void *from_ptr, size_t size,\n memcpy_direction direction = automatic,\n sycl::queue &q = dpct::get_default_queue())\n {\n detail::dpct_memcpy(q, to_ptr, from_ptr, size, direction);\n }\n\n static inline unsigned int select_device(unsigned int id)\n {\n dev_mgr::instance().select_device(id);\n return id;\n }\n\n template \n T permute_sub_group_by_xor(sycl::sub_group g, T x, unsigned int mask,\n unsigned int logical_sub_group_size = 32)\n {\n unsigned int id = g.get_local_linear_id();\n unsigned int start_index =\n id / logical_sub_group_size * logical_sub_group_size;\n unsigned int target_offset = (id % logical_sub_group_size) ^ mask;\n return sycl::select_from_group(g, x,\n target_offset < logical_sub_group_size\n ? start_index + target_offset\n : id);\n }\n\n template \n using dot_product_acc_t = std::conditional_t<\n std::is_unsigned_v && std::is_unsigned_v,\n uint32_t,\n int32_t>;\n\n template \n sycl::vec extract_and_sign_or_zero_extend4(T val) {\n return sycl::vec(val)\n .template as, int8_t, uint8_t>,\n 4>>()\n .template convert();\n }\n\n template \n inline auto dp4a(T1 a, T2 b, T3 c) {\n dot_product_acc_t res = c;\n auto va = extract_and_sign_or_zero_extend4(a);\n auto vb = extract_and_sign_or_zero_extend4(b);\n res += va[0] * vb[0];\n res += va[1] * vb[1];\n res += va[2] * vb[2];\n res += va[3] * vb[3];\n return res;\n }\n\n struct sub_sat\n {\n template \n auto operator()(const T x, const T y) const\n {\n return sycl::sub_sat(x, y);\n }\n };\n\n template \n inline T vectorized_min(T a, T b)\n {\n sycl::vec v0{a}, v1{b};\n auto v2 = v0.template as();\n auto v3 = v1.template as();\n auto v4 = sycl::min(v2, v3);\n v0 = v4.template as>();\n return v0;\n }\n\n inline float pow(const float a, const int b) { return sycl::pown(a, b); }\n inline double pow(const double a, const int b) { return sycl::pown(a, b); }\n inline float pow(const float a, const float b) { return sycl::pow(a, b); }\n inline double pow(const double a, const double b) { return sycl::pow(a, b); }\n template \n inline typename std::enable_if_t, T>\n pow(const T a, const U b)\n {\n return sycl::pow(a, static_cast(b));\n }\n template \n inline typename std::enable_if_t, double>\n pow(const T a, const U b)\n {\n return sycl::pow(static_cast(a), static_cast(b));\n }\n\n inline double min(const double a, const float b)\n {\n return sycl::fmin(a, static_cast(b));\n }\n inline double min(const float a, const double b)\n {\n return sycl::fmin(static_cast(a), b);\n }\n inline float min(const float a, const float b) { return sycl::fmin(a, b); }\n inline double min(const double a, const double b) { return sycl::fmin(a, b); }\n inline std::uint32_t min(const std::uint32_t a, const std::int32_t b)\n {\n return sycl::min(a, static_cast(b));\n }\n inline std::uint32_t min(const std::int32_t a, const std::uint32_t b)\n {\n return sycl::min(static_cast(a), b);\n }\n inline std::int32_t min(const std::int32_t a, const std::int32_t b)\n {\n return sycl::min(a, b);\n }\n inline std::uint32_t min(const std::uint32_t a, const std::uint32_t b)\n {\n return sycl::min(a, b);\n }\n inline std::uint64_t min(const std::uint64_t a, const std::int64_t b)\n {\n return sycl::min(a, static_cast(b));\n }\n inline std::uint64_t min(const std::int64_t a, const std::uint64_t b)\n {\n return sycl::min(static_cast(a), b);\n }\n inline std::int64_t min(const std::int64_t a, const std::int64_t b)\n {\n return sycl::min(a, b);\n }\n inline std::uint64_t min(const std::uint64_t a, const std::uint64_t b)\n {\n return sycl::min(a, b);\n }\n inline std::uint64_t min(const std::uint64_t a, const std::int32_t b)\n {\n return sycl::min(a, static_cast(b));\n }\n inline std::uint64_t min(const std::int32_t a, const std::uint64_t b)\n {\n return sycl::min(static_cast(a), b);\n }\n inline std::uint64_t min(const std::uint64_t a, const std::uint32_t b)\n {\n return sycl::min(a, static_cast(b));\n }\n inline std::uint64_t min(const std::uint32_t a, const std::uint64_t b)\n {\n return sycl::min(static_cast(a), b);\n }\n // max function overloads.\n // For floating-point types, `float` or `double` arguments are acceptable.\n // For integer types, `std::uint32_t`, `std::int32_t`, `std::uint64_t` or\n // `std::int64_t` type arguments are acceptable.\n inline double max(const double a, const float b)\n {\n return sycl::fmax(a, static_cast(b));\n }\n inline double max(const float a, const double b)\n {\n return sycl::fmax(static_cast(a), b);\n }\n inline float max(const float a, const float b) { return sycl::fmax(a, b); }\n inline double max(const double a, const double b) { return sycl::fmax(a, b); }\n inline std::uint32_t max(const std::uint32_t a, const std::int32_t b)\n {\n return sycl::max(a, static_cast(b));\n }\n inline std::uint32_t max(const std::int32_t a, const std::uint32_t b)\n {\n return sycl::max(static_cast(a), b);\n }\n inline std::int32_t max(const std::int32_t a, const std::int32_t b)\n {\n return sycl::max(a, b);\n }\n inline std::uint32_t max(const std::uint32_t a, const std::uint32_t b)\n {\n return sycl::max(a, b);\n }\n inline std::uint64_t max(const std::uint64_t a, const std::int64_t b)\n {\n return sycl::max(a, static_cast(b));\n }\n inline std::uint64_t max(const std::int64_t a, const std::uint64_t b)\n {\n return sycl::max(static_cast(a), b);\n }\n inline std::int64_t max(const std::int64_t a, const std::int64_t b)\n {\n return sycl::max(a, b);\n }\n inline std::uint64_t max(const std::uint64_t a, const std::uint64_t b)\n {\n return sycl::max(a, b);\n }\n inline std::uint64_t max(const std::uint64_t a, const std::int32_t b)\n {\n return sycl::max(a, static_cast(b));\n }\n inline std::uint64_t max(const std::int32_t a, const std::uint64_t b)\n {\n return sycl::max(static_cast(a), b);\n }\n inline std::uint64_t max(const std::uint64_t a, const std::uint32_t b)\n {\n return sycl::max(a, static_cast(b));\n }\n inline std::uint64_t max(const std::uint32_t a, const std::uint64_t b)\n {\n return sycl::max(static_cast(a), b);\n }\n\n inline void\n has_capability_or_fail(const sycl::device &dev,\n const std::initializer_list &props)\n {\n for (const auto &it : props)\n {\n if (dev.has(it))\n continue;\n switch (it)\n {\n case sycl::aspect::fp64:\n throw std::runtime_error(\"'double' is not supported in '\" +\n dev.get_info() +\n \"' device\");\n break;\n case sycl::aspect::fp16:\n throw std::runtime_error(\"'half' is not supported in '\" +\n dev.get_info() +\n \"' device\");\n break;\n default:\n#define __SYCL_ASPECT(ASPECT, ID) \\\n case sycl::aspect::ASPECT: \\\n return #ASPECT;\n#define __SYCL_ASPECT_DEPRECATED(ASPECT, ID, MESSAGE) __SYCL_ASPECT(ASPECT, ID)\n#define __SYCL_ASPECT_DEPRECATED_ALIAS(ASPECT, ID, MESSAGE)\n auto getAspectNameStr = [](sycl::aspect AspectNum) -> std::string\n {\n switch (AspectNum)\n {\n#include \n#include \n default:\n return \"unknown aspect\";\n }\n };\n#undef __SYCL_ASPECT_DEPRECATED_ALIAS\n#undef __SYCL_ASPECT_DEPRECATED\n#undef __SYCL_ASPECT\n throw std::runtime_error(\n \"'\" + getAspectNameStr(it) + \"' is not supported in '\" +\n dev.get_info() + \"' device\");\n }\n break;\n }\n }\n\n static inline unsigned int get_current_device_id()\n {\n return dev_mgr::instance().current_device_id();\n }\n\n static inline device_ext &get_current_device()\n {\n return dev_mgr::instance().current_device();\n }\n\n static inline device_ext &get_device(unsigned int id)\n {\n return dev_mgr::instance().get_device(id);\n }\n\n static inline sycl::queue &get_in_order_queue()\n {\n return dev_mgr::instance().current_device().in_order_queue();\n }\n\n static sycl::event\n dpct_memcpy(sycl::queue &q, void *to_ptr, const void *from_ptr, size_t size,\n memcpy_direction direction,\n const std::vector &dep_events = {})\n {\n if (!size)\n return sycl::event{};\n return q.memcpy(to_ptr, from_ptr, size, dep_events);\n GGML_UNUSED(direction);\n }\n\n // Get actual copy range and make sure it will not exceed range.\n static inline size_t get_copy_range(sycl::range<3> size, size_t slice,\n size_t pitch)\n {\n return slice * (size.get(2) - 1) + pitch * (size.get(1) - 1) + size.get(0);\n }\n\n static inline size_t get_offset(sycl::id<3> id, size_t slice,\n size_t pitch)\n {\n return slice * id.get(2) + pitch * id.get(1) + id.get(0);\n }\n\n /// copy 3D matrix specified by \\p size from 3D matrix specified by \\p from_ptr\n /// and \\p from_range to another specified by \\p to_ptr and \\p to_range.\n static inline std::vector\n dpct_memcpy(sycl::queue &q, void *to_ptr, const void *from_ptr,\n sycl::range<3> to_range, sycl::range<3> from_range,\n sycl::id<3> to_id, sycl::id<3> from_id,\n sycl::range<3> size, memcpy_direction direction,\n const std::vector &dep_events = {})\n {\n // RAII for host pointer\n class host_buffer\n {\n void *_buf;\n size_t _size;\n sycl::queue &_q;\n const std::vector &_deps; // free operation depends\n\n public:\n host_buffer(size_t size, sycl::queue &q,\n const std::vector &deps)\n : _buf(std::malloc(size)), _size(size), _q(q), _deps(deps) {}\n void *get_ptr() const { return _buf; }\n size_t get_size() const { return _size; }\n ~host_buffer()\n {\n if (_buf)\n {\n _q.submit([&](sycl::handler &cgh)\n {\n cgh.depends_on(_deps);\n cgh.host_task([buf = _buf] { std::free(buf); }); });\n }\n }\n };\n std::vector event_list;\n\n size_t to_slice = to_range.get(1) * to_range.get(0),\n from_slice = from_range.get(1) * from_range.get(0);\n unsigned char *to_surface =\n (unsigned char *)to_ptr + get_offset(to_id, to_slice, to_range.get(0));\n const unsigned char *from_surface =\n (const unsigned char *)from_ptr +\n get_offset(from_id, from_slice, from_range.get(0));\n\n if (to_slice == from_slice && to_slice == size.get(1) * size.get(0))\n {\n return {dpct_memcpy(q, to_surface, from_surface, to_slice * size.get(2),\n direction, dep_events)};\n }\n direction = detail::deduce_memcpy_direction(q, to_ptr, from_ptr, direction);\n size_t size_slice = size.get(1) * size.get(0);\n switch (direction)\n {\n case host_to_host:\n for (size_t z = 0; z < size.get(2); ++z)\n {\n unsigned char *to_ptr = to_surface;\n const unsigned char *from_ptr = from_surface;\n if (to_range.get(0) == from_range.get(0) &&\n to_range.get(0) == size.get(0))\n {\n event_list.push_back(dpct_memcpy(q, to_ptr, from_ptr, size_slice,\n direction, dep_events));\n }\n else\n {\n for (size_t y = 0; y < size.get(1); ++y)\n {\n event_list.push_back(dpct_memcpy(q, to_ptr, from_ptr, size.get(0),\n direction, dep_events));\n to_ptr += to_range.get(0);\n from_ptr += from_range.get(0);\n }\n }\n to_surface += to_slice;\n from_surface += from_slice;\n }\n break;\n case host_to_device:\n {\n host_buffer buf(get_copy_range(size, to_slice, to_range.get(0)), q,\n event_list);\n std::vector host_events;\n if (to_slice == size_slice)\n {\n // Copy host data to a temp host buffer with the shape of target.\n host_events =\n dpct_memcpy(q, buf.get_ptr(), from_surface, to_range, from_range,\n sycl::id<3>(0, 0, 0), sycl::id<3>(0, 0, 0), size,\n host_to_host, dep_events);\n }\n else\n {\n // Copy host data to a temp host buffer with the shape of target.\n host_events = dpct_memcpy(\n q, buf.get_ptr(), from_surface, to_range, from_range,\n sycl::id<3>(0, 0, 0), sycl::id<3>(0, 0, 0), size, host_to_host,\n // If has padding data, not sure whether it is useless. So fill temp\n // buffer with it.\n std::vector{\n dpct_memcpy(q, buf.get_ptr(), to_surface, buf.get_size(),\n device_to_host, dep_events)});\n }\n // Copy from temp host buffer to device with only one submit.\n event_list.push_back(dpct_memcpy(q, to_surface, buf.get_ptr(),\n buf.get_size(), host_to_device,\n host_events));\n break;\n }\n case device_to_host:\n {\n host_buffer buf(get_copy_range(size, from_slice, from_range.get(0)), q,\n event_list);\n // Copy from host temp buffer to host target with reshaping.\n event_list = dpct_memcpy(\n q, to_surface, buf.get_ptr(), to_range, from_range, sycl::id<3>(0, 0, 0),\n sycl::id<3>(0, 0, 0), size, host_to_host,\n // Copy from device to temp host buffer with only one submit.\n std::vector{dpct_memcpy(q, buf.get_ptr(), from_surface,\n buf.get_size(),\n device_to_host, dep_events)});\n break;\n }\n case device_to_device:\n event_list.push_back(q.submit([&](sycl::handler &cgh)\n {\n cgh.depends_on(dep_events);\n cgh.parallel_for(\n size,\n [=](sycl::id<3> id) {\n to_surface[get_offset(id, to_slice, to_range.get(0))] =\n from_surface[get_offset(id, from_slice, from_range.get(0))];\n }); }));\n break;\n default:\n throw std::runtime_error(\"dpct_memcpy: invalid direction value\");\n }\n return event_list;\n }\n\n /// memcpy 2D/3D matrix specified by pitched_data.\n static inline std::vector\n dpct_memcpy(sycl::queue &q, pitched_data to, sycl::id<3> to_id,\n pitched_data from, sycl::id<3> from_id, sycl::range<3> size,\n memcpy_direction direction = automatic)\n {\n return dpct_memcpy(q, to.get_data_ptr(), from.get_data_ptr(),\n sycl::range<3>(to.get_pitch(), to.get_y(), 1),\n sycl::range<3>(from.get_pitch(), from.get_y(), 1), to_id, from_id,\n size, direction);\n }\n\n /// memcpy 2D matrix with pitch.\n static inline std::vector\n dpct_memcpy(sycl::queue &q, void *to_ptr, const void *from_ptr,\n size_t to_pitch, size_t from_pitch, size_t x, size_t y,\n memcpy_direction direction = automatic)\n {\n return dpct_memcpy(q, to_ptr, from_ptr, sycl::range<3>(to_pitch, y, 1),\n sycl::range<3>(from_pitch, y, 1),\n sycl::id<3>(0, 0, 0), sycl::id<3>(0, 0, 0),\n sycl::range<3>(x, y, 1), direction);\n }\n\n inline void gemm(sycl::queue & q, oneapi::mkl::transpose a_trans, oneapi::mkl::transpose b_trans, int m, int n,\n int k, const void * alpha, const void * a, library_data_t a_type, int lda, const void * b,\n library_data_t b_type, int ldb, const void * beta, void * c, library_data_t c_type, int ldc,\n library_data_t scaling_type) {\n if (scaling_type == library_data_t::real_float &&\n c_type == library_data_t::complex_float)\n {\n scaling_type = library_data_t::complex_float;\n }\n else if (scaling_type == library_data_t::real_double &&\n c_type == library_data_t::complex_double)\n {\n scaling_type = library_data_t::complex_double;\n }\n\n std::uint64_t key =\n detail::get_type_combination_id(a_type, b_type, c_type, scaling_type);\n switch (key)\n {\n case detail::get_type_combination_id(\n library_data_t::real_float, library_data_t::real_float,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_double, library_data_t::real_double,\n library_data_t::real_double, library_data_t::real_double):\n {\n detail::gemm_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::complex_float, library_data_t::complex_float,\n library_data_t::complex_float, library_data_t::complex_float):\n {\n detail::gemm_impl, std::complex,\n std::complex, std::complex>(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::complex_double, library_data_t::complex_double,\n library_data_t::complex_double, library_data_t::complex_double):\n {\n detail::gemm_impl, std::complex,\n std::complex, std::complex>(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_half, library_data_t::real_half,\n library_data_t::real_half, library_data_t::real_half):\n {\n detail::gemm_impl(q, a_trans, b_trans, m, n, k, alpha, a,\n lda, b, ldb, beta, c, ldc);\n break;\n }\n#ifdef __INTEL_MKL__\n case detail::get_type_combination_id(\n library_data_t::real_bfloat16, library_data_t::real_bfloat16,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_half, library_data_t::real_half,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_half, library_data_t::real_half,\n library_data_t::real_half, library_data_t::real_float):\n {\n float alpha_value =\n dpct::get_value(reinterpret_cast(alpha), q);\n float beta_value =\n dpct::get_value(reinterpret_cast(beta), q);\n sycl::half alpha_half(alpha_value);\n sycl::half beta_half(beta_value);\n detail::gemm_impl(q, a_trans, b_trans, m, n, k, &alpha_half,\n a, lda, b, ldb, &beta_half, c, ldc);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_int8, library_data_t::real_int8,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_bfloat16, library_data_t::real_bfloat16,\n library_data_t::real_bfloat16, library_data_t::real_float):\n {\n detail::gemm_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_int8, library_data_t::real_int8,\n library_data_t::real_int32, library_data_t::real_int32):\n {\n float alpha_float =\n dpct::get_value(reinterpret_cast(alpha), q);\n float beta_float =\n dpct::get_value(reinterpret_cast(beta), q);\n detail::gemm_impl(\n q, a_trans, b_trans, m, n, k, &alpha_float, a, lda, b, ldb, &beta_float, c, ldc);\n break;\n }\n#endif // __INTEL_MKL__\n default:\n throw std::runtime_error(\"the combination of data type is unsupported\");\n }\n } // gemm()\n\n /// Computes a batch of matrix-matrix product with general matrices.\n /// \\param [in] q The queue where the routine should be executed.\n /// \\param [in] a_trans Specifies the operation applied to A.\n /// \\param [in] b_trans Specifies the operation applied to B.\n /// \\param [in] m Specifies the number of rows of the matrix op(A) and of the matrix C.\n /// \\param [in] n Specifies the number of columns of the matrix op(B) and of the matrix C.\n /// \\param [in] k Specifies the number of columns of the matrix op(A) and the number of rows of the matrix op(B).\n /// \\param [in] alpha Scaling factor for the matrix-matrix product.\n /// \\param [in] a Input matrix A.\n /// \\param [in] a_type Data type of the matrix A.\n /// \\param [in] lda Leading dimension of A.\n /// \\param [in] b Input matrix B.\n /// \\param [in] b_type Data type of the matrix B.\n /// \\param [in] ldb Leading dimension of B.\n /// \\param [in] beta Scaling factor for matrix C.\n /// \\param [in, out] c Input/Output matrix C.\n /// \\param [in] c_type Data type of the matrix C.\n /// \\param [in] ldc Leading dimension of C.\n /// \\param [in] batch_size Specifies the number of matrix multiply operations to perform.\n /// \\param [in] scaling_type Data type of the scaling factors.\n inline void gemm_batch(sycl::queue & q, oneapi::mkl::transpose a_trans, oneapi::mkl::transpose b_trans, int m,\n int n, int k, const void * alpha, const void * a[], library_data_t a_type, int lda,\n const void * b[], library_data_t b_type, int ldb, const void * beta, void * c[],\n library_data_t c_type, int ldc, int batch_size, library_data_t scaling_type,\n matrix_info_t * matrix_info) {\n std::uint64_t key =\n detail::get_type_combination_id(a_type, b_type, c_type, scaling_type);\n switch (key)\n {\n case detail::get_type_combination_id(\n library_data_t::real_float, library_data_t::real_float,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_batch_impl(q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb,\n beta, c, ldc, batch_size, matrix_info);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_double, library_data_t::real_double,\n library_data_t::real_double, library_data_t::real_double):\n {\n detail::gemm_batch_impl(q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb,\n beta, c, ldc, batch_size, matrix_info);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_half, library_data_t::real_half,\n library_data_t::real_half, library_data_t::real_half):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc, batch_size, matrix_info);\n break;\n }\n#ifdef __INTEL_MKL__\n case detail::get_type_combination_id(\n library_data_t::real_bfloat16, library_data_t::real_bfloat16,\n library_data_t::real_bfloat16, library_data_t::real_float):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc, batch_size, matrix_info);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_bfloat16, library_data_t::real_bfloat16,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc, batch_size, matrix_info);\n break;\n }\n#endif\n case detail::get_type_combination_id(\n library_data_t::real_int8, library_data_t::real_int8,\n library_data_t::real_int32, library_data_t::real_int32):\n {\n float alpha_float =\n dpct::get_value(reinterpret_cast(alpha), q);\n float beta_float =\n dpct::get_value(reinterpret_cast(beta), q);\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, &alpha_float, a, lda, b, ldb, &beta_float, c, ldc, batch_size,\n matrix_info);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_int8, library_data_t::real_int8,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc, batch_size, matrix_info);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_half, library_data_t::real_half,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc, batch_size, matrix_info);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_half, library_data_t::real_half,\n library_data_t::real_half, library_data_t::real_float):\n {\n float alpha_value =\n dpct::get_value(reinterpret_cast(alpha), q);\n float beta_value =\n dpct::get_value(reinterpret_cast(beta), q);\n sycl::half alpha_half(alpha_value);\n sycl::half beta_half(beta_value);\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, &alpha_half, a, lda, b, ldb, &beta_half, c, ldc, batch_size, matrix_info);\n break;\n }\n default:\n throw std::runtime_error(\"the combination of data type is unsupported\");\n }\n }\n\n /// Computes a batch of matrix-matrix product with general matrices.\n /// \\param [in] q The queue where the routine should be executed.\n /// \\param [in] a_trans Specifies the operation applied to A.\n /// \\param [in] b_trans Specifies the operation applied to B.\n /// \\param [in] m Specifies the number of rows of the matrix op(A) and of the matrix C.\n /// \\param [in] n Specifies the number of columns of the matrix op(B) and of the matrix C.\n /// \\param [in] k Specifies the number of columns of the matrix op(A) and the number of rows of the matrix op(B).\n /// \\param [in] alpha Scaling factor for the matrix-matrix product.\n /// \\param [in] a Input matrix A.\n /// \\param [in] a_type Data type of the matrix A.\n /// \\param [in] lda Leading dimension of A.\n /// \\param [in] stride_a Stride between the different A matrices.\n /// \\param [in] b Input matrix B.\n /// \\param [in] b_type Data type of the matrix B.\n /// \\param [in] ldb Leading dimension of B.\n /// \\param [in] stride_b Stride between the different B matrices.\n /// \\param [in] beta Scaling factor for matrix C.\n /// \\param [in, out] c Input/Output matrix C.\n /// \\param [in] c_type Data type of the matrix C.\n /// \\param [in] ldc Leading dimension of C.\n /// \\param [in] stride_c Stride between the different C matrices.\n /// \\param [in] batch_size Specifies the number of matrix multiply operations to perform.\n /// \\param [in] scaling_type Data type of the scaling factors.\n inline void gemm_batch(sycl::queue & q, oneapi::mkl::transpose a_trans, oneapi::mkl::transpose b_trans, int m,\n int n, int k, const void * alpha, const void * a, library_data_t a_type, int lda,\n long long int stride_a, const void * b, library_data_t b_type, int ldb,\n long long int stride_b, const void * beta, void * c, library_data_t c_type, int ldc,\n long long int stride_c, int batch_size, library_data_t scaling_type) {\n if (scaling_type == library_data_t::real_float &&\n c_type == library_data_t::complex_float)\n {\n scaling_type = library_data_t::complex_float;\n }\n else if (scaling_type == library_data_t::real_double &&\n c_type == library_data_t::complex_double)\n {\n scaling_type = library_data_t::complex_double;\n }\n\n std::uint64_t key =\n detail::get_type_combination_id(a_type, b_type, c_type, scaling_type);\n switch (key)\n {\n case detail::get_type_combination_id(\n library_data_t::real_float, library_data_t::real_float,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, stride_a, b, ldb, stride_b,\n beta, c, ldc, stride_c, batch_size);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_double, library_data_t::real_double,\n library_data_t::real_double, library_data_t::real_double):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, stride_a, b, ldb, stride_b,\n beta, c, ldc, stride_c, batch_size);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::complex_float, library_data_t::complex_float,\n library_data_t::complex_float, library_data_t::complex_float):\n {\n detail::gemm_batch_impl, std::complex,\n std::complex, std::complex>(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, stride_a, b, ldb, stride_b,\n beta, c, ldc, stride_c, batch_size);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::complex_double, library_data_t::complex_double,\n library_data_t::complex_double, library_data_t::complex_double):\n {\n detail::gemm_batch_impl, std::complex,\n std::complex, std::complex>(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, stride_a, b, ldb, stride_b,\n beta, c, ldc, stride_c, batch_size);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_half, library_data_t::real_half,\n library_data_t::real_half, library_data_t::real_half):\n {\n detail::gemm_batch_impl(q, a_trans, b_trans, m, n, k, alpha,\n a, lda, stride_a, b, ldb, stride_b,\n beta, c, ldc, stride_c, batch_size);\n break;\n }\n#ifdef __INTEL_MKL__\n case detail::get_type_combination_id(\n library_data_t::real_bfloat16, library_data_t::real_bfloat16,\n library_data_t::real_bfloat16, library_data_t::real_float):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, stride_a, b, ldb, stride_b, beta, c, ldc, stride_c,\n batch_size);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_bfloat16, library_data_t::real_bfloat16,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, stride_a, b, ldb, stride_b, beta, c, ldc, stride_c,\n batch_size);\n break;\n }\n#endif\n case detail::get_type_combination_id(\n library_data_t::real_int8, library_data_t::real_int8,\n library_data_t::real_int32, library_data_t::real_int32):\n {\n detail::gemm_batch_impl(q, a_trans, b_trans, m, n, k, alpha,\n a, lda, stride_a, b, ldb, stride_b,\n beta, c, ldc, stride_c, batch_size);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_int8, library_data_t::real_int8,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, stride_a, b, ldb, stride_b,\n beta, c, ldc, stride_c, batch_size);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_half, library_data_t::real_half,\n library_data_t::real_float, library_data_t::real_float):\n {\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, alpha, a, lda, stride_a, b, ldb, stride_b,\n beta, c, ldc, stride_c, batch_size);\n break;\n }\n case detail::get_type_combination_id(\n library_data_t::real_half, library_data_t::real_half,\n library_data_t::real_half, library_data_t::real_float):\n {\n float alpha_value =\n dpct::get_value(reinterpret_cast(alpha), q);\n float beta_value =\n dpct::get_value(reinterpret_cast(beta), q);\n sycl::half alpha_half(alpha_value);\n sycl::half beta_half(beta_value);\n detail::gemm_batch_impl(\n q, a_trans, b_trans, m, n, k, &alpha_half, a, lda, stride_a, b, ldb, stride_b,\n &beta_half, c, ldc, stride_c, batch_size);\n break;\n }\n default:\n throw std::runtime_error(\"the combination of data type is unsupported\");\n }\n }\n\n static inline void\n async_dpct_memcpy(void *to_ptr, size_t to_pitch, const void *from_ptr,\n size_t from_pitch, size_t x, size_t y,\n memcpy_direction direction = automatic,\n sycl::queue &q = get_default_queue())\n {\n detail::dpct_memcpy(q, to_ptr, from_ptr, to_pitch, from_pitch, x, y,\n direction);\n }\n\n using err0 = detail::generic_error_type;\n using err1 = detail::generic_error_type;\n\n static inline void dpct_free(void *ptr, sycl::queue &q = get_default_queue()) {\n detail::dpct_free(ptr, q);\n }\n\n /// dpct accessor used as device function parameter.\n template class accessor;\n template class accessor {\n public:\n using memory_t = detail::memory_traits;\n using element_t = typename memory_t::element_t;\n using pointer_t = typename memory_t::pointer_t;\n using accessor_t = typename memory_t::template accessor_t<3>;\n accessor(pointer_t data, const sycl::range<3> &in_range)\n : _data(data), _range(in_range) {}\n template \n accessor(typename std::enable_if::type &acc)\n : accessor(acc, acc.get_range()) {}\n accessor(const accessor_t &acc, const sycl::range<3> &in_range)\n : accessor(acc.get_pointer(), in_range) {}\n accessor operator[](size_t index) const {\n sycl::range<2> sub(_range.get(1), _range.get(2));\n return accessor(_data + index * sub.size(), sub);\n }\n\n pointer_t get_ptr() const { return _data; }\n\n private:\n pointer_t _data;\n sycl::range<3> _range;\n };\n template class accessor {\n public:\n using memory_t = detail::memory_traits;\n using element_t = typename memory_t::element_t;\n using pointer_t = typename memory_t::pointer_t;\n using accessor_t = typename memory_t::template accessor_t<2>;\n accessor(pointer_t data, const sycl::range<2> &in_range)\n : _data(data), _range(in_range) {}\n template \n accessor(typename std::enable_if::type &acc)\n : accessor(acc, acc.get_range()) {}\n accessor(const accessor_t &acc, const sycl::range<2> &in_range)\n : accessor(acc.get_pointer(), in_range) {}\n\n pointer_t operator[](size_t index) const {\n return _data + _range.get(1) * index;\n }\n\n pointer_t get_ptr() const { return _data; }\n\n private:\n pointer_t _data;\n sycl::range<2> _range;\n };\n\n namespace detail {\n /// Device variable with address space of shared, global or constant.\n template class device_memory {\n public:\n using accessor_t =\n typename detail::memory_traits::template accessor_t;\n using value_t = typename detail::memory_traits::value_t;\n using dpct_accessor_t = dpct::accessor;\n\n device_memory() : device_memory(sycl::range(1)) {}\n\n /// Constructor of 1-D array with initializer list\n device_memory(const sycl::range &in_range,\n std::initializer_list &&init_list)\n : device_memory(in_range) {\n assert(init_list.size() <= in_range.size());\n _host_ptr = (value_t *)std::malloc(_size);\n std::memset(_host_ptr, 0, _size);\n std::memcpy(_host_ptr, init_list.begin(), init_list.size() * sizeof(T));\n }\n\n /// Constructor of 2-D array with initializer list\n template \n device_memory(\n const typename std::enable_if>::type &in_range,\n std::initializer_list> &&init_list)\n : device_memory(in_range) {\n assert(init_list.size() <= in_range[0]);\n _host_ptr = (value_t *)std::malloc(_size);\n std::memset(_host_ptr, 0, _size);\n auto tmp_data = _host_ptr;\n for (auto sub_list : init_list) {\n assert(sub_list.size() <= in_range[1]);\n std::memcpy(tmp_data, sub_list.begin(),\n sub_list.size() * sizeof(T));\n tmp_data += in_range[1];\n }\n }\n\n /// Constructor with range\n device_memory(const sycl::range &range_in)\n : _size(range_in.size() * sizeof(T)), _range(range_in),\n _reference(false), _host_ptr(nullptr), _device_ptr(nullptr) {\n static_assert(\n (Memory == global) || (Memory == constant) || (Memory == shared),\n \"device memory region should be global, constant or shared\");\n // Make sure that singleton class mem_mgr and dev_mgr will destruct\n // later than this.\n detail::mem_mgr::instance();\n dev_mgr::instance();\n }\n\n /// Constructor with range\n template \n device_memory(Args... Arguments)\n : device_memory(sycl::range(Arguments...)) {}\n\n ~device_memory() {\n if (_device_ptr && !_reference)\n dpct::dpct_free(_device_ptr);\n if (_host_ptr)\n std::free(_host_ptr);\n }\n\n /// Allocate memory with default queue, and init memory if has initial\n /// value.\n void init() { init(dpct::get_default_queue()); }\n /// Allocate memory with specified queue, and init memory if has initial\n /// value.\n void init(sycl::queue &q) {\n if (_device_ptr)\n return;\n if (!_size)\n return;\n allocate_device(q);\n if (_host_ptr)\n detail::dpct_memcpy(q, _device_ptr, _host_ptr, _size,\n host_to_device);\n }\n\n /// The variable is assigned to a device pointer.\n void assign(value_t *src, size_t size) {\n this->~device_memory();\n new (this) device_memory(src, size);\n }\n\n /// Get memory pointer of the memory object, which is virtual pointer when\n /// usm is not used, and device pointer when usm is used.\n value_t *get_ptr() { return get_ptr(get_default_queue()); }\n /// Get memory pointer of the memory object, which is virtual pointer when\n /// usm is not used, and device pointer when usm is used.\n value_t *get_ptr(sycl::queue &q) {\n init(q);\n return _device_ptr;\n }\n\n /// Get the device memory object size in bytes.\n size_t get_size() { return _size; }\n\n template \n typename std::enable_if::type &operator[](size_t index) {\n init();\n return _device_ptr[index];\n }\n\n /// Get dpct::accessor with dimension info for the device memory object\n /// when usm is used and dimension is greater than 1.\n template \n typename std::enable_if::type\n get_access([[maybe_unused]] sycl::handler &cgh) {\n return dpct_accessor_t((T *)_device_ptr, _range);\n }\n\n private:\n device_memory(value_t *memory_ptr, size_t size)\n : _size(size), _range(size / sizeof(T)), _reference(true),\n _device_ptr(memory_ptr) {}\n\n void allocate_device(sycl::queue &q) {\n #ifndef DPCT_USM_LEVEL_NONE\n if (Memory == shared) {\n _device_ptr = (value_t *)sycl::malloc_shared(_size, q.get_device(),\n q.get_context());\n return;\n }\n #ifdef SYCL_EXT_ONEAPI_USM_DEVICE_READ_ONLY\n if (Memory == constant) {\n _device_ptr = (value_t *)sycl::malloc_device(\n _size, q.get_device(), q.get_context(),\n sycl::ext::oneapi::property::usm::device_read_only());\n return;\n }\n #endif\n #endif\n _device_ptr = (value_t *)detail::dpct_malloc(_size, q);\n }\n\n size_t _size;\n sycl::range _range;\n bool _reference;\n value_t *_host_ptr;\n value_t *_device_ptr;\n };\n template \n class device_memory : public device_memory {\n public:\n using base = device_memory;\n using value_t = typename base::value_t;\n using accessor_t =\n typename detail::memory_traits::template accessor_t<0>;\n\n /// Constructor with initial value.\n device_memory(const value_t &val) : base(sycl::range<1>(1), {val}) {}\n\n /// Default constructor\n device_memory() : base(1) {}\n };\n } // namespace detail\n\n template \n using global_memory = detail::device_memory;\n template \n using constant_memory = detail::device_memory;\n template \n using shared_memory = detail::device_memory;\n\n\n template \n inline T atomic_fetch_add(T *addr, T operand) {\n auto atm =\n sycl::atomic_ref(addr[0]);\n return atm.fetch_add(operand);\n }\n\n template \n inline T1 atomic_fetch_add(T1 *addr, T2 operand) {\n auto atm =\n sycl::atomic_ref(addr[0]);\n return atm.fetch_add(operand);\n }\n\n template \n inline T atomic_fetch_add(T *addr, T operand,\n sycl::memory_order memoryOrder) {\n switch (memoryOrder) {\n case sycl::memory_order::relaxed:\n return atomic_fetch_add(addr, operand);\n case sycl::memory_order::acq_rel:\n return atomic_fetch_add(addr, operand);\n case sycl::memory_order::seq_cst:\n return atomic_fetch_add(addr, operand);\n default:\n assert(false && \"Invalid memory_order for atomics. Valid memory_order for \"\n \"atomics are: sycl::memory_order::relaxed, \"\n \"sycl::memory_order::acq_rel, sycl::memory_order::seq_cst!\");\n }\n }\n\n template \n inline T1 atomic_fetch_add(T1 *addr, T2 operand,\n sycl::memory_order memoryOrder) {\n atomic_fetch_add(addr, operand, memoryOrder);\n }\n\n inline unsigned int byte_level_permute(\n unsigned int a, unsigned int b, unsigned int s) {\n unsigned int ret;\n ret = ((((std::uint64_t)b << 32 | a) >> (s & 0x7) * 8) & 0xff) |\n (((((std::uint64_t)b << 32 | a) >> ((s >> 4) & 0x7) * 8) & 0xff)\n << 8) |\n (((((std::uint64_t)b << 32 | a) >> ((s >> 8) & 0x7) * 8) & 0xff)\n << 16) |\n (((((std::uint64_t)b << 32 | a) >> ((s >> 12) & 0x7) * 8) & 0xff)\n << 24);\n return ret;\n }\n\n inline uint32_t byte_level_permute_custom(\n uint32_t low32, uint32_t high32, uint32_t sel, int mode = 0) {\n constexpr uint16_t lookup[6][4] = {\n {0x3210, 0x4321, 0x5432, 0x6543}, // Forward 4-byte extract\n {0x5670, 0x6701, 0x7012, 0x0123}, // Backward 4-byte extract\n {0x0000, 0x1111, 0x2222, 0x3333}, // Replicate 8-bit values\n {0x3210, 0x3211, 0x3222, 0x3333}, // Edge clamp left\n {0x0000, 0x1110, 0x2210, 0x3210}, // Edge clamp right\n {0x1010, 0x3232, 0x1010, 0x3232} // Replicate 16-bit values\n };\n\n if (mode >= 1 && mode <= 6) {\n return byte_level_permute(low32, high32, lookup[mode - 1][sel & 0x3]);\n } else if (!mode) {\n return byte_level_permute(low32, high32, sel);\n }\n return 0;\n }\n\n template \n class args_selector;\n\n /// args_selector is a helper class for extracting arguments from an\n /// array of pointers to arguments or buffer of arguments to pass to a\n /// kernel function.\n ///\n /// \\param R(Ts...) The type of the kernel\n /// \\param n_nondefault_params The number of nondefault parameters of the\n /// kernel (excluding parameters that like sycl::nd_item, etc.) \\param\n /// n_default_params The number of default parameters of the kernel\n ///\n /// Example usage:\n /// With the following kernel:\n /// void foo(sycl::float2 *x, int n, sycl::nd_item<3> item_ct1, float\n /// f=.1) {}\n /// and with the declaration:\n /// args_selector<2, 1, decltype(foo)> selector(kernelParams, extra);\n /// we have:\n /// selector.get<0>() returns a reference to sycl::float*,\n /// selector.get<1>() returns a reference to int,\n /// selector.get<2>() returns a reference to float\n template \n class args_selector {\n private:\n void **kernel_params;\n char *args_buffer;\n\n template static constexpr int account_for_default_params() {\n constexpr int n_total_params = sizeof...(Ts);\n if constexpr (i >= n_nondefault_params) {\n return n_total_params - n_default_params +\n (i - n_nondefault_params);\n } else {\n return i;\n }\n }\n\n public:\n /// Get the type of the ith argument of R(Ts...)\n /// \\param [in] i Index of parameter to get\n /// \\returns Type of ith parameter\n template \n using arg_type = std::tuple_element_t(),\n std::tuple>;\n static constexpr int params_num = sizeof...(Ts);\n\n private:\n template static constexpr int get_offset() {\n if constexpr (i == 0) {\n // we can assume args_buffer is properly aligned to the\n // first argument\n return 0;\n } else {\n constexpr int prev_off = get_offset();\n constexpr int prev_past_end =\n prev_off + sizeof(arg_type);\n using T = arg_type;\n // is the past-the-end of the i-1st element properly aligned\n // with the ith element's alignment?\n if constexpr (prev_past_end % alignof(T) == 0) {\n return prev_past_end;\n }\n // otherwise bump prev_past_end to match alignment\n else {\n return prev_past_end +\n (alignof(T) - (prev_past_end % alignof(T)));\n }\n }\n }\n\n static char *get_args_buffer(void **extra) {\n if (!extra)\n return nullptr;\n for (; (std::size_t)*extra != 0; ++extra) {\n if ((std::size_t)*extra == 1) {\n return static_cast(*(extra + 1));\n }\n }\n return nullptr;\n }\n\n public:\n /// If kernel_params is nonnull, then args_selector will\n /// extract arguments from kernel_params. Otherwise, it\n /// will extract them from extra.\n /// \\param [in] kernel_params Array of pointers to arguments\n /// a or null pointer.\n /// \\param [in] extra Array containing pointer to argument buffer.\n args_selector(void **kernel_params, void **extra)\n : kernel_params(kernel_params),\n args_buffer(get_args_buffer(extra)) {}\n\n /// Get a reference to the ith argument extracted from kernel_params\n /// or extra.\n /// \\param [in] i Index of argument to get\n /// \\returns Reference to the ith argument\n template arg_type &get() {\n if (kernel_params) {\n return *static_cast *>(kernel_params[i]);\n } else {\n return *reinterpret_cast *>(args_buffer +\n get_offset());\n }\n }\n }; // COPY from DPCT head file\n // /opt/intel/oneapi/dpcpp-ct/latest/include/dpct/util.hpp\n\n /// Utility class for launching SYCL kernels through kernel\n /// function wrapper.\n /// For example:\n /// A SYCL kernel function:\n /// void kernel_func(int *ptr, sycl::nd_item<3> item);\n /// Kernel function wrapper:\n /// void kernel_func_wrapper(int *ptr) {\n /// sycl::queue queue = *dpct::kernel_launcher::_que;\n /// unsigned int localMemSize = dpct::kernel_launcher::_local_mem_size;\n /// sycl::nd_range<3> nr = dpct::kernel_launcher::_nr;\n /// queue.parallel_for(\n /// nr,\n /// [=](sycl::nd_item<3> item_ct1) {\n /// kernel_func(ptr, item_ct1);\n /// });\n /// }\n /// Then launch the kernel through wrapper like:\n /// typedef void(*fpt)(int *);\n /// fpt fp = kernel_func_wrapper;\n /// dpct::kernel_launcher::launch(fp, dpct::dim3(1), dpct::dim3(1), 0, 0,\n /// device_ptr);\n /// If the origin function type is erased, then need to register it first:\n /// void *fp = (void *)wrapper_register(&kernel_func_wrapper).get();\n /// dpct::kernel_launcher::launch(fp, dpct::dim3(1), dpct::dim3(1), args,\n /// 0, 0);\n class kernel_launcher {\n template \n static void launch_helper(FuncT &&func, ArgSelector &selector,\n std::index_sequence) {\n func(selector.template get()...);\n }\n static void set_execution_config(dim3 group_range, dim3 local_range,\n unsigned int local_mem_size,\n queue_ptr que) {\n if (que) {\n _que = que;\n } else {\n _que = &get_default_queue();\n }\n _nr = sycl::nd_range<3>(\n static_cast>(group_range * local_range),\n static_cast>(local_range));\n _local_mem_size = local_mem_size;\n\n\n };\n static inline std::mutex kernel_function_ptr_map_mutex;\n\n public:\n /// Variables for storing execution configuration.\n static inline thread_local sycl::queue *_que = nullptr;\n static inline thread_local sycl::nd_range<3> _nr = sycl::nd_range<3>();\n static inline thread_local unsigned int _local_mem_size = 0;\n /// Map for retrieving launchable functor from a raw pointer.\n static inline std::map<\n const void *,\n std::function>\n kernel_function_ptr_map = {};\n\n /// Registers a kernel function pointer with a corresponding launchable\n /// functor.\n /// \\param [in] func Pointer to the kernel function.\n /// \\param [in] launcher Functor to handle kernel invocation.\n static void register_kernel_ptr(\n const void *func,\n std::function\n launcher) {\n std::lock_guard lock(kernel_function_ptr_map_mutex);\n kernel_function_ptr_map[func] = std::move(launcher);\n }\n /// Launches a kernel function with arguments provided directly through\n /// kernel function wrapper.\n /// \\tparam FuncT Type of the kernel function wrapper.\n /// \\tparam ArgsT Types of kernel arguments.\n /// \\param [in] func Pointer to the kernel function wrapper.\n /// \\param [in] group_range SYCL group range.\n /// \\param [in] local_range SYCL local range.\n /// \\param [in] local_mem_size The size of local memory required by the\n /// kernel function. \\param [in] que SYCL queue used to execute kernel.\n /// \\param [in] args Kernel arguments.\n template \n static std::enable_if_t, void>\n launch(FuncT *func, dim3 group_range, dim3 local_range,\n unsigned int local_mem_size, queue_ptr que, ArgsT... args) {\n set_execution_config(group_range, local_range, local_mem_size, que);\n func(args...);\n }\n /// Launches a kernel function through registered kernel function\n /// wrapper. \\param [in] func Pointer to the registered kernel function\n /// wrapper. \\param [in] group_range SYCL group range. \\param [in]\n /// local_range SYCL local range. \\param [in] args Array of pointers to\n /// kernel arguments. \\param [in] local_mem_size The size of local\n /// memory required by the kernel function. \\param [in] que SYCL queue\n /// used to execute kernel.\n static void launch(const void *func, dim3 group_range, dim3 local_range,\n void **args, unsigned int local_mem_size,\n queue_ptr que) {\n std::lock_guard lock(kernel_function_ptr_map_mutex);\n auto Iter = kernel_function_ptr_map.find(func);\n if (Iter == kernel_function_ptr_map.end()) {\n throw std::runtime_error(\"dpct::launch() : no registered \"\n \"kernel function wrapper found.\");\n }\n (Iter->second)(group_range, local_range, args, local_mem_size, que);\n }\n /// Launches a kernel function with packed arguments through kernel\n /// function wrapper.\n /// \\tparam FuncT Type of the kernel function wrapper.\n /// \\param [in] func Pointer to the kernel function wrapper.\n /// \\param [in] group_range SYCL group range.\n /// \\param [in] local_range SYCL local range.\n /// \\param [in] args Array of pointers to kernel arguments.\n /// \\param [in] local_mem_size The size of local memory required by the\n /// kernel function. \\param [in] que SYCL queue used to execute kernel.\n template \n static std::enable_if_t, void>\n launch(FuncT *func, dim3 group_range, dim3 local_range, void **args,\n unsigned int local_mem_size, queue_ptr que) {\n constexpr size_t p_num = args_selector<0, 0, FuncT>::params_num;\n set_execution_config(group_range, local_range, local_mem_size, que);\n args_selector selector(args, nullptr);\n launch_helper(func, selector, std::make_index_sequence{});\n }\n }; // COPY from DPCT head file\n // /opt/intel/oneapi/dpcpp-ct/latest/include/dpct/kernel.hpp\n\n // /opt/intel/oneapi/dpcpp-ct/latest/include/dpct/util.hpp\n template \n T select_from_sub_group(\n sycl::sub_group g,\n T x,\n int remote_local_id,\n int logical_sub_group_size = 32) {\n unsigned int start_index = g.get_local_linear_id() /\n logical_sub_group_size *\n logical_sub_group_size;\n return sycl::select_from_group(\n g, x, start_index + remote_local_id % logical_sub_group_size);\n }\n\n // /opt/intel/oneapi/dpcpp-ct/latest/include/dpct/math.hpp\n template \n void ldmatrix(uintptr_t addr, T* m, bool trans = false, unsigned mat = 0) {\n auto sg = sycl::ext::oneapi::this_work_item::get_sub_group();\n int lane = sg.get_local_linear_id();\n\n int lane_group8_row = lane / 8;\n int lane_group8_col = lane % 8;\n\n if (!trans) {\n // calculate the source lane\n int src_lane = 2 * lane_group8_row;\n if (lane_group8_col >= 4)\n src_lane += 1;\n\n // Broadcast the address from the source lane\n auto recv_addr_uintp =\n dpct::select_from_sub_group(sg, addr, mat * 8 + src_lane);\n\n // Cast the received address from uintptr_t to the type of 'm'\n auto recv_addr = reinterpret_cast(recv_addr_uintp);\n\n // Non-transposed load\n *m = recv_addr[lane_group8_col % 4];\n } else {\n // calculate the source lane\n int src_lane = (lane % 4) * 2;\n\n // Broadcast the address from the source lane\n auto recv_addr_uintp_1 =\n dpct::select_from_sub_group(sg, addr, mat * 8 + src_lane);\n auto recv_addr_uintp_2 =\n dpct::select_from_sub_group(sg, addr, mat * 8 + src_lane + 1);\n\n // Cast the received address from uintptr_t to 'half *'\n auto recv_addr_1 = reinterpret_cast(recv_addr_uintp_1);\n auto recv_addr_2 = reinterpret_cast(recv_addr_uintp_2);\n\n // Transposed load\n int index = lane / 4;\n sycl::half val0 = recv_addr_1[index];\n sycl::half val1 = recv_addr_2[index];\n\n // Combine the two 16-bits into one 32-bit value\n sycl::half2 val = sycl::half2(val0, val1);\n *m = *reinterpret_cast(&val);\n }\n }\n\n template \n void ldmatrix(uintptr_t addr, T* m1, T* m2, bool trans = false) {\n // Load 1st matrix\n ldmatrix(addr, m1, trans, 0);\n // Load 2nd matrix\n ldmatrix(addr, m2, trans, 1);\n }\n\n template \n void ldmatrix(\n uintptr_t addr, T* m1, T* m2, T* m3, T* m4, bool trans = false) {\n // Load 1st matrix\n ldmatrix(addr, m1, trans, 0);\n // Load 2nd matrix\n ldmatrix(addr, m2, trans, 1);\n // Load 3rd matrix\n ldmatrix(addr, m3, trans, 2);\n // Load 4th matrix\n ldmatrix(addr, m4, trans, 3);\n }\n\n // /opt/intel/oneapi/dpcpp-ct/latest/include/dpct/math.hpp\n\n /// A helper struct that defines the pack type for the input matrix\n /// fragments\n /// of mma() function based on the type of input matrix fragments.\n /// The MMAType struct is specialized for different types of input matrices.\n /// Currently, the specialization for f16, bf16 and s8 types is defined\n /// below. \\tparam [in] T The type of the input matrix fragments\n template \n struct MMAType {\n using PackType = uint32_t;\n };\n\n /// Each work item of a sub-group (limited to size 32) calling this function\n /// calculates a subset fragment for the output matrix D using MAD operation\n /// on A, B & C matrix fragments (D = A * B + C). Current supported shapes &\n /// types:\n /// - m8n8k4 (f32.f16.f16.f32)\n /// - m8n8k16 (s32.s8.s8.s32)\n /// - m16n8k8 (f32.f16.f16.f32 & f32.bf16.bf16.f32)\n /// - m16n8k16 (f32.f16.f16.f32 & s32.s8.s8.s32)\n /// - m16n8k32 (s32.s8.s8.s32)\n /// Here, m, n & k define the shapes of A, B & C matrices respectively\n /// (A = [m x k], B = [k x n], C = [m x n]).\n /// \\tparam [in] M The rows of A, C & D matrices\n /// \\tparam [in] N The columns of B, C, D matrices\n /// \\tparam [in] K The columns & rows of A & B matrices respectively\n /// \\tparam [in] ABType The type of the input matrix (A & B) fragment\n /// \\tparam [in] CDType The type of the output matrix (C & D) fragment\n /// \\param [out] d_mat_frag The fragment of the output matrix D to store the\n /// result of A * B + C\n /// \\param [in] a_mat_frag The fragment of the input matrix A to be\n /// multiplied with B matrix fragment \\param [in] b_mat_frag The fragment of\n /// the input matrix B to be multiplied with A matrix fragment \\param [in]\n /// c_mat_frag The fragment of the input matrix C to be added with the\n /// result of A * B fragments\n template \n void mma(\n volatile void** d_mat_frag,\n void* a_mat_frag,\n void* b_mat_frag,\n void* c_mat_frag) {\n auto d = reinterpret_cast(d_mat_frag);\n auto a =\n reinterpret_cast::PackType*>(a_mat_frag);\n auto b =\n reinterpret_cast::PackType*>(b_mat_frag);\n auto c = reinterpret_cast(c_mat_frag);\n\n auto sg = sycl::ext::oneapi::this_work_item::get_sub_group();\n int lane = sg.get_local_linear_id();\n\n static_assert(\n (M == 8 && N == 8 && K == 4) || (M == 8 && N == 8 && K == 16) ||\n (M == 16 && N == 8 && K == 8) || (M == 16 && N == 8 && K == 16) ||\n (M == 16 && N == 8 && K == 32),\n \"Unsupported MMA shape!\");\n\n short row_load_offset = 4 * (lane >> 2);\n short col_load_offset = 8 * (lane % 4);\n\n if constexpr (M == 8 && N == 8 && K == 4) {\n if constexpr (std::is_floating_point_v) {\n col_load_offset = row_load_offset % 16;\n\n // Init D matrix with fragments of C matrix\n *d[0] = c[0];\n *d[1] = c[1];\n *d[2] = c[2];\n *d[3] = c[3];\n *d[4] = c[4];\n *d[5] = c[5];\n *d[6] = c[6];\n *d[7] = c[7];\n\n // Calculate the row and col offset indices to iterate through the row\n // & col fragments of A & B matrices\n int r_ind = (lane % 2) ? 1 : 0;\n int c_ind = ((lane % 4) / 2) ? 2 : 0;\n\n // Each sub-group is responsible for computing a fragment size of 8*8\n // elements of matrix D for each of 4 MMA computations.\n // Each work item computes 8 elements of matrix D by gathering\n // their corresponding col & row matrix fragments of length k (4)\n // from A & B matrices respectively using below mapping logic:\n // row0 = (i % 4) if (lane < 16) else (i % 4) + 4\n // col0 = (lane % 4)\n // As each row & col fragment of A & B matrices is distributed across\n // 4 work items, each iteration of below loop loads a partial fragment\n // of matrix A (row) and matrix B (col) using the row & col offsets.\n typename MMAType::PackType recv_a[2], recv_b[2];\n\n for (int i = 0; i < 4; i++) {\n // Load partial fragment from col0 of matrix A ({a0, a1})\n recv_a[0] =\n dpct::select_from_sub_group(sg, a[0], row_load_offset + i);\n // Load partial fragment from col0 of matrix A ({a2, a3})\n recv_a[1] =\n dpct::select_from_sub_group(sg, a[1], row_load_offset + i);\n\n // Load partial fragment from row0 of matrix B ({b0, b1})\n recv_b[0] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i);\n // Load partial fragment from row0 of matrix B ({b2, b3})\n recv_b[1] =\n dpct::select_from_sub_group(sg, b[1], col_load_offset + i);\n\n auto ra = reinterpret_cast(recv_a);\n auto rb = reinterpret_cast(recv_b);\n\n // Each work item calculates a partial product of A & B matrix\n // fragments and adds it to the corresponding D matrix fragment (for\n // even work item indices) d0 += col0{ a0 } * row0{ b0 } d1 += col0{\n // a0 } * row0{ b1 } d2 += col1{ a2 } * row0{ b0 } d3 += col1{ a2 }\n // * row0{ b1 } (for odd work item indices) d0 += col0{ a1 } * row0{\n // b2 } d1 += col0{ a1 } * row0{ b3 } d2 += col1{ a3 } * row0{ b2 }\n // d3 += col1{ a3 } * row0{ b3 }\n *d[0] +=\n static_cast(ra[r_ind]) * static_cast(rb[c_ind]);\n *d[1] += static_cast(ra[r_ind]) *\n static_cast(rb[c_ind + 1]);\n *d[2] += static_cast(ra[r_ind + 2]) *\n static_cast(rb[c_ind]);\n *d[3] += static_cast(ra[r_ind + 2]) *\n static_cast(rb[c_ind + 1]);\n\n // Load partial fragment from row1 of matrix B ({b0, b1})\n recv_b[0] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i + 16);\n // Load partial fragment from row1 of matrix B ({b2, b3})\n recv_b[1] =\n dpct::select_from_sub_group(sg, b[1], col_load_offset + i + 16);\n\n // (for even work item indices)\n // d0 += col0{ a0 } * row1{ b0 }\n // d1 += col0{ a0 } * row1{ b1 }\n // d2 += col1{ a2 } * row1{ b0 }\n // d3 += col1{ a2 } * row1{ b1 }\n // (for odd work item indices)\n // d0 += col0{ a1 } * row1{ b2 }\n // d1 += col0{ a1 } * row1{ b3 }\n // d2 += col1{ a3 } * row1{ b2 }\n // d3 += col1{ a3 } * row1{ b3 }\n *d[4] +=\n static_cast(ra[r_ind]) * static_cast(rb[c_ind]);\n *d[5] += static_cast(ra[r_ind]) *\n static_cast(rb[c_ind + 1]);\n *d[6] += static_cast(ra[r_ind + 2]) *\n static_cast(rb[c_ind]);\n *d[7] += static_cast(ra[r_ind + 2]) *\n static_cast(rb[c_ind + 1]);\n }\n }\n } else if constexpr (M == 8 && N == 8 && K == 16) {\n if constexpr (std::is_integral_v) {\n // Init D matrix with fragments of C matrix\n *d[0] = c[0];\n *d[1] = c[1];\n\n // Each sub-group is responsible for computing a fragment size of 16*8\n // elements of matrix D.\n // Each work item computes 2 elements of matrix D by gathering\n // their corresponding row & col matrix fragments of length k (16)\n // from A & B matrices respectively using below mapping logic:\n // row0 = ((lane % 4) * 4) + i\n // col0 = (lane >> 2)\n // As each row & col fragment of A & B matrices is distributed across\n // 4 work items, each iteration of below loop loads a partial fragment\n // of matrix A (row) and matrix B (col) using the row & col offsets.\n for (int i = 0; i < 4; i++) {\n typename MMAType::PackType recv_a, recv_b[2];\n\n // Load partial fragment from row0 of matrix A ({a0, a1, a2, a3})\n recv_a = dpct::select_from_sub_group(sg, a[0], row_load_offset + i);\n // Load partial fragment from col0 of matrix B ({b0, b1, b2, b3})\n recv_b[0] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i);\n // Load partial fragment from col1 of matrix B ({b0, b1, b2, b3})\n recv_b[1] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i + 4);\n\n auto a = reinterpret_cast(&recv_a);\n auto b = reinterpret_cast(recv_b);\n\n // Each work item calculates a partial product of A & B matrix\n // fragments and adds it to the corresponding D matrix fragment d0\n // += row0{ a0, a1, a2, a3 } * col0{ b0, b1, b2, b3 } d1 += row0{\n // a0, a1, a2, a3 } * col1{ b0, b1, b2, b3 } d2 += row0{ a0, a1, a2,\n // a3 } * col0{ b0, b1, b2, b3 } d3 += row0{ a0, a1, a2, a3 } *\n // col1{ b0, b1, b2, b3 }\n for (int j = 0; j < 4; j++) {\n *d[0] += a[j] * b[j];\n *d[1] += a[j] * b[j + 4];\n }\n }\n }\n } else if constexpr (M == 16 && N == 8 && K == 8) {\n if constexpr (std::is_floating_point_v) {\n // Init D matrix fragment with C matrix fragment\n *d[0] = c[0];\n *d[1] = c[1];\n *d[2] = c[2];\n *d[3] = c[3];\n\n // Each sub-group is responsible for computing a fragment size of 16*8\n // elements of matrix D.\n // Each work item computes 4 elements of matrix D by gathering\n // their corresponding row & col matrix fragments of length k (8)\n // from A & B matrices respectively using below mapping logic:\n // row0 = (lane >> 2) & row1 = (lane >> 2) + 8\n // col0 = (lane % 4) * 2 + (i & 0x1)\n // As each row & col fragment of A & B matrices is distributed across\n // 4 work items, each iteration of below loop loads a partial fragment\n // of matrix A (row) and matrix B (col) using the row & col offsets.\n for (int i = 0; i < 4; i++) {\n typename MMAType::PackType recv_a[2], recv_b[2];\n\n // Load partial fragment from row0 of matrix A ({a0, a1})\n recv_a[0] =\n dpct::select_from_sub_group(sg, a[0], row_load_offset + i);\n // Load partial fragment from row1 of matrix A ({a2, a3})\n recv_a[1] =\n dpct::select_from_sub_group(sg, a[1], row_load_offset + i);\n // Load partial fragment from col0 of matrix B ({b0, b1})\n recv_b[0] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i);\n // Load partial fragment from col1 of matrix B ({b0, b1})\n recv_b[1] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i + 4);\n\n auto ra = reinterpret_cast(recv_a);\n auto rb = reinterpret_cast(recv_b);\n\n // Each work item calculates a partial product of A & B matrix\n // fragments and adds it to the corresponding D matrix fragment d0\n // += row0{ a0, a1 } * col0{ b0, b1 } d1 += row0{ a0, a1 } * col1{\n // b0, b1 } d2 += row1{ a2, a3 } * col0{ b0, b1 } d3 += row1{ a2, a3\n // } * col1{ b0, b1 }\n for (int j = 0; j < 2; j++) {\n *d[0] += static_cast(ra[j]) * static_cast(rb[j]);\n *d[1] +=\n static_cast(ra[j]) * static_cast(rb[j + 2]);\n *d[2] +=\n static_cast(ra[j + 2]) * static_cast(rb[j]);\n *d[3] +=\n static_cast(ra[j + 2]) * static_cast(rb[j + 2]);\n }\n }\n }\n } else if constexpr (M == 16 && N == 8 && K == 16) {\n if constexpr (std::is_floating_point_v) {\n // Init D matrix fragment with C matrix fragment\n *d[0] = c[0];\n *d[1] = c[1];\n *d[2] = c[2];\n *d[3] = c[3];\n\n // Each sub-group is responsible for computing a fragment size of 16*8\n // elements of matrix D.\n // Each work item computes 4 elements of matrix D by gathering\n // their corresponding row & col matrix fragments of length k (8)\n // from A & B matrices respectively using below mapping logic:\n // row0 = (lane >> 2) & row1 = (lane >> 2) + 8\n // col0 = (lane % 4) * 2 & col1 = (lane % 4) * 2 + 1\n // As each row & col fragment of A & B matrices is distributed across\n // 4 work items, each iteration of below loop loads a partial fragment\n // of matrix A (row) and matrix B (col) using the row & col offsets.\n for (int i = 0; i < 4; i++) {\n typename MMAType::PackType recv_a[4], recv_b[4];\n\n // Load partial fragment from row0 of matrix A ({a0, a1})\n recv_a[0] =\n dpct::select_from_sub_group(sg, a[0], row_load_offset + i);\n // Load partial fragment from row0 of matrix A ({a2, a3})\n recv_a[1] =\n dpct::select_from_sub_group(sg, a[2], row_load_offset + i);\n // Load partial fragment from row1 of matrix A ({a0, a1})\n recv_a[2] =\n dpct::select_from_sub_group(sg, a[1], row_load_offset + i);\n // Load partial fragment from row1 of matrix A ({a2, a3})\n recv_a[3] =\n dpct::select_from_sub_group(sg, a[3], row_load_offset + i);\n\n // Load partial fragment from col0 of matrix B ({b0, b1})\n recv_b[0] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i);\n // Load partial fragment from col0 of matrix B ({b2, b3})\n recv_b[1] =\n dpct::select_from_sub_group(sg, b[1], col_load_offset + i);\n // Load partial fragment from col1 of matrix B ({b0, b1})\n recv_b[2] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + 4 + i);\n // Load partial fragment from col1 of matrix B ({b2, b3})\n recv_b[3] =\n dpct::select_from_sub_group(sg, b[1], col_load_offset + 4 + i);\n\n auto ra = reinterpret_cast(recv_a);\n auto rb = reinterpret_cast(recv_b);\n\n // Each work item calculates a partial product of A & B matrix\n // fragments and adds it to the corresponding D matrix fragment d0\n // += row0{ a0, a1, a2, a3 } * col0{ b0, b1, b2, b3 } d1 += row0{\n // a0, a1, a2, a3 } * col1{ b0, b1, b2, b3 } d2 += row1{ a0, a1, a2,\n // a3 } * col0{ b0, b1, b2, b3 } d3 += row1{ a0, a1, a2, a3 } *\n // col1{ b0, b1, b2, b3 }\n for (int j = 0; j < 4; j++) {\n *d[0] += static_cast(ra[j]) * static_cast(rb[j]);\n *d[1] +=\n static_cast(ra[j]) * static_cast(rb[j + 4]);\n *d[2] +=\n static_cast(ra[j + 4]) * static_cast(rb[j]);\n *d[3] += static_cast(ra[j + 4]) *\n static_cast(rb[j + 4]);\n }\n }\n } else if constexpr (std::is_integral_v) {\n // Init D matrix with fragments of C matrix\n *d[0] = c[0];\n *d[1] = c[1];\n *d[2] = c[2];\n *d[3] = c[3];\n\n // Each sub-group is responsible for computing a fragment size of 16*8\n // elements of matrix D.\n // Each work item computes 4 elements of matrix D by gathering\n // their corresponding row & col matrix fragments of length k (8)\n // from A & B matrices respectively using below mapping logic:\n // row0 = (lane >> 2) & row1 = (lane >> 2) + 8\n // col0 = (lane % 4) * 2 & col1 = (lane % 4) * 2 + 1\n // As each row & col fragment of A & B matrices is distributed across\n // 4 work items, each iteration of below loop loads a partial fragment\n // of matrix A (row) and matrix B (col) using the row & col offsets.\n for (int i = 0; i < 4; i++) {\n typename MMAType::PackType recv_a[2], recv_b[2];\n\n // Load partial fragment from row0 of matrix A ({a0, a1, a2, a3})\n recv_a[0] =\n dpct::select_from_sub_group(sg, a[0], row_load_offset + i);\n // Load partial fragment from row1 of matrix A ({a4, a5, a6, a7})\n recv_a[1] =\n dpct::select_from_sub_group(sg, a[1], row_load_offset + i);\n // Load partial fragment from col0 of matrix B ({b0, b1, b2, b3})\n recv_b[0] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i);\n // Load partial fragment from col1 of matrix B ({b4, b5, b6, b7})\n recv_b[1] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i + 4);\n\n auto ra = reinterpret_cast(recv_a);\n auto rb = reinterpret_cast(recv_b);\n\n // Each work item calculates a partial product of A & B matrix\n // fragments and adds it to the corresponding D matrix fragment d0\n // += row0{ a0, a1, a2, a3 } * col0{ b0, b1, b2, b3 } d1 += row0{\n // a0, a1, a2, a3 } * col1{ b4, b5, b6, b7 } d2 += row1{ a4, a5, a6,\n // a7 } * col0{ b0, b1, b2, b3 } d3 += row1{ a4, a5, a6, a7 } *\n // col1{ b4, b5, b6, b7 }\n for (int i = 0; i < 4; i++) {\n *d[0] += ra[i] * rb[i];\n *d[1] += ra[i] * rb[i + 4];\n *d[2] += ra[i + 4] * rb[i];\n *d[3] += ra[i + 4] * rb[i + 4];\n }\n }\n }\n } else if constexpr (M == 16 && N == 8 && K == 32) {\n if constexpr (std::is_integral_v) {\n // Init D matrix with fragments of C matrix\n *d[0] = c[0];\n *d[1] = c[1];\n *d[2] = c[2];\n *d[3] = c[3];\n\n // Each sub-group is responsible for computing a fragment size of 16*8\n // elements of matrix D.\n // Each work item computes 4 elements of matrix D by gathering\n // their corresponding row & col matrix fragments of length k (32)\n // from A & B matrices respectively using below mapping logic:\n // row0 = (lane >> 2) & row1 = (lane >> 2) + 8\n // col0 = ((lane % 4) * 4) + (i & 0x3) & col1 = ((lane % 4) * 4) + (i\n // & 0x3) As each row & col fragment of A & B matrices is distributed\n // across 4 work items, each iteration of below loop loads a partial\n // fragment of matrix A (row) and matrix B (col) using the row & col\n // offsets.\n for (int i = 0; i < 4; i++) {\n typename MMAType::PackType recv_a[2], recv_b[2];\n\n // Load partial fragment from row0 of matrix A ({a0, a1, a2, a3})\n recv_a[0] =\n dpct::select_from_sub_group(sg, a[0], row_load_offset + i);\n // Load partial fragment from row1 of matrix A ({a4, a5, a6, a7})\n recv_a[1] =\n dpct::select_from_sub_group(sg, a[1], row_load_offset + i);\n // Load partial fragment from col0 of matrix B ({b0, b1, b2, b3})\n recv_b[0] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i);\n // Load partial fragment from col1 of matrix B ({b0, b1, b2, b3})\n recv_b[1] =\n dpct::select_from_sub_group(sg, b[0], col_load_offset + i + 4);\n\n auto a = reinterpret_cast(recv_a);\n auto b = reinterpret_cast(recv_b);\n\n // Each work item calculates a partial product of A & B matrix\n // fragments and adds it to the corresponding D matrix fragment d0\n // += row0{ a0, a1, a2, a3 } * col0{ b0, b1, b2, b3 } d1 += row0{\n // a0, a1, a2, a3 } * col1{ b0, b1, b2, b3 } d2 += row1{ a4, a5, a6,\n // a7 } * col0{ b0, b1, b2, b3 } d3 += row1{ a4, a5, a6, a7 } *\n // col1{ b0, b1, b2, b3 }\n for (int j = 0; j < 4; j++) {\n *d[0] += a[j] * b[j];\n *d[1] += a[j] * b[j + 4];\n *d[2] += a[j + 4] * b[j];\n *d[3] += a[j + 4] * b[j + 4];\n }\n }\n\n for (int i = 0; i < 4; i++) {\n typename MMAType::PackType recv_a[2], recv_b[2];\n\n // Load partial fragment from row0 of matrix A ({a8, a9, a10, a11})\n recv_a[0] =\n dpct::select_from_sub_group(sg, a[2], row_load_offset + i);\n // Load partial fragment from row1 of matrix A ({a12, a13, a14,\n // a15})\n recv_a[1] =\n dpct::select_from_sub_group(sg, a[3], row_load_offset + i);\n // Load partial fragment from col0 of matrix B ({b4, b5, b6, b7})\n recv_b[0] =\n dpct::select_from_sub_group(sg, b[1], col_load_offset + i);\n // Load partial fragment from col1 of matrix B ({b4, b5, b6, b7})\n recv_b[1] =\n dpct::select_from_sub_group(sg, b[1], col_load_offset + i + 4);\n\n auto a = reinterpret_cast(recv_a);\n auto b = reinterpret_cast(recv_b);\n\n // Each work item calculates a partial product of A & B matrix\n // fragments and adds it to the corresponding D matrix fragment d0\n // += row0{ a8, a9, a10, a11 } * col0{ b4, b5, b6, b7 } d1 += row0{\n // a8, a9, a10, a11 } * col1{ b4, b5, b6, b7 } d2 += row1{ a12, a13,\n // a14, a15 } * col0{ b4, b5, b6, b7 } d3 += row1{ a12, a13, a14,\n // a15 } * col1{ b4, b5, b6, b7 }\n for (int j = 0; j < 4; j++) {\n *d[0] += a[j] * b[j];\n *d[1] += a[j] * b[j + 4];\n *d[2] += a[j + 4] * b[j];\n *d[3] += a[j + 4] * b[j + 4];\n }\n }\n }\n }\n }\n} // COPY from DPCT head files\n\n#endif // GGML_SYCL_DPCT_HELPER_HPP\n" }, { "path": "src/ggml-sycl/element_wise.cpp", "content": "#include \"common.hpp\"\n#include \"ggml-sycl/presets.hpp\"\n#include \"ggml.h\"\n#include \"element_wise.hpp\"\n\n#define SYCL_GLOBAL_ID_LOOP(K, ITEM) \\\n for (auto i = ITEM.get_global_id(0); i < (size_t)K; i += ITEM.get_global_range(0))\n\n#define SYCL_LOCAL_ID_CALC(ITEM, IDX) \\\n (ITEM.get_local_range(IDX) * ITEM.get_group(IDX) + ITEM.get_local_id(IDX))\n\nstatic void acc_f32(const float * x, const float * y, float * dst, const int64_t ne,\n const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,\n const int64_t s11, const int64_t s12, const int64_t s13, const int64_t offset) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const int64_t i = SYCL_LOCAL_ID_CALC(item_ct1, 2);\n\n if (i >= ne) {\n return;\n }\n\n int64_t src1_idx = i - offset;\n\n int64_t tmp = src1_idx;\n const int64_t i13 = tmp / s13;\n tmp -= i13 * s13;\n const int64_t i12 = tmp / s12;\n tmp -= i12 * s12;\n const int64_t i11 = tmp / s11;\n tmp -= i11 * s11;\n const int64_t i10 = tmp;\n\n float val = x[i];\n if (src1_idx >= 0 && i10 < ne10 && i11 < ne11 && i12 < ne12 && i13 < ne13) {\n val += y[((i13*ne12 + i12) * ne11 + i11) * ne10 + i10];\n }\n dst[i] = val;\n}\n\n/* Unary OP funcs */\ntemplate\nstatic __dpct_inline__ T op_sgn(T x) {\n return x > static_cast(0.f) ? static_cast(1.f) : ((x < static_cast(0.f) ? static_cast(-1.f) : static_cast(0.f)));\n}\n\ntemplate\nstatic __dpct_inline__ T op_abs(T x) {\n return sycl::fabs(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_elu(T x) {\n return (x > static_cast(0.f)) ? x : sycl::expm1(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_gelu(T x) {\n const T GELU_COEF_A = static_cast(0.044715f);\n const T SQRT_2_OVER_PI = static_cast(0.79788456080286535587989211986876f);\n return static_cast(0.5f) * x *\n (static_cast(1.0f) +\n sycl::tanh(SQRT_2_OVER_PI * x * (static_cast(1.0f) + GELU_COEF_A * x * x)));\n}\n\ntemplate\nstatic __dpct_inline__ T op_silu(T x) {\n return x / (static_cast(1.0f) + sycl::native::exp(-x));\n}\n\ntemplate\nstatic __dpct_inline__ T op_gelu_quick(T x) {\n const T GELU_QUICK_COEF_LOCAL = static_cast(-1.702f);\n return x * (static_cast(1.0f) / (static_cast(1.0f) + sycl::native::exp(GELU_QUICK_COEF_LOCAL * x)));\n}\n\ntemplate\nstatic __dpct_inline__ T op_gelu_erf(T x) {\n const T SQRT_2_INV = static_cast(0.70710678118654752440084436210484f);\n return static_cast(0.5f) * x * (static_cast(1.0f) + sycl::erf(x * SQRT_2_INV));\n}\n\ntemplate\nstatic __dpct_inline__ T op_tanh(T x) {\n return sycl::tanh(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_relu(T x) {\n return sycl::fmax(x, static_cast(0));\n}\n\ntemplate\nstatic __dpct_inline__ T op_sigmoid(T x) {\n return static_cast(1.0f) / (static_cast(1.0f) + sycl::native::exp(-x));\n}\n\ntemplate\nstatic __dpct_inline__ T op_sqrt(T x) {\n return sycl::sqrt(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_sin(T x) {\n return sycl::sin(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_cos(T x) {\n return sycl::cos(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_hardsigmoid(T x) {\n return sycl::fmin(static_cast(1.0f), sycl::fmax(static_cast(0.0f), (x + static_cast(3.0f)) / static_cast(6.0f)));\n}\n\ntemplate\nstatic __dpct_inline__ T op_hardswish(T x) {\n return x * sycl::fmin(static_cast(1.0f), sycl::fmax(static_cast(0.0f), (x + static_cast(3.0f)) / static_cast(6.0f)));\n}\n\ntemplate\nstatic __dpct_inline__ T op_exp(T x) {\n return sycl::exp(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_log(T x) {\n if (x <= static_cast(0)) {\n return neg_infinity();\n }\n return sycl::log(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_softplus(T x) {\n const float xf = (float) x;\n const float ax = sycl::fabs(xf);\n const float m = sycl::fmax(xf, 0.0f);\n const float y = m + sycl::log1p(sycl::exp(-ax));\n return (T) y;\n}\n\ntemplate\nstatic __dpct_inline__ T op_neg(T x) {\n return -x;\n}\n\ntemplate\nstatic __dpct_inline__ T op_step(T x) {\n return (x > static_cast(0.0f)) ? static_cast(1.0f) : static_cast(0.0f);\n}\n\ntemplate\nstatic __dpct_inline__ T op_leaky_relu(T x, float negative_slope) {\n T neg_slope_T = static_cast(negative_slope);\n return sycl::fmax(x, static_cast(0)) +\n sycl::fmin(x, static_cast(0.0f)) * neg_slope_T;\n}\n\ntemplate\nstatic __dpct_inline__ T op_sqr(T x) {\n return x * x;\n}\n\ntemplate\nstatic __dpct_inline__ T op_clamp(T x, float min_val, float max_val) {\n return x < static_cast(min_val) ? static_cast(min_val) : (x > static_cast(max_val) ? static_cast(max_val) : x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_floor(T x) {\n return sycl::floor(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_ceil(T x) {\n return sycl::ceil(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_round(T x) {\n return sycl::round(x);\n}\n\ntemplate\nstatic __dpct_inline__ T op_trunc(T x) {\n return sycl::trunc(x);\n}\n\ntemplate\nstatic void unary_op_generic_kernel(\n const T * x,\n T * dst,\n const int k,\n const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3,\n const size_t nb0, const size_t nb1, const size_t nb2, const size_t nb3,\n const size_t nbd0, const size_t nbd1, const size_t nbd2, const size_t nbd3,\n const sycl::nd_item<1> & item_ct1,\n F func) {\n\n (void) ne3;\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n const int64_t i0 = i % ne0;\n const int64_t i1 = (i / ne0) % ne1;\n const int64_t i2 = (i / (ne0*ne1)) % ne2;\n const int64_t i3 = i / (ne0*ne1*ne2);\n\n const char * src_base = (const char *) x;\n char * dst_base = (char *) dst;\n\n const T * srcp = (const T *)(src_base + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3 );\n T * dstp = (T *)(dst_base + i0*nbd0 + i1*nbd1 + i2*nbd2 + i3*nbd3);\n\n *dstp = func(*srcp);\n }\n}\n\ntemplate\nstatic void unary_op_sqrt_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_sqrt(x[i]);\n }\n}\n\ntemplate\nstatic void unary_op_sin_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_sin(x[i]);\n }\n}\n\ntemplate\nstatic void unary_op_cos_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_cos(x[i]);\n }\n}\n\ntemplate\nstatic void unary_op_log_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_log(x[i]);\n }\n}\n\n\ntemplate\nstatic void unary_op_leaky_relu_kernel(const T * x, T * dst, const int k, float negative_slope, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_leaky_relu(x[i], negative_slope);\n }\n}\n\ntemplate\nstatic void unary_op_sqr_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_sqr(x[i]);\n }\n}\n\ntemplate\nstatic void unary_op_clamp_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1, float min_val, float max_val) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_clamp(x[i], min_val, max_val);\n }\n}\n\ntemplate\nstatic void unary_op_floor_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_floor(x[i]);\n }\n}\n\ntemplate\nstatic void unary_op_ceil_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_ceil(x[i]);\n }\n}\n\ntemplate\nstatic void unary_op_round_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_round(x[i]);\n }\n}\n\ntemplate\nstatic void unary_op_trunc_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = op_trunc(x[i]);\n }\n}\n\ntemplate\nstatic void upscale(const T *x, T *dst, const int nb00, const int nb01,\n const int nb02, const int nb03, const int ne10, const int ne11,\n const int ne12, const int ne13, const float sf0, const float sf1,\n const float sf2, const float sf3, const sycl::nd_item<1> &item_ct1) {\n int index = item_ct1.get_local_id(0) +\n item_ct1.get_group(0) * item_ct1.get_local_range(0);\n if (index >= ne10 * ne11 * ne12 * ne13) {\n return;\n }\n // operation\n int i10 = index % ne10;\n int i11 = (index / ne10) % ne11;\n int i12 = (index / (ne10 * ne11)) % ne12;\n int i13 = (index / (ne10 * ne11 * ne12)) % ne13;\n\n int i00 = static_cast(i10 / sf0);\n int i01 = static_cast(i11 / sf1);\n int i02 = static_cast(i12 / sf2);\n int i03 = static_cast(i13 / sf3);\n\n dst[index] = *(const T *)((const char *)x + i03 * nb03 + i02 * nb02 + i01 * nb01 + i00 * nb00);\n}\n\ntemplate\nstatic void clamp(const T * x, T * dst, const float min, const float max, const int k,\n const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = x[i] < static_cast(min) ? static_cast(min) : (x[i] > static_cast(max) ? static_cast(max) : x[i]);\n }\n}\n\ntemplate\nstatic void gated_op_fused_geglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n const int64_t j0 = (i / n) * o0 + (i % n);\n const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);\n dst[i] = op_gelu(x[j0]) * g[j1];\n }\n}\n\ntemplate\nstatic void gated_op_fused_reglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n const int64_t j0 = (i / n) * o0 + (i % n);\n const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);\n dst[i] = op_relu(x[j0]) * g[j1];\n }\n}\n\ntemplate\nstatic void gated_op_fused_swiglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n const int64_t j0 = (i / n) * o0 + (i % n);\n const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);\n dst[i] = op_silu(x[j0]) * g[j1];\n }\n}\n\ntemplate\nstatic void gated_op_fused_geglu_erf(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n const int64_t j0 = (i / n) * o0 + (i % n);\n const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);\n dst[i] = op_gelu_erf(x[j0]) * g[j1];\n }\n}\n\ntemplate\nstatic void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n const int64_t j0 = (i / n) * o0 + (i % n);\n const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);\n dst[i] = op_gelu_quick(x[j0]) * g[j1];\n }\n}\n\nnamespace ggml_sycl_detail {\nstatic void acc_f32_sycl(const float *x, const float *y, float *dst,\n const int64_t n_elements, const int64_t ne10, const int64_t ne11,\n const int64_t ne12, const int64_t ne13, const int64_t s1, const int64_t s2, const int64_t s3,\n const int64_t offset, queue_ptr stream) {\n const int num_blocks = (n_elements + SYCL_ACC_BLOCK_SIZE - 1) / SYCL_ACC_BLOCK_SIZE;\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n acc_f32(x, y, dst, n_elements, ne10, ne11, ne12, ne13, s1, s2, s3, offset);\n });\n}\n\ntemplate\nstatic void arange_kernel(T * dst, const int k, T start, T step,\n const sycl::nd_item<1> &item_ct1) {\n SYCL_GLOBAL_ID_LOOP(k, item_ct1) {\n dst[i] = start + static_cast(i) * step;\n }\n}\n\ntemplate\nstatic void upscale_sycl(const T *x, T *dst, const int nb00, const int nb01,\n const int nb02, const int nb03, const int ne10, const int ne11,\n const int ne12, const int ne13, const float sf0, const float sf1,\n const float sf2, const float sf3, queue_ptr stream) {\n int dst_size = ne10 * ne11 * ne12 * ne13;\n int num_blocks = ceil_div(dst_size, SYCL_UPSCALE_BLOCK_SIZE);\n sycl::range<1> gridDim(num_blocks * SYCL_UPSCALE_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<1>(gridDim, sycl::range<1>(SYCL_UPSCALE_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {\n upscale(x, dst, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3, item_ct1);\n });\n}\n\ntemplate\nstatic inline void dispatch_ggml_sycl_op_unary(ggml_backend_sycl_context & ctx, ggml_tensor * dst, KernelInvoker kernel_invoker, Args&&... args) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);\n GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n GGML_ASSERT(dst->src[0]->type == dst->type);\n\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n switch (dst->type) {\n case GGML_TYPE_F16:\n {\n auto data_pts = cast_data(dst);\n kernel_invoker(data_pts.src, data_pts.dst, (int)ggml_nelements(dst->src[0]), main_stream, std::forward(args)...);\n break;\n }\n case GGML_TYPE_F32:\n {\n auto data_pts = cast_data(dst);\n kernel_invoker(data_pts.src, data_pts.dst, (int)ggml_nelements(dst->src[0]), main_stream, std::forward(args)...);\n break;\n }\n default:\n GGML_ABORT(\"GGML tensor type not supported!\\n\");\n }\n}\n\ntemplate\nstatic inline void dispatch_ggml_sycl_op_fused_glu(ggml_backend_sycl_context & ctx, ggml_tensor * dst, KernelInvoker kernel_invoker, Args&&... args) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);\n GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n GGML_ASSERT(dst->src[0]->type == dst->type);\n\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2;;\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_is_contiguous_1(dst->src[0]));\n GGML_ASSERT(ggml_is_contiguous(dst));\n const int32_t swapped = ((const int32_t *) dst->op_params)[1];\n void * src0_d = src0->data;\n void * src1_d = src1 ? src1->data : src0->data;\n const int64_t src0_o = src0->nb[1];\n const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n void * dst_d = dst->data;\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src1->nb[0] == ggml_element_size(src1));\n GGML_ASSERT(src1->ne[0] == nc);\n GGML_ASSERT(src0->type == src1->type);\n }\n switch (dst->type) {\n case GGML_TYPE_F16:\n {\n sycl::half * src0_p = (sycl::half *) src0_d;\n sycl::half * src1_p = (sycl::half *) src1_d;\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n kernel_invoker(src0_p,\n src1_p,\n (sycl::half *) dst_d,\n ggml_nelements(dst),\n nc,\n src0_o / sizeof(sycl::half),\n src1_o / sizeof(sycl::half),\n main_stream,\n std::forward(args)...);\n break;\n }\n case GGML_TYPE_F32:\n {\n float * src0_p = (float *) src0_d;\n float * src1_p = (float *) src1_d;\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n kernel_invoker(src0_p,\n src1_p,\n (float *) dst_d,\n ggml_nelements(dst),\n nc,\n src0_o / sizeof(float),\n src1_o / sizeof(float),\n main_stream,\n std::forward(args)...);\n break;\n }\n default:\n GGML_ABORT(\"GGML tensor type not supported!\\n\");\n }\n}\n\ntemplate\nstatic inline void dispatch_ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst, KernelInvoker kernel_invoker, Args&&... args) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);\n GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n\n GGML_ASSERT(dst->src[0]->type == dst->type);\n\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n\n const float sf0 = (float) dst->ne[0] / dst->src[0]->ne[0];\n const float sf1 = (float) dst->ne[1] / dst->src[0]->ne[1];\n const float sf2 = (float) dst->ne[2] / dst->src[0]->ne[2];\n const float sf3 = (float) dst->ne[3] / dst->src[0]->ne[3];\n switch (dst->type) {\n case GGML_TYPE_F16:\n {\n auto data_pts = cast_data(dst);\n kernel_invoker(data_pts.src, data_pts.dst, (int)dst->src[0]->nb[0], (int)dst->src[0]->nb[1], (int)dst->src[0]->nb[2],\n (int)dst->src[0]->nb[3], (int)dst->ne[0], (int)dst->ne[1], (int)dst->ne[2], (int)dst->ne[3], sf0, sf1, sf2, sf3,\n main_stream, std::forward(args)...);\n break;\n }\n case GGML_TYPE_F32:\n {\n auto data_pts = cast_data(dst);\n kernel_invoker(data_pts.src, data_pts.dst, (int)dst->src[0]->nb[0], (int)dst->src[0]->nb[1], (int)dst->src[0]->nb[2],\n (int)dst->src[0]->nb[3], (int)dst->ne[0], (int)dst->ne[1], (int)dst->ne[2], (int)dst->ne[3], sf0, sf1, sf2, sf3,\n main_stream, std::forward(args)...);\n break;\n }\n default:\n GGML_ABORT(\"GGML tensor type not supported!\\n\");\n }\n}\n\ntemplate\nstatic inline void ggml_sycl_op_unary(\n ggml_backend_sycl_context & ctx, ggml_tensor * dst, F func) {\n\n ggml_tensor * src0 = dst->src[0];\n\n const int64_t ne0 = dst->ne[0];\n const int64_t ne1 = dst->ne[1];\n const int64_t ne2 = dst->ne[2];\n const int64_t ne3 = dst->ne[3];\n\n const size_t nb0 = src0->nb[0];\n const size_t nb1 = src0->nb[1];\n const size_t nb2 = src0->nb[2];\n const size_t nb3 = src0->nb[3];\n\n const size_t nbd0 = dst->nb[0];\n const size_t nbd1 = dst->nb[1];\n const size_t nbd2 = dst->nb[2];\n const size_t nbd3 = dst->nb[3];\n\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [=](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {\n\n const int num_blocks = ceil_div(k_elements, 256);\n\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),\n sycl::range<1>(256)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_generic_kernel(\n src, dst_ptr, k_elements,\n ne0, ne1, ne2, ne3,\n nb0, nb1, nb2, nb3,\n nbd0, nbd1, nbd2, nbd3,\n item_ct1,\n func\n );\n });\n });\n}\n\n\nstatic inline void ggml_sycl_op_arange(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n float start, stop, step;\n memcpy(&start, dst->op_params, sizeof(float));\n memcpy(&stop, (float *) dst->op_params + 1, sizeof(float));\n memcpy(&step, (float *) dst->op_params + 2, sizeof(float));\n dpct::queue_ptr stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n float * dst_ptr = (float *)dst->data;\n const int k = (int)ggml_nelements(dst);\n const int num_blocks = ceil_div(k, SYCL_ARANGE_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_ARANGE_BLOCK_SIZE),\n sycl::range<1>(SYCL_ARANGE_BLOCK_SIZE)),\n [=](sycl::nd_item<1> item_ct1) {\n arange_kernel(dst_ptr, k, start, step, item_ct1);\n });\n}\n\n} // namespace ggml_sycl_detail\n\n\n\nstatic inline void ggml_sycl_op_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_sgn(x);\n });\n}\n\n\nstatic inline void ggml_sycl_op_abs(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_abs(x);\n });\n}\n\nstatic inline void ggml_sycl_op_elu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_elu(x);\n });\n}\nstatic inline void ggml_sycl_op_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_silu(x);\n });\n}\n\nstatic inline void ggml_sycl_op_gelu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_gelu(x);\n });\n}\n\nstatic inline void ggml_sycl_op_gelu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_gelu_quick(x);\n });\n}\n\nstatic inline void ggml_sycl_op_gelu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_gelu_erf(x);\n });\n}\n\nstatic inline void ggml_sycl_op_tanh(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_tanh(x);\n });\n}\n\nstatic inline void ggml_sycl_op_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_relu(x);\n });\n}\n\nstatic inline void ggml_sycl_op_hardsigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_hardsigmoid(x);\n });\n}\n\nstatic inline void ggml_sycl_op_hardswish(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_hardswish(x);\n });\n}\n\nstatic inline void ggml_sycl_op_exp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_exp(x);\n });\n}\n\nstatic inline void ggml_sycl_op_log(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {\n const int num_blocks = ceil_div(k_elements, SYCL_EXP_BLOCK_SIZE); // Using EXP block size\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_EXP_BLOCK_SIZE),\n sycl::range<1>(SYCL_EXP_BLOCK_SIZE)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_log_kernel(src, dst_ptr, k_elements, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_softplus(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_softplus(x);\n });\n}\n\nstatic inline void ggml_sycl_op_neg(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_neg(x);\n });\n}\n\n\nstatic inline void ggml_sycl_op_step(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_step(x);\n });\n}\n\nstatic inline void ggml_sycl_op_sigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_sigmoid(x);\n });\n}\n\nstatic inline void ggml_sycl_op_sqrt(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {\n const int num_blocks = ceil_div(k_elements, SYCL_SQRT_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SQRT_BLOCK_SIZE),\n sycl::range<1>(SYCL_SQRT_BLOCK_SIZE)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_sqrt_kernel(src, dst_ptr, k_elements, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_sin(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {\n const int num_blocks = ceil_div(k_elements, SYCL_SIN_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SIN_BLOCK_SIZE),\n sycl::range<1>(SYCL_SIN_BLOCK_SIZE)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_sin_kernel(src, dst_ptr, k_elements, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_cos(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {\n const int num_blocks = ceil_div(k_elements, SYCL_SIN_BLOCK_SIZE); // Using SIN block size\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SIN_BLOCK_SIZE),\n sycl::range<1>(SYCL_SIN_BLOCK_SIZE)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_cos_kernel(src, dst_ptr, k_elements, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_leaky_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n float negative_slope;\n memcpy(&negative_slope, dst->op_params, sizeof(float));\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream, float slope) {\n const int num_blocks = ceil_div(k_elements, SYCL_RELU_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_RELU_BLOCK_SIZE),\n sycl::range<1>(SYCL_RELU_BLOCK_SIZE)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_leaky_relu_kernel(src, dst_ptr, k_elements, slope, item_ct1);\n });\n }, negative_slope);\n}\n\nstatic inline void ggml_sycl_op_sqr(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {\n const int num_blocks = ceil_div(k_elements, SYCL_SQR_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SQR_BLOCK_SIZE),\n sycl::range<1>(SYCL_SQR_BLOCK_SIZE)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_sqr_kernel(src, dst_ptr, k_elements, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_upscale(ctx, dst,\n [](const auto* src, auto* dst_ptr, int nb00, int nb01, int nb02, int nb03,\n int ne10, int ne11, int ne12, int ne13, float sf0, float sf1, float sf2, float sf3,\n queue_ptr stream) {\n ggml_sycl_detail::upscale_sycl(src, dst_ptr, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3, stream);\n });\n}\n\nstatic inline void ggml_sycl_op_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n float min_val;\n float max_val;\n memcpy(&min_val, dst->op_params, sizeof(float));\n memcpy(&max_val, (float *) dst->op_params + 1, sizeof(float));\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream, float min_arg, float max_arg) {\n const int num_blocks = ceil_div(k_elements, SYCL_CLAMP_BLOCK_SIZE);\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE),\n sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE)),\n [=](sycl::nd_item<1> item_ct1) {\n clamp(src, dst_ptr, min_arg, max_arg, k_elements, item_ct1);\n });\n }, min_val, max_val);\n}\n\nstatic inline void ggml_sycl_op_floor(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {\n const int num_blocks = ceil_div(k_elements, 256);\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),\n sycl::range<1>(256)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_floor_kernel(src, dst_ptr, k_elements, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_ceil(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) {\n return op_ceil(x);\n });\n}\n\nstatic inline void ggml_sycl_op_round(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {\n const int num_blocks = ceil_div(k_elements, 256);\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),\n sycl::range<1>(256)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_round_kernel(src, dst_ptr, k_elements, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_trunc(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,\n [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {\n const int num_blocks = ceil_div(k_elements, 256);\n stream->parallel_for(\n sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),\n sycl::range<1>(256)),\n [=](sycl::nd_item<1> item_ct1) {\n unary_op_trunc_kernel(src, dst_ptr, k_elements, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n\n const float * src0_d = (const float *) src0->data;\n const float * src1_d = (const float *) src1->data;\n float * dst_d = (float *) dst->data;\n\n dpct::queue_ptr stream = ctx.stream();\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(src1->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(ggml_is_contiguous(src1));\n GGML_ASSERT(dst->nb[0] == ggml_element_size(dst));\n GGML_ASSERT(ggml_is_contiguously_allocated(dst));\n\n const int64_t s1 = dst->op_params[0] / sizeof(float);\n const int64_t s2 = dst->op_params[1] / sizeof(float);\n const int64_t s3 = dst->op_params[2] / sizeof(float);\n const int64_t offset = dst->op_params[3] / sizeof(float);\n\n ggml_sycl_detail::acc_f32_sycl(src0_d, src1_d, dst_d, ggml_nelements(dst),\n src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],\n s1, s2, s3, offset, stream);\n}\n\nstatic inline void ggml_sycl_op_geglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,\n [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {\n const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);\n main_stream->parallel_for(\n sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {\n gated_op_fused_geglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_reglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,\n [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {\n const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_RELU_BLOCK_SIZE); // Using RELU block size for reglu\n main_stream->parallel_for(\n sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {\n gated_op_fused_reglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_swiglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,\n [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {\n const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_SILU_BLOCK_SIZE); // Using SILU block size for swiglu\n main_stream->parallel_for(\n sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {\n gated_op_fused_swiglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);\n });\n });\n}\n\n__dpct_inline__ float ggml_sycl_op_swiglu_oai_single(float x, float g, float alpha = 1.702f, float limit = 7.0f) {\n x = sycl::fmin(x, limit);\n g = sycl::fmax(sycl::fmin(g, limit), -limit);\n\n float out_glu = x / (1.0f + sycl::native::exp(-x * alpha));\n out_glu = out_glu * (1.0f + g);\n return out_glu;\n}\n\n\ntemplate \nstatic void swiglu_oai_kernel(const T * x, const T * g, T * dst, const int64_t k,\n const int64_t n, const int64_t o0, const int64_t o1,\n float alpha, float limit, sycl::nd_item<3> item_ct1) {\n const int64_t i = int64_t(item_ct1.get_local_range(2)) * item_ct1.get_group(2) + item_ct1.get_local_id(2);\n\n if (i >= k) {\n return;\n }\n\n const int64_t j0 = (i / n) * o0 + (i % n);\n const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);\n\n float xi = x[j0];\n float gi = g[j1];\n\n dst[i] = ggml_sycl_op_swiglu_oai_single(xi, gi, alpha, limit);\n}\n\ntemplate \nstatic void swiglu_oai_sycl(const T * x,\n const T * g,\n T * dst,\n const int64_t k,\n const int64_t n,\n const int64_t o0,\n const int64_t o1,\n const float alpha,\n const float limit,\n dpct::queue_ptr stream) {\n const int64_t num_blocks = (k + SYCL_GLU_BLOCK_SIZE - 1) / SYCL_GLU_BLOCK_SIZE;\n stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n swiglu_oai_kernel(x, g, dst, k, n, o0, o1, alpha, limit, item_ct1);\n });\n}\n\nvoid ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n const ggml_tensor * src1 = dst->src[1];\n void * src0_d = src0->data;\n void * src1_d = src1 ? src1->data : src0->data;\n const int64_t src0_o = src0->nb[1];\n const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n void * dst_d = dst->data;\n const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n dpct::queue_ptr stream = ctx.stream();\n\n GGML_ASSERT(ggml_is_contiguous_1(src0));\n GGML_ASSERT(src0->nb[0] == ggml_element_size(src0));\n GGML_ASSERT(ggml_is_contiguous(dst));\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n GGML_ASSERT(src0->type == dst->type);\n GGML_ASSERT(dst->ne[0] == nc);\n GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0));\n\n if (src1) {\n GGML_ASSERT(ggml_is_contiguous_1(src1));\n GGML_ASSERT(src1->nb[0] == ggml_element_size(src1));\n GGML_ASSERT(src1->ne[0] == nc);\n GGML_ASSERT(src0->type == src1->type);\n }\n\n //const int32_t swapped = ((const int32_t *) dst->op_params)[1];\n const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n const float alpha = ggml_get_op_params_f32(dst, 2);\n const float limit = ggml_get_op_params_f32(dst, 3);\n\n float * src0_p = (float *) src0_d;\n float * src1_p = (float *) src1_d;\n\n if (!src1) {\n src0_p += swapped ? nc : 0;\n src1_p += swapped ? 0 : nc;\n }\n\n swiglu_oai_sycl(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream);\n}\n\nstatic inline void ggml_sycl_op_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,\n [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {\n const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);\n main_stream->parallel_for(\n sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {\n gated_op_fused_geglu_erf(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);\n });\n });\n}\n\nstatic inline void ggml_sycl_op_geglu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,\n [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {\n const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);\n main_stream->parallel_for(\n sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {\n gated_op_fused_geglu_quick(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);\n });\n });\n}\n\n\nvoid ggml_sycl_sqrt(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_sqrt(ctx, dst);\n}\n\nvoid ggml_sycl_sin(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_sin(ctx, dst);\n}\n\nvoid ggml_sycl_cos(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_cos(ctx, dst);\n}\n\nvoid ggml_sycl_acc(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);\n ggml_sycl_op_acc(ctx, dst);\n}\n\nvoid ggml_sycl_gelu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_gelu(ctx, dst);\n}\n\nvoid ggml_sycl_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_silu(ctx, dst);\n}\n\nvoid ggml_sycl_gelu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_gelu_quick(ctx, dst);\n}\n\nvoid ggml_sycl_gelu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_gelu_erf(ctx, dst);\n}\n\nvoid ggml_sycl_tanh(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_tanh(ctx, dst);\n}\n\nvoid ggml_sycl_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_relu(ctx, dst);\n}\n\nvoid ggml_sycl_sigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_sigmoid(ctx, dst);\n}\n\nvoid ggml_sycl_hardsigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_hardsigmoid(ctx, dst);\n}\n\nvoid ggml_sycl_hardswish(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_hardswish(ctx, dst);\n}\n\nvoid ggml_sycl_exp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_exp(ctx, dst);\n}\n\nvoid ggml_sycl_log(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_log(ctx, dst);\n}\n\nvoid ggml_sycl_softplus(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_softplus(ctx, dst);\n}\n\nvoid ggml_sycl_neg(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_neg(ctx, dst);\n}\n\nvoid ggml_sycl_step(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_step(ctx, dst);\n}\n\nvoid ggml_sycl_leaky_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_leaky_relu(ctx, dst);\n}\n\nvoid ggml_sycl_sqr(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_sqr(ctx, dst);\n}\n\nvoid ggml_sycl_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_upscale(ctx, dst);\n}\n\n\nvoid ggml_sycl_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_clamp(ctx, dst);\n}\n\nvoid ggml_sycl_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_sgn(ctx, dst);\n}\n\nvoid ggml_sycl_abs(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_abs(ctx, dst);\n}\n\nvoid ggml_sycl_elu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_elu(ctx, dst);\n}\n\nvoid ggml_sycl_geglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_geglu(ctx, dst);\n}\n\nvoid ggml_sycl_reglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_reglu(ctx, dst);\n}\n\nvoid ggml_sycl_swiglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_swiglu(ctx, dst);\n}\n\nvoid ggml_sycl_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_swiglu_oai(ctx, dst);\n}\n\nvoid ggml_sycl_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_geglu_erf(ctx, dst);\n}\n\nvoid ggml_sycl_geglu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_geglu_quick(ctx, dst);\n}\n\nvoid ggml_sycl_arange(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/0);\n ggml_sycl_detail::ggml_sycl_op_arange(ctx, dst);\n}\n\nvoid ggml_sycl_floor(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_floor(ctx, dst);\n}\n\nvoid ggml_sycl_ceil(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_ceil(ctx, dst);\n}\n\nvoid ggml_sycl_round(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_round(ctx, dst);\n}\n\nvoid ggml_sycl_trunc(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);\n ggml_sycl_op_trunc(ctx, dst);\n}\n" }, { "path": "src/ggml-sycl/element_wise.hpp", "content": "#ifndef GGML_SYCL_ELEMENTWISE_HPP\n#define GGML_SYCL_ELEMENTWISE_HPP\n\n#include \"common.hpp\"\n#include \"ggml.h\"\n#include // For std::numeric_limits\n\n#define SYCL_GLU_BLOCK_SIZE 256\n\ntemplate \nT neg_infinity() {\n return -std::numeric_limits::infinity();\n}\n\ntemplate\nstruct typed_data {\n const T_Src * src;\n T_Dst * dst;\n};\n\ntemplate\ntyped_data cast_data(ggml_tensor * dst) {\n return {\n /* .src = */ static_cast(dst->src[0]->data),\n /* .dst = */ static_cast(dst->data)\n };\n}\n\nconst float GELU_QUICK_COEF = -1.702f;\n\n\nvoid ggml_sycl_sqrt(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_sin(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_cos(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_acc(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_gelu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_gelu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_gelu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_tanh(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_sigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_hardsigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_hardswish(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_exp(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_log(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_softplus(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_neg(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_step(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_leaky_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_sqr(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_abs(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_elu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_geglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\nvoid ggml_sycl_reglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\nvoid ggml_sycl_swiglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\nvoid ggml_sycl_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\nvoid ggml_sycl_geglu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\nvoid ggml_sycl_floor(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\nvoid ggml_sycl_ceil(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\nvoid ggml_sycl_round(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\nvoid ggml_sycl_trunc(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nvoid ggml_sycl_arange(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\n#endif // GGML_SYCL_ELEMENTWISE_HPP\n" }, { "path": "src/ggml-sycl/fattn-common.hpp", "content": "#pragma once\n\n#include \n#include \"dpct/helper.hpp\"\n#include \"common.hpp\"\n#include \"convert.hpp\"\n#include \"vecdotq.hpp\"\n\n#include \"ggml.h\"\n\n#include \n#include \n#include \n\n\n#define FATTN_KQ_STRIDE 256\n#define HALF_MAX_HALF sycl::half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction.\n#define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs.\n#define FATTN_KQ_MAX_OFFSET (3.0f*0.6931f)\n\ntypedef void (*fattn_kernel_t)(\n const char* Q,\n const char* K,\n const char* V,\n const char* mask,\n const char* sinks,\n const int* KV_max,\n float* dst,\n sycl::float2* dst_meta,\n const float scale,\n const float max_bias,\n const float m0,\n const float m1,\n const uint32_t n_head_log2,\n const float logit_softcap,\n const int32_t ne00,\n const sycl::uint3 ne01,\n const int32_t ne02,\n const int32_t ne03,\n const int32_t nb01,\n const int32_t nb02,\n const int32_t nb03,\n const int32_t ne10,\n const int32_t ne11,\n const int32_t ne12,\n const int32_t ne13,\n const int32_t nb11,\n const int32_t nb12,\n const int64_t nb13,\n const int32_t nb21,\n const int32_t nb22,\n const int64_t nb23,\n const int32_t ne31,\n const int32_t ne32,\n const int32_t ne33,\n const int32_t nb31,\n const int32_t nb32,\n const int64_t nb33);\n\ntypedef float (*vec_dot_KQ_t)(\n const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds);\n\ntemplate \nstatic __dpct_inline__ float vec_dot_fattn_vec_KQ_f16(const char * __restrict__ K_c,\n const void * __restrict__ Q_v,\n const int * __restrict__ Q_q8,\n const void * __restrict__ Q_ds_v) {\n const sycl::half2 * K_h2 = (const sycl::half2 *) K_c;\n GGML_UNUSED(Q_q8);\n GGML_UNUSED(Q_ds_v);\n\n constexpr int cpy_nb = ggml_sycl_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += nthreads*cpy_ne) {\n sycl::half2 tmp[cpy_ne];\n ggml_sycl_memcpy_1(\n tmp,\n K_h2 + k_KQ_0 + (sycl::ext::oneapi::this_work_item::get_nd_item<3>().get_local_id(2) % nthreads) * cpy_ne);\n#pragma unroll\n for (int k_KQ_1 = 0; k_KQ_1 < cpy_ne; ++k_KQ_1) {\n#ifdef GGML_SYCL_F16\n ggml_sycl_mad(sum, tmp[k_KQ_1] , ((const sycl::half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]);\n#else\n ggml_sycl_mad(sum, __half22float2(tmp[k_KQ_1]), ((const sycl::float2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]);\n#endif // GGML_SYCL_F16\n }\n }\n\n return sum;\n}\n\ntemplate \nstatic __dpct_inline__ float vec_dot_fattn_vec_KQ_q4_0(const char * __restrict__ K_c,\n const void * __restrict__ Q_v,\n const int * __restrict__ Q_q8,\n const void * __restrict__ Q_ds_v) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n\n const block_q4_0 * K_q4_0 = (const block_q4_0 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ =\n k_KQ_0 + (nthreads == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads);\n\n const int ib = k_KQ / QI8_1;\n const int iqs4 = k_KQ % QI4_0;\n const int shift = k_KQ & (QI8_1/2);\n\n int v;\n ggml_sycl_memcpy_1(&v, K_q4_0[ib].qs + sizeof(int)*iqs4);\n v = (v >> shift) & 0x0F0F0F0F;\n const int u = Q_q8[k_KQ_0/nthreads];\n\n const int sumi = ggml_sycl_dp4a(v, u, 0);\n\n const sycl::float2 Q_ds = ((const sycl::float2 *) Q_ds_v)[k_KQ_0 / nthreads];\n sum += __half2float(K_q4_0[ib].d) * (sumi*Q_ds.x() - (8/QI8_1)*Q_ds.y());\n }\n\n return sum;\n}\n\ntemplate \nstatic __dpct_inline__ float vec_dot_fattn_vec_KQ_q4_1(const char * __restrict__ K_c,\n const void * __restrict__ Q_v,\n const int * __restrict__ Q_q8,\n const void * __restrict__ Q_ds_v) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const block_q4_1 * K_q4_1 = (const block_q4_1 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ =\n k_KQ_0 + (nthreads == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads);\n\n const int ib = k_KQ / QI8_1;\n const int iqs4 = k_KQ % QI4_1;\n const int shift = k_KQ & (QI8_1/2);\n\n int v;\n ggml_sycl_memcpy_1(&v, K_q4_1[ib].qs + sizeof(int)*iqs4);\n v = (v >> shift) & 0x0F0F0F0F;\n const int u = Q_q8[k_KQ_0/nthreads];\n\n const int sumi = ggml_sycl_dp4a(v, u, 0);\n\n const sycl::float2 K_dm = (K_q4_1[ib].dm).template convert();\n const sycl::float2 Q_ds = ((const sycl::float2 *) Q_ds_v)[k_KQ_0 / nthreads];\n\n sum += K_dm.x()*Q_ds.x()*sumi + K_dm.y()*Q_ds.y()/QI8_1;\n }\n\n return sum;\n}\n\ntemplate \nstatic __dpct_inline__ float vec_dot_fattn_vec_KQ_q5_0(const char * __restrict__ K_c,\n const void * __restrict__ Q_v,\n const int * __restrict__ Q_q8,\n const void * __restrict__ Q_ds_v) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const block_q5_0 * K_q5_0 = (const block_q5_0 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ =\n k_KQ_0 + (nthreads == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads);\n\n const int ib = k_KQ / QI8_1;\n const int iqs4 = k_KQ % QI5_0;\n const int iqs8 = k_KQ % QI8_1;\n const int shift = k_KQ & (QI8_1/2);\n\n int v;\n ggml_sycl_memcpy_1(&v, K_q5_0[ib].qs + sizeof(int)*iqs4);\n v = (v >> shift) & 0x0F0F0F0F;\n\n {\n int vh;\n ggml_sycl_memcpy_1(&vh, K_q5_0[ib].qh);\n vh >>= iqs8 * QI5_0;\n\n v |= (vh << 4) & 0x00000010; // 0 -> 4\n v |= (vh << 11) & 0x00001000; // 1 -> 12\n v |= (vh << 18) & 0x00100000; // 2 -> 20\n v |= (vh << 25) & 0x10000000; // 3 -> 28\n }\n\n const int u = Q_q8[k_KQ_0/nthreads];\n\n const int sumi = ggml_sycl_dp4a(v, u, 0);\n\n const sycl::float2 Q_ds = ((const sycl::float2 *) Q_ds_v)[k_KQ_0 / nthreads];\n\n sum += __half2float(K_q5_0[ib].d) * (sumi*Q_ds.x() - (16/QI8_1)*Q_ds.y());\n }\n\n return sum;\n}\n\ntemplate \nstatic __dpct_inline__ float vec_dot_fattn_vec_KQ_q5_1(const char * __restrict__ K_c,\n const void * __restrict__ Q_v,\n const int * __restrict__ Q_q8,\n const void * __restrict__ Q_ds_v) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const block_q5_1 * K_q5_1 = (const block_q5_1 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ =\n k_KQ_0 + (nthreads == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads);\n\n const int ib = k_KQ / QI8_1;\n const int iqs4 = k_KQ % QI5_1;\n const int iqs8 = k_KQ % QI8_1;\n const int shift = k_KQ & (QI8_1/2);\n\n int v;\n ggml_sycl_memcpy_1(&v, K_q5_1[ib].qs + sizeof(int)*iqs4);\n v = (v >> shift) & 0x0F0F0F0F;\n\n {\n int vh;\n ggml_sycl_memcpy_1(&vh, K_q5_1[ib].qh);\n vh >>= iqs8 * QI5_0;\n\n v |= (vh << 4) & 0x00000010; // 0 -> 4\n v |= (vh << 11) & 0x00001000; // 1 -> 12\n v |= (vh << 18) & 0x00100000; // 2 -> 20\n v |= (vh << 25) & 0x10000000; // 3 -> 28\n }\n\n const int u = Q_q8[k_KQ_0/nthreads];\n\n const int sumi = ggml_sycl_dp4a(v, u, 0);\n\n const sycl::float2 K_dm = (K_q5_1[ib].dm).template convert();\n const sycl::float2 Q_ds = ((const sycl::float2 *) Q_ds_v)[k_KQ_0 / nthreads];\n\n sum += K_dm.x()*Q_ds.x()*sumi + K_dm.y()*Q_ds.y()/QI8_1;\n }\n\n return sum;\n}\n\ntemplate \nstatic __dpct_inline__ float vec_dot_fattn_vec_KQ_q8_0(const char * __restrict__ K_c,\n const void * __restrict__ Q_v,\n const int * __restrict__ Q_q8,\n const void * __restrict__ Q_ds_v) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const block_q8_0 * K_q8_0 = (const block_q8_0 *) K_c;\n GGML_UNUSED(Q_v);\n\n float sum = 0.0f;\n\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) {\n const int k_KQ =\n k_KQ_0 + (nthreads == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads);\n\n const int ib = k_KQ / QI8_0;\n const int iqs = k_KQ % QI8_0;\n\n int v;\n ggml_sycl_memcpy_1(&v, K_q8_0[ib].qs + 4*iqs);\n\n const sycl::float2 * Q_ds = (const sycl::float2 *) Q_ds_v;\n const float Q_d = Q_ds[k_KQ_0 / nthreads].x();\n\n sum += vec_dot_q8_0_q8_1_impl(&v, &Q_q8[k_KQ_0/nthreads], K_q8_0[ib].d, Q_d);\n }\n\n return sum;\n}\n\ntemplate \nstatic __dpct_inline__ void quantize_q8_1_to_shared(const float * __restrict__ x,\n const float scale,\n int * __restrict__ yq32,\n void * __restrict__ yds) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n\n float vals[sizeof(int)] = { 0.0f };\n#pragma unroll\n for (int l = 0; l < int(sizeof(int)); ++l) {\n vals[l] =\n (ni == warp_size || item_ct1.get_local_id(2) < ni) ? scale * x[4 * item_ct1.get_local_id(2) + l] : 0.0f;\n }\n\n float amax = sycl::fabs(vals[0]);\n float sum = vals[0];\n#pragma unroll\n for (int l = 1; l < int(sizeof(int)); ++l) {\n amax = sycl::fmax(amax, sycl::fabs(vals[l]));\n sum += vals[l];\n }\n#pragma unroll\n for (int mask = QI8_1/2; mask > 0; mask >>= 1) {\n amax = sycl::fmax(\n amax, dpct::permute_sub_group_by_xor(sycl::ext::oneapi::this_work_item::get_sub_group(), amax, mask));\n sum += dpct::permute_sub_group_by_xor(sycl::ext::oneapi::this_work_item::get_sub_group(), sum, mask);\n }\n\n const float d = amax / 127;\n int q32 = 0;\n int8_t * q8 = (int8_t *) &q32;\n\n if (d != 0.0f) {\n#pragma unroll\n for (int l = 0; l < int(sizeof(int)); ++l) {\n q8[l] = sycl::round(vals[l] / d);\n }\n }\n\n yq32[item_ct1.get_local_id(2)] = q32;\n if (item_ct1.get_local_id(2) % QI8_1 == 0 && (ni == warp_size || item_ct1.get_local_id(2) < ni)) {\n if (std::is_same::value) {\n ((sycl::half2 *) yds)[item_ct1.get_local_id(2)/QI8_1] = make_half2(d, sum);\n } else {\n ((sycl::float2 *) yds)[item_ct1.get_local_id(2)/QI8_1] = make_float2(d, sum);\n }\n }\n}\n\ntypedef void (*dequantize_V_t)(const void *, void *, const int64_t);\n\ntemplate \nstatic __dpct_inline__ void dequantize_V_f16(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n if constexpr (std::is_same_v) {\n ggml_sycl_memcpy_1(dst, (const sycl::half *) vx + i0);\n } else if constexpr (std::is_same_v) {\n static_assert(ne % 2 == 0, \"bad ne\");\n sycl::half2 tmp[ne / 2];\n ggml_sycl_memcpy_1(tmp, (const sycl::half *) vx + i0);\n sycl::float2 * dst_f2 = (sycl::float2 *) dst;\n#pragma unroll\n for (int l = 0; l < ne/2; ++l) {\n dst_f2[l] = tmp[l].template convert();\n }\n } else {\n static_assert(std::is_same_v, \"unsupported type\");\n }\n}\n\ntemplate \nstatic __dpct_inline__ void dequantize_V_q4_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q4_0 * x = (const block_q4_0 *) vx;\n\n const int64_t ib = i0 / QK4_0;\n const int iqs = i0 % (QK4_0/2);\n const int shift = (i0 % QK4_0) / (QK4_0/2);\n\n int q;\n static_assert(ne == 2 || ne == 4, \"bad ne\");\n ggml_sycl_memcpy_1(&q, x[ib].qs + iqs);\n q >>= 4*shift;\n q &= 0x0F0F0F0F;\n q = dpct::vectorized_binary(q, 0x08080808, dpct::sub_sat());\n\n const int8_t * q8 = (const int8_t *) &q;\n\n#ifdef GGML_SYCL_F16\n if constexpr (std::is_same_v) {\n const sycl::half2 d = sycl::half2(x[ib].d);\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((sycl::half2 *) dst)[l0 / 2] = d * sycl::half2(q8[l0 + 0], q8[l0 + 1]);\n }\n } else\n#endif // GGML_SYCL_F16\n if constexpr (std::is_same_v) {\n const float d = x[ib].d;\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = d * q8[l];\n }\n } else {\n static_assert(std::is_same_v, \"bad type\");\n }\n}\n\ntemplate \nstatic __dpct_inline__ void dequantize_V_q4_1(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q4_1 * x = (const block_q4_1 *) vx;\n\n const int64_t ib = i0 / QK4_1;\n const int iqs = i0 % (QK4_1/2);\n const int shift = (i0 % QK4_1) / (QK4_1/2);\n\n int q;\n static_assert(ne == 2 || ne == 4, \"bad ne\");\n ggml_sycl_memcpy_1(&q, x[ib].qs + iqs);\n q >>= 4*shift;\n q &= 0x0F0F0F0F;\n\n const int8_t * q8 = (const int8_t *) &q;\n\n#ifdef GGML_SYCL_F16\n if constexpr (std::is_same_v) {\n const sycl::half2 dm = x[ib].dm;\n const sycl::half2 d = sycl::half2(dm[0]);\n const sycl::half2 m = sycl::half2(dm[1]);\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((sycl::half2 *) dst)[l0 / 2] = d * sycl::half2(q8[l0 + 0], q8[l0 + 1]) + m;\n }\n } else\n#endif // GGML_SYCL_F16\n if constexpr (std::is_same_v) {\n const sycl::float2 dm = (x[ib].dm).template convert();\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = dm.x() * q8[l] + dm.y();\n }\n } else {\n static_assert(std::is_same_v, \"bad type\");\n }\n}\n\ntemplate \nstatic __dpct_inline__ void dequantize_V_q5_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q5_0 * x = (const block_q5_0 *) vx;\n\n const int64_t ib = i0 / QK5_0;\n const int idq = i0 % QK5_0;\n const int iqs = i0 % (QK5_0/2);\n const int shift = (i0 % QK5_0) / (QK5_0/2);\n\n int q;\n static_assert(ne == 2 || ne == 4, \"bad ne\");\n ggml_sycl_memcpy_1(&q, x[ib].qs + iqs);\n q >>= 4*shift;\n q &= 0x0F0F0F0F;\n\n {\n int qh;\n ggml_sycl_memcpy_1(&qh, x[ib].qh);\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n q |= ((qh >> (idq + l)) & 0x00000001) << (8*l + 4);\n }\n }\n\n q = dpct::vectorized_binary(q, 0x10101010, dpct::sub_sat());\n\n const int8_t * q8 = (const int8_t *) &q;\n\n#ifdef GGML_SYCL_F16\n if constexpr (std::is_same_v) {\n const sycl::half2 d = sycl::half2(x[ib].d);\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((sycl::half2 *) dst)[l0 / 2] = d * sycl::half2(q8[l0 + 0], q8[l0 + 1]);\n }\n } else\n#endif // GGML_SYCL_F16\n if constexpr (std::is_same_v) {\n const float d = x[ib].d;\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = d * q8[l];\n }\n } else {\n static_assert(std::is_same_v, \"bad type\");\n }\n}\n\ntemplate \nstatic __dpct_inline__ void dequantize_V_q5_1(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q5_1 * x = (const block_q5_1 *) vx;\n\n const int64_t ib = i0 / QK5_1;\n const int idq = i0 % QK5_1;\n const int iqs = i0 % (QK5_1/2);\n const int shift = (i0 % QK5_1) / (QK5_1/2);\n\n int q;\n static_assert(ne == 2 || ne == 4, \"bad ne\");\n ggml_sycl_memcpy_1(&q, x[ib].qs + iqs);\n q >>= 4*shift;\n q &= 0x0F0F0F0F;\n\n {\n int qh;\n ggml_sycl_memcpy_1(&qh, x[ib].qh);\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n q |= ((qh >> (idq + l)) & 0x00000001) << (8*l + 4);\n }\n }\n\n const int8_t * q8 = (const int8_t *) &q;\n\n#ifdef GGML_SYCL_F16\n if constexpr (std::is_same_v) {\n const sycl::half2 dm = x[ib].dm;\n const sycl::half2 d = sycl::half2(dm[0]);\n const sycl::half2 m = sycl::half2(dm[1]);\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((sycl::half2 *) dst)[l0 / 2] = d * sycl::half2(q8[l0 + 0], q8[l0 + 1]) + m;\n }\n } else\n#endif // GGML_SYCL_F16\n if constexpr (std::is_same_v) {\n const sycl::float2 dm = (x[ib].dm).template convert();\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = dm.x() * q8[l] + dm.y();\n }\n } else {\n static_assert(std::is_same_v, \"bad type\");\n }\n}\n\ntemplate \nstatic __dpct_inline__ void dequantize_V_q8_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {\n const block_q8_0 * x = (const block_q8_0 *) vx;\n\n const int64_t ib = i0 / QK8_0;\n const int iqs = i0 % QK8_0;\n\n static_assert(ne % 2 == 0, \"bad ne\");\n int8_t qs[ne];\n ggml_sycl_memcpy_1(qs, x[ib].qs + iqs);\n\n#ifdef GGML_SYCL_F16\n if constexpr (std::is_same::value) {\n const sycl::half2 d = sycl::half2(x[ib].d);\n\n#pragma unroll\n for (int l0 = 0; l0 < ne; l0 += 2) {\n ((sycl::half2 *) dst)[l0 / 2] = d * make_half2(qs[l0 + 0], qs[l0 + 1]);\n }\n } else\n#endif // GGML_SYCL_F16\n if constexpr (std::is_same::value) {\n const float d = x[ib].d;\n\n#pragma unroll\n for (int l = 0; l < ne; ++l) {\n ((float *) dst)[l] = d * qs[l];\n }\n } else {\n static_assert(std::is_same_v, \"unsupported type\");\n }\n}\n\ntemplate \nconstexpr vec_dot_KQ_t get_vec_dot_KQ() {\n if constexpr (type_K == GGML_TYPE_F16) {\n return vec_dot_fattn_vec_KQ_f16;\n } else if constexpr (type_K == GGML_TYPE_Q4_0) {\n return vec_dot_fattn_vec_KQ_q4_0;\n } else if constexpr (type_K == GGML_TYPE_Q4_1) {\n return vec_dot_fattn_vec_KQ_q4_1;\n } else if constexpr (type_K == GGML_TYPE_Q5_0) {\n return vec_dot_fattn_vec_KQ_q5_0;\n } else if constexpr (type_K == GGML_TYPE_Q5_1) {\n return vec_dot_fattn_vec_KQ_q5_1;\n } else if constexpr (type_K == GGML_TYPE_Q8_0) {\n return vec_dot_fattn_vec_KQ_q8_0;\n } else {\n static_assert(type_K == -1, \"bad type\");\n return nullptr;\n }\n}\n\ntemplate \nconstexpr dequantize_V_t get_dequantize_V() {\n if constexpr (type_V == GGML_TYPE_F16) {\n return dequantize_V_f16;\n } else if constexpr (type_V == GGML_TYPE_Q4_0) {\n return dequantize_V_q4_0;\n } else if constexpr (type_V == GGML_TYPE_Q4_1) {\n return dequantize_V_q4_1;\n } else if constexpr (type_V == GGML_TYPE_Q5_0) {\n return dequantize_V_q5_0;\n } else if constexpr (type_V == GGML_TYPE_Q5_1) {\n return dequantize_V_q5_1;\n } else if constexpr (type_V == GGML_TYPE_Q8_0) {\n return dequantize_V_q8_0;\n } else {\n static_assert(type_V == -1, \"bad type\");\n return nullptr;\n }\n}\n\ntemplate \nstatic void flash_attn_mask_to_KV_max(const sycl::half2 * __restrict__ mask,\n int * __restrict__ KV_max,\n const int ne30,\n const int s31,\n const int s33,\n int * buf_iw) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const int ne31 = item_ct1.get_group_range(2);\n const int tid = item_ct1.get_local_id(2);\n const int sequence = item_ct1.get_group(1);\n const int jt = item_ct1.get_group(2);\n\n mask += sequence*s33 + jt*ncols1*s31;\n\n if (tid < warp_size) {\n buf_iw[tid] = 1;\n }\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n int KV_max_sj = (ne30 - 1) * FATTN_KQ_STRIDE;\n for (; KV_max_sj >= 0; KV_max_sj -= FATTN_KQ_STRIDE) {\n int all_inf = 1;\n\n#pragma unroll\n for (int j = 0; j < ncols1; ++j) {\n const sycl::float2 tmp =\n mask[j * s31 + KV_max_sj / 2 + tid].template convert();\n all_inf = all_inf && int(sycl::isinf((float) (tmp.x()))) && int(sycl::isinf((float) (tmp.y())));\n }\n\n all_inf = warp_reduce_all(all_inf);\n if (tid % warp_size == 0) {\n buf_iw[tid / warp_size] = all_inf;\n }\n item_ct1.barrier(sycl::access::fence_space::local_space);\n all_inf = buf_iw[tid % warp_size];\n item_ct1.barrier(sycl::access::fence_space::local_space);\n all_inf = warp_reduce_all(all_inf);\n\n if (!all_inf) {\n break;\n }\n }\n\n // If the break in the loop was not triggered, KV_max_sj is now -FATTN_KQ_STRIDE.\n // If the break was triggered it's the lower edge of the tile with the first non-masked values.\n // In either case, walk back the decrementation by FATTN_KQ_STRIDE.\n KV_max_sj += FATTN_KQ_STRIDE;\n\n if (item_ct1.get_local_id(2) != 0) {\n return;\n }\n\n KV_max[sequence*ne31 + jt] = KV_max_sj;\n}\n\ntemplate // D == head size\n\nstatic void flash_attn_stream_k_fixup(float * __restrict__ dst,\n const sycl::float2 * __restrict__ dst_fixup,\n const int ne01,\n const int ne02,\n const int ne03,\n const int ne11,\n const int ne12,\n const int nbatch_fa) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n constexpr int ncols = ncols1 * ncols2;\n\n const int bidx0 = item_ct1.get_group(2);\n const int j = item_ct1.get_group(1);\n const int c = item_ct1.get_group(0);\n const int jc = j*ncols2 + c;\n const int tid = item_ct1.get_local_id(2);\n\n const float * dst_fixup_data = ((const float *) dst_fixup) + item_ct1.get_group_range(2) * (2 * 2 * ncols);\n\n const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n\n const int iter_k = (ne11 + (nbatch_fa - 1)) / nbatch_fa;\n const int iter_j = (ne01 + (ncols1 - 1)) / ncols1;\n const int iter_z_gqa = (gqa_ratio + (ncols2 - 1)) / ncols2;\n\n const int kbc0 = int64_t(bidx0 + 0) * (iter_k * iter_j * iter_z_gqa * ne12 * ne03) / item_ct1.get_group_range(2);\n const int kbc0_stop =\n int64_t(bidx0 + 1) * (iter_k * iter_j * iter_z_gqa * ne12 * ne03) / item_ct1.get_group_range(2);\n\n const bool did_not_have_any_data = kbc0 == kbc0_stop;\n const bool wrote_beginning_of_tile = kbc0 % iter_k == 0;\n const bool did_not_write_last = kbc0/iter_k == kbc0_stop/iter_k && kbc0_stop % iter_k != 0;\n if (did_not_have_any_data || wrote_beginning_of_tile || did_not_write_last) {\n return;\n }\n\n // z_KV == K/V head index, zt_gqa = Q head start index per K/V head, jt = token position start index\n const int sequence = kbc0 /(iter_k*iter_j*iter_z_gqa*ne12);\n const int z_KV = (kbc0 - iter_k*iter_j*iter_z_gqa*ne12 * sequence)/(iter_k*iter_j*iter_z_gqa);\n const int zt_gqa = (kbc0 - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV)/(iter_k*iter_j);\n const int jt = (kbc0 - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV - iter_k*iter_j * zt_gqa) / iter_k;\n\n const int zt_Q = z_KV*gqa_ratio + zt_gqa*ncols2; // Global Q head start index.\n\n if (jt*ncols1 + j >= ne01 || zt_gqa*ncols2 + c >= gqa_ratio) {\n return;\n }\n\n dst += sequence*ne02*ne01*D + jt*ne02*(ncols1*D) + zt_Q*D + (j*ne02 + c)*D + tid;\n\n // Load the partial result that needs a fixup:\n float dst_val = 0.0f;\n float max_val = 0.0f;\n float rowsum = 0.0f;\n {\n dst_val = *dst;\n\n const sycl::float2 tmp = dst_fixup[bidx0 * ncols + jc];\n max_val = tmp.x();\n rowsum = tmp.y();\n }\n\n // Iterate over previous blocks and compute the combined results.\n // All SYCL blocks that get here must have a previous block that needs a fixup.\n int bidx = bidx0 - 1;\n int kbc_stop = kbc0;\n while(true) {\n const int kbc = int64_t(bidx) * (iter_k * iter_j * iter_z_gqa * ne12 * ne03) / item_ct1.get_group_range(2);\n if (kbc == kbc_stop) { // Did not have any data.\n bidx--;\n kbc_stop = kbc;\n continue;\n }\n\n const float dst_add = dst_fixup_data[bidx*ncols*D + jc*D + tid];\n\n const sycl::float2 tmp = dst_fixup[(item_ct1.get_group_range(2) + bidx) * ncols + jc];\n\n // Scale the current and new value accumulators depending on the max. values.\n const float max_val_new = sycl::fmax(max_val, tmp.x());\n\n const float diff_val = max_val - max_val_new;\n const float diff_add = tmp.x() - max_val_new;\n\n const float scale_val = diff_val >= SOFTMAX_FTZ_THRESHOLD ? sycl::native::exp(diff_val) : 0.0f;\n const float scale_add = diff_add >= SOFTMAX_FTZ_THRESHOLD ? sycl::native::exp(diff_add) : 0.0f;\n\n dst_val = scale_val*dst_val + scale_add*dst_add;\n rowsum = scale_val * rowsum + scale_add * tmp.y();\n\n max_val = max_val_new;\n\n // If this block started in a previous tile we are done and don't need to combine additional partial results.\n if (kbc % iter_k == 0 || kbc/iter_k < kbc0/iter_k) {\n break;\n }\n bidx--;\n kbc_stop = kbc;\n }\n\n // Write back final result:\n *dst = dst_val / rowsum;\n}\n\ntemplate // D == head size\n\nstatic void flash_attn_combine_results(const float * __restrict__ VKQ_parts,\n const sycl::float2 * __restrict__ VKQ_meta,\n float * __restrict__ dst,\n const int parallel_blocks,\n uint8_t * dpct_local) {\n // Dimension 0: threadIdx.x\n // Dimension 1: blockIdx.x\n // Dimension 2: blockIdx.y\n // Dimension 3: blockIdx.z\n // Memory layout is permuted with [0, 2, 1, 3]\n\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const int ne01 = item_ct1.get_group_range(2);\n const int ne02 = item_ct1.get_group_range(1);\n\n const int col = item_ct1.get_group(2);\n const int head = item_ct1.get_group(1);\n const int sequence = item_ct1.get_group(0);\n\n const int j_dst_unrolled = (sequence*ne01 + col)*ne02 + head;\n\n VKQ_parts += j_dst_unrolled * parallel_blocks*D;\n VKQ_meta += j_dst_unrolled * parallel_blocks;\n dst += j_dst_unrolled * D;\n\n const int tid = item_ct1.get_local_id(2);\n __builtin_assume(tid < D);\n\n auto meta = (sycl::float2 *) dpct_local;\n for (int i = tid; i < 2*parallel_blocks; i += D) {\n ((float *) meta)[i] = ((const float *)VKQ_meta) [i];\n }\n\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n float kqmax = meta[0].x();\n for (int l = 1; l < parallel_blocks; ++l) {\n kqmax = sycl::max(kqmax, meta[l].x());\n }\n\n float VKQ_numerator = 0.0f;\n float VKQ_denominator = 0.0f;\n for (int l = 0; l < parallel_blocks; ++l) {\n const float KQ_max_scale = sycl::native::exp(meta[l].x() - kqmax);\n\n VKQ_numerator += KQ_max_scale * VKQ_parts[l*D + tid];\n VKQ_denominator += KQ_max_scale * meta[l].y();\n }\n\n dst[tid] = VKQ_numerator / VKQ_denominator;\n}\n\ntemplate \nstatic void lauch_kernel(\n dpct::dim3 group_range,\n dpct::dim3 local_range,\n queue_ptr q,\n unsigned int local_mem_size,\n const char* __restrict__ Q,\n const char* __restrict__ K,\n const char* __restrict__ V,\n const char* __restrict__ mask,\n const char* __restrict__ sinks,\n const int* __restrict__ KV_max,\n float* __restrict__ dst,\n sycl::float2* __restrict__ dst_meta,\n const float scale,\n const float max_bias,\n const float m0,\n const float m1,\n const uint32_t n_head_log2,\n const float logit_softcap,\n const int32_t ne00,\n const sycl::uint3 ne01,\n const int32_t ne02,\n const int32_t ne03,\n const int32_t nb01,\n const int32_t nb02,\n const int32_t nb03,\n const int32_t ne10,\n const int32_t ne11,\n const int32_t ne12,\n const int32_t ne13,\n const int32_t nb11,\n const int32_t nb12,\n const int64_t nb13,\n const int32_t nb21,\n const int32_t nb22,\n const int64_t nb23,\n const int32_t ne31,\n const int32_t ne32,\n const int32_t ne33,\n const int32_t nb31,\n const int32_t nb32,\n const int64_t nb33) {\n GGML_UNUSED(local_mem_size);\n q->submit([&](sycl::handler &cgh) {\n cgh.parallel_for(\n sycl::nd_range<3>(\n static_cast>(group_range * local_range),\n static_cast>(local_range)),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] {\n GGML_UNUSED(item_ct1);\n fattn_kernel(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap, ne00,\n ne01, ne02, ne03, nb01, nb02, nb03, ne10, ne11,\n ne12, ne13, nb11, nb12, nb13, nb21, nb22, nb23,\n ne31, ne32, ne33, nb31, nb32, nb33);\n });\n });\n}\n\ntemplate \nvoid launch_fattn(\n ggml_backend_sycl_context & ctx, ggml_tensor * dst, const int nwarps, const size_t nbytes_shared,\n const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k) {\n\n constexpr int ncols = ncols1 * ncols2;\n\n const ggml_tensor * Q = dst->src[0];\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * V = dst->src[2];\n\n const bool V_is_K_view = V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs));\n\n const ggml_tensor * mask = dst->src[3];\n const ggml_tensor * sinks = dst->src[4];\n\n ggml_tensor * KQV = dst;\n\n GGML_ASSERT(Q->type == GGML_TYPE_F32);\n GGML_ASSERT(KQV->type == GGML_TYPE_F32);\n\n GGML_ASSERT(Q->nb[0] == ggml_element_size(Q));\n GGML_ASSERT(K->nb[0] == ggml_element_size(K));\n GGML_ASSERT(V->nb[0] == ggml_element_size(V));\n\n GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16);\n\n ggml_sycl_pool & pool = ctx.pool();\n dpct::queue_ptr main_stream = ctx.stream();\n const int id = ggml_sycl_get_device();\n const int nsm = ggml_sycl_info().devices[id].nsm;\n\n ggml_sycl_pool_alloc K_f16(pool);\n ggml_sycl_pool_alloc V_f16(pool);\n ggml_sycl_pool_alloc KV_max(pool);\n ggml_sycl_pool_alloc dst_tmp(pool);\n ggml_sycl_pool_alloc dst_tmp_meta(pool);\n\n const char * K_data = (const char *) K->data;\n size_t nb11 = K->nb[1];\n size_t nb12 = K->nb[2];\n size_t nb13 = K->nb[3];\n\n const char * V_data = (const char *) V->data;\n size_t nb21 = V->nb[1];\n size_t nb22 = V->nb[2];\n size_t nb23 = V->nb[3];\n\n if (need_f16_K && K->type != GGML_TYPE_F16) {\n const size_t bs = ggml_blck_size(K->type);\n const size_t ts = ggml_type_size(K->type);\n\n K_f16.alloc(ggml_nelements(K));\n if (ggml_is_contiguously_allocated(K)) {\n to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(K->type, dst);\n to_fp16(K_data, K_f16.ptr, ggml_nelements(K), main_stream);\n\n nb11 = nb11 * bs * sizeof(sycl::half) / ts;\n nb12 = nb12 * bs * sizeof(sycl::half) / ts;\n nb13 = nb13 * bs * sizeof(sycl::half) / ts;\n } else {\n GGML_ASSERT(K->nb[0] == ts);\n to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(K->type);\n const int64_t s01 = nb11 / ts;\n const int64_t s02 = nb12 / ts;\n const int64_t s03 = nb13 / ts;\n to_fp16(K_data, K_f16.ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], s01, s02, s03, main_stream);\n\n nb11 = K->ne[0] * sizeof(sycl::half);\n nb12 = K->ne[1] * nb11;\n nb13 = K->ne[2] * nb12;\n }\n K_data = (char *) K_f16.ptr;\n }\n\n if (need_f16_V && V->type != GGML_TYPE_F16) {\n if (V_is_K_view) {\n V_data = K_data;\n nb21 = nb11;\n nb22 = nb12;\n nb23 = nb13;\n } else {\n const size_t bs = ggml_blck_size(V->type);\n const size_t ts = ggml_type_size(V->type);\n\n V_f16.alloc(ggml_nelements(V));\n if (ggml_is_contiguously_allocated(V)) {\n to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(V->type, dst);\n to_fp16(V_data, V_f16.ptr, ggml_nelements(V), main_stream);\n V_data = (char *) V_f16.ptr;\n\n nb21 = nb21 * bs * sizeof(sycl::half) / ts;\n nb22 = nb22 * bs * sizeof(sycl::half) / ts;\n nb23 = nb23 * bs * sizeof(sycl::half) / ts;\n } else {\n GGML_ASSERT(V->nb[0] == ts);\n to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(V->type);\n const int64_t s01 = nb21 / ts;\n const int64_t s02 = nb22 / ts;\n const int64_t s03 = nb23 / ts;\n to_fp16(V_data, V_f16.ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], s01, s02, s03, main_stream);\n\n nb21 = V->ne[0] * sizeof(sycl::half);\n nb22 = V->ne[1] * nb21;\n nb23 = V->ne[2] * nb22;\n }\n V_data = (char *) V_f16.ptr;\n }\n }\n\n const int ntiles_x = ((Q->ne[1] + ncols1 - 1) / ncols1);\n const int gqa_ratio = Q->ne[2] / K->ne[2];\n const int ntiles_z_gqa = ((gqa_ratio + ncols2 - 1) / ncols2);\n const int ntiles_total = ntiles_x * ntiles_z_gqa * K->ne[2] * Q->ne[3];\n\n // Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.\n // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or\n // multiple sequences of possibly different lengths.\n if (mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) {\n const int s31 = mask->nb[1] / sizeof(sycl::half2);\n const int s33 = mask->nb[3] / sizeof(sycl::half2);\n\n const dpct::dim3 blocks_num_KV_max(ntiles_x, Q->ne[3], 1);\n const dpct::dim3 block_dim_KV_max(FATTN_KQ_STRIDE / 2, 1, 1);\n\n const int ne_KV_max = blocks_num_KV_max.x*blocks_num_KV_max.y;\n const int iter_k = K->ne[1] / FATTN_KQ_STRIDE;\n\n KV_max.alloc(ne_KV_max);\n {\n dpct::has_capability_or_fail(main_stream->get_device(), { sycl::aspect::fp16 });\n\n main_stream->submit([&](sycl::handler & cgh) {\n sycl::local_accessor buf_iw_acc_ct1(sycl::range<1>(warp_size), cgh);\n\n auto mask_data_ct0 = (const sycl::half2 *) mask->data;\n auto KV_max_ptr_ct1 = KV_max.ptr;\n\n cgh.parallel_for(sycl::nd_range<3>(blocks_num_KV_max * block_dim_KV_max, block_dim_KV_max),\n [=](sycl::nd_item<3> item_ct1) {\n GGML_UNUSED(item_ct1);\n flash_attn_mask_to_KV_max(\n mask_data_ct0, KV_max_ptr_ct1, iter_k, s31, s33,\n buf_iw_acc_ct1.get_multi_ptr().get());\n });\n });\n }\n SYCL_CHECK(0);\n }\n\n const dpct::dim3 block_dim(warp_size, nwarps, 1);\n\n // Max. number of active blocks limited by occupancy.\n int max_blocks_per_sm = ggml_sycl_info().devices[id].max_wg_per_cu;\n int parallel_blocks = max_blocks_per_sm;\n dpct::dim3 blocks_num;\n if (stream_k) {\n // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup.\n const int max_blocks = max_blocks_per_sm*nsm;\n const int nblocks_stream_k = max_blocks;\n const bool use_stream_k = true;\n\n blocks_num.x = use_stream_k ? nblocks_stream_k : ntiles_total;\n blocks_num.y = 1;\n blocks_num.z = 1;\n\n if (ntiles_total % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles.\n dst_tmp_meta.alloc((size_t(blocks_num.x) * ncols * (2 + DV/2)));\n }\n } else {\n const int ntiles_KQ = (K->ne[1] + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by tensor size.\n\n // parallel_blocks must not be larger than what the tensor size allows:\n parallel_blocks = std::min(parallel_blocks, ntiles_KQ);\n // todo fix the hard code change\n // parallel_blocks = ntiles_KQ;\n\n // If ntiles_total % blocks_per_wave != 0 then some efficiency is lost due to tail effects.\n // Test whether parallel_blocks can be set to a higher value for better efficiency.\n const int blocks_per_wave = nsm * max_blocks_per_sm;\n int nwaves_best = 0;\n int efficiency_percent_best = 0;\n for (int parallel_blocks_test = parallel_blocks; parallel_blocks_test <= ntiles_KQ; ++parallel_blocks_test) {\n const int nblocks_total = ntiles_total * parallel_blocks_test;\n const int nwaves = (nblocks_total + blocks_per_wave - 1) / blocks_per_wave;\n const int efficiency_percent = 100 * nblocks_total / (nwaves*blocks_per_wave);\n\n // Stop trying configurations with more waves if we already have good efficiency to avoid excessive overhead.\n if (efficiency_percent_best >= 95 && nwaves > nwaves_best) {\n break;\n }\n\n if (efficiency_percent > efficiency_percent_best) {\n nwaves_best = nwaves;\n efficiency_percent_best = efficiency_percent;\n parallel_blocks = parallel_blocks_test;\n }\n }\n\n blocks_num.x = ntiles_x;\n blocks_num.y = parallel_blocks;\n blocks_num.z = ntiles_z_gqa*K->ne[2]*Q->ne[3];\n\n if (parallel_blocks > 1) {\n dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));\n dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));\n }\n }\n\n float scale = 1.0f;\n float max_bias = 0.0f;\n float logit_softcap = 0.0f;\n\n memcpy(&scale, (const float *) KQV->op_params + 0, sizeof(float));\n memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));\n memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n if (logit_softcap != 0.0f) {\n scale /= logit_softcap;\n }\n\n const uint32_t n_head = Q->ne[2];\n const uint32_t n_head_log2 = 1u << uint32_t(floorf(log2f(float(n_head))));\n\n const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);\n const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n // TODO other tensor dimensions after removal of WMMA kernel:\n const sycl::uint3 ne01 = init_fastdiv_values(Q->ne[1]);\n\n GGML_ASSERT(block_dim.x % warp_size == 0);\n\n lauch_kernel(\n blocks_num, block_dim, main_stream, (unsigned int) nbytes_shared, (const char *) Q->data, K_data, V_data,\n mask ? ((const char *) mask->data) : nullptr, sinks ? ((const char *) sinks->data) : nullptr, KV_max.ptr,\n !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, (sycl::float2 *)dst_tmp_meta.ptr, scale, max_bias, m0, m1,\n n_head_log2, logit_softcap, Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->ne[0],\n K->ne[1], K->ne[2], K->ne[3], nb11, nb12, nb13, nb21, nb22, nb23, mask ? mask->ne[1] : 0,\n mask ? mask->ne[2] : 0, mask ? mask->ne[3] : 0, mask ? mask->nb[1] : 0, mask ? mask->nb[2] : 0,\n mask ? mask->nb[3] : 0);\n SYCL_CHECK(0);\n\n if (stream_k) {\n if (ntiles_total % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles.\n const dpct::dim3 block_dim_combine(DV, 1, 1);\n const dpct::dim3 blocks_num_combine = { blocks_num.x, ncols1, ncols2 };\n\n main_stream->submit([&](sycl::handler & cgh) {\n auto KQV_data_ct0 = (float *) KQV->data;\n auto dst_tmp_meta_ptr_ct1 = dst_tmp_meta.ptr;\n auto Q_ne_ct2 = Q->ne[1];\n auto Q_ne_ct3 = Q->ne[2];\n auto Q_ne_ct4 = Q->ne[3];\n auto K_ne_ct5 = K->ne[1];\n auto K_ne_ct6 = K->ne[2];\n\n cgh.parallel_for(sycl::nd_range<3>(blocks_num_combine * block_dim_combine, block_dim_combine),\n [=](sycl::nd_item<3> item_ct1) {\n GGML_UNUSED(item_ct1);\n flash_attn_stream_k_fixup(KQV_data_ct0, dst_tmp_meta_ptr_ct1,\n Q_ne_ct2, Q_ne_ct3, Q_ne_ct4,\n K_ne_ct5, K_ne_ct6, nbatch_fa);\n });\n });\n }\n } else if (parallel_blocks > 1) {\n const dpct::dim3 block_dim_combine(DV, 1, 1);\n const dpct::dim3 blocks_num_combine(Q->ne[1], Q->ne[2], Q->ne[3]);\n const size_t nbytes_shared_combine = parallel_blocks * sizeof(sycl::float2);\n main_stream->submit([&](sycl::handler & cgh) {\n sycl::local_accessor dpct_local_acc_ct1(sycl::range<1>(nbytes_shared_combine), cgh);\n\n auto dst_tmp_ptr_ct0 = dst_tmp.ptr;\n auto dst_tmp_meta_ptr_ct1 = dst_tmp_meta.ptr;\n auto KQV_data_ct2 = (float *) KQV->data;\n\n cgh.parallel_for(sycl::nd_range<3>(blocks_num_combine * block_dim_combine, block_dim_combine),\n [=](sycl::nd_item<3> item_ct1) {\n GGML_UNUSED(item_ct1);\n flash_attn_combine_results(\n dst_tmp_ptr_ct0, dst_tmp_meta_ptr_ct1, KQV_data_ct2, parallel_blocks,\n dpct_local_acc_ct1.get_multi_ptr().get());\n });\n });\n }\n SYCL_CHECK(0);\n}\n" }, { "path": "src/ggml-sycl/fattn-tile.cpp", "content": "#include \n#include \n#include \"dpct/helper.hpp\"\n#include \"common.hpp\"\n#include \"fattn-common.hpp\"\n#include \"fattn-tile.hpp\"\n#include \n#include \nnamespace syclex = sycl::ext::oneapi::experimental;\n\nvoid ggml_sycl_flash_attn_ext_tile(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * V = dst->src[2];\n switch (K->ne[0]) {\n case 40: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_sycl_flash_attn_ext_tile_case< 40, 40>(ctx, dst);\n } break;\n case 64: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_sycl_flash_attn_ext_tile_case< 64, 64>(ctx, dst);\n } break;\n case 72: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_sycl_flash_attn_ext_tile_case< 72, 72>(ctx, dst);\n } break;\n case 80: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_sycl_flash_attn_ext_tile_case< 80, 80>(ctx, dst);\n } break;\n case 96: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_sycl_flash_attn_ext_tile_case< 96, 96>(ctx, dst);\n } break;\n case 112: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_sycl_flash_attn_ext_tile_case<112, 112>(ctx, dst);\n } break;\n case 128: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_sycl_flash_attn_ext_tile_case<128, 128>(ctx, dst);\n } break;\n case 256: {\n GGML_ASSERT(V->ne[0] == K->ne[0]);\n ggml_sycl_flash_attn_ext_tile_case<256, 256>(ctx, dst);\n } break;\n case 576: {\n GGML_ASSERT(V->ne[0] == 512);\n ggml_sycl_flash_attn_ext_tile_case<576, 512>(ctx, dst);\n } break;\n default: {\n GGML_ABORT(\"Unsupported head size\");\n } break;\n }\n}\n" }, { "path": "src/ggml-sycl/fattn-tile.hpp", "content": "#include \n#include \n#include \"dpct/helper.hpp\"\n#include \"common.hpp\"\n#include \"fattn-common.hpp\"\n\n#include \n#include \n\nnamespace syclex = sycl::ext::oneapi::experimental;\n\n#define GGML_SYCL_FATTN_TILE_CONFIG_CASE(DKQ_, DV_, ncols_, nthreads, occupancy, nbatch_fa, nbatch_K) \\\n if (DKQ == (DKQ_) && DV == (DV_) && ncols == (ncols_)) { \\\n static_assert((nthreads) <= 512, \"bad nthreads\"); \\\n static_assert((occupancy) <= 8, \"bad occupancy\"); \\\n static_assert((nbatch_fa) <= 256, \"bad nbatch_fa\"); \\\n static_assert((nbatch_K) <= 256, \"bad nbatch_K\"); \\\n return ((nthreads) << 0) | ((occupancy) << 10) | ((nbatch_fa) << 14) | ((nbatch_K) << 23); \\\n } \\\n\nstatic constexpr uint32_t ggml_sycl_fattn_tile_get_config_fp16(const int DKQ, const int DV, const int ncols) {\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 64, 2, 64, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 64, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 256, 2, 64, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 16, 256, 2, 64, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 32, 256, 2, 64, 40)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 2, 64, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 4, 128, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 64)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 64, 2, 64, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 2, 64, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 8, 256, 2, 64, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 16, 256, 2, 64, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 32, 256, 2, 64, 72)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 64, 2, 64, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 4, 128, 2, 64, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 64, 40)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 2, 64, 2, 64, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 4, 128, 2, 64, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 8, 256, 2, 64, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 16, 256, 2, 64, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 64, 48)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 2, 64, 2, 64, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 4, 128, 2, 64, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 2, 64, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 64, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 64, 56)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 2, 64, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 4, 128, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 8, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 16, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 2, 64, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 64, 64)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 4, 128, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 2, 64, 64)\n\n return 0;\n}\n\nstatic constexpr uint32_t ggml_sycl_fattn_tile_get_config_fp32(const int DKQ, const int DV, const int ncols) {\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 64, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 16, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 32, 256, 2, 32, 40)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 2, 128, 3, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 4, 128, 3, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 128, 3, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 16, 128, 3, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 64)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 64, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 8, 256, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 16, 256, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 32, 256, 2, 32, 72)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 64, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 4, 128, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 8, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 16, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 32, 40)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 2, 64, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 4, 128, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 8, 256, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 16, 256, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 32, 48)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 2, 64, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 4, 128, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 32, 56)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 2, 128, 3, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 4, 128, 3, 32, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 8, 128, 3, 64, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 16, 128, 3, 32, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 2, 128, 3, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 3, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 32, 256)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 32, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 32, 64)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 4, 128, 2, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 2, 32, 64)\n\n return 0;\n}\n\nstatic constexpr uint32_t ggml_sycl_fattn_tile_get_config_amd(const int DKQ, const int DV, const int ncols) {\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 64, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 16, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 32, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 64, 256, 2, 32, 40)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 2, 64, 3, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 4, 128, 3, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 128, 2, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 16, 256, 2, 128, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 64, 256, 2, 64, 64)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 64, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 8, 256, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 16, 256, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 32, 256, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 64, 256, 2, 32, 72)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 64, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 4, 128, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 8, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 16, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 64, 256, 2, 32, 40)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 2, 64, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 4, 128, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 8, 256, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 16, 256, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 64, 256, 2, 32, 48)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 2, 64, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 4, 128, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 64, 256, 2, 32, 56)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 2, 256, 2, 128, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 4, 128, 2, 64, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 8, 256, 2, 64, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 16, 256, 2, 64, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 2, 64, 32)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 2, 256, 2, 128, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 4, 256, 2, 64, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 64, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 32, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 32, 128)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 4, 128, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 32, 512, 1, 128, 64)\n\n return 0;\n}\n\nstatic constexpr uint32_t ggml_sycl_fattn_tile_get_config_amd_rdna(const int DKQ, const int DV, const int ncols) {\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 64, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 16, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 32, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 40, 40, 64, 256, 2, 32, 40)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 2, 64, 8, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 4, 64, 8, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 128, 5, 128, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 16, 128, 5, 128, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 32, 128, 4, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 64, 64, 64, 128, 5, 64, 64)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 64, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 8, 256, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 16, 256, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 32, 256, 2, 32, 72)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 72, 72, 64, 256, 2, 32, 72)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 64, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 4, 128, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 8, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 16, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 32, 40)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 80, 80, 64, 256, 2, 32, 40)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 2, 64, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 4, 128, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 8, 256, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 16, 256, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 32, 48)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE( 96, 96, 64, 256, 2, 32, 48)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 2, 64, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 4, 128, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 32, 56)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(112, 112, 64, 256, 2, 32, 56)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 2, 64, 8, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 4, 128, 8, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 8, 128, 8, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 16, 256, 3, 128, 128)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 3, 128, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 3, 64, 64)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 2, 64, 8, 32, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 6, 32, 256)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 8, 128, 6, 32, 256)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 5, 32, 256)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 3, 64, 128)\n\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 4, 128, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 4, 64, 64)\n GGML_SYCL_FATTN_TILE_CONFIG_CASE(576, 512, 32, 256, 2, 128, 64)\n\n return 0;\n}\n\nstatic constexpr uint32_t ggml_sycl_fattn_tile_get_config(const int DKQ, const int DV, const int ncols, const int cc) {\n if(fast_fp16_available(cc))\n return ggml_sycl_fattn_tile_get_config_fp16(DKQ, DV, ncols);\n else\n return ggml_sycl_fattn_tile_get_config_fp32(DKQ, DV, ncols);\n}\n\nstatic constexpr uint32_t ggml_sycl_fattn_tile_get_config(const int DKQ, const int DV, const int ncols) {\n#ifdef SYCL_FAST_FP16\n return ggml_sycl_fattn_tile_get_config_fp16(DKQ, DV, ncols);\n#else\n return ggml_sycl_fattn_tile_get_config_fp32(DKQ, DV, ncols);\n#endif // SYCL_FAST_FP16\n}\n\nstatic int ggml_sycl_fattn_tile_get_nthreads(const int DKQ, const int DV, const int ncols, const int cc) {\n return (ggml_sycl_fattn_tile_get_config(DKQ, DV, ncols, cc) >> 0) & ((1 << 10) - 1);\n}\n\nstatic constexpr int ggml_sycl_fattn_tile_get_nthreads(const int DKQ, const int DV, const int ncols) {\n return (ggml_sycl_fattn_tile_get_config(DKQ, DV, ncols) >> 0) & ((1 << 10) - 1);\n}\n\nstatic int ggml_sycl_fattn_tile_get_occupancy(const int DKQ, const int DV, const int ncols, const int cc) {\n return (ggml_sycl_fattn_tile_get_config(DKQ, DV, ncols, cc) >> 10) & ((1 << 4) - 1);\n}\n\nstatic constexpr int ggml_sycl_fattn_tile_get_occupancy(const int DKQ, const int DV, const int ncols) {\n return (ggml_sycl_fattn_tile_get_config(DKQ, DV, ncols) >> 10) & ((1 << 4) - 1);\n}\n\nstatic int ggml_sycl_fattn_tile_get_nbatch_fa(const int DKQ, const int DV, const int ncols, const int cc) {\n return (ggml_sycl_fattn_tile_get_config(DKQ, DV, ncols, cc) >> 14) & ((1 << 9) - 1);\n}\n\nstatic constexpr int ggml_sycl_fattn_tile_get_nbatch_fa(const int DKQ, const int DV, const int ncols) {\n return (ggml_sycl_fattn_tile_get_config(DKQ, DV, ncols) >> 14) & ((1 << 9) - 1);\n}\n\nstatic int ggml_sycl_fattn_tile_get_nbatch_K(const int DKQ, const int DV, const int ncols, const int cc) {\n return (ggml_sycl_fattn_tile_get_config(DKQ, DV, ncols, cc) >> 23) & ((1 << 9) - 1);\n}\n\nstatic constexpr int ggml_sycl_fattn_tile_get_nbatch_K(const int DKQ, const int DV, const int ncols) {\n return (ggml_sycl_fattn_tile_get_config(DKQ, DV, ncols) >> 23) & ((1 << 9) - 1);\n}\n\ntemplate \nstatic __dpct_inline__ void flash_attn_tile_load_tile(const sycl::half2 * const __restrict__ KV,\n sycl::half2 * const __restrict__ tile_KV,\n const int stride_KV,\n const int i_sup) {\n constexpr int cpy_nb = ggml_sycl_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n auto load = [&] (const int n) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const int stride_j = warp_size >> n;\n\n if (stride_j == 0) {\n return;\n }\n\n const int j0_start = stride_j == warp_size ? 0 : ((J/2)/cpy_ne) - ((J/2)/cpy_ne) % (2*stride_j);\n const int j0_stop = ((J/2)/cpy_ne) - ((J/2)/cpy_ne) % (1*stride_j);\n const int stride_i = warp_size / stride_j;\n\n if (j0_start == j0_stop) {\n return;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < I; i0 += nwarps*stride_i) {\n const int i = i0 + item_ct1.get_local_id(1) * stride_i +\n (stride_j == warp_size ? 0 : item_ct1.get_local_id(2) / stride_j);\n\n if (i0 + nwarps*stride_i <= I || i < I) {\n#pragma unroll\n for (int j0 = j0_start; j0 < j0_stop; j0 += stride_j) {\n const int j = j0 * cpy_ne + (stride_j == warp_size ? item_ct1.get_local_id(2) :\n item_ct1.get_local_id(2) % stride_j) *\n cpy_ne;\n\n const __dpct_align__(16) sycl::half2 zero[cpy_ne] = {\n { 0.0f, 0.0f }\n };\n ggml_sycl_memcpy_1(\n tile_KV + i*(J/2 + J_padding) + j,\n !oob_check || i < i_sup ? KV + i*stride_KV + j : zero);\n }\n }\n }\n };\n // 1: max 64*16=512 bytes, 512 half\n // 2: max 32*16=512 bytes, 256 half\n // 3: max 16*16=256 bytes, 128 half\n // 4: max 8*16=128 bytes, 64 half\n // 5: max 4*16= 64 bytes, 32 half\n // 6: max 2*16= 32 bytes, 16 half\n // 7: max 1*16= 16 bytes, 8 half\n static_assert(J % 8 == 0, \"bad J\");\n static_assert((J/2) % cpy_ne == 0, \"bad J\");\n ggml_sycl_unroll<7>{}(load);\n}\n\ntemplate \nstatic __dpct_inline__ void flash_attn_tile_load_tile(const sycl::half2 * const __restrict__ KV,\n float * const __restrict__ tile_KV,\n const int stride_KV,\n const int i_sup) {\n constexpr int cpy_nb = ggml_sycl_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n auto load = [&] (const int n) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const int stride_j = warp_size >> n;\n\n if (stride_j == 0) {\n return;\n }\n\n const int j0_start = stride_j == warp_size ? 0 : (J/cpy_ne) - (J/cpy_ne) % (2*stride_j);\n const int j0_stop = (J/cpy_ne) - (J/cpy_ne) % (1*stride_j);\n const int stride_i = warp_size / stride_j;\n\n if (j0_start == j0_stop) {\n return;\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < I; i0 += nwarps*stride_i) {\n const int i = i0 + item_ct1.get_local_id(1) * stride_i +\n (stride_j == warp_size ? 0 : item_ct1.get_local_id(2) / stride_j);\n\n if (i0 + nwarps*stride_i <= I || i < I) {\n#pragma unroll\n for (int j0 = j0_start; j0 < j0_stop; j0 += stride_j) {\n const int j = j0 * (cpy_ne / 2) + (stride_j == warp_size ? item_ct1.get_local_id(2) :\n item_ct1.get_local_id(2) % stride_j) *\n (cpy_ne / 2);\n\n const sycl::half2 zero[cpy_ne / 2] = {\n { 0.0f, 0.0f }\n };\n __dpct_align__(16) sycl::half2 tmp_h2[cpy_ne / 2];\n ggml_sycl_memcpy_1(\n tmp_h2, !oob_check || i < i_sup ? KV + i*stride_KV + j : zero);\n\n __dpct_align__(16) sycl::float2 tmp_f2[cpy_ne / 2];\n#pragma unroll\n for (int l = 0; l < cpy_ne/2; ++l) {\n tmp_f2[l] = tmp_h2[l].template convert();\n }\n ggml_sycl_memcpy_1(tile_KV + i*(J + J_padding) + 2*j, tmp_f2);\n }\n }\n }\n };\n // 1: max 32*16=512 bytes, 128 float\n // 2: max 16*16=256 bytes, 64 float\n // 3: max 8*16=128 bytes, 32 float\n // 4: max 4*16= 64 bytes, 16 float\n // 5: max 2*16= 32 bytes, 8 float\n static_assert(J % 8 == 0, \"bad J\");\n static_assert(J % cpy_ne == 0, \"bad J\");\n ggml_sycl_unroll<5>{}(load);\n}\n\n// Function that performs a single iteration in for the KQ matrix multiplication:\ntemplate \nstatic __dpct_inline__ void flash_attn_tile_iter_KQ(T_vec_dot * const Q_tmp,\n const sycl::half2 * const __restrict__ K_h2,\n T_vec_dot * const KV_tmp,\n const int stride_K2,\n const int k_VKQ_0,\n const int k_VKQ_sup,\n const int k_KQ_0,\n float * KQ_acc) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n constexpr int cpy_nb = ggml_sycl_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n constexpr int ncols = ncols1*ncols2;\n constexpr int cpw = ncols > nwarps ? ncols/nwarps : 1; // Q columns per warp\n constexpr int np = nwarps > ncols ? nwarps/ncols : 1; // number of parallel warps per Q column\n\n flash_attn_tile_load_tile\n (K_h2 + int64_t(k_VKQ_0)*stride_K2 + k_KQ_0/2, KV_tmp, stride_K2, k_VKQ_sup);\n item_ct1.barrier();\n\n#ifdef SYCL_FAST_FP16\n static_assert((nbatch_K/2) % cpy_ne == 0, \"bad nbatch_K\");\n#pragma unroll\n for (int k_KQ_1 = 0; k_KQ_1 < nbatch_K/2; k_KQ_1 += cpy_ne) {\n __dpct_align__(16) sycl::half2 K_k[nbatch_fa / (np * warp_size)][cpy_ne];\n __dpct_align__(16) sycl::half2 Q_k[cpw][cpy_ne];\n#else\n static_assert(nbatch_K % cpy_ne == 0, \"bad nbatch_K\");\n#pragma unroll\n for (int k_KQ_1 = 0; k_KQ_1 < nbatch_K; k_KQ_1 += cpy_ne) {\n __dpct_align__(16) float K_k[nbatch_fa/(np*warp_size)][cpy_ne];\n __dpct_align__(16) float Q_k[cpw][cpy_ne];\n#endif // SYCL_FAST_FP16\n\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) {\n const int i_KQ = i_KQ_0 + (item_ct1.get_local_id(1) % np) * warp_size + item_ct1.get_local_id(2);\n\n#ifdef SYCL_FAST_FP16\n ggml_sycl_memcpy_1(&K_k[i_KQ_0/(np*warp_size)], &KV_tmp[i_KQ*(nbatch_K/2 + cpy_ne) + k_KQ_1]);\n#else\n ggml_sycl_memcpy_1(&K_k[i_KQ_0/(np*warp_size)], &KV_tmp[i_KQ*(nbatch_K + cpy_ne) + k_KQ_1]);\n#endif // SYCL_FAST_FP16\n }\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int jc = jc0 + (item_ct1.get_local_id(1) / np) * cpw;\n\n#ifdef SYCL_FAST_FP16\n ggml_sycl_memcpy_1(&Q_k[jc0], &Q_tmp[jc*(DKQ/2) + k_KQ_0/2 + k_KQ_1]);\n#else\n ggml_sycl_memcpy_1(&Q_k[jc0], &Q_tmp[jc* DKQ + k_KQ_0 + k_KQ_1]);\n#endif // SYCL_FAST_FP16\n }\n\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) {\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n#pragma unroll\n for (int k = 0; k < cpy_ne; ++k) {\n ggml_sycl_mad(KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0], K_k[i_KQ_0/(np*warp_size)][k], Q_k[jc0][k]);\n }\n }\n }\n }\n\n if (k_KQ_0 + nbatch_K < DKQ) {\n item_ct1.barrier(); // Sync not needed on last iteration.\n }\n}\n\n// Function that performs a single iteration of the main loop over up to nbatch_fa tokens.\ntemplate \n/*\nThe total declared local variable size in device function flash_attn_tile_iter exceeds 128 bytes and may cause high register pressure. Consult with your hardware vendor to find the total register size available and adjust the code, or use smaller sub-group size to avoid high register pressure.\n*/\nstatic __dpct_inline__ void flash_attn_tile_iter(T_vec_dot * const Q_tmp,\n const sycl::half2 * const __restrict__ K_h2,\n const sycl::half2 * const __restrict__ V_h2,\n const sycl::half * const __restrict__ mask,\n const sycl::uint3 ne01,\n const float logit_softcap,\n const float slope,\n T_KQ * const KQ,\n T_vec_dot * const KV_tmp,\n const int stride_K2,\n const int stride_V2,\n const int stride_mask,\n float * const KQ_max,\n float * const KQ_sum,\n T_acc * const VKQ,\n const int k_VKQ_0,\n const int k_VKQ_max,\n const int col_Q_0,\n float * KQ_max_new_shared) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n constexpr int cpy_nb = ggml_sycl_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n constexpr int ncols = ncols1*ncols2;\n constexpr int cpw = ncols > nwarps ? ncols/nwarps : 1; // Q columns per warp\n constexpr int np = nwarps > ncols ? nwarps/ncols : 1; // number of parallel warps per Q column\n\n constexpr int DVp = (DV + 2*warp_size - 1) & ~(2*warp_size - 1); // DV padded to multiple of 2*warp_size.\n\n#ifdef SYCL_FAST_FP16\n constexpr int KQ_cs = cpw < 2*cpy_ne ? cpw : 2*cpy_ne;\n#else\n constexpr int KQ_cs = cpw < 1*cpy_ne ? cpw : 1*cpy_ne;\n#endif // SYCL_FAST_FP16\n static_assert(cpw % KQ_cs == 0, \"bad KQ_cs\");\n const int k_VKQ_sup = k_VKQ_max - k_VKQ_0; // k supremum, only smaller k values have valid KV data\n\n float KQ_max_new[cpw];\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n KQ_max_new[jc0] = KQ_max[jc0];\n }\n\n float KQ_acc[nbatch_fa/(np*warp_size) * cpw] = {0.0f}; // Accumulators for KQ matrix multiplication.\n\n // KQ = K @ Q matrix multiplication:\n constexpr int nbatch_K_last = DKQ % nbatch_K;\n#pragma unroll\n for (int k_KQ_0 = 0; k_KQ_0 < DKQ - nbatch_K_last; k_KQ_0 += nbatch_K) {\n flash_attn_tile_iter_KQ(\n Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);\n }\n if (nbatch_K_last > 0) {\n constexpr int k_KQ_0 = DKQ - nbatch_K_last;\n flash_attn_tile_iter_KQ(\n Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);\n }\n\n // Apply logit softcap + mask, update KQ_max:\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int j = fastmodulo(col_Q_0 + (jc0 + (item_ct1.get_local_id(1) / np) * cpw) / ncols2, ne01);\n\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) {\n const int i_KQ = i_KQ_0 + (item_ct1.get_local_id(1) % np) * warp_size + item_ct1.get_local_id(2);\n\n#if defined(SYCL_FAST_FP16) && !defined(GGML_SYCL_F16)\n // Without the v_dot2_f32_f16 instruction there is a higher risk of numerical overflow in the KQ calculation.\n // Therefore, scale down Q values and apply the inverse scale the FP32 KQ values afterwards again.\n KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0] *= 4.0f;\n#endif // defined(SYCL_FAST_FP16) && !defined(GGML_SYCL_F16)\n\n if (use_logit_softcap) {\n KQ_acc[(i_KQ_0 / (np * warp_size)) * cpw + jc0] =\n logit_softcap * sycl::tanh((float) KQ_acc[(i_KQ_0 / (np * warp_size)) * cpw + jc0]);\n }\n\n if (!oob_check || i_KQ < k_VKQ_sup) {\n KQ_acc[(i_KQ_0 / (np * warp_size)) * cpw + jc0] +=\n (ncols2 > 1 || mask) ? slope * sycl::vec(mask[j * stride_mask + k_VKQ_0 + i_KQ])\n .convert()[0] :\n 0.0f;\n\n KQ_max_new[jc0] =\n sycl::fmax((float) KQ_max_new[jc0],\n (float) (KQ_acc[(i_KQ_0 / (np * warp_size)) * cpw + jc0] + FATTN_KQ_MAX_OFFSET));\n }\n }\n\n KQ_max_new[jc0] = warp_reduce_max(KQ_max_new[jc0]);\n }\n\n if constexpr (np == 1) {\n item_ct1.barrier();\n } else {\n static_assert(cpw == 1, \"bad cpw\");\n\n if (item_ct1.get_local_id(2) == 0) {\n KQ_max_new_shared[item_ct1.get_local_id(1)] = KQ_max_new[0];\n }\n item_ct1.barrier();\n KQ_max_new[0] = KQ_max_new_shared[(item_ct1.get_local_id(1) & ~(np - 1)) + item_ct1.get_local_id(2) % np];\n KQ_max_new[0] = warp_reduce_max(KQ_max_new[0]);\n }\n\n // Calculate KQ softmax, write to shared KQ buffer, re-scale VKQ accumulators:\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; jc0 += KQ_cs) {\n#ifdef SYCL_FAST_FP16\n __dpct_align__(16) sycl::half tmp[nbatch_fa / (np * warp_size)][KQ_cs];\n#else\n __dpct_align__(16) float tmp[nbatch_fa/(np*warp_size)][KQ_cs];\n#endif // SYCL_FAST_FP16\n\n#pragma unroll\n for (int jc1 = 0; jc1 < KQ_cs; ++jc1) {\n const int jc = jc0 + jc1;\n\n const float KQ_max_scale = sycl::native::exp((float) (KQ_max[jc] - KQ_max_new[jc]));\n KQ_max[jc] = KQ_max_new[jc];\n\n float KQ_sum_add = 0.0f;\n#pragma unroll\n for (int i0 = 0; i0 < nbatch_fa; i0 += np*warp_size) {\n const float val =\n !oob_check || i0 + (item_ct1.get_local_id(1) % np) * warp_size + item_ct1.get_local_id(2) <\n static_cast(k_VKQ_sup) ?\n sycl::native::exp((float) (KQ_acc[(i0 / (np * warp_size)) * cpw + jc] - KQ_max[jc])) :\n 0.0f;\n KQ_sum_add += val;\n tmp[i0/(np*warp_size)][jc1] = val;\n }\n KQ_sum[jc] = KQ_sum[jc]*KQ_max_scale + KQ_sum_add;\n\n#ifdef SYCL_FAST_FP16\n const sycl::half2 KQ_max_scale_h2 = sycl::half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n VKQ[jc*((DVp/2)/warp_size) + i0/warp_size].x() *= KQ_max_scale_h2.x();\n VKQ[jc*((DVp/2)/warp_size) + i0/warp_size].y() *= KQ_max_scale_h2.y();\n }\n#else\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n VKQ[jc*((DVp/2)/warp_size) + i0/warp_size].x() *= KQ_max_scale;\n VKQ[jc*((DVp/2)/warp_size) + i0/warp_size].y() *= KQ_max_scale;\n }\n#endif // SYCL_FAST_FP16\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < nbatch_fa; i0 += np*warp_size) {\n const int i = i0 + (item_ct1.get_local_id(1) % np) * warp_size + item_ct1.get_local_id(2);\n\n ggml_sycl_memcpy_1(\n KQ + (jc0 / KQ_cs + (item_ct1.get_local_id(1) / np) * (cpw / KQ_cs)) * (nbatch_fa * KQ_cs) + i * KQ_cs,\n tmp[i0 / (np * warp_size)]);\n }\n }\n\n // VKQ = V @ KQ matrix multiplication:\n static_assert(DV <= DKQ, \"bad DV\");\n static_assert(DV % nbatch_K == 0 || (nbatch_K % 3 == 0 && DV % (nbatch_K*2/3) == 0), \"bad nbatch_K\");\n constexpr int nbatch_V = (DV % nbatch_K == 0 ? nbatch_K : nbatch_K*2/3) * nbatch_fa / DV; // Number of V columns that fit in SRAM for K.\n static_assert(nbatch_fa % nbatch_V == 0, \"bad nbatch_V\");\n static_assert(nbatch_V % np == 0, \"bad nbatch_V\");\n#pragma unroll\n for (int k0 = 0; k0 < nbatch_fa; k0 += nbatch_V) {\n flash_attn_tile_load_tile\n (V_h2 + int64_t(k_VKQ_0 + k0)*stride_V2, KV_tmp, stride_V2, k_VKQ_sup - k0);\n item_ct1.barrier();\n\n#ifdef SYCL_FAST_FP16\n#pragma unroll\n for (int k1 = 0; k1 < nbatch_V; k1 += np) {\n __dpct_align__(16) sycl::half2 V_k[(DVp / 2) / warp_size];\n __dpct_align__(16) sycl::half2 KQ_k[cpw];\n\n constexpr int cpy_ne_D = cpy_ne/2 < (DVp/2)/warp_size ? cpy_ne/2 : (DVp/2)/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size*cpy_ne_D) {\n ggml_sycl_memcpy_1(&V_k[i0 / warp_size],\n &KV_tmp[(k1 + item_ct1.get_local_id(1) % np) * (DV / 2) + i0 +\n item_ct1.get_local_id(2) * cpy_ne_D]);\n }\n#pragma unroll\n for (int jc_VKQ_0 = 0; jc_VKQ_0 < cpw; jc_VKQ_0 += KQ_cs) {\n const int jc_KQ = jc_VKQ_0 / KQ_cs + (item_ct1.get_local_id(1) / np) * (cpw / KQ_cs);\n\n __dpct_align__(16) sycl::half tmp[KQ_cs];\n ggml_sycl_memcpy_1(\n &tmp, KQ + jc_KQ * (nbatch_fa * KQ_cs) + (k0 + k1 + item_ct1.get_local_id(1) % np) * KQ_cs);\n#pragma unroll\n for (int jc_VKQ_1 = 0; jc_VKQ_1 < KQ_cs; ++jc_VKQ_1) {\n KQ_k[jc_VKQ_0 + jc_VKQ_1] = sycl::half2(tmp[jc_VKQ_1]);\n }\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n#pragma unroll\n for (int jc_VKQ_0 = 0; jc_VKQ_0 < cpw; ++jc_VKQ_0) {\n VKQ[jc_VKQ_0*((DVp/2)/warp_size) + i0/warp_size].x() +=\n V_k[i0/warp_size].x()*KQ_k[jc_VKQ_0].x();\n VKQ[jc_VKQ_0*((DVp/2)/warp_size) + i0/warp_size].y() +=\n V_k[i0/warp_size].y()*KQ_k[jc_VKQ_0].y();\n }\n }\n }\n#else\n#pragma unroll\n for (int k1 = 0; k1 < nbatch_V; k1 += np) {\n __dpct_align__(16) sycl::float2 V_k[(DVp/2)/warp_size];\n __dpct_align__(16) float KQ_k[cpw];\n\n constexpr int cpy_ne_D = cpy_ne < DVp/warp_size ? cpy_ne : DVp/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp; i0 += warp_size*cpy_ne_D) {\n ggml_sycl_memcpy_1(&V_k[i0/(2*warp_size)], &KV_tmp[(k1 + item_ct1.get_local_id(1) % np)*DV + i0 + item_ct1.get_local_id(2)*cpy_ne_D]);\n }\n#pragma unroll\n for (int jc_VKQ_0 = 0; jc_VKQ_0 < cpw; jc_VKQ_0 += KQ_cs) {\n const int jc_KQ = jc_VKQ_0/KQ_cs + (item_ct1.get_local_id(1) / np)*(cpw/KQ_cs);\n\n ggml_sycl_memcpy_1(\n &KQ_k[jc_VKQ_0], KQ + jc_KQ*(nbatch_fa*KQ_cs) + (k0 + k1 + item_ct1.get_local_id(1) % np)*KQ_cs);\n }\n\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n#pragma unroll\n for (int jc_VKQ_0 = 0; jc_VKQ_0 < cpw; ++jc_VKQ_0) {\n VKQ[jc_VKQ_0*((DVp/2)/warp_size) + i0/warp_size].x() += V_k[i0/warp_size].x()*KQ_k[jc_VKQ_0];\n VKQ[jc_VKQ_0*((DVp/2)/warp_size) + i0/warp_size].y() += V_k[i0/warp_size].y()*KQ_k[jc_VKQ_0];\n }\n }\n }\n#endif // SYCL_FAST_FP16\n item_ct1.barrier();\n }\n}\n\ntemplate // D == head size\n/*\nThe total declared local variable size in device function flash_attn_tile exceeds 128 bytes and may cause high register pressure. Consult with your hardware vendor to find the total register size available and adjust the code, or use smaller sub-group size to avoid high register pressure.\n*/\nstatic void flash_attn_tile(const char * Q,\n const char * K,\n const char * V,\n const char * mask,\n const char * sinks,\n const int * KV_max,\n float * dst,\n sycl::float2 * dst_meta,\n const float scale,\n const float max_bias,\n const float m0,\n const float m1,\n const uint32_t n_head_log2,\n const float logit_softcap,\n const int32_t ne00,\n const sycl::uint3 ne01,\n const int32_t ne02,\n const int32_t ne03,\n const int32_t nb01,\n const int32_t nb02,\n const int32_t nb03,\n const int32_t ne10,\n const int32_t ne11,\n const int32_t ne12,\n const int32_t ne13,\n const int32_t nb11,\n const int32_t nb12,\n const int64_t nb13,\n const int32_t nb21,\n const int32_t nb22,\n const int64_t nb23,\n const int32_t ne31,\n const int32_t ne32,\n const int32_t ne33,\n const int32_t nb31,\n const int32_t nb32,\n const int64_t nb33) {\n#ifdef SYCL_FLASH_ATTN\n // Skip unused kernel variants for faster compilation:\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n if ((use_logit_softcap && !(DV == 128 || DV == 256))) {\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n return;\n }\n\n static_assert(ggml_sycl_fattn_tile_get_config(DKQ, DV, ncols1*ncols2) != 0, \"kernel config not defined\");\n\n constexpr int ncols = ncols1*ncols2;\n\n constexpr int nwarps = ggml_sycl_fattn_tile_get_nthreads (DKQ, DV, ncols1*ncols2) / warp_size;\n constexpr int nbatch_fa = ggml_sycl_fattn_tile_get_nbatch_fa(DKQ, DV, ncols1*ncols2);\n constexpr int nbatch_K = ggml_sycl_fattn_tile_get_nbatch_K (DKQ, DV, ncols1*ncols2);\n\n // In this kernel Q, K, V are matrices while i, j, k are matrix indices.\n\n const int col_Q_0 = item_ct1.get_group(2) * ncols1; // Index of the first Q column for this SYCL block to work on.\n\n const int sequence = item_ct1.get_group(0) / (ne02 / ncols2);\n const int head0 = item_ct1.get_group(0) * ncols2 - sequence * ne02; // == item_ct1.get_group(0) % (ne02/ncols2)\n const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n const float * Q_f = (const float *) (Q + nb03*sequence + nb02* head0);\n const sycl::half2 * K_h2 = (const sycl::half2 *) (K + nb13 * sequence + nb12 * (head0 / gqa_ratio));\n const sycl::half2 * V_h2 =\n (const sycl::half2 *) (V + nb23 * sequence + nb22 * (head0 / gqa_ratio)); // K and V have same shape\n\n const sycl::half * maskh = mask ? (const sycl::half *) (mask + nb33 * (sequence % ne33)) : nullptr;\n\n const int stride_K2 = nb11 / sizeof(sycl::half2);\n const int stride_V2 = nb21 / sizeof(sycl::half2);\n const int stride_mask = nb31 / sizeof(sycl::half);\n\n const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, head0, n_head_log2, m0, m1) : 1.0f;\n\n constexpr int cpy_nb = ggml_sycl_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n constexpr int cpw = ncols > nwarps ? ncols/nwarps : 1; // Q columns per warp.\n constexpr int np = nwarps > ncols ? nwarps/ncols : 1; // Number of parallel warps per Q column.\n\n static_assert(cpw == 1 || np == 1, \"bad cpw / np\");\n static_assert(nbatch_fa % (np*warp_size) == 0, \"nbatch_fa % (np*warp_size) != 0\");\n\n constexpr int DKQp = (DKQ + 2*warp_size - 1) & ~(2*warp_size - 1); // DKQ padded to multiple of 2*warp_size.\n constexpr int DVp = (DV + 2*warp_size - 1) & ~(2*warp_size - 1); // DV padded to multiple of 2*warp_size.\n\n // Q_tmp == SRAM buffer to hold Q data for the entire lifetime of the kernel.\n // KV_tmp == SRAM buffer to hold fragments of K/V data while iterating over ne11.\n // KV_tmp is padded to avoid memory conflicts for K (cpy_ne) and OOB accesses for V (DVp-DV).\n // KQ == SRAM buffer to hold KQ fragments between KQ and VKQ matrix multiplications.\n // VKQ == Accumulators in registers for the final VKQ result.\n\n\n#ifdef SYCL_FAST_FP16\n constexpr size_t lsm_size1 = ncols * DKQ/2 ;\n constexpr size_t lsm_size2 = nbatch_fa * (nbatch_K/2 + cpy_ne) + DVp-DV ;\n constexpr size_t lsm_size3 = ncols * nbatch_fa;\n constexpr size_t lsm_size4 = nwarps;\n\n constexpr size_t local_share_mem_size = lsm_size1 * sizeof(sycl::half2) +\n lsm_size2 * sizeof(sycl::half2) +\n lsm_size3 * sizeof(sycl::half) +\n lsm_size4 * sizeof(float);\n\n syclex::work_group_static lsm;\n\n sycl::half2 *Q_tmp = (sycl::half2 *)&lsm;\n sycl::half2 *KV_tmp = (sycl::half2*)(Q_tmp +lsm_size1);\n sycl::half *KQ = (sycl::half *)(KV_tmp+lsm_size2);\n float *KQ_max_new_shared = (float *)(KQ+lsm_size3);\n\n __dpct_align__(16) sycl::half2 VKQ[cpw * ((DVp / 2) / warp_size)] = {\n { 0.0f, 0.0f }\n };\n#else\n constexpr size_t lsm_size1 = ncols * DKQ ;\n constexpr size_t lsm_size2 = nbatch_fa * (nbatch_K + cpy_ne) + DVp-DV;\n constexpr size_t lsm_size3 = ncols * nbatch_fa;\n constexpr size_t lsm_size4 = nwarps;\n\n constexpr size_t local_share_mem_size = (lsm_size1 + lsm_size2 +lsm_size3 + lsm_size4) * sizeof(float);\n\n syclex::work_group_static lsm;\n\n float *Q_tmp = (float *)&lsm;\n float *KV_tmp = Q_tmp +lsm_size1;\n float *KQ = KV_tmp+lsm_size2;\n float *KQ_max_new_shared = KQ+lsm_size3;\n\n __dpct_align__(16) sycl::float2 VKQ[cpw * ((DVp/2)/warp_size)] = {{0.0f, 0.0f}};\n\n\n#endif // SYCL_FAST_FP16\n\n float KQ_max[cpw] = {};\n\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n KQ_max[j0/nwarps] = -FLT_MAX/2.0f;\n }\n float KQ_sum[cpw] = {0.0f};\n\n // Load Q data, convert to FP16 if fast:\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int jc = jc0 + (item_ct1.get_local_id(1) / np) * cpw;\n\n const int j = jc / ncols2;\n const int c = jc % ncols2;\n\n constexpr int cpy_ne_D = cpy_ne < DKQp/warp_size ? cpy_ne : DKQp/warp_size;\n\n#pragma unroll\n for (int i0 = 0; i0 < DKQp; i0 += np*warp_size*cpy_ne_D) {\n if (i0 + np * warp_size * cpy_ne_D <= DKQ ||\n i0 + (item_ct1.get_local_id(1) % np) * (warp_size * cpy_ne_D) + item_ct1.get_local_id(2) * cpy_ne_D <\n DKQ) {\n __dpct_align__(16) float tmp_f[cpy_ne_D] = { 0.0f };\n ggml_sycl_memcpy_1(\n tmp_f, &Q_f[c * (nb02 / sizeof(float)) + fastmodulo(col_Q_0 + j, ne01) * (nb01 / sizeof(float)) +\n i0 + (item_ct1.get_local_id(1) % np) * (warp_size * cpy_ne_D) +\n item_ct1.get_local_id(2) * cpy_ne_D]);\n\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; ++i1) {\n tmp_f[i1] *= scale;\n }\n\n#ifdef SYCL_FAST_FP16\n __dpct_align__(16) sycl::half2 tmp_h2[cpy_ne_D / 2];\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; i1 += 2) {\n tmp_h2[i1/2] = make_half2(tmp_f[i1 + 0], tmp_f[i1 + 1]);\n#if defined(SYCL_FAST_FP16) && !defined(GGML_SYCL_F16)\n // Without the v_dot2_f32_f16 instruction there is a higher risk of numerical overflow in the KQ calculation.\n // Therefore, scale down Q values and apply the inverse scale the FP32 KQ values afterwards again.\n tmp_h2[i1 / 2] *= sycl::half2(0.25f, 0.25f);\n#endif // defined(SYCL_FAST_FP16) && !defined(GGML_SYCL_F16)\n }\n ggml_sycl_memcpy_1(\n &Q_tmp[jc * (DKQ / 2) + i0 / 2 + (item_ct1.get_local_id(1) % np) * (warp_size * cpy_ne_D / 2) +\n item_ct1.get_local_id(2) * (cpy_ne_D / 2)],\n tmp_h2);\n#else\n ggml_sycl_memcpy_1(\n &Q_tmp[jc* DKQ + i0 + (item_ct1.get_local_id(1) % np)*(warp_size*cpy_ne_D) + item_ct1.get_local_id(2)* cpy_ne_D],\n tmp_f);\n#endif // SYCL_FAST_FP16\n }\n }\n }\n\n item_ct1.barrier();\n\n // Main loop over KV cache:\n const int k_VKQ_max = KV_max ? KV_max[sequence * item_ct1.get_group_range(2) + item_ct1.get_group(2)] : ne11;\n if (ncols2 == 1) {\n // Branch with out-of-bounds checks.\n int k_VKQ_0 = item_ct1.get_group(1) * nbatch_fa;\n while (k_VKQ_0 < k_VKQ_max - nbatch_fa) {\n constexpr bool oob_check = false;\n flash_attn_tile_iter(Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, stride_K2,\n stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0,\n KQ_max_new_shared);\n k_VKQ_0 += item_ct1.get_group_range(1) * nbatch_fa;\n }\n if (k_VKQ_0 < k_VKQ_max) {\n constexpr bool oob_check = true;\n flash_attn_tile_iter(Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, stride_K2,\n stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0,\n KQ_max_new_shared);\n }\n } else {\n // Branch without out-of-bounds checks.\n for (int k_VKQ_0 = item_ct1.get_group(1) * nbatch_fa; k_VKQ_0 < k_VKQ_max;\n k_VKQ_0 += item_ct1.get_group_range(1) * nbatch_fa) {\n\n constexpr bool oob_check = false;\n flash_attn_tile_iter(Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, stride_K2,\n stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0,\n KQ_max_new_shared);\n }\n }\n\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n KQ_sum[jc0] = warp_reduce_sum(KQ_sum[jc0]);\n }\n\n if constexpr (np > 1) {\n static_assert(cpw == 1, \"bad cpw\");\n static_assert(nbatch_fa*nbatch_K >= nwarps*DVp, \"KV_tmp too small\");\n\n#ifdef SYCL_FAST_FP16\n sycl::half2 * VKQ_combine = (sycl::half2 *) KV_tmp;\n#else\n float * VKQ_combine = (float *) KV_tmp;\n#endif // SYCL_FAST_FP16\n\n float * KQ_sum_combine = (float *) Q_tmp;\n\n if (item_ct1.get_local_id(1) % np != 0) {\n\n#ifdef SYCL_FAST_FP16\n constexpr int cpy_ne_D = cpy_ne < (DVp/2)/warp_size ? cpy_ne : (DVp/2)/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size*cpy_ne_D) {\n ggml_sycl_memcpy_1(\n &VKQ_combine[item_ct1.get_local_id(1) * (DVp / 2) + i0 + item_ct1.get_local_id(2) * cpy_ne_D],\n &VKQ[i0 / warp_size]);\n }\n#else\n\n constexpr int cpy_ne_D = cpy_ne < DVp/warp_size ? cpy_ne : DVp/warp_size;\n\n#pragma unroll\n for (int i0 = 0; i0 < DVp; i0 += warp_size*cpy_ne_D) {\n ggml_sycl_memcpy_1(\n &VKQ_combine[item_ct1.get_local_id(1)*DVp + i0 + item_ct1.get_local_id(2)*cpy_ne_D], ((const float *) VKQ) + i0/warp_size);\n }\n#endif // SYCL_FAST_FP16\n\n if (item_ct1.get_local_id(2) == 0) {\n KQ_sum_combine[item_ct1.get_local_id(1)] = KQ_sum[0];\n }\n return;\n }\n\n item_ct1.barrier();\n\n#pragma unroll\n for (int ip = 1; ip < np; ++ip) {\n#ifdef SYCL_FAST_FP16\n constexpr int cpy_ne_D = cpy_ne < (DVp/2)/warp_size ? cpy_ne : (DVp/2)/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size*cpy_ne_D) {\n __dpct_align__(16) sycl::half2 tmp[cpy_ne_D];\n ggml_sycl_memcpy_1(tmp, &VKQ_combine[(item_ct1.get_local_id(1) + ip) * (DVp / 2) + i0 +\n item_ct1.get_local_id(2) * cpy_ne_D]);\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; ++i1) {\n VKQ[i0/warp_size + i1] += tmp[i1];\n }\n }\n#else\n constexpr int cpy_ne_D = cpy_ne < DVp/warp_size ? cpy_ne : DVp/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp; i0 += warp_size*cpy_ne_D) {\n __dpct_align__(16) float tmp[cpy_ne_D];\n ggml_sycl_memcpy_1(tmp, &VKQ_combine[(item_ct1.get_local_id(1) + ip)*DVp + i0 + item_ct1.get_local_id(2)*cpy_ne_D]);\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; ++i1) {\n ((float *)VKQ)[i0/warp_size + i1] += tmp[i1];\n }\n }\n#endif // SYCL_FAST_FP16\n\n KQ_sum[0] += KQ_sum_combine[item_ct1.get_local_id(1) + ip];\n }\n }\n\n // Attention sink: adjust KQ max and sum only for the first of all parallel blocks:\n if (sinks && item_ct1.get_group(1) == 0) {\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int jc = jc0 + (item_ct1.get_local_id(1) / np) * cpw;\n const float sink = ((const float *) sinks)[head0 + jc % ncols2];\n\n float KQ_max_new_j = sycl::fmax((float) KQ_max[jc0], sink);\n const float KQ_max_scale = sycl::native::exp((float) (KQ_max[jc0] - KQ_max_new_j));\n KQ_max[jc0] = KQ_max_new_j;\n\n const float val = sycl::native::exp((float) (sink - KQ_max[jc0]));\n KQ_sum[jc0] = KQ_sum[jc0]*KQ_max_scale + val;\n\n#ifdef SYCL_FAST_FP16\n const sycl::half2 KQ_max_scale_h2 = sycl::half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n VKQ[jc0*((DVp/2)/warp_size) + i0/warp_size] *= KQ_max_scale_h2;\n }\n#else\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size) {\n VKQ[jc0*((DVp/2)/warp_size) + i0/warp_size].x() *= KQ_max_scale;\n VKQ[jc0*((DVp/2)/warp_size) + i0/warp_size].y() *= KQ_max_scale;\n }\n#endif // SYCL_FAST_FP16\n }\n }\n\n // Write back results:\n#pragma unroll\n for (int jc0 = 0; jc0 < cpw; ++jc0) {\n const int jc = jc0 + (item_ct1.get_local_id(1) / np) * cpw;\n\n const int j = jc / ncols2;\n const int c = jc % ncols2;\n\n if (ncols1 > 1 && col_Q_0 + j >= int(ne01.z())) {\n return;\n }\n\n const float scale = item_ct1.get_group_range(1) == 1 ? 1.0f / KQ_sum[jc0] : 1.0f;\n\n const int j_dst_unrolled =\n ((sequence * int(ne01.z()) + col_Q_0 + j) * ne02 + head0 + c) * item_ct1.get_group_range(1) +\n item_ct1.get_group(1);\n\n#ifdef SYCL_FAST_FP16\n constexpr int cpy_ne_D = cpy_ne/2 < (DVp/2)/warp_size ? cpy_ne/2 : (DVp/2)/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp/2; i0 += warp_size*cpy_ne_D) {\n __dpct_align__(16) sycl::float2 tmp[cpy_ne_D];\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D; ++i1) {\n tmp[i1] = VKQ[jc0 * ((DVp / 2) / warp_size) + i0 / warp_size + i1]\n .template convert();\n tmp[i1].x() *= scale;\n tmp[i1].y() *= scale;\n }\n if (i0 + warp_size * cpy_ne_D <= DV / 2 || i0 + item_ct1.get_local_id(2) * cpy_ne_D < DV / 2) {\n ggml_sycl_memcpy_1(\n &dst[j_dst_unrolled * DV + 2 * i0 + item_ct1.get_local_id(2) * (2 * cpy_ne_D)], tmp);\n }\n }\n#else\n constexpr int cpy_ne_D = cpy_ne < DVp/warp_size ? cpy_ne : DVp/warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < DVp; i0 += warp_size*cpy_ne_D) {\n if (i0 + warp_size*cpy_ne_D <= DV || i0 + item_ct1.get_local_id(2)*cpy_ne_D < DV) {\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne_D/2; ++i1) {\n VKQ[jc0*((DVp/2)/warp_size) + i0/(2*warp_size) + i1].x() *= scale;\n VKQ[jc0*((DVp/2)/warp_size) + i0/(2*warp_size) + i1].y() *= scale;\n }\n ggml_sycl_memcpy_1(\n &dst[j_dst_unrolled*DV + i0 + item_ct1.get_local_id(2)*cpy_ne_D],\n &VKQ[jc0*((DVp/2)/warp_size) + i0/(2*warp_size)]);\n }\n }\n#endif // SYCL_FAST_FP16\n\n if (item_ct1.get_group_range(1) != 1 && item_ct1.get_local_id(2) == 0) {\n dst_meta[j_dst_unrolled] = make_float2(KQ_max[jc0], KQ_sum[jc0]);\n }\n }\n#else\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n#endif // SYCL_FLASH_ATTN\n}\n\ntemplate \nstatic void launch_fattn_tile_switch_ncols1(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * Q = dst->src[0];\n\n const int id = ggml_sycl_get_device();\n const int cc = ggml_sycl_info().devices[id].cc;\n const int warp_size = WARP_32_SIZE; //can't support WARP_16_SIZE\n\n constexpr size_t nbytes_shared = 0;\n\n if constexpr (DV <= 256) {\n if (Q->ne[1] > 16/ncols2) {\n constexpr int cols_per_block = 32;\n const int nwarps = ggml_sycl_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_sycl_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n launch_fattn, warp_size>\n (ctx, dst, nwarps, nbytes_shared, nbatch_fa, true, true, false);\n return;\n }\n }\n\n if (Q->ne[1] > 8/ncols2) {\n constexpr int cols_per_block = 16;\n const int nwarps = ggml_sycl_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_sycl_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n launch_fattn, warp_size>\n (ctx, dst, nwarps, nbytes_shared, nbatch_fa, true, true, false);\n return;\n }\n\n if constexpr (ncols2 <= 8) {\n if (Q->ne[1] > 4/ncols2) {\n constexpr int cols_per_block = 8;\n const int nwarps = ggml_sycl_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_sycl_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n launch_fattn, warp_size>\n (ctx, dst, nwarps, nbytes_shared, nbatch_fa, true, true, false);\n return;\n }\n }\n\n if constexpr (ncols2 <= 4) {\n if (Q->ne[1] > 2/ncols2) {\n constexpr int cols_per_block = 4;\n const int nwarps = ggml_sycl_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_sycl_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n launch_fattn, warp_size>\n (ctx, dst, nwarps, nbytes_shared, nbatch_fa, true, true, false);\n return;\n }\n }\n\n if constexpr (ncols2 <= 2) {\n constexpr int cols_per_block = 2;\n const int nwarps = ggml_sycl_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;\n const int nbatch_fa = ggml_sycl_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);\n launch_fattn, warp_size>\n (ctx, dst, nwarps, nbytes_shared, nbatch_fa, true, true, false);\n return;\n }\n\n GGML_ABORT(\"fatal error\");\n}\n\ntemplate \nstatic void launch_fattn_tile_switch_ncols2(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * KQV = dst;\n const ggml_tensor * Q = dst->src[0];\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * mask = dst->src[3];\n\n float max_bias = 0.0f;\n memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));\n\n GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);\n const int gqa_ratio = Q->ne[2] / K->ne[2];\n\n // On NVIDIA (Pascal and older) the GQA optimizations seem to be detrimental in some cases.\n // However, for DKQ == 576, DV == 512 only the kernel variant with GQA optimizations is implemented.\n //const bool nvidia = GGML_SYCL_CC_IS_NVIDIA(ggml_sycl_info().devices[ggml_sycl_get_device()].cc);\n const int gqa_limit = gqa_ratio <= 4 && DV <= 256 ? 16 : INT_MAX;\n const bool use_gqa_opt = mask && max_bias == 0.0f && Q->ne[1] <= gqa_limit && K->ne[1] % FATTN_KQ_STRIDE == 0;\n\n if constexpr (DV == 512) {\n if (use_gqa_opt && gqa_ratio % 16 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n if (use_gqa_opt && gqa_ratio % 4 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n }\n\n if constexpr (DV <= 256) {\n if (use_gqa_opt && gqa_ratio % 8 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n\n if (use_gqa_opt && gqa_ratio % 4 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n\n if (use_gqa_opt && gqa_ratio % 2 == 0) {\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n\n launch_fattn_tile_switch_ncols1(ctx, dst);\n return;\n }\n GGML_ABORT(\"fatal error\");\n}\n\ntemplate \nvoid ggml_sycl_flash_attn_ext_tile_case(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * KQV = dst;\n\n float logit_softcap;\n memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n if (logit_softcap == 0.0f) {\n constexpr bool use_logit_softcap = false;\n launch_fattn_tile_switch_ncols2(ctx, dst);\n } else {\n constexpr bool use_logit_softcap = true;\n launch_fattn_tile_switch_ncols2(ctx, dst);\n }\n}\n\nvoid ggml_sycl_flash_attn_ext_tile(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\n#define DECL_FATTN_TILE_CASE(DKQ, DV) \\\n template void ggml_sycl_flash_attn_ext_tile_case \\\n (ggml_backend_sycl_context & ctx, ggml_tensor * dst) \\\n\nextern DECL_FATTN_TILE_CASE( 40, 40);\nextern DECL_FATTN_TILE_CASE( 64, 64);\nextern DECL_FATTN_TILE_CASE( 72, 72);\nextern DECL_FATTN_TILE_CASE( 80, 80);\nextern DECL_FATTN_TILE_CASE( 96, 96);\nextern DECL_FATTN_TILE_CASE(112, 112);\nextern DECL_FATTN_TILE_CASE(128, 128);\nextern DECL_FATTN_TILE_CASE(256, 256);\nextern DECL_FATTN_TILE_CASE(576, 512);\n\n" }, { "path": "src/ggml-sycl/fattn-vec.hpp", "content": "#ifndef GGML_SYCL_FATTN_VEC_HPP\n#define GGML_SYCL_FATTN_VEC_HPP\n\n#include \n#include \n#include \n#include \n\n#include \"dpct/helper.hpp\"\n#include \"common.hpp\"\n#include \"ggml.h\"\n#include \"fattn-common.hpp\"\n#include \n#include \n\nnamespace syclex = sycl::ext::oneapi::experimental;\n\nstatic int ggml_sycl_fattn_vec_get_nthreads_host(const int cc) {\n return 128;\n GGML_UNUSED(cc);\n}\n\nstatic constexpr int ggml_sycl_fattn_vec_get_nthreads_device() {\n return 128;\n}\n\n// Currenlty llvm with the amdgcn target dose not support unrolling loops\n// that contain a break that can not be resolved at compile time.\n#ifdef __clang__\n#pragma clang diagnostic push\n#pragma clang diagnostic ignored \"-Wpass-failed\"\n#endif // __clang__\n\ntemplate // D == head size\nstatic void flash_attn_ext_vec(const char* __restrict__ Q,\n const char* __restrict__ K,\n const char* __restrict__ V,\n const char* __restrict__ mask,\n const char* __restrict__ sinks,\n const int* __restrict__ KV_max,\n float* __restrict__ dst,\n sycl::float2* __restrict__ dst_meta,\n const float scale,\n const float max_bias,\n const float m0,\n const float m1,\n const uint32_t n_head_log2,\n const float logit_softcap,\n const int32_t ne00,\n const sycl::uint3 ne01,\n const int32_t ne02,\n const int32_t ne03,\n const int32_t nb01,\n const int32_t nb02,\n const int32_t nb03,\n const int32_t ne10,\n const int32_t ne11,\n const int32_t ne12,\n const int32_t ne13,\n const int32_t nb11,\n const int32_t nb12,\n const int64_t nb13,\n const int32_t nb21,\n const int32_t nb22,\n const int64_t nb23,\n const int32_t ne31,\n const int32_t ne32,\n const int32_t ne33,\n const int32_t nb31,\n const int32_t nb32,\n const int64_t nb33) {\n#ifdef SYCL_FLASH_ATTN\n // Skip unused kernel variants for faster compilation:\n\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n if (use_logit_softcap && !(D == 128 || D == 256)) {\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n return;\n }\n\n //In this kernel Q, K, V are matrices while i, j, k are matrix indices.\n\n constexpr int cpy_nb = ggml_sycl_get_max_cpy_bytes();\n constexpr int cpy_ne = cpy_nb / 4;\n\n constexpr int nthreads_KQ_q = (D/4 < warp_size ? D/4 : warp_size);\n constexpr int nthreads_V_q = (D/4 < warp_size ? D/4 : warp_size);\n\n constexpr int nthreads = ggml_sycl_fattn_vec_get_nthreads_device();\n constexpr int nthreads_KQ = type_K == GGML_TYPE_F16 ? 128 / cpy_nb : nthreads_KQ_q;\n constexpr int nthreads_V = type_V == GGML_TYPE_F16 ? 128 / cpy_nb : nthreads_V_q;\n\n static_assert(warp_size % nthreads_KQ == 0, \"bad nthreads_K\");\n static_assert(warp_size % nthreads_V == 0, \"bad nthreads_V\");\n\n constexpr int V_rows_per_thread = type_V == GGML_TYPE_F16 ? 2*cpy_ne : 4;\n constexpr int V_cols_per_iter = warp_size / nthreads_V;\n\n constexpr vec_dot_KQ_t vec_dot_KQ = get_vec_dot_KQ();\n constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16;\n#ifdef GGML_SYCL_F16\n constexpr dequantize_V_t dequantize_V = get_dequantize_V();\n#else\n constexpr dequantize_V_t dequantize_V = get_dequantize_V();\n#endif // GGML_SYCL_F16\n\n const int ic0 = item_ct1.get_group(2) * ncols; // Index of the Q/QKV column to work on.\n\n const int sequence = item_ct1.get_group(0) / ne02;\n const int head = item_ct1.get_group(0) - sequence * ne02;\n const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n Q += nb03*sequence + nb02* head + nb01*ic0;\n K += nb13*sequence + nb12*(head / gqa_ratio);\n V += nb23*sequence + nb22*(head / gqa_ratio);\n\n const sycl::half * maskh = (const sycl::half *) (mask + nb33 * (sequence % ne33) + nb31 * ic0);\n\n const float slope = get_alibi_slope(max_bias, head, n_head_log2, m0, m1);\n\n static_assert(D % (2*warp_size) == 0, \"D not divisible by 2*warp_size == 64.\");\n constexpr int nwarps = nthreads / warp_size;\n const int tid = warp_size * item_ct1.get_local_id(1) + item_ct1.get_local_id(2);\n __builtin_assume(tid < nthreads);\n\n constexpr int ne_KQ = ncols*D;\n constexpr int ne_combine = nwarps*V_cols_per_iter*D;\n\n constexpr size_t lsm_size1 = ncols * warp_size;\n constexpr size_t lsm_size2 = ncols * warp_size;\n#ifdef GGML_SYCL_F16\n sycl::half2 VKQ[ncols][(D / 2) / nthreads_V] = { { { 0.0f, 0.0f } } };\n constexpr size_t lsm_size3 = (ne_KQ > ne_combine ? ne_KQ : ne_combine);\n constexpr size_t local_share_mem_size = (lsm_size1 + lsm_size2)*sizeof(float) + lsm_size3*sizeof(sycl::half);\n\n syclex::work_group_static lsm;\n\n float *KQ_max_shared = (float *)&lsm;\n float *KQ_sum_shared = KQ_max_shared+lsm_size1;\n sycl::half* KQ = (sycl::half*)(KQ_sum_shared + lsm_size2);\n\n\n#else\n sycl::float2 VKQ[ncols][(D/2)/nthreads_V] = {{{0.0f, 0.0f}}};\n\n constexpr size_t lsm_size3 = (ne_KQ > ne_combine ? ne_KQ : ne_combine);\n constexpr size_t local_share_mem_size = (lsm_size1 + lsm_size2 + lsm_size3)*sizeof(float);\n\n\n syclex::work_group_static lsm;\n float *KQ_max_shared = (float *)&lsm;\n float *KQ_sum_shared = KQ_max_shared+lsm_size1;\n float* KQ = KQ_sum_shared + lsm_size2;\n\n#endif // GGML_SYCL_F16\n\n float KQ_max[ncols];\n float KQ_sum[ncols];\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n KQ_max[j] = -FLT_MAX/2.0f;\n KQ_sum[j] = 0.0f;\n }\n\n // Convert Q to float2 (f16 K) or q8_1 (quantized K) and store in registers:\n#ifdef GGML_SYCL_F16\n sycl::half2 Q_reg[ncols][(D / 2) / nthreads_KQ] = {{{0.0f, 0.0f}}}; // Will be initialized completely.\n#else\n sycl::float2 Q_reg[ncols][(D/2)/nthreads_KQ] = {{{0.0f, 0.0f}}}; // May be only partially initialized.\n#endif // GGML_SYCL_F16\n int Q_i32[ncols][1 > D/(sizeof(int)*nthreads_KQ) ? 1 : D/(sizeof(int)*nthreads_KQ)];\n sycl::float2 Q_ds[ncols][1 > D / (sizeof(int) * nthreads_KQ) ? 1 : D / (sizeof(int) * nthreads_KQ)];\n if constexpr (Q_q8_1) {\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j = j0 + item_ct1.get_local_id(1);\n\n if (j0 + nwarps > ncols && j >= ncols) {\n break;\n }\n\n // Reuse KQ as temporary storage for converting Q to q8_1:\n int * tmp_q_i32 = (int *) &KQ[j*D];\n sycl::float2 * tmp_q_ds = (sycl::float2 *) (tmp_q_i32 + D / sizeof(int));\n\n // Set memory to zero if out of bounds:\n if (ncols > 1 && ic0 + j >= int(ne01.z())) {\n#pragma unroll\n for (int i0 = 0; i0 < int(D/sizeof(int)); i0 += warp_size) {\n const int i = i0 + item_ct1.get_local_id(2);\n\n if (i0 + warp_size <= int(D/sizeof(int)) || i < int(D/sizeof(int))) {\n tmp_q_i32[i] = 0;\n }\n }\n if (item_ct1.get_local_id(2) < D/QK8_1) {\n tmp_q_ds[item_ct1.get_local_id(2)] = sycl::float2(0.0f, 0.0f);\n }\n } else {\n const float * Q_f = (const float *) (Q + j*nb01);\n constexpr int nthreads_quantize = D/sizeof(int) < warp_size ? D/sizeof(int) : warp_size;\n#pragma unroll\n for (int i0 = 0; i0 < int(D/sizeof(int)); i0 += nthreads_quantize) {\n quantize_q8_1_to_shared\n (Q_f + i0*sizeof(int), scale, tmp_q_i32 + i0, tmp_q_ds + i0/QI8_1);\n }\n }\n }\n\n\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n int * tmp_q_i32 = (int *) &KQ[j*D];\n sycl::float2 * tmp_q_ds = (sycl::float2 *) (tmp_q_i32 + D / sizeof(int));\n\n#pragma unroll\n for (int i0 = 0; i0 < int(D/sizeof(int)); i0 += nthreads_KQ) {\n const int i =\n i0 + (nthreads_KQ == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads_KQ);\n\n Q_i32[j][i0/nthreads_KQ] = tmp_q_i32[i];\n Q_ds[j][i0/nthreads_KQ] = tmp_q_ds[i/QI8_1];\n }\n }\n\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n } else {\n#ifdef GGML_SYCL_F16\n const sycl::half2 scale_h2 = sycl::half2(scale, scale);\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n const sycl::float2 * Q_j = (const sycl::float2 *) (Q + j * nb01);\n#pragma unroll\n for (int i0 = 0; i0 < D/2; i0 += nthreads_KQ*cpy_ne) {\n const int i = i0 + (nthreads_KQ == warp_size ? item_ct1.get_local_id(2) :\n item_ct1.get_local_id(2) % nthreads_KQ) *\n cpy_ne;\n\n sycl::float2 tmp[cpy_ne] = {\n { 0.0f, 0.0f }\n };\n if (ncols == 1 || ic0 + j < int(ne01.z())) {\n ggml_sycl_memcpy_1(tmp, &Q_j[i]);\n ggml_sycl_memcpy_1(tmp + cpy_ne/2, &Q_j[i + cpy_ne/2]);\n }\n#pragma unroll\n for (int i1 = 0; i1 < cpy_ne; ++i1) {\n Q_reg[j][i0 / nthreads_KQ + i1] = sycl::half2(tmp[i1].x(), tmp[i1].y());\n }\n }\n#pragma unroll\n for (int k = 0; k < (D/2)/nthreads_KQ; ++k) {\n Q_reg[j][k] *= scale_h2;\n }\n }\n#else\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n const sycl::float2 * Q_j = (const sycl::float2 *) (Q + j*nb01);\n#pragma unroll\n for (int i0 = 0; i0 < D/2; i0 += nthreads_KQ*cpy_ne) {\n const int i = i0 + (nthreads_KQ == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads_KQ)*cpy_ne;\n if (ncols == 1 || ic0 + j < int(ne01.z())) {\n ggml_sycl_memcpy_1(&Q_reg[j][i0/nthreads_KQ], &Q_j[i]);\n ggml_sycl_memcpy_1(&Q_reg[j][i0/nthreads_KQ + cpy_ne/2], &Q_j[i + cpy_ne/2]);\n }\n }\n#pragma unroll\n for (int k = 0; k < (D/2)/nthreads_KQ; ++k) {\n Q_reg[j][k].x() *= scale;\n Q_reg[j][k].y() *= scale;\n }\n }\n#endif // GGML_SYCL_F16\n }\n\n const int k_VKQ_max = KV_max ? KV_max[sequence * item_ct1.get_group_range(2) + item_ct1.get_group(2)] : ne11;\n K += item_ct1.get_group(1) * nthreads * nb11;\n V += item_ct1.get_group(1) * nthreads * nb21;\n maskh += item_ct1.get_group(1) * nthreads;\n for (int k_VKQ_0 = item_ct1.get_group(1) * nthreads; k_VKQ_0 < k_VKQ_max;\n k_VKQ_0 += item_ct1.get_group_range(1) * nthreads,\n // Increment pointers after each loop:\n K += item_ct1.get_group_range(1) * nthreads * nb11, V += item_ct1.get_group_range(1) * nthreads * nb21,\n maskh += item_ct1.get_group_range(1) * nthreads) {\n // Calculate KQ tile and keep track of new maximum KQ values:\n float KQ_reg[ncols]={}; // KQ in registers.\n float KQ_max_new[ncols]={};\n\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n KQ_max_new[j] = KQ_max[j];\n }\n\n#pragma unroll\n for (int i_KQ_0 = 0; i_KQ_0 < nthreads_KQ; ++i_KQ_0) {\n const int i_KQ = item_ct1.get_local_id(1) * warp_size +\n (nthreads_KQ == warp_size ? 0 : (item_ct1.get_local_id(2) & ~(nthreads_KQ - 1))) + i_KQ_0;\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n float sum = vec_dot_KQ(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j]);\n sum = warp_reduce_sum(sum);\n\n if (use_logit_softcap) {\n sum = logit_softcap * sycl::tanh(sum);\n }\n if (mask) {\n sum += slope * sycl::vec(maskh[j * ne11 + i_KQ])\n .convert()[0];\n }\n\n KQ_max_new[j] = sycl::fmax((float) KQ_max_new[j], sum);\n\n if (int(nthreads_KQ == warp_size ? item_ct1.get_local_id(2)\n : item_ct1.get_local_id(2) %\n nthreads_KQ) == i_KQ_0) {\n KQ_reg[j] = sum;\n }\n }\n }\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n#pragma unroll\n for (int offset = nthreads_KQ; offset < warp_size; offset <<= 1) {\n KQ_max_new[j] = sycl::fmax(\n (float)KQ_max_new[j],\n (float)dpct::permute_sub_group_by_xor(\n sycl::ext::oneapi::this_work_item::get_sub_group(),\n KQ_max_new[j],\n offset,\n warp_size));\n }\n const float KQ_max_scale = sycl::native::exp((float) (KQ_max[j] - KQ_max_new[j]));\n KQ_max[j] = KQ_max_new[j];\n\n KQ_reg[j] = sycl::native::exp((float) (KQ_reg[j] - KQ_max[j]));\n KQ_sum[j] = KQ_sum[j]*KQ_max_scale + KQ_reg[j];\n KQ[j*nthreads + tid] = KQ_reg[j];\n\n#ifdef GGML_SYCL_F16\n const sycl::half2 KQ_max_scale_h2 = sycl::half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j][i_VKQ_0/nthreads_V] *= KQ_max_scale_h2;\n }\n#else\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j][i_VKQ_0/nthreads_V].x() *= KQ_max_scale;\n VKQ[j][i_VKQ_0/nthreads_V].y() *= KQ_max_scale;\n }\n#endif // GGML_SYCL_F16\n }\n\n sycl::group_barrier(sycl::ext::oneapi::this_work_item::get_sub_group());\n\n#pragma unroll\n for (int k0 = 0; k0 < warp_size; k0 += V_cols_per_iter) {\n const int k = item_ct1.get_local_id(1) * warp_size + k0 +\n (nthreads_V == warp_size ? 0 : item_ct1.get_local_id(2) / nthreads_V);\n\n#ifdef GGML_SYCL_F16\n sycl::half2 KQ_k[ncols];\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n KQ_k[j] = sycl::half2(KQ[j * nthreads + k]);\n }\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n sycl::half2 tmp[V_rows_per_thread / 2];\n dequantize_V(V + k * nb21, tmp,\n 2 * i_VKQ_0 + (nthreads_V == warp_size ? item_ct1.get_local_id(2) :\n item_ct1.get_local_id(2) % nthreads_V) *\n V_rows_per_thread);\n#pragma unroll\n for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) {\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n VKQ[j][i_VKQ_0/nthreads_V + i_VKQ_1] += tmp[i_VKQ_1]*KQ_k[j];\n }\n }\n }\n#else\n float KQ_k[ncols];\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n KQ_k[j] = KQ[j*nthreads + k];\n }\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n sycl::float2 tmp[V_rows_per_thread/2];\n dequantize_V(V + k*nb21, tmp,\n 2*i_VKQ_0 + (nthreads_V == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads_V)*V_rows_per_thread);\n#pragma unroll\n for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) {\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n VKQ[j][i_VKQ_0/nthreads_V + i_VKQ_1].x() += tmp[i_VKQ_1].x()*KQ_k[j];\n VKQ[j][i_VKQ_0/nthreads_V + i_VKQ_1].y() += tmp[i_VKQ_1].y()*KQ_k[j];\n }\n }\n }\n#endif // GGML_SYCL_F16\n }\n }\n\n if (sinks && item_ct1.get_group(1) == 0) {\n const float sink = ((const float *) sinks)[head];\n\n#pragma unroll\n for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n const int j = j0 + item_ct1.get_local_id(1);\n\n if (j0 + nwarps > ncols && j >= ncols) {\n break;\n }\n const float kqmax_new_j = sycl::fmax(sink, (float) KQ_max[j]);\n const float KQ_max_scale = sycl::native::exp((float) (KQ_max[j] - kqmax_new_j));\n KQ_max[j] = kqmax_new_j;\n\n KQ_sum[j] = KQ_sum[j] * KQ_max_scale +\n (item_ct1.get_local_id(2) == 0 ? sycl::native::exp((float) (sink - KQ_max[j])) : 0.0f);\n#ifdef GGML_SYCL_F16\n const sycl::half2 KQ_max_scale_h2 = sycl::half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j][i_VKQ_0/nthreads_V] *= KQ_max_scale_h2;\n }\n#else\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j][i_VKQ_0/nthreads_V].x() *= KQ_max_scale;\n VKQ[j][i_VKQ_0/nthreads_V].y() *= KQ_max_scale;\n }\n#endif // GGML_SYCL_F16\n }\n }\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n if (item_ct1.get_local_id(1) == 0) {\n KQ_max_shared[j*warp_size+item_ct1.get_local_id(2)] = -FLT_MAX / 2.0f;\n KQ_sum_shared[j*warp_size+item_ct1.get_local_id(2)] = 0.0f;\n }\n }\n\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n#pragma unroll\n for (int j = 0; j < ncols; ++j) {\n if (item_ct1.get_local_id(2) == 0) {\n KQ_max_shared[j*warp_size+item_ct1.get_local_id(1)] = KQ_max[j];\n }\n }\n\n\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n#pragma unroll\n for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) {\n if (ncols > 1 && ic0 + j_VKQ >= int(ne01.z())) {\n break;\n }\n\n float kqmax_new = KQ_max_shared[j_VKQ*warp_size+item_ct1.get_local_id(2)];\n kqmax_new = warp_reduce_max(kqmax_new);\n const float kqmax_scale = sycl::native::exp((float) (KQ_max[j_VKQ] - kqmax_new));\n KQ_max[j_VKQ] = kqmax_new;\n\n#ifdef GGML_SYCL_F16\n sycl::half2 * VKQ_tmp = (sycl::half2 *) KQ + item_ct1.get_local_id(1) * (V_cols_per_iter * D / 2) +\n (nthreads_V == warp_size ? 0 : item_ct1.get_local_id(2) / nthreads_V) * (D / 2);\n\n const sycl::half2 kqmax_scale_h2 = sycl::half2(kqmax_scale, kqmax_scale);\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j_VKQ][i_VKQ_0/nthreads_V] *= kqmax_scale_h2;\n }\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n const int i_VKQ =\n i_VKQ_0 + (nthreads_V == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads_V) *\n (V_rows_per_thread / 2);\n\n ggml_sycl_memcpy_1(VKQ_tmp + i_VKQ,\n &VKQ[j_VKQ][i_VKQ_0 / nthreads_V]);\n }\n#else\n sycl::float2 * VKQ_tmp = (sycl::float2 *) KQ + item_ct1.get_local_id(1)*(V_cols_per_iter*D/2)\n + (nthreads_V == warp_size ? 0 : item_ct1.get_local_id(2) / nthreads_V)*(D/2);\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n VKQ[j_VKQ][i_VKQ_0/nthreads_V].x() *= kqmax_scale;\n VKQ[j_VKQ][i_VKQ_0/nthreads_V].y() *= kqmax_scale;\n }\n#pragma unroll\n for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n const int i_VKQ = i_VKQ_0 + (nthreads_V == warp_size ? item_ct1.get_local_id(2) : item_ct1.get_local_id(2) % nthreads_V)*(V_rows_per_thread/2);\n\n ggml_sycl_memcpy_1(VKQ_tmp + i_VKQ, &VKQ[j_VKQ][i_VKQ_0/nthreads_V]);\n ggml_sycl_memcpy_1(VKQ_tmp + i_VKQ + V_rows_per_thread/4, &VKQ[j_VKQ][i_VKQ_0/nthreads_V + V_rows_per_thread/4]);\n }\n#endif // GGML_SYCL_F16\n\n KQ_sum[j_VKQ] *= kqmax_scale;\n KQ_sum[j_VKQ] = warp_reduce_sum(KQ_sum[j_VKQ]);\n if (item_ct1.get_local_id(2) == 0) {\n KQ_sum_shared[j_VKQ*warp_size+item_ct1.get_local_id(1)] = KQ_sum[j_VKQ];\n }\n\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n\n if (nthreads <= D || tid < D) {\n KQ_sum[j_VKQ] = KQ_sum_shared[j_VKQ*warp_size+item_ct1.get_local_id(2)];\n KQ_sum[j_VKQ] = warp_reduce_sum(KQ_sum[j_VKQ]);\n\n#pragma unroll\n for (int i0 = 0; i0 < D; i0 += nthreads) {\n float dst_val = 0;\n#pragma unroll\n for (int w = 0; w < nwarps; ++w) {\n#pragma unroll\n for (int v = 0; v < V_cols_per_iter; ++v) {\n dst_val += float(KQ[w*V_cols_per_iter*D + v*D + i0 + tid]);\n }\n }\n if (item_ct1.get_group_range(1) == 1) {\n dst_val /= KQ_sum[j_VKQ];\n }\n dst[(((sequence * int(ne01.z()) + ic0 + j_VKQ) * ne02 + head) * item_ct1.get_group_range(1) +\n item_ct1.get_group(1)) *\n D +\n i0 + tid] = dst_val;\n }\n }\n\n if (j_VKQ < ncols-1) {\n item_ct1.barrier(sycl::access::fence_space::local_space);\n }\n\n }\n\n if (item_ct1.get_group_range(1) != 1 && tid < ncols && (ncols == 1 || ic0 + tid < int(ne01.z()))) {\n dst_meta[((sequence * int(ne01.z()) + ic0 + tid) * ne02 + head) * item_ct1.get_group_range(1) +\n item_ct1.get_group(1)] = make_float2(KQ_max[tid], KQ_sum[tid]);\n }\n#else\n GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n max_bias, m0, m1, n_head_log2, logit_softcap,\n ne00, ne01, ne02, ne03,\n nb01, nb02, nb03,\n ne10, ne11, ne12, ne13,\n nb11, nb12, nb13,\n nb21, nb22, nb23,\n ne31, ne32, ne33,\n nb31, nb32, nb33);\n\n#endif // SYCL_FLASH_ATTN\n}\n#ifdef __clang__\n#pragma clang diagnostic pop\n#endif // __clang__\n\n\ntemplate \nvoid ggml_sycl_flash_attn_ext_vec_case_impl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n\n const int warp_size = WARP_16_SIZE; //better performance than WARP_32_SIZE\n\n const int cc = ggml_sycl_info().devices[ggml_sycl_get_device()].cc;\n\n const int nthreads = ggml_sycl_fattn_vec_get_nthreads_host(cc);\n const int nwarps = nthreads / warp_size;\n\n const bool need_f16_K = type_K == GGML_TYPE_F16;\n const bool need_f16_V = type_V == GGML_TYPE_F16;\n constexpr size_t nbytes_shared = 0;\n\n launch_fattn, warp_size>(\n ctx, dst, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);\n}\n\ntemplate \nvoid ggml_sycl_flash_attn_ext_vec_case(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * KQV = dst;\n const ggml_tensor * Q = dst->src[0];\n\n float logit_softcap;\n memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n if (Q->ne[1] == 1) {\n constexpr int cols_per_block = 1;\n if (logit_softcap == 0.0f) {\n constexpr bool use_logit_softcap = false;\n ggml_sycl_flash_attn_ext_vec_case_impl(ctx, dst);\n } else {\n constexpr bool use_logit_softcap = true;\n ggml_sycl_flash_attn_ext_vec_case_impl(ctx, dst);\n }\n return;\n }\n\n constexpr int cols_per_block = 2;\n if (logit_softcap == 0.0f) {\n constexpr bool use_logit_softcap = false;\n ggml_sycl_flash_attn_ext_vec_case_impl(ctx, dst);\n } else {\n constexpr bool use_logit_softcap = true;\n ggml_sycl_flash_attn_ext_vec_case_impl(ctx, dst);\n }\n}\n\n#define DECL_FATTN_VEC_CASE(D, type_K, type_V) \\\n template void ggml_sycl_flash_attn_ext_vec_case \\\n (ggml_backend_sycl_context & ctx, ggml_tensor * dst) \\\n\n#define EXTERN_DECL_FATTN_VEC_CASES(D, type_K) \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_F16); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q4_0); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q4_1); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_0); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_1); \\\n extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q8_0); \\\n\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F16)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_0)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_1)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_0)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_1)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q8_0)\n\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F16)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_0)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_1)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_0)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_1)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q8_0)\n\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F16)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_0)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_1)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_0)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_1)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q8_0)\n\n#endif // GGML_SYCL_FATTN_VEC_HPP\n" }, { "path": "src/ggml-sycl/fattn.cpp", "content": "//\n// MIT license\n// Copyright (C) 2025 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n\n#include \n#include \"dpct/helper.hpp\"\n#include \"common.hpp\"\n#include \"fattn-common.hpp\"\n#include \"fattn-tile.hpp\"\n#include \"fattn-vec.hpp\"\n#include \"fattn.hpp\"\n\n\n#define FATTN_VEC_CASE(D, type_K, type_V) \\\n { \\\n const bool type_K_okay = K->type == (type_K) || (K->type == GGML_TYPE_F32 && (type_K) == GGML_TYPE_F16); \\\n const bool type_V_okay = V->type == (type_V) || (V->type == GGML_TYPE_F32 && (type_V) == GGML_TYPE_F16); \\\n if (Q->ne[0] == (D) && type_K_okay && type_V_okay) { \\\n ggml_sycl_flash_attn_ext_vec_case(ctx, dst); \\\n return; \\\n } \\\n } \\\n\n#define FATTN_VEC_CASES_ALL_D(type_K, type_V) \\\n FATTN_VEC_CASE( 64, type_K, type_V) \\\n FATTN_VEC_CASE(128, type_K, type_V) \\\n FATTN_VEC_CASE(256, type_K, type_V) \\\n\nstatic void ggml_sycl_flash_attn_ext_vec(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_tensor * Q = dst->src[0];\n ggml_tensor * K = dst->src[1];\n ggml_tensor * V = dst->src[2];\n\n#ifdef GGML_SYCL_FA_ALL_QUANTS\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_F16)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_1)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_0)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_1)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_1)\n\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q8_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)\n#else\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)\n FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)\n#endif // GGML_SYCL_FA_ALL_QUANTS\n\n GGML_ABORT(\"Not match KV type in vec\");\n}\n\n// Best FlashAttention kernel for a specific GPU:\nenum best_fattn_kernel {\n BEST_FATTN_KERNEL_NONE = 0,\n BEST_FATTN_KERNEL_VEC = 100,\n BEST_FATTN_KERNEL_TILE = 200,\n};\n\nstatic best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const ggml_tensor * dst) {\n GGML_UNUSED(device);\n#ifndef SYCL_FLASH_ATTN\n GGML_UNUSED(dst);\n return BEST_FATTN_KERNEL_NONE;\n#endif// SYCL_FLASH_ATTN\n\n if(!g_ggml_sycl_enable_flash_attention) return BEST_FATTN_KERNEL_NONE;\n\n const ggml_tensor * KQV = dst;\n const ggml_tensor * Q = dst->src[0];\n const ggml_tensor * K = dst->src[1];\n const ggml_tensor * V = dst->src[2];\n const ggml_tensor * mask = dst->src[3];\n\n const int gqa_ratio = Q->ne[2] / K->ne[2];\n GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);\n\n float max_bias = 0.0f;\n memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));\n\n bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0;\n for (const ggml_tensor * t : {Q, K, V, mask}) {\n if (t == nullptr || ggml_is_quantized(t->type)) {\n continue;\n }\n for (size_t i = 1; i < GGML_MAX_DIMS; ++i) {\n if (t->nb[i] % 16 != 0) {\n gqa_opt_applies = false;\n break;\n }\n }\n }\n\n switch (K->ne[0]) {\n case 40:\n case 64:\n case 72:\n case 80:\n case 96:\n case 128:\n case 112:\n case 256:\n if (V->ne[0] != K->ne[0]) {\n return BEST_FATTN_KERNEL_NONE;\n }\n break;\n case 576:\n if (V->ne[0] != 512) {\n return BEST_FATTN_KERNEL_NONE;\n }\n if (!gqa_opt_applies) {\n return BEST_FATTN_KERNEL_NONE;\n }\n break;\n default:\n return BEST_FATTN_KERNEL_NONE;\n }\n\n#ifndef GGML_SYCL_FA_ALL_QUANTS\n if (K->type != V->type) {\n return BEST_FATTN_KERNEL_NONE;\n }\n#endif // GGML_SYCL_FA_ALL_QUANTS\n\n switch (K->type) {\n case GGML_TYPE_F32:\n case GGML_TYPE_F16:\n break;\n case GGML_TYPE_Q4_1:\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n#ifndef GGML_SYCL_FA_ALL_QUANTS\n return BEST_FATTN_KERNEL_NONE;\n#endif // GGML_SYCL_FA_ALL_QUANTS\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q8_0:\n break;\n default:\n return BEST_FATTN_KERNEL_NONE;\n }\n\n if (mask && mask->ne[2] != 1) {\n return BEST_FATTN_KERNEL_NONE;\n }\n\n // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes:\n const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0;\n\n // Todo: Use the XMX kernel if possible:\n\n // If there are no tensor cores available, use the generic tile kernel:\n if (can_use_vector_kernel) {\n if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) {\n if (Q->ne[1] == 1) {\n if (!gqa_opt_applies) {\n return BEST_FATTN_KERNEL_VEC;\n }\n }\n } else {\n if (Q->ne[1] <= 2) {\n return BEST_FATTN_KERNEL_VEC;\n }\n }\n }\n return BEST_FATTN_KERNEL_TILE;\n}\n\nvoid ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_sycl_set_device(ctx.device);\n switch (ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst)) {\n case BEST_FATTN_KERNEL_NONE:\n GGML_ABORT(\"Not support Flash-Attention\");\n case BEST_FATTN_KERNEL_TILE:\n ggml_sycl_flash_attn_ext_tile(ctx, dst);\n break;\n case BEST_FATTN_KERNEL_VEC:\n ggml_sycl_flash_attn_ext_vec(ctx, dst);\n break;\n }\n}\n\nbool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst) {\n return ggml_sycl_get_best_fattn_kernel(device, dst) != BEST_FATTN_KERNEL_NONE;\n}\n" }, { "path": "src/ggml-sycl/fattn.hpp", "content": "//\n// MIT license\n// Copyright (C) 2025 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_FATTN_HPP\n#define GGML_SYCL_FATTN_HPP\n\n#include \"common.hpp\"\n\nvoid ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n\nbool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst);\n\n#endif // GGML_SYCL_FATTN_HPP\n" }, { "path": "src/ggml-sycl/gated_delta_net.cpp", "content": "#include \n#include \"dpct/helper.hpp\"\n#include \"common.hpp\"\n#include \"ggml.h\"\n#include \"gated_delta_net.hpp\"\n#include \n\n\ntemplate \nvoid gated_delta_net_sycl(const float * q,\n const float * k,\n const float * v,\n const float * g,\n const float * beta,\n const float * curr_state,\n float * dst,\n int64_t H,\n int64_t n_tokens,\n int64_t n_seqs,\n int64_t sq1,\n int64_t sq2,\n int64_t sq3,\n int64_t sv1,\n int64_t sv2,\n int64_t sv3,\n int64_t sb1,\n int64_t sb2,\n int64_t sb3,\n const sycl::uint3 neqk1_magic,\n const sycl::uint3 rq3_magic,\n float scale) {\n auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();\n const uint32_t h_idx = item_ct1.get_group(2);\n const uint32_t sequence = item_ct1.get_group(1);\n // each warp owns one column, using warp-level primitives to reduce across rows\n const int lane = item_ct1.get_local_id(2);\n const int col = item_ct1.get_group(0) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1);\n\n const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic);\n const uint32_t iq3 = fastdiv(sequence, rq3_magic);\n\n const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs;\n float * attn_data = dst;\n float * state = dst + attn_score_elems;\n\n const int64_t state_offset = (sequence * H + h_idx) * S_v * S_v;\n state += state_offset;\n curr_state += state_offset;\n attn_data += (sequence * n_tokens * H + h_idx) * S_v;\n\n constexpr int warp_size = ggml_sycl_get_physical_warp_size() < S_v ? ggml_sycl_get_physical_warp_size() : S_v;\n static_assert(S_v % warp_size == 0, \"S_v must be a multiple of warp_size\");\n constexpr int rows_per_lane = (S_v + warp_size - 1) / warp_size;\n float s_shard[rows_per_lane];\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n s_shard[r] = curr_state[i * S_v + col];\n }\n\n for (int t = 0; t < n_tokens; t++) {\n const float * q_t = q + iq3 * sq3 + t * sq2 + iq1 * sq1;\n const float * k_t = k + iq3 * sq3 + t * sq2 + iq1 * sq1;\n const float * v_t = v + sequence * sv3 + t * sv2 + h_idx * sv1;\n\n const int64_t gb_offset = sequence * sb3 + t * sb2 + h_idx * sb1;\n const float * beta_t = beta + gb_offset;\n const float * g_t = g + gb_offset * (KDA ? S_v : 1);\n\n const float beta_val = *beta_t;\n\n if constexpr (!KDA) {\n const float g_val = sycl::native::exp(*g_t);\n\n // kv[col] = (S^T @ k)[col] = sum_i S[i][col] * k[i]\n float kv_shard = 0.0f;\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n kv_shard += s_shard[r] * k_t[i];\n }\n float kv_col = warp_reduce_sum(kv_shard);\n\n // delta[col] = (v[col] - g * kv[col]) * beta\n float delta_col = (v_t[col] - g_val * kv_col) * beta_val;\n\n // fused: S[i][col] = g * S[i][col] + k[i] * delta[col]\n // attn[col] = (S^T @ q)[col] = sum_i S[i][col] * q[i]\n float attn_partial = 0.0f;\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n s_shard[r] = g_val * s_shard[r] + k_t[i] * delta_col;\n attn_partial += s_shard[r] * q_t[i];\n }\n\n float attn_col = warp_reduce_sum(attn_partial);\n\n if (lane == 0) {\n attn_data[col] = attn_col * scale;\n }\n } else {\n // kv[col] = sum_i g[i] * S[i][col] * k[i]\n float kv_shard = 0.0f;\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n kv_shard += sycl::native::exp(g_t[i]) * s_shard[r] * k_t[i];\n }\n\n float kv_col = warp_reduce_sum(kv_shard);\n\n // delta[col] = (v[col] - kv[col]) * beta\n float delta_col = (v_t[col] - kv_col) * beta_val;\n\n // fused: S[i][col] = g[i] * S[i][col] + k[i] * delta[col]\n // attn[col] = (S^T @ q)[col] = sum_i S[i][col] * q[i]\n float attn_partial = 0.0f;\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n s_shard[r] = sycl::native::exp(g_t[i]) * s_shard[r] + k_t[i] * delta_col;\n attn_partial += s_shard[r] * q_t[i];\n }\n\n float attn_col = warp_reduce_sum(attn_partial);\n\n if (lane == 0) {\n attn_data[col] = attn_col * scale;\n }\n }\n\n attn_data += S_v * H;\n }\n\n // Write state back to global memory\n#pragma unroll\n for (int r = 0; r < rows_per_lane; r++) {\n const int i = r * warp_size + lane;\n state[i * S_v + col] = s_shard[r];\n }\n}\n\ntemplate \nstatic void launch_gated_delta_net(const float * q_d,\n const float * k_d,\n const float * v_d,\n const float * g_d,\n const float * b_d,\n const float * s_d,\n float * dst_d,\n int64_t S_v,\n int64_t H,\n int64_t n_tokens,\n int64_t n_seqs,\n int64_t sq1,\n int64_t sq2,\n int64_t sq3,\n int64_t sv1,\n int64_t sv2,\n int64_t sv3,\n int64_t sb1,\n int64_t sb2,\n int64_t sb3,\n int64_t neqk1,\n int64_t rq3,\n float scale,\n dpct::queue_ptr stream) {\n //TODO: Add chunked kernel for even faster pre-fill\n const int warp_size = ggml_sycl_info().devices[ggml_sycl_get_device()].warp_size;\n\n const int num_warps = 4;\n dpct::dim3 grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps);\n dpct::dim3 block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1);\n\n const sycl::uint3 neqk1_magic = init_fastdiv_values(neqk1);\n const sycl::uint3 rq3_magic = init_fastdiv_values(rq3);\n\n int cc = ggml_sycl_info().devices[ggml_sycl_get_device()].cc;\n\n switch (S_v) {\n case 16:\n {\n constexpr int sv = 16;\n stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, n_tokens,\n n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2,\n sb3, neqk1_magic, rq3_magic, scale);\n });\n }\n break;\n case 32:\n {\n constexpr int sv = 32;\n stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, n_tokens,\n n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2,\n sb3, neqk1_magic, rq3_magic, scale);\n });\n }\n break;\n case 64: {\n {\n constexpr int sv = 64;\n stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n gated_delta_net_sycl(\n q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, n_tokens, n_seqs, sq1, sq2,\n sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale);\n });\n }\n break;\n }\n case 128: {\n {\n constexpr int sv = 128;\n stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n gated_delta_net_sycl(\n q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, n_tokens, n_seqs, sq1, sq2,\n sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale);\n });\n }\n break;\n }\n default:\n GGML_ABORT(\"fatal error\");\n break;\n }\n}\n\nvoid ggml_sycl_op_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n ggml_tensor * src_q = dst->src[0];\n ggml_tensor * src_k = dst->src[1];\n ggml_tensor * src_v = dst->src[2];\n ggml_tensor * src_g = dst->src[3];\n ggml_tensor * src_beta = dst->src[4];\n ggml_tensor * src_state = dst->src[5];\n\n GGML_TENSOR_LOCALS(int64_t, neq, src_q, ne);\n GGML_TENSOR_LOCALS(size_t , nbq, src_q, nb);\n GGML_TENSOR_LOCALS(int64_t, nek, src_k, ne);\n GGML_TENSOR_LOCALS(size_t , nbk, src_k, nb);\n GGML_TENSOR_LOCALS(int64_t, nev, src_v, ne);\n GGML_TENSOR_LOCALS(size_t, nbv, src_v, nb);\n GGML_TENSOR_LOCALS(size_t, nbb, src_beta, nb);\n\n const int64_t S_v = nev0;\n const int64_t H = nev1;\n const int64_t n_tokens = nev2;\n const int64_t n_seqs = nev3;\n\n const bool kda = (src_g->ne[0] == S_v);\n\n GGML_ASSERT(neq1 == nek1);\n const int64_t neqk1 = neq1;\n\n const int64_t rq3 = nev3 / neq3;\n\n const float * q_d = (const float *) src_q->data;\n const float * k_d = (const float *) src_k->data;\n const float * v_d = (const float *) src_v->data;\n const float * g_d = (const float *) src_g->data;\n const float * b_d = (const float *) src_beta->data;\n\n const float * s_d = (const float *) src_state->data;\n float * dst_d = (float *) dst->data;\n\n GGML_ASSERT(ggml_is_contiguous_rows(src_q));\n GGML_ASSERT(ggml_is_contiguous_rows(src_k));\n GGML_ASSERT(ggml_is_contiguous_rows(src_v));\n GGML_ASSERT(ggml_are_same_stride(src_q, src_k));\n GGML_ASSERT(src_g->ne[0] == 1 || kda);\n GGML_ASSERT(ggml_is_contiguous(src_g));\n GGML_ASSERT(ggml_is_contiguous(src_beta));\n GGML_ASSERT(ggml_is_contiguous(src_state));\n\n // strides in floats (beta strides used for both g and beta offset computation)\n const int64_t sq1 = nbq1 / sizeof(float);\n const int64_t sq2 = nbq2 / sizeof(float);\n const int64_t sq3 = nbq3 / sizeof(float);\n const int64_t sv1 = nbv1 / sizeof(float);\n const int64_t sv2 = nbv2 / sizeof(float);\n const int64_t sv3 = nbv3 / sizeof(float);\n const int64_t sb1 = nbb1 / sizeof(float);\n const int64_t sb2 = nbb2 / sizeof(float);\n const int64_t sb3 = nbb3 / sizeof(float);\n\n const float scale = 1.0f / sqrtf((float) S_v);\n\n dpct::queue_ptr stream = ctx.stream();\n\n if (kda) {\n launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d,\n S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,\n sb1, sb2, sb3, neqk1, rq3, scale, stream);\n } else {\n launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d,\n S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3,\n sb1, sb2, sb3, neqk1, rq3, scale, stream);\n }\n}\n\nvoid ggml_sycl_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/6);\n ggml_sycl_op_gated_delta_net(ctx, dst);\n}\n" }, { "path": "src/ggml-sycl/gated_delta_net.hpp", "content": "#pragma once\n\n#include \n#include \"dpct/helper.hpp\"\n#include \"common.hpp\"\n#include \"ggml.h\"\n\nvoid ggml_sycl_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst);\n" }, { "path": "src/ggml-sycl/gemm.hpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_GEMM_HPP\n#define GGML_SYCL_GEMM_HPP\n\n#include \"ggml-sycl.h\"\n\n#if GGML_SYCL_DNNL\n\n#include \"dnnl.hpp\"\n#include \"dnnl_sycl.hpp\"\n\nclass DnnlGemmWrapper {\npublic:\n using dt = dnnl::memory::data_type;\n using tag = dnnl::memory::format_tag;\n\n template\n static constexpr dt to_dt() {\n if constexpr (std::is_same_v) return dt::f32;\n else if constexpr (std::is_same_v) return dt::f16;\n else static_assert(0);\n }\n\n static void gemm(ggml_backend_sycl_context & ctx, int m, int n, int k,\n const void * a, dt at, dnnl_dim_t stra0, dnnl_dim_t stra1, dnnl_dim_t stra2,\n const void * b, dt bt, dnnl_dim_t strb0, dnnl_dim_t strb1, dnnl_dim_t strb2,\n void * c, dt ct, const queue_ptr & q, dnnl_dim_t batches_a, dnnl_dim_t batches_b) {\n\n auto stream = ctx.stream_dnnl(q);\n auto eng = ctx.engine_dnnl(q);\n\n dnnl::memory::dims a_dims = {batches_a, m, k };\n dnnl::memory::dims a_strides = {stra2, stra1, stra0};\n const auto a_in_md = dnnl::memory::desc(a_dims, at, a_strides);\n\n dnnl::memory::dims b_dims = {batches_b, k, n };\n dnnl::memory::dims b_strides = {strb2, strb0, strb1};\n const auto b_in_md = dnnl::memory::desc(b_dims, bt, b_strides);\n\n dnnl::memory::dims c_dims = { std::max(batches_a, batches_b), m, n};\n dnnl::memory::dims c_strides = {m*n, 1, m };\n const auto c_md = dnnl::memory::desc(c_dims, ct, c_strides);\n dnnl::primitive_attr primitive_attr;\n primitive_attr.set_scratchpad_mode(dnnl::scratchpad_mode::user);\n\n#ifdef GGML_SYCL_F16\n primitive_attr.set_fpmath_mode(dnnl::fpmath_mode::f16);\n#endif\n\n auto a_mem = dnnl::memory(a_in_md, eng, const_cast(a));\n auto b_mem = dnnl::memory(b_in_md, eng, const_cast(b));\n auto matmul_pd = dnnl::matmul::primitive_desc(eng, a_in_md, b_in_md, c_md, primitive_attr);\n auto c_mem = dnnl::memory(matmul_pd.dst_desc(), eng, c);\n\n auto scratchpad_md = matmul_pd.scratchpad_desc();\n auto scratchpad_mem = ctx.get_scratchpad_mem(scratchpad_md, eng, q);\n\n auto matmul_prim = dnnl::matmul(matmul_pd);\n\n std::unordered_map matmul_args;\n matmul_args.insert({ DNNL_ARG_SRC, a_mem });\n matmul_args.insert({ DNNL_ARG_WEIGHTS, b_mem });\n\n matmul_args.insert({ DNNL_ARG_DST, c_mem });\n matmul_args.insert({ DNNL_ARG_SCRATCHPAD, scratchpad_mem });\n\n matmul_prim.execute(stream, matmul_args);\n }\n\n static void row_gemm(ggml_backend_sycl_context & ctx, int m, int n, int k,\n const void * a, dt at, const void * b, dt bt, void * c, dt ct, const queue_ptr & q) {\n\n gemm(ctx, m, n, k, a, at, 1, k, k * m, b, bt, 1, k, n * k, c, ct, q, 1, 1);\n }\n};\n\n#endif\n\n#endif // GGML_SYCL_GEMM_HPP\n" }, { "path": "src/ggml-sycl/getrows.cpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#include \"ggml-impl.h\"\n#include \"common.hpp\"\n#include \"dequantize.hpp\"\n#include \"getrows.hpp\"\n\n\ntemplate\nstatic void k_get_rows(\n const void * src0, const int32_t * src1, dst_t * dst,\n int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/\n /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/\n /*size_t s0,*/ size_t s1, size_t s2, size_t s3,\n /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,\n size_t s10, size_t s11, size_t s12,\n const sycl::nd_item<3> &item_ct1/*, size_t s13*/) {\n\n const int i00 = (item_ct1.get_group(2) * item_ct1.get_local_range(2) +\n item_ct1.get_local_id(2)) *\n 2;\n const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) +\n item_ct1.get_local_id(1);\n const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) +\n item_ct1.get_local_id(0)) /\n ne12;\n const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) +\n item_ct1.get_local_id(0)) %\n ne12;\n\n if (i00 >= ne00) {\n return;\n }\n\n const int i01 = src1[i10*s10 + i11*s11 + i12*s12];\n\n dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;\n const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03;\n\n const int ib = i00/qk; // block index\n const int iqs = (i00%qk)/qr; // quant index\n const int iybs = i00 - i00%qk; // dst block start index\n const int y_offset = qr == 1 ? 1 : qk/2;\n\n // dequantize\n dfloat2 v;\n dequantize_kernel(src0_row, ib, iqs, v);\n\n dst_row[iybs + iqs + 0] = v.x();\n dst_row[iybs + iqs + y_offset] = v.y();\n}\n\ntemplate\nstatic void k_get_rows_float(\n const src0_t * src0, const int32_t * src1, dst_t * dst,\n int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/\n /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/\n /*size_t s0,*/ size_t s1, size_t s2, size_t s3,\n /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,\n size_t s10, size_t s11, size_t s12,\n const sycl::nd_item<3> &item_ct1/*, size_t s13*/) {\n\n const int i00 = item_ct1.get_group(2) * item_ct1.get_local_range(2) +\n item_ct1.get_local_id(2);\n const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) +\n item_ct1.get_local_id(1);\n const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) +\n item_ct1.get_local_id(0)) /\n ne12;\n const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) +\n item_ct1.get_local_id(0)) %\n ne12;\n\n if (i00 >= ne00) {\n return;\n }\n\n const int i01 = src1[i10*s10 + i11*s11 + i12*s12];\n\n dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;\n const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03);\n\n dst_row[i00] = src0_row[i00];\n}\n\ntemplate \nstatic void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1,\n ggml_tensor *dst, const void *src0_dd,\n const int32_t *src1_dd, float *dst_dd,\n queue_ptr stream) {\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE);\n const int block_num_x = (ne00 + 2*SYCL_GET_ROWS_BLOCK_SIZE - 1) / (2*SYCL_GET_ROWS_BLOCK_SIZE);\n const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x);\n\n // strides in elements\n //const size_t s0 = nb0 / ggml_element_size(dst);\n const size_t s1 = nb1 / ggml_element_size(dst);\n const size_t s2 = nb2 / ggml_element_size(dst);\n const size_t s3 = nb3 / ggml_element_size(dst);\n\n const size_t s10 = nb10 / ggml_element_size(src1);\n const size_t s11 = nb11 / ggml_element_size(src1);\n const size_t s12 = nb12 / ggml_element_size(src1);\n //const size_t s13 = nb13 / ggml_element_size(src1);\n\n GGML_ASSERT(ne00 % 2 == 0);\n\n stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) {\n k_get_rows(\n src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2,\n s3, nb01, nb02, nb03, s10, s11, s12, item_ct1);\n });\n\n GGML_UNUSED(dst);\n GGML_UNUSED(ctx);\n}\n\ntemplate \nstatic void get_rows_sycl_float(ggml_backend_sycl_context & ctx, const ggml_tensor *src0,\n const ggml_tensor *src1, ggml_tensor *dst,\n const src0_t *src0_dd, const int32_t *src1_dd,\n float *dst_dd, queue_ptr stream) {\n\n GGML_TENSOR_BINARY_OP_LOCALS\n\n const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE);\n const int block_num_x = (ne00 + SYCL_GET_ROWS_BLOCK_SIZE - 1) / SYCL_GET_ROWS_BLOCK_SIZE;\n const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x);\n\n // strides in elements\n //const size_t s0 = nb0 / ggml_element_size(dst);\n const size_t s1 = nb1 / ggml_element_size(dst);\n const size_t s2 = nb2 / ggml_element_size(dst);\n const size_t s3 = nb3 / ggml_element_size(dst);\n\n const size_t s10 = nb10 / ggml_element_size(src1);\n const size_t s11 = nb11 / ggml_element_size(src1);\n const size_t s12 = nb12 / ggml_element_size(src1);\n //const size_t s13 = nb13 / ggml_element_size(src1);\n\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) {\n k_get_rows_float(src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2,\n s3, nb01, nb02, nb03, s10, s11, s12, item_ct1);\n });\n }\n\n GGML_UNUSED(dst);\n GGML_UNUSED(ctx);\n}\n\nvoid ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->src[1]->type == GGML_TYPE_I32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n GGML_ASSERT(dst->src[0]->nb[0] == ggml_type_size(dst->src[0]->type));\n GGML_ASSERT(dst->src[1]->nb[0] == ggml_type_size(dst->src[1]->type));\n GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));\n\n const int32_t * src1_i32 = (const int32_t *) dst->src[1]->data;\n /* TODO: Refactor and remove duplicates */\n switch (dst->src[0]->type) {\n case GGML_TYPE_F16:\n get_rows_sycl_float(ctx, dst->src[0], dst->src[1], dst, (const sycl::half *)dst->src[0]->data,\n src1_i32, (float *)dst->data, ctx.stream());\n break;\n case GGML_TYPE_F32:\n get_rows_sycl_float(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,\n src1_i32, (float *)dst->data, ctx.stream());\n break;\n case GGML_TYPE_Q4_0:\n get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,\n src1_i32, (float *)dst->data, ctx.stream());\n break;\n case GGML_TYPE_Q4_1:\n get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,\n src1_i32, (float *)dst->data, ctx.stream());\n break;\n case GGML_TYPE_Q5_0:\n get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,\n src1_i32, (float *)dst->data, ctx.stream());\n break;\n case GGML_TYPE_Q5_1:\n get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,\n src1_i32, (float *)dst->data, ctx.stream());\n break;\n case GGML_TYPE_Q8_0:\n get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,\n src1_i32, (float *)dst->data, ctx.stream());\n break;\n default:\n // TODO: k-quants\n GGML_LOG_ERROR(\"%s: unsupported type: %s\\n\", __func__, ggml_type_name(dst->src[0]->type));\n GGML_ABORT(\"fatal error\");\n }\n}\n" }, { "path": "src/ggml-sycl/getrows.hpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#ifndef GGML_SYCL_GETROWS_HPP\n#define GGML_SYCL_GETROWS_HPP\n\n#include \"common.hpp\"\n\nvoid ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor *dst);\n\n#endif // GGML_SYCL_GETROWS_HPP\n" }, { "path": "src/ggml-sycl/ggml-sycl.cpp", "content": "//\n// MIT license\n// Copyright (C) 2024 Intel Corporation\n// SPDX-License-Identifier: MIT\n//\n\n//\n// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.\n// See https://llvm.org/LICENSE.txt for license information.\n// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n//\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC\n# include \n#endif\n#include \n\n#include \"ggml.h\"\n#include \"ggml-sycl.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-sycl/add-id.hpp\"\n#include \"ggml-sycl/backend.hpp\"\n#include \"ggml-sycl/common.hpp\"\n#include \"ggml-sycl/element_wise.hpp\"\n#include \"ggml-sycl/gated_delta_net.hpp\"\n#include \"ggml-sycl/gemm.hpp\"\n#include \"ggml-sycl/getrows.hpp\"\n#include \"ggml-sycl/norm.hpp\"\n#include \"ggml-sycl/presets.hpp\"\n#include \"ggml-sycl/quantize.hpp\"\n#include \"ggml-sycl/repeat_back.hpp\"\n#include \"ggml-sycl/set_rows.hpp\"\n#include \"ggml-sycl/set.hpp\"\n#include \"ggml-sycl/ssm_conv.hpp\"\n#include \"ggml-sycl/sycl_hw.hpp\"\n\n\nstatic bool g_sycl_loaded = false;\nint g_ggml_sycl_debug = 0;\nint g_ggml_sycl_disable_optimize = 0;\nint g_ggml_sycl_disable_graph = 0;\nint g_ggml_sycl_disable_dnn = 0;\nint g_ggml_sycl_prioritize_dmmv = 0;\nint g_ggml_sycl_use_async_mem_op = 0;\nint g_ggml_sycl_enable_flash_attention = 1;\n\n\nstatic ggml_sycl_device_info ggml_sycl_init() {\n ggml_sycl_device_info info = {};\n\n info.device_count = dpct::dev_mgr::instance().device_count();\n if (info.device_count == 0) {\n GGML_LOG_ERROR(\"%s: failed to initialize: %s\\n\", GGML_SYCL_NAME, __func__);\n return info;\n }\n\n GGML_ASSERT(info.device_count <= GGML_SYCL_MAX_DEVICES);\n\n int64_t total_vram = 0;\n/* This is a bit misleading; reserved for later */\n// #if defined(SYCL_USE_XMX)\n// GGML_LOG_INFO(\"%s: SYCL_USE_XMX: yes\\n\", __func__);\n// #else\n// GGML_LOG_INFO(\"%s: SYCL_USE_XMX: no\\n\", __func__);\n// #endif\n for (int i = 0; i < info.device_count; ++i) {\n info.devices[i].vmm = 0;\n dpct::device_info prop;\n sycl::device device = dpct::dev_mgr::instance().get_device(i);\n\n SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info(\n prop, device)));\n\n info.default_tensor_split[i] = total_vram;\n total_vram += prop.get_global_mem_size();\n\n info.devices[i].cc =\n 100 * prop.get_major_version() + 10 * prop.get_minor_version();\n info.devices[i].nsm = prop.get_max_compute_units() / 16; //16: Number of Xe Cores\n info.devices[i].opt_feature.reorder = device.ext_oneapi_architecture_is(syclex::arch_category::intel_gpu);\n info.devices[i].smpbo = prop.get_local_mem_size();\n info.devices[i].warp_size = WARP_SIZE;\n\n info.max_work_group_sizes[i] = prop.get_max_work_group_size();\n info.devices[i].max_wg_per_cu = info.max_work_group_sizes[i] / prop.get_max_compute_units();\n\n }\n\n for (int id = 0; id < info.device_count; ++id) {\n info.default_tensor_split[id] /= total_vram;\n }\n return info;\n}\n\nconst ggml_sycl_device_info & ggml_sycl_info() {\n static ggml_sycl_device_info info = ggml_sycl_init();\n return info;\n}\n\nstatic void print_device_detail(int id, sycl::device &device, std::string device_type) {\n\n dpct::device_info prop;\n SYCL_CHECK(CHECK_TRY_ERROR(\n dpct::get_device_info(prop, device)));\n\n std::string version;\n version += std::to_string(prop.get_major_version());\n version += \".\";\n version += std::to_string(prop.get_minor_version());\n\n device_type = std::regex_replace(device_type, std::regex(\"ext_oneapi_\"), \"\");\n std::string name = std::string(prop.get_name());\n name = std::regex_replace(name, std::regex(\"\\\\(R\\\\)\"), \"\");\n name = std::regex_replace(name, std::regex(\"\\\\(TM\\\\)\"), \"\");\n\n auto global_mem_size = prop.get_global_mem_size()/1000000;\n GGML_LOG_INFO(\"|%2d|%19s|%39s|%7s|%7d|%8d|%5d|%6luM|%21s|\\n\", id, device_type.c_str(),\n name.c_str(), version.c_str(), prop.get_max_compute_units(),\n prop.get_max_work_group_size(), prop.get_max_sub_group_size(),\n global_mem_size, device.get_info().c_str());\n}\n\nstatic void print_device_opt_feature(int device_count) {\n GGML_LOG_INFO(\"SYCL Optimization Feature:\\n\");\n GGML_LOG_INFO(\n \"|ID| Device Type|Reorder|\\n\");\n GGML_LOG_INFO(\n \"|--|-------------------|-------|\\n\");\n std::map DeviceNums;\n for (int id = 0; id < device_count; ++id) {\n sycl::device device = dpct::dev_mgr::instance().get_device(id);\n std::string backend_type = get_device_backend_and_type(device);\n int type_id = DeviceNums[backend_type]++;\n std::stringstream device_type;\n device_type << \"[\" << backend_type << \":\" << std::to_string(type_id)\n << \"]\";\n std::string device_type_s = device_type.str();\n device_type_s = std::regex_replace(device_type_s, std::regex(\"ext_oneapi_\"), \"\");\n GGML_LOG_INFO(\"|%2d|%19s|%7s|\\n\", id, device_type_s.c_str(),\n ggml_sycl_info().devices[id].opt_feature.reorder ? \"Y\": \"N\");\n }\n\n}\nvoid ggml_backend_sycl_print_sycl_devices() {\n GGML_SYCL_DEBUG(\"[SYCL] call ggml_backend_sycl_print_sycl_devices\\n\");\n int device_count = dpct::dev_mgr::instance().device_count();\n std::map DeviceNums;\n GGML_LOG_INFO(\"Found %d SYCL devices:\\n\", device_count);\n\n GGML_LOG_INFO(\n \"| | | | \"\n \" |Max | |Max |Global | |\\n\");\n GGML_LOG_INFO(\n \"| | | | \"\n \" |compute|Max work|sub |mem | |\\n\");\n GGML_LOG_INFO(\n \"|ID| Device Type| \"\n \"Name|Version|units |group |group|size | Driver version|\\n\");\n GGML_LOG_INFO(\n \"|--|-------------------|---------------------------------------|------\"\n \"-|-------|--------|-----|-------|---------------------|\\n\");\n\n for (int id = 0; id < device_count; ++id) {\n sycl::device device = dpct::dev_mgr::instance().get_device(id);\n std::string backend_type = get_device_backend_and_type(device);\n int type_id = DeviceNums[backend_type]++;\n std::stringstream device_type;\n device_type << \"[\" << backend_type << \":\" << std::to_string(type_id)\n << \"]\";\n print_device_detail(id, device, device_type.str());\n }\n\n print_device_opt_feature(device_count);\n}\n\nstatic inline int get_sycl_env(const char *env_name, int default_val) {\n char *user_device_string = getenv(env_name);\n int user_number = default_val;\n\n unsigned n;\n if (user_device_string != NULL &&\n sscanf(user_device_string, \" %u\", &n) == 1) {\n user_number = (int)n;\n } else {\n user_number = default_val;\n }\n return user_number;\n}\n\nstatic void ggml_check_sycl() try {\n static bool initialized = false;\n\n if (!initialized) {\n g_ggml_sycl_debug = get_sycl_env(\"GGML_SYCL_DEBUG\", 0);\n g_ggml_sycl_disable_optimize = get_sycl_env(\"GGML_SYCL_DISABLE_OPT\", 0);\n g_ggml_sycl_disable_graph = get_sycl_env(\"GGML_SYCL_DISABLE_GRAPH\", 1);\n g_ggml_sycl_disable_dnn = get_sycl_env(\"GGML_SYCL_DISABLE_DNN\", 0);\n g_ggml_sycl_prioritize_dmmv = get_sycl_env(\"GGML_SYCL_PRIORITIZE_DMMV\", 0);\n\n#ifdef SYCL_FLASH_ATTN\n g_ggml_sycl_enable_flash_attention = get_sycl_env(\"GGML_SYCL_ENABLE_FLASH_ATTN\", 1);\n#else\n g_ggml_sycl_enable_flash_attention = 0;\n#endif\n\n GGML_SYCL_DEBUG(\"[SYCL] call ggml_check_sycl\\n\");\n\n GGML_LOG_INFO(\"Build with Macros:\\n\");\n#if defined(GGML_SYCL_FORCE_MMQ)\n GGML_LOG_INFO(\" GGML_SYCL_FORCE_MMQ: yes\\n\");\n#else\n GGML_LOG_INFO(\" GGML_SYCL_FORCE_MMQ: no\\n\");\n#endif\n#if defined(GGML_SYCL_F16)\n GGML_LOG_INFO(\" GGML_SYCL_F16: yes\\n\");\n#else\n GGML_LOG_INFO(\" GGML_SYCL_F16: no\\n\");\n#endif\n#if defined(GGML_SYCL_GRAPH)\n GGML_LOG_INFO(\" GGML_SYCL_GRAPH: yes\\n\");\n#else\n GGML_LOG_INFO(\" GGML_SYCL_GRAPH: no\\n\");\n#endif\n#if defined(GGML_SYCL_DNNL)\n GGML_LOG_INFO(\" GGML_SYCL_DNNL: yes\\n\");\n#else\n GGML_LOG_INFO(\" GGML_SYCL_DNNL: no\\n\");\n#endif\n\n GGML_LOG_INFO(\"Running with Environment Variables:\\n\");\n GGML_LOG_INFO(\" GGML_SYCL_DEBUG: %d\\n\", g_ggml_sycl_debug);\n GGML_LOG_INFO(\" GGML_SYCL_DISABLE_OPT: %d\\n\", g_ggml_sycl_disable_optimize);\n#ifdef GGML_SYCL_GRAPH\n GGML_LOG_INFO(\" GGML_SYCL_DISABLE_GRAPH: %d\\n\", g_ggml_sycl_disable_graph);\n#else\n GGML_LOG_INFO(\" GGML_SYCL_DISABLE_GRAPH: graph disabled by compile flag\\n\");\n#endif\n#if GGML_SYCL_DNNL\n GGML_LOG_INFO(\" GGML_SYCL_DISABLE_DNN: %d\\n\", g_ggml_sycl_disable_dnn);\n#else\n GGML_LOG_INFO(\" GGML_SYCL_DISABLE_DNN: DNN disabled by compile flag\\n\");\n#endif\n GGML_LOG_INFO(\" GGML_SYCL_PRIORITIZE_DMMV: %d\\n\", g_ggml_sycl_prioritize_dmmv);\n\n#ifdef SYCL_FLASH_ATTN\n GGML_LOG_INFO(\" GGML_SYCL_ENABLE_FLASH_ATTN: %d\\n\", g_ggml_sycl_enable_flash_attention);\n#else\n GGML_LOG_INFO(\" GGML_SYCL_ENABLE_FLASH_ATTN: %d disabled by compile flag\\n\",\n g_ggml_sycl_enable_flash_attention);\n#endif\n\n/* NOT REMOVE, keep it for next optimize for XMX.\n#if defined(SYCL_USE_XMX)\n fprintf(stderr, \"%s: SYCL_USE_XMX: yes\\n\", __func__);\n#else\n fprintf(stderr, \"%s: SYCL_USE_XMX: no\\n\", __func__);\n#endif\n*/\n // Currently, we only use async malloc / free when graphs are enabled as it is required for the calls to be\n // properly recorded. As this SYCL extension matures it may be beneficial to enable as the default path and in\n // other places.\n#if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC\n g_ggml_sycl_use_async_mem_op = !g_ggml_sycl_disable_graph;\n if (g_ggml_sycl_use_async_mem_op) {\n for (unsigned int i = 0; i < dpct::dev_mgr::instance().device_count(); ++i) {\n if (!dpct::dev_mgr::instance().get_device(i).has(sycl::aspect::ext_oneapi_async_memory_alloc)) {\n g_ggml_sycl_use_async_mem_op = 0;\n break;\n }\n }\n }\n#endif\n if (CHECK_TRY_ERROR(g_all_sycl_device_count =\n dpct::dev_mgr::instance().device_count()) != 0) {\n initialized = true;\n g_sycl_loaded = false;\n return;\n }\n GGML_ASSERT(g_all_sycl_device_count <= GGML_SYCL_MAX_DEVICES);\n\n initialized = true;\n g_sycl_loaded = true;\n ggml_backend_sycl_print_sycl_devices();\n }\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\n/*\ndevice_index: device index from 0 to n (continue numbers).\n It is used for device select/set in SYCL backend internal data structure.\n*/\ninline void check_allow_gpu_index(const int device_index) {\n if (device_index >= ggml_sycl_info().device_count) {\n char error_buf[256];\n snprintf(\n error_buf,\n sizeof(error_buf),\n \"%s error: device_index:%d is out of range: [0-%d]\",\n __func__,\n device_index,\n ggml_sycl_info().device_count - 1);\n GGML_LOG_ERROR(\"%s\\n\", error_buf);\n assert(false);\n }\n}\n\nGGML_API void ggml_backend_sycl_get_gpu_list(int *id_list, int max_len) try {\n GGML_SYCL_DEBUG(\"[SYCL] call ggml_backend_sycl_get_gpu_list\\n\");\n for(int i=0;i=max_len) break;\n id_list[i] = i;\n }\n return;\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\n// sycl buffer\n\nstruct ggml_backend_sycl_buffer_context {\n int device;\n void * dev_ptr = nullptr;\n queue_ptr stream;\n std::string name;\n optimize_feature opt_feature;\n std::vector tensor_extras;\n\n ggml_backend_sycl_buffer_context(int device, void * dev_ptr, queue_ptr stream) :\n device(device), dev_ptr(dev_ptr), stream(stream) {\n check_allow_gpu_index(device);\n name = (GGML_SYCL_NAME + std::to_string(device));\n opt_feature = ggml_sycl_info().devices[device].opt_feature;\n }\n\n ~ggml_backend_sycl_buffer_context() {\n if (dev_ptr != nullptr) {\n ggml_sycl_set_device(device);\n SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(dev_ptr, *stream)));\n }\n\n //release extra used by tensors\n for (ggml_tensor_extra_gpu * extra : tensor_extras) {\n release_extra_gpu(extra);\n }\n\n }\n};\n\nstatic const char * ggml_backend_sycl_buffer_type_get_name(ggml_backend_buffer_type_t buft);\n\nstatic bool ggml_backend_buffer_is_sycl(ggml_backend_buffer_t buffer) {\n return buffer->buft->iface.get_name == ggml_backend_sycl_buffer_type_get_name;\n}\n\nstatic void\nggml_backend_sycl_buffer_free_buffer(ggml_backend_buffer_t buffer) try {\n ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context;\n ggml_sycl_set_device(ctx->device);\n\n delete ctx;\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void * ggml_backend_sycl_buffer_get_base(ggml_backend_buffer_t buffer) {\n ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context;\n return ctx->dev_ptr;\n}\n\nstatic enum ggml_status\nggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer,\n ggml_tensor *tensor) try {\n GGML_SYCL_DEBUG(\"[SYCL] call %s\", __func__);\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\": tensor\", tensor, \"\\n\").c_str());\n ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *)buffer->context;\n\n if (tensor->view_src != NULL) {\n assert(tensor->view_src->buffer->buft == buffer->buft);\n return GGML_STATUS_SUCCESS;\n }\n if ((tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q4_K || tensor->type == GGML_TYPE_Q6_K) &&\n !g_ggml_sycl_disable_optimize) {\n ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};\n tensor->extra = extra;\n ctx->tensor_extras.push_back(extra); //used to release it when destroy ctx.\n }\n\n if (ggml_is_quantized(tensor->type)) {\n // initialize padding to 0 to avoid possible NaN values\n size_t original_size = ggml_nbytes(tensor);\n size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);\n\n if (padded_size > original_size && tensor->view_src == nullptr) {\n SYCL_CHECK(CHECK_TRY_ERROR(ctx->stream->memset(\n (char *)tensor->data + original_size, 0,\n padded_size - original_size).wait()));\n }\n }\n return GGML_STATUS_SUCCESS;\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer,\n ggml_tensor *tensor,\n const void *data, size_t offset,\n size_t size) try {\n GGML_SYCL_DEBUG(\"[SYCL] call %s\", __func__);\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\": tensor\", tensor).c_str());\n GGML_SYCL_DEBUG(\" size=%zu offset=%zu\\n\", size, offset);\n ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context;\n ggml_sycl_set_device(ctx->device);\n auto stream = &(dpct::dev_mgr::instance().get_device(ctx->device).default_queue());\n SYCL_CHECK(CHECK_TRY_ERROR(dpct::dev_mgr::instance().get_device(ctx->device).queues_wait_and_throw()));\n#ifndef _WIN32\n // Note: Use host buffer to save the data from mmap(), then copy to device. It's workaround for mmap() issue on PVC GPU.\n // This function will be called during load model from disk. Use memory buffer replace dynamic won't save more time and brings potential memory leak risk here.\n char * host_buf = (char *) malloc(size);\n memcpy(host_buf, data, size);\n SYCL_CHECK(CHECK_TRY_ERROR((*stream).memcpy((char *) tensor->data + offset, host_buf, size).wait()));\n free(host_buf);\n#else\n SYCL_CHECK(CHECK_TRY_ERROR((*stream).memcpy((char *) tensor->data + offset, data, size).wait()));\n#endif\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void ggml_backend_sycl_buffer_get_tensor(ggml_backend_buffer_t buffer,\n const ggml_tensor *tensor,\n void *data, size_t offset,\n size_t size) try {\n GGML_SYCL_DEBUG(\"[SYCL] call %s\", __func__);\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\": tensor\", tensor).c_str());\n GGML_SYCL_DEBUG(\" size=%zu offset=%zu\\n\", size, offset);\n ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context;\n\n ggml_sycl_set_device(ctx->device);\n auto stream = dpct::dev_mgr::instance().get_device(ctx->device).default_queue();\n\n SYCL_CHECK(CHECK_TRY_ERROR(\n stream.memcpy(data, (const char *)tensor->data + offset, size)\n .wait()));\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void dev2dev_memcpy(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst,\n const void *ptr_src, size_t size) {\n char *host_buf = (char *)malloc(size);\n q_src.memcpy(host_buf, (const char *)ptr_src, size).wait();\n q_dst.memcpy((char *)ptr_dst, host_buf, size).wait();\n free(host_buf);\n}\n\nstatic bool\nggml_backend_sycl_buffer_cpy_tensor(ggml_backend_buffer_t buffer,\n const ggml_tensor *src,\n ggml_tensor *dst) try {\n bool is_cpy_supported = ggml_backend_buffer_is_sycl(src->buffer);\n GGML_SYCL_DEBUG(\"[SYCL] call %s\", __func__);\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\": dst\", dst).c_str());\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\" src\", src).c_str());\n GGML_SYCL_DEBUG(\" is_cpy_supported=%d\\n\", is_cpy_supported);\n if (is_cpy_supported) {\n ggml_backend_sycl_buffer_context * src_ctx = (ggml_backend_sycl_buffer_context *)src->buffer->context;\n ggml_backend_sycl_buffer_context * dst_ctx = (ggml_backend_sycl_buffer_context *)dst->buffer->context;\n\n ggml_sycl_set_device(src_ctx->device);\n /*\n DPCT1009:198: SYCL uses exceptions to report errors and does not use the\n error codes. The original code was commented out and a warning string\n was inserted. You need to rewrite this code.\n */\n SYCL_CHECK(CHECK_TRY_ERROR(\n dpct::dev_mgr::instance().get_device(src_ctx->device).queues_wait_and_throw()));\n ggml_sycl_set_device(dst_ctx->device);\n /*\n DPCT1009:199: SYCL uses exceptions to report errors and does not use the\n error codes. The original code was commented out and a warning string\n was inserted. You need to rewrite this code.\n */\n SYCL_CHECK(CHECK_TRY_ERROR(\n dpct::dev_mgr::instance().get_device(dst_ctx->device).queues_wait_and_throw()));\n /*\n DPCT1009:200: SYCL uses exceptions to report errors and does not use the\n error codes. The original code was commented out and a warning string\n was inserted. You need to rewrite this code.\n */\n\n queue_ptr stream_dst = dst_ctx->stream;\n queue_ptr stream_src = src_ctx->stream;\n size_t size = ggml_nbytes(src);\n\n //todo. it's dirty solutino to walkaroud known issue:device2device cross GPUs.\n dev2dev_memcpy(*stream_dst, *stream_src, dst->data, src->data, size);\n\n//todo, it's known issue：error in device2device cross GPUs. reused when the issue is fixed. DON\"T remove\n#if 0\n SYCL_CHECK(CHECK_TRY_ERROR((*stream).memcpy(\n (char *)dst->data, (const char *)src->data, size).wait()));\n\n /*\n DPCT1009:201: SYCL uses exceptions to report errors and does not use the\n error codes. The original code was commented out and a warning string\n was inserted. You need to rewrite this code.\n */\n SYCL_CHECK(CHECK_TRY_ERROR(\n dpct::dev_mgr::instance().get_device(dst_ctx->device).queues_wait_and_throw()));\n#endif\n return true;\n }\n return false;\n GGML_UNUSED(buffer);\n} catch (const sycl::exception & exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__ << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void ggml_backend_sycl_buffer_clear(ggml_backend_buffer_t buffer,\n uint8_t value) try {\n GGML_SYCL_DEBUG(\"[SYCL] call %s: size=%zu\\n\", __func__, buffer->size);\n ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *) buffer->context;\n\n ggml_sycl_set_device(ctx->device);\n queue_ptr stream = ctx->stream;\n SYCL_CHECK(\n CHECK_TRY_ERROR(dpct::get_current_device().queues_wait_and_throw()));\n\n SYCL_CHECK(CHECK_TRY_ERROR((*stream)\n .memset(ctx->dev_ptr, value, buffer->size)\n .wait()));\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void ggml_backend_sycl_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value,\n size_t offset, size_t size) {\n GGML_SYCL_DEBUG(\"[SYCL] call %s\", __func__);\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\": tensor\", tensor).c_str());\n GGML_SYCL_DEBUG(\" size=%zu offset=%zu value=%u\\n\", size, offset, value);\n ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *) buffer->context;\n SYCL_CHECK(ggml_sycl_set_device(ctx->device));\n auto stream = &(dpct::dev_mgr::instance().get_device(ctx->device).default_queue());\n if (size == 0) {\n return; // Nothing to do\n }\n if (tensor->data == nullptr) {\n GGML_ABORT(\"Error: Tensor data pointer is null.\\n\");\n }\n void * target_ptr = static_cast(tensor->data) + offset;\n SYCL_CHECK(CHECK_TRY_ERROR((*stream).memset(target_ptr, value, size)));\n SYCL_CHECK(CHECK_TRY_ERROR((*stream).wait()));\n}\n\nstatic void ggml_backend_sycl_buffer_reset(ggml_backend_buffer_t buffer) {\n GGML_SYCL_DEBUG(\"[SYCL] call %s\\n\", __func__);\n if (buffer == nullptr) {\n return;\n }\n\n ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *) buffer->context;\n\n if (ctx != nullptr) {\n for (ggml_tensor_extra_gpu * extra : ctx->tensor_extras) {\n release_extra_gpu(extra);\n }\n ctx->tensor_extras.clear(); // reset the tensor_extras vector\n }\n}\n\nstatic const ggml_backend_buffer_i ggml_backend_sycl_buffer_interface = {\n /* .free_buffer = */ ggml_backend_sycl_buffer_free_buffer,\n /* .get_base = */ ggml_backend_sycl_buffer_get_base,\n /* .init_tensor = */ ggml_backend_sycl_buffer_init_tensor,\n /* .memset_tensor = */ ggml_backend_sycl_buffer_memset_tensor,\n /* .set_tensor = */ ggml_backend_sycl_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_sycl_buffer_get_tensor,\n /* .cpy_tensor = */ ggml_backend_sycl_buffer_cpy_tensor,\n /* .clear = */ ggml_backend_sycl_buffer_clear,\n /* .reset = */ ggml_backend_sycl_buffer_reset,\n};\n\n// sycl buffer type\nstruct ggml_backend_sycl_buffer_type_context {\n int device;\n std::string name;\n\n // each buffer type has its own stream\n queue_ptr stream = nullptr;\n};\n\nstatic const char * ggml_backend_sycl_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;\n\n return ctx->name.c_str();\n}\n\nstatic ggml_backend_buffer_t\nggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,\n size_t size) try {\n ggml_backend_sycl_buffer_type_context * buft_ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;\n ggml_sycl_set_device(buft_ctx->device);\n const queue_ptr stream = buft_ctx->stream;\n size = std::max(size, (size_t)1); // syclMalloc returns null for size 0\n\n void * dev_ptr;\n SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)sycl::malloc_device(\n size, *stream)));\n if (!dev_ptr) {\n GGML_LOG_ERROR(\"%s: can't allocate %lu Bytes of memory on device\\n\", __func__, size);\n return nullptr;\n }\n ggml_backend_sycl_buffer_context * ctx = new ggml_backend_sycl_buffer_context(buft_ctx->device, dev_ptr, buft_ctx->stream);\n return ggml_backend_buffer_init(buft, ggml_backend_sycl_buffer_interface, ctx, size);\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic size_t ggml_backend_sycl_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n return 128;\n GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {\n return dpct::get_current_device().get_max_mem_alloc_size();\n\n GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n size_t size = ggml_nbytes(tensor);\n int64_t ne0 = tensor->ne[0];\n\n if (ggml_is_quantized(tensor->type)) {\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n }\n\n return size;\n\n GGML_UNUSED(buft);\n}\n\nstatic const ggml_backend_buffer_type_i ggml_backend_sycl_buffer_type_interface = {\n /* .get_name = */ ggml_backend_sycl_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_sycl_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_sycl_buffer_type_get_alignment,\n /* .get_max_size = */ ggml_backend_sycl_buffer_type_get_max_size,\n /* .get_alloc_size = */ ggml_backend_sycl_buffer_type_get_alloc_size,\n /* .is_host = */ NULL,\n};\n\nggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n\n auto dev_count = ggml_backend_sycl_get_device_count();\n\n if (device>=dev_count or device<0) {\n GGML_LOG_ERROR(\"ggml_backend_sycl_buffer_type error: device_index:%d is out of range [0, %d], miss to call ggml_backend_sycl_set_single_device()\\n\",\n device, dev_count-1);\n GGML_ASSERT(devicedevice;\n if (device>=ggml_sycl_info().device_count or device<0) {\n GGML_LOG_ERROR(\"ggml_backend_sycl_buffer_type error: device_index:%d is out of range [0, %d], miss to call ggml_backend_sycl_set_single_device()\\n\",\n device, ggml_sycl_info().device_count-1);\n GGML_ASSERT(devicestream(i, 0)},\n };\n }\n ggml_backend_sycl_buffer_type_initialized = true;\n }\n return &ggml_backend_sycl_buffer_types[device];\n}\n\n// sycl split buffer\n\nstatic int64_t get_row_rounding(ggml_type type, const std::array & tensor_split) {\n int64_t min_compute_capability = INT_MAX;\n int64_t max_compute_capability = INT_MIN;\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n if (tensor_split[i] < (i + 1 < ggml_sycl_info().device_count ? tensor_split[i + 1] : 1.0f)) {\n if (min_compute_capability > ggml_sycl_info().devices[i].cc) {\n min_compute_capability = ggml_sycl_info().devices[i].cc;\n }\n if (max_compute_capability < ggml_sycl_info().devices[i].cc) {\n max_compute_capability = ggml_sycl_info().devices[i].cc;\n }\n }\n }\n\n switch(type) {\n case GGML_TYPE_Q4_0:\n case GGML_TYPE_Q4_1:\n return max_compute_capability >= VER_GEN9 ? 128 : 64;\n case GGML_TYPE_Q5_0:\n case GGML_TYPE_Q5_1:\n case GGML_TYPE_Q8_0:\n return 64;\n case GGML_TYPE_F16:\n case GGML_TYPE_F32:\n return 1;\n case GGML_TYPE_Q2_K:\n case GGML_TYPE_Q3_K:\n case GGML_TYPE_Q4_K:\n case GGML_TYPE_Q5_K:\n case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n case GGML_TYPE_IQ2_S:\n case GGML_TYPE_IQ1_S:\n case GGML_TYPE_IQ1_M:\n case GGML_TYPE_IQ3_XXS:\n case GGML_TYPE_IQ4_XS:\n case GGML_TYPE_IQ4_NL:\n return max_compute_capability >= VER_GEN9 ? 128 : 64;\n case GGML_TYPE_IQ3_S:\n return max_compute_capability >= VER_GEN9 ? 128 : 64;\n case GGML_TYPE_Q6_K:\n return 64;\n default:\n GGML_ABORT(\"fatal error\");\n }\n}\n\nstatic void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array & tensor_split, int id) {\n const int64_t nrows = ggml_nrows(tensor);\n const int64_t rounding = get_row_rounding(tensor->type, tensor_split);\n\n *row_low = id == 0 ? 0 : nrows*tensor_split[id];\n *row_low -= *row_low % rounding;\n if (id == ggml_sycl_info().device_count - 1) {\n *row_high = nrows;\n } else {\n *row_high = nrows*tensor_split[id + 1];\n *row_high -= *row_high % rounding;\n }\n}\n\nstatic size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) {\n static_assert(GGML_MAX_DIMS == 4, \"GGML_MAX_DIMS is not 4 - update this function\");\n\n return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]);\n}\n\nstruct ggml_backend_sycl_split_buffer_type_context {\n std::array tensor_split;\n};\n\nstruct ggml_backend_sycl_split_buffer_context {\n ~ggml_backend_sycl_split_buffer_context() try {\n for (ggml_tensor_extra_gpu * extra : tensor_extras) {\n release_extra_gpu(extra, streams);\n }\n }\n catch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n }\n\n std::vector tensor_extras;\n std::vector streams;\n};\n\nstatic void ggml_backend_sycl_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n ggml_backend_sycl_split_buffer_context * ctx = (ggml_backend_sycl_split_buffer_context *)buffer->context;\n delete ctx;\n}\n\nstatic void * ggml_backend_sycl_split_buffer_get_base(ggml_backend_buffer_t buffer) {\n // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced\n return (void *)0x1000;\n\n GGML_UNUSED(buffer);\n}\n\nstatic enum ggml_status\nggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer,\n ggml_tensor *tensor) try {\n GGML_SYCL_DEBUG(\"[SYCL] call %s\", __func__);\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\": tensor\", tensor, \"\\n\").c_str());\n GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported\n\n ggml_backend_sycl_split_buffer_context * ctx = (ggml_backend_sycl_split_buffer_context *)buffer->context;\n ggml_backend_sycl_split_buffer_type_context * buft_ctx = (ggml_backend_sycl_split_buffer_type_context *)buffer->buft->context;\n\n const int64_t ne0 = tensor->ne[0];\n\n ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};\n\n ctx->tensor_extras.push_back(extra);\n ctx->streams.push_back(&(dpct::get_current_device().default_queue()));\n\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n int64_t row_low, row_high;\n get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, i);\n\n int64_t nrows_split = row_high - row_low;\n if (nrows_split == 0) {\n continue;\n }\n\n size_t size = ggml_nbytes_split(tensor, nrows_split);\n const size_t original_size = size;\n\n // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n\n // FIXME: do not crash if SYCL Buffer alloc fails\n // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first\n ggml_sycl_set_device(i);\n const queue_ptr stream = ctx->streams[i];\n char * buf;\n /*\n DPCT1009:208: SYCL uses exceptions to report errors and does not use the\n error codes. The original code was commented out and a warning string\n was inserted. You need to rewrite this code.\n */\n SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)sycl::malloc_device(\n size, *stream)));\n if (!buf) {\n char err_buf[1024];\n snprintf(err_buf, 1023, \"%s: can't allocate %lu Bytes of memory on device\\n\", __func__, size);\n throw std::runtime_error(err_buf);\n }\n // set padding to 0 to avoid possible NaN values\n if (size > original_size) {\n /*\n DPCT1009:209: SYCL uses exceptions to report errors and does not use\n the error codes. The original code was commented out and a warning\n string was inserted. You need to rewrite this code.\n */\n SYCL_CHECK(CHECK_TRY_ERROR(\n (*stream)\n .memset(buf + original_size, 0, size - original_size)\n .wait()));\n }\n\n extra->data_device[i] = buf;\n\n for (int64_t is = 0; is < GGML_SYCL_MAX_STREAMS; ++is) {\n /*\n DPCT1009:210: SYCL uses exceptions to report errors and does not use\n the error codes. The original code was commented out and a warning\n string was inserted. You need to rewrite this code.\n */\n SYCL_CHECK(\n CHECK_TRY_ERROR(extra->events[i][is] = new sycl::event()));\n }\n }\n tensor->extra = extra;\n return GGML_STATUS_SUCCESS;\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void\nggml_backend_sycl_split_buffer_set_tensor(ggml_backend_buffer_t buffer,\n ggml_tensor *tensor, const void *data,\n size_t offset, size_t size) try {\n GGML_SYCL_DEBUG(\"[SYCL] call %s\", __func__);\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\": tensor\", tensor).c_str());\n GGML_SYCL_DEBUG(\" size=%zu offset=%zu\\n\", size, offset);\n // split tensors must always be set in their entirety at once\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(size == ggml_nbytes(tensor));\n\n ggml_backend_sycl_split_buffer_context * ctx = (ggml_backend_sycl_split_buffer_context *)buffer->context;\n ggml_backend_sycl_split_buffer_type_context * buft_ctx = (ggml_backend_sycl_split_buffer_type_context *)buffer->buft->context;\n\n const int64_t ne0 = tensor->ne[0];\n const size_t nb1 = tensor->nb[1];\n ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;\n\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n int64_t row_low, row_high;\n get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, i);\n\n int64_t nrows_split = row_high - row_low;\n if (nrows_split == 0) {\n continue;\n }\n\n const size_t offset_split = row_low*nb1;\n size_t size = ggml_nbytes_split(tensor, nrows_split);\n const size_t original_size = size;\n\n // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n\n const char * buf_host = (const char *)data + offset_split;\n /*\n DPCT1009:211: SYCL uses exceptions to report errors and does not use the\n error codes. The original code was commented out and a warning string\n was inserted. You need to rewrite this code.\n */\n ggml_sycl_set_device(i);\n const queue_ptr stream = ctx->streams[i];\n SYCL_CHECK(CHECK_TRY_ERROR(\n (*stream)\n .memcpy(extra->data_device[i], buf_host, original_size)\n .wait()));\n }\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void\nggml_backend_sycl_split_buffer_get_tensor(ggml_backend_buffer_t buffer,\n const ggml_tensor *tensor, void *data,\n size_t offset, size_t size) try {\n GGML_SYCL_DEBUG(\"[SYCL] call %s\", __func__);\n GGML_SYCL_DEBUG(\"%s\", debug_get_tensor_str(\": tensor\", tensor).c_str());\n GGML_SYCL_DEBUG(\" size=%zu offset=%zu\\n\", size, offset);\n // split tensors must always be set in their entirety at once\n GGML_ASSERT(offset == 0);\n GGML_ASSERT(size == ggml_nbytes(tensor));\n\n ggml_backend_sycl_split_buffer_context * ctx = (ggml_backend_sycl_split_buffer_context *)buffer->context;\n ggml_backend_sycl_split_buffer_type_context * buft_ctx = (ggml_backend_sycl_split_buffer_type_context *)buffer->buft->context;\n\n const int64_t ne0 = tensor->ne[0];\n const size_t nb1 = tensor->nb[1];\n ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;\n\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n int64_t row_low, row_high;\n get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, i);\n\n int64_t nrows_split = row_high - row_low;\n if (nrows_split == 0) {\n continue;\n }\n\n const size_t offset_split = row_low*nb1;\n size_t size = ggml_nbytes_split(tensor, nrows_split);\n const size_t original_size = size;\n\n // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n\n char * buf_host = (char *)data + offset_split;\n /*\n DPCT1009:212: SYCL uses exceptions to report errors and does not use the\n error codes. The original code was commented out and a warning string\n was inserted. You need to rewrite this code.\n */\n ggml_sycl_set_device(i);\n const queue_ptr stream = ctx->streams[i];\n SYCL_CHECK(CHECK_TRY_ERROR(\n (*stream)\n .memcpy(buf_host, extra->data_device[i], original_size)\n .wait()));\n }\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void ggml_backend_sycl_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n GGML_UNUSED(buffer);\n GGML_UNUSED(value);\n}\n\nstatic struct ggml_backend_buffer_i ggml_backend_sycl_split_buffer_interface = {\n /* .free_buffer = */ ggml_backend_sycl_split_buffer_free_buffer,\n /* .get_base = */ ggml_backend_sycl_split_buffer_get_base,\n /* .init_tensor = */ ggml_backend_sycl_split_buffer_init_tensor,\n /* .memset_tensor = */ NULL,\n /* .set_tensor = */ ggml_backend_sycl_split_buffer_set_tensor,\n /* .get_tensor = */ ggml_backend_sycl_split_buffer_get_tensor,\n /* .cpy_tensor = */ NULL,\n /* .clear = */ ggml_backend_sycl_split_buffer_clear,\n /* .reset = */ NULL,\n};\n\n// sycl split buffer type\n\nstatic const char * ggml_backend_sycl_split_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n return GGML_SYCL_NAME \"_Split\";\n\n GGML_UNUSED(buft);\n}\n\nstatic bool ggml_backend_buffer_is_sycl_split(ggml_backend_buffer_t buffer) {\n return buffer->buft->iface.get_name == ggml_backend_sycl_split_buffer_type_get_name;\n}\n\nstatic ggml_backend_buffer_t ggml_backend_sycl_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point\n // instead, we allocate them for each tensor separately in init_tensor\n // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,\n // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct.\n ggml_backend_sycl_split_buffer_context * ctx = new ggml_backend_sycl_split_buffer_context();\n\n return ggml_backend_buffer_init(buft, ggml_backend_sycl_split_buffer_interface, ctx, size);\n}\n\nstatic size_t ggml_backend_sycl_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n return 128;\n GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_sycl_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n ggml_backend_sycl_split_buffer_type_context * ctx = (ggml_backend_sycl_split_buffer_type_context *)buft->context;\n\n size_t total_size = 0;\n\n const int64_t ne0 = tensor->ne[0];\n\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n int64_t row_low, row_high;\n get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, i);\n\n int64_t nrows_split = row_high - row_low;\n if (nrows_split == 0) {\n continue;\n }\n\n total_size += ggml_nbytes_split(tensor, nrows_split);\n\n // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses\n if (ne0 % MATRIX_ROW_PADDING != 0) {\n total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n }\n }\n\n return total_size;\n}\n\nstatic bool ggml_backend_sycl_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) {\n return false;\n\n GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_type_i ggml_backend_sycl_split_buffer_type_interface = {\n /* .get_name = */ ggml_backend_sycl_split_buffer_type_get_name,\n /* .alloc_buffer = */ ggml_backend_sycl_split_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_sycl_split_buffer_type_get_alignment,\n /* .get_max_size = */ NULL, // defaults to SIZE_MAX\n /* .get_alloc_size = */ ggml_backend_sycl_split_buffer_type_get_alloc_size,\n /* .is_host = */ ggml_backend_sycl_split_buffer_type_is_host,\n};\n\nggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split) {\n static std::mutex mutex;\n std::lock_guard lock(mutex);\n\n GGML_SYCL_DEBUG(\"[SYCL] call ggml_backend_sycl_split_buffer_type\\n\");\n ggml_check_sycl();\n // FIXME: this is not thread safe\n static std::map, struct ggml_backend_buffer_type> buft_map;\n\n std::array tensor_split_arr = {};\n\n bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_SYCL_MAX_DEVICES, [](float x) { return x == 0.0f; });\n if (all_zero) {\n tensor_split_arr = ggml_sycl_info().default_tensor_split;\n } else {\n float split_sum = 0.0f;\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n tensor_split_arr[i] = split_sum;\n split_sum += tensor_split[i];\n }\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n tensor_split_arr[i] /= split_sum;\n }\n }\n\n auto it = buft_map.find(tensor_split_arr);\n if (it != buft_map.end()) {\n return &it->second;\n }\n\n struct ggml_backend_buffer_type buft {\n /* .iface = */ ggml_backend_sycl_split_buffer_type_interface,\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_sycl_reg(), 0),\n /* .context = */ new ggml_backend_sycl_split_buffer_type_context{tensor_split_arr},\n };\n\n auto result = buft_map.emplace(tensor_split_arr, buft);\n return &result.first->second;\n}\n\n// host buffer type\n\nstatic const char * ggml_backend_sycl_host_buffer_type_name(ggml_backend_buffer_type_t buft) {\n return GGML_SYCL_NAME \"_Host\";\n\n GGML_UNUSED(buft);\n}\n\ninline void * aligned_malloc_host(size_t alignment, size_t size) {\n#ifdef _WIN32\n return _aligned_malloc(size, alignment);\n#else\n return aligned_alloc(alignment, size);\n#endif\n}\n\ninline void free_aligned_mem_host(void * memblock) {\n#ifdef _WIN32\n _aligned_free(memblock);\n#else\n free(memblock);\n#endif\n}\n\nstatic void ggml_backend_sycl_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n free_aligned_mem_host((void *)buffer->context);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_sycl_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n void * ptr = aligned_malloc_host(TENSOR_ALIGNMENT, size);\n if (ptr == nullptr) {\n // fallback to cpu buffer\n return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n }\n\n // FIXME: this is a hack to avoid having to implement a new buffer type\n ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);\n buffer->buft = buft;\n buffer->iface.free_buffer = ggml_backend_sycl_host_buffer_free_buffer;\n\n return buffer;\n}\n\nggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() {\n GGML_SYCL_DEBUG(\"[SYCL] call ggml_backend_sycl_host_buffer_type\\n\");\n static struct ggml_backend_buffer_type ggml_backend_sycl_buffer_type_host = {\n /* .iface = */ {\n /* .get_name = */ ggml_backend_sycl_host_buffer_type_name,\n /* .alloc_buffer = */ ggml_backend_sycl_host_buffer_type_alloc_buffer,\n /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,\n /* .get_max_size = */ NULL, // TODO: return device.maxBufferLength\n /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,\n /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,\n },\n /* .device = */ ggml_backend_reg_dev_get(ggml_backend_sycl_reg(), 0),\n /* .context = */ nullptr,\n };\n\n return &ggml_backend_sycl_buffer_type_host;\n}\n\n// buffer pool for sycl (legacy)\nstruct ggml_sycl_pool_leg : public ggml_sycl_pool {\n static const int MAX_SYCL_BUFFERS = 256;\n\n int device;\n queue_ptr qptr;\n struct ggml_sycl_buffer {\n void * ptr = nullptr;\n size_t size = 0;\n };\n\n ggml_sycl_buffer buffer_pool[MAX_SYCL_BUFFERS] = {};\n size_t pool_size = 0;\n\n explicit ggml_sycl_pool_leg(queue_ptr qptr_, int device_) : device(device_), qptr(qptr_) {}\n\n ~ggml_sycl_pool_leg() {\n for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) {\n ggml_sycl_buffer & b = buffer_pool[i];\n if (b.ptr != nullptr) {\n SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr)));\n pool_size -= b.size;\n }\n }\n GGML_ASSERT(pool_size == 0);\n }\n\n void * alloc(size_t size, size_t * actual_size) override {\n#ifdef DEBUG_sycl_MALLOC\n int nnz = 0;\n size_t max_size = 0;\n#endif\n size_t best_diff = 1ull << 36;\n int ibest = -1;\n for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) {\n ggml_sycl_buffer& b = buffer_pool[i];\n if (b.ptr != nullptr) {\n#ifdef DEBUG_sycl_MALLOC\n ++nnz;\n if (b.size > max_size) max_size = b.size;\n#endif\n if (b.size >= size) {\n size_t diff = b.size - size;\n if (diff < best_diff) {\n best_diff = diff;\n ibest = i;\n if (!best_diff) {\n void * ptr = b.ptr;\n *actual_size = b.size;\n b.ptr = nullptr;\n b.size = 0;\n return ptr;\n }\n }\n }\n }\n }\n if (ibest >= 0) {\n ggml_sycl_buffer& b = buffer_pool[ibest];\n void * ptr = b.ptr;\n *actual_size = b.size;\n b.ptr = nullptr;\n b.size = 0;\n return ptr;\n }\n void * ptr;\n size_t look_ahead_size = (size_t) (1.05 * size);\n\n SYCL_CHECK(\n CHECK_TRY_ERROR(ptr = (void *)sycl::malloc_device(\n look_ahead_size, *qptr)));\n if (!ptr) {\n GGML_LOG_ERROR(\"%s: can't allocate %lu Bytes of memory on device/GPU\\n\", __func__, look_ahead_size);\n return nullptr;\n }\n\n *actual_size = look_ahead_size;\n pool_size += look_ahead_size;\n\n#ifdef DEBUG_SYCL_MALLOC\n GGML_LOG_DEBUG(\"%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\\n\", __func__, id, nnz,\n (uint32_t)(max_size/1024/1024), (uint32_t)(g_sycl_pool_size[id]/1024/1024), (uint32_t)(size/1024/1024));\n#endif\n\n // GGML_SYCL_DEBUG(\"ggml_sycl_pool_malloc_leg look_ahead_size=%lu, return %p\\n\", look_ahead_size, ptr);\n return ptr;\n }\n\n void free(void * ptr, size_t size) override {\n for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) {\n ggml_sycl_buffer& b = buffer_pool[i];\n if (b.ptr == nullptr) {\n b.ptr = ptr;\n b.size = size;\n return;\n }\n }\n GGML_LOG_WARN(\"WARNING: sycl buffer pool full, increase MAX_sycl_BUFFERS\\n\");\n SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr)));\n pool_size -= size;\n }\n};\n\nstruct ggml_sycl_pool_host : public ggml_sycl_pool {\n queue_ptr qptr;\n int device;\n\n inline static int counter{ 0 };\n\n struct ggml_sycl_buffer {\n void * ptr = nullptr;\n size_t size = 0;\n };\n\n // Set arbitrarly to 64\n static constexpr int MAX_POOL_SIZE{ 64 };\n std::vector buffer_pool = std::vector(MAX_POOL_SIZE);\n size_t pool_size = 0;\n\n explicit ggml_sycl_pool_host(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {}\n\n ~ggml_sycl_pool_host() {\n for (int i = 0; i < MAX_POOL_SIZE; ++i) {\n ggml_sycl_buffer & b = buffer_pool[i];\n if (b.ptr != nullptr) {\n SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr)));\n b.ptr = nullptr;\n pool_size -= b.size;\n b.size = 0;\n }\n }\n counter = 0;\n }\n\n void * alloc(size_t size, size_t * actual_size) override {\n if (counter == MAX_POOL_SIZE) {\n ggml_sycl_buffer b = buffer_pool[0];\n void * ptr = b.ptr;\n *actual_size = b.size;\n counter = 1;\n return ptr;\n }\n ggml_sycl_buffer & b = buffer_pool[counter];\n\n if (b.ptr == nullptr) {\n void * ptr;\n\n SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));\n if (!ptr) {\n GGML_LOG_ERROR(\"%s: can't allocate %lu Bytes of memory on host\\n\", __func__, size);\n return nullptr;\n }\n pool_size += size;\n *actual_size = size;\n counter = counter + 1;\n return ptr;\n } else {\n ++counter;\n b.size = size;\n return b.ptr;\n }\n }\n\n void free(void * ptr, size_t size) override {\n // if the pool is not completed add the pointer to it in place of the first nullptr found.\n // Otherwise do nothing, pointers will be freed once the pool is deallocated.\n for (int i = 0; i < MAX_POOL_SIZE; ++i) {\n ggml_sycl_buffer & b = buffer_pool[i];\n if (b.ptr == nullptr) {\n b.ptr = ptr;\n b.size = size;\n return;\n }\n }\n }\n};\n\nstd::unique_ptr ggml_backend_sycl_context::new_pool_for_host(queue_ptr qptr, int device) {\n // return pool for the host to speed up memory management\n return std::unique_ptr(new ggml_sycl_pool_host(qptr, device));\n}\n\nstd::unique_ptr ggml_backend_sycl_context::new_pool_for_device(queue_ptr qptr, int device) {\n // TBD: NO VMM support\n // if (ggml_sycl_info().devices[device].vmm) {\n // return std::unique_ptr(new ggml_sycl_pool_vmm(device));\n // }\n return std::unique_ptr(new ggml_sycl_pool_leg(qptr, device));\n}\n\n// TBD pool with virtual memory management\n// struct ggml_sycl_pool_vmm : public ggml_sycl_pool\n\n/// kernels\ntypedef void (*ggml_sycl_op_mul_mat_t)(\n ggml_backend_sycl_context & ctx,\n const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst,\n const char *src0_dd_i, const float *src1_ddf_i, const char *src1_ddq_i,\n float *dst_dd_i, const int64_t row_low, const int64_t row_high,\n const int64_t src1_ncols, const int64_t src1_padded_row_size,\n const queue_ptr &stream);\n\n\n\nstatic void mul_mat_p021_f16_f32(\n const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst,\n const int ncols_x, const int nrows_x, const int nchannels_x, const int nchannels_y,\n const sycl::nd_item<3> &item_ct1) {\n\n const sycl::half *x = (const sycl::half *)vx;\n\n const int row_x = item_ct1.get_local_range(1) * item_ct1.get_group(1) +\n item_ct1.get_local_id(1);\n const int channel = item_ct1.get_local_range(0) * item_ct1.get_group(0) +\n item_ct1.get_local_id(0);\n const int channel_x = channel / (nchannels_y / nchannels_x);\n\n const int nrows_y = ncols_x;\n const int nrows_dst = nrows_x;\n const int row_dst = row_x;\n\n float tmp = 0.0f;\n\n for (int col_x0 = 0; col_x0 < ncols_x;\n col_x0 += item_ct1.get_local_range(2)) {\n const int col_x = col_x0 + item_ct1.get_local_id(2);\n\n if (col_x >= ncols_x) {\n break;\n }\n\n // x is transposed and permuted\n const int ix = row_x*nchannels_x*ncols_x + channel_x*ncols_x + col_x;\n const float xi =\n sycl::vec(x[ix])\n .convert()[0];\n\n const int row_y = col_x;\n\n\n // y is not transposed but permuted\n const int iy = channel*nrows_y + row_y;\n\n tmp += xi * y[iy];\n }\n\n // dst is not transposed and not permuted\n const int idst = channel*nrows_dst + row_dst;\n\n // sum up partial sums and write back result\n#pragma unroll\n for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {\n tmp +=\n dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);\n }\n\n if (item_ct1.get_local_id(2) == 0) {\n dst[idst] = tmp;\n }\n}\n\nstatic void mul_mat_vec_nc_f16_f32( // nc == non-contiguous\n const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst, const int ncols_x, const int nrows_x,\n const int row_stride_x, const int channel_stride_x,const int channel_stride_y, const int channel_x_divisor,\n const sycl::nd_item<3> &item_ct1) {\n\n const sycl::half *x = (const sycl::half *)vx;\n\n const int row_x = item_ct1.get_local_range(1) * item_ct1.get_group(1) +\n item_ct1.get_local_id(1);\n const int channel = item_ct1.get_local_range(0) * item_ct1.get_group(0) +\n item_ct1.get_local_id(0);\n const int channel_x = channel / channel_x_divisor;\n\n const int nrows_dst = nrows_x;\n const int row_dst = row_x;\n\n const int idst = channel*nrows_dst + row_dst;\n\n float tmp = 0.0f;\n\n for (int col_x0 = 0; col_x0 < ncols_x;\n col_x0 += item_ct1.get_local_range(2)) {\n const int col_x = col_x0 + item_ct1.get_local_id(2);\n\n if (col_x >= ncols_x) {\n break;\n }\n\n const int row_y = col_x;\n\n const int ix = channel_x*channel_stride_x + row_x*row_stride_x + col_x;\n const int iy = channel * channel_stride_y + row_y;\n\n const float xi =\n sycl::vec(x[ix])\n .convert()[0];\n\n tmp += xi * y[iy];\n }\n\n // sum up partial sums and write back result\n#pragma unroll\n for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {\n tmp +=\n dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);\n }\n\n if (item_ct1.get_local_id(2) == 0) {\n dst[idst] = tmp;\n }\n}\n\nstatic void k_sum_rows_f32(const float * x, float * dst, const int ncols,\n const sycl::nd_item<3> &item_ct1) {\n const int row = item_ct1.get_group(1);\n const int col = item_ct1.get_local_id(2);\n\n float sum = 0.0f;\n for (int i = col; i < ncols; i += item_ct1.get_local_range(2)) {\n sum += x[row * ncols + i];\n }\n\n sum = warp_reduce_sum(sum, item_ct1);\n\n if (col == 0) {\n dst[row] = sum;\n }\n}\n\n\ntemplate\nstatic inline void ggml_sycl_swap(T & a, T & b) {\n T tmp = a;\n a = b;\n b = tmp;\n}\n\ntemplate \n__dpct_inline__ static void\nk_argsort_f32_i32(const float *x, int *dst, const int ncols, int ncols_pad,\n const int tasks_per_thread, const sycl::nd_item<3> &item_ct1,\n uint8_t *dpct_local) {\n // bitonic sort\n int col_index = item_ct1.get_local_id(2);\n int row = item_ct1.get_group(1);\n\n for (int i = 0; i < tasks_per_thread; i++) {\n int col = col_index * tasks_per_thread + i;\n if (col >= ncols_pad) {\n return;\n }\n }\n\n const float * x_row = x + row * ncols;\n auto dst_row = (int *)dpct_local;\n\n // initialize indices\n for (int i=0;i 0; j /= 2) {\n for (int i = 0; i < tasks_per_thread; i++) {\n int col = col_index * tasks_per_thread + i;\n int ixj = col ^ j;\n if (ixj > col) {\n if ((col & k) == 0) {\n if (dst_row[col] >= ncols ||\n (dst_row[ixj] < ncols &&\n (order == GGML_SORT_ORDER_ASC\n ? x_row[dst_row[col]] > x_row[dst_row[ixj]]\n : x_row[dst_row[col]] <\n x_row[dst_row[ixj]]))) {\n ggml_sycl_swap(dst_row[col], dst_row[ixj]);\n }\n } else {\n if (dst_row[ixj] >= ncols ||\n (dst_row[col] < ncols &&\n (order == GGML_SORT_ORDER_ASC\n ? x_row[dst_row[col]] < x_row[dst_row[ixj]]\n : x_row[dst_row[col]] >\n x_row[dst_row[ixj]]))) {\n ggml_sycl_swap(dst_row[col], dst_row[ixj]);\n }\n }\n }\n item_ct1.barrier(sycl::access::fence_space::local_space);\n }\n }\n }\n\n // copy the result to dst without the padding\n for (int i = 0; i < tasks_per_thread; i++) {\n int col = col_index * tasks_per_thread + i;\n if (col < ncols) {\n dst[row * ncols + col] = dst_row[col];\n }\n }\n}\n\nstatic void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past,\n const sycl::nd_item<3> &item_ct1) {\n const int col = item_ct1.get_local_range(1) * item_ct1.get_group(1) +\n item_ct1.get_local_id(1);\n const int row = item_ct1.get_local_range(2) * item_ct1.get_group(2) +\n item_ct1.get_local_id(2);\n\n if (col >= ncols) {\n return;\n }\n\n const int i = row*ncols + col;\n //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];\n //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU\n dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;\n}\n\nstatic void scale_f32(const float * x, float * dst, const float scale, const float bias, const int k,\n const sycl::nd_item<3> &item_ct1) {\n const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +\n item_ct1.get_local_id(2);\n\n if (i >= k) {\n return;\n }\n\n dst[i] = scale * x[i] + bias;\n}\n\n\ntemplate \nstatic void pool2d_nchw_kernel(\n const int ih, const int iw, const int oh, const int ow,\n const int kh, const int kw, const int sh, const int sw,\n const int ph, const int pw, const int parallel_elements,\n const Ti* src, To* dst, const enum ggml_op_pool op,\n const sycl::nd_item<3> &item_ct1) {\n int idx = item_ct1.get_local_id(2) +\n item_ct1.get_group(2) * item_ct1.get_local_range(2);\n if (idx >= parallel_elements) {\n return;\n }\n\n const int I_HW = ih * iw;\n const int O_HW = oh * ow;\n const int nc = idx / O_HW;\n const int cur_oh = idx % O_HW / ow;\n const int cur_ow = idx % O_HW % ow;\n const Ti* i_ptr = src + nc * I_HW;\n To* o_ptr = dst + nc * O_HW;\n const int start_h = cur_oh * sh - ph;\n const int bh = sycl::max(0, start_h);\n const int eh = sycl::min(ih, start_h + kh);\n const int start_w = cur_ow * sw - pw;\n const int bw = sycl::max(0, start_w);\n const int ew = sycl::min(iw, start_w + kw);\n\n To res = 0;\n\n switch (op) {\n case GGML_OP_POOL_AVG: res = 0; break;\n case GGML_OP_POOL_MAX: res = -FLT_MAX; break;\n default:\n res = (To) sycl::nan(uint32_t(0));\n break;\n }\n\n for (int i = bh; i < eh; i += 1) {\n for (int j = bw; j < ew; j += 1) {\n#if DPCT_COMPATIBILITY_TEMP >= 350\n /*\n DPCT1098:106: The '*' expression is used instead of the __ldg\n call. These two expressions do not provide the exact same\n functionality. Check the generated code for potential precision\n and/or performance issues.\n */\n Ti cur = *(i_ptr + i * iw + j);\n#else\n Ti cur = i_ptr[i * iw + j];\n#endif\n switch (op) {\n case GGML_OP_POOL_AVG: res += (cur / (kh * kw)); break;\n case GGML_OP_POOL_MAX: res = sycl::max(res, (To)cur); break;\n default:\n res = (To) sycl::nan(uint32_t(0));\n break;\n }\n }\n }\n o_ptr[cur_oh * ow + cur_ow] = res;\n}\n\n\nstatic void ggml_mul_mat_p021_f16_f32_sycl(const void *vx, const float *y,\n float *dst, const int ncols_x,\n const int nrows_x,\n const int nchannels_x,\n const int nchannels_y,\n queue_ptr stream) {\n\n const sycl::range<3> block_nums(nchannels_y, nrows_x, 1);\n const sycl::range<3> block_dims(1, 1, WARP_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n mul_mat_p021_f16_f32(vx, y, dst, ncols_x, nrows_x, nchannels_x,\n nchannels_y, item_ct1);\n });\n }\n}\n\nstatic void ggml_mul_mat_vec_nc_f16_f32_sycl(\n const void *vx, const float *y, float *dst, const int ncols_x,\n const int nrows_x, const int row_stride_x, const int nchannels_x,\n const int nchannels_y, const int channel_stride_x, const int channel_stride_y, queue_ptr stream) {\n\n const sycl::range<3> block_nums(nchannels_y, nrows_x, 1);\n const sycl::range<3> block_dims(1, 1, WARP_SIZE);\n {\n dpct::has_capability_or_fail(stream->get_device(),\n {sycl::aspect::fp16});\n\n stream->parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n mul_mat_vec_nc_f16_f32(vx, y, dst, ncols_x, nrows_x,\n row_stride_x, channel_stride_x, channel_stride_y,\n nchannels_y / nchannels_x, item_ct1);\n });\n }\n}\n\n\n\nstatic void scale_f32_sycl(const float *x, float *dst, const float scale, const float bias,\n const int k, queue_ptr stream) {\n const int num_blocks = (k + SYCL_SCALE_BLOCK_SIZE - 1) / SYCL_SCALE_BLOCK_SIZE;\n stream->parallel_for(\n sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *\n sycl::range<3>(1, 1, SYCL_SCALE_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_SCALE_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n scale_f32(x, dst, scale, bias, k, item_ct1);\n });\n}\n\n\nstatic void sum_rows_f32_sycl(const float *x, float *dst, const int ncols,\n const int nrows, queue_ptr stream) {\n const sycl::range<3> block_dims(1, 1, WARP_SIZE);\n const sycl::range<3> block_nums(1, nrows, 1);\n stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1)\n [[sycl::reqd_sub_group_size(WARP_SIZE)]] {\n k_sum_rows_f32(x, dst, ncols, item_ct1);\n });\n}\n\nstatic int next_power_of_2(int x) {\n int n = 1;\n while (n < x) {\n n *= 2;\n }\n return n;\n}\n\nstatic void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols,\n const int nrows, ggml_sort_order order,\n queue_ptr stream, int device) {\n // bitonic sort requires ncols to be power of 2\n const int ncols_pad = next_power_of_2(ncols);\n\n int nth = 1;\n int max_block_size = ggml_sycl_info().max_work_group_sizes[device];\n while (nth < ncols_pad && nth < max_block_size)\n nth *= 2;\n if (nth > max_block_size)\n nth = max_block_size;\n\n const int tasks_per_thread = ncols_pad / nth;\n\n const sycl::range<3> block_dims(1, 1, nth);\n const sycl::range<3> block_nums(1, nrows, 1);\n const size_t shared_mem = ncols_pad * sizeof(int);\n GGML_ASSERT(shared_mem<=ggml_sycl_info().devices[device].smpbo);\n\n if (order == GGML_SORT_ORDER_ASC) {\n stream->submit([&](sycl::handler &cgh) {\n sycl::local_accessor dpct_local_acc_ct1(\n sycl::range<1>(shared_mem), cgh);\n\n cgh.parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) {\n k_argsort_f32_i32(\n x, dst, ncols, ncols_pad, tasks_per_thread, item_ct1,\n dpct_local_acc_ct1\n .get_multi_ptr()\n .get());\n });\n });\n } else if (order == GGML_SORT_ORDER_DESC) {\n stream->submit([&](sycl::handler &cgh) {\n sycl::local_accessor dpct_local_acc_ct1(\n sycl::range<1>(shared_mem), cgh);\n\n cgh.parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) {\n k_argsort_f32_i32(\n x, dst, ncols, ncols_pad, tasks_per_thread, item_ct1,\n dpct_local_acc_ct1\n .get_multi_ptr()\n .get());\n });\n });\n } else {\n GGML_ABORT(\"fatal error\");\n }\n}\n\nstatic void top_k_f32_sycl(\n const float * src,\n int32_t * dst_indices,\n const int64_t ncols,\n const int64_t nrows,\n const int k,\n dpct::queue_ptr main_stream\n) {\n const int block_size = 128;\n\n const sycl::range<1> block_dims(block_size);\n const sycl::range<1> grid_dims(nrows);\n\n main_stream->submit([&](sycl::handler &cgh) {\n sycl::local_accessor shared_vals(sycl::range<1>(block_size * k), cgh);\n sycl::local_accessor shared_idx(sycl::range<1>(block_size * k), cgh);\n\n cgh.parallel_for(\n sycl::nd_range<1>(grid_dims * block_dims, block_dims),\n [=](sycl::nd_item<1> item_ct1) {\n const int row = item_ct1.get_group(0);\n const int tid = item_ct1.get_local_id(0);\n\n if (row >= nrows) return;\n\n const float * src_row = src + row * ncols;\n int32_t * dst_idx_row = dst_indices + row * k;\n\n float local_vals[32];\n int local_idx[32];\n\n for (int i = 0; i < k; i++) {\n local_vals[i] = -FLT_MAX;\n local_idx[i] = -1;\n }\n\n for (int col = tid; col < ncols; col += block_size) {\n float val = src_row[col];\n\n if (val > local_vals[k-1]) {\n int pos = k - 1;\n while (pos > 0 && val > local_vals[pos - 1]) {\n pos--;\n }\n\n for (int i = k - 1; i > pos; i--) {\n local_vals[i] = local_vals[i - 1];\n local_idx[i] = local_idx[i - 1];\n }\n local_vals[pos] = val;\n local_idx[pos] = col;\n }\n }\n\n for (int i = 0; i < k; i++) {\n shared_vals[tid * k + i] = local_vals[i];\n shared_idx[tid * k + i] = local_idx[i];\n }\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n if (tid == 0) {\n float final_vals[32];\n int final_idx[32];\n\n for (int i = 0; i < k; i++) {\n final_vals[i] = -FLT_MAX;\n final_idx[i] = -1;\n }\n\n for (int t = 0; t < block_size; t++) {\n for (int i = 0; i < k; i++) {\n float val = shared_vals[t * k + i];\n int idx = shared_idx[t * k + i];\n\n if (val > final_vals[k-1]) {\n int pos = k - 1;\n while (pos > 0 && val > final_vals[pos - 1]) {\n pos--;\n }\n\n for (int j = k - 1; j > pos; j--) {\n final_vals[j] = final_vals[j - 1];\n final_idx[j] = final_idx[j - 1];\n }\n final_vals[pos] = val;\n final_idx[pos] = idx;\n }\n }\n }\n\n for (int i = 0; i < k; i++) {\n dst_idx_row[i] = final_idx[i];\n }\n\n if (k > 1) {\n int32_t temp = dst_idx_row[0];\n dst_idx_row[0] = dst_idx_row[1];\n dst_idx_row[1] = temp;\n }\n }\n });\n });\n}\n\nstatic void argmax_f32_i32_sycl(const float *x, int *dst, const int ncols,\n const int nrows, queue_ptr stream) {\n const sycl::range<3> block_dims(1, 1, SYCL_ARGMAX_BLOCK_SIZE);\n const sycl::range<3> block_nums(1, nrows, 1);\n const size_t shared_mem = 256 * sizeof(float);\n\n stream->submit([&](sycl::handler &cgh) {\n sycl::local_accessor shared_data(\n sycl::range<1>(shared_mem/sizeof(float)), cgh);\n sycl::local_accessor shared_indices(\n sycl::range<1>(shared_mem/sizeof(float)), cgh);\n\n cgh.parallel_for(\n sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) {\n const int tid = item_ct1.get_local_id(2);\n const int row = item_ct1.get_global_id(1);\n\n float max_val = -INFINITY;\n int max_idx = -1;\n\n for (int col = tid; col < ncols; col += 256) {\n float val = x[row * ncols + col];\n if (val > max_val) {\n max_val = val;\n max_idx = col;\n }\n }\n\n shared_data[tid] = max_val;\n shared_indices[tid] = max_idx;\n item_ct1.barrier(sycl::access::fence_space::local_space);\n\n for (int stride = 256/2; stride > 0; stride >>= 1) {\n if (tid < stride) {\n float val1 = shared_data[tid];\n float val2 = shared_data[tid + stride];\n if (val2 > val1) {\n shared_data[tid] = val2;\n shared_indices[tid] = shared_indices[tid + stride];\n }\n }\n item_ct1.barrier(sycl::access::fence_space::local_space);\n }\n\n\n if (tid == 0) {\n dst[row] = shared_indices[0];\n }\n });\n });\n}\nstatic void diag_mask_inf_f32_sycl(const float *x, float *dst,\n const int ncols_x, const int nrows_x,\n const int rows_per_channel, const int n_past,\n queue_ptr stream) {\n const sycl::range<3> block_dims(1, SYCL_DIAG_MASK_INF_BLOCK_SIZE, 1);\n const int block_num_x = (ncols_x + SYCL_DIAG_MASK_INF_BLOCK_SIZE - 1) / SYCL_DIAG_MASK_INF_BLOCK_SIZE;\n const sycl::range<3> block_nums(1, block_num_x, nrows_x);\n stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),\n [=](sycl::nd_item<3> item_ct1) {\n diag_mask_inf_f32(x, dst, ncols_x,\n rows_per_channel, n_past,\n item_ct1);\n });\n}\n\nstatic dpct::err0 ggml_sycl_cpy_tensor_2d(void *dst,\n const struct ggml_tensor *src,\n int64_t i3, int64_t i2,\n int64_t i1_low, int64_t i1_high,\n queue_ptr stream) try {\n\n dpct::memcpy_direction kind;\n char * src_ptr;\n if (ggml_backend_buffer_is_host(src->buffer)) {\n kind = dpct::host_to_device;\n //GGML_SYCL_DEBUG(\"%s: Host buffer type src tensor\\n\", __func__);\n src_ptr = (char *) src->data;\n // GGML_SYCL_DEBUG(\"ggml_sycl_cpy_tensor_2d GGML_BACKEND_TYPE_CPU src_ptr %p\\n\", src_ptr);\n } else if (ggml_backend_buffer_is_sycl(src->buffer)) {\n // If buffer is a SYCL buffer\n //GGML_SYCL_DEBUG(\"%s: SYCL buffer type src tensor\\n\", __func__);\n kind = dpct::device_to_device;\n src_ptr = (char *) src->data;\n } else if (ggml_backend_buffer_is_sycl_split(src->buffer)) {\n /*\n If buffer is a SYCL split buffer\n */\n //GGML_SYCL_DEBUG(\"%s: Split buffer type src tensor\\n\", __func__);\n GGML_ASSERT(i1_low == 0 && i1_high == src->ne[1]);\n kind = dpct::device_to_device;\n ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src->extra;\n int id;\n SYCL_CHECK(CHECK_TRY_ERROR(\n id = get_current_device_id()));\n // GGML_SYCL_DEBUG(\"current device index %d\\n\", id);\n src_ptr = (char *) extra->data_device[id];\n } else {\n // GGML_SYCL_DEBUG(\"GGML_ABORT(\"fatal error\")\\n\");\n GGML_ABORT(\"fatal error\");\n }\n char * dst_ptr = (char *) dst;\n\n GGML_TENSOR_LOCALS_1(int64_t, ne, src, ne);\n GGML_TENSOR_LOCALS(int64_t, nb, src, nb);\n const enum ggml_type type = src->type;\n const int64_t ts = ggml_type_size(type);\n const int64_t bs = ggml_blck_size(type);\n int64_t i1_diff = i1_high - i1_low;\n\n const char * x = src_ptr + i1_low*nb1 + i2*nb2 + i3*nb3;\n if (nb0 == ts && nb1 == ts*ne0/bs) {\n // GGML_SYCL_DEBUG(\"stream->memcpy: dst_ptr=%p, x=%p, size=%lu\\n\", dst_ptr, x, i1_diff * nb1);\n // return CHECK_TRY_ERROR(stream->memcpy(dst_ptr, x, i1_diff * nb1));\n return CHECK_TRY_ERROR(dpct::async_dpct_memcpy(dst_ptr, x, i1_diff * nb1,\n kind, *stream));\n\n } else if (nb0 == ts) {\n return CHECK_TRY_ERROR(\n dpct::async_dpct_memcpy(dst_ptr, ts * ne0 / bs, x, nb1,\n ts * ne0 / bs, i1_diff, kind, *stream));\n } else {\n for (int64_t i1 = 0; i1 < i1_diff; i1++) {\n const void * rx = (const void *) ((const char *) x + i1*nb1);\n void * rd = (void *) (dst_ptr + i1*ts*ne0/bs);\n // pretend the row is a matrix with cols=1\n dpct::err0 r = CHECK_TRY_ERROR(dpct::async_dpct_memcpy(\n rd, ts / bs, rx, nb0, ts / bs, ne0, kind, *stream));\n /*\n DPCT1001:85: The statement could not be removed.\n */\n /*\n DPCT1000:86: Error handling if-stmt was detected but could not be\n rewritten.\n */\n if (r != 0) return r;\n }\n return 0;\n }\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\ninline void ggml_sycl_op_mul_mat_sycl(\n ggml_backend_sycl_context & ctx,\n const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst,\n const char *src0_dd_i, const float *src1_ddf_i, const char *src1_ddq_i,\n float *dst_dd_i, const int64_t row_low, const int64_t row_high,\n const int64_t src1_ncols, const int64_t src1_padded_row_size,\n const queue_ptr &stream) try {\n\n GGML_ASSERT(src0_dd_i != nullptr);\n GGML_ASSERT(src1_ddf_i != nullptr);\n GGML_ASSERT(dst_dd_i != nullptr);\n\n const int64_t ne00 = src0->ne[0];\n const int64_t ne10 = src1->ne[0];\n GGML_ASSERT(ne00 == ne10);\n\n const int64_t row_diff = row_high - row_low;\n\n int id;\n SYCL_CHECK(\n CHECK_TRY_ERROR(id = get_current_device_id()));\n\n const int64_t ne0 = dst->ne[0]; // used by MKL only\n // the main device has a larger memory buffer to hold the results from all GPUs\n // ldc == nrows of the matrix that cuBLAS writes into\n int ldc = id == ctx.device ? ne0 : row_diff; // used by MKL only\n\n#ifdef GGML_SYCL_F16\n bool use_fp16 = true; // TODO(Yu) SYCL capability check\n#else\n bool use_fp16 = false;\n#endif\n if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && use_fp16 && ggml_is_contiguous(src0) &&\n row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) {\n ggml_sycl_pool_alloc src0_as_f16(ctx.pool());\n if (src0->type != GGML_TYPE_F16) {\n scope_op_debug_print scope_dbg_print(__func__, \"/to_fp16_sycl\", dst, /*num_src=*/2,\n \" : converting src0 to fp16\");\n const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src0->type, dst);\n GGML_ASSERT(to_fp16_sycl != nullptr);\n size_t ne = row_diff*ne00;\n src0_as_f16.alloc(ne);\n to_fp16_sycl(src0_dd_i, src0_as_f16.get(), ne, stream);\n }\n const sycl::half *src0_ptr = src0->type == GGML_TYPE_F16\n ? (const sycl::half *)src0_dd_i\n : src0_as_f16.get();\n\n ggml_sycl_pool_alloc src1_as_f16(ctx.pool());\n if (src1->type != GGML_TYPE_F16) {\n scope_op_debug_print scope_dbg_print(__func__, \"/to_fp16_sycl\", dst, /*num_src=*/2,\n \" : converting src1 to fp16\");\n const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst);\n GGML_ASSERT(to_fp16_sycl != nullptr);\n size_t ne = src1_ncols*ne10;\n src1_as_f16.alloc(ne);\n to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream);\n }\n const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16\n ? (const sycl::half *)src1->data + src1_padded_row_size\n : src1_as_f16.get();\n\n#if GGML_SYCL_DNNL\n if (!g_ggml_sycl_disable_dnn) {\n DnnlGemmWrapper::row_gemm(ctx,row_diff, src1_ncols , ne10, src0_ptr,\n DnnlGemmWrapper::to_dt(), src1_ptr, DnnlGemmWrapper::to_dt(),\n dst_dd_i, DnnlGemmWrapper::to_dt(), stream);\n }\n else\n#endif\n {\n ggml_sycl_pool_alloc dst_f16(ctx.pool(), row_diff * src1_ncols);\n\n const sycl::half alpha_f16 = 1.0f;\n const sycl::half beta_f16 = 0.0f;\n SYCL_CHECK(CHECK_TRY_ERROR(dpct::gemm(\n *stream, oneapi::mkl::transpose::trans,\n oneapi::mkl::transpose::nontrans, row_diff, src1_ncols, ne10,\n &alpha_f16, src0_ptr, dpct::library_data_t::real_half, ne00,\n src1_ptr, dpct::library_data_t::real_half, ne10, &beta_f16,\n dst_f16.get(), dpct::library_data_t::real_half, ldc,\n dpct::library_data_t::real_half)));\n scope_op_debug_print scope_dbg_print(__func__, \"/to_fp32_sycl\", dst, /*num_src=*/2,\n \" : converting dst to fp32\");\n const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16, dst);\n to_fp32_sycl(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream);\n }\n } else {\n ggml_sycl_pool_alloc src0_ddq_as_f32(ctx.pool());\n ggml_sycl_pool_alloc src1_ddq_as_f32(ctx.pool());\n if (src0->type != GGML_TYPE_F32) {\n scope_op_debug_print scope_dbg_print(__func__, \"/to_fp32_sycl\", dst, /*num_src=*/2,\n \" : converting src0 to fp32\");\n const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src0->type, dst);\n GGML_ASSERT(to_fp32_sycl != nullptr);\n src0_ddq_as_f32.alloc(row_diff*ne00);\n to_fp32_sycl(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream);\n }\n if (src1->type != GGML_TYPE_F32) {\n scope_op_debug_print scope_dbg_print(__func__, \"/to_fp32_sycl\", dst, /*num_src=*/2,\n \" : converting src1 to fp32\");\n const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src1->type, dst);\n GGML_ASSERT(to_fp32_sycl != nullptr);\n src1_ddq_as_f32.alloc(src1_ncols*ne10);\n to_fp32_sycl(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream);\n }\n const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get();\n const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get();\n\n#if GGML_SYCL_DNNL\n if (!g_ggml_sycl_disable_dnn) {\n DnnlGemmWrapper::row_gemm(ctx, row_diff, src1_ncols, ne10, src0_ddf_i,\n DnnlGemmWrapper::to_dt(), src1_ddf1_i, DnnlGemmWrapper::to_dt(),\n dst_dd_i, DnnlGemmWrapper::to_dt(), stream);\n }\n else\n#endif\n {\n const float alpha = 1.0f;\n const float beta = 0.0f;\n SYCL_CHECK(CHECK_TRY_ERROR(oneapi::mkl::blas::column_major::gemm(\n *stream, oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, row_diff,\n src1_ncols, ne10, dpct::get_value(&alpha, *stream), src0_ddf_i, ne00, src1_ddf1_i, ne10,\n dpct::get_value(&beta, *stream), dst_dd_i, ldc)));\n }\n }\n GGML_UNUSED(dst);\n GGML_UNUSED(src1_ddq_i);\n GGML_UNUSED(src1_padded_row_size);\n}\ncatch (sycl::exception const &exc) {\n std::cerr << exc.what() << \"Exception caught at file:\" << __FILE__\n << \", line:\" << __LINE__ << std::endl;\n std::exit(1);\n}\n\nstatic void ggml_sycl_op_pool2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n const float * src0_dd = static_cast(dst->src[0]->data);\n float * dst_dd = static_cast(dst->data);\n\n const int32_t * opts = (const int32_t *)dst->op_params;\n enum ggml_op_pool op = static_cast(opts[0]);\n const int k0 = opts[1];\n const int k1 = opts[2];\n const int s0 = opts[3];\n const int s1 = opts[4];\n const int p0 = opts[5];\n const int p1 = opts[6];\n\n const int64_t IH = dst->src[0]->ne[1];\n const int64_t IW = dst->src[0]->ne[0];\n\n const int64_t N = dst->ne[3];\n const int64_t OC = dst->ne[2];\n const int64_t OH = dst->ne[1];\n const int64_t OW = dst->ne[0];\n\n const int parallel_elements = N * OC * OH * OW;\n const int num_blocks = (parallel_elements + SYCL_POOL2D_BLOCK_SIZE - 1) / SYCL_POOL2D_BLOCK_SIZE;\n sycl::range<3> block_nums(1, 1, num_blocks);\n main_stream->parallel_for(\n sycl::nd_range<3>(block_nums *\n sycl::range<3>(1, 1, SYCL_IM2COL_BLOCK_SIZE),\n sycl::range<3>(1, 1, SYCL_IM2COL_BLOCK_SIZE)),\n [=](sycl::nd_item<3> item_ct1) {\n pool2d_nchw_kernel(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0,\n parallel_elements, src0_dd, dst_dd, op,\n item_ct1);\n });\n}\n\ninline void ggml_sycl_op_sum(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n const float * src0_dd = static_cast(dst->src[0]->data);\n float * dst_dd = static_cast(dst->data);\n\n const int64_t ne = ggml_nelements(dst->src[0]);\n\n sum_rows_f32_sycl(src0_dd, dst_dd, ne, 1, main_stream);\n}\n\ninline void ggml_sycl_op_sum_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n const float * src0_dd = static_cast(dst->src[0]->data);\n float * dst_dd = static_cast(dst->data);\n\n const int64_t ncols = dst->src[0]->ne[0];\n const int64_t nrows = ggml_nrows(dst->src[0]);\n\n sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);\n}\n\ninline void ggml_sycl_op_mean(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n\n const float * src0_dd = static_cast(dst->src[0]->data);\n float * dst_dd = static_cast(dst->data);\n\n const int64_t ncols = dst->src[0]->ne[0];\n const int64_t nrows = ggml_nrows(dst->src[0]);\n\n sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);\n\n main_stream->parallel_for(\n sycl::range<1>(nrows),\n [=](sycl::id<1> row) {\n dst_dd[row] /= ncols;\n }\n );\n}\n\n\ninline void ggml_sycl_op_argsort(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_I32);\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n const float * src0_dd = static_cast(dst->src[0]->data);\n int32_t * dst_dd = static_cast(dst->data);\n\n\n const int64_t ncols = dst->src[0]->ne[0];\n const int64_t nrows = ggml_nrows(dst->src[0]);\n\n enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];\n\n argsort_f32_i32_sycl(src0_dd, (int *)dst_dd, ncols, nrows, order,\n main_stream, ctx.device);\n}\n\nstatic void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n\n GGML_ASSERT(src0);\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_I32);\n GGML_ASSERT(ggml_is_contiguous(src0));\n\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n\n const float * src0_dd = static_cast(src0->data);\n int32_t * dst_dd = static_cast(dst->data);\n\n const int k = dst->ne[0];\n const int64_t ncols = src0->ne[0];\n const int64_t nrows = ggml_nrows(src0);\n\n GGML_ASSERT(k > 0 && k <= 32);\n GGML_ASSERT(k <= ncols);\n\n top_k_f32_sycl(src0_dd, dst_dd, ncols, nrows, k, main_stream);\n}\n\ninline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_I32);\n\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n const float * src0_dd = static_cast(dst->src[0]->data);\n int32_t * dst_dd = static_cast(dst->data);\n\n const int64_t ncols = dst->src[0]->ne[0];\n const int64_t nrows = ggml_nrows(dst->src[0]);\n\n argmax_f32_i32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);\n}\n\ninline void ggml_sycl_op_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n const float * src0_dd = static_cast(dst->src[0]->data);\n float * dst_dd = static_cast(dst->data);\n\n const int64_t ne00 = dst->src[0]->ne[0];\n const int64_t ne01 = dst->src[0]->ne[1];\n const int nrows0 = ggml_nrows(dst->src[0]);\n\n const int n_past = ((int32_t *) dst->op_params)[0];\n\n diag_mask_inf_f32_sycl(src0_dd, dst_dd, ne00, nrows0, ne01, n_past, main_stream);\n}\n\nstatic void tri_f32_sycl(\n const float * src,\n float * dst,\n const int64_t ne0,\n const int64_t ne1,\n const int64_t ne2,\n const int64_t ne3,\n const ggml_tri_type ttype,\n dpct::queue_ptr main_stream\n) {\n const size_t total = (size_t) ne0 * (size_t) ne1 * (size_t) ne2 * (size_t) ne3;\n\n main_stream->parallel_for(sycl::range<1>(total), [=](sycl::id<1> tid) {\n const int64_t idx = (int64_t) tid[0];\n\n const int64_t i0 = idx % ne0;\n const int64_t t1 = idx / ne0;\n const int64_t i1 = t1 % ne1;\n\n bool keep = false;\n switch (ttype) {\n case GGML_TRI_TYPE_LOWER: keep = (i0 < i1); break;\n case GGML_TRI_TYPE_LOWER_DIAG: keep = (i0 <= i1); break;\n case GGML_TRI_TYPE_UPPER: keep = (i0 > i1); break;\n case GGML_TRI_TYPE_UPPER_DIAG: keep = (i0 >= i1); break;\n default: keep = false; break;\n }\n\n dst[idx] = keep ? src[idx] : 0.0f;\n });\n}\n\nstatic void ggml_sycl_op_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n const ggml_tensor * src0 = dst->src[0];\n GGML_ASSERT(src0);\n\n GGML_ASSERT(src0->type == GGML_TYPE_F32);\n GGML_ASSERT(dst->type == GGML_TYPE_F32);\n GGML_ASSERT(ggml_is_contiguous(src0));\n GGML_ASSERT(ggml_is_contiguous(dst));\n GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n\n const float * src0_dd = static_cast(src0->data);\n float * dst_dd = static_cast(dst->data);\n\n const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0);\n\n const int64_t ne0 = src0->ne[0];\n const int64_t ne1 = src0->ne[1];\n const int64_t ne2 = src0->ne[2];\n const int64_t ne3 = src0->ne[3];\n\n tri_f32_sycl(src0_dd, dst_dd, ne0, ne1, ne2, ne3, ttype, main_stream);\n}\n\n\ninline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {\n GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);\n GGML_ASSERT( dst->type == GGML_TYPE_F32);\n dpct::queue_ptr main_stream = ctx.stream();\n SYCL_CHECK(ggml_sycl_set_device(ctx.device));\n const float * src0_dd = static_cast(dst->src[0]->data);\n float * dst_dd = static_cast(dst->data);\n\n float scale;\n float bias;\n memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));\n memcpy(&bias, (float *) dst->op_params + 1, sizeof(float));\n\n scale_f32_sycl(src0_dd, dst_dd, scale, bias, ggml_nelements(dst->src[0]), main_stream);\n /*\n DPCT1010:87: SYCL uses exceptions to report errors and does not use the\n error codes. The call was replaced with 0. You need to rewrite this code.\n */\n SYCL_CHECK(0);\n}\n\nstatic void ggml_sycl_set_peer_access(const int n_tokens, int main_device) {\n static bool peer_access_enabled = false;\n\n const bool enable_peer_access = n_tokens <= GGML_SYCL_PEER_MAX_BATCH_SIZE;\n\n if (peer_access_enabled == enable_peer_access) {\n return;\n }\n\n#ifdef NDEBUG\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n SYCL_CHECK(ggml_sycl_set_device(i));\n }\n\n for (int i = 0; i < ggml_sycl_info().device_count; ++i) {\n SYCL_CHECK(ggml_sycl_set_device(i));\n\n for (int id_other = 0; id_other < ggml_sycl_info().device_count; ++id_other) {\n if (i == id_other) {\n continue;\n }\n if (i != main_device && id_other != main_device) {\n continue;\n }\n\n // int can_access_peer;\n // SYCL_CHECK(syclDeviceCanAccessPeer(&can_access_peer, id, id_other));\n // if (can_access_peer) {\n // if (enable_peer_access) {\n // SYCL_CHECK(syclDeviceEnablePeerAccess(id_other, 0));\n // } else {\n // SYCL_CHECK(syclDeviceDisablePeerAccess(id_other));\n // }\n // }\n }\n }\n#endif // NDEBUG\n\n peer_access_enabled = enable_peer_access;\n}\n\ntemplate